From 5066b694f14d626199aba94a311915540dc9dd06 Mon Sep 17 00:00:00 2001 From: tcompa Date: Fri, 20 Oct 2023 06:41:27 +0000 Subject: [PATCH] Deployed 1df67ca with MkDocs version: 1.5.2 --- .nojekyll | 0 404.html | 7 + assets/_mkdocstrings.css | 64 + assets/images/favicon.png | Bin 0 -> 1870 bytes assets/javascripts/bundle.220ee61c.min.js | 29 + assets/javascripts/bundle.220ee61c.min.js.map | 8 + assets/javascripts/lunr/min/lunr.ar.min.js | 1 + assets/javascripts/lunr/min/lunr.da.min.js | 18 + assets/javascripts/lunr/min/lunr.de.min.js | 18 + assets/javascripts/lunr/min/lunr.du.min.js | 18 + assets/javascripts/lunr/min/lunr.es.min.js | 18 + assets/javascripts/lunr/min/lunr.fi.min.js | 18 + assets/javascripts/lunr/min/lunr.fr.min.js | 18 + assets/javascripts/lunr/min/lunr.hi.min.js | 1 + assets/javascripts/lunr/min/lunr.hu.min.js | 18 + assets/javascripts/lunr/min/lunr.hy.min.js | 1 + assets/javascripts/lunr/min/lunr.it.min.js | 18 + assets/javascripts/lunr/min/lunr.ja.min.js | 1 + assets/javascripts/lunr/min/lunr.jp.min.js | 1 + assets/javascripts/lunr/min/lunr.kn.min.js | 1 + assets/javascripts/lunr/min/lunr.ko.min.js | 1 + assets/javascripts/lunr/min/lunr.multi.min.js | 1 + assets/javascripts/lunr/min/lunr.nl.min.js | 18 + assets/javascripts/lunr/min/lunr.no.min.js | 18 + assets/javascripts/lunr/min/lunr.pt.min.js | 18 + assets/javascripts/lunr/min/lunr.ro.min.js | 18 + assets/javascripts/lunr/min/lunr.ru.min.js | 18 + assets/javascripts/lunr/min/lunr.sa.min.js | 1 + .../lunr/min/lunr.stemmer.support.min.js | 1 + assets/javascripts/lunr/min/lunr.sv.min.js | 18 + assets/javascripts/lunr/min/lunr.ta.min.js | 1 + assets/javascripts/lunr/min/lunr.te.min.js | 1 + assets/javascripts/lunr/min/lunr.th.min.js | 1 + assets/javascripts/lunr/min/lunr.tr.min.js | 18 + assets/javascripts/lunr/min/lunr.vi.min.js | 1 + assets/javascripts/lunr/min/lunr.zh.min.js | 1 + assets/javascripts/lunr/tinyseg.js | 206 + assets/javascripts/lunr/wordcut.js | 6708 +++++++++++++++++ .../workers/search.74e28a9f.min.js | 42 + .../workers/search.74e28a9f.min.js.map | 8 + assets/stylesheets/main.eebd395e.min.css | 1 + assets/stylesheets/main.eebd395e.min.css.map | 1 + assets/stylesheets/palette.ecc896b0.min.css | 1 + .../stylesheets/palette.ecc896b0.min.css.map | 1 + changelog/index.html | 1414 ++++ custom_task/index.html | 1491 ++++ development/index.html | 1220 +++ doc-requirements.txt | 8 + extra.css | 17 + gen_ref_pages.py | 81 + index.html | 1172 +++ install/index.html | 1144 +++ manifest/index.html | 2342 ++++++ objects.inv | Bin 0 -> 2200 bytes overrides/404.html | 8 + .../fractal_tasks_core/SUMMARY/index.html | 1087 +++ .../dev/check_manifest/index.html | 1327 ++++ reference/fractal_tasks_core/dev/index.html | 1104 +++ .../dev/lib_args_schemas/index.html | 1697 +++++ .../dev/lib_descriptions/index.html | 1984 +++++ .../dev/lib_signature_constraints/index.html | 1427 ++++ .../dev/lib_task_docs/index.html | 1444 ++++ .../dev/lib_titles/index.html | 1382 ++++ .../dev/update_manifest/index.html | 1141 +++ reference/fractal_tasks_core/index.html | 1093 +++ .../lib_ROI_overlaps/index.html | 3189 ++++++++ .../lib_channels/index.html | 3224 ++++++++ .../fractal_tasks_core/lib_glob/index.html | 1302 ++++ .../lib_input_models/index.html | 2001 +++++ .../lib_masked_loading/index.html | 1975 +++++ .../lib_metadata_parsing/index.html | 2422 ++++++ .../fractal_tasks_core/lib_ngff/index.html | 3491 +++++++++ .../lib_parse_filename_metadata/index.html | 1474 ++++ .../lib_pyramid_creation/index.html | 1461 ++++ .../lib_read_fractal_metadata/index.html | 1479 ++++ .../lib_regions_of_interest/index.html | 4168 ++++++++++ .../lib_upscale_array/index.html | 1778 +++++ .../fractal_tasks_core/lib_write/index.html | 2556 +++++++ .../lib_zattrs_utils/index.html | 1516 ++++ .../tasks/_utils/index.html | 1465 ++++ .../index.html | 1888 +++++ .../apply_registration_to_image/index.html | 2153 ++++++ .../index.html | 2166 ++++++ .../tasks/cellpose_segmentation/index.html | 3079 ++++++++ .../tasks/compress_tif/index.html | 1326 ++++ .../tasks/copy_ome_zarr/index.html | 1710 +++++ .../tasks/create_ome_zarr/index.html | 2248 ++++++ .../create_ome_zarr_multiplex/index.html | 2331 ++++++ .../tasks/illumination_correction/index.html | 1964 +++++ .../tasks/import_ome_zarr/index.html | 1983 +++++ reference/fractal_tasks_core/tasks/index.html | 1103 +++ .../maximum_intensity_projection/index.html | 1465 ++++ .../tasks/napari_workflows_wrapper/index.html | 2653 +++++++ .../tasks/yokogawa_to_ome_zarr/index.html | 1792 +++++ search/search_index.json | 1 + sitemap.xml | 3 + sitemap.xml.gz | Bin 0 -> 127 bytes tasks_in_fractal/index.html | 1080 +++ tasks_in_scripts/index.html | 1108 +++ 99 files changed, 93498 insertions(+) create mode 100644 .nojekyll create mode 100644 404.html create mode 100644 assets/_mkdocstrings.css create mode 100644 assets/images/favicon.png create mode 100644 assets/javascripts/bundle.220ee61c.min.js create mode 100644 assets/javascripts/bundle.220ee61c.min.js.map create mode 100644 assets/javascripts/lunr/min/lunr.ar.min.js create mode 100644 assets/javascripts/lunr/min/lunr.da.min.js create mode 100644 assets/javascripts/lunr/min/lunr.de.min.js create mode 100644 assets/javascripts/lunr/min/lunr.du.min.js create mode 100644 assets/javascripts/lunr/min/lunr.es.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hu.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hy.min.js create mode 100644 assets/javascripts/lunr/min/lunr.it.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ja.min.js create mode 100644 assets/javascripts/lunr/min/lunr.jp.min.js create mode 100644 assets/javascripts/lunr/min/lunr.kn.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ko.min.js create mode 100644 assets/javascripts/lunr/min/lunr.multi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.nl.min.js create mode 100644 assets/javascripts/lunr/min/lunr.no.min.js create mode 100644 assets/javascripts/lunr/min/lunr.pt.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ro.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ru.min.js create mode 100644 assets/javascripts/lunr/min/lunr.sa.min.js create mode 100644 assets/javascripts/lunr/min/lunr.stemmer.support.min.js create mode 100644 assets/javascripts/lunr/min/lunr.sv.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ta.min.js create mode 100644 assets/javascripts/lunr/min/lunr.te.min.js create mode 100644 assets/javascripts/lunr/min/lunr.th.min.js create mode 100644 assets/javascripts/lunr/min/lunr.tr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.vi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.zh.min.js create mode 100644 assets/javascripts/lunr/tinyseg.js create mode 100644 assets/javascripts/lunr/wordcut.js create mode 100644 assets/javascripts/workers/search.74e28a9f.min.js create mode 100644 assets/javascripts/workers/search.74e28a9f.min.js.map create mode 100644 assets/stylesheets/main.eebd395e.min.css create mode 100644 assets/stylesheets/main.eebd395e.min.css.map create mode 100644 assets/stylesheets/palette.ecc896b0.min.css create mode 100644 assets/stylesheets/palette.ecc896b0.min.css.map create mode 100644 changelog/index.html create mode 100644 custom_task/index.html create mode 100644 development/index.html create mode 100644 doc-requirements.txt create mode 100644 extra.css create mode 100644 gen_ref_pages.py create mode 100644 index.html create mode 100644 install/index.html create mode 100644 manifest/index.html create mode 100644 objects.inv create mode 100644 overrides/404.html create mode 100644 reference/fractal_tasks_core/SUMMARY/index.html create mode 100644 reference/fractal_tasks_core/dev/check_manifest/index.html create mode 100644 reference/fractal_tasks_core/dev/index.html create mode 100644 reference/fractal_tasks_core/dev/lib_args_schemas/index.html create mode 100644 reference/fractal_tasks_core/dev/lib_descriptions/index.html create mode 100644 reference/fractal_tasks_core/dev/lib_signature_constraints/index.html create mode 100644 reference/fractal_tasks_core/dev/lib_task_docs/index.html create mode 100644 reference/fractal_tasks_core/dev/lib_titles/index.html create mode 100644 reference/fractal_tasks_core/dev/update_manifest/index.html create mode 100644 reference/fractal_tasks_core/index.html create mode 100644 reference/fractal_tasks_core/lib_ROI_overlaps/index.html create mode 100644 reference/fractal_tasks_core/lib_channels/index.html create mode 100644 reference/fractal_tasks_core/lib_glob/index.html create mode 100644 reference/fractal_tasks_core/lib_input_models/index.html create mode 100644 reference/fractal_tasks_core/lib_masked_loading/index.html create mode 100644 reference/fractal_tasks_core/lib_metadata_parsing/index.html create mode 100644 reference/fractal_tasks_core/lib_ngff/index.html create mode 100644 reference/fractal_tasks_core/lib_parse_filename_metadata/index.html create mode 100644 reference/fractal_tasks_core/lib_pyramid_creation/index.html create mode 100644 reference/fractal_tasks_core/lib_read_fractal_metadata/index.html create mode 100644 reference/fractal_tasks_core/lib_regions_of_interest/index.html create mode 100644 reference/fractal_tasks_core/lib_upscale_array/index.html create mode 100644 reference/fractal_tasks_core/lib_write/index.html create mode 100644 reference/fractal_tasks_core/lib_zattrs_utils/index.html create mode 100644 reference/fractal_tasks_core/tasks/_utils/index.html create mode 100644 reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/index.html create mode 100644 reference/fractal_tasks_core/tasks/apply_registration_to_image/index.html create mode 100644 reference/fractal_tasks_core/tasks/calculate_registration_image_based/index.html create mode 100644 reference/fractal_tasks_core/tasks/cellpose_segmentation/index.html create mode 100644 reference/fractal_tasks_core/tasks/compress_tif/index.html create mode 100644 reference/fractal_tasks_core/tasks/copy_ome_zarr/index.html create mode 100644 reference/fractal_tasks_core/tasks/create_ome_zarr/index.html create mode 100644 reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/index.html create mode 100644 reference/fractal_tasks_core/tasks/illumination_correction/index.html create mode 100644 reference/fractal_tasks_core/tasks/import_ome_zarr/index.html create mode 100644 reference/fractal_tasks_core/tasks/index.html create mode 100644 reference/fractal_tasks_core/tasks/maximum_intensity_projection/index.html create mode 100644 reference/fractal_tasks_core/tasks/napari_workflows_wrapper/index.html create mode 100644 reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/index.html create mode 100644 search/search_index.json create mode 100644 sitemap.xml create mode 100644 sitemap.xml.gz create mode 100644 tasks_in_fractal/index.html create mode 100644 tasks_in_scripts/index.html diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 000000000..e69de29bb diff --git a/404.html b/404.html new file mode 100644 index 000000000..77c59b660 --- /dev/null +++ b/404.html @@ -0,0 +1,7 @@ + + +

404 - Not found

diff --git a/assets/_mkdocstrings.css b/assets/_mkdocstrings.css new file mode 100644 index 000000000..049a254b9 --- /dev/null +++ b/assets/_mkdocstrings.css @@ -0,0 +1,64 @@ + +/* Avoid breaking parameter names, etc. in table cells. */ +.doc-contents td code { + word-break: normal !important; +} + +/* No line break before first paragraph of descriptions. */ +.doc-md-description, +.doc-md-description>p:first-child { + display: inline; +} + +/* Max width for docstring sections tables. */ +.doc .md-typeset__table, +.doc .md-typeset__table table { + display: table !important; + width: 100%; +} + +.doc .md-typeset__table tr { + display: table-row; +} + +/* Defaults in Spacy table style. */ +.doc-param-default { + float: right; +} + +/* Keep headings consistent. */ +h1.doc-heading, +h2.doc-heading, +h3.doc-heading, +h4.doc-heading, +h5.doc-heading, +h6.doc-heading { + font-weight: 400; + line-height: 1.5; + color: inherit; + text-transform: none; +} + +h1.doc-heading { + font-size: 1.6rem; +} + +h2.doc-heading { + font-size: 1.2rem; +} + +h3.doc-heading { + font-size: 1.15rem; +} + +h4.doc-heading { + font-size: 1.10rem; +} + +h5.doc-heading { + font-size: 1.05rem; +} + +h6.doc-heading { + font-size: 1rem; +} \ No newline at end of file diff --git a/assets/images/favicon.png b/assets/images/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..1cf13b9f9d978896599290a74f77d5dbe7d1655c GIT binary patch literal 1870 zcmV-U2eJ5xP)Gc)JR9QMau)O=X#!i9;T z37kk-upj^(fsR36MHs_+1RCI)NNu9}lD0S{B^g8PN?Ww(5|~L#Ng*g{WsqleV}|#l zz8@ri&cTzw_h33bHI+12+kK6WN$h#n5cD8OQt`5kw6p~9H3()bUQ8OS4Q4HTQ=1Ol z_JAocz`fLbT2^{`8n~UAo=#AUOf=SOq4pYkt;XbC&f#7lb$*7=$na!mWCQ`dBQsO0 zLFBSPj*N?#u5&pf2t4XjEGH|=pPQ8xh7tpx;US5Cx_Ju;!O`ya-yF`)b%TEt5>eP1ZX~}sjjA%FJF?h7cX8=b!DZl<6%Cv z*G0uvvU+vmnpLZ2paivG-(cd*y3$hCIcsZcYOGh{$&)A6*XX&kXZd3G8m)G$Zz-LV z^GF3VAW^Mdv!)4OM8EgqRiz~*Cji;uzl2uC9^=8I84vNp;ltJ|q-*uQwGp2ma6cY7 z;`%`!9UXO@fr&Ebapfs34OmS9^u6$)bJxrucutf>`dKPKT%%*d3XlFVKunp9 zasduxjrjs>f8V=D|J=XNZp;_Zy^WgQ$9WDjgY=z@stwiEBm9u5*|34&1Na8BMjjgf3+SHcr`5~>oz1Y?SW^=K z^bTyO6>Gar#P_W2gEMwq)ot3; zREHn~U&Dp0l6YT0&k-wLwYjb?5zGK`W6S2v+K>AM(95m2C20L|3m~rN8dprPr@t)5lsk9Hu*W z?pS990s;Ez=+Rj{x7p``4>+c0G5^pYnB1^!TL=(?HLHZ+HicG{~4F1d^5Awl_2!1jICM-!9eoLhbbT^;yHcefyTAaqRcY zmuctDopPT!%k+}x%lZRKnzykr2}}XfG_ne?nRQO~?%hkzo;@RN{P6o`&mMUWBYMTe z6i8ChtjX&gXl`nvrU>jah)2iNM%JdjqoaeaU%yVn!^70x-flljp6Q5tK}5}&X8&&G zX3fpb3E(!rH=zVI_9Gjl45w@{(ITqngWFe7@9{mX;tO25Z_8 zQHEpI+FkTU#4xu>RkN>b3Tnc3UpWzPXWm#o55GKF09j^Mh~)K7{QqbO_~(@CVq! zS<8954|P8mXN2MRs86xZ&Q4EfM@JB94b=(YGuk)s&^jiSF=t3*oNK3`rD{H`yQ?d; ztE=laAUoZx5?RC8*WKOj`%LXEkgDd>&^Q4M^z`%u0rg-It=hLCVsq!Z%^6eB-OvOT zFZ28TN&cRmgU}Elrnk43)!>Z1FCPL2K$7}gwzIc48NX}#!A1BpJP?#v5wkNprhV** z?Cpalt1oH&{r!o3eSKc&ap)iz2BTn_VV`4>9M^b3;(YY}4>#ML6{~(4mH+?%07*qo IM6N<$f(jP3KmY&$ literal 0 HcmV?d00001 diff --git a/assets/javascripts/bundle.220ee61c.min.js b/assets/javascripts/bundle.220ee61c.min.js new file mode 100644 index 000000000..116072a11 --- /dev/null +++ b/assets/javascripts/bundle.220ee61c.min.js @@ -0,0 +1,29 @@ +"use strict";(()=>{var Ci=Object.create;var gr=Object.defineProperty;var Ri=Object.getOwnPropertyDescriptor;var ki=Object.getOwnPropertyNames,Ht=Object.getOwnPropertySymbols,Hi=Object.getPrototypeOf,yr=Object.prototype.hasOwnProperty,nn=Object.prototype.propertyIsEnumerable;var rn=(e,t,r)=>t in e?gr(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,P=(e,t)=>{for(var r in t||(t={}))yr.call(t,r)&&rn(e,r,t[r]);if(Ht)for(var r of Ht(t))nn.call(t,r)&&rn(e,r,t[r]);return e};var on=(e,t)=>{var r={};for(var n in e)yr.call(e,n)&&t.indexOf(n)<0&&(r[n]=e[n]);if(e!=null&&Ht)for(var n of Ht(e))t.indexOf(n)<0&&nn.call(e,n)&&(r[n]=e[n]);return r};var Pt=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var Pi=(e,t,r,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of ki(t))!yr.call(e,o)&&o!==r&&gr(e,o,{get:()=>t[o],enumerable:!(n=Ri(t,o))||n.enumerable});return e};var yt=(e,t,r)=>(r=e!=null?Ci(Hi(e)):{},Pi(t||!e||!e.__esModule?gr(r,"default",{value:e,enumerable:!0}):r,e));var sn=Pt((xr,an)=>{(function(e,t){typeof xr=="object"&&typeof an!="undefined"?t():typeof define=="function"&&define.amd?define(t):t()})(xr,function(){"use strict";function e(r){var n=!0,o=!1,i=null,s={text:!0,search:!0,url:!0,tel:!0,email:!0,password:!0,number:!0,date:!0,month:!0,week:!0,time:!0,datetime:!0,"datetime-local":!0};function a(O){return!!(O&&O!==document&&O.nodeName!=="HTML"&&O.nodeName!=="BODY"&&"classList"in O&&"contains"in O.classList)}function f(O){var Qe=O.type,De=O.tagName;return!!(De==="INPUT"&&s[Qe]&&!O.readOnly||De==="TEXTAREA"&&!O.readOnly||O.isContentEditable)}function c(O){O.classList.contains("focus-visible")||(O.classList.add("focus-visible"),O.setAttribute("data-focus-visible-added",""))}function u(O){O.hasAttribute("data-focus-visible-added")&&(O.classList.remove("focus-visible"),O.removeAttribute("data-focus-visible-added"))}function p(O){O.metaKey||O.altKey||O.ctrlKey||(a(r.activeElement)&&c(r.activeElement),n=!0)}function m(O){n=!1}function d(O){a(O.target)&&(n||f(O.target))&&c(O.target)}function h(O){a(O.target)&&(O.target.classList.contains("focus-visible")||O.target.hasAttribute("data-focus-visible-added"))&&(o=!0,window.clearTimeout(i),i=window.setTimeout(function(){o=!1},100),u(O.target))}function v(O){document.visibilityState==="hidden"&&(o&&(n=!0),Y())}function Y(){document.addEventListener("mousemove",N),document.addEventListener("mousedown",N),document.addEventListener("mouseup",N),document.addEventListener("pointermove",N),document.addEventListener("pointerdown",N),document.addEventListener("pointerup",N),document.addEventListener("touchmove",N),document.addEventListener("touchstart",N),document.addEventListener("touchend",N)}function B(){document.removeEventListener("mousemove",N),document.removeEventListener("mousedown",N),document.removeEventListener("mouseup",N),document.removeEventListener("pointermove",N),document.removeEventListener("pointerdown",N),document.removeEventListener("pointerup",N),document.removeEventListener("touchmove",N),document.removeEventListener("touchstart",N),document.removeEventListener("touchend",N)}function N(O){O.target.nodeName&&O.target.nodeName.toLowerCase()==="html"||(n=!1,B())}document.addEventListener("keydown",p,!0),document.addEventListener("mousedown",m,!0),document.addEventListener("pointerdown",m,!0),document.addEventListener("touchstart",m,!0),document.addEventListener("visibilitychange",v,!0),Y(),r.addEventListener("focus",d,!0),r.addEventListener("blur",h,!0),r.nodeType===Node.DOCUMENT_FRAGMENT_NODE&&r.host?r.host.setAttribute("data-js-focus-visible",""):r.nodeType===Node.DOCUMENT_NODE&&(document.documentElement.classList.add("js-focus-visible"),document.documentElement.setAttribute("data-js-focus-visible",""))}if(typeof window!="undefined"&&typeof document!="undefined"){window.applyFocusVisiblePolyfill=e;var t;try{t=new CustomEvent("focus-visible-polyfill-ready")}catch(r){t=document.createEvent("CustomEvent"),t.initCustomEvent("focus-visible-polyfill-ready",!1,!1,{})}window.dispatchEvent(t)}typeof document!="undefined"&&e(document)})});var cn=Pt(Er=>{(function(e){var t=function(){try{return!!Symbol.iterator}catch(c){return!1}},r=t(),n=function(c){var u={next:function(){var p=c.shift();return{done:p===void 0,value:p}}};return r&&(u[Symbol.iterator]=function(){return u}),u},o=function(c){return encodeURIComponent(c).replace(/%20/g,"+")},i=function(c){return decodeURIComponent(String(c).replace(/\+/g," "))},s=function(){var c=function(p){Object.defineProperty(this,"_entries",{writable:!0,value:{}});var m=typeof p;if(m!=="undefined")if(m==="string")p!==""&&this._fromString(p);else if(p instanceof c){var d=this;p.forEach(function(B,N){d.append(N,B)})}else if(p!==null&&m==="object")if(Object.prototype.toString.call(p)==="[object Array]")for(var h=0;hd[0]?1:0}),c._entries&&(c._entries={});for(var p=0;p1?i(d[1]):"")}})})(typeof global!="undefined"?global:typeof window!="undefined"?window:typeof self!="undefined"?self:Er);(function(e){var t=function(){try{var o=new e.URL("b","http://a");return o.pathname="c d",o.href==="http://a/c%20d"&&o.searchParams}catch(i){return!1}},r=function(){var o=e.URL,i=function(f,c){typeof f!="string"&&(f=String(f)),c&&typeof c!="string"&&(c=String(c));var u=document,p;if(c&&(e.location===void 0||c!==e.location.href)){c=c.toLowerCase(),u=document.implementation.createHTMLDocument(""),p=u.createElement("base"),p.href=c,u.head.appendChild(p);try{if(p.href.indexOf(c)!==0)throw new Error(p.href)}catch(O){throw new Error("URL unable to set base "+c+" due to "+O)}}var m=u.createElement("a");m.href=f,p&&(u.body.appendChild(m),m.href=m.href);var d=u.createElement("input");if(d.type="url",d.value=f,m.protocol===":"||!/:/.test(m.href)||!d.checkValidity()&&!c)throw new TypeError("Invalid URL");Object.defineProperty(this,"_anchorElement",{value:m});var h=new e.URLSearchParams(this.search),v=!0,Y=!0,B=this;["append","delete","set"].forEach(function(O){var Qe=h[O];h[O]=function(){Qe.apply(h,arguments),v&&(Y=!1,B.search=h.toString(),Y=!0)}}),Object.defineProperty(this,"searchParams",{value:h,enumerable:!0});var N=void 0;Object.defineProperty(this,"_updateSearchParams",{enumerable:!1,configurable:!1,writable:!1,value:function(){this.search!==N&&(N=this.search,Y&&(v=!1,this.searchParams._fromString(this.search),v=!0))}})},s=i.prototype,a=function(f){Object.defineProperty(s,f,{get:function(){return this._anchorElement[f]},set:function(c){this._anchorElement[f]=c},enumerable:!0})};["hash","host","hostname","port","protocol"].forEach(function(f){a(f)}),Object.defineProperty(s,"search",{get:function(){return this._anchorElement.search},set:function(f){this._anchorElement.search=f,this._updateSearchParams()},enumerable:!0}),Object.defineProperties(s,{toString:{get:function(){var f=this;return function(){return f.href}}},href:{get:function(){return this._anchorElement.href.replace(/\?$/,"")},set:function(f){this._anchorElement.href=f,this._updateSearchParams()},enumerable:!0},pathname:{get:function(){return this._anchorElement.pathname.replace(/(^\/?)/,"/")},set:function(f){this._anchorElement.pathname=f},enumerable:!0},origin:{get:function(){var f={"http:":80,"https:":443,"ftp:":21}[this._anchorElement.protocol],c=this._anchorElement.port!=f&&this._anchorElement.port!=="";return this._anchorElement.protocol+"//"+this._anchorElement.hostname+(c?":"+this._anchorElement.port:"")},enumerable:!0},password:{get:function(){return""},set:function(f){},enumerable:!0},username:{get:function(){return""},set:function(f){},enumerable:!0}}),i.createObjectURL=function(f){return o.createObjectURL.apply(o,arguments)},i.revokeObjectURL=function(f){return o.revokeObjectURL.apply(o,arguments)},e.URL=i};if(t()||r(),e.location!==void 0&&!("origin"in e.location)){var n=function(){return e.location.protocol+"//"+e.location.hostname+(e.location.port?":"+e.location.port:"")};try{Object.defineProperty(e.location,"origin",{get:n,enumerable:!0})}catch(o){setInterval(function(){e.location.origin=n()},100)}}})(typeof global!="undefined"?global:typeof window!="undefined"?window:typeof self!="undefined"?self:Er)});var qr=Pt((Mt,Nr)=>{/*! + * clipboard.js v2.0.11 + * https://clipboardjs.com/ + * + * Licensed MIT © Zeno Rocha + */(function(t,r){typeof Mt=="object"&&typeof Nr=="object"?Nr.exports=r():typeof define=="function"&&define.amd?define([],r):typeof Mt=="object"?Mt.ClipboardJS=r():t.ClipboardJS=r()})(Mt,function(){return function(){var e={686:function(n,o,i){"use strict";i.d(o,{default:function(){return Ai}});var s=i(279),a=i.n(s),f=i(370),c=i.n(f),u=i(817),p=i.n(u);function m(j){try{return document.execCommand(j)}catch(T){return!1}}var d=function(T){var E=p()(T);return m("cut"),E},h=d;function v(j){var T=document.documentElement.getAttribute("dir")==="rtl",E=document.createElement("textarea");E.style.fontSize="12pt",E.style.border="0",E.style.padding="0",E.style.margin="0",E.style.position="absolute",E.style[T?"right":"left"]="-9999px";var H=window.pageYOffset||document.documentElement.scrollTop;return E.style.top="".concat(H,"px"),E.setAttribute("readonly",""),E.value=j,E}var Y=function(T,E){var H=v(T);E.container.appendChild(H);var I=p()(H);return m("copy"),H.remove(),I},B=function(T){var E=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body},H="";return typeof T=="string"?H=Y(T,E):T instanceof HTMLInputElement&&!["text","search","url","tel","password"].includes(T==null?void 0:T.type)?H=Y(T.value,E):(H=p()(T),m("copy")),H},N=B;function O(j){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?O=function(E){return typeof E}:O=function(E){return E&&typeof Symbol=="function"&&E.constructor===Symbol&&E!==Symbol.prototype?"symbol":typeof E},O(j)}var Qe=function(){var T=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},E=T.action,H=E===void 0?"copy":E,I=T.container,q=T.target,Me=T.text;if(H!=="copy"&&H!=="cut")throw new Error('Invalid "action" value, use either "copy" or "cut"');if(q!==void 0)if(q&&O(q)==="object"&&q.nodeType===1){if(H==="copy"&&q.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if(H==="cut"&&(q.hasAttribute("readonly")||q.hasAttribute("disabled")))throw new Error(`Invalid "target" attribute. You can't cut text from elements with "readonly" or "disabled" attributes`)}else throw new Error('Invalid "target" value, use a valid Element');if(Me)return N(Me,{container:I});if(q)return H==="cut"?h(q):N(q,{container:I})},De=Qe;function $e(j){"@babel/helpers - typeof";return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?$e=function(E){return typeof E}:$e=function(E){return E&&typeof Symbol=="function"&&E.constructor===Symbol&&E!==Symbol.prototype?"symbol":typeof E},$e(j)}function Ei(j,T){if(!(j instanceof T))throw new TypeError("Cannot call a class as a function")}function tn(j,T){for(var E=0;E0&&arguments[0]!==void 0?arguments[0]:{};this.action=typeof I.action=="function"?I.action:this.defaultAction,this.target=typeof I.target=="function"?I.target:this.defaultTarget,this.text=typeof I.text=="function"?I.text:this.defaultText,this.container=$e(I.container)==="object"?I.container:document.body}},{key:"listenClick",value:function(I){var q=this;this.listener=c()(I,"click",function(Me){return q.onClick(Me)})}},{key:"onClick",value:function(I){var q=I.delegateTarget||I.currentTarget,Me=this.action(q)||"copy",kt=De({action:Me,container:this.container,target:this.target(q),text:this.text(q)});this.emit(kt?"success":"error",{action:Me,text:kt,trigger:q,clearSelection:function(){q&&q.focus(),window.getSelection().removeAllRanges()}})}},{key:"defaultAction",value:function(I){return vr("action",I)}},{key:"defaultTarget",value:function(I){var q=vr("target",I);if(q)return document.querySelector(q)}},{key:"defaultText",value:function(I){return vr("text",I)}},{key:"destroy",value:function(){this.listener.destroy()}}],[{key:"copy",value:function(I){var q=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body};return N(I,q)}},{key:"cut",value:function(I){return h(I)}},{key:"isSupported",value:function(){var I=arguments.length>0&&arguments[0]!==void 0?arguments[0]:["copy","cut"],q=typeof I=="string"?[I]:I,Me=!!document.queryCommandSupported;return q.forEach(function(kt){Me=Me&&!!document.queryCommandSupported(kt)}),Me}}]),E}(a()),Ai=Li},828:function(n){var o=9;if(typeof Element!="undefined"&&!Element.prototype.matches){var i=Element.prototype;i.matches=i.matchesSelector||i.mozMatchesSelector||i.msMatchesSelector||i.oMatchesSelector||i.webkitMatchesSelector}function s(a,f){for(;a&&a.nodeType!==o;){if(typeof a.matches=="function"&&a.matches(f))return a;a=a.parentNode}}n.exports=s},438:function(n,o,i){var s=i(828);function a(u,p,m,d,h){var v=c.apply(this,arguments);return u.addEventListener(m,v,h),{destroy:function(){u.removeEventListener(m,v,h)}}}function f(u,p,m,d,h){return typeof u.addEventListener=="function"?a.apply(null,arguments):typeof m=="function"?a.bind(null,document).apply(null,arguments):(typeof u=="string"&&(u=document.querySelectorAll(u)),Array.prototype.map.call(u,function(v){return a(v,p,m,d,h)}))}function c(u,p,m,d){return function(h){h.delegateTarget=s(h.target,p),h.delegateTarget&&d.call(u,h)}}n.exports=f},879:function(n,o){o.node=function(i){return i!==void 0&&i instanceof HTMLElement&&i.nodeType===1},o.nodeList=function(i){var s=Object.prototype.toString.call(i);return i!==void 0&&(s==="[object NodeList]"||s==="[object HTMLCollection]")&&"length"in i&&(i.length===0||o.node(i[0]))},o.string=function(i){return typeof i=="string"||i instanceof String},o.fn=function(i){var s=Object.prototype.toString.call(i);return s==="[object Function]"}},370:function(n,o,i){var s=i(879),a=i(438);function f(m,d,h){if(!m&&!d&&!h)throw new Error("Missing required arguments");if(!s.string(d))throw new TypeError("Second argument must be a String");if(!s.fn(h))throw new TypeError("Third argument must be a Function");if(s.node(m))return c(m,d,h);if(s.nodeList(m))return u(m,d,h);if(s.string(m))return p(m,d,h);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList")}function c(m,d,h){return m.addEventListener(d,h),{destroy:function(){m.removeEventListener(d,h)}}}function u(m,d,h){return Array.prototype.forEach.call(m,function(v){v.addEventListener(d,h)}),{destroy:function(){Array.prototype.forEach.call(m,function(v){v.removeEventListener(d,h)})}}}function p(m,d,h){return a(document.body,m,d,h)}n.exports=f},817:function(n){function o(i){var s;if(i.nodeName==="SELECT")i.focus(),s=i.value;else if(i.nodeName==="INPUT"||i.nodeName==="TEXTAREA"){var a=i.hasAttribute("readonly");a||i.setAttribute("readonly",""),i.select(),i.setSelectionRange(0,i.value.length),a||i.removeAttribute("readonly"),s=i.value}else{i.hasAttribute("contenteditable")&&i.focus();var f=window.getSelection(),c=document.createRange();c.selectNodeContents(i),f.removeAllRanges(),f.addRange(c),s=f.toString()}return s}n.exports=o},279:function(n){function o(){}o.prototype={on:function(i,s,a){var f=this.e||(this.e={});return(f[i]||(f[i]=[])).push({fn:s,ctx:a}),this},once:function(i,s,a){var f=this;function c(){f.off(i,c),s.apply(a,arguments)}return c._=s,this.on(i,c,a)},emit:function(i){var s=[].slice.call(arguments,1),a=((this.e||(this.e={}))[i]||[]).slice(),f=0,c=a.length;for(f;f{"use strict";/*! + * escape-html + * Copyright(c) 2012-2013 TJ Holowaychuk + * Copyright(c) 2015 Andreas Lubbe + * Copyright(c) 2015 Tiancheng "Timothy" Gu + * MIT Licensed + */var rs=/["'&<>]/;Yo.exports=ns;function ns(e){var t=""+e,r=rs.exec(t);if(!r)return t;var n,o="",i=0,s=0;for(i=r.index;i0&&i[i.length-1])&&(c[0]===6||c[0]===2)){r=0;continue}if(c[0]===3&&(!i||c[1]>i[0]&&c[1]=e.length&&(e=void 0),{value:e&&e[n++],done:!e}}};throw new TypeError(t?"Object is not iterable.":"Symbol.iterator is not defined.")}function W(e,t){var r=typeof Symbol=="function"&&e[Symbol.iterator];if(!r)return e;var n=r.call(e),o,i=[],s;try{for(;(t===void 0||t-- >0)&&!(o=n.next()).done;)i.push(o.value)}catch(a){s={error:a}}finally{try{o&&!o.done&&(r=n.return)&&r.call(n)}finally{if(s)throw s.error}}return i}function D(e,t,r){if(r||arguments.length===2)for(var n=0,o=t.length,i;n1||a(m,d)})})}function a(m,d){try{f(n[m](d))}catch(h){p(i[0][3],h)}}function f(m){m.value instanceof et?Promise.resolve(m.value.v).then(c,u):p(i[0][2],m)}function c(m){a("next",m)}function u(m){a("throw",m)}function p(m,d){m(d),i.shift(),i.length&&a(i[0][0],i[0][1])}}function pn(e){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var t=e[Symbol.asyncIterator],r;return t?t.call(e):(e=typeof Ee=="function"?Ee(e):e[Symbol.iterator](),r={},n("next"),n("throw"),n("return"),r[Symbol.asyncIterator]=function(){return this},r);function n(i){r[i]=e[i]&&function(s){return new Promise(function(a,f){s=e[i](s),o(a,f,s.done,s.value)})}}function o(i,s,a,f){Promise.resolve(f).then(function(c){i({value:c,done:a})},s)}}function C(e){return typeof e=="function"}function at(e){var t=function(n){Error.call(n),n.stack=new Error().stack},r=e(t);return r.prototype=Object.create(Error.prototype),r.prototype.constructor=r,r}var It=at(function(e){return function(r){e(this),this.message=r?r.length+` errors occurred during unsubscription: +`+r.map(function(n,o){return o+1+") "+n.toString()}).join(` + `):"",this.name="UnsubscriptionError",this.errors=r}});function Ve(e,t){if(e){var r=e.indexOf(t);0<=r&&e.splice(r,1)}}var Ie=function(){function e(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._finalizers=null}return e.prototype.unsubscribe=function(){var t,r,n,o,i;if(!this.closed){this.closed=!0;var s=this._parentage;if(s)if(this._parentage=null,Array.isArray(s))try{for(var a=Ee(s),f=a.next();!f.done;f=a.next()){var c=f.value;c.remove(this)}}catch(v){t={error:v}}finally{try{f&&!f.done&&(r=a.return)&&r.call(a)}finally{if(t)throw t.error}}else s.remove(this);var u=this.initialTeardown;if(C(u))try{u()}catch(v){i=v instanceof It?v.errors:[v]}var p=this._finalizers;if(p){this._finalizers=null;try{for(var m=Ee(p),d=m.next();!d.done;d=m.next()){var h=d.value;try{ln(h)}catch(v){i=i!=null?i:[],v instanceof It?i=D(D([],W(i)),W(v.errors)):i.push(v)}}}catch(v){n={error:v}}finally{try{d&&!d.done&&(o=m.return)&&o.call(m)}finally{if(n)throw n.error}}}if(i)throw new It(i)}},e.prototype.add=function(t){var r;if(t&&t!==this)if(this.closed)ln(t);else{if(t instanceof e){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._finalizers=(r=this._finalizers)!==null&&r!==void 0?r:[]).push(t)}},e.prototype._hasParent=function(t){var r=this._parentage;return r===t||Array.isArray(r)&&r.includes(t)},e.prototype._addParent=function(t){var r=this._parentage;this._parentage=Array.isArray(r)?(r.push(t),r):r?[r,t]:t},e.prototype._removeParent=function(t){var r=this._parentage;r===t?this._parentage=null:Array.isArray(r)&&Ve(r,t)},e.prototype.remove=function(t){var r=this._finalizers;r&&Ve(r,t),t instanceof e&&t._removeParent(this)},e.EMPTY=function(){var t=new e;return t.closed=!0,t}(),e}();var Sr=Ie.EMPTY;function jt(e){return e instanceof Ie||e&&"closed"in e&&C(e.remove)&&C(e.add)&&C(e.unsubscribe)}function ln(e){C(e)?e():e.unsubscribe()}var Le={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1};var st={setTimeout:function(e,t){for(var r=[],n=2;n0},enumerable:!1,configurable:!0}),t.prototype._trySubscribe=function(r){return this._throwIfClosed(),e.prototype._trySubscribe.call(this,r)},t.prototype._subscribe=function(r){return this._throwIfClosed(),this._checkFinalizedStatuses(r),this._innerSubscribe(r)},t.prototype._innerSubscribe=function(r){var n=this,o=this,i=o.hasError,s=o.isStopped,a=o.observers;return i||s?Sr:(this.currentObservers=null,a.push(r),new Ie(function(){n.currentObservers=null,Ve(a,r)}))},t.prototype._checkFinalizedStatuses=function(r){var n=this,o=n.hasError,i=n.thrownError,s=n.isStopped;o?r.error(i):s&&r.complete()},t.prototype.asObservable=function(){var r=new F;return r.source=this,r},t.create=function(r,n){return new xn(r,n)},t}(F);var xn=function(e){ie(t,e);function t(r,n){var o=e.call(this)||this;return o.destination=r,o.source=n,o}return t.prototype.next=function(r){var n,o;(o=(n=this.destination)===null||n===void 0?void 0:n.next)===null||o===void 0||o.call(n,r)},t.prototype.error=function(r){var n,o;(o=(n=this.destination)===null||n===void 0?void 0:n.error)===null||o===void 0||o.call(n,r)},t.prototype.complete=function(){var r,n;(n=(r=this.destination)===null||r===void 0?void 0:r.complete)===null||n===void 0||n.call(r)},t.prototype._subscribe=function(r){var n,o;return(o=(n=this.source)===null||n===void 0?void 0:n.subscribe(r))!==null&&o!==void 0?o:Sr},t}(x);var Et={now:function(){return(Et.delegate||Date).now()},delegate:void 0};var wt=function(e){ie(t,e);function t(r,n,o){r===void 0&&(r=1/0),n===void 0&&(n=1/0),o===void 0&&(o=Et);var i=e.call(this)||this;return i._bufferSize=r,i._windowTime=n,i._timestampProvider=o,i._buffer=[],i._infiniteTimeWindow=!0,i._infiniteTimeWindow=n===1/0,i._bufferSize=Math.max(1,r),i._windowTime=Math.max(1,n),i}return t.prototype.next=function(r){var n=this,o=n.isStopped,i=n._buffer,s=n._infiniteTimeWindow,a=n._timestampProvider,f=n._windowTime;o||(i.push(r),!s&&i.push(a.now()+f)),this._trimBuffer(),e.prototype.next.call(this,r)},t.prototype._subscribe=function(r){this._throwIfClosed(),this._trimBuffer();for(var n=this._innerSubscribe(r),o=this,i=o._infiniteTimeWindow,s=o._buffer,a=s.slice(),f=0;f0?e.prototype.requestAsyncId.call(this,r,n,o):(r.actions.push(this),r._scheduled||(r._scheduled=ut.requestAnimationFrame(function(){return r.flush(void 0)})))},t.prototype.recycleAsyncId=function(r,n,o){var i;if(o===void 0&&(o=0),o!=null?o>0:this.delay>0)return e.prototype.recycleAsyncId.call(this,r,n,o);var s=r.actions;n!=null&&((i=s[s.length-1])===null||i===void 0?void 0:i.id)!==n&&(ut.cancelAnimationFrame(n),r._scheduled=void 0)},t}(Wt);var Sn=function(e){ie(t,e);function t(){return e!==null&&e.apply(this,arguments)||this}return t.prototype.flush=function(r){this._active=!0;var n=this._scheduled;this._scheduled=void 0;var o=this.actions,i;r=r||o.shift();do if(i=r.execute(r.state,r.delay))break;while((r=o[0])&&r.id===n&&o.shift());if(this._active=!1,i){for(;(r=o[0])&&r.id===n&&o.shift();)r.unsubscribe();throw i}},t}(Dt);var Oe=new Sn(wn);var M=new F(function(e){return e.complete()});function Vt(e){return e&&C(e.schedule)}function Cr(e){return e[e.length-1]}function Ye(e){return C(Cr(e))?e.pop():void 0}function Te(e){return Vt(Cr(e))?e.pop():void 0}function zt(e,t){return typeof Cr(e)=="number"?e.pop():t}var pt=function(e){return e&&typeof e.length=="number"&&typeof e!="function"};function Nt(e){return C(e==null?void 0:e.then)}function qt(e){return C(e[ft])}function Kt(e){return Symbol.asyncIterator&&C(e==null?void 0:e[Symbol.asyncIterator])}function Qt(e){return new TypeError("You provided "+(e!==null&&typeof e=="object"?"an invalid object":"'"+e+"'")+" where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.")}function zi(){return typeof Symbol!="function"||!Symbol.iterator?"@@iterator":Symbol.iterator}var Yt=zi();function Gt(e){return C(e==null?void 0:e[Yt])}function Bt(e){return un(this,arguments,function(){var r,n,o,i;return $t(this,function(s){switch(s.label){case 0:r=e.getReader(),s.label=1;case 1:s.trys.push([1,,9,10]),s.label=2;case 2:return[4,et(r.read())];case 3:return n=s.sent(),o=n.value,i=n.done,i?[4,et(void 0)]:[3,5];case 4:return[2,s.sent()];case 5:return[4,et(o)];case 6:return[4,s.sent()];case 7:return s.sent(),[3,2];case 8:return[3,10];case 9:return r.releaseLock(),[7];case 10:return[2]}})})}function Jt(e){return C(e==null?void 0:e.getReader)}function U(e){if(e instanceof F)return e;if(e!=null){if(qt(e))return Ni(e);if(pt(e))return qi(e);if(Nt(e))return Ki(e);if(Kt(e))return On(e);if(Gt(e))return Qi(e);if(Jt(e))return Yi(e)}throw Qt(e)}function Ni(e){return new F(function(t){var r=e[ft]();if(C(r.subscribe))return r.subscribe(t);throw new TypeError("Provided object does not correctly implement Symbol.observable")})}function qi(e){return new F(function(t){for(var r=0;r=2;return function(n){return n.pipe(e?A(function(o,i){return e(o,i,n)}):de,ge(1),r?He(t):Dn(function(){return new Zt}))}}function Vn(){for(var e=[],t=0;t=2,!0))}function pe(e){e===void 0&&(e={});var t=e.connector,r=t===void 0?function(){return new x}:t,n=e.resetOnError,o=n===void 0?!0:n,i=e.resetOnComplete,s=i===void 0?!0:i,a=e.resetOnRefCountZero,f=a===void 0?!0:a;return function(c){var u,p,m,d=0,h=!1,v=!1,Y=function(){p==null||p.unsubscribe(),p=void 0},B=function(){Y(),u=m=void 0,h=v=!1},N=function(){var O=u;B(),O==null||O.unsubscribe()};return y(function(O,Qe){d++,!v&&!h&&Y();var De=m=m!=null?m:r();Qe.add(function(){d--,d===0&&!v&&!h&&(p=$r(N,f))}),De.subscribe(Qe),!u&&d>0&&(u=new rt({next:function($e){return De.next($e)},error:function($e){v=!0,Y(),p=$r(B,o,$e),De.error($e)},complete:function(){h=!0,Y(),p=$r(B,s),De.complete()}}),U(O).subscribe(u))})(c)}}function $r(e,t){for(var r=[],n=2;ne.next(document)),e}function K(e,t=document){return Array.from(t.querySelectorAll(e))}function z(e,t=document){let r=ce(e,t);if(typeof r=="undefined")throw new ReferenceError(`Missing element: expected "${e}" to be present`);return r}function ce(e,t=document){return t.querySelector(e)||void 0}function _e(){return document.activeElement instanceof HTMLElement&&document.activeElement||void 0}function tr(e){return L(b(document.body,"focusin"),b(document.body,"focusout")).pipe(ke(1),l(()=>{let t=_e();return typeof t!="undefined"?e.contains(t):!1}),V(e===_e()),J())}function Xe(e){return{x:e.offsetLeft,y:e.offsetTop}}function Kn(e){return L(b(window,"load"),b(window,"resize")).pipe(Ce(0,Oe),l(()=>Xe(e)),V(Xe(e)))}function rr(e){return{x:e.scrollLeft,y:e.scrollTop}}function dt(e){return L(b(e,"scroll"),b(window,"resize")).pipe(Ce(0,Oe),l(()=>rr(e)),V(rr(e)))}var Yn=function(){if(typeof Map!="undefined")return Map;function e(t,r){var n=-1;return t.some(function(o,i){return o[0]===r?(n=i,!0):!1}),n}return function(){function t(){this.__entries__=[]}return Object.defineProperty(t.prototype,"size",{get:function(){return this.__entries__.length},enumerable:!0,configurable:!0}),t.prototype.get=function(r){var n=e(this.__entries__,r),o=this.__entries__[n];return o&&o[1]},t.prototype.set=function(r,n){var o=e(this.__entries__,r);~o?this.__entries__[o][1]=n:this.__entries__.push([r,n])},t.prototype.delete=function(r){var n=this.__entries__,o=e(n,r);~o&&n.splice(o,1)},t.prototype.has=function(r){return!!~e(this.__entries__,r)},t.prototype.clear=function(){this.__entries__.splice(0)},t.prototype.forEach=function(r,n){n===void 0&&(n=null);for(var o=0,i=this.__entries__;o0},e.prototype.connect_=function(){!Wr||this.connected_||(document.addEventListener("transitionend",this.onTransitionEnd_),window.addEventListener("resize",this.refresh),va?(this.mutationsObserver_=new MutationObserver(this.refresh),this.mutationsObserver_.observe(document,{attributes:!0,childList:!0,characterData:!0,subtree:!0})):(document.addEventListener("DOMSubtreeModified",this.refresh),this.mutationEventsAdded_=!0),this.connected_=!0)},e.prototype.disconnect_=function(){!Wr||!this.connected_||(document.removeEventListener("transitionend",this.onTransitionEnd_),window.removeEventListener("resize",this.refresh),this.mutationsObserver_&&this.mutationsObserver_.disconnect(),this.mutationEventsAdded_&&document.removeEventListener("DOMSubtreeModified",this.refresh),this.mutationsObserver_=null,this.mutationEventsAdded_=!1,this.connected_=!1)},e.prototype.onTransitionEnd_=function(t){var r=t.propertyName,n=r===void 0?"":r,o=ba.some(function(i){return!!~n.indexOf(i)});o&&this.refresh()},e.getInstance=function(){return this.instance_||(this.instance_=new e),this.instance_},e.instance_=null,e}(),Gn=function(e,t){for(var r=0,n=Object.keys(t);r0},e}(),Jn=typeof WeakMap!="undefined"?new WeakMap:new Yn,Xn=function(){function e(t){if(!(this instanceof e))throw new TypeError("Cannot call a class as a function.");if(!arguments.length)throw new TypeError("1 argument required, but only 0 present.");var r=ga.getInstance(),n=new La(t,r,this);Jn.set(this,n)}return e}();["observe","unobserve","disconnect"].forEach(function(e){Xn.prototype[e]=function(){var t;return(t=Jn.get(this))[e].apply(t,arguments)}});var Aa=function(){return typeof nr.ResizeObserver!="undefined"?nr.ResizeObserver:Xn}(),Zn=Aa;var eo=new x,Ca=$(()=>k(new Zn(e=>{for(let t of e)eo.next(t)}))).pipe(g(e=>L(ze,k(e)).pipe(R(()=>e.disconnect()))),X(1));function he(e){return{width:e.offsetWidth,height:e.offsetHeight}}function ye(e){return Ca.pipe(S(t=>t.observe(e)),g(t=>eo.pipe(A(({target:r})=>r===e),R(()=>t.unobserve(e)),l(()=>he(e)))),V(he(e)))}function bt(e){return{width:e.scrollWidth,height:e.scrollHeight}}function ar(e){let t=e.parentElement;for(;t&&(e.scrollWidth<=t.scrollWidth&&e.scrollHeight<=t.scrollHeight);)t=(e=t).parentElement;return t?e:void 0}var to=new x,Ra=$(()=>k(new IntersectionObserver(e=>{for(let t of e)to.next(t)},{threshold:0}))).pipe(g(e=>L(ze,k(e)).pipe(R(()=>e.disconnect()))),X(1));function sr(e){return Ra.pipe(S(t=>t.observe(e)),g(t=>to.pipe(A(({target:r})=>r===e),R(()=>t.unobserve(e)),l(({isIntersecting:r})=>r))))}function ro(e,t=16){return dt(e).pipe(l(({y:r})=>{let n=he(e),o=bt(e);return r>=o.height-n.height-t}),J())}var cr={drawer:z("[data-md-toggle=drawer]"),search:z("[data-md-toggle=search]")};function no(e){return cr[e].checked}function Ke(e,t){cr[e].checked!==t&&cr[e].click()}function Ue(e){let t=cr[e];return b(t,"change").pipe(l(()=>t.checked),V(t.checked))}function ka(e,t){switch(e.constructor){case HTMLInputElement:return e.type==="radio"?/^Arrow/.test(t):!0;case HTMLSelectElement:case HTMLTextAreaElement:return!0;default:return e.isContentEditable}}function Ha(){return L(b(window,"compositionstart").pipe(l(()=>!0)),b(window,"compositionend").pipe(l(()=>!1))).pipe(V(!1))}function oo(){let e=b(window,"keydown").pipe(A(t=>!(t.metaKey||t.ctrlKey)),l(t=>({mode:no("search")?"search":"global",type:t.key,claim(){t.preventDefault(),t.stopPropagation()}})),A(({mode:t,type:r})=>{if(t==="global"){let n=_e();if(typeof n!="undefined")return!ka(n,r)}return!0}),pe());return Ha().pipe(g(t=>t?M:e))}function le(){return new URL(location.href)}function ot(e){location.href=e.href}function io(){return new x}function ao(e,t){if(typeof t=="string"||typeof t=="number")e.innerHTML+=t.toString();else if(t instanceof Node)e.appendChild(t);else if(Array.isArray(t))for(let r of t)ao(e,r)}function _(e,t,...r){let n=document.createElement(e);if(t)for(let o of Object.keys(t))typeof t[o]!="undefined"&&(typeof t[o]!="boolean"?n.setAttribute(o,t[o]):n.setAttribute(o,""));for(let o of r)ao(n,o);return n}function fr(e){if(e>999){let t=+((e-950)%1e3>99);return`${((e+1e-6)/1e3).toFixed(t)}k`}else return e.toString()}function so(){return location.hash.substring(1)}function Dr(e){let t=_("a",{href:e});t.addEventListener("click",r=>r.stopPropagation()),t.click()}function Pa(e){return L(b(window,"hashchange"),e).pipe(l(so),V(so()),A(t=>t.length>0),X(1))}function co(e){return Pa(e).pipe(l(t=>ce(`[id="${t}"]`)),A(t=>typeof t!="undefined"))}function Vr(e){let t=matchMedia(e);return er(r=>t.addListener(()=>r(t.matches))).pipe(V(t.matches))}function fo(){let e=matchMedia("print");return L(b(window,"beforeprint").pipe(l(()=>!0)),b(window,"afterprint").pipe(l(()=>!1))).pipe(V(e.matches))}function zr(e,t){return e.pipe(g(r=>r?t():M))}function ur(e,t={credentials:"same-origin"}){return ue(fetch(`${e}`,t)).pipe(fe(()=>M),g(r=>r.status!==200?Ot(()=>new Error(r.statusText)):k(r)))}function We(e,t){return ur(e,t).pipe(g(r=>r.json()),X(1))}function uo(e,t){let r=new DOMParser;return ur(e,t).pipe(g(n=>n.text()),l(n=>r.parseFromString(n,"text/xml")),X(1))}function pr(e){let t=_("script",{src:e});return $(()=>(document.head.appendChild(t),L(b(t,"load"),b(t,"error").pipe(g(()=>Ot(()=>new ReferenceError(`Invalid script: ${e}`))))).pipe(l(()=>{}),R(()=>document.head.removeChild(t)),ge(1))))}function po(){return{x:Math.max(0,scrollX),y:Math.max(0,scrollY)}}function lo(){return L(b(window,"scroll",{passive:!0}),b(window,"resize",{passive:!0})).pipe(l(po),V(po()))}function mo(){return{width:innerWidth,height:innerHeight}}function ho(){return b(window,"resize",{passive:!0}).pipe(l(mo),V(mo()))}function bo(){return G([lo(),ho()]).pipe(l(([e,t])=>({offset:e,size:t})),X(1))}function lr(e,{viewport$:t,header$:r}){let n=t.pipe(ee("size")),o=G([n,r]).pipe(l(()=>Xe(e)));return G([r,t,o]).pipe(l(([{height:i},{offset:s,size:a},{x:f,y:c}])=>({offset:{x:s.x-f,y:s.y-c+i},size:a})))}(()=>{function e(n,o){parent.postMessage(n,o||"*")}function t(...n){return n.reduce((o,i)=>o.then(()=>new Promise(s=>{let a=document.createElement("script");a.src=i,a.onload=s,document.body.appendChild(a)})),Promise.resolve())}var r=class extends EventTarget{constructor(n){super(),this.url=n,this.m=i=>{i.source===this.w&&(this.dispatchEvent(new MessageEvent("message",{data:i.data})),this.onmessage&&this.onmessage(i))},this.e=(i,s,a,f,c)=>{if(s===`${this.url}`){let u=new ErrorEvent("error",{message:i,filename:s,lineno:a,colno:f,error:c});this.dispatchEvent(u),this.onerror&&this.onerror(u)}};let o=document.createElement("iframe");o.hidden=!0,document.body.appendChild(this.iframe=o),this.w.document.open(),this.w.document.write(` + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Changelog

+ +

Note: Numbers like (#123) point to closed Pull Requests on the fractal-tasks-core repository.

+

0.13.0

+
    +
  • Tasks:
      +
    • New task and helper functions:
        +
      • Introduce import_ome_zarr task (#557, #579).
        • +
      • Introduce get_single_image_ROI and get_image_grid_ROIs (#557).
        • +
      • Introduce detect_ome_ngff_type (#557).
        • +
      • Introduce update_omero_channels (#579).
        • +
      +
      • +
    • Make maximum_intensity_projection independent from ROI tables (#557).
      • +
    • Make Cellpose task work when input_ROI_table is empty (#566).
      • +
    • Fix bug of missing attributes in ROI-table Zarr group (#573).
      • +
    +
    • +
  • Dependencies:
      +
    • Restrict Pillow version to <10.1 (#571).
      • +
    • Support AnnData 0.10 (#574).
      • +
    +
    • +
  • Testing:
      +
    • Align with new Zenodo API (#568).
      • +
    • Use ubuntu-22 for GitHub CI (#576).
      • +
    +
    • +
+

0.12.2

+
    +
  • Relax check_valid_ROI_indices to support search-first scenario (#555).
    • +
  • Do not install docs dependencies in GitHub CI (#551).
    • +
+

0.12.1

+
    +
  • Make Channel.window attribute optional in lib_ngff.py (#548).
    • +
  • Automate procedure for publishing package to PyPI (#545).
    • +
+

0.12.0

+

This release includes work on Pydantic models for NGFF specs and on ROI tables.

+
    +
  • NGFF Pydantic models:
      +
    • Introduce Pydantic models for NGFF metadata in lib_ngff.py (#528).
      • +
    • Extract num_levels and coarsening_xy parameters from NGFF objects, rather than from metadata task input (#528).
      • +
    • Transform several lib_zattrs_utils.py functions (get_axes_names, extract_zyx_pixel_sizes and get_acquisition_paths) into lib_ngff.py methods (#528).
      • +
    • Load Zarr attributes from groups, rather than from .zattrs files (#528).
      • +
    +
    • +
  • Regions of interest:
      +
    • Set FOV_ROI_table and well_ROI_table ZYX origin to zero (#524).
      • +
    • Remove heuristics to determine whether to reset origin, in cellpose_segmentation task (#524).
      • +
    • Remove obsolete reset_origin argument from convert_ROI_table_to_indices function (#524).
      • +
    • Remove redundant reset_origin call from apply_registration_to_ROI_tables task (#524).
      • +
    • Add check on non-negative ROI indices (#534).
      • +
    • Add check on ROI indices not starting at (0,0,0), to highlight v0.12/v0.11 incompatibility (#534).
      • +
    • Fix bug in creation of bounding-box ROIs when cellpose_segmentation loops of FOVs (#524).
      • +
    • Update type of metadata parameter of prepare_FOV_ROI_table and prepare_well_ROI_table functions (#524).
      • +
    • Fix reset_origin so that it returns an updated copy of its input (#524).
      • +
    +
    • +
  • Dependencies:
      +
    • Relax fsspec<=2023.6 constraint into fsspec!=2023.9.0 (#539).
      • +
    +
    • +
+

0.11.0

+
    +
  • Tasks:
      +
    • (major) Introduce new tasks for registration of multiplexing cycles: calculate_registration_image_based, apply_registration_to_ROI_tables, apply_registration_to_image (#487).
      • +
    • (major) Introduce new overwrite argument for tasks create_ome_zarr, create_ome_zarr_multiplex, yokogawa_to_ome_zarr, copy_ome_zarr, maximum_intensity_projection, cellpose_segmentation, napari_workflows_wrapper (#499).
      • +
    • (major) Rename illumination_correction parameter from overwrite to overwrite_input (#499).
      • +
    • Fix plate-selection bug in copy_ome_zarr task (#513).
      • +
    • Fix bug in definition of metadata["plate"] in create_ome_zarr_multiplex task (#513).
      • +
    • Introduce new helper functions write_table, prepare_label_group and open_zarr_group_with_overwrite (#499).
      • +
    • Introduce new helper functions are_ROI_table_columns_valid, convert_indices_to_regions, reset_origin, is_standard_roi_table, get_acquisition_paths, get_table_path_dict, get_axes_names, add_zero_translation_columns, calculate_min_max_across_dfs, apply_registration_to_single_ROI_table, write_registered_zarr, calculate_physical_shifts, get_ROI_table_with_translation (#487).
      • +
    +
    • +
  • Testing:
      +
    • Add tests for overwrite-related task behaviors (#499).
      • +
    • Introduce mock-up of napari_skimage_regionprops package, for testing of + napari_workflows_wrapper task (#499).
      • +
    +
    • +
  • Dependencies:
      +
    • Require fsspec version to be <=2023.6 (#509).
      • +
    +
    • +
+

0.10.1

+
    +
  • Tasks:
      +
    • Improve validation for OmeroChannel.color field (#488).
      • +
    • Include image-label/source/image OME-NGFF attribute when creating labels (#478).
      • +
    • Update default values for tolerance (tol) in lib_ROI_overlaps.py functions (#466).
      • +
    +
    • +
  • Development tools:
      +
    • Include docs_info and docs_link attributes in manifest tasks (#486).
      • +
    • Rename and revamp scripts to update/check the manifest (#486).
      • +
    • Improve logging and error-handling in tools for args-schema creation (#469).
      • +
    +
    • +
  • Documentation:
      +
    • Convert docstrings to Google style (#473, #479).
      • +
    • Switch from sphinx to mkdocs for documentation (#479).
      • +
    • Update generic type hints (#462, #479).
      • +
    • Align examples to recent package version, and mention them in the documentation (#470).
      • +
    +
    • +
  • Testing:
      +
    • Improve coverage of core library (#459, #467, #468).
      • +
    • Update Zenodo datasets used in tests (#454).
      • +
    • Run tests both for the poetry-installed and pip-installed package (#455).
      • +
    +
    • +
  • Dependencies:
      +
    • Relax numpy required version to <2 (#477).
      • +
    • Relax dask required version to >=2023.1.0 (#455).
      • +
    • Relax zarr required version to >=2.13.6,<3 (#455).
      • +
    • Relax pandas required version to >=1.2.0,<2 (#455).
      • +
    • Relax Pillow required version to >=9.1.1,<10.0.0 (#455).
      • +
    • Full update of poetry.lock file (mutiple PRs, e.g. #472).
      • +
    • Include requests and wget in the dev poetry dependency group (#455).
      • +
    +
    • +
+

0.10.0

+
    +
  • Restructure the package and repository:
      +
    • Move tasks to tasks subpackage (#390)
      • +
    • Create new dev subpackage (#384).
      • +
    • Make tasks-related dependencies optional, and installable via fractal-tasks extra (#390).
      • +
    • Remove tools package extra (#384), and split the subpackage content into lib_ROI_overlaps and examples (#390).
      • +
    +
    • +
  • (major) Modify task arguments
      +
    • Add Pydantic model lib_channels.OmeroChannel (#410, #422);
      • +
    • Add Pydantic model tasks._input_models.Channel (#422);
      • +
    • Add Pydantic model tasks._input_models.NapariWorkflowsInput (#422);
      • +
    • Add Pydantic model tasks._input_models.NapariWorkflowsOutput (#422);
      • +
    • Move all Pydantic models to main package (#438).
      • +
    • Modify arguments of illumination_correction task (#431);
      • +
    • Modify arguments of create_ome_zarr and create_ome_zarr_multiplex (#433).
      • +
    • Modify argument default for ROI_table_names, in copy_ome_zarr (#449).
      • +
    • Remove the delete option from yokogawa to ome zarr (#443).
      • +
    • Reorder task inputs (#451).
      • +
    +
    • +
  • JSON Schemas for task arguments:
      +
    • Add JSON Schemas for task arguments in the package manifest (#369, #384).
      • +
    • Add JSON Schemas for attributes of custom task-argument Pydantic models (#436).
      • +
    • Make schema-generation tools more general, when handling custom Pydantic models (#445).
      • +
    • Include titles for custom-model-typed arguments and argument attributes (#447).
      • +
    • Remove TaskArguments models and switch to Pydantic V1 validate_arguments (#369).
      • +
    • Make coercing&validating task arguments required, rather than optional (#408).
      • +
    • Remove default_args from manifest (#379, #393).
      • +
    +
    • +
  • Other:
      +
    • Make pydantic dependency required for running tasks, and pin it to V1 (#408).
      • +
    • Remove legacy executor definitions from manifest (#361).
      • +
    • Add GitHub action for testing pip install with/without fractal-tasks extra (#390).
      • +
    • Remove sqlmodel from dev dependencies (#374).
      • +
    • Relax constraint on torch version, from ==1.12.1 to <=2.0.0 (#406).
      • +
    • Review task docstrings and improve documentation (#413, #416).
      • +
    • Update anndata dependency requirements (from ^0.8.0 to >=0.8.0,<=0.9.1), and replace anndata.experimental.write_elem with anndata._io.specs.write_elem (#428).
      • +
    +
    • +
+

0.9.4

+
    +
  • Relax constraint on scikit-image version, by only requiring a version >=0.19 (#367).
    • +
+

0.9.3

+
    +
  • For labeling tasks (cellpose_segmentation or napari_worfklows_wrapper), allow empty ROI tables as input or output (#365).
    • +
  • Relax constraint related to the presence of channels in create_zarr_structure_multiplex task (#365).
    • +
+

0.9.2

+
    +
  • Increase memory requirements for some tasks in manifest (#363).
    • +
+

0.9.1

+
    +
  • Add use_gpu argument for cellpose_segmentation task (#350).
    • +
  • Add dummy return object to napari-workflows task (#359).
    • +
  • Include memory/cpu/gpu requirements in manifest, in view of new fractal-server SLURM backend (#360).
    • +
+

0.9.0

+
    +
  • Introduce a module for masked loading of ROIs, and update the cellpose_segmentation task accordingly (#306).
    • +
  • Rename task arguments: ROI_table_name->input_ROI_table and bounding_box_ROI_table_name->output_ROI_table (#306).
    • +
  • Implement part of the proposed table support in OME-NGFF specs, both for the tables zarr group and then for each table subgroup (#306).
    • +
  • Rename module: lib_remove_FOV_overlaps.py->lib_ROI_overlaps.py (#306).
    • +
  • Add new functions to existing modules: lib_regions_of_interest.convert_region_to_low_res, lib_ROI_overlaps.find_overlaps_in_ROI_indices (#306).
    • +
+

0.8.1

+
    +
  • Disable bugged validation of model_type argument in cellpose_segmentation (#344).
    • +
  • Raise an error if the user provides an unexpected argument to a task (#337); this applies to the case of running a task as a script, with a pydantic model for task-argument validation.
    • +
+

0.8.0

+
    +
  • (major) Update task interface: remove filename extension from input_paths and output_path for all tasks, and add new arguments (image_extension,image_glob_pattern) to create_ome_zarr task (#323).
    • +
  • Implement logic for handling image_glob_patterns argument, both when globbing images and in Yokogawa metadata parsing (#326).
    • +
  • Fix minor bugs in task arguments (#329).
    • +
+

0.7.5

+
    +
  • Update cellpose_segmentation defaults and parse additional parameters (#316).
    • +
  • Add dual-channel input for cellpose_segmentation task (#315).
    • +
+

0.7.4

+
    +
  • Add tests for python 3.10 (#309).
    • +
  • Drop support for python 3.8 (#319).
    • +
  • Update task interface: use string arguments instead of pathlib.Path, and only set defaults in function call signatures (#303).
    • +
+

0.7.3

+
    +
  • Add reset_origin argument to convert_ROI_table_to_indices (#305).
    • +
  • Do not overwrite existing labels in cellpose_segmentation task (#308).
    • +
+

0.7.2

+
    +
  • Remove pyqt5-related dependencies (#288).
    • +
+

0.7.1

+

Missing

+

0.7.0

+
    +
  • Replace dask.array.core.get_mapper() with zarr.storage.FSStore() (#282).
    • +
  • Pin dask version to >=2023.1.0, <2023.2.
    • +
  • Pin zarr version to >=2.13.6, <2.14.
    • +
  • Pin numpy version to >=1.23.5,<1.24.
    • +
  • Pin cellpose version to >=2.2,<2.3.
    • +
+

0.6.5

+
    +
  • Remove FOV overlaps with more flexibility (#265).
    • +
+

0.6.4

+
    +
  • Created tools submodule and installation extra (#262).
    • +
+

0.6.3

+
    +
  • Added napari dependency, pinned to 0.4.16 version.
    • +
  • Fixed type-hinting bug in task to create multiplexing OME-Zarr structure (#258).
    • +
+

0.6.2

+
    +
  • Support passing a pre-made metadata table to tasks creating the OME-Zarr structure (#252).
    • +
+

0.6.1

+
    +
  • Add option for padding an array with zeros in upscale_array (#251).
    • +
  • Simplified imagecodecs and PyQt5 dependencies (#248).
    • +
+

0.6.0

+
    +
  • (major) Refactor of how to address channels (#239).
    • +
  • Fix bug in well ROI table (#245).
    • +
+

0.5.1

+
    +
  • Fix sorting of image files when number of Z planes passes 100 (#237).
    • +
+

0.5.0

+
    +
  • (major) Deprecate measurement task (#235).
    • +
  • (major) Use more uniform names for tasks, both in python modules and manifest (#235).
    • +
  • Remove deprecated manifest from __init__.py (#233).
    • +
+

0.4.6

+
    +
  • Skip image files if filename is not parsable (#219).
    • +
  • Preserve order of input_paths for multiplexing subfolders (#222).
    • +
  • Major refactor of replicate_zarr_structure, also enabling support for zarr files with multiple images (#223).
    • +
+

0.4.5

+
    +
  • Replace Cellpose wrapper with CellposeModel, to support pretrained_model argument (#218).
    • +
  • Update cellpose version (it was pinned to 2.0, in previous versions) (#218).
    • +
  • Pin torch dependency to version 1.12.1, to support CUDA version 10.2 (#218).
    • +
+

0.4.4

+

Missing due to releasing error.

+

0.4.3

+
    +
  • In create_zarr_structure_multiplex, always use/require strings for acquisition field (#217).
    • +
+

0.4.2

+
    +
  • Bugfixes
    • +
+

0.4.1

+
    +
  • Only use strings as keys of channel_parameters (in create_zarr_structure_multiplex).
    • +
+

0.4.0

+
    +
  • (major) Rename well to image (both in metadata list and in manifest) and add an actual well field (#210).
    • +
  • Add create_ome_zarr_multiplexing, and adapt yokogawa_to_zarr (#210).
    • +
  • Relax constraint about outputs in napari_worfklows_wrapper (#209).
    • +
+

0.3.4

+
    +
  • Always log START/END times for each task (#204).
    • +
  • Add label_name argument to cellpose_segmentation (#207).
    • +
  • Add pretrained_model argument to cellpose_segmentation (#207).
    • +
+

0.3.3

+
    +
  • Added napari_worfklows_wrapper to manifest.
    • +
+

0.3.2

+
    +
  • Compute bounding boxes of labels, in cellpose_segmentation (#192).
    • +
  • Parse image filenames in a more robust way (#191).
    • +
  • Update manifest, moving parallelization_level and executor to meta attribute.
    • +
+

0.3.1

+
    +
  • Fix executable fields in manifest.
    • +
  • Remove graphviz dependency.
    • +
+

0.3.0

+
    +
  • Conform to Fractal v1, through new task manifest (#162) and standard input/output interface (#155, #157).
    • +
  • Add several type hints (#148) and validate them in the standard task interface (#175).
    • +
  • Update napari_worfklows_wrapper: pyramid level for labeling worfklows (#148), label-only inputs (#163, #171), relabeling (#167), 2D/3D handling (#166).
    • +
  • Deprecate dummy and dummy_fail tasks.
    • +
+

0.2.6

+ +

0.2.5

+
    +
  • Add napari_workflows_wrapper task (#141).
    • +
  • Add lib_upscale_array.py module (#141).
    • +
+

0.2.4

+
    +
  • Major updates to metadata_parsing.py (#136).
    • +
+ + + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/custom_task/index.html b/custom_task/index.html new file mode 100644 index 000000000..7a26afd17 --- /dev/null +++ b/custom_task/index.html @@ -0,0 +1,1491 @@ + + + + + + + + + + + + + + + + + + + + + + Write a custom task - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

How to write a Fractal-compatible custom task

+

The fractal-tasks-core repository is the reference implementation for Fractal +tasks and for Fractal task packages, but the Fractal platform can also be +used to execute custom tasks. This page lists the Fractal-compatibility +requirements, for a single custom task or for a task +package.

+
+

Note: these specifications evolve frequently, see e.g. discussion at +https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/151.

+
+

A Fractal task is mainly formed by two components:

+
    +
  1. A set of metadata, which are stored in the task table of the database of a + fractal-server instance, see Task metadata.
    2. +
  2. An executable command, which can take some specific command-line arguments + (see Command-line interface); the standard + example is a Python script.
    3. +
+

In the following we explain what are the Fractal-compatibility requirements for +a single task, and then for a task package.

+

Single custom task

+

We describe how to define the multiple aspects of a task, and provide a Full task example.

+

Task metadata

+

Each task must be associated to some metadata, so that it can be used in +Fractal. The full specification is +here, +and the required attributes are:

+
    +
  • name: the task name, e.g. "Create OME-Zarr structure";
    • +
  • command: a command that can be executed from the command line;
    • +
  • input_type: this can be any string (typical examples: "image" or "zarr"); + the special value "Any" means that Fractal won't perform any check of the + input_type when applying the task to a dataset.
    • +
  • output_type: same logic as input_type.
    • +
  • source: this is meant to be as close as possible to unique task identifier; + for custom tasks, it can be anything (e.g. "my_task"), but for task that + are collected automatically from a package (see Task package this + attribute will have a very specific form (e.g. + "pip_remote:fractal_tasks_core:0.10.0:fractal-tasks::convert_yokogawa_to_ome-zarr").
    • +
  • meta: a JSON object (similar to a Python dictionary) with some additional + information, see Task meta-parameters.
    • +
+

There are multiple ways to get the appropriate metadata into the database, +including a POST request to the fractal-server API (see Tasks section in +the fractal-server API +documentation) +or the automated addition of a whole set of tasks through specific API +endpoints (see Task package).

+

Command-line interface

+

Some examples of task commands may look like

+
    +
  • python3 /some/path/my_task.py,
    • +
  • /some/absolute/path/python3.10 /some/other/absolute/path/my_task.py,
    • +
  • /some/path/my_executable_task.py,
    • +
  • any other executable command (not necessarily based on Python).
    • +
+

Given a task command, Fractal will add two additional command-line arguments to it:

+
    +
  • -j /some/path/input-arguments.json
    • +
  • --metadata-out /some/path/output-metadata-update.json
    • +
+

Therefore the task command must accept these additional command-line arguments. +If the task is a Python script, this can be achieved easily by using the +run_fractal_task function - which is available as part of +fractal_tasks_core.tasks._utils.

+

Task meta-parameters

+

The meta attribute of tasks (see the corresponding item in Task +metadata) is where we specify some requirements on how the +task should be run. This notably includes:

+
    +
  • If the task has to be run in parallel (e.g. over multiple wells of an + OME-Zarr dataset), then meta should include a key-value pair like + {"parallelization_level": "well"}. If the parallelization_level key is + missing, the task is considered as non-parallel.
    • +
  • If Fractal is configured to run on a SLURM cluster, meta may include + additional information on the SLRUM requirements (more info on the Fractal + SLURM backend + here).
    • +
+

Task input parameters

+

When a task is run via Fractal, its input parameters (i.e. the ones in the file +specified via the -j command-line otion) will always include a set of keyword +arguments with specific names:

+
    +
  • input_paths
    • +
  • output_path
    • +
  • metadata
    • +
  • component (only for parallel tasks)
    • +
+

Task output

+

The only task output which will be visible to Fractal is what goes in the +output metadata-update file (i.e. the one specified through the +--metadata-out command-line option). Note that this only holds for +non-parallel tasks, while (for the moment) Fractal fully ignores the output of +parallel tasks.

+
+

IMPORTANT: This means that each task must always write any output to +disk, before ending.

+
+

Advanced features

+

The description of other advanced features is not yet available in this page.

+
    +
  1. Also other attributes of the Task metadata exist, and they + would be recognized by other Fractal components (e.g. fractal-server or + fractal-web). These include JSON Schemas for input parameters and additional + documentation-related attributes.
    2. +
  2. In fractal-tasks-core, we use pydantic + v1 to fully coerce and validate the input + parameters into a set of given types.
    3. +
+

Full task example

+

Here we describe a simplified example of a Fractal-compatible Python task (for +more realistic examples see the fractal-task-core tasks +folder).

+

The script /some/path/my_task.py may look like +

# Import a helper function from fractal_tasks_core
+from fractal_tasks_core.tasks._utils import run_fractal_task
+
+def my_task_function(
+    # Reserved Fractal arguments
+    input_paths,
+    output_path,
+    metadata,
+    # Task-specific arguments
+    argument_A,
+    argument_B = "default_B_value",
+):
+    # Do something, based on the task parameters
+    print("Here we go, we are in `my_task_function`")
+    with open(f"{output_path}/output.txt", "w") as f:
+        f.write(f"argument_A={argument_A}\n")
+        f.write(f"argument_B={argument_B}\n")
+    # Compile the output metadata update and return
+    output_metadata_update = {"nothing": "to add"}
+    return output_metadata_update
+
+# Thi block is executed when running the Python script directly
+if __name__ == "__main__":
+    run_fractal_task(task_function=my_task_function)
+
+where we use run_fractal_task so that we don't have to take care of the command-line arguments.

+

Some valid metadata attributes for this task would be: +

name="My Task"
+command="python3 /some/path/my_task.py"
+input_type="Any"
+output_type="Any"
+source="my_custom_task"
+meta={}
+

+
+

Note that this was an example of a non-parallel tasks; to have a parallel +one, we would also need to:

+
    +
  1. Set meta={"parallelization_level": "something"};
    2. +
  2. Include component in the input arguments of my_task_function.
    3. +
+
+

Task package

+

Given a set of Python scripts corresponding to Fractal tasks, it is useful to +combine them into a single Python package, using the standard +tools or +other options (e.g. for fractal-tasks-core we use +poetry).

+

Reasons

+

Creating a package is often a good practice, for reasons unrelated to Fractal:

+
    +
  1. It makes it simple to assign a global version to the package, and to host it + on a public index like PyPI;
    2. +
  2. It may reduce code duplication:
      +
    • The scripts may have a shared set of external dependencies, which are + defined in a single place for a package.
      • +
    • The scripts may import functions from a shared set of auxiliary Python + modules, which can be included in the package.
      • +
    +
    3. +
+

Moreover, having a single package also streamlines some Fractal-related +operations. Given the package MyTasks (available on PyPI, or locally), the +Fractal platform offers a feature that automatically:

+
    +
  1. Downloads the wheel file of package MyTasks (if it's on a public index, + rather than a local file);
    2. +
  2. Creates a Python virtual environment (venv) which is specific for a given + version of the MyTasks package, and installs the MyTasks package in that + venv;
    3. +
  3. Populates all the corresponding entries in the task database table with + the appropriate Task metadata, which are extracted from + the package manifest.
    4. +
+

This feature is currently exposed in the /api/v1/task/collect/pip/ endpoint of fractal-server (see API documentation).

+

Requirements

+

To be compatible with Fractal, a task package must satisfy some additional requirements:

+
    +
  • The package is built as a a wheel file, and can be installed via pip.
    • +
  • The __FRACTAL_MANIFEST__.json file is bundled in the package, in its root + folder. If you are using poetry, no special operation is needed. If you + are using a setup.cfg file, see + this + comment.
    • +
  • Include JSON Schemas. The tools in fractal_tasks_core.dev are used to + generate JSON Schema's for the input parameters of each task in + fractal-tasks-core. They are meant to be flexible and re-usable to perform + the same operation on an independent package, but they are not thoroughly + documented/tested for more general use; feel free to open an issue if something + is not clear.
    • +
  • Include additional task metadata like docs_info or docs_link, which will + be displayed in the Fractal web-client. Note: this feature is not yet + implemented.
    • +
+
+

The ones in the list are the main requirements; if you hit unexpected +behaviors, also have a look at +https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/151 +or open a new issue.

+
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/development/index.html b/development/index.html new file mode 100644 index 000000000..4d397bfd6 --- /dev/null +++ b/development/index.html @@ -0,0 +1,1220 @@ + + + + + + + + + + + + + + + + + + + + + + Development - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Development

+

Setting up environment

+

We use poetry v1.5 to manage the development environment and the dependencies. A simple way to install it is pipx install poetry==1.5, or you can look at the installation section here.

+

Running:

+
poetry install [--with dev] [--with docs]
+
+

will take care of installing all the dependencies in a separate environment, optionally installing also the dependencies for developement and to build the documentation.

+

Testing

+

We use pytest for unit and integration testing of Fractal. If you installed the development dependencies, you may run the test suite by invoking: +

poetry run pytest
+

+

The tests files are in the tests folder of the repository, and they are also +run through GitHub Actions; both the main fractal_tasks_core tests (in +tests/) and the fractal_tasks_core.tasks tests (in tests/tasks/) are run +with Python 3.9, 3.10 and 3.11.

+

Documentation

+

The documentations is built with mkdocs. +To build the documentation locally, setup a development python environment (e.g. with poetry install --with docs) and then run one of these commands: +

poetry run mkdocs serve --config-file mkdocs.yml  # serves the docs at http://127.0.0.1:8000
+poetry run mkdocs build --config-file mkdocs.yml  # creates a build in the `site` folder
+

+

Mypy

+

We do not enforce strict mypy compliance, but we do run it as part of a specific GitHub Action. +You can run mypy locally for instance as: +

poetry run mypy --package fractal_tasks_core --ignore-missing-imports --warn-redundant-casts --warn-unused-ignores --warn-unreachable --pretty
+

+

How to release

+

Preliminary check-list:

+
    +
  1. The main branch is checked out.
    2. +
  2. You reviewed dependencies and dev dependencies and the lock file is up to date with pyproject.toml (it is useful to have a look at the output of deptry . -v, where deptry is already installed as part of the dev dependencies - NOTE: deptry should be installed independently, e.g. via pipx install deptry).
    3. +
  3. The current HEAD of the main branch passes all the tests (note: make sure that you are using the poetry-installed local package).
    4. +
  4. Update changelog. First look at the list of commits since the last tag, via: +
    git log --pretty="[%cs] %h - %s" `git tag --sort version:refname | tail -n 1`..HEAD
+
    +then add the upcoming release to docs/source/changelog.rst with the main information about it, using standard categories like "New features", "Fixes" and "Other changes", and including PR numbers when relevant. Commit docs/source/changelog.rst and push.
    5. +
  5. If appropriate (e.g. if you added some new task arguments, or if you modified some of their descriptions), update the JSON Schemas in the manifest via: +
    poetry run python fractal_tasks_core/dev/update_manifest.py
+
    6. +
+

Actual release

+
    +
  1. Use: +
    poetry run bumpver update --[tag-num|patch|minor] --tag-commit --commit --dry
+
    +to test updating the version bump.
    2. +
  2. If the previous step looks good, use: +
    poetry run bumpver update --[tag-num|patch|minor] --tag-commit --commit
+
    +to actually bump the version. This will trigger a dedicated GitHub +action to build the new package and publish it to PyPI.
    3. +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/doc-requirements.txt b/doc-requirements.txt new file mode 100644 index 000000000..1fc3cce45 --- /dev/null +++ b/doc-requirements.txt @@ -0,0 +1,8 @@ +mkdocs==1.5.2 +mkdocs-material==9.1.21 +mkdocs-autorefs==0.5.0 +mkdocs-gen-files==0.4.0 +mkdocs-literate-nav==0.5.0 +mkdocs-section-index==0.3.5 +mkdocstrings[python]==0.22.0 +mkdocs-include-markdown-plugin==4.0.4 diff --git a/extra.css b/extra.css new file mode 100644 index 000000000..be13614b9 --- /dev/null +++ b/extra.css @@ -0,0 +1,17 @@ +/* Custom style for blockquotes */ +blockquote { + background-color: #e7e3e3d8; /* Light gray background */ + border: 1px solid #9c27b0; /* Purple border */ + padding: 10px; + margin: 20px 0; + border-radius: 4px; + font-size: 16px; + line-height: 1.6; + box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1); /* Optional: Add a subtle shadow */ +} + +/* Style the text inside blockquotes */ +blockquote p { + margin: 0; + color: #333; /* Dark text color */ +} diff --git a/gen_ref_pages.py b/gen_ref_pages.py new file mode 100644 index 000000000..a5b3a6ea6 --- /dev/null +++ b/gen_ref_pages.py @@ -0,0 +1,81 @@ +from pathlib import Path +from typing import Iterable +from typing import Mapping + +import mkdocs_gen_files +from mkdocs_gen_files import Nav + + +class CustomNav(Nav): + """ + The original Nav class is part of mkdocs_gen_files + (https://github.com/oprypin/mkdocs-gen-files) + Original Copyright 2020 Oleh Prypin + License: MIT + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @classmethod + def _items(cls, data: Mapping, level: int) -> Iterable[Nav.Item]: + """ + Custom modification: rather than looping over data.items(), we loop + over keys/values in a custom order (that is, we first include "tasks", + then "dev", then all the rest) + """ + sorted_keys = list(data.keys()) + if None in sorted_keys: + sorted_keys.remove(None) + sorted_keys = sorted(sorted_keys, key=str.casefold) + if "dev" in sorted_keys: + sorted_keys.remove("dev") + sorted_keys = ["dev"] + sorted_keys + if "tasks" in sorted_keys: + sorted_keys.remove("tasks") + sorted_keys = ["tasks"] + sorted_keys + + for key in sorted_keys: + value = data[key] + if key is not None: + yield cls.Item( + level=level, title=key, filename=value.get(None) + ) + yield from cls._items(value, level + 1) + + +nav = CustomNav() + +for path in sorted(Path("fractal_tasks_core").rglob("*.py")): + module_path = path.relative_to(".").with_suffix("") + doc_path = path.relative_to(".").with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = list(module_path.parts) + + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + + # Remove fractal_tasks_core from doc_path + doc_path = Path("/".join(doc_path.as_posix().split("/")[1:])) + + # Remove fractal_tasks_core from parts, and skip the case where + # parts=["fractal_tasks_core"] + if parts[1:]: + nav[parts[1:]] = doc_path.as_posix() + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + identifier = ".".join(parts) + fd.write(f"::: {identifier}") + + mkdocs_gen_files.set_edit_path(full_doc_path, path) + + +with mkdocs_gen_files.open( + "reference/fractal_tasks_core/SUMMARY.md", "w" +) as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/index.html b/index.html new file mode 100644 index 000000000..c04f29ae0 --- /dev/null +++ b/index.html @@ -0,0 +1,1172 @@ + + + + + + + + + + + + + + + + + + + + Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + + + +
+
+
+ + + + +
+
+
+ + + + + + + + + +
+
+ + + + + + + +

Welcome to Fractal Tasks Core's documentation!

+

Fractal is a framework to process high content imaging data at scale and prepare it for interactive visualization.

+
+

This project is under active development 🔨. If you need help or found a bug, open an issue here.

+
+

Fractal provides distributed workflows that convert TBs of image data into OME-Zar files. +The platform then processes the 3D image data by applying tasks like illumination correction, maximum intensity projection, 3D segmentation using cellpose and measurements using napari workflows. +The pyramidal OME-Zarr files enable interactive visualization in the napari viewer.

+

Fractal overview

+

The fractal-tasks-core package contains the python tasks that parse Yokogawa CV7000 images into OME-Zarr and process OME-Zarr files. Find more information about Fractal in general and the other repositories at this link. All tasks are written as Python functions and are optimized for usage in Fractal workflows, but they can also be used as standalone functions to parse data or process OME-Zarr files. We heavily use regions of interest (ROIs) in our OME-Zarr files to store the positions of field of views. ROIs are saved as AnnData tables following this spec proposal. We save wells as large Zarr arrays instead of a collection of arrays for each field of view (see details here).

+

Here is an example of the interactive visualization in napari using the newly-proposed async loading in NAP4 and the napari-ome-zarr plugin:

+

Napari plate overview

+

Available tasks

+

Currently, the following tasks are available:

+
    +
  • Create Zarr Structure: Task to generate the zarr structure based on Yokogawa metadata files
    • +
  • Yokogawa to Zarr: Parses the Yokogawa CV7000 image data and saves it to the Zarr file
    • +
  • Illumination Correction: Applies an illumination correction based on a flatfield image & subtracts a background from the image.
    • +
  • Image Labeling (& Image Labeling Whole Well): Applies a cellpose network to the image of a single ROI or the whole well. cellpose parameters can be tuned for optimal performance.
    • +
  • Maximum Intensity Projection: Creates a maximum intensity projection of the whole plate.
    • +
  • Measurement: Make some standard measurements (intensity & morphology) using napari workflows, saving results to AnnData tables.
    • +
+

Some additional tasks are currently being worked on and some older tasks are still present in the fractal_tasks_core folder. See the package page for the detailed description of all tasks.

+

Contributors

+

Fractal was conceived in the Liberali Lab at the Friedrich Miescher Institute for Biomedical Research and in the Pelkmans Lab at the University of Zurich (both in Switzerland). The project lead is with @gusqgm & @jluethi. The core development is done under contract by eXact lab S.r.l..

+

License

+

Fractal is released according to a BSD 3-Clause License, see LICENSE.

+ + + + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/install/index.html b/install/index.html new file mode 100644 index 000000000..ef58e9a76 --- /dev/null +++ b/install/index.html @@ -0,0 +1,1144 @@ + + + + + + + + + + + + + + + + + + + + + + Install - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

How to install

+

The fractal_tasks_core Python package is hosted on PyPI (https://pypi.org/project/fractal-tasks-core), and can be installed via pip. +It includes three (sub)packages:

+
    +
  1. The main fractal_tasks_core package: a set of helper functions to be used in the Fractal tasks (and possibly in other independent packages).
    2. +
  2. The fractal_tasks_core.tasks subpackage: a set of standard Fractal tasks.
    3. +
  3. The fractal_tasks_core.dev subpackage: a set of developement tools (mostly related to creation of JSON Schemas for task arguments).
    4. +
+

Minimal installation

+

The minimal installation command is +

pip install fractal-tasks-core
+
+which only installs the dependencies necessary for the main package and for the dev subpackage.

+

Full installation

+

In order to also use the tasks subpackage, the additional extra fractal-tasks must be included, as in +

pip install fractal-tasks-core[fractal-tasks]
+
+Warning: This command installs heavier dependencies (e.g. torch).

+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/manifest/index.html b/manifest/index.html new file mode 100644 index 000000000..d7fbb1ec1 --- /dev/null +++ b/manifest/index.html @@ -0,0 +1,2342 @@ + + + + + + + + + + + + + + + + + + + + + + Task manifest - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Task manifest

+

To be imported automatically within Fractal, tasks must be included in a standard manifest. Here is the one for fractal-task-core's tasks.

+
{
+  "args_schema_version": "pydantic_v1",
+  "has_args_schema": true,
+  "has_args_schemas": true,
+  "manifest_version": "1",
+  "task_list": [
+    {
+      "args_schema": {
+        "title": "CreateOmeZarr",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data from the microscope is stored (as TIF or PNG). Should point to the parent folder containing the images and the metadata files `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present). Example: `[\"/some/path/\"]`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Path were the output of this task is stored. Example: \"/some/path/\" => puts the new OME-Zarr file in the \"/some/path/\". (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "allowed_channels": {
+            "title": "Allowed Channels",
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/OmeroChannel"
+            },
+            "description": "A list of `OmeroChannel` s, where each channel must include the `wavelength_id` attribute and where the `wavelength_id` values must be unique across the list."
+          },
+          "image_glob_patterns": {
+            "title": "Image Glob Patterns",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: `image_glob_pattern=[\"*_B03_*\"]` => only process well B03 `image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"]` => only process well C09, field of view 16 and Z planes 0-59."
+          },
+          "num_levels": {
+            "title": "Num Levels",
+            "default": 5,
+            "type": "integer",
+            "description": "Number of resolution-pyramid levels. If set to `5`, there will be the full-resolution level and 4 levels of downsampled images."
+          },
+          "coarsening_xy": {
+            "title": "Coarsening Xy",
+            "default": 2,
+            "type": "integer",
+            "description": "Linear coarsening factor between subsequent levels. If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled etc."
+          },
+          "image_extension": {
+            "title": "Image Extension",
+            "default": "tif",
+            "type": "string",
+            "description": "Filename extension of images (e.g. `\"tif\"` or `\"png\"`)"
+          },
+          "metadata_table_file": {
+            "title": "Metadata Table File",
+            "type": "string",
+            "description": "If `None`, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, the full path to a csv file containing the parsed metadata table."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "metadata",
+          "allowed_channels"
+        ],
+        "additionalProperties": false,
+        "definitions": {
+          "Window": {
+            "title": "Window",
+            "description": "Custom class for Omero-channel window, based on OME-NGFF v0.4.",
+            "type": "object",
+            "properties": {
+              "min": {
+                "title": "Min",
+                "type": "integer",
+                "description": "Do not change. It will be set to `0` by default."
+              },
+              "max": {
+                "title": "Max",
+                "type": "integer",
+                "description": "Do not change. It will be set according to bit-depth of the images by default (e.g. 65535 for 16 bit images)."
+              },
+              "start": {
+                "title": "Start",
+                "type": "integer",
+                "description": "Lower-bound rescaling value for visualization."
+              },
+              "end": {
+                "title": "End",
+                "type": "integer",
+                "description": "Upper-bound rescaling value for visualization."
+              }
+            },
+            "required": [
+              "start",
+              "end"
+            ]
+          },
+          "OmeroChannel": {
+            "title": "OmeroChannel",
+            "description": "Custom class for Omero channels, based on OME-NGFF v0.4.",
+            "type": "object",
+            "properties": {
+              "wavelength_id": {
+                "title": "Wavelength Id",
+                "type": "string",
+                "description": "Unique ID for the channel wavelength, e.g. `A01_C01`."
+              },
+              "index": {
+                "title": "Index",
+                "type": "integer",
+                "description": "Do not change. For internal use only."
+              },
+              "label": {
+                "title": "Label",
+                "type": "string",
+                "description": "Name of the channel."
+              },
+              "window": {
+                "$ref": "#/definitions/Window",
+                "title": "Window",
+                "description": "Optional `Window` object to set default display settings for napari."
+              },
+              "color": {
+                "title": "Color",
+                "type": "string",
+                "description": "Optional hex colormap to display the channel in napari (it must be of length 6, e.g. `00FFFF`)."
+              },
+              "active": {
+                "title": "Active",
+                "default": true,
+                "type": "boolean",
+                "description": "Should this channel be shown in the viewer?"
+              },
+              "coefficient": {
+                "title": "Coefficient",
+                "default": 1,
+                "type": "integer",
+                "description": "Do not change. Omero-channel attribute."
+              },
+              "inverted": {
+                "title": "Inverted",
+                "default": false,
+                "type": "boolean",
+                "description": "Do not change. Omero-channel attribute."
+              }
+            },
+            "required": [
+              "wavelength_id"
+            ]
+          }
+        }
+      },
+      "executable": "tasks/create_ome_zarr.py",
+      "input_type": "image",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000
+      },
+      "name": "Create OME-Zarr structure",
+      "output_type": "zarr",
+      "docs_info": "Create a OME-NGFF zarr folder, without reading/writing image data.\n\nFind plates (for each folder in input_paths):\n\n- glob image files,\n- parse metadata from image filename to identify plates,\n- identify populated channels.\n\nCreate a zarr folder (for each plate):\n\n- parse mlf metadata,\n- identify wells and field of view (FOV),\n- create FOV ZARR,\n- verify that channels are uniform (i.e., same channels).",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/create_ome_zarr/#fractal_tasks_core.tasks.create_ome_zarr.create_ome_zarr"
+    },
+    {
+      "args_schema": {
+        "title": "YokogawaToOmeZarr",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Unclear. Should be the same as `input_path`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"` (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata. `original_paths`: list of paths that correspond to the `input_paths` of the `create_ome_zarr` task (=> where the microscopy image are stored); `num_levels (int)`: number of pyramid levels in the image (this determines how many pyramid levels are built for the segmentation); `coarsening_xy (int)`: coarsening factor in XY of the downsampling when building the pyramid; `image_extension`: filename extension of images (e.g. `\"tif\"` or `\"png\"`); `image_glob_patterns`: parameter of `create_ome_zarr` task (if specified, only parse images with filenames that match with all these patterns). (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata"
+        ],
+        "additionalProperties": false
+      },
+      "executable": "tasks/yokogawa_to_ome_zarr.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000,
+        "parallelization_level": "image"
+      },
+      "name": "Convert Yokogawa to OME-Zarr",
+      "output_type": "zarr",
+      "docs_info": "Convert Yokogawa output (png, tif) to zarr file.\n\nThis task is typically run after Create OME-Zarr or\nCreate OME-Zarr Multiplexing and populates the empty OME-Zarr files that\nwere prepared.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/#fractal_tasks_core.tasks.yokogawa_to_ome_zarr.yokogawa_to_ome_zarr"
+    },
+    {
+      "args_schema": {
+        "title": "CopyOmeZarr",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Path were the output of this task is stored. Example: `\"/some/path/\"` => puts the new OME-Zarr file in the same folder as the input OME-Zarr file `\"/some/new_path\"` => puts the new OME-Zarr file into a new folder at `/some/new_path`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata: `plate`: List of plates (e.g. `[\"MyPlate.zarr\"]`); `well`: List of wells in the OME-Zarr plate (e.g. `[\"MyPlate.zarr/B/03/MyPlate.zarr/B/05\"]`); \"image\": List of images in the OME-Zarr plate (e.g. `[\"MyPlate.zarr/B/03/0\", \"MyPlate.zarr/B/05/0\"]`). standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "project_to_2D": {
+            "title": "Project To 2D",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, apply a 3D->2D projection to the ROI tables that are copied to the new OME-Zarr."
+          },
+          "suffix": {
+            "title": "Suffix",
+            "default": "mip",
+            "type": "string",
+            "description": "The suffix that is used to transform `plate.zarr` into `plate_suffix.zarr`. Note that `None` is not currently supported."
+          },
+          "ROI_table_names": {
+            "title": "Roi Table Names",
+            "default": [
+              "FOV_ROI_table",
+              "well_ROI_table"
+            ],
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of Anndata table names to be copied. Note: copying non-ROI tables may fail if `project_to_2D=True`."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "metadata"
+        ],
+        "additionalProperties": false
+      },
+      "executable": "tasks/copy_ome_zarr.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 1000
+      },
+      "name": "Copy OME-Zarr structure",
+      "output_type": "zarr",
+      "docs_info": "Duplicate an input zarr structure to a new path.\n\nThis task copies all the structure, but none of the image data:\n\n- For each plate, create a new zarr group with the same attributes as\n   the original one.\n- For each well (in each plate), create a new zarr subgroup with the\n   same attributes as the original one.\n- For each image (in each well), create a new zarr subgroup with the\n   same attributes as the original one.\n- For each image (in each well), copy the relevant AnnData tables from\n   the original source.\n\nNote: this task makes use of methods from the `Attributes` class, see\nhttps://zarr.readthedocs.io/en/stable/api/attrs.html.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/copy_ome_zarr/#fractal_tasks_core.tasks.copy_ome_zarr.copy_ome_zarr"
+    },
+    {
+      "args_schema": {
+        "title": "MaximumIntensityProjection",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "This parameter is not used by this task. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Path were the output of this task is stored. Example: `\"/some/path/\"` => puts the new OME-Zarr file in that folder. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Component is typically changed by the `copy_ome_zarr` task before, to point to a new mip Zarr file. Example: `\"some_plate_mip.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "Dictionary containing metadata about the OME-Zarr. This task requires the key `copy_ome_zarr` to be present in the metadata (as defined in `copy_ome_zarr` task). (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata"
+        ],
+        "additionalProperties": false
+      },
+      "executable": "tasks/maximum_intensity_projection.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000,
+        "parallelization_level": "image"
+      },
+      "name": "Maximum Intensity Projection",
+      "output_type": "zarr",
+      "docs_info": "Perform maximum-intensity projection along Z axis.\n\nNote: this task stores the output in a new zarr file.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/maximum_intensity_projection/#fractal_tasks_core.tasks.maximum_intensity_projection.maximum_intensity_projection"
+    },
+    {
+      "args_schema": {
+        "title": "CellposeSegmentation",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "level": {
+            "title": "Level",
+            "type": "integer",
+            "description": "Pyramid level of the image to be segmented. Choose `0` to process at full resolution."
+          },
+          "channel": {
+            "$ref": "#/definitions/Channel",
+            "title": "Channel",
+            "description": "Primary channel for segmentation; requires either `wavelength_id` (e.g. `A01_C01`) or `label` (e.g. `DAPI`)."
+          },
+          "channel2": {
+            "$ref": "#/definitions/Channel",
+            "title": "Channel2",
+            "description": "Second channel for segmentation (in the same format as `channel`). If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker."
+          },
+          "input_ROI_table": {
+            "title": "Input Roi Table",
+            "default": "FOV_ROI_table",
+            "type": "string",
+            "description": "Name of the ROI table over which the task loops to apply Cellpose segmentation. Examples: `FOV_ROI_table` => loop over the field of views, `organoid_ROI_table` => loop over the organoid ROI table (generated by another task), `well_ROI_table` => process the whole well as one image."
+          },
+          "output_ROI_table": {
+            "title": "Output Roi Table",
+            "type": "string",
+            "description": "If provided, a ROI table with that name is created, which will contain the bounding boxes of the newly segmented labels. ROI tables should have `ROI` in their name."
+          },
+          "output_label_name": {
+            "title": "Output Label Name",
+            "type": "string",
+            "description": "Name of the output label image (e.g. `\"organoids\"`)."
+          },
+          "use_masks": {
+            "title": "Use Masks",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, try to use masked loading and fall back to `use_masks=False` if the ROI table is not suitable. Masked loading is relevant when only a subset of the bounding box should actually be processed (e.g. running within `organoid_ROI_table`)."
+          },
+          "relabeling": {
+            "title": "Relabeling",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, apply relabeling so that label values are unique for all objects in the well."
+          },
+          "diameter_level0": {
+            "title": "Diameter Level0",
+            "default": 30.0,
+            "type": "number",
+            "description": "Expected diameter of the objects that should be segmented in pixels at level 0. Initial diameter is rescaled using the `level` that was selected. The rescaled value is passed as the diameter to the `CellposeModel.eval` method."
+          },
+          "model_type": {
+            "title": "Model Type",
+            "default": "cyto2",
+            "type": "string",
+            "description": "Parameter of `CellposeModel` class. Defines which model should be used. Typical choices are `nuclei`, `cyto`, `cyto2`, etc."
+          },
+          "pretrained_model": {
+            "title": "Pretrained Model",
+            "type": "string",
+            "description": "Parameter of `CellposeModel` class (takes precedence over `model_type`). Allows you to specify the path of a custom trained cellpose model."
+          },
+          "cellprob_threshold": {
+            "title": "Cellprob Threshold",
+            "default": 0.0,
+            "type": "number",
+            "description": "Parameter of `CellposeModel.eval` method. Valid values between -6 to 6. From Cellpose documentation: \"Decrease this threshold if cellpose is not returning as many ROIs as you\u2019d expect. Similarly, increase this threshold if cellpose is returning too ROIs particularly from dim areas.\""
+          },
+          "flow_threshold": {
+            "title": "Flow Threshold",
+            "default": 0.4,
+            "type": "number",
+            "description": "Parameter of `CellposeModel.eval` method. Valid values between 0.0 and 1.0. From Cellpose documentation: \"Increase this threshold if cellpose is not returning as many ROIs as you\u2019d expect. Similarly, decrease this threshold if cellpose is returning too many ill-shaped ROIs.\""
+          },
+          "anisotropy": {
+            "title": "Anisotropy",
+            "type": "number",
+            "description": "Ratio of the pixel sizes along Z and XY axis (ignored if the image is not three-dimensional). If `None`, it is inferred from the OME-NGFF metadata."
+          },
+          "min_size": {
+            "title": "Min Size",
+            "default": 15,
+            "type": "integer",
+            "description": "Parameter of `CellposeModel` class. Minimum size of the segmented objects (in pixels). Use `-1` to turn off the size filter."
+          },
+          "augment": {
+            "title": "Augment",
+            "default": false,
+            "type": "boolean",
+            "description": "Parameter of `CellposeModel` class. Whether to use cellpose augmentation to tile images with overlap."
+          },
+          "net_avg": {
+            "title": "Net Avg",
+            "default": false,
+            "type": "boolean",
+            "description": "Parameter of `CellposeModel` class. Whether to use cellpose net averaging to run the 4 built-in networks (useful for `nuclei`, `cyto` and `cyto2`, not sure it works for the others)."
+          },
+          "use_gpu": {
+            "title": "Use Gpu",
+            "default": true,
+            "type": "boolean",
+            "description": "If `False`, always use the CPU; if `True`, use the GPU if possible (as defined in `cellpose.core.use_gpu()`) and fall-back to the CPU otherwise."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata",
+          "level",
+          "channel"
+        ],
+        "additionalProperties": false,
+        "definitions": {
+          "Channel": {
+            "title": "Channel",
+            "description": "A channel which is specified by either `wavelength_id` or `label`.",
+            "type": "object",
+            "properties": {
+              "wavelength_id": {
+                "title": "Wavelength Id",
+                "type": "string",
+                "description": "Unique ID for the channel wavelength, e.g. `A01_C01`."
+              },
+              "label": {
+                "title": "Label",
+                "type": "string",
+                "description": "Name of the channel."
+              }
+            }
+          }
+        }
+      },
+      "executable": "tasks/cellpose_segmentation.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 4,
+        "mem": 16000,
+        "needs_gpu": true,
+        "parallelization_level": "image"
+      },
+      "name": "Cellpose Segmentation",
+      "output_type": "zarr",
+      "docs_info": "Run cellpose segmentation on the ROIs of a single OME-Zarr image.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/cellpose_segmentation/#fractal_tasks_core.tasks.cellpose_segmentation.cellpose_segmentation"
+    },
+    {
+      "args_schema": {
+        "title": "IlluminationCorrection",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Path were the output of this task is stored. Examples: `\"/some/path/\"` => puts the new OME-Zarr file in the same folder as the input OME-Zarr file; `\"/some/new_path\"` => puts the new OME-Zarr file into a new folder at `/some/new_path`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "illumination_profiles_folder": {
+            "title": "Illumination Profiles Folder",
+            "type": "string",
+            "description": "Path of folder of illumination profiles."
+          },
+          "dict_corr": {
+            "title": "Dict Corr",
+            "type": "object",
+            "additionalProperties": {
+              "type": "string"
+            },
+            "description": "Dictionary where keys match the `wavelength_id` attributes of existing channels (e.g. `A01_C01` ) and values are the filenames of the corresponding illumination profiles."
+          },
+          "background": {
+            "title": "Background",
+            "default": 110,
+            "type": "integer",
+            "description": "Background value that is subtracted from the image before the illumination correction is applied. Set it to `0` if you don't want any background subtraction."
+          },
+          "overwrite_input": {
+            "title": "Overwrite Input",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, the results of this task will overwrite the input image data. In the current version, `overwrite_input=False` is not implemented."
+          },
+          "new_component": {
+            "title": "New Component",
+            "type": "string",
+            "description": "Not implemented yet. This is not implemented well in Fractal server at the moment, it's unclear how a user would specify fitting new components. If the results shall not overwrite the input data and the output path is the same as the input path, a new component needs to be provided. Example: `myplate_new_name.zarr/B/03/0/`."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata",
+          "illumination_profiles_folder",
+          "dict_corr"
+        ],
+        "additionalProperties": false
+      },
+      "executable": "tasks/illumination_correction.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000,
+        "parallelization_level": "image"
+      },
+      "name": "Illumination correction",
+      "output_type": "zarr",
+      "docs_info": "Applies illumination correction to the images in the OME-Zarr.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/illumination_correction/#fractal_tasks_core.tasks.illumination_correction.illumination_correction"
+    },
+    {
+      "args_schema": {
+        "title": "NapariWorkflowsWrapper",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. his task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "workflow_file": {
+            "title": "Workflow File",
+            "type": "string",
+            "description": "Absolute path to napari-workflows YAML file"
+          },
+          "input_specs": {
+            "title": "Input Specs",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/definitions/NapariWorkflowsInput"
+            },
+            "description": "A dictionary of `NapariWorkflowsInput` values."
+          },
+          "output_specs": {
+            "title": "Output Specs",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/definitions/NapariWorkflowsOutput"
+            },
+            "description": "A dictionary of `NapariWorkflowsOutput` values."
+          },
+          "input_ROI_table": {
+            "title": "Input Roi Table",
+            "default": "FOV_ROI_table",
+            "type": "string",
+            "description": "Name of the ROI table over which the task loops to apply napari workflows. Examples: `FOV_ROI_table` => loop over the field of views; `organoid_ROI_table` => loop over the organoid ROI table (generated by another task); `well_ROI_table` => process the whole well as one image."
+          },
+          "level": {
+            "title": "Level",
+            "default": 0,
+            "type": "integer",
+            "description": "Pyramid level of the image to be used as input for napari-workflows. Choose `0` to process at full resolution. Levels > 0 are currently only supported for workflows that only have intensity images as input and only produce a label images as output."
+          },
+          "relabeling": {
+            "title": "Relabeling",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, apply relabeling so that label values are unique across all ROIs in the well."
+          },
+          "expected_dimensions": {
+            "title": "Expected Dimensions",
+            "default": 3,
+            "type": "integer",
+            "description": "Expected dimensions (either `2` or `3`). Useful when loading 2D images that are stored in a 3D array with shape `(1, size_x, size_y)` [which is the default way Fractal stores 2D images], but you want to make sure the napari workflow gets a 2D array to process. Also useful to set to `2` when loading a 2D OME-Zarr that is saved as `(size_x, size_y)`."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": true,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata",
+          "workflow_file",
+          "input_specs",
+          "output_specs"
+        ],
+        "additionalProperties": false,
+        "definitions": {
+          "Channel": {
+            "title": "Channel",
+            "description": "A channel which is specified by either `wavelength_id` or `label`.",
+            "type": "object",
+            "properties": {
+              "wavelength_id": {
+                "title": "Wavelength Id",
+                "type": "string",
+                "description": "Unique ID for the channel wavelength, e.g. `A01_C01`."
+              },
+              "label": {
+                "title": "Label",
+                "type": "string",
+                "description": "Name of the channel."
+              }
+            }
+          },
+          "NapariWorkflowsInput": {
+            "title": "NapariWorkflowsInput",
+            "description": "A value of the `input_specs` argument in `napari_workflows_wrapper`.",
+            "type": "object",
+            "properties": {
+              "type": {
+                "title": "Type",
+                "enum": [
+                  "image",
+                  "label"
+                ],
+                "type": "string",
+                "description": "Input type (either `image` or `label`)."
+              },
+              "label_name": {
+                "title": "Label Name",
+                "type": "string",
+                "description": "Label name (for label inputs only)."
+              },
+              "channel": {
+                "$ref": "#/definitions/Channel",
+                "title": "Channel",
+                "description": "Channel object (for image inputs only)."
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
+          "NapariWorkflowsOutput": {
+            "title": "NapariWorkflowsOutput",
+            "description": "A value of the `output_specs` argument in `napari_workflows_wrapper`.",
+            "type": "object",
+            "properties": {
+              "type": {
+                "title": "Type",
+                "enum": [
+                  "label",
+                  "dataframe"
+                ],
+                "type": "string",
+                "description": "Output type (either `label` or `dataframe`)."
+              },
+              "label_name": {
+                "title": "Label Name",
+                "type": "string",
+                "description": "Label name (for label outputs only)."
+              },
+              "table_name": {
+                "title": "Table Name",
+                "type": "string",
+                "description": "Table name (for dataframe outputs only)."
+              }
+            },
+            "required": [
+              "type"
+            ]
+          }
+        }
+      },
+      "executable": "tasks/napari_workflows_wrapper.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 8,
+        "mem": 32000,
+        "parallelization_level": "image"
+      },
+      "name": "Napari workflows wrapper",
+      "output_type": "zarr",
+      "docs_info": "Run a napari-workflow on the ROIs of a single OME-NGFF image.\n\nThis task takes images and labels and runs a napari-workflow on them that\ncan produce a label and tables as output.\n\nExamples of allowed entries for `input_specs` and `output_specs`:\n\n```\ninput_specs = {\n    \"in_1\": {\"type\": \"image\", \"channel\": {\"wavelength_id\": \"A01_C02\"}},\n    \"in_2\": {\"type\": \"image\", \"channel\": {\"label\": \"DAPI\"}},\n    \"in_3\": {\"type\": \"label\", \"label_name\": \"label_DAPI\"},\n}\n\noutput_specs = {\n    \"out_1\": {\"type\": \"label\", \"label_name\": \"label_DAPI_new\"},\n    \"out_2\": {\"type\": \"dataframe\", \"table_name\": \"measurements\"},\n}\n```",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/napari_workflows_wrapper/#fractal_tasks_core.tasks.napari_workflows_wrapper.napari_workflows_wrapper"
+    },
+    {
+      "args_schema": {
+        "title": "CreateOmeZarrMultiplex",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data from the microscope is stored (as TIF or PNG). Each element of the list is treated as another cycle of the multiplexing data, the cycles are ordered by their order in this list. Should point to the parent folder containing the images and the metadata files `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present). Example: `[\"/path/cycle1/\", \"/path/cycle2/\"]`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Path were the output of this task is stored. Example: `\"/some/path/\"` => puts the new OME-Zarr file in the `/some/path/`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "allowed_channels": {
+            "title": "Allowed Channels",
+            "type": "object",
+            "additionalProperties": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/OmeroChannel"
+              }
+            },
+            "description": "A dictionary of lists of `OmeroChannel`s, where each channel must include the `wavelength_id` attribute and where the `wavelength_id` values must be unique across each list. Dictionary keys represent channel indices (`\"0\",\"1\",..`)."
+          },
+          "image_glob_patterns": {
+            "title": "Image Glob Patterns",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: `image_glob_pattern=[\"*_B03_*\"]` => only process well B03 `image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"]` => only process well C09, field of view 16 and Z planes 0-59."
+          },
+          "num_levels": {
+            "title": "Num Levels",
+            "default": 5,
+            "type": "integer",
+            "description": "Number of resolution-pyramid levels. If set to `5`, there will be the full-resolution level and 4 levels of downsampled images."
+          },
+          "coarsening_xy": {
+            "title": "Coarsening Xy",
+            "default": 2,
+            "type": "integer",
+            "description": "Linear coarsening factor between subsequent levels. If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled etc."
+          },
+          "image_extension": {
+            "title": "Image Extension",
+            "default": "tif",
+            "type": "string",
+            "description": "Filename extension of images (e.g. `\"tif\"` or `\"png\"`)."
+          },
+          "metadata_table_files": {
+            "title": "Metadata Table Files",
+            "type": "object",
+            "additionalProperties": {
+              "type": "string"
+            },
+            "description": "If `None`, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, a dictionary of key-value pairs like `(acquisition, path)` with `acquisition` a string and `path` pointing to a csv file containing the parsed metadata table."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "If `True`, overwrite the task output."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "metadata",
+          "allowed_channels"
+        ],
+        "additionalProperties": false,
+        "definitions": {
+          "Window": {
+            "title": "Window",
+            "description": "Custom class for Omero-channel window, based on OME-NGFF v0.4.",
+            "type": "object",
+            "properties": {
+              "min": {
+                "title": "Min",
+                "type": "integer",
+                "description": "Do not change. It will be set to `0` by default."
+              },
+              "max": {
+                "title": "Max",
+                "type": "integer",
+                "description": "Do not change. It will be set according to bit-depth of the images by default (e.g. 65535 for 16 bit images)."
+              },
+              "start": {
+                "title": "Start",
+                "type": "integer",
+                "description": "Lower-bound rescaling value for visualization."
+              },
+              "end": {
+                "title": "End",
+                "type": "integer",
+                "description": "Upper-bound rescaling value for visualization."
+              }
+            },
+            "required": [
+              "start",
+              "end"
+            ]
+          },
+          "OmeroChannel": {
+            "title": "OmeroChannel",
+            "description": "Custom class for Omero channels, based on OME-NGFF v0.4.",
+            "type": "object",
+            "properties": {
+              "wavelength_id": {
+                "title": "Wavelength Id",
+                "type": "string",
+                "description": "Unique ID for the channel wavelength, e.g. `A01_C01`."
+              },
+              "index": {
+                "title": "Index",
+                "type": "integer",
+                "description": "Do not change. For internal use only."
+              },
+              "label": {
+                "title": "Label",
+                "type": "string",
+                "description": "Name of the channel."
+              },
+              "window": {
+                "$ref": "#/definitions/Window",
+                "title": "Window",
+                "description": "Optional `Window` object to set default display settings for napari."
+              },
+              "color": {
+                "title": "Color",
+                "type": "string",
+                "description": "Optional hex colormap to display the channel in napari (it must be of length 6, e.g. `00FFFF`)."
+              },
+              "active": {
+                "title": "Active",
+                "default": true,
+                "type": "boolean",
+                "description": "Should this channel be shown in the viewer?"
+              },
+              "coefficient": {
+                "title": "Coefficient",
+                "default": 1,
+                "type": "integer",
+                "description": "Do not change. Omero-channel attribute."
+              },
+              "inverted": {
+                "title": "Inverted",
+                "default": false,
+                "type": "boolean",
+                "description": "Do not change. Omero-channel attribute."
+              }
+            },
+            "required": [
+              "wavelength_id"
+            ]
+          }
+        }
+      },
+      "executable": "tasks/create_ome_zarr_multiplex.py",
+      "input_type": "image",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000
+      },
+      "name": "Create OME-ZARR structure (multiplexing)",
+      "output_type": "zarr",
+      "docs_info": "Create OME-NGFF structure and metadata to host a multiplexing dataset.\n\nThis task takes a set of image folders (i.e. different acquisition cycles)\nand build the internal structure and metadata of a OME-NGFF zarr group,\nwithout actually loading/writing the image data.\n\nEach element in input_paths should be treated as a different acquisition.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/#fractal_tasks_core.tasks.create_ome_zarr_multiplex.create_ome_zarr_multiplex"
+    },
+    {
+      "executable": "tasks/calculate_registration_image_based.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 8000,
+        "parallelization_level": "image"
+      },
+      "name": "Calculate registration (image-based)",
+      "output_type": "zarr",
+      "args_schema": {
+        "title": "CalculateRegistrationImageBased",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "wavelength_id": {
+            "title": "Wavelength Id",
+            "type": "string",
+            "description": "Wavelength that will be used for image-based registration; e.g. `A01_C01` for Yokogawa, `C01` for MD."
+          },
+          "roi_table": {
+            "title": "Roi Table",
+            "default": "FOV_ROI_table",
+            "type": "string",
+            "description": "Name of the ROI table over which the task loops to calculate the registration. Examples: `FOV_ROI_table` => loop over the field of views, `well_ROI_table` => process the whole well as one image."
+          },
+          "reference_cycle": {
+            "title": "Reference Cycle",
+            "default": 0,
+            "type": "integer",
+            "description": "Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well (usually the first cycle that was provided)."
+          },
+          "level": {
+            "title": "Level",
+            "default": 2,
+            "type": "integer",
+            "description": "Pyramid level of the image to be segmented. Choose `0` to process at full resolution."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata",
+          "wavelength_id"
+        ],
+        "additionalProperties": false
+      },
+      "docs_info": "Calculate registration based on images\n\nThis task consists of 3 parts:\n\n1. Loading the images of a given ROI (=> loop over ROIs)\n2. Calculating the transformation for that ROI\n3. Storing the calculated transformation in the ROI table\n\nParallelization level: image",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/calculate_registration_image_based/#fractal_tasks_core.tasks.calculate_registration_image_based.calculate_registration_image_based"
+    },
+    {
+      "executable": "tasks/apply_registration_to_ROI_tables.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 1000,
+        "parallelization_level": "well"
+      },
+      "name": "Apply Registration to ROI Tables",
+      "output_type": "zarr",
+      "args_schema": {
+        "title": "ApplyRegistrationToRoiTables",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "roi_table": {
+            "title": "Roi Table",
+            "default": "FOV_ROI_table",
+            "type": "string",
+            "description": "Name of the ROI table over which the task loops to calculate the registration. Examples: `FOV_ROI_table` => loop over the field of views, `well_ROI_table` => process the whole well as one image."
+          },
+          "reference_cycle": {
+            "title": "Reference Cycle",
+            "default": 0,
+            "type": "integer",
+            "description": "Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided"
+          },
+          "new_roi_table": {
+            "title": "New Roi Table",
+            "type": "string",
+            "description": "Optional name for the new, registered ROI table. If no name is given, it will default to \"registered_\" + `roi_table`"
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata"
+        ],
+        "additionalProperties": false
+      },
+      "docs_info": "Applies pre-calculated registration to ROI tables.\n\nApply pre-calculated registration such that resulting ROIs contain\nthe consensus align region between all cycles.\n\nParallelization level: well",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/#fractal_tasks_core.tasks.apply_registration_to_ROI_tables.apply_registration_to_ROI_tables"
+    },
+    {
+      "executable": "tasks/apply_registration_to_image.py",
+      "input_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000,
+        "parallelization_level": "image"
+      },
+      "name": "Apply Registration to Image",
+      "output_type": "zarr",
+      "args_schema": {
+        "title": "ApplyRegistrationToImage",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\"/some/path/\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "component": {
+            "title": "Component",
+            "type": "string",
+            "description": "Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\"some_plate.zarr/B/03/0\"`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "registered_roi_table": {
+            "title": "Registered Roi Table",
+            "type": "string",
+            "description": "Name of the ROI table which has been registered and will be applied to mask and shift the images. Examples: `registered_FOV_ROI_table` => loop over the field of views, `registered_well_ROI_table` => process the whole well as one image."
+          },
+          "reference_cycle": {
+            "title": "Reference Cycle",
+            "default": "0",
+            "type": "string",
+            "description": "Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided"
+          },
+          "overwrite_input": {
+            "title": "Overwrite Input",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether the old image data should be replaced with the newly registered image data. Currently only implemented for `overwrite_input=True`."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "component",
+          "metadata",
+          "registered_roi_table"
+        ],
+        "additionalProperties": false
+      },
+      "docs_info": "Apply registration to images by using a registered ROI table\n\nThis task consists of 4 parts:\n\n1. Mask all regions in images that are not available in the\nregistered ROI table and store each cycle aligned to the\nreference_cycle (by looping over ROIs).\n2. Do the same for all label images.\n3. Copy all tables from the non-aligned image to the aligned image\n(currently only works well if the only tables are well & FOV ROI tables\n(registered and original). Not implemented for measurement tables and\nother ROI tables).\n4. Clean up: Delete the old, non-aligned image and rename the new,\naligned image to take over its place.\n\nParallelization level: image",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/apply_registration_to_image/#fractal_tasks_core.tasks.apply_registration_to_image.apply_registration_to_image"
+    },
+    {
+      "name": "Import OME-Zarr",
+      "executable": "tasks/import_ome_zarr.py",
+      "input_type": "zarr",
+      "output_type": "zarr",
+      "meta": {
+        "cpus_per_task": 1,
+        "mem": 4000
+      },
+      "args_schema": {
+        "title": "ImportOmeZarr",
+        "type": "object",
+        "properties": {
+          "input_paths": {
+            "title": "Input Paths",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "A length-one list with the parent folder of the OME-Zarr to be imported; e.g. `input_paths=[\"/somewhere\"]`, if the OME-Zarr path is `/somewhere/array.zarr`. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "output_path": {
+            "title": "Output Path",
+            "type": "string",
+            "description": "Not used in this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "metadata": {
+            "title": "Metadata",
+            "type": "object",
+            "description": "Not used in this task. (standard argument for Fractal tasks, managed by Fractal server)."
+          },
+          "zarr_name": {
+            "title": "Zarr Name",
+            "type": "string",
+            "description": "The OME-Zarr name, without its parent folder; e.g. `zarr_name=\"array.zarr\"`, if the OME-Zarr path is `/somewhere/array.zarr`."
+          },
+          "add_image_ROI_table": {
+            "title": "Add Image Roi Table",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether to add a `image_ROI_table` table to each image, with a single ROI covering the whole image."
+          },
+          "add_grid_ROI_table": {
+            "title": "Add Grid Roi Table",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether to add a `grid_ROI_table` table to each image, with the image split into a rectangular grid of ROIs."
+          },
+          "grid_y_shape": {
+            "title": "Grid Y Shape",
+            "default": 2,
+            "type": "integer",
+            "description": "Y shape of the ROI grid in `grid_ROI_table`."
+          },
+          "grid_x_shape": {
+            "title": "Grid X Shape",
+            "default": 2,
+            "type": "integer",
+            "description": "X shape of the ROI grid in `grid_ROI_table`."
+          },
+          "update_omero_metadata": {
+            "title": "Update Omero Metadata",
+            "default": true,
+            "type": "boolean",
+            "description": "Whether to update Omero-channels metadata, to make them Fractal-compatible."
+          },
+          "overwrite": {
+            "title": "Overwrite",
+            "default": false,
+            "type": "boolean",
+            "description": "Whether new ROI tables (added when `add_image_ROI_table` and/or `add_grid_ROI_table` are `True`) can overwite existing ones."
+          }
+        },
+        "required": [
+          "input_paths",
+          "output_path",
+          "metadata",
+          "zarr_name"
+        ],
+        "additionalProperties": false
+      },
+      "docs_info": "Import an OME-Zarr into Fractal.\n\nThe current version of this task:\n\n1. Creates the appropriate components-related metadata, needed for\n   processing an existing OME-Zarr through Fractal.\n2. Optionally adds new ROI tables to the existing OME-Zarr.",
+      "docs_link": "https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/import_ome_zarr/#fractal_tasks_core.tasks.import_ome_zarr.import_ome_zarr"
+    }
+  ]
+}
+
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/objects.inv b/objects.inv new file mode 100644 index 0000000000000000000000000000000000000000..a8bdc3c88a3ecb2724c875fc4cf745a099725ce7 GIT binary patch literal 2200 zcmV;J2xs>rAX9K?X>NERX>N99Zgg*Qc_4OWa&u{KZXhxWBOp+6Z)#;@bUGkLa$#e1 zVQe5&VRLJ9AVY6*WeOu8R%LQ?X>V>iATTa4E-(rsAXI2&AaZ4GVQFq;WpW^IW*~HE zX>%ZEX>4U6X>%ZBZ*6dLWpi_7WFU2OX>MmAdTeQ8E(&1|pX-Y7k%m(31S?y8uX;k{{xd1nFLE@qzumT>y(+JXji@J1&{y z_SUjo8+cP6{-|`(NO<%1je!yjs2qMPAHDe7&%eC+^{2=8GbuLg@9Qh3@4!f2+u;(= zOAdy}n6TYNE!391{lJ8B9=U8(?u1s$VZU1t8zJR^*&9(h1Z7$QD};fFx(I{(2J&0B z11VXl4f}NcPxovY{8Fgmc;#*2pW;1{gI$Kf7Tihw*IbzOkS(^qWzZjWxm93)Uk7$6 zBO9pjcfVUL)T&LRx(gQicrw`hhARb28qtz#~)v``=lFHd^ARfKAKBz?fh`p z6&T%OE(DX^aVZLR1ABZ?YO)8SA%8~?t9J=iW8SwZ*s(^59}O_nu7R{V#gVMV+wnV) zptkOYiNY=c9EQ;L9+Q02_K0+_1>o2TO#4Q3(h1W+`bzCoERC+1sQ4DxJvU}yU@5Fi z4|!tS1u&bs;BJvB-AcWoIc#6zqR7nuVLO4{u4<$cwFHc6oCBlCdRzn)s%{)Z4akE} zzvN(uY|MZ#2`Q@@*Kmm$0_=0ySb>p5*BH^LMr;~KkCMkx`oe2&#CL6OOR0DE%0FAc z5>LTqoeeOis|{G7XKOQ(39g(j}2&mC%5(IT1S8}vh` z%+S$tl!&*Iva9eC3MMr#FzujusnfCqn2gohy4q;Wp)u__HVcY|Q@{;$*g`D|eR|1u z2D3ddDe=K^-;!{UGnQV#XQfF{S}=@c)4w^sV0V9X~!_mV^ObJlrjX1iJpAX8ZBv(LVQB1wg3OzDuQ;hVYE1_w+WkiMOmj_eJFk@$jtf(<`I^msEXc_KgK zd$1vbwDY%wQ>0vbr>=bbvOlB-D`JyRu2gl>98)6IVviYz757+Q_lJENINxa#HoH!I z2rPL1qY)U>VCV5uDL09_Q<#fRpwCxd>F=L$4GHi9fDFrpxIFeINvu}8i-Ax^(<&I%KQZ6D}`nS_3@Jv=CTM#vc8xW-b44A4KIDm*8Z)BB4s zIT&7vg5?;3cb%G04QHiEP}!ytvgmG;m8C$3{Gxj{uEjkmf)3hl)Tki%i{79_)qgHN zq49G+gXh55xGFR@0UpC?>qA?nOTSqNrN^ZqO0_%^Kn2XWfv`*;=QndLn+o3u3(0f~ z%t6QS19O^f{2ILvU^jXn$*n;rPEo~W^$DAbW}%|h9T?ZEEVR<=;M4Y;cON_@`evLt z*|Y}nMB3NZU;|a{4zHvMdmk5<*taIOV!IX%KS71H$T_MFd!l?pN|8^ zUL1Nye@1wA)B^5?8xdcANl+W#S6qpblzn-*r)3E+89t~&%l1`6vJw&3828SPE>gM%3gLrui7vhk`37@;F{A zE*ItzW{{x044azTAl(;CPUYPGw6@tVO@fNEo?$wiMVB+RAxxyw!A0x67BYp-E%ukV zBWXbrLRlXnzLNTB*}R%6vgr7oxQG7j`60KiJxTClOjaneS)1y!&?TeAjVy zkzALTAe=Y+h&D$P7CG>95c;u66* zj5U>FoIs4vkU>E{&pMUd!%2>l(s8YA!z~oly1XoLGNs-t_07Qwyo?_*8PR!{rTdbR zUmtHDq(7{BhS~d`i(i8?NLgzO%)+(;<%rv_S{6?U83X)09imxgBh+3{7s)5k(3L-S zYB7K=$@S1HOMrQVNz{^pS|7qb)y^r?wCfe706m7KDo6m1+308;Pm9yQo&qOoItBjt zk%0X~tsex9CGaw}Nj4cwPDl*b{f6}zyPMvJz$mtPM$9#FLy7F@|c+MJgvH1_nd1V6aF)cCx literal 0 HcmV?d00001 diff --git a/overrides/404.html b/overrides/404.html new file mode 100644 index 000000000..03caa9b52 --- /dev/null +++ b/overrides/404.html @@ -0,0 +1,8 @@ + +{% block content %} +

404 - Not found

+{% endblock %} diff --git a/reference/fractal_tasks_core/SUMMARY/index.html b/reference/fractal_tasks_core/SUMMARY/index.html new file mode 100644 index 000000000..457b21533 --- /dev/null +++ b/reference/fractal_tasks_core/SUMMARY/index.html @@ -0,0 +1,1087 @@ + + + + + + + + + + + + + + + + + + SUMMARY - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

SUMMARY

+ + + + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/check_manifest/index.html b/reference/fractal_tasks_core/dev/check_manifest/index.html new file mode 100644 index 000000000..997efca4d --- /dev/null +++ b/reference/fractal_tasks_core/dev/check_manifest/index.html @@ -0,0 +1,1327 @@ + + + + + + + + + + + + + + + + + + + + + + check_manifest - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

check_manifest

+ +
+ + + + +
+ +

Script to check that JSON schemas for task arguments (as reported in the +package manfest) are up-to-date.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _compare_dicts(old, new, path=[]) + +

+ + +
+ +

Provide more informative comparison of two (possibly nested) dictionaries.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
old +
+

TBD

+
+

+ + TYPE: + dict[str, Any] + +

+
new +
+

TBD

+
+

+ + TYPE: + dict[str, Any] + +

+
path +
+

TBD

+
+

+ + TYPE: + list[str] + + + DEFAULT: + [] + +

+
+ +
+ Source code in fractal_tasks_core/dev/check_manifest.py + 
31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
def _compare_dicts(
+    old: dict[str, Any], new: dict[str, Any], path: list[str] = []
+):
+    """
+    Provide more informative comparison of two (possibly nested) dictionaries.
+
+    Args:
+        old: TBD
+        new: TBD
+        path: TBD
+    """
+    path_str = "/".join(path)
+    keys_old = set(old.keys())
+    keys_new = set(new.keys())
+    if not keys_old == keys_new:
+        msg = (
+            "\n\n"
+            f"Dictionaries at path {path_str} have different keys:\n\n"
+            f"OLD KEYS:\n{keys_old}\n\n"
+            f"NEW KEYS:\n{keys_new}\n\n"
+        )
+        raise ValueError(msg)
+    for key, value_old in old.items():
+        value_new = new[key]
+        if type(value_old) != type(value_new):
+            msg = (
+                "\n\n"
+                f"Values at path {path_str}/{key} "
+                "have different types:\n\n"
+                f"OLD TYPE:\n{type(value_old)}\n\n"
+                f"NEW TYPE:\n{type(value_new)}\n\n"
+            )
+            raise ValueError(msg)
+        if isinstance(value_old, dict):
+            _compare_dicts(value_old, value_new, path=path + [key])
+        else:
+            if value_old != value_new:
+                msg = (
+                    "\n\n"
+                    f"Values at path {path_str}/{key} "
+                    "are different:\n\n"
+                    f"OLD VALUE:\n{value_old}\n\n"
+                    f"NEW VALUE:\n{value_new}\n\n"
+                )
+                raise ValueError(msg)
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/index.html b/reference/fractal_tasks_core/dev/index.html new file mode 100644 index 000000000..85c0f7830 --- /dev/null +++ b/reference/fractal_tasks_core/dev/index.html @@ -0,0 +1,1104 @@ + + + + + + + + + + + + + + + + + + + + + + dev - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

dev

+ +
+ + + + +
+ +

Development-tools subpackage (e.g. for the creation of JSON Schemas for task +parameters).

+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/lib_args_schemas/index.html b/reference/fractal_tasks_core/dev/lib_args_schemas/index.html new file mode 100644 index 000000000..e89ca107f --- /dev/null +++ b/reference/fractal_tasks_core/dev/lib_args_schemas/index.html @@ -0,0 +1,1697 @@ + + + + + + + + + + + + + + + + + + + + + + lib_args_schemas - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_args_schemas

+ +
+ + + + +
+ +

Helper functions to handle JSON schemas for task arguments.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _remove_args_kwargs_properties(old_schema) + +

+ + +
+ +

Remove args and kwargs schema properties.

+

Pydantic v1 automatically includes args and kwargs properties in +JSON Schemas generated via ValidatedFunction(task_function, +config=None).model.schema(), with some default (empty) values -- see see +https://github.com/pydantic/pydantic/blob/1.10.X-fixes/pydantic/decorator.py.

+

Verify that these properties match with their expected default values, and +then remove them from the schema.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
old_schema +
+

TBD

+
+

+ + TYPE: + _Schema + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_args_schemas.py + 
58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
def _remove_args_kwargs_properties(old_schema: _Schema) -> _Schema:
+    """
+    Remove `args` and `kwargs` schema properties.
+
+    Pydantic v1 automatically includes `args` and `kwargs` properties in
+    JSON Schemas generated via `ValidatedFunction(task_function,
+    config=None).model.schema()`, with some default (empty) values -- see see
+    https://github.com/pydantic/pydantic/blob/1.10.X-fixes/pydantic/decorator.py.
+
+    Verify that these properties match with their expected default values, and
+    then remove them from the schema.
+
+    Args:
+        old_schema: TBD
+    """
+    new_schema = old_schema.copy()
+    args_property = new_schema["properties"].pop("args")
+    kwargs_property = new_schema["properties"].pop("kwargs")
+    expected_args_property = {"title": "Args", "type": "array", "items": {}}
+    expected_kwargs_property = {"title": "Kwargs", "type": "object"}
+    if args_property != expected_args_property:
+        raise ValueError(
+            f"{args_property=}\ndiffers from\n{expected_args_property=}"
+        )
+    if kwargs_property != expected_kwargs_property:
+        raise ValueError(
+            f"{kwargs_property=}\ndiffers from\n"
+            f"{expected_kwargs_property=}"
+        )
+    logging.info("[_remove_args_kwargs_properties] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ _remove_attributes_from_descriptions(old_schema) + +

+ + +
+ +

Keeps only the description part of the docstrings: e.g from +

'Custom class for Omero-channel window, based on OME-NGFF v0.4.\n'
+'\n'
+'Attributes:\n'
+'min: Do not change. It will be set to `0` by default.\n'
+'max: Do not change. It will be set according to bitdepth of the images\n'
+'    by default (e.g. 65535 for 16 bit images).\n'
+'start: Lower-bound rescaling value for visualization.\n'
+'end: Upper-bound rescaling value for visualization.'
+
+to 'Custom class for Omero-channel window, based on OME-NGFF v0.4.\n'.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
old_schema +
+

TBD

+
+

+ + TYPE: + _Schema + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_args_schemas.py + 
110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
def _remove_attributes_from_descriptions(old_schema: _Schema) -> _Schema:
+    """
+    Keeps only the description part of the docstrings: e.g from
+    ```
+    'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'
+    '\\n'
+    'Attributes:\\n'
+    'min: Do not change. It will be set to `0` by default.\\n'
+    'max: Do not change. It will be set according to bitdepth of the images\\n'
+    '    by default (e.g. 65535 for 16 bit images).\\n'
+    'start: Lower-bound rescaling value for visualization.\\n'
+    'end: Upper-bound rescaling value for visualization.'
+    ```
+    to `'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'`.
+
+    Args:
+        old_schema: TBD
+    """
+    new_schema = old_schema.copy()
+    if "definitions" in new_schema:
+        for name, definition in new_schema["definitions"].items():
+            parsed_docstring = docparse(definition["description"])
+            new_schema["definitions"][name][
+                "description"
+            ] = parsed_docstring.short_description
+    logging.info("[_remove_attributes_from_descriptions] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ _remove_pydantic_internals(old_schema) + +

+ + +
+ +

Remove schema properties that are only used internally by Pydantic V1.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
old_schema +
+

TBD

+
+

+ + TYPE: + _Schema + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_args_schemas.py + 
 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
def _remove_pydantic_internals(old_schema: _Schema) -> _Schema:
+    """
+    Remove schema properties that are only used internally by Pydantic V1.
+
+    Args:
+        old_schema: TBD
+    """
+    new_schema = old_schema.copy()
+    for key in (
+        V_POSITIONAL_ONLY_NAME,
+        V_DUPLICATE_KWARGS,
+        ALT_V_ARGS,
+        ALT_V_KWARGS,
+    ):
+        new_schema["properties"].pop(key, None)
+    logging.info("[_remove_pydantic_internals] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ create_schema_for_single_task(executable, package='fractal_tasks_core', custom_pydantic_models=None) + +

+ + +
+ +

Main function to create a JSON Schema of task arguments

+ +
+ Source code in fractal_tasks_core/dev/lib_args_schemas.py + 
139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
def create_schema_for_single_task(
+    executable: str,
+    package: str = "fractal_tasks_core",
+    custom_pydantic_models: Optional[list[tuple[str, str, str]]] = None,
+) -> _Schema:
+    """
+    Main function to create a JSON Schema of task arguments
+    """
+
+    logging.info("[create_schema_for_single_task] START")
+
+    # Extract the function name. Note: this could be made more general, but for
+    # the moment we assume the function has the same name as the module)
+    function_name = Path(executable).with_suffix("").name
+    logging.info(f"[create_schema_for_single_task] {function_name=}")
+
+    # Extract function from module
+    task_function = _extract_function(
+        package_name=package,
+        module_relative_path=executable,
+        function_name=function_name,
+    )
+
+    logging.info(f"[create_schema_for_single_task] {task_function=}")
+
+    # Validate function signature against some custom constraints
+    _validate_function_signature(task_function)
+
+    # Create and clean up schema
+    vf = ValidatedFunction(task_function, config=None)
+    schema = vf.model.schema()
+    schema = _remove_args_kwargs_properties(schema)
+    schema = _remove_pydantic_internals(schema)
+    schema = _remove_attributes_from_descriptions(schema)
+
+    # Include titles for custom-model-typed arguments
+    schema = _include_titles(schema)
+
+    # Include descriptions of function arguments
+    function_args_descriptions = _get_function_args_descriptions(
+        package_name=package,
+        module_relative_path=executable,
+        function_name=function_name,
+    )
+    schema = _insert_function_args_descriptions(
+        schema=schema, descriptions=function_args_descriptions
+    )
+
+    # Merge lists of fractal-tasks-core and user-provided Pydantic models
+    user_provided_models = custom_pydantic_models or []
+    pydantic_models = FRACTAL_TASKS_CORE_PYDANTIC_MODELS + user_provided_models
+
+    # Check that model names are unique
+    pydantic_models_names = [item[2] for item in pydantic_models]
+    duplicate_class_names = [
+        name
+        for name, count in Counter(pydantic_models_names).items()
+        if count > 1
+    ]
+    if duplicate_class_names:
+        pydantic_models_str = "  " + "\n  ".join(map(str, pydantic_models))
+        raise ValueError(
+            "Cannot parse docstrings for models with non-unique names "
+            f"{duplicate_class_names}, in\n{pydantic_models_str}"
+        )
+
+    # Extract model-attribute descriptions and insert them into schema
+    for package_name, module_relative_path, class_name in pydantic_models:
+        attrs_descriptions = _get_class_attrs_descriptions(
+            package_name=package_name,
+            module_relative_path=module_relative_path,
+            class_name=class_name,
+        )
+        schema = _insert_class_attrs_descriptions(
+            schema=schema,
+            class_name=class_name,
+            descriptions=attrs_descriptions,
+        )
+
+    logging.info("[create_schema_for_single_task] END")
+    return schema
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/lib_descriptions/index.html b/reference/fractal_tasks_core/dev/lib_descriptions/index.html new file mode 100644 index 000000000..c522c053c --- /dev/null +++ b/reference/fractal_tasks_core/dev/lib_descriptions/index.html @@ -0,0 +1,1984 @@ + + + + + + + + + + + + + + + + + + + + + + lib_descriptions - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_descriptions

+ +
+ + + + +
+ + + +
+ + + + + + + + + + +
+ + + + +

+ _get_class_attrs_descriptions(package_name, module_relative_path, class_name) + +

+ + +
+ +

Extract attribute descriptions from a class.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
package_name +
+

Example fractal_tasks_core.

+
+

+ + TYPE: + str + +

+
module_relative_path +
+

Example lib_channels.py.

+
+

+ + TYPE: + str + +

+
class_name +
+

Example OmeroChannel.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
def _get_class_attrs_descriptions(
+    package_name: str, module_relative_path: str, class_name: str
+) -> dict[str, str]:
+    """
+    Extract attribute descriptions from a class.
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `lib_channels.py`.
+        class_name: Example `OmeroChannel`.
+    """
+
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"Module {module_relative_path} must end with '.py'")
+
+    # Get the class ast.ClassDef object
+    package_path = Path(import_module(package_name).__file__).parent
+    module_path = package_path / module_relative_path
+    tree = ast.parse(module_path.read_text())
+    try:
+        _class = next(
+            c
+            for c in ast.walk(tree)
+            if (isinstance(c, ast.ClassDef) and c.name == class_name)
+        )
+    except StopIteration:
+        raise RuntimeError(
+            f"Cannot find {class_name=} for {package_name=} "
+            f"and {module_relative_path=}"
+        )
+    docstring = ast.get_docstring(_class)
+    parsed_docstring = docparse(docstring)
+    descriptions = {
+        x.arg_name: _sanitize_description(x.description)
+        if x.description
+        else "Missing description"
+        for x in parsed_docstring.params
+    }
+    logging.info(f"[_get_class_attrs_descriptions] END ({class_name=})")
+    return descriptions
+
+
+
+ +
+ + +
+ + + + +

+ _get_function_args_descriptions(package_name, module_relative_path, function_name) + +

+ + +
+ +

Extract argument descriptions from a function.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
package_name +
+

Example fractal_tasks_core.

+
+

+ + TYPE: + str + +

+
module_relative_path +
+

Example tasks/create_ome_zarr.py.

+
+

+ + TYPE: + str + +

+
function_name +
+

Example create_ome_zarr.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
def _get_function_args_descriptions(
+    package_name: str, module_relative_path: str, function_name: str
+) -> dict[str, str]:
+    """
+    Extract argument descriptions from a function.
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `tasks/create_ome_zarr.py`.
+        function_name: Example `create_ome_zarr`.
+    """
+
+    # Extract docstring from ast.FunctionDef
+    docstring = _get_function_docstring(
+        package_name, module_relative_path, function_name
+    )
+
+    # Parse docstring (via docstring_parser) and prepare output
+    parsed_docstring = docparse(docstring)
+    descriptions = {
+        param.arg_name: _sanitize_description(param.description)
+        for param in parsed_docstring.params
+    }
+    logging.info(f"[_get_function_args_descriptions] END ({function_name=})")
+    return descriptions
+
+
+
+ +
+ + +
+ + + + +

+ _get_function_docstring(package_name, module_relative_path, function_name) + +

+ + +
+ +

Extract docstring from a function.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
package_name +
+

Example fractal_tasks_core.

+
+

+ + TYPE: + str + +

+
module_relative_path +
+

Example tasks/create_ome_zarr.py.

+
+

+ + TYPE: + str + +

+
function_name +
+

Example create_ome_zarr.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
def _get_function_docstring(
+    package_name: str, module_relative_path: str, function_name: str
+) -> str:
+    """
+    Extract docstring from a function.
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `tasks/create_ome_zarr.py`.
+        function_name: Example `create_ome_zarr`.
+    """
+
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"Module {module_relative_path} must end with '.py'")
+
+    # Get the function ast.FunctionDef object
+    package_path = Path(import_module(package_name).__file__).parent
+    module_path = package_path / module_relative_path
+    tree = ast.parse(module_path.read_text())
+    _function = next(
+        f
+        for f in ast.walk(tree)
+        if (isinstance(f, ast.FunctionDef) and f.name == function_name)
+    )
+
+    # Extract docstring from ast.FunctionDef
+    return ast.get_docstring(_function)
+
+
+
+ +
+ + +
+ + + + +

+ _insert_class_attrs_descriptions(*, schema, class_name, descriptions) + +

+ + +
+ +

Merge the descriptions obtained via _get_attributes_models_descriptions +into the class_name definition, within an existing JSON Schema

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
schema +
+

TBD

+
+

+ + TYPE: + dict + +

+
class_name +
+

TBD

+
+

+ + TYPE: + str + +

+
descriptions +
+

TBD

+
+

+ + TYPE: + dict + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
def _insert_class_attrs_descriptions(
+    *, schema: dict, class_name: str, descriptions: dict
+):
+    """
+    Merge the descriptions obtained via `_get_attributes_models_descriptions`
+    into the `class_name` definition, within an existing JSON Schema
+
+    Args:
+        schema: TBD
+        class_name: TBD
+        descriptions: TBD
+    """
+    new_schema = schema.copy()
+    if "definitions" not in schema:
+        return new_schema
+    else:
+        new_definitions = schema["definitions"].copy()
+    # Loop over existing definitions
+    for name, definition in schema["definitions"].items():
+        if name == class_name:
+            for prop in definition["properties"]:
+                if "description" in new_definitions[name]["properties"][prop]:
+                    raise ValueError(
+                        f"Property {name}.{prop} already has description"
+                    )
+                else:
+                    new_definitions[name]["properties"][prop][
+                        "description"
+                    ] = descriptions[prop]
+    new_schema["definitions"] = new_definitions
+    logging.info("[_insert_class_attrs_descriptions] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ _insert_function_args_descriptions(*, schema, descriptions) + +

+ + +
+ +

Merge the descriptions obtained via _get_args_descriptions into the +properties of an existing JSON Schema.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
schema +
+

TBD

+
+

+ + TYPE: + dict + +

+
descriptions +
+

TBD

+
+

+ + TYPE: + dict + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
def _insert_function_args_descriptions(*, schema: dict, descriptions: dict):
+    """
+    Merge the descriptions obtained via `_get_args_descriptions` into the
+    properties of an existing JSON Schema.
+
+    Args:
+        schema: TBD
+        descriptions: TBD
+    """
+    new_schema = schema.copy()
+    new_properties = schema["properties"].copy()
+    for key, value in schema["properties"].items():
+        if "description" in value:
+            raise ValueError("Property already has description")
+        else:
+            if key in descriptions:
+                value["description"] = descriptions[key]
+            else:
+                value["description"] = "Missing description"
+            new_properties[key] = value
+    new_schema["properties"] = new_properties
+    logging.info("[_insert_function_args_descriptions] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ _sanitize_description(string) + +

+ + +
+ +

Sanitize a description string.

+

This is a provisional helper function that replaces newlines with spaces +and reduces multiple contiguous whitespace characters to a single one. +Future iterations of the docstrings format/parsing may render this function +not-needed or obsolete.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
string +
+

TBD

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_descriptions.py + 
19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
def _sanitize_description(string: str) -> str:
+    """
+    Sanitize a description string.
+
+    This is a provisional helper function that replaces newlines with spaces
+    and reduces multiple contiguous whitespace characters to a single one.
+    Future iterations of the docstrings format/parsing may render this function
+    not-needed or obsolete.
+
+    Args:
+        string: TBD
+    """
+    # Replace newline with space
+    new_string = string.replace("\n", " ")
+    # Replace N-whitespace characterss with a single one
+    while "  " in new_string:
+        new_string = new_string.replace("  ", " ")
+    return new_string
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/lib_signature_constraints/index.html b/reference/fractal_tasks_core/dev/lib_signature_constraints/index.html new file mode 100644 index 000000000..adcd3f79d --- /dev/null +++ b/reference/fractal_tasks_core/dev/lib_signature_constraints/index.html @@ -0,0 +1,1427 @@ + + + + + + + + + + + + + + + + + + + + + + lib_signature_constraints - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_signature_constraints

+ +
+ + + + +
+ + + +
+ + + + + + + + + + +
+ + + + +

+ _extract_function(module_relative_path, function_name, package_name='fractal_tasks_core') + +

+ + +
+ +

Extract function from a module with the same name.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
package_name +
+

Example fractal_tasks_core.

+
+

+ + TYPE: + str + + + DEFAULT: + 'fractal_tasks_core' + +

+
module_relative_path +
+

Example tasks/create_ome_zarr.py.

+
+

+ + TYPE: + str + +

+
function_name +
+

Example create_ome_zarr.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_signature_constraints.py + 
33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
def _extract_function(
+    module_relative_path: str,
+    function_name: str,
+    package_name: str = "fractal_tasks_core",
+) -> Callable:
+    """
+    Extract function from a module with the same name.
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `tasks/create_ome_zarr.py`.
+        function_name: Example `create_ome_zarr`.
+    """
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"{module_relative_path=} must end with '.py'")
+    module_relative_path_no_py = str(
+        Path(module_relative_path).with_suffix("")
+    )
+    module_relative_path_dots = module_relative_path_no_py.replace("/", ".")
+    module = import_module(f"{package_name}.{module_relative_path_dots}")
+    task_function = getattr(module, function_name)
+    return task_function
+
+
+
+ +
+ + +
+ + + + +

+ _validate_function_signature(function) + +

+ + +
+ +

Validate the function signature.

+

Implement a set of checks for type hints that do not play well with the +creation of JSON Schema, see +https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/399.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
function +
+

TBD

+
+

+ + TYPE: + Callable + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_signature_constraints.py + 
57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
def _validate_function_signature(function: Callable):
+    """
+    Validate the function signature.
+
+    Implement a set of checks for type hints that do not play well with the
+    creation of JSON Schema, see
+    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/399.
+
+    Args:
+        function: TBD
+    """
+    sig = signature(function)
+    for param in sig.parameters.values():
+
+        # CASE 1: Check that name is not forbidden
+        if param.name in FORBIDDEN_PARAM_NAMES:
+            raise ValueError(
+                f"Function {function} has argument with name {param.name}"
+            )
+
+        # CASE 2: Raise an error for unions
+        if str(param.annotation).startswith(("typing.Union[", "Union[")):
+            raise ValueError("typing.Union is not supported")
+
+        # CASE 3: Raise an error for "|"
+        if "|" in str(param.annotation):
+            raise ValueError('Use of "|" in type hints is not supported')
+
+        # CASE 4: Raise an error for optional parameter with given (non-None)
+        # default, e.g. Optional[str] = "asd"
+        is_annotation_optional = str(param.annotation).startswith(
+            ("typing.Optional[", "Optional[")
+        )
+        default_given = (param.default is not None) and (
+            param.default != inspect._empty
+        )
+        if default_given and is_annotation_optional:
+            raise ValueError("Optional parameter has non-None default value")
+
+    logging.info("[_validate_function_signature] END")
+    return sig
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/lib_task_docs/index.html b/reference/fractal_tasks_core/dev/lib_task_docs/index.html new file mode 100644 index 000000000..643580c3b --- /dev/null +++ b/reference/fractal_tasks_core/dev/lib_task_docs/index.html @@ -0,0 +1,1444 @@ + + + + + + + + + + + + + + + + + + + + + + lib_task_docs - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_task_docs

+ +
+ + + + +
+ + + +
+ + + + + + + + + + +
+ + + + +

+ _get_function_description(package_name, module_relative_path, function_name) + +

+ + +
+ +

Extract function description from its docstring.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
package_name +
+

Example fractal_tasks_core.

+
+

+ + TYPE: + str + +

+
module_relative_path +
+

Example tasks/create_ome_zarr.py.

+
+

+ + TYPE: + str + +

+
function_name +
+

Example create_ome_zarr.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_task_docs.py + 
19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
def _get_function_description(
+    package_name: str, module_relative_path: str, function_name: str
+) -> str:
+    """
+    Extract function description from its docstring.
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `tasks/create_ome_zarr.py`.
+        function_name: Example `create_ome_zarr`.
+    """
+    # Extract docstring from ast.FunctionDef
+    docstring = _get_function_docstring(
+        package_name, module_relative_path, function_name
+    )
+    # Parse docstring (via docstring_parser)
+    parsed_docstring = docparse(docstring)
+    # Combine short/long descriptions (if present)
+    short_description = parsed_docstring.short_description
+    long_description = parsed_docstring.long_description
+    items = []
+    if short_description:
+        items.append(short_description)
+    if long_description:
+        items.append(long_description)
+    if items:
+        if parsed_docstring.blank_after_short_description:
+            return "\n\n".join(items)
+        else:
+            return "\n".join(items)
+    else:
+        return ""
+
+
+
+ +
+ + +
+ + + + +

+ create_docs_info(executable, package='fractal_tasks_core') + +

+ + +
+ +

Return task description based on function docstring.

+ +
+ Source code in fractal_tasks_core/dev/lib_task_docs.py + 
53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
def create_docs_info(
+    executable: str,
+    package: str = "fractal_tasks_core",
+) -> str:
+    """
+    Return task description based on function docstring.
+    """
+    logging.info("[create_docs_info] START")
+    # Extract the function name. Note: this could be made more general, but for
+    # the moment we assume the function has the same name as the module)
+    function_name = Path(executable).with_suffix("").name
+    logging.info(f"[create_docs_info] {function_name=}")
+    # Get function description
+    docs_info = _get_function_description(
+        package_name=package,
+        module_relative_path=executable,
+        function_name=function_name,
+    )
+    logging.info("[create_docs_info] END")
+    return docs_info
+
+
+
+ +
+ + +
+ + + + + + + +
+ +

Return link to docs page for a fractal_tasks_core task.

+ +
+ Source code in fractal_tasks_core/dev/lib_task_docs.py + 
75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
def create_docs_link(executable: str) -> str:
+    """
+    Return link to docs page for a fractal_tasks_core task.
+    """
+    logging.info("[create_docs_link] START")
+
+    # Extract the function name. Note: this could be made more general, but for
+    # the moment we assume the function has the same name as the module)
+    function_name = Path(executable).with_suffix("").name
+    logging.info(f"[create_docs_link] {function_name=}")
+    # Define docs_link
+    docs_link = (
+        "https://fractal-analytics-platform.github.io/fractal-tasks-core/"
+        f"reference/fractal_tasks_core/tasks/{function_name}/"
+        f"#fractal_tasks_core.tasks.{function_name}.{function_name}"
+    )
+    logging.info("[create_docs_link] END")
+    return docs_link
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/lib_titles/index.html b/reference/fractal_tasks_core/dev/lib_titles/index.html new file mode 100644 index 000000000..2b586b199 --- /dev/null +++ b/reference/fractal_tasks_core/dev/lib_titles/index.html @@ -0,0 +1,1382 @@ + + + + + + + + + + + + + + + + + + + + + + lib_titles - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_titles

+ +
+ + + + +
+ +

Module to include titles in JSON Schema properties.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _include_titles(schema) + +

+ + +
+ +

Include property titles, when missing.

+

This handles both:

+
    +
  • first-level JSON Schema properties (corresponding to task + arguments);
    • +
  • properties of JSON Schema definitions (corresponding to + task-argument attributes).
    • +
+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
schema +
+

TBD

+
+

+ + TYPE: + _Schema + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_titles.py + 
43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
def _include_titles(schema: _Schema) -> _Schema:
+    """
+    Include property titles, when missing.
+
+    This handles both:
+
+    - first-level JSON Schema properties (corresponding to task
+        arguments);
+    - properties of JSON Schema definitions (corresponding to
+        task-argument attributes).
+
+    Args:
+        schema: TBD
+    """
+    new_schema = schema.copy()
+
+    # Update first-level properties (that is, task arguments)
+    new_properties = _include_titles_for_properties(schema["properties"])
+    new_schema["properties"] = new_properties
+
+    # Update properties of definitions
+    if "definitions" in schema.keys():
+        new_definitions = schema["definitions"].copy()
+        for def_name, def_schema in new_definitions.items():
+            new_properties = _include_titles_for_properties(
+                def_schema["properties"]
+            )
+            new_definitions[def_name]["properties"] = new_properties
+        new_schema["definitions"] = new_definitions
+
+    logging.info("[_include_titles] END")
+    return new_schema
+
+
+
+ +
+ + +
+ + + + +

+ _include_titles_for_properties(properties) + +

+ + +
+ +

Scan through properties of a JSON Schema, and set their title when it is +missing.

+

The title is set to name.title(), where title is a standard string +method - see https://docs.python.org/3/library/stdtypes.html#str.title.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
properties +
+

TBD

+
+

+ + TYPE: + dict[str, dict] + +

+
+ +
+ Source code in fractal_tasks_core/dev/lib_titles.py + 
21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
def _include_titles_for_properties(
+    properties: dict[str, dict]
+) -> dict[str, dict]:
+    """
+    Scan through properties of a JSON Schema, and set their title when it is
+    missing.
+
+    The title is set to `name.title()`, where `title` is a standard string
+    method - see https://docs.python.org/3/library/stdtypes.html#str.title.
+
+    Args:
+        properties: TBD
+    """
+    new_properties = properties.copy()
+    for prop_name, prop in properties.items():
+        if "title" not in prop.keys():
+            new_prop = prop.copy()
+            new_prop["title"] = prop_name.title()
+            new_properties[prop_name] = new_prop
+    return new_properties
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/dev/update_manifest/index.html b/reference/fractal_tasks_core/dev/update_manifest/index.html new file mode 100644 index 000000000..348e19b61 --- /dev/null +++ b/reference/fractal_tasks_core/dev/update_manifest/index.html @@ -0,0 +1,1141 @@ + + + + + + + + + + + + + + + + + + + + + + update_manifest - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

update_manifest

+ +
+ + + + +
+ +

Script to generate JSON schemas for task arguments afresh, and write them +to the package manifest.

+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/index.html b/reference/fractal_tasks_core/index.html new file mode 100644 index 000000000..db986affb --- /dev/null +++ b/reference/fractal_tasks_core/index.html @@ -0,0 +1,1093 @@ + + + + + + + + + + + + + + + + + + Index - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Index

+ +
+ + + + +
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_ROI_overlaps/index.html b/reference/fractal_tasks_core/lib_ROI_overlaps/index.html new file mode 100644 index 000000000..7ff33d139 --- /dev/null +++ b/reference/fractal_tasks_core/lib_ROI_overlaps/index.html @@ -0,0 +1,3189 @@ + + + + + + + + + + + + + + + + + + + + + + lib_ROI_overlaps - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_ROI_overlaps

+ +
+ + + + +
+ +

Functions to identify and remove overlaps between regions of interest.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _is_overlapping_1D_int(line1, line2) + +

+ + +
+ +

Given two integer intervals, find whether they overlap

+

This is the same as is_overlapping_1D (based on +https://stackoverflow.com/a/70023212/19085332), for integer-valued +intervals.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
line1 +
+

The boundaries of the first interval , written as +[x_min, x_max].

+
+

+ + TYPE: + Sequence[int] + +

+
line2 +
+

The boundaries of the second interval , written as +[x_min, x_max].

+
+

+ + TYPE: + Sequence[int] + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
def _is_overlapping_1D_int(
+    line1: Sequence[int],
+    line2: Sequence[int],
+) -> bool:
+    """
+    Given two integer intervals, find whether they overlap
+
+    This is the same as `is_overlapping_1D` (based on
+    https://stackoverflow.com/a/70023212/19085332), for integer-valued
+    intervals.
+
+    Args:
+        line1: The boundaries of the first interval , written as
+            `[x_min, x_max]`.
+        line2: The boundaries of the second interval , written as
+            `[x_min, x_max]`.
+    """
+    return line1[0] < line2[1] and line2[0] < line1[1]
+
+
+
+ +
+ + +
+ + + + +

+ _is_overlapping_3D_int(box1, box2) + +

+ + +
+ +

Given two three-dimensional integer boxes, find whether they overlap.

+

This is the same as is_overlapping_3D (based on +https://stackoverflow.com/a/70023212/19085332), for integer-valued +boxes.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
box1 +
+

The boundaries of the first box, written as +[x_min, y_min, z_min, x_max, y_max, z_max].

+
+

+ + TYPE: + list[int] + +

+
box2 +
+

The boundaries of the second box, written as +[x_min, y_min, z_min, x_max, y_max, z_max].

+
+

+ + TYPE: + list[int] + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
def _is_overlapping_3D_int(box1: list[int], box2: list[int]) -> bool:
+    """
+    Given two three-dimensional integer boxes, find whether they overlap.
+
+    This is the same as is_overlapping_3D (based on
+    https://stackoverflow.com/a/70023212/19085332), for integer-valued
+    boxes.
+
+    Args:
+        box1: The boundaries of the first box, written as
+            `[x_min, y_min, z_min, x_max, y_max, z_max]`.
+        box2: The boundaries of the second box, written as
+            `[x_min, y_min, z_min, x_max, y_max, z_max]`.
+    """
+    overlap_x = _is_overlapping_1D_int([box1[0], box1[3]], [box2[0], box2[3]])
+    overlap_y = _is_overlapping_1D_int([box1[1], box1[4]], [box2[1], box2[4]])
+    overlap_z = _is_overlapping_1D_int([box1[2], box1[5]], [box2[2], box2[5]])
+    return overlap_x and overlap_y and overlap_z
+
+
+
+ +
+ + +
+ + + + +

+ apply_shift_in_one_direction(tmp_df_well, line_1, line_2, mu, tol=1e-10) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
tmp_df_well +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
line_1 +
+

TBD

+
+

+ + TYPE: + Sequence[float] + +

+
line_2 +
+

TBD

+
+

+ + TYPE: + Sequence[float] + +

+
mu +
+

TBD

+
+

+ + TYPE: + str + +

+
tol +
+

TBD

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
def apply_shift_in_one_direction(
+    tmp_df_well: pd.DataFrame,
+    line_1: Sequence[float],
+    line_2: Sequence[float],
+    mu: str,
+    tol: float = 1e-10,
+):
+    """
+    TBD
+
+    Args:
+        tmp_df_well: TBD
+        line_1: TBD
+        line_2: TBD
+        mu: TBD
+        tol: TBD
+    """
+    min_1, max_1 = line_1[:]
+    min_2, max_2 = line_2[:]
+    min_max = min(max_1, max_2)
+    max_min = max(min_1, min_2)
+    shift = min_max - max_min
+    logging.debug(f"{mu}-shifting by {shift=}")
+    ind = tmp_df_well.loc[:, f"{mu}min"] >= max_min - tol
+    if not (shift > 0.0 and ind.to_numpy().max() > 0):
+        raise ValueError(
+            "Something wrong in apply_shift_in_one_direction\n"
+            f"{mu=}\n{shift=}\n{ind.to_numpy()=}"
+        )
+    tmp_df_well.loc[ind, f"{mu}min"] += shift
+    tmp_df_well.loc[ind, f"{mu}max"] += shift
+    tmp_df_well.loc[ind, f"{mu}_micrometer"] += shift
+    return tmp_df_well
+
+
+
+ +
+ + +
+ + + + +

+ check_well_for_FOV_overlap(site_metadata, selected_well, plotting_function, tol=1e-10) + +

+ + +
+ +

This function is currently only used in tests and examples.

+

The plotting_function parameter is exposed so that other tools (see +examples in this repository) may use it to show the FOV ROIs.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
site_metadata +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
selected_well +
+

TBD

+
+

+ + TYPE: + str + +

+
plotting_function +
+

TBD

+
+

+ + TYPE: + Callable + +

+
tol +
+

TBD

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
def check_well_for_FOV_overlap(
+    site_metadata: pd.DataFrame,
+    selected_well: str,
+    plotting_function: Callable,
+    tol: float = 1e-10,
+):
+    """
+    This function is currently only used in tests and examples.
+
+    The `plotting_function` parameter is exposed so that other tools (see
+    examples in this repository) may use it to show the FOV ROIs.
+
+    Args:
+        site_metadata: TBD
+        selected_well: TBD
+        plotting_function: TBD
+        tol: TBD
+    """
+
+    df = site_metadata.loc[selected_well].copy()
+    df["xmin"] = df["x_micrometer"]
+    df["ymin"] = df["y_micrometer"]
+    df["xmax"] = df["x_micrometer"] + df["pixel_size_x"] * df["x_pixel"]
+    df["ymax"] = df["y_micrometer"] + df["pixel_size_y"] * df["y_pixel"]
+
+    xmin = list(df.loc[:, "xmin"])
+    ymin = list(df.loc[:, "ymin"])
+    xmax = list(df.loc[:, "xmax"])
+    ymax = list(df.loc[:, "ymax"])
+    num_lines = len(xmin)
+
+    list_overlapping_FOVs = []
+    for line_1 in range(num_lines):
+        min_x_1, max_x_1 = [a[line_1] for a in [xmin, xmax]]
+        min_y_1, max_y_1 = [a[line_1] for a in [ymin, ymax]]
+        for line_2 in range(line_1):
+            min_x_2, max_x_2 = [a[line_2] for a in [xmin, xmax]]
+            min_y_2, max_y_2 = [a[line_2] for a in [ymin, ymax]]
+            overlap = is_overlapping_2D(
+                (min_x_1, min_y_1, max_x_1, max_y_1),
+                (min_x_2, min_y_2, max_x_2, max_y_2),
+                tol=tol,
+            )
+            if overlap:
+                list_overlapping_FOVs.append(line_1)
+                list_overlapping_FOVs.append(line_2)
+
+    # Call plotting_function
+    plotting_function(
+        xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well
+    )
+
+    if len(list_overlapping_FOVs) > 0:
+        # Increase values by one to switch from index to the label plotted
+        return {selected_well: [x + 1 for x in list_overlapping_FOVs]}
+
+
+
+ +
+ + +
+ + + + +

+ find_overlaps_in_ROI_indices(list_indices) + +

+ + +
+ +

Given a list of integer ROI indices, find whether there are overlaps.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
list_indices +
+

List of ROI indices, where each element in the list +should look like +[start_z, end_z, start_y, end_y, start_x, end_x].

+
+

+ + TYPE: + list[list[int]] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + Optional[tuple[int, int]] + + +
+

None if no overlap was detected, otherwise a tuple with the +positional indices of a pair of overlapping ROIs.

+
+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
def find_overlaps_in_ROI_indices(
+    list_indices: list[list[int]],
+) -> Optional[tuple[int, int]]:
+    """
+    Given a list of integer ROI indices, find whether there are overlaps.
+
+    Args:
+        list_indices: List of ROI indices, where each element in the list
+            should look like
+            `[start_z, end_z, start_y, end_y, start_x, end_x]`.
+
+    Returns:
+        `None` if no overlap was detected, otherwise a tuple with the
+            positional indices of a pair of overlapping ROIs.
+    """
+
+    for ind_1, ROI_1 in enumerate(list_indices):
+        s_z, e_z, s_y, e_y, s_x, e_x = ROI_1[:]
+        box_1 = [s_x, s_y, s_z, e_x, e_y, e_z]
+        for ind_2 in range(ind_1):
+            ROI_2 = list_indices[ind_2]
+            s_z, e_z, s_y, e_y, s_x, e_x = ROI_2[:]
+            box_2 = [s_x, s_y, s_z, e_x, e_y, e_z]
+            if _is_overlapping_3D_int(box_1, box_2):
+                return (ind_1, ind_2)
+    return None
+
+
+
+ +
+ + +
+ + + + +

+ get_overlapping_pair(tmp_df, tol=1e-10) + +

+ + +
+ +

Finds the indices for the next overlapping FOVs pair.

+

Note: the returned indices are positional indices, starting from 0.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
tmp_df +
+

Dataframe with columns ["xmin", "ymin", "xmax", "ymax"].

+
+

+ + TYPE: + DataFrame + +

+
tol +
+

Finite tolerance for floating-point comparisons.

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
def get_overlapping_pair(
+    tmp_df: pd.DataFrame, tol: float = 1e-10
+) -> Union[tuple[int, int], bool]:
+    """
+    Finds the indices for the next overlapping FOVs pair.
+
+    Note: the returned indices are positional indices, starting from 0.
+
+    Args:
+        tmp_df: Dataframe with columns `["xmin", "ymin", "xmax", "ymax"]`.
+        tol: Finite tolerance for floating-point comparisons.
+    """
+
+    num_lines = len(tmp_df.index)
+    for pos_ind_1 in range(num_lines):
+        for pos_ind_2 in range(pos_ind_1):
+            if is_overlapping_2D(
+                tmp_df.iloc[pos_ind_1], tmp_df.iloc[pos_ind_2], tol=tol
+            ):
+                return (pos_ind_1, pos_ind_2)
+    return False
+
+
+
+ +
+ + +
+ + + + +

+ get_overlapping_pairs_3D(tmp_df, full_res_pxl_sizes_zyx) + +

+ + +
+ +

Finds the indices for the all overlapping FOVs pair, in three dimensions.

+

Note: the returned indices are positional indices, starting from 0.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
tmp_df +
+

Dataframe with columns {x,y,z}_micrometer and +len_{x,y,z}_micrometer.

+
+

+ + TYPE: + DataFrame + +

+
full_res_pxl_sizes_zyx +
+

TBD

+
+

+ + TYPE: + Sequence[float] + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
def get_overlapping_pairs_3D(
+    tmp_df: pd.DataFrame,
+    full_res_pxl_sizes_zyx: Sequence[float],
+):
+    """
+    Finds the indices for the all overlapping FOVs pair, in three dimensions.
+
+    Note: the returned indices are positional indices, starting from 0.
+
+    Args:
+        tmp_df: Dataframe with columns `{x,y,z}_micrometer` and
+            `len_{x,y,z}_micrometer`.
+        full_res_pxl_sizes_zyx: TBD
+    """
+
+    tol = 1e-10
+    if tol > min(full_res_pxl_sizes_zyx) / 1e3:
+        raise ValueError(f"{tol=} but {full_res_pxl_sizes_zyx=}")
+
+    new_tmp_df = tmp_df.copy()
+
+    new_tmp_df["x_micrometer_max"] = (
+        new_tmp_df["x_micrometer"] + new_tmp_df["len_x_micrometer"]
+    )
+    new_tmp_df["y_micrometer_max"] = (
+        new_tmp_df["y_micrometer"] + new_tmp_df["len_y_micrometer"]
+    )
+    new_tmp_df["z_micrometer_max"] = (
+        new_tmp_df["z_micrometer"] + new_tmp_df["len_z_micrometer"]
+    )
+    # Remove columns which are not necessary for overlap checks
+    list_columns = [
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "label",
+    ]
+    new_tmp_df.drop(labels=list_columns, axis=1, inplace=True)
+
+    # Loop over all pairs, and construct list of overlapping ones
+    num_lines = len(new_tmp_df.index)
+    overlapping_list = []
+    for pos_ind_1 in range(num_lines):
+        for pos_ind_2 in range(pos_ind_1):
+            overlap = is_overlapping_3D(
+                new_tmp_df.iloc[pos_ind_1], new_tmp_df.iloc[pos_ind_2], tol=tol
+            )
+            if overlap:
+                overlapping_list.append((pos_ind_1, pos_ind_2))
+    return overlapping_list
+
+
+
+ +
+ + +
+ + + + +

+ is_overlapping_1D(line1, line2, tol=1e-10) + +

+ + +
+ +

Given two intervals, finds whether they overlap.

+

This is based on https://stackoverflow.com/a/70023212/19085332, and we +additionally use a finite tolerance for floating-point comparisons.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
line1 +
+

The boundaries of the first interval, written as +[x_min, x_max].

+
+

+ + TYPE: + Sequence[float] + +

+
line2 +
+

The boundaries of the second interval, written as +[x_min, x_max].

+
+

+ + TYPE: + Sequence[float] + +

+
tol +
+

Finite tolerance for floating-point comparisons.

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
def is_overlapping_1D(
+    line1: Sequence[float], line2: Sequence[float], tol: float = 1e-10
+) -> bool:
+    """
+    Given two intervals, finds whether they overlap.
+
+    This is based on https://stackoverflow.com/a/70023212/19085332, and we
+    additionally use a finite tolerance for floating-point comparisons.
+
+    Args:
+        line1: The boundaries of the first interval, written as
+            `[x_min, x_max]`.
+        line2: The boundaries of the second interval, written as
+            `[x_min, x_max]`.
+        tol: Finite tolerance for floating-point comparisons.
+    """
+    return line1[0] <= line2[1] - tol and line2[0] <= line1[1] - tol
+
+
+
+ +
+ + +
+ + + + +

+ is_overlapping_2D(box1, box2, tol=1e-10) + +

+ + +
+ +

Given two rectangular boxes, finds whether they overlap.

+

This is based on https://stackoverflow.com/a/70023212/19085332, and we +additionally use a finite tolerance for floating-point comparisons.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
box1 +
+

The boundaries of the first rectangle, written as +[x_min, y_min, x_max, y_max].

+
+

+ + TYPE: + Sequence[float] + +

+
box2 +
+

The boundaries of the second rectangle, written as +[x_min, y_min, x_max, y_max].

+
+

+ + TYPE: + Sequence[float] + +

+
tol +
+

Finite tolerance for floating-point comparisons.

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
def is_overlapping_2D(
+    box1: Sequence[float], box2: Sequence[float], tol: float = 1e-10
+) -> bool:
+    """
+    Given two rectangular boxes, finds whether they overlap.
+
+    This is based on https://stackoverflow.com/a/70023212/19085332, and we
+    additionally use a finite tolerance for floating-point comparisons.
+
+    Args:
+        box1: The boundaries of the first rectangle, written as
+            `[x_min, y_min, x_max, y_max]`.
+        box2: The boundaries of the second rectangle, written as
+            `[x_min, y_min, x_max, y_max]`.
+        tol: Finite tolerance for floating-point comparisons.
+    """
+    overlap_x = is_overlapping_1D(
+        [box1[0], box1[2]], [box2[0], box2[2]], tol=tol
+    )
+    overlap_y = is_overlapping_1D(
+        [box1[1], box1[3]], [box2[1], box2[3]], tol=tol
+    )
+    return overlap_x and overlap_y
+
+
+
+ +
+ + +
+ + + + +

+ is_overlapping_3D(box1, box2, tol=1e-10) + +

+ + +
+ +

Given two three-dimensional boxes, finds whether they overlap.

+

This is based on https://stackoverflow.com/a/70023212/19085332, and we +additionally use a finite tolerance for floating-point comparisons.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
box1 +
+

The boundaries of the first box, written as +[x_min, y_min, z_min, x_max, y_max, z_max].

+
+

+ + TYPE: + Sequence[float] + +

+
box2 +
+

The boundaries of the second box, written as +[x_min, y_min, z_min, x_max, y_max, z_max].

+
+

+ + TYPE: + Sequence[float] + +

+
tol +
+

Finite tolerance for floating-point comparisons.

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
def is_overlapping_3D(
+    box1: Sequence[float], box2: Sequence[float], tol: float = 1e-10
+) -> bool:
+    """
+    Given two three-dimensional boxes, finds whether they overlap.
+
+    This is based on https://stackoverflow.com/a/70023212/19085332, and we
+    additionally use a finite tolerance for floating-point comparisons.
+
+    Args:
+        box1: The boundaries of the first box, written as
+            `[x_min, y_min, z_min, x_max, y_max, z_max]`.
+        box2: The boundaries of the second box, written as
+            `[x_min, y_min, z_min, x_max, y_max, z_max]`.
+        tol: Finite tolerance for floating-point comparisons.
+    """
+
+    overlap_x = is_overlapping_1D(
+        [box1[0], box1[3]], [box2[0], box2[3]], tol=tol
+    )
+    overlap_y = is_overlapping_1D(
+        [box1[1], box1[4]], [box2[1], box2[4]], tol=tol
+    )
+    overlap_z = is_overlapping_1D(
+        [box1[2], box1[5]], [box2[2], box2[5]], tol=tol
+    )
+    return overlap_x and overlap_y and overlap_z
+
+
+
+ +
+ + +
+ + + + +

+ remove_FOV_overlaps(df) + +

+ + +
+ +

Given a metadata dataframe, shift its columns to remove FOV overlaps.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
df +
+

Metadata dataframe.

+
+

+ + TYPE: + DataFrame + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
def remove_FOV_overlaps(df: pd.DataFrame):
+    """
+    Given a metadata dataframe, shift its columns to remove FOV overlaps.
+
+    Args:
+        df: Metadata dataframe.
+    """
+
+    # Set tolerance (this should be much smaller than pixel size or expected
+    # round-offs), and maximum number of iterations in constraint solver
+    tol = 1e-10
+    max_iterations = 200
+
+    # Create a local copy of the dataframe
+    df = df.copy()
+
+    # Create temporary columns (to streamline overlap removals), which are
+    # then removed at the end of the remove_FOV_overlaps function
+    df["xmin"] = df["x_micrometer"]
+    df["ymin"] = df["y_micrometer"]
+    df["xmax"] = df["x_micrometer"] + df["pixel_size_x"] * df["x_pixel"]
+    df["ymax"] = df["y_micrometer"] + df["pixel_size_y"] * df["y_pixel"]
+    list_columns = ["xmin", "ymin", "xmax", "ymax"]
+
+    # Create columns with the original positions (not to be removed)
+    df["x_micrometer_original"] = df["x_micrometer"]
+    df["y_micrometer_original"] = df["y_micrometer"]
+
+    # Check that tolerance is much smaller than pixel sizes
+    min_pixel_size = df[["pixel_size_x", "pixel_size_y"]].min().min()
+    if tol > min_pixel_size / 1e3:
+        raise ValueError(
+            f"In remove_FOV_overlaps, {tol=} but {min_pixel_size=}"
+        )
+
+    # Loop over wells
+    wells = sorted(list(set([ind[0] for ind in df.index])))
+    for well in wells:
+
+        logger.info(f"removing FOV overlaps for {well=}")
+        df_well = df.loc[well].copy()
+
+        # NOTE: these are positional indices (i.e. starting from 0)
+        pair_pos_indices = get_overlapping_pair(df_well[list_columns], tol=tol)
+
+        # Keep going until there are no overlaps, or until iteration reaches
+        # max_iterations
+        iteration = 0
+        while pair_pos_indices:
+            iteration += 1
+
+            # Identify overlapping FOVs
+            pos_ind_1, pos_ind_2 = pair_pos_indices
+            fov_id_1 = df_well.index[pos_ind_1]
+            fov_id_2 = df_well.index[pos_ind_2]
+            xmin_1, ymin_1, xmax_1, ymax_1 = df_well[list_columns].iloc[
+                pos_ind_1
+            ]
+            xmin_2, ymin_2, xmax_2, ymax_2 = df_well[list_columns].iloc[
+                pos_ind_2
+            ]
+            logger.debug(
+                f"{well=}, {iteration=}, removing overlap between"
+                f" {fov_id_1=} and {fov_id_2=}"
+            )
+
+            # Check what kind of overlap is there (X, Y, or XY)
+            is_x_equal = abs(xmin_1 - xmin_2) < tol and (xmax_1 - xmax_2) < tol
+            is_y_equal = abs(ymin_1 - ymin_2) < tol and (ymax_1 - ymax_2) < tol
+            is_x_overlap = is_overlapping_1D(
+                [xmin_1, xmax_1], [xmin_2, xmax_2], tol=tol
+            )
+            is_y_overlap = is_overlapping_1D(
+                [ymin_1, ymax_1], [ymin_2, ymax_2], tol=tol
+            )
+
+            if is_x_equal and is_y_overlap:
+                # Y overlap
+                df_well = apply_shift_in_one_direction(
+                    df_well,
+                    [ymin_1, ymax_1],
+                    [ymin_2, ymax_2],
+                    mu="y",
+                    tol=tol,
+                )
+            elif is_y_equal and is_x_overlap:
+                # X overlap
+                df_well = apply_shift_in_one_direction(
+                    df_well,
+                    [xmin_1, xmax_1],
+                    [xmin_2, xmax_2],
+                    mu="x",
+                    tol=tol,
+                )
+            elif not (is_x_equal or is_y_equal) and (
+                is_x_overlap and is_y_overlap
+            ):
+                # XY overlap
+                df_well = apply_shift_in_one_direction(
+                    df_well,
+                    [xmin_1, xmax_1],
+                    [xmin_2, xmax_2],
+                    mu="x",
+                    tol=tol,
+                )
+                df_well = apply_shift_in_one_direction(
+                    df_well,
+                    [ymin_1, ymax_1],
+                    [ymin_2, ymax_2],
+                    mu="y",
+                    tol=tol,
+                )
+            else:
+                raise ValueError(
+                    "Trying to remove overlap which is not there."
+                )
+
+            # Look for next overlapping FOV pair
+            pair_pos_indices = get_overlapping_pair(
+                df_well[list_columns], tol=tol
+            )
+
+            # Enforce maximum number of iterations
+            if iteration >= max_iterations:
+                raise ValueError(f"Reached {max_iterations=} for {well=}")
+
+        # Note: using df.loc[well] = df_well leads to a NaN dataframe, see
+        # for instance https://stackoverflow.com/a/28432733/19085332
+        df.loc[well, :] = df_well.values
+
+    # Remove temporary columns that were added only as part of this function
+    df.drop(list_columns, axis=1, inplace=True)
+
+    return df
+
+
+
+ +
+ + +
+ + + + +

+ run_overlap_check(site_metadata, tol=1e-10, plotting_function=None) + +

+ + +
+ +

Run an overlap check over all wells and optionally plots overlaps.

+

This function is currently only used in tests and examples.

+

The plotting_function parameter is exposed so that other tools (see +examples in this repository) may use it to show the FOV ROIs. Its arguments +are: [xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well].

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
site_metadata +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
tol +
+

TBD

+
+

+ + TYPE: + float + + + DEFAULT: + 1e-10 + +

+
plotting_function +
+

TBD

+
+

+ + TYPE: + Optional[Callable] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_ROI_overlaps.py + 
470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
def run_overlap_check(
+    site_metadata: pd.DataFrame,
+    tol: float = 1e-10,
+    plotting_function: Optional[Callable] = None,
+):
+    """
+    Run an overlap check over all wells and optionally plots overlaps.
+
+    This function is currently only used in tests and examples.
+
+    The `plotting_function` parameter is exposed so that other tools (see
+    examples in this repository) may use it to show the FOV ROIs. Its arguments
+    are: `[xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well]`.
+
+    Args:
+        site_metadata: TBD
+        tol: TBD
+        plotting_function: TBD
+    """
+
+    if plotting_function is None:
+
+        def plotting_function(
+            xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well
+        ):
+            pass
+
+    wells = site_metadata.index.unique(level="well_id")
+    overlapping_FOVs = []
+    for selected_well in wells:
+        overlap_curr_well = check_well_for_FOV_overlap(
+            site_metadata,
+            selected_well=selected_well,
+            tol=tol,
+            plotting_function=plotting_function,
+        )
+        if overlap_curr_well:
+            print(selected_well)
+            overlapping_FOVs.append(overlap_curr_well)
+
+    return overlapping_FOVs
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_channels/index.html b/reference/fractal_tasks_core/lib_channels/index.html new file mode 100644 index 000000000..69bdee5bb --- /dev/null +++ b/reference/fractal_tasks_core/lib_channels/index.html @@ -0,0 +1,3224 @@ + + + + + + + + + + + + + + + + + + + + + + lib_channels - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_channels

+ +
+ + + + +
+ +

Helper functions to address channels via OME-NGFF/OMERO metadata.

+ + + +
+ + + + + + + + +
+ + + + +

+ ChannelNotFoundError + + +

+ + +
+

+ Bases: ValueError

+ + +

Custom error for when get_channel_from_list fails, +that can be captured and handled upstream if needed.

+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
104
+105
+106
+107
+108
+109
+110
class ChannelNotFoundError(ValueError):
+    """
+    Custom error for when `get_channel_from_list` fails,
+    that can be captured and handled upstream if needed.
+    """
+
+    pass
+
+
+ +
+ +
+ +
+ + + + +

+ OmeroChannel + + +

+ + +
+

+ Bases: BaseModel

+ + +

Custom class for Omero channels, based on OME-NGFF v0.4.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ATTRIBUTEDESCRIPTION
wavelength_id +
+

Unique ID for the channel wavelength, e.g. A01_C01.

+
+

+ + TYPE: + str + +

+
index +
+

Do not change. For internal use only.

+
+

+ + TYPE: + Optional[int] + +

+
label +
+

Name of the channel.

+
+

+ + TYPE: + Optional[str] + +

+
window +
+

Optional Window object to set default display settings for +napari.

+
+

+ + TYPE: + Optional[Window] + +

+
color +
+

Optional hex colormap to display the channel in napari (it +must be of length 6, e.g. 00FFFF).

+
+

+ + TYPE: + Optional[str] + +

+
active +
+

Should this channel be shown in the viewer?

+
+

+ + TYPE: + bool + +

+
coefficient +
+

Do not change. Omero-channel attribute.

+
+

+ + TYPE: + int + +

+
inverted +
+

Do not change. Omero-channel attribute.

+
+

+ + TYPE: + bool + +

+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
class OmeroChannel(BaseModel):
+    """
+    Custom class for Omero channels, based on OME-NGFF v0.4.
+
+    Attributes:
+        wavelength_id: Unique ID for the channel wavelength, e.g. `A01_C01`.
+        index: Do not change. For internal use only.
+        label: Name of the channel.
+        window: Optional `Window` object to set default display settings for
+            napari.
+        color: Optional hex colormap to display the channel in napari (it
+            must be of length 6, e.g. `00FFFF`).
+        active: Should this channel be shown in the viewer?
+        coefficient: Do not change. Omero-channel attribute.
+        inverted: Do not change. Omero-channel attribute.
+    """
+
+    # Custom
+
+    wavelength_id: str
+    index: Optional[int]
+
+    # From OME-NGFF v0.4 transitional metadata
+
+    label: Optional[str]
+    window: Optional[Window]
+    color: Optional[str]
+    active: bool = True
+    coefficient: int = 1
+    inverted: bool = False
+
+    @validator("color", always=True)
+    def valid_hex_color(cls, v, values):
+        """
+        Check that `color` is made of exactly six elements which are letters
+        (a-f or A-F) or digits (0-9).
+        """
+        if v is None:
+            return v
+        if len(v) != 6:
+            raise ValueError(f'color must have length 6 (given: "{v}")')
+        allowed_characters = "abcdefABCDEF0123456789"
+        for character in v:
+            if character not in allowed_characters:
+                raise ValueError(
+                    "color must only include characters from "
+                    f'"{allowed_characters}" (given: "{v}")'
+                )
+        return v
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ valid_hex_color(v, values) + +

+ + +
+ +

Check that color is made of exactly six elements which are letters +(a-f or A-F) or digits (0-9).

+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
@validator("color", always=True)
+def valid_hex_color(cls, v, values):
+    """
+    Check that `color` is made of exactly six elements which are letters
+    (a-f or A-F) or digits (0-9).
+    """
+    if v is None:
+        return v
+    if len(v) != 6:
+        raise ValueError(f'color must have length 6 (given: "{v}")')
+    allowed_characters = "abcdefABCDEF0123456789"
+    for character in v:
+        if character not in allowed_characters:
+            raise ValueError(
+                "color must only include characters from "
+                f'"{allowed_characters}" (given: "{v}")'
+            )
+    return v
+
+
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + +

+ Window + + +

+ + +
+

+ Bases: BaseModel

+ + +

Custom class for Omero-channel window, based on OME-NGFF v0.4.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ATTRIBUTEDESCRIPTION
min +
+

Do not change. It will be set to 0 by default.

+
+

+ + TYPE: + Optional[int] + +

+
max +
+

Do not change. It will be set according to bit-depth of the images +by default (e.g. 65535 for 16 bit images).

+
+

+ + TYPE: + Optional[int] + +

+
start +
+

Lower-bound rescaling value for visualization.

+
+

+ + TYPE: + int + +

+
end +
+

Upper-bound rescaling value for visualization.

+
+

+ + TYPE: + int + +

+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
class Window(BaseModel):
+    """
+    Custom class for Omero-channel window, based on OME-NGFF v0.4.
+
+    Attributes:
+        min: Do not change. It will be set to `0` by default.
+        max:
+            Do not change. It will be set according to bit-depth of the images
+            by default (e.g. 65535 for 16 bit images).
+        start: Lower-bound rescaling value for visualization.
+        end: Upper-bound rescaling value for visualization.
+    """
+
+    min: Optional[int]
+    max: Optional[int]
+    start: int
+    end: int
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + +
+ + + + +

+ _get_new_unique_value(value, existing_values) + +

+ + +
+ +

Produce a string value that is not present in a given list

+

Append _1, _2, ... to a given string, if needed, until finding a value +which is not already present in existing_values.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
value +
+

The first guess for the new value

+
+

+ + TYPE: + str + +

+
existing_values +
+

The list of existing values

+
+

+ + TYPE: + list[str] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + str + + +
+

A string value which is not present in existing_values

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
def _get_new_unique_value(
+    value: str,
+    existing_values: list[str],
+) -> str:
+    """
+    Produce a string value that is not present in a given list
+
+    Append `_1`, `_2`, ... to a given string, if needed, until finding a value
+    which is not already present in `existing_values`.
+
+    Args:
+        value: The first guess for the new value
+        existing_values: The list of existing values
+
+    Returns:
+        A string value which is not present in `existing_values`
+    """
+    counter = 1
+    new_value = value
+    while new_value in existing_values:
+        new_value = f"{value}-{counter}"
+        counter += 1
+    return new_value
+
+
+
+ +
+ + +
+ + + + +

+ check_unique_wavelength_ids(channels) + +

+ + +
+ +

Check that the wavelength_id attributes of a channel list are unique.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
channels +
+

TBD

+
+

+ + TYPE: + list[OmeroChannel] + +

+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
def check_unique_wavelength_ids(channels: list[OmeroChannel]):
+    """
+    Check that the `wavelength_id` attributes of a channel list are unique.
+
+    Args:
+        channels: TBD
+    """
+    wavelength_ids = [c.wavelength_id for c in channels]
+    if len(set(wavelength_ids)) < len(wavelength_ids):
+        raise ValueError(
+            f"Non-unique wavelength_id's in {wavelength_ids}\n" f"{channels=}"
+        )
+
+
+
+ +
+ + +
+ + + + +

+ check_well_channel_labels(*, well_zarr_path) + +

+ + +
+ +

Check that the channel labels for a well are unique.

+

First identify the channel-labels list for each image in the well, then +compare lists and verify their intersection is empty.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
well_zarr_path +
+

path to an OME-NGFF well zarr group.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
def check_well_channel_labels(*, well_zarr_path: str) -> None:
+    """
+    Check that the channel labels for a well are unique.
+
+    First identify the channel-labels list for each image in the well, then
+    compare lists and verify their intersection is empty.
+
+    Args:
+        well_zarr_path: path to an OME-NGFF well zarr group.
+    """
+
+    # Iterate over all images (multiplexing cycles, multi-FOVs, ...)
+    group = zarr.open_group(well_zarr_path, mode="r+")
+    image_paths = [image["path"] for image in group.attrs["well"]["images"]]
+    list_of_channel_lists = []
+    for image_path in image_paths:
+        channels = get_omero_channel_list(
+            image_zarr_path=f"{well_zarr_path}/{image_path}"
+        )
+        list_of_channel_lists.append(channels[:])
+
+    # For each pair of channel-labels lists, verify they do not overlap
+    for ind_1, channels_1 in enumerate(list_of_channel_lists):
+        labels_1 = set([c.label for c in channels_1])
+        for ind_2 in range(ind_1):
+            channels_2 = list_of_channel_lists[ind_2]
+            labels_2 = set([c.label for c in channels_2])
+            intersection = labels_1 & labels_2
+            if intersection:
+                hint = (
+                    "Are you parsing fields of view into separate OME-Zarr "
+                    "images? This could lead to non-unique channel labels, "
+                    "and then could be the reason of the error"
+                )
+                raise ValueError(
+                    "Non-unique channel labels\n"
+                    f"{labels_1=}\n{labels_2=}\n{hint}"
+                )
+
+
+
+ +
+ + +
+ + + + +

+ define_omero_channels(*, channels, bit_depth, label_prefix=None) + +

+ + +
+ +

Update a channel list to use it in the OMERO/channels metadata.

+

Given a list of channel dictionaries, update each one of them by: + 1. Adding a label (if missing); + 2. Adding a set of OMERO-specific attributes; + 3. Discarding all other attributes.

+

The new_channels output can be used in the attrs["omero"]["channels"] +attribute of an image group.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
channels +
+

A list of channel dictionaries (each one must include the +wavelength_id key).

+
+

+ + TYPE: + list[OmeroChannel] + +

+
bit_depth +
+

bit depth.

+
+

+ + TYPE: + int + +

+
label_prefix +
+

TBD

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + list[dict[str, Union[str, int, bool, dict[str, int]]]] + + +
+

new_channels, a new list of consistent channel dictionaries that +can be written to OMERO metadata.

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
def define_omero_channels(
+    *,
+    channels: list[OmeroChannel],
+    bit_depth: int,
+    label_prefix: Optional[str] = None,
+) -> list[dict[str, Union[str, int, bool, dict[str, int]]]]:
+    """
+    Update a channel list to use it in the OMERO/channels metadata.
+
+    Given a list of channel dictionaries, update each one of them by:
+        1. Adding a label (if missing);
+        2. Adding a set of OMERO-specific attributes;
+        3. Discarding all other attributes.
+
+    The `new_channels` output can be used in the `attrs["omero"]["channels"]`
+    attribute of an image group.
+
+    Args:
+        channels: A list of channel dictionaries (each one must include the
+            `wavelength_id` key).
+        bit_depth: bit depth.
+        label_prefix: TBD
+
+    Returns:
+        `new_channels`, a new list of consistent channel dictionaries that
+            can be written to OMERO metadata.
+    """
+
+    new_channels = [c.copy(deep=True) for c in channels]
+    default_colors = ["00FFFF", "FF00FF", "FFFF00"]
+
+    for channel in new_channels:
+        wavelength_id = channel.wavelength_id
+
+        # If channel.label is None, set it to a default value
+        if channel.label is None:
+            default_label = wavelength_id
+            if label_prefix:
+                default_label = f"{label_prefix}_{default_label}"
+            logging.warning(
+                f"Missing label for {channel=}, using {default_label=}"
+            )
+            channel.label = default_label
+
+        # If channel.color is None, set it to a default value (use the default
+        # ones for the first three channels, or gray otherwise)
+        if channel.color is None:
+            try:
+                channel.color = default_colors.pop()
+            except IndexError:
+                channel.color = "808080"
+
+        # Set channel.window attribute
+        if channel.window:
+            channel.window.min = 0
+            channel.window.max = 2**bit_depth - 1
+
+    # Check that channel labels are unique for this image
+    labels = [c.label for c in new_channels]
+    if len(set(labels)) < len(labels):
+        raise ValueError(f"Non-unique labels in {new_channels=}")
+
+    new_channels_dictionaries = [
+        c.dict(exclude={"index"}, exclude_unset=True) for c in new_channels
+    ]
+
+    return new_channels_dictionaries
+
+
+
+ +
+ + +
+ + + + +

+ get_channel_from_image_zarr(*, image_zarr_path, label=None, wavelength_id=None) + +

+ + +
+ +

Extract a channel from OME-NGFF zarr attributes.

+

This is a helper function that combines get_omero_channel_list with +get_channel_from_list.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_zarr_path +
+

Path to an OME-NGFF image zarr group.

+
+

+ + TYPE: + str + +

+
label +
+

label attribute of the channel to be extracted.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
wavelength_id +
+

wavelength_id attribute of the channel to be +extracted.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + OmeroChannel + + +
+

A single channel dictionary.

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
def get_channel_from_image_zarr(
+    *,
+    image_zarr_path: str,
+    label: Optional[str] = None,
+    wavelength_id: Optional[str] = None,
+) -> OmeroChannel:
+    """
+    Extract a channel from OME-NGFF zarr attributes.
+
+    This is a helper function that combines `get_omero_channel_list` with
+    `get_channel_from_list`.
+
+    Args:
+        image_zarr_path: Path to an OME-NGFF image zarr group.
+        label: `label` attribute of the channel to be extracted.
+        wavelength_id: `wavelength_id` attribute of the channel to be
+            extracted.
+
+    Returns:
+        A single channel dictionary.
+    """
+    omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path)
+    channel = get_channel_from_list(
+        channels=omero_channels, label=label, wavelength_id=wavelength_id
+    )
+    return channel
+
+
+
+ +
+ + +
+ + + + +

+ get_channel_from_list(*, channels, label=None, wavelength_id=None) + +

+ + +
+ +

Find matching channel in a list.

+

Find the channel that has the required values of label and/or +wavelength_id, and identify its positional index (which also +corresponds to its index in the zarr array).

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
channels +
+

A list of channel dictionary, where each channel includes (at +least) the label and wavelength_id keys.

+
+

+ + TYPE: + list[OmeroChannel] + +

+
label +
+

The label to look for in the list of channels.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
wavelength_id +
+

The wavelength_id to look for in the list of channels.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + OmeroChannel + + +
+

A single channel dictionary.

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
def get_channel_from_list(
+    *,
+    channels: list[OmeroChannel],
+    label: Optional[str] = None,
+    wavelength_id: Optional[str] = None,
+) -> OmeroChannel:
+    """
+    Find matching channel in a list.
+
+    Find the channel that has the required values of `label` and/or
+    `wavelength_id`, and identify its positional index (which also
+    corresponds to its index in the zarr array).
+
+    Args:
+        channels: A list of channel dictionary, where each channel includes (at
+            least) the `label` and `wavelength_id` keys.
+        label: The label to look for in the list of channels.
+        wavelength_id: The wavelength_id to look for in the list of channels.
+
+    Returns:
+        A single channel dictionary.
+    """
+
+    # Identify matching channels
+    if label:
+        if wavelength_id:
+            # Both label and wavelength_id are specified
+            matching_channels = [
+                c
+                for c in channels
+                if (c.label == label and c.wavelength_id == wavelength_id)
+            ]
+        else:
+            # Only label is specified
+            matching_channels = [c for c in channels if c.label == label]
+    else:
+        if wavelength_id:
+            # Only wavelength_id is specified
+            matching_channels = [
+                c for c in channels if c.wavelength_id == wavelength_id
+            ]
+        else:
+            # Neither label or wavelength_id are specified
+            raise ValueError(
+                "get_channel requires at least one in {label,wavelength_id} "
+                "arguments"
+            )
+
+    # Verify that there is one and only one matching channel
+    if len(matching_channels) == 0:
+        required_match = [f"{label=}", f"{wavelength_id=}"]
+        required_match_string = " and ".join(
+            [x for x in required_match if "None" not in x]
+        )
+        raise ChannelNotFoundError(
+            f"ChannelNotFoundError: No channel found in {channels}"
+            f" for {required_match_string}"
+        )
+    if len(matching_channels) > 1:
+        raise ValueError(f"Inconsistent set of channels: {channels}")
+
+    channel = matching_channels[0]
+    channel.index = channels.index(channel)
+    return channel
+
+
+
+ +
+ + +
+ + + + +

+ get_omero_channel_list(*, image_zarr_path) + +

+ + +
+ +

Extract the list of channels from OME-NGFF zarr attributes.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_zarr_path +
+

Path to an OME-NGFF image zarr group.

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + list[OmeroChannel] + + +
+

A list of channel dictionaries.

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
def get_omero_channel_list(*, image_zarr_path: str) -> list[OmeroChannel]:
+    """
+    Extract the list of channels from OME-NGFF zarr attributes.
+
+    Args:
+        image_zarr_path: Path to an OME-NGFF image zarr group.
+
+    Returns:
+        A list of channel dictionaries.
+    """
+    group = zarr.open_group(image_zarr_path, mode="r+")
+    channels_dicts = group.attrs["omero"]["channels"]
+    channels = [OmeroChannel(**c) for c in channels_dicts]
+    return channels
+
+
+
+ +
+ + +
+ + + + +

+ update_omero_channels(old_channels) + +

+ + +
+ +

Make an existing list of Omero channels Fractal-compatible

+

The output channels all have keys label, wavelength_id and color; +the wavelength_id values are unique across the channel list.

+

See https://ngff.openmicroscopy.org/0.4/index.html#omero-md for the +definition of NGFF Omero metadata.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
old_channels +
+

Existing list of Omero-channel dictionaries

+
+

+ + TYPE: + list[dict[str, Any]] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + list[dict[str, Any]] + + +
+

New list of Fractal-compatible Omero-channel dictionaries

+
+
+ +
+ Source code in fractal_tasks_core/lib_channels.py + 
371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
def update_omero_channels(
+    old_channels: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """
+    Make an existing list of Omero channels Fractal-compatible
+
+    The output channels all have keys `label`, `wavelength_id` and `color`;
+    the `wavelength_id` values are unique across the channel list.
+
+    See https://ngff.openmicroscopy.org/0.4/index.html#omero-md for the
+    definition of NGFF Omero metadata.
+
+    Args:
+        old_channels: Existing list of Omero-channel dictionaries
+
+    Returns:
+        New list of Fractal-compatible Omero-channel dictionaries
+    """
+    new_channels = deepcopy(old_channels)
+    existing_wavelength_ids: list[str] = []
+    handled_channels = []
+
+    default_colors = ["00FFFF", "FF00FF", "FFFF00"]
+
+    def _get_next_color() -> str:
+        try:
+            return default_colors.pop(0)
+        except IndexError:
+            return "808080"
+
+    # Channels that contain the key "wavelength_id"
+    for ind, old_channel in enumerate(old_channels):
+        if "wavelength_id" in old_channel.keys():
+            handled_channels.append(ind)
+            existing_wavelength_ids.append(old_channel["wavelength_id"])
+            new_channel = old_channel.copy()
+            try:
+                label = old_channel["label"]
+            except KeyError:
+                label = str(ind + 1)
+            new_channel["label"] = label
+            if "color" not in old_channel:
+                new_channel["color"] = _get_next_color()
+            new_channels[ind] = new_channel
+
+    # Channels that contain the key "label" but do not contain the key
+    # "wavelength_id"
+    for ind, old_channel in enumerate(old_channels):
+        if ind in handled_channels:
+            continue
+        if "label" not in old_channel.keys():
+            continue
+        handled_channels.append(ind)
+        label = old_channel["label"]
+        wavelength_id = _get_new_unique_value(
+            label,
+            existing_wavelength_ids,
+        )
+        existing_wavelength_ids.append(wavelength_id)
+        new_channel = old_channel.copy()
+        new_channel["wavelength_id"] = wavelength_id
+        if "color" not in old_channel:
+            new_channel["color"] = _get_next_color()
+        new_channels[ind] = new_channel
+
+    # Channels that do not contain the key "label" nor the key "wavelength_id"
+    # NOTE: these channels must be treated last, as they have lower priority
+    # w.r.t. existing "wavelength_id" or "label" values
+    for ind, old_channel in enumerate(old_channels):
+        if ind in handled_channels:
+            continue
+        label = str(ind + 1)
+        wavelength_id = _get_new_unique_value(
+            label,
+            existing_wavelength_ids,
+        )
+        existing_wavelength_ids.append(wavelength_id)
+        new_channel = old_channel.copy()
+        new_channel["label"] = label
+        new_channel["wavelength_id"] = wavelength_id
+        if "color" not in old_channel:
+            new_channel["color"] = _get_next_color()
+        new_channels[ind] = new_channel
+
+    # Log old/new values of label, wavelength_id and color
+    for ind, old_channel in enumerate(old_channels):
+        label = old_channel.get("label")
+        color = old_channel.get("color")
+        wavelength_id = old_channel.get("wavelength_id")
+        old_attributes = (
+            f"Old attributes: {label=}, {wavelength_id=}, {color=}"
+        )
+        label = new_channels[ind]["label"]
+        wavelength_id = new_channels[ind]["wavelength_id"]
+        color = new_channels[ind]["color"]
+        new_attributes = (
+            f"New attributes: {label=}, {wavelength_id=}, {color=}"
+        )
+        logging.info(
+            "Omero channel update:\n"
+            f"    {old_attributes}\n"
+            f"    {new_attributes}"
+        )
+
+    return new_channels
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_glob/index.html b/reference/fractal_tasks_core/lib_glob/index.html new file mode 100644 index 000000000..8d3a1425f --- /dev/null +++ b/reference/fractal_tasks_core/lib_glob/index.html @@ -0,0 +1,1302 @@ + + + + + + + + + + + + + + + + + + + + + + lib_glob - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_glob

+ +
+ + + + +
+ +

Auxiliary functions related to globbing (i.e. listing some items from a +directory).

+ + + +
+ + + + + + + + + + +
+ + + + +

+ glob_with_multiple_patterns(*, folder, patterns=None) + +

+ + +
+ +

List all the items (files and folders) in a given folder that +simultaneously match a series of glob patterns.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
folder +
+

Base folder where items will be searched.

+
+

+ + TYPE: + str + +

+
patterns +
+

If specified, the list of patterns (defined as in +https://docs.python.org/3/library/fnmatch.html) that item +names will match with.

+
+

+ + TYPE: + Sequence[str] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_glob.py + 
20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
def glob_with_multiple_patterns(
+    *,
+    folder: str,
+    patterns: Sequence[str] = None,
+) -> set[str]:
+    """
+    List all the items (files and folders) in a given folder that
+    simultaneously match a series of glob patterns.
+
+    Args:
+        folder: Base folder where items will be searched.
+        patterns: If specified, the list of patterns (defined as in
+            https://docs.python.org/3/library/fnmatch.html) that item
+            names will match with.
+    """
+
+    # Sanitize base-folder path
+    if folder.endswith("/"):
+        actual_folder = folder[:-1]
+    else:
+        actual_folder = folder[:]
+
+    # If not pattern is specified, look for *all* items in the base folder
+    if not patterns:
+        patterns = ["*"]
+
+    # Combine multiple glob searches (via set intersection)
+    logging.info(f"[glob_with_multiple_patterns] {patterns=}")
+    items = None
+    for pattern in patterns:
+        new_matches = glob(f"{actual_folder}/{pattern}")
+        if items is None:
+            items = set(new_matches)
+        else:
+            items = items.intersection(new_matches)
+    items = items or set()
+    logging.info(f"[glob_with_multiple_patterns] Found {len(items)} items")
+
+    return items
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_input_models/index.html b/reference/fractal_tasks_core/lib_input_models/index.html new file mode 100644 index 000000000..bd97b121c --- /dev/null +++ b/reference/fractal_tasks_core/lib_input_models/index.html @@ -0,0 +1,2001 @@ + + + + + + + + + + + + + + + + + + + + + + lib_input_models - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_input_models

+ +
+ + + + +
+ +

Pydantic models for some task parameters.

+ + + +
+ + + + + + + + +
+ + + + +

+ Channel + + +

+ + +
+

+ Bases: BaseModel

+ + +

A channel which is specified by either wavelength_id or label.

+ + + + + + + + + + + + + + + + + + + + +
ATTRIBUTEDESCRIPTION
wavelength_id +
+

Unique ID for the channel wavelength, e.g. A01_C01.

+
+

+ + TYPE: + Optional[str] + +

+
label +
+

Name of the channel.

+
+

+ + TYPE: + Optional[str] + +

+
+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
class Channel(BaseModel):
+    """
+    A channel which is specified by either `wavelength_id` or `label`.
+
+    Attributes:
+        wavelength_id: Unique ID for the channel wavelength, e.g. `A01_C01`.
+        label: Name of the channel.
+    """
+
+    wavelength_id: Optional[str] = None
+    label: Optional[str] = None
+
+    @validator("label", always=True)
+    def mutually_exclusive_channel_attributes(cls, v, values):
+        """
+        Check that either `label` or `wavelength_id` is set.
+        """
+        wavelength_id = values.get("wavelength_id")
+        label = v
+        if wavelength_id and v:
+            raise ValueError(
+                "`wavelength_id` and `label` cannot be both set "
+                f"(given {wavelength_id=} and {label=})."
+            )
+        if wavelength_id is None and v is None:
+            raise ValueError(
+                "`wavelength_id` and `label` cannot be both `None`"
+            )
+        return v
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ mutually_exclusive_channel_attributes(v, values) + +

+ + +
+ +

Check that either label or wavelength_id is set.

+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
@validator("label", always=True)
+def mutually_exclusive_channel_attributes(cls, v, values):
+    """
+    Check that either `label` or `wavelength_id` is set.
+    """
+    wavelength_id = values.get("wavelength_id")
+    label = v
+    if wavelength_id and v:
+        raise ValueError(
+            "`wavelength_id` and `label` cannot be both set "
+            f"(given {wavelength_id=} and {label=})."
+        )
+    if wavelength_id is None and v is None:
+        raise ValueError(
+            "`wavelength_id` and `label` cannot be both `None`"
+        )
+    return v
+
+
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + +

+ NapariWorkflowsInput + + +

+ + +
+

+ Bases: BaseModel

+ + +

A value of the input_specs argument in napari_workflows_wrapper.

+ + + + + + + + + + + + + + + + + + + + + + + + +
ATTRIBUTEDESCRIPTION
type +
+

Input type (either image or label).

+
+

+ + TYPE: + Literal['image', 'label'] + +

+
label_name +
+

Label name (for label inputs only).

+
+

+ + TYPE: + Optional[str] + +

+
channel +
+

Channel object (for image inputs only).

+
+

+ + TYPE: + Optional[Channel] + +

+
+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
class NapariWorkflowsInput(BaseModel):
+    """
+    A value of the `input_specs` argument in `napari_workflows_wrapper`.
+
+    Attributes:
+        type: Input type (either `image` or `label`).
+        label_name: Label name (for label inputs only).
+        channel: Channel object (for image inputs only).
+    """
+
+    type: Literal["image", "label"]
+    label_name: Optional[str]
+    channel: Optional[Channel]
+
+    @validator("label_name", always=True)
+    def label_name_is_present(cls, v, values):
+        """
+        Check that label inputs have `label_name` set.
+        """
+        _type = values.get("type")
+        if _type == "label" and not v:
+            raise ValueError(
+                f"Input item has type={_type} but label_name={v}."
+            )
+        return v
+
+    @validator("channel", always=True)
+    def channel_is_present(cls, v, values):
+        """
+        Check that image inputs have `channel` set.
+        """
+        _type = values.get("type")
+        if _type == "image" and not v:
+            raise ValueError(f"Input item has type={_type} but channel={v}.")
+        return v
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ channel_is_present(v, values) + +

+ + +
+ +

Check that image inputs have channel set.

+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
78
+79
+80
+81
+82
+83
+84
+85
+86
@validator("channel", always=True)
+def channel_is_present(cls, v, values):
+    """
+    Check that image inputs have `channel` set.
+    """
+    _type = values.get("type")
+    if _type == "image" and not v:
+        raise ValueError(f"Input item has type={_type} but channel={v}.")
+    return v
+
+
+
+ +
+ + +
+ + + + +

+ label_name_is_present(v, values) + +

+ + +
+ +

Check that label inputs have label_name set.

+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
@validator("label_name", always=True)
+def label_name_is_present(cls, v, values):
+    """
+    Check that label inputs have `label_name` set.
+    """
+    _type = values.get("type")
+    if _type == "label" and not v:
+        raise ValueError(
+            f"Input item has type={_type} but label_name={v}."
+        )
+    return v
+
+
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + +

+ NapariWorkflowsOutput + + +

+ + +
+

+ Bases: BaseModel

+ + +

A value of the output_specs argument in napari_workflows_wrapper.

+ + + + + + + + + + + + + + + + + + + + + + + + +
ATTRIBUTEDESCRIPTION
type +
+

Output type (either label or dataframe).

+
+

+ + TYPE: + Literal['label', 'dataframe'] + +

+
label_name +
+

Label name (for label outputs only).

+
+

+ + TYPE: + Optional[str] + +

+
table_name +
+

Table name (for dataframe outputs only).

+
+

+ + TYPE: + Optional[str] + +

+
+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
class NapariWorkflowsOutput(BaseModel):
+    """
+    A value of the `output_specs` argument in `napari_workflows_wrapper`.
+
+    Attributes:
+        type: Output type (either `label` or `dataframe`).
+        label_name: Label name (for label outputs only).
+        table_name: Table name (for dataframe outputs only).
+    """
+
+    type: Literal["label", "dataframe"]
+    label_name: Optional[str] = None
+    table_name: Optional[str] = None
+
+    @validator("label_name", always=True)
+    def label_name_only_for_label_type(cls, v, values):
+        """
+        Check that label_name is set only for label outputs.
+        """
+        _type = values.get("type")
+        if (_type == "label" and (not v)) or (_type != "label" and v):
+            raise ValueError(
+                f"Output item has type={_type} but label_name={v}."
+            )
+        return v
+
+    @validator("table_name", always=True)
+    def table_name_only_for_dataframe_type(cls, v, values):
+        """
+        Check that table_name is set only for dataframe outputs.
+        """
+        _type = values.get("type")
+        if (_type == "dataframe" and (not v)) or (_type != "dataframe" and v):
+            raise ValueError(
+                f"Output item has type={_type} but table_name={v}."
+            )
+        return v
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ label_name_only_for_label_type(v, values) + +

+ + +
+ +

Check that label_name is set only for label outputs.

+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
@validator("label_name", always=True)
+def label_name_only_for_label_type(cls, v, values):
+    """
+    Check that label_name is set only for label outputs.
+    """
+    _type = values.get("type")
+    if (_type == "label" and (not v)) or (_type != "label" and v):
+        raise ValueError(
+            f"Output item has type={_type} but label_name={v}."
+        )
+    return v
+
+
+
+ +
+ + +
+ + + + +

+ table_name_only_for_dataframe_type(v, values) + +

+ + +
+ +

Check that table_name is set only for dataframe outputs.

+ +
+ Source code in fractal_tasks_core/lib_input_models.py + 
115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
@validator("table_name", always=True)
+def table_name_only_for_dataframe_type(cls, v, values):
+    """
+    Check that table_name is set only for dataframe outputs.
+    """
+    _type = values.get("type")
+    if (_type == "dataframe" and (not v)) or (_type != "dataframe" and v):
+        raise ValueError(
+            f"Output item has type={_type} but table_name={v}."
+        )
+    return v
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_masked_loading/index.html b/reference/fractal_tasks_core/lib_masked_loading/index.html new file mode 100644 index 000000000..6af6b87e5 --- /dev/null +++ b/reference/fractal_tasks_core/lib_masked_loading/index.html @@ -0,0 +1,1975 @@ + + + + + + + + + + + + + + + + + + + + + + lib_masked_loading - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_masked_loading

+ +
+ + + + +
+ +

Functions to use masked loading of ROIs before/after processing.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _postprocess_output(*, modified_array, original_array, background) + +

+ + +
+ +

Postprocess cellpose output, mainly to restore its original background.

+

NOTE: The pre/post-processing functions and the +masked_loading_wrapper are currently meant to work as part of the +cellpose_segmentation task, with the plan of then making them more +flexible; see +https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
modified_array +
+

The 3D (ZYX) array with the correct object data and +wrong background data.

+
+

+ + TYPE: + ndarray + +

+
original_array +
+

The 3D (ZYX) array with the wrong object data and +correct background data.

+
+

+ + TYPE: + ndarray + +

+
background +
+

The 3D (ZYX) boolean array that defines the background.

+
+

+ + TYPE: + ndarray + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + ndarray + + +
+

The postprocessed array.

+
+
+ +
+ Source code in fractal_tasks_core/lib_masked_loading.py + 
178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
def _postprocess_output(
+    *,
+    modified_array: np.ndarray,
+    original_array: np.ndarray,
+    background: np.ndarray,
+) -> np.ndarray:
+    """
+    Postprocess cellpose output, mainly to restore its original background.
+
+    **NOTE**: The pre/post-processing functions and the
+    masked_loading_wrapper are currently meant to work as part of the
+    cellpose_segmentation task, with the plan of then making them more
+    flexible; see
+    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.
+
+    Args:
+        modified_array: The 3D (ZYX) array with the correct object data and
+            wrong background data.
+        original_array: The 3D (ZYX) array with the wrong object data and
+            correct background data.
+        background: The 3D (ZYX) boolean array that defines the background.
+
+    Returns:
+        The postprocessed array.
+    """
+    # Restore background
+    modified_array[background] = original_array[background]
+    return modified_array
+
+
+
+ +
+ + +
+ + + + +

+ _preprocess_input(image_array, *, region, current_label_path, ROI_table_path, ROI_positional_index) + +

+ + +
+ +

Preprocess a four-dimensional cellpose input.

+

This involves :

+
    +
  • Loading the masking label array for the appropriate ROI;
    • +
  • Extracting the appropriate label value from the ROI_table.obs + dataframe;
    • +
  • Constructing the background mask, where the masking label matches with a + specific label value;
    • +
  • Setting the background of image_array to 0;
    • +
  • Loading the array which will be needed in postprocessing to restore + background.
    • +
+

NOTE 1: This function relies on a change to OME-NGFF table specs +(https://github.com/ome/ngff/pull/64) which is still in-progress.

+

NOTE 2: The pre/post-processing functions and the +masked_loading_wrapper are currently meant to work as part of the +cellpose_segmentation task, with the plan of then making them more +flexible; see +https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.

+

Naming of variables refers to a two-steps labeling, as in "first identify +organoids, then look for nuclei inside each organoid") :

+
    +
  • "masking" refers to the labels that are used to identify the object + vs background (e.g. the organoid labels); these labels already exist.
    • +
  • "current" refers to the labels that are currently being computed in + the cellpose_segmentation task, e.g. the nuclear labels.
    • +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_array +
+

The 4D CZYX array with image data for a specific ROI.

+
+

+ + TYPE: + ndarray + +

+
region +
+

The ZYX indices of the ROI, in a form like +(slice(0, 1), slice(1000, 2000), slice(1000, 2000)).

+
+

+ + TYPE: + tuple[slice, ...] + +

+
current_label_path +
+

Path to the image used as current label, in a form +like /somewhere/plate.zarr/A/01/0/labels/nuclei_in_organoids/0.

+
+

+ + TYPE: + str + +

+
ROI_table_path +
+

Path of the AnnData table for the masking-label ROIs; +this is used (together with ROI_positional_index) to extract +label_value.

+
+

+ + TYPE: + str + +

+
ROI_positional_index +
+

Index of the current ROI, which is used to +extract label_value from ROI_table_obs.

+
+

+ + TYPE: + int + +

+
+

Returns: + A tuple with three arrays: the preprocessed image array, the background + mask, the current label.

+ +
+ Source code in fractal_tasks_core/lib_masked_loading.py + 
 30
+ 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
def _preprocess_input(
+    image_array: np.ndarray,
+    *,
+    region: tuple[slice, ...],
+    current_label_path: str,
+    ROI_table_path: str,
+    ROI_positional_index: int,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """
+    Preprocess a four-dimensional cellpose input.
+
+    This involves :
+
+    - Loading the masking label array for the appropriate ROI;
+    - Extracting the appropriate label value from the `ROI_table.obs`
+      dataframe;
+    - Constructing the background mask, where the masking label matches with a
+      specific label value;
+    - Setting the background of `image_array` to `0`;
+    - Loading the array which will be needed in postprocessing to restore
+      background.
+
+    **NOTE 1**: This function relies on a change to OME-NGFF table specs
+    (https://github.com/ome/ngff/pull/64) which is still in-progress.
+
+    **NOTE 2**: The pre/post-processing functions and the
+    masked_loading_wrapper are currently meant to work as part of the
+    cellpose_segmentation task, with the plan of then making them more
+    flexible; see
+    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.
+
+    Naming of variables refers to a two-steps labeling, as in "first identify
+    organoids, then look for nuclei inside each organoid") :
+
+    - `"masking"` refers to the labels that are used to identify the object
+      vs background (e.g. the organoid labels); these labels already exist.
+    - `"current"` refers to the labels that are currently being computed in
+      the `cellpose_segmentation` task, e.g. the nuclear labels.
+
+    Args:
+        image_array: The 4D CZYX array with image data for a specific ROI.
+        region: The ZYX indices of the ROI, in a form like
+            `(slice(0, 1), slice(1000, 2000), slice(1000, 2000))`.
+        current_label_path: Path to the image used as current label, in a form
+            like `/somewhere/plate.zarr/A/01/0/labels/nuclei_in_organoids/0`.
+        ROI_table_path: Path of the AnnData table for the masking-label ROIs;
+            this is used (together with `ROI_positional_index`) to extract
+            `label_value`.
+        ROI_positional_index: Index of the current ROI, which is used to
+            extract `label_value` from `ROI_table_obs`.
+    Returns:
+        A tuple with three arrays: the preprocessed image array, the background
+            mask, the current label.
+    """
+
+    logger.info(f"[_preprocess_input] {image_array.shape=}")
+    logger.info(f"[_preprocess_input] {region=}")
+
+    # Check that image data are 4D (CZYX) - FIXME issue 340
+    if not image_array.ndim == 4:
+        raise ValueError(
+            "_preprocess_input requires a 4D "
+            f"image_array argument, but {image_array.shape=}"
+        )
+
+    # Load the ROI table and its metadata attributes
+    ROI_table = ad.read_zarr(ROI_table_path)
+    attrs = zarr.group(ROI_table_path).attrs
+    logger.info(f"[_preprocess_input] {ROI_table_path=}")
+    logger.info(f"[_preprocess_input] {attrs.asdict()=}")
+    if not attrs["type"] == "ngff:region_table":
+        raise ValueError("Wrong attributes for {ROI_table_path}:\n{attrs}")
+    label_relative_path = attrs["region"]["path"]
+    column_name = attrs["instance_key"]
+
+    # Check that ROI_table.obs has the right column and extract label_value
+    if column_name not in ROI_table.obs.columns:
+        raise ValueError(
+            'In _preprocess_input, "{column_name}" '
+            f" missing in {ROI_table.obs.columns=}"
+        )
+    label_value = int(ROI_table.obs[column_name][ROI_positional_index])
+
+    # Load masking-label array (lazily)
+    masking_label_path = str(
+        Path(ROI_table_path).parent / label_relative_path / "0"
+    )
+    logger.info(f"{masking_label_path=}")
+    masking_label_array = da.from_zarr(masking_label_path)
+    logger.info(
+        f"[_preprocess_input] {masking_label_path=}, "
+        f"{masking_label_array.shape=}"
+    )
+
+    # Load current-label array (lazily)
+    current_label_array = da.from_zarr(current_label_path)
+    logger.info(
+        f"[_preprocess_input] {current_label_path=}, "
+        f"{current_label_array.shape=}"
+    )
+
+    # Load ROI data for current label array
+    current_label_region = current_label_array[region].compute()
+
+    # Load ROI data for masking label array, with or without upscaling
+    if masking_label_array.shape != current_label_array.shape:
+        logger.info("Upscaling of masking label is needed")
+        lowres_region = convert_region_to_low_res(
+            highres_region=region,
+            highres_shape=current_label_array.shape,
+            lowres_shape=masking_label_array.shape,
+        )
+        masking_label_region = masking_label_array[lowres_region].compute()
+        masking_label_region = upscale_array(
+            array=masking_label_region,
+            target_shape=current_label_region.shape,
+        )
+    else:
+        masking_label_region = masking_label_array[region].compute()
+
+    # Check that all shapes match
+    shapes = (
+        masking_label_region.shape,
+        current_label_region.shape,
+        image_array.shape[1:],
+    )
+    if len(set(shapes)) > 1:
+        raise ValueError(
+            "Shape mismatch:\n"
+            f"{current_label_region.shape=}\n"
+            f"{masking_label_region.shape=}\n"
+            f"{image_array.shape=}"
+        )
+
+    # Compute background mask
+    background_3D = masking_label_region != label_value
+    if (masking_label_region == label_value).sum() == 0:
+        raise ValueError(
+            f"Label {label_value} is not present in the extracted ROI"
+        )
+
+    # Set image background to zero
+    background_4D = np.expand_dims(background_3D, axis=0)
+    image_array[background_4D] = 0
+
+    return (image_array, background_3D, current_label_region)
+
+
+
+ +
+ + +
+ + + + +

+ masked_loading_wrapper(*, function, image_array, kwargs=None, use_masks, preprocessing_kwargs=None) + +

+ + +
+ +

Wrap a function with some pre/post-processing functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
function +
+

The callable function to be wrapped.

+
+

+ + TYPE: + Callable + +

+
image_array +
+

The image array to be preprocessed and then used as +positional argument for function.

+
+

+ + TYPE: + ndarray + +

+
kwargs +
+

Keyword arguments for function.

+
+

+ + TYPE: + Optional[dict] + + + DEFAULT: + None + +

+
use_masks +
+

If False, the wrapper only calls +function(*args, **kwargs).

+
+

+ + TYPE: + bool + +

+
preprocessing_kwargs +
+

Keyword arguments for the preprocessing function +(see call signature of _preprocess_input()).

+
+

+ + TYPE: + Optional[dict] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_masked_loading.py + 
208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
def masked_loading_wrapper(
+    *,
+    function: Callable,
+    image_array: np.ndarray,
+    kwargs: Optional[dict] = None,
+    use_masks: bool,
+    preprocessing_kwargs: Optional[dict] = None,
+):
+    """
+    Wrap a function with some pre/post-processing functions
+
+    Args:
+        function: The callable function to be wrapped.
+        image_array: The image array to be preprocessed and then used as
+            positional argument for `function`.
+        kwargs: Keyword arguments for `function`.
+        use_masks: If `False`, the wrapper only calls
+            `function(*args, **kwargs)`.
+        preprocessing_kwargs: Keyword arguments for the preprocessing function
+            (see call signature of `_preprocess_input()`).
+    """
+    # Optional preprocessing
+    if use_masks:
+        preprocessing_kwargs = preprocessing_kwargs or {}
+        (
+            image_array,
+            background_3D,
+            current_label_region,
+        ) = _preprocess_input(image_array, **preprocessing_kwargs)
+    # Run function
+    kwargs = kwargs or {}
+    new_label_img = function(image_array, **kwargs)
+    # Optional postprocessing
+    if use_masks:
+        new_label_img = _postprocess_output(
+            modified_array=new_label_img,
+            original_array=current_label_region,
+            background=background_3D,
+        )
+    return new_label_img
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_metadata_parsing/index.html b/reference/fractal_tasks_core/lib_metadata_parsing/index.html new file mode 100644 index 000000000..4c9bfc52f --- /dev/null +++ b/reference/fractal_tasks_core/lib_metadata_parsing/index.html @@ -0,0 +1,2422 @@ + + + + + + + + + + + + + + + + + + + + + + lib_metadata_parsing - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_metadata_parsing

+ +
+ + + + +
+ +

Functions to create a metadata dataframe from Yokogawa files.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ calculate_steps(site_series) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
site_series +
+

TBD

+
+

+ + TYPE: + Series + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
def calculate_steps(site_series: pd.Series):
+    """
+    TBD
+
+    Args:
+        site_series: TBD
+    """
+
+    # site_series is the z_micrometer series for a given site of a given
+    # channel. This function calculates the step size in Z
+
+    # First diff is always NaN because there is nothing to compare it to
+    steps = site_series.diff().dropna().astype(float)
+    if not np.allclose(steps.iloc[0], np.array(steps)):
+        raise NotImplementedError(
+            "When parsing the Yokogawa mlf file, some sites "
+            "had varying step size in Z. "
+            "That is not supported for the OME-Zarr parsing"
+        )
+    return steps.mean()
+
+
+
+ +
+ + +
+ + + + +

+ check_group_consistency(grouped_df, message='') + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
grouped_df +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
message +
+

TBD

+
+

+ + TYPE: + str + + + DEFAULT: + '' + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
def check_group_consistency(grouped_df: pd.DataFrame, message: str = ""):
+    """
+    TBD
+
+    Args:
+        grouped_df: TBD
+        message: TBD
+    """
+
+    # Check consistency in grouped df for multi-index, multi-column dataframes
+    # raises an exception if there is variability
+    diff_df = grouped_df.max() - grouped_df.min()
+    if not np.isclose(np.sum(np.sum(diff_df)), 0.0):
+        raise ValueError(
+            "During metadata parsing, a consistency check failed: \n"
+            f"{message}\n"
+            f"Difference dataframe: \n{diff_df}"
+        )
+
+
+
+ +
+ + +
+ + + + +

+ get_earliest_time_per_site(mlf_frame) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mlf_frame +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
def get_earliest_time_per_site(mlf_frame: pd.DataFrame) -> pd.DataFrame:
+    """
+    TBD
+
+    Args:
+        mlf_frame: TBD
+    """
+
+    # Get the time information per site
+    # Because a site will contain time information for each plane
+    # of each channel, we just return the earliest time infromation
+    # per site.
+    return pd.to_datetime(
+        mlf_frame.groupby(["well_id", "FieldIndex"]).min()["Time"], utc=True
+    )
+
+
+
+ +
+ + +
+ + + + +

+ get_z_steps(mlf_frame) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mlf_frame +
+

TBD

+
+

+ + TYPE: + DataFrame + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
def get_z_steps(mlf_frame: pd.DataFrame) -> pd.DataFrame:
+    """
+    TBD
+
+    Args:
+        mlf_frame: TBD
+    """
+
+    # Process mlf_frame to extract Z information (pixel size & steps).
+    # Run checks on consistencies & return site-based z step dataframe
+    # Group by well, field & channel
+    grouped_sites_z = (
+        mlf_frame.loc[
+            :,
+            ["well_id", "FieldIndex", "ActionIndex", "Ch", "Z"],
+        ]
+        .set_index(["well_id", "FieldIndex", "ActionIndex", "Ch"])
+        .groupby(level=[0, 1, 2, 3])
+    )
+
+    # If there is only 1 Z step, set the Z spacing to the count of planes => 1
+    if grouped_sites_z.count()["Z"].max() == 1:
+        z_data = grouped_sites_z.count().groupby(["well_id", "FieldIndex"])
+    else:
+        # Group the whole site (combine channels), because Z steps need to be
+        # consistent between channels for OME-Zarr.
+        z_data = grouped_sites_z.apply(calculate_steps).groupby(
+            ["well_id", "FieldIndex"]
+        )
+
+    check_group_consistency(
+        z_data, message="Comparing Z steps between channels"
+    )
+
+    # Ensure that channels have the same number of z planes and
+    # reduce it to one value.
+    # Only check if there is more than one channel available
+    if any(
+        grouped_sites_z.count().groupby(["well_id", "FieldIndex"]).count() > 1
+    ):
+        check_group_consistency(
+            grouped_sites_z.count().groupby(["well_id", "FieldIndex"]),
+            message="Checking number of Z steps between channels",
+        )
+
+    z_steps = (
+        grouped_sites_z.count()
+        .groupby(["well_id", "FieldIndex"])
+        .mean()
+        .astype(int)
+    )
+
+    # Combine the two dataframes
+    z_frame = pd.concat([z_data.mean(), z_steps], axis=1)
+    z_frame.columns = ["pixel_size_z", "z_pixel"]
+    return z_frame
+
+
+
+ +
+ + +
+ + + + +

+ parse_yokogawa_metadata(mrf_path, mlf_path, *, filename_patterns=None) + +

+ + +
+ +

Parse Yokogawa CV7000 metadata files and prepare site-level metadata.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mrf_path +
+

Full path to MeasurementDetail.mrf metadata file.

+
+

+ + TYPE: + Union[str, Path] + +

+
mlf_path +
+

Full path to MeasurementData.mlf metadata file.

+
+

+ + TYPE: + Union[str, Path] + +

+
filename_patterns +
+

List of patterns to filter the image filenames in the mlf metadata +table. Patterns must be defined as in +https://docs.python.org/3/library/fnmatch.html

+
+

+ + TYPE: + Optional[list[str]] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
 28
+ 29
+ 30
+ 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
def parse_yokogawa_metadata(
+    mrf_path: Union[str, Path],
+    mlf_path: Union[str, Path],
+    *,
+    filename_patterns: Optional[list[str]] = None,
+) -> tuple[pd.DataFrame, dict[str, int]]:
+    """
+    Parse Yokogawa CV7000 metadata files and prepare site-level metadata.
+
+    Args:
+        mrf_path: Full path to MeasurementDetail.mrf metadata file.
+        mlf_path: Full path to MeasurementData.mlf metadata file.
+        filename_patterns:
+            List of patterns to filter the image filenames in the mlf metadata
+            table. Patterns must be defined as in
+            https://docs.python.org/3/library/fnmatch.html
+    """
+
+    # Convert paths to strings
+    mrf_str = Path(mrf_path).as_posix()
+    mlf_str = Path(mlf_path).as_posix()
+
+    mrf_frame, mlf_frame, error_count = read_metadata_files(
+        mrf_str, mlf_str, filename_patterns
+    )
+
+    # Aggregate information from the mlf file
+    per_site_parameters = ["X", "Y"]
+
+    grouping_params = ["well_id", "FieldIndex"]
+    grouped_sites = mlf_frame.loc[
+        :, grouping_params + per_site_parameters
+    ].groupby(by=grouping_params)
+
+    check_group_consistency(grouped_sites, message="X & Y stage positions")
+    site_metadata = grouped_sites.mean()
+    site_metadata.columns = ["x_micrometer", "y_micrometer"]
+    site_metadata["z_micrometer"] = 0
+
+    site_metadata = pd.concat(
+        [
+            site_metadata,
+            get_z_steps(mlf_frame),
+            get_earliest_time_per_site(mlf_frame),
+        ],
+        axis=1,
+    )
+
+    # Aggregate information from the mrf file
+    mrf_columns = [
+        "horiz_pixel_dim",
+        "vert_pixel_dim",
+        "horiz_pixels",
+        "vert_pixels",
+        "bit_depth",
+    ]
+    check_group_consistency(
+        mrf_frame.loc[:, mrf_columns], message="Image dimensions"
+    )
+    site_metadata["pixel_size_x"] = mrf_frame.loc[:, "horiz_pixel_dim"].max()
+    site_metadata["pixel_size_y"] = mrf_frame.loc[:, "vert_pixel_dim"].max()
+    site_metadata["x_pixel"] = int(mrf_frame.loc[:, "horiz_pixels"].max())
+    site_metadata["y_pixel"] = int(mrf_frame.loc[:, "vert_pixels"].max())
+    site_metadata["bit_depth"] = int(mrf_frame.loc[:, "bit_depth"].max())
+
+    if error_count > 0:
+        logger.info(
+            f"There were {error_count} ERR entries in the metadatafile. "
+            f"Still succesfully parsed {len(site_metadata)} sites. "
+        )
+
+    # Compute expected number of image files for each well
+    list_of_wells = set(site_metadata.index.get_level_values("well_id"))
+    number_of_files = {}
+    for this_well_id in list_of_wells:
+        num_images = (mlf_frame.well_id == this_well_id).sum()
+        logger.info(
+            f"Expected number of images for well {this_well_id}: {num_images}"
+        )
+        number_of_files[this_well_id] = num_images
+    # Check that the sum of per-well file numbers correspond to the total
+    # file number
+    if not sum(number_of_files.values()) == len(mlf_frame):
+        raise ValueError(
+            "Error while counting the number of image files per well.\n"
+            f"{len(mlf_frame)=}\n"
+            f"{number_of_files=}"
+        )
+
+    return site_metadata, number_of_files
+
+
+
+ +
+ + +
+ + + + +

+ read_metadata_files(mrf_path, mlf_path, filename_patterns=None) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mrf_path +
+

Full path to MeasurementDetail.mrf metadata file.

+
+

+ + TYPE: + str + +

+
mlf_path +
+

Full path to MeasurementData.mlf metadata file.

+
+

+ + TYPE: + str + +

+
filename_patterns +
+

List of patterns to filter the image filenames in +the mlf metadata table. Patterns must be defined as in +https://docs.python.org/3/library/fnmatch.html.

+
+

+ + TYPE: + Optional[list[str]] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
def read_metadata_files(
+    mrf_path: str,
+    mlf_path: str,
+    filename_patterns: Optional[list[str]] = None,
+) -> tuple[pd.DataFrame, pd.DataFrame, int]:
+    """
+    TBD
+
+    Args:
+        mrf_path: Full path to MeasurementDetail.mrf metadata file.
+        mlf_path: Full path to MeasurementData.mlf metadata file.
+        filename_patterns: List of patterns to filter the image filenames in
+            the mlf metadata table. Patterns must be defined as in
+            https://docs.python.org/3/library/fnmatch.html.
+    """
+
+    # parsing of mrf & mlf files are based on the
+    # yokogawa_image_collection_task v0.5 in drogon, written by Dario Vischi.
+    # https://github.com/fmi-basel/job-system-workflows/blob/00bbf34448972d27f258a2c28245dd96180e8229/src/gliberal_workflows/tasks/yokogawa_image_collection_task/versions/version_0_5.py  # noqa
+    # Now modified for Fractal use
+
+    mrf_frame = read_mrf_file(mrf_path)
+    # TODO: filter_position & filter_wheel_position are parsed, but not
+    # processed further. Figure out how to save them as relevant metadata for
+    # use e.g. during illumination correction
+
+    mlf_frame, error_count = read_mlf_file(mlf_path, filename_patterns)
+    # TODO: Time points are parsed as part of the mlf_frame, but currently not
+    # processed further. Once we tackle time-resolved data, parse from here.
+
+    return mrf_frame, mlf_frame, error_count
+
+
+
+ +
+ + +
+ + + + +

+ read_mlf_file(mlf_path, filename_patterns=None) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mlf_path +
+

Full path to MeasurementData.mlf metadata file.

+
+

+ + TYPE: + str + +

+
filename_patterns +
+

List of patterns to filter the image filenames in +the mlf metadata table. Patterns must be defined as in +https://docs.python.org/3/library/fnmatch.html.

+
+

+ + TYPE: + Optional[list[str]] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
def read_mlf_file(
+    mlf_path: str,
+    filename_patterns: Optional[list[str]] = None,
+) -> tuple[pd.DataFrame, int]:
+    """
+    TBD
+
+    Args:
+        mlf_path: Full path to MeasurementData.mlf metadata file.
+        filename_patterns: List of patterns to filter the image filenames in
+            the mlf metadata table. Patterns must be defined as in
+            https://docs.python.org/3/library/fnmatch.html.
+    """
+
+    # Load the whole MeasurementData.mlf file
+    mlf_frame_raw = pd.read_xml(mlf_path)
+
+    # Remove all rows that do not match the given patterns
+    logger.info(
+        f"Read {mlf_path}, and apply following patterns to "
+        f"image filenames: {filename_patterns}"
+    )
+    if filename_patterns:
+        filenames = mlf_frame_raw.MeasurementRecord
+        keep_row = None
+        for pattern in filename_patterns:
+            actual_pattern = fnmatch.translate(pattern)
+            new_matches = filenames.str.fullmatch(actual_pattern)
+            if new_matches.sum() == 0:
+                raise ValueError(
+                    f"In {mlf_path} there is no image filename "
+                    f'matching "{actual_pattern}".'
+                )
+            if keep_row is None:
+                keep_row = new_matches.copy()
+            else:
+                keep_row = keep_row & new_matches
+        if keep_row.sum() == 0:
+            raise ValueError(
+                f"In {mlf_path} there is no image filename "
+                f"matching {filename_patterns}."
+            )
+        mlf_frame_matching = mlf_frame_raw[keep_row.values].copy()
+    else:
+        mlf_frame_matching = mlf_frame_raw.copy()
+
+    # Create a well ID column
+    row_str = [chr(x) for x in (mlf_frame_matching["Row"] + 64)]
+    mlf_frame_matching["well_id"] = [
+        f"{a}{b:02}" for a, b in zip(row_str, mlf_frame_matching["Column"])
+    ]
+
+    # Flip Y axis to align to image coordinate system
+    mlf_frame_matching["Y"] = -mlf_frame_matching["Y"]
+
+    # Compute number or errors
+    error_count = (mlf_frame_matching["Type"] == "ERR").sum()
+
+    # We're only interested in the image metadata
+    mlf_frame = mlf_frame_matching[mlf_frame_matching["Type"] == "IMG"]
+
+    return mlf_frame, error_count
+
+
+
+ +
+ + +
+ + + + +

+ read_mrf_file(mrf_path) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mrf_path +
+

Full path to MeasurementDetail.mrf metadata file.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/lib_metadata_parsing.py + 
153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
def read_mrf_file(mrf_path: str):
+    """
+    TBD
+
+    Args:
+        mrf_path: Full path to MeasurementDetail.mrf metadata file.
+    """
+
+    # Prepare mrf dataframe
+    mrf_columns = [
+        "channel_id",
+        "horiz_pixel_dim",
+        "vert_pixel_dim",
+        "camera_no",
+        "bit_depth",
+        "horiz_pixels",
+        "vert_pixels",
+        "filter_wheel_position",
+        "filter_position",
+        "shading_corr_src",
+    ]
+    mrf_frame = pd.DataFrame(columns=mrf_columns)
+
+    mrf_xml = ElementTree.parse(mrf_path).getroot()
+    # Read mrf file
+    ns = {"bts": "http://www.yokogawa.co.jp/BTS/BTSSchema/1.0"}
+    for channel in mrf_xml.findall("bts:MeasurementChannel", namespaces=ns):
+        mrf_frame.loc[channel.get("{%s}Ch" % ns["bts"])] = [
+            channel.get("{%s}Ch" % ns["bts"]),
+            float(channel.get("{%s}HorizontalPixelDimension" % ns["bts"])),
+            float(channel.get("{%s}VerticalPixelDimension" % ns["bts"])),
+            int(channel.get("{%s}CameraNumber" % ns["bts"])),
+            int(channel.get("{%s}InputBitDepth" % ns["bts"])),
+            int(channel.get("{%s}HorizontalPixels" % ns["bts"])),
+            int(channel.get("{%s}VerticalPixels" % ns["bts"])),
+            int(channel.get("{%s}FilterWheelPosition" % ns["bts"])),
+            int(channel.get("{%s}FilterPosition" % ns["bts"])),
+            channel.get("{%s}ShadingCorrectionSource" % ns["bts"]),
+        ]
+
+    return mrf_frame
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_ngff/index.html b/reference/fractal_tasks_core/lib_ngff/index.html new file mode 100644 index 000000000..f44f16162 --- /dev/null +++ b/reference/fractal_tasks_core/lib_ngff/index.html @@ -0,0 +1,3491 @@ + + + + + + + + + + + + + + + + + + + + + + lib_ngff - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_ngff

+ +
+ + + + +
+ +

Pydantic models related to OME-NGFF 0.4 specs.

+ + + +
+ + + + + + + + +
+ + + + +

+ Axis + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for an element of Multiscale.axes.

+

See https://ngff.openmicroscopy.org/0.4/#axes-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
56
+57
+58
+59
+60
+61
+62
+63
+64
+65
class Axis(BaseModel):
+    """
+    Model for an element of `Multiscale.axes`.
+
+    See https://ngff.openmicroscopy.org/0.4/#axes-md.
+    """
+
+    name: str
+    type: Optional[str] = None
+    unit: Optional[str] = None
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ Channel + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for an element of Omero.channels.

+

See https://ngff.openmicroscopy.org/0.4/#omero-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
class Channel(BaseModel):
+    """
+    Model for an element of `Omero.channels`.
+
+    See https://ngff.openmicroscopy.org/0.4/#omero-md.
+    """
+
+    window: Optional[Window] = None
+    label: Optional[str] = None
+    family: Optional[str] = None
+    color: str
+    active: Optional[bool] = None
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ Dataset + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for an element of Multiscale.datasets.

+

See https://ngff.openmicroscopy.org/0.4/#multiscale-md

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
class Dataset(BaseModel):
+    """
+    Model for an element of `Multiscale.datasets`.
+
+    See https://ngff.openmicroscopy.org/0.4/#multiscale-md
+    """
+
+    path: str
+    coordinateTransformations: list[
+        Union[
+            ScaleCoordinateTransformation, TranslationCoordinateTransformation
+        ]
+    ] = Field(..., min_items=1)
+
+    @property
+    def scale_transformation(self) -> ScaleCoordinateTransformation:
+        """
+        Extract the unique scale transformation, or fail otherwise.
+        """
+        _transformations = [
+            t for t in self.coordinateTransformations if t.type == "scale"
+        ]
+        if len(_transformations) == 0:
+            raise ValueError(
+                "Missing scale transformation in dataset.\n"
+                "Current coordinateTransformations:\n"
+                f"{self.coordinateTransformations}"
+            )
+        elif len(_transformations) > 1:
+            raise ValueError(
+                "More than one scale transformation in dataset.\n"
+                "Current coordinateTransformations:\n"
+                f"{self.coordinateTransformations}"
+            )
+        else:
+            return _transformations[0]
+
+
+ + + +
+ + + + + + + +
+ + + + +

+ scale_transformation: ScaleCoordinateTransformation + + + property + + +

+ + +
+ +

Extract the unique scale transformation, or fail otherwise.

+
+ +
+ + + + + +
+ +
+ +
+ +
+ + + + +

+ ImageInWell + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for an element of Well.images.

+

Note 1: The NGFF image is defined in a different model +(NgffImageMeta), while the Image model only refere to an item of +Well.images.

+

Note 2: We deviate from NGFF specs, since we allow path to be an +arbitrary string. +TODO: include a check like constr(regex=r'^[A-Za-z0-9]+$'), through a +Pydantic validator.

+

See https://ngff.openmicroscopy.org/0.4/#well-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
class ImageInWell(BaseModel):
+    """
+    Model for an element of `Well.images`.
+
+    **Note 1:** The NGFF image is defined in a different model
+    (`NgffImageMeta`), while the `Image` model only refere to an item of
+    `Well.images`.
+
+    **Note 2:** We deviate from NGFF specs, since we allow `path` to be an
+    arbitrary string.
+    TODO: include a check like `constr(regex=r'^[A-Za-z0-9]+$')`, through a
+    Pydantic validator.
+
+    See https://ngff.openmicroscopy.org/0.4/#well-md.
+    """
+
+    acquisition: Optional[int] = Field(
+        None, description="A unique identifier within the context of the plate"
+    )
+    path: str = Field(
+        ..., description="The path for this field of view subgroup"
+    )
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ Multiscale + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for an element of NgffImageMeta.multiscales.

+

See https://ngff.openmicroscopy.org/0.4/#multiscale-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
class Multiscale(BaseModel):
+    """
+    Model for an element of `NgffImageMeta.multiscales`.
+
+    See https://ngff.openmicroscopy.org/0.4/#multiscale-md.
+    """
+
+    name: Optional[str] = None
+    datasets: list[Dataset] = Field(..., min_items=1)
+    version: Optional[str] = None
+    axes: list[Axis] = Field(..., max_items=5, min_items=2, unique_items=True)
+    coordinateTransformations: Optional[
+        list[
+            Union[
+                ScaleCoordinateTransformation,
+                TranslationCoordinateTransformation,
+            ]
+        ]
+    ] = None
+
+    @validator("coordinateTransformations", always=True)
+    def _no_global_coordinateTransformations(cls, v):
+        """
+        Fail if Multiscale has a (global) coordinateTransformations attribute.
+        """
+        if v is not None:
+            raise NotImplementedError(
+                "Global coordinateTransformations at the multiscales "
+                "level are not currently supported in the fractal-tasks-core "
+                "model for the NGFF multiscale."
+            )
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ _no_global_coordinateTransformations(v) + +

+ + +
+ +

Fail if Multiscale has a (global) coordinateTransformations attribute.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
@validator("coordinateTransformations", always=True)
+def _no_global_coordinateTransformations(cls, v):
+    """
+    Fail if Multiscale has a (global) coordinateTransformations attribute.
+    """
+    if v is not None:
+        raise NotImplementedError(
+            "Global coordinateTransformations at the multiscales "
+            "level are not currently supported in the fractal-tasks-core "
+            "model for the NGFF multiscale."
+        )
+
+
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + +

+ NgffImageMeta + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for the metadata of a NGFF image.

+

See https://ngff.openmicroscopy.org/0.4/#image-layout.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
class NgffImageMeta(BaseModel):
+    """
+    Model for the metadata of a NGFF image.
+
+    See https://ngff.openmicroscopy.org/0.4/#image-layout.
+    """
+
+    multiscales: list[Multiscale] = Field(
+        ...,
+        description="The multiscale datasets for this image",
+        min_items=1,
+        unique_items=True,
+    )
+    omero: Optional[Omero] = None
+
+    @property
+    def multiscale(self) -> Multiscale:
+        """
+        The single element of `self.multiscales`.
+
+        Raises:
+            NotImplementedError:
+                If there are no multiscales or more than one.
+        """
+        if len(self.multiscales) > 1:
+            raise NotImplementedError(
+                "Only images with one multiscale are supported "
+                f"(given: {len(self.multiscales)}"
+            )
+        return self.multiscales[0]
+
+    @property
+    def datasets(self) -> list[Dataset]:
+        """
+        The `datasets` attribute of `self.multiscale`.
+        """
+        return self.multiscale.datasets
+
+    @property
+    def num_levels(self) -> int:
+        return len(self.datasets)
+
+    @property
+    def axes_names(self) -> list[str]:
+        """
+        List of axes names.
+        """
+        return [ax.name for ax in self.multiscale.axes]
+
+    @property
+    def pixel_sizes_zyx(self) -> list[list[float]]:
+        """
+        Pixel sizes extracted from scale transformations of datasets.
+
+        Raises:
+            ValueError:
+                If pixel sizes are below a given threshold (1e-9).
+        """
+        x_index = self.axes_names.index("x")
+        y_index = self.axes_names.index("y")
+        try:
+            z_index = self.axes_names.index("z")
+        except ValueError:
+            z_index = None
+            logging.warning(
+                f"Z axis is not present (axes: {self.axes_names}), and Z pixel"
+                " size is set to 1. This may work, by accident, but it is "
+                "not fully supported."
+            )
+        _pixel_sizes_zyx = []
+        for level in range(self.num_levels):
+            scale = self.datasets[level].scale_transformation.scale
+            pixel_size_x = scale[x_index]
+            pixel_size_y = scale[y_index]
+            if z_index is not None:
+                pixel_size_z = scale[z_index]
+            else:
+                pixel_size_z = 1.0
+            _pixel_sizes_zyx.append([pixel_size_z, pixel_size_y, pixel_size_x])
+            if min(_pixel_sizes_zyx[-1]) < 1e-9:
+                raise ValueError(
+                    f"Pixel sizes at level {level} are too small: "
+                    f"{_pixel_sizes_zyx[-1]}"
+                )
+
+        return _pixel_sizes_zyx
+
+    def get_pixel_sizes_zyx(self, *, level: int = 0) -> list[float]:
+        return self.pixel_sizes_zyx[level]
+
+    @property
+    def coarsening_xy(self) -> int:
+        """
+        Linear coarsening factor in the YX plane.
+
+        We only support coarsening factors that are homogeneous (both in the
+        X/Y directions and across pyramid levels).
+
+        Raises:
+            NotImplementedError:
+                If coarsening ratios are not homogeneous.
+        """
+        current_ratio = None
+        for ind in range(1, self.num_levels):
+            ratio_x = round(
+                self.pixel_sizes_zyx[ind][2] / self.pixel_sizes_zyx[ind - 1][2]
+            )
+            ratio_y = round(
+                self.pixel_sizes_zyx[ind][1] / self.pixel_sizes_zyx[ind - 1][1]
+            )
+            if ratio_x != ratio_y:
+                raise NotImplementedError(
+                    "Inhomogeneous coarsening in X/Y directions "
+                    "is not supported.\n"
+                    f"ZYX pixel sizes:\n {self.pixel_sizes_zyx}"
+                )
+            if current_ratio is None:
+                current_ratio = ratio_x
+            else:
+                if current_ratio != ratio_x:
+                    raise NotImplementedError(
+                        "Inhomogeneous coarsening across levels "
+                        "is not supported.\n"
+                        f"ZYX pixel sizes:\n {self.pixel_sizes_zyx}"
+                    )
+
+        return current_ratio
+
+
+ + + +
+ + + + + + + +
+ + + + +

+ axes_names: list[str] + + + property + + +

+ + +
+ +

List of axes names.

+
+ +
+ +
+ + + + +

+ coarsening_xy: int + + + property + + +

+ + +
+ +

Linear coarsening factor in the YX plane.

+

We only support coarsening factors that are homogeneous (both in the +X/Y directions and across pyramid levels).

+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + NotImplementedError + + +
+

If coarsening ratios are not homogeneous.

+
+
+
+ +
+ +
+ + + + +

+ datasets: list[Dataset] + + + property + + +

+ + +
+ +

The datasets attribute of self.multiscale.

+
+ +
+ +
+ + + + +

+ multiscale: Multiscale + + + property + + +

+ + +
+ +

The single element of self.multiscales.

+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + NotImplementedError + + +
+

If there are no multiscales or more than one.

+
+
+
+ +
+ +
+ + + + +

+ pixel_sizes_zyx: list[list[float]] + + + property + + +

+ + +
+ +

Pixel sizes extracted from scale transformations of datasets.

+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + ValueError + + +
+

If pixel sizes are below a given threshold (1e-9).

+
+
+
+ +
+ + + + + +
+ +
+ +
+ +
+ + + + +

+ NgffWellMeta + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for the metadata of a NGFF well.

+

See https://ngff.openmicroscopy.org/0.4/#well-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
class NgffWellMeta(BaseModel):
+    """
+    Model for the metadata of a NGFF well.
+
+    See https://ngff.openmicroscopy.org/0.4/#well-md.
+    """
+
+    well: Optional[Well] = None
+
+    def get_acquisition_paths(self) -> dict[int, str]:
+        """
+        Create mapping from acquisition indices to corresponding paths.
+
+        Runs on the well zarr attributes and loads the relative paths in the
+        well.
+
+        Returns:
+            Dictionary with `(acquisition index: image path)` key/value pairs.
+
+        Raises:
+            ValueError:
+                If an element of `self.well.images` has no `acquisition`
+                    attribute.
+            NotImplementedError:
+                If acquisitions are not unique.
+        """
+        acquisition_dict = {}
+        for image in self.well.images:
+            if image.acquisition is None:
+                raise ValueError(
+                    "Cannot get acquisition paths for Zarr files without "
+                    "'acquisition' metadata at the well level"
+                )
+            if image.acquisition in acquisition_dict:
+                raise NotImplementedError(
+                    "The `NgffWellMeta.get_acquisition_paths` method (in "
+                    "fractal-tasks-core) does not support wells with "
+                    "multiple images of the same acquisition."
+                )
+            acquisition_dict[image.acquisition] = image.path
+        return acquisition_dict
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ get_acquisition_paths() + +

+ + +
+ +

Create mapping from acquisition indices to corresponding paths.

+

Runs on the well zarr attributes and loads the relative paths in the +well.

+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[int, str] + + +
+

Dictionary with (acquisition index: image path) key/value pairs.

+
+
+ + + + + + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + ValueError + + +
+

If an element of self.well.images has no acquisition + attribute.

+
+
+ + NotImplementedError + + +
+

If acquisitions are not unique.

+
+
+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
def get_acquisition_paths(self) -> dict[int, str]:
+    """
+    Create mapping from acquisition indices to corresponding paths.
+
+    Runs on the well zarr attributes and loads the relative paths in the
+    well.
+
+    Returns:
+        Dictionary with `(acquisition index: image path)` key/value pairs.
+
+    Raises:
+        ValueError:
+            If an element of `self.well.images` has no `acquisition`
+                attribute.
+        NotImplementedError:
+            If acquisitions are not unique.
+    """
+    acquisition_dict = {}
+    for image in self.well.images:
+        if image.acquisition is None:
+            raise ValueError(
+                "Cannot get acquisition paths for Zarr files without "
+                "'acquisition' metadata at the well level"
+            )
+        if image.acquisition in acquisition_dict:
+            raise NotImplementedError(
+                "The `NgffWellMeta.get_acquisition_paths` method (in "
+                "fractal-tasks-core) does not support wells with "
+                "multiple images of the same acquisition."
+            )
+        acquisition_dict[image.acquisition] = image.path
+    return acquisition_dict
+
+
+
+ +
+ + + +
+ +
+ +
+ +
+ + + + +

+ Omero + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for NgffImageMeta.omero.

+

See https://ngff.openmicroscopy.org/0.4/#omero-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
46
+47
+48
+49
+50
+51
+52
+53
class Omero(BaseModel):
+    """
+    Model for `NgffImageMeta.omero`.
+
+    See https://ngff.openmicroscopy.org/0.4/#omero-md.
+    """
+
+    channels: list[Channel]
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ ScaleCoordinateTransformation + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for a scale transformation.

+

This corresponds to scale-type elements of +Dataset.coordinateTransformations or +Multiscale.coordinateTransformations. +See https://ngff.openmicroscopy.org/0.4/#trafo-md

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
class ScaleCoordinateTransformation(BaseModel):
+    """
+    Model for a scale transformation.
+
+    This corresponds to scale-type elements of
+    `Dataset.coordinateTransformations` or
+    `Multiscale.coordinateTransformations`.
+    See https://ngff.openmicroscopy.org/0.4/#trafo-md
+    """
+
+    type: Literal["scale"]
+    scale: list[float] = Field(..., min_items=2)
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ TranslationCoordinateTransformation + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for a translation transformation.

+

This corresponds to translation-type elements of +Dataset.coordinateTransformations or +Multiscale.coordinateTransformations. +See https://ngff.openmicroscopy.org/0.4/#trafo-md

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
class TranslationCoordinateTransformation(BaseModel):
+    """
+    Model for a translation transformation.
+
+    This corresponds to translation-type elements of
+    `Dataset.coordinateTransformations` or
+    `Multiscale.coordinateTransformations`.
+    See https://ngff.openmicroscopy.org/0.4/#trafo-md
+    """
+
+    type: Literal["translation"]
+    translation: list[float] = Field(..., min_items=2)
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ Well + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for NgffWellMeta.well.

+

See https://ngff.openmicroscopy.org/0.4/#well-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
class Well(BaseModel):
+    """
+    Model for `NgffWellMeta.well`.
+
+    See https://ngff.openmicroscopy.org/0.4/#well-md.
+    """
+
+    images: list[ImageInWell] = Field(
+        ...,
+        description="The images included in this well",
+        min_items=1,
+        unique_items=True,
+    )
+    version: Optional[str] = Field(
+        None, description="The version of the specification"
+    )
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + + +

+ Window + + +

+ + +
+

+ Bases: BaseModel

+ + +

Model for Channel.window.

+

Note that we deviate by NGFF specs by making start and end optional. +See https://ngff.openmicroscopy.org/0.4/#omero-md.

+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
class Window(BaseModel):
+    """
+    Model for `Channel.window`.
+
+    Note that we deviate by NGFF specs by making `start` and `end` optional.
+    See https://ngff.openmicroscopy.org/0.4/#omero-md.
+    """
+
+    max: float
+    min: float
+    start: Optional[float] = None
+    end: Optional[float] = None
+
+
+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + +
+ + + + +

+ detect_ome_ngff_type(group) + +

+ + +
+ +

Given a Zarr group, find whether it is an OME-NGFF plate, well or image.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
group +
+

Zarr group

+
+

+ + TYPE: + Group + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + str + + +
+

The detected OME-NGFF type (plate, well or image).

+
+
+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
def detect_ome_ngff_type(group: zarr.Group) -> str:
+    """
+    Given a Zarr group, find whether it is an OME-NGFF plate, well or image.
+
+    Args:
+        group: Zarr group
+
+    Returns:
+        The detected OME-NGFF type (`plate`, `well` or `image`).
+    """
+    attrs = group.attrs.asdict()
+    if "plate" in attrs.keys():
+        ngff_type = "plate"
+    elif "well" in attrs.keys():
+        ngff_type = "well"
+    elif "multiscales" in attrs.keys():
+        ngff_type = "image"
+    else:
+        error_msg = (
+            "Zarr group at cannot be identified as one "
+            "of OME-NGFF plate/well/image groups."
+        )
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+    logger.info(f"Zarr group identified as OME-NGFF {ngff_type}.")
+    return ngff_type
+
+
+
+ +
+ + +
+ + + + +

+ load_NgffImageMeta(zarr_path) + +

+ + +
+ +

Load the attributes of a zarr group and cast them to NgffImageMeta.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
zarr_path +
+

Path to the zarr group.

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + NgffImageMeta + + +
+

A new NgffImageMeta object.

+
+
+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
def load_NgffImageMeta(zarr_path: str) -> NgffImageMeta:
+    """
+    Load the attributes of a zarr group and cast them to `NgffImageMeta`.
+
+    Args:
+        zarr_path: Path to the zarr group.
+
+    Returns:
+        A new `NgffImageMeta` object.
+    """
+    zarr_group = zarr.open_group(zarr_path, mode="r")
+    zarr_attrs = zarr_group.attrs.asdict()
+    try:
+        return NgffImageMeta(**zarr_attrs)
+    except Exception as e:
+        logging.error(
+            f"Contents of {zarr_path} cannot be cast to NgffImageMeta.\n"
+            f"Original error:\n{str(e)}"
+        )
+        raise e
+
+
+
+ +
+ + +
+ + + + +

+ load_NgffWellMeta(zarr_path) + +

+ + +
+ +

Load the attributes of a zarr group and cast them to NgffWellMeta.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
zarr_path +
+

Path to the zarr group.

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + NgffWellMeta + + +
+

A new NgffWellMeta object.

+
+
+ +
+ Source code in fractal_tasks_core/lib_ngff.py + 
403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
def load_NgffWellMeta(zarr_path: str) -> NgffWellMeta:
+    """
+    Load the attributes of a zarr group and cast them to `NgffWellMeta`.
+
+    Args:
+        zarr_path: Path to the zarr group.
+
+    Returns:
+        A new `NgffWellMeta` object.
+    """
+    zarr_group = zarr.open_group(zarr_path, mode="r")
+    zarr_attrs = zarr_group.attrs.asdict()
+    try:
+        return NgffWellMeta(**zarr_attrs)
+    except Exception as e:
+        logging.error(
+            f"Contents of {zarr_path} cannot be cast to NgffWellMeta.\n"
+            f"Original error:\n{str(e)}"
+        )
+        raise e
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_parse_filename_metadata/index.html b/reference/fractal_tasks_core/lib_parse_filename_metadata/index.html new file mode 100644 index 000000000..c2198e2d3 --- /dev/null +++ b/reference/fractal_tasks_core/lib_parse_filename_metadata/index.html @@ -0,0 +1,1474 @@ + + + + + + + + + + + + + + + + + + + + + + lib_parse_filename_metadata - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_parse_filename_metadata

+ +
+ + + + +
+ +

Extract metadata from image filename.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _get_plate_name(plate_prefix) + +

+ + +
+ +

Two kinds of plate_prefix values are handled in a special way:

+
    +
  1. Filenames from FMI, with successful barcode reading: + 210305NAR005AAN_210416_164828 with plate name 210305NAR005AAN;
    2. +
  2. Filenames from FMI, with failed barcode reading: + yymmdd_hhmmss_210416_164828 with plate name RS{yymmddhhmmss}.
    3. +
+

For all non-matching filenames, plate name is plate_prefix.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
plate_prefix +
+

TBD

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/lib_parse_filename_metadata.py + 
19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
def _get_plate_name(plate_prefix: str) -> str:
+    """
+    Two kinds of plate_prefix values are handled in a special way:
+
+    1. Filenames from FMI, with successful barcode reading:
+       `210305NAR005AAN_210416_164828` with plate name `210305NAR005AAN`;
+    2. Filenames from FMI, with failed barcode reading:
+       `yymmdd_hhmmss_210416_164828` with plate name `RS{yymmddhhmmss}`.
+
+    For all non-matching filenames, plate name is `plate_prefix`.
+
+    Args:
+        plate_prefix: TBD
+    """
+
+    fields = plate_prefix.split("_")
+
+    # FMI (successful barcode reading)
+    if (
+        len(fields) == 3
+        and len(fields[1]) == 6
+        and len(fields[2]) == 6
+        and fields[1].isdigit()
+        and fields[2].isdigit()
+    ):
+        barcode, img_date, img_time = fields[:]
+        plate = barcode
+    # FMI (failed barcode reading)
+    elif (
+        len(fields) == 4
+        and len(fields[0]) == 6
+        and len(fields[1]) == 6
+        and len(fields[2]) == 6
+        and len(fields[3]) == 6
+        and fields[0].isdigit()
+        and fields[1].isdigit()
+        and fields[2].isdigit()
+        and fields[3].isdigit()
+    ):
+        scan_date, scan_time, img_date, img_time = fields[:]
+        plate = f"RS{scan_date + scan_time}"
+    # All non-matching cases
+    else:
+        plate = plate_prefix
+
+    return plate
+
+
+
+ +
+ + +
+ + + + +

+ parse_filename(filename) + +

+ + +
+ +

Parse image metadata from filename.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
filename +
+

Name of the image.

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[str, str] + + +
+

Metadata dictionary.

+
+
+ +
+ Source code in fractal_tasks_core/lib_parse_filename_metadata.py + 
 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
def parse_filename(filename: str) -> dict[str, str]:
+    """
+    Parse image metadata from filename.
+
+    Args:
+        filename: Name of the image.
+
+    Returns:
+        Metadata dictionary.
+    """
+
+    # Remove extension and folder from filename
+    filename = Path(filename).with_suffix("").name
+
+    output = {}
+
+    # Split filename into plate_prefix + well + TFLAZC
+    filename_fields = filename.split("_")
+    if len(filename_fields) < 3:
+        raise ValueError(f"{filename} not valid")
+    output["plate_prefix"] = "_".join(filename_fields[:-2])
+    output["plate"] = _get_plate_name(output["plate_prefix"])
+
+    # Assign well
+    output["well"] = filename_fields[-2]
+
+    # Assign TFLAZC
+    TFLAZC = filename_fields[-1]
+    metadata = re.split(r"([0-9]+)", TFLAZC)
+    if metadata[-1] != "" or len(metadata) != 13:
+        raise ValueError(f"Something wrong with {filename=}, {TFLAZC=}")
+    # Remove 13-th (and last) element of the metadata list (an empty string)
+    metadata = metadata[:-1]
+    # Fill output dictionary
+    for ind, key in enumerate(metadata[::2]):
+        value = metadata[2 * ind + 1]
+        if key.isdigit() or not value.isdigit():
+            raise ValueError(
+                f"Something wrong with {filename=}, for {key=} {value=}"
+            )
+        output[key] = value
+    return output
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_pyramid_creation/index.html b/reference/fractal_tasks_core/lib_pyramid_creation/index.html new file mode 100644 index 000000000..1924da72a --- /dev/null +++ b/reference/fractal_tasks_core/lib_pyramid_creation/index.html @@ -0,0 +1,1461 @@ + + + + + + + + + + + + + + + + + + + + + + lib_pyramid_creation - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_pyramid_creation

+ +
+ + + + +
+ +

Construct and write pyramid of lower-resolution levels.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ build_pyramid(*, zarrurl, overwrite=False, num_levels=2, coarsening_xy=2, chunksize=None, aggregation_function=None) + +

+ + +
+ +

Starting from on-disk highest-resolution data, build and write to disk a +pyramid with (num_levels - 1) coarsened levels. +This function works for 2D, 3D or 4D arrays.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
zarrurl +
+

Path of the zarr group, which must already include level 0 +(e.g. "some/path/plate.zarr/B/03/0").

+
+

+ + TYPE: + Union[str, Path] + +

+
overwrite +
+

Whether to overwrite existing pyramid levels.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
num_levels +
+

Total number of pyramid levels (including 0).

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
coarsening_xy +
+

Linear coarsening factor between subsequent levels.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
chunksize +
+

Shape of a single chunk.

+
+

+ + TYPE: + Optional[Sequence[int]] + + + DEFAULT: + None + +

+
aggregation_function +
+

Function to be used when downsampling.

+
+

+ + TYPE: + Optional[Callable] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/lib_pyramid_creation.py + 
 28
+ 29
+ 30
+ 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
def build_pyramid(
+    *,
+    zarrurl: Union[str, pathlib.Path],
+    overwrite: bool = False,
+    num_levels: int = 2,
+    coarsening_xy: int = 2,
+    chunksize: Optional[Sequence[int]] = None,
+    aggregation_function: Optional[Callable] = None,
+) -> None:
+
+    """
+    Starting from on-disk highest-resolution data, build and write to disk a
+    pyramid with `(num_levels - 1)` coarsened levels.
+    This function works for 2D, 3D or 4D arrays.
+
+    Args:
+        zarrurl: Path of the zarr group, which must already include level 0
+            (e.g. `"some/path/plate.zarr/B/03/0"`).
+        overwrite: Whether to overwrite existing pyramid levels.
+        num_levels: Total number of pyramid levels (including 0).
+        coarsening_xy: Linear coarsening factor between subsequent levels.
+        chunksize: Shape of a single chunk.
+        aggregation_function: Function to be used when downsampling.
+    """
+
+    # Clean up zarrurl
+    zarrurl = str(pathlib.Path(zarrurl))  # FIXME
+    zarrurl_highres = f"{zarrurl}/0"
+    logger.info(f"[build_pyramid] High-resolution path: {zarrurl_highres}")
+
+    # Lazily load highest-resolution data
+    data_highres = da.from_zarr(zarrurl_highres)
+    logger.info(f"[build_pyramid] High-resolution data: {str(data_highres)}")
+
+    # Check the number of axes and identify YX dimensions
+    ndims = len(data_highres.shape)
+    if ndims not in [2, 3, 4]:
+        raise ValueError(f"{data_highres.shape=}, ndims not in [2,3,4]")
+    y_axis = ndims - 2
+    x_axis = ndims - 1
+
+    # Set aggregation_function
+    if aggregation_function is None:
+        aggregation_function = np.mean
+
+    # Compute and write lower-resolution levels
+    previous_level = data_highres
+    for ind_level in range(1, num_levels):
+        # Verify that coarsening is doable
+        if min(previous_level.shape[-2:]) < coarsening_xy:
+            raise ValueError(
+                f"ERROR: at {ind_level}-th level, "
+                f"coarsening_xy={coarsening_xy} "
+                f"but previous level has shape {previous_level.shape}"
+            )
+        # Apply coarsening
+        newlevel = da.coarsen(
+            aggregation_function,
+            previous_level,
+            {y_axis: coarsening_xy, x_axis: coarsening_xy},
+            trim_excess=True,
+        ).astype(data_highres.dtype)
+
+        # Apply rechunking
+        if chunksize is None:
+            newlevel_rechunked = newlevel
+        else:
+            newlevel_rechunked = newlevel.rechunk(chunksize)
+        logger.info(
+            f"[build_pyramid] Level {ind_level} data: "
+            f"{str(newlevel_rechunked)}"
+        )
+
+        # Write zarr and store output (useful to construct next level)
+        previous_level = newlevel_rechunked.to_zarr(
+            zarrurl,
+            component=f"{ind_level}",
+            overwrite=overwrite,
+            compute=True,
+            return_stored=True,
+            write_empty_chunks=False,
+            dimension_separator="/",
+        )
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_read_fractal_metadata/index.html b/reference/fractal_tasks_core/lib_read_fractal_metadata/index.html new file mode 100644 index 000000000..f9c821080 --- /dev/null +++ b/reference/fractal_tasks_core/lib_read_fractal_metadata/index.html @@ -0,0 +1,1479 @@ + + + + + + + + + + + + + + + + + + + + + + lib_read_fractal_metadata - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_read_fractal_metadata

+ +
+ + + + +
+ +

Helper functions to extract parameters from metadata dictionary.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ find_omengff_acquisition(image_zarr_path) + +

+ + +
+ +

Discover the acquisition index based on OME-NGFF metadata.

+

Given the path to a zarr image folder (e.g. /path/plate.zarr/B/03/0), +extract the acquisition index from the .zattrs file of the parent +folder (i.e. at the well level), or return None if acquisition is not +specified.

+

Notes:

+
    +
  1. For non-multiplexing datasets, acquisition is not a required + information in the metadata. If it is not there, this function + returns None.
    2. +
  2. This function fails if we use an image that does not belong to + an OME-NGFF well.
    3. +
+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_zarr_path +
+

Full path to an OME-NGFF image folder.

+
+

+ + TYPE: + Path + +

+
+ +
+ Source code in fractal_tasks_core/lib_read_fractal_metadata.py + 
23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
def find_omengff_acquisition(image_zarr_path: Path) -> Union[int, None]:
+    """
+    Discover the acquisition index based on OME-NGFF metadata.
+
+    Given the path to a zarr image folder (e.g. `/path/plate.zarr/B/03/0`),
+    extract the acquisition index from the `.zattrs` file of the parent
+    folder (i.e. at the well level), or return `None` if acquisition is not
+    specified.
+
+    Notes:
+
+    1. For non-multiplexing datasets, acquisition is not a required
+       information in the metadata. If it is not there, this function
+       returns `None`.
+    2. This function fails if we use an image that does not belong to
+       an OME-NGFF well.
+
+    Args:
+        image_zarr_path: Full path to an OME-NGFF image folder.
+    """
+
+    # Identify well path and attrs
+    well_zarr_path = image_zarr_path.parent
+    if not (well_zarr_path / ".zattrs").exists():
+        raise ValueError(
+            f"{str(well_zarr_path)} must be an OME-NGFF well "
+            "folder, but it does not include a .zattrs file."
+        )
+    well_group = zarr.open_group(str(well_zarr_path))
+    attrs_images = well_group.attrs["well"]["images"]
+
+    # Loook for the acquisition of the current image (if any)
+    acquisition = None
+    for img_dict in attrs_images:
+        if (
+            img_dict["path"] == image_zarr_path.name
+            and "acquisition" in img_dict.keys()
+        ):
+            acquisition = img_dict["acquisition"]
+            break
+
+    return acquisition
+
+
+
+ +
+ + +
+ + + + +

+ get_parameters_from_metadata(*, keys, metadata, image_zarr_path) + +

+ + +
+ +

Flexibly extract parameters from metadata dictionary

+

This covers both parameters which are acquisition-specific (if the image +belongs to an OME-NGFF array and its acquisition is specified) or simply +available in the dictionary. +The two cases are handled as: +

metadata[acquisition]["some_parameter"]  # acquisition available
+metadata["some_parameter"]               # acquisition not available
+

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
keys +
+

list of required parameters.

+
+

+ + TYPE: + Sequence[str] + +

+
metadata +
+

metadata dictionary.

+
+

+ + TYPE: + Dict[str, Any] + +

+
image_zarr_path +
+

full path to image, e.g. /path/plate.zarr/B/03/0.

+
+

+ + TYPE: + Path + +

+
+ +
+ Source code in fractal_tasks_core/lib_read_fractal_metadata.py + 
 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
def get_parameters_from_metadata(
+    *,
+    keys: Sequence[str],
+    metadata: Dict[str, Any],
+    image_zarr_path: Path,
+) -> Dict[str, Any]:
+    """
+    Flexibly extract parameters from metadata dictionary
+
+    This covers both parameters which are acquisition-specific (if the image
+    belongs to an OME-NGFF array and its acquisition is specified) or simply
+    available in the dictionary.
+    The two cases are handled as:
+    ```
+    metadata[acquisition]["some_parameter"]  # acquisition available
+    metadata["some_parameter"]               # acquisition not available
+    ```
+
+    Args:
+        keys: list of required parameters.
+        metadata: metadata dictionary.
+        image_zarr_path: full path to image, e.g. `/path/plate.zarr/B/03/0`.
+    """
+
+    parameters = {}
+    acquisition = find_omengff_acquisition(image_zarr_path)
+    if acquisition is not None:
+        parameters["acquisition"] = acquisition
+
+    for key in keys:
+        if acquisition is None:
+            parameter = metadata[key]
+        else:
+            try:
+                parameter = metadata[key][str(acquisition)]
+            except TypeError:
+                parameter = metadata[key]
+            except KeyError:
+                parameter = metadata[key]
+        parameters[key] = parameter
+    return parameters
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_regions_of_interest/index.html b/reference/fractal_tasks_core/lib_regions_of_interest/index.html new file mode 100644 index 000000000..5c764b380 --- /dev/null +++ b/reference/fractal_tasks_core/lib_regions_of_interest/index.html @@ -0,0 +1,4168 @@ + + + + + + + + + + + + + + + + + + + + + + lib_regions_of_interest - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_regions_of_interest

+ +
+ + + + +
+ +

Functions to handle regions of interests (via pandas and AnnData).

+ + + +
+ + + + + + + + + + +
+ + + + +

+ are_ROI_table_columns_valid(*, table) + +

+ + +
+ +

Verify some validity assumptions on a ROI table.

+

This function reflects our current working assumptions (e.g. the presence +of some specific columns); this may change in future versions.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
table +
+

AnnData table to be checked

+
+

+ + TYPE: + AnnData + +

+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
def are_ROI_table_columns_valid(*, table: ad.AnnData) -> None:
+    """
+    Verify some validity assumptions on a ROI table.
+
+    This function reflects our current working assumptions (e.g. the presence
+    of some specific columns); this may change in future versions.
+
+    Args:
+        table: AnnData table to be checked
+    """
+
+    # Hard constraint: table columns must include some expected ones
+    columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+    ]
+    for column in columns:
+        if column not in table.var_names:
+            raise ValueError(f"Column {column} is not present in ROI table")
+
+
+
+ +
+ + +
+ + + + +

+ array_to_bounding_box_table(mask_array, pxl_sizes_zyx, origin_zyx=(0, 0, 0)) + +

+ + +
+ +

Construct bounding-box ROI table for a mask array.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
mask_array +
+

Original array to construct bounding boxes.

+
+

+ + TYPE: + ndarray + +

+
pxl_sizes_zyx +
+

Physical-unit pixel ZYX sizes.

+
+

+ + TYPE: + list[float] + +

+
origin_zyx +
+

Shift ROI origin by this amount of ZYX pixels.

+
+

+ + TYPE: + tuple[int, int, int] + + + DEFAULT: + (0, 0, 0) + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + DataFrame + + +
+

DataFrame with each line representing the bounding-box ROI that +corresponds to a unique value of mask_array. ROI properties are +expressed in physical units (with columns defined as elsewhere this +module - see e.g. prepare_well_ROI_table), and positions are +optionally shifted (if origin_zyx is set). An additional column +label keeps track of the mask_array value corresponding to each +ROI.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
def array_to_bounding_box_table(
+    mask_array: np.ndarray,
+    pxl_sizes_zyx: list[float],
+    origin_zyx: tuple[int, int, int] = (0, 0, 0),
+) -> pd.DataFrame:
+    """
+    Construct bounding-box ROI table for a mask array.
+
+    Args:
+        mask_array: Original array to construct bounding boxes.
+        pxl_sizes_zyx: Physical-unit pixel ZYX sizes.
+        origin_zyx: Shift ROI origin by this amount of ZYX pixels.
+
+    Returns:
+        DataFrame with each line representing the bounding-box ROI that
+            corresponds to a unique value of `mask_array`. ROI properties are
+            expressed in physical units (with columns defined as elsewhere this
+            module - see e.g. `prepare_well_ROI_table`), and positions are
+            optionally shifted (if `origin_zyx` is set). An additional column
+            `label` keeps track of the `mask_array` value corresponding to each
+            ROI.
+    """
+
+    pxl_sizes_zyx_array = np.array(pxl_sizes_zyx)
+    z_origin, y_origin, x_origin = origin_zyx[:]
+
+    labels = np.unique(mask_array)
+    labels = labels[labels > 0]
+    elem_list = []
+    for label in labels:
+
+        # Compute bounding box
+        label_match = np.where(mask_array == label)
+        zmin, ymin, xmin = np.min(label_match, axis=1) * pxl_sizes_zyx_array
+        zmax, ymax, xmax = (
+            np.max(label_match, axis=1) + 1
+        ) * pxl_sizes_zyx_array
+
+        # Compute bounding-box edges
+        length_x = xmax - xmin
+        length_y = ymax - ymin
+        length_z = zmax - zmin
+
+        # Shift origin
+        zmin += z_origin * pxl_sizes_zyx[0]
+        ymin += y_origin * pxl_sizes_zyx[1]
+        xmin += x_origin * pxl_sizes_zyx[2]
+
+        elem_list.append((xmin, ymin, zmin, length_x, length_y, length_z))
+
+    df_columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+    ]
+
+    if len(elem_list) == 0:
+        df = pd.DataFrame(columns=[x for x in df_columns] + ["label"])
+    else:
+        df = pd.DataFrame(np.array(elem_list), columns=df_columns)
+        df["label"] = labels
+
+    return df
+
+
+
+ +
+ + +
+ + + + +

+ check_valid_ROI_indices(list_indices, ROI_table_name) + +

+ + +
+ +

Check that list of indices has zero origin on each axis.

+

See fractal-tasks-core issues #530 and #554.

+

This helper function is meant to provide informative error messages when +ROI tables created with fractal-tasks-core up to v0.11 are used in v0.12. +This function will be deprecated and removed as soon as the v0.11/v0.12 +transition advances.

+

Note that only FOV_ROI_table and well_ROI_table have to fulfill this +constraint, while ROI tables obtained through segmentation may have +arbitrary (non-negative) indices.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
list_indices +
+

Output of convert_ROI_table_to_indices; each item is like +[start_z, end_z, start_y, end_y, start_x, end_x].

+
+

+ + TYPE: + list[list[int]] + +

+
ROI_table_name +
+

Name of the ROI table.

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + ValueError + + +
+

If the table name is FOV_ROI_table or well_ROI_table and the + minimum value of start_x, start_y and start_z are not all + zero.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
def check_valid_ROI_indices(
+    list_indices: list[list[int]],
+    ROI_table_name: str,
+) -> None:
+    """
+    Check that list of indices has zero origin on each axis.
+
+    See fractal-tasks-core issues #530 and #554.
+
+    This helper function is meant to provide informative error messages when
+    ROI tables created with fractal-tasks-core up to v0.11 are used in v0.12.
+    This function will be deprecated and removed as soon as the v0.11/v0.12
+    transition advances.
+
+    Note that only `FOV_ROI_table` and `well_ROI_table` have to fulfill this
+    constraint, while ROI tables obtained through segmentation may have
+    arbitrary (non-negative) indices.
+
+    Args:
+        list_indices:
+            Output of `convert_ROI_table_to_indices`; each item is like
+            `[start_z, end_z, start_y, end_y, start_x, end_x]`.
+        ROI_table_name: Name of the ROI table.
+
+    Raises:
+        ValueError:
+            If the table name is `FOV_ROI_table` or `well_ROI_table` and the
+                minimum value of `start_x`, `start_y` and `start_z` are not all
+                zero.
+    """
+    if ROI_table_name not in ["FOV_ROI_table", "well_ROI_table"]:
+        # This validation function only applies to the FOV/well ROI tables
+        # generated with fractal-tasks-core
+        return
+
+    # Find minimum index along ZYX
+    min_start_z = min(item[0] for item in list_indices)
+    min_start_y = min(item[2] for item in list_indices)
+    min_start_x = min(item[4] for item in list_indices)
+
+    # Check that minimum indices are all zero
+    for (ind, min_index) in enumerate((min_start_z, min_start_y, min_start_x)):
+        if min_index != 0:
+            axis = ["Z", "Y", "X"][ind]
+            raise ValueError(
+                f"{axis} component of ROI indices for table `{ROI_table_name}`"
+                f" do not start with 0, but with {min_index}.\n"
+                "Hint: As of fractal-tasks-core v0.12, FOV/well ROI "
+                "tables with non-zero origins (e.g. the ones created with "
+                "v0.11) are not supported."
+            )
+
+
+
+ +
+ + +
+ + + + +

+ convert_ROI_table_to_indices(ROI, full_res_pxl_sizes_zyx, level=0, coarsening_xy=2, cols_xyz_pos=['x_micrometer', 'y_micrometer', 'z_micrometer'], cols_xyz_len=['len_x_micrometer', 'len_y_micrometer', 'len_z_micrometer']) + +

+ + +
+ +

Convert a ROI AnnData table into integer array indices.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
ROI +
+

AnnData table with list of ROIs.

+
+

+ + TYPE: + AnnData + +

+
full_res_pxl_sizes_zyx +
+

Physical-unit pixel ZYX sizes at the full-resolution pyramid level.

+
+

+ + TYPE: + Sequence[float] + +

+
level +
+

Pyramid level.

+
+

+ + TYPE: + int + + + DEFAULT: + 0 + +

+
coarsening_xy +
+

Linear coarsening factor in the YX plane.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
cols_xyz_pos +
+

Column names for XYZ ROI positions.

+
+

+ + TYPE: + Sequence[str] + + + DEFAULT: + ['x_micrometer', 'y_micrometer', 'z_micrometer'] + +

+
cols_xyz_len +
+

Column names for XYZ ROI edges.

+
+

+ + TYPE: + Sequence[str] + + + DEFAULT: + ['len_x_micrometer', 'len_y_micrometer', 'len_z_micrometer'] + +

+
+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + ValueError + + +
+

If any of the array indices is negative.

+
+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + list[list[int]] + + +
+

Nested list of indices. The main list has one item per ROI. Each ROI +item is a list of six integers as in [start_z, end_z, start_y, +end_y, start_x, end_x]. The array-index interval for a given ROI +is start_x:end_x along X, and so on for Y and Z.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
def convert_ROI_table_to_indices(
+    ROI: ad.AnnData,
+    full_res_pxl_sizes_zyx: Sequence[float],
+    level: int = 0,
+    coarsening_xy: int = 2,
+    cols_xyz_pos: Sequence[str] = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+    ],
+    cols_xyz_len: Sequence[str] = [
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+    ],
+) -> list[list[int]]:
+    """
+    Convert a ROI AnnData table into integer array indices.
+
+    Args:
+        ROI: AnnData table with list of ROIs.
+        full_res_pxl_sizes_zyx:
+            Physical-unit pixel ZYX sizes at the full-resolution pyramid level.
+        level: Pyramid level.
+        coarsening_xy: Linear coarsening factor in the YX plane.
+        cols_xyz_pos: Column names for XYZ ROI positions.
+        cols_xyz_len: Column names for XYZ ROI edges.
+
+    Raises:
+        ValueError:
+            If any of the array indices is negative.
+
+    Returns:
+        Nested list of indices. The main list has one item per ROI. Each ROI
+            item is a list of six integers as in `[start_z, end_z, start_y,
+            end_y, start_x, end_x]`. The array-index interval for a given ROI
+            is `start_x:end_x` along X, and so on for Y and Z.
+    """
+    # Handle empty ROI table
+    if len(ROI) == 0:
+        return []
+
+    # Set pyramid-level pixel sizes
+    pxl_size_z, pxl_size_y, pxl_size_x = full_res_pxl_sizes_zyx
+    prefactor = coarsening_xy**level
+    pxl_size_x *= prefactor
+    pxl_size_y *= prefactor
+
+    x_pos, y_pos, z_pos = cols_xyz_pos[:]
+    x_len, y_len, z_len = cols_xyz_len[:]
+
+    list_indices = []
+    for ROI_name in ROI.obs_names:
+
+        # Extract data from anndata table
+        x_micrometer = ROI[ROI_name, x_pos].X[0, 0]
+        y_micrometer = ROI[ROI_name, y_pos].X[0, 0]
+        z_micrometer = ROI[ROI_name, z_pos].X[0, 0]
+        len_x_micrometer = ROI[ROI_name, x_len].X[0, 0]
+        len_y_micrometer = ROI[ROI_name, y_len].X[0, 0]
+        len_z_micrometer = ROI[ROI_name, z_len].X[0, 0]
+
+        # Identify indices along the three dimensions
+        start_x = x_micrometer / pxl_size_x
+        end_x = (x_micrometer + len_x_micrometer) / pxl_size_x
+        start_y = y_micrometer / pxl_size_y
+        end_y = (y_micrometer + len_y_micrometer) / pxl_size_y
+        start_z = z_micrometer / pxl_size_z
+        end_z = (z_micrometer + len_z_micrometer) / pxl_size_z
+        indices = [start_z, end_z, start_y, end_y, start_x, end_x]
+
+        # Round indices to lower integer
+        indices = list(map(round, indices))
+
+        # Fail for negative indices
+        if min(indices) < 0:
+            raise ValueError(
+                f"ROI {ROI_name} converted into negative array indices.\n"
+                f"ZYX position: {z_micrometer}, {y_micrometer}, "
+                f"{x_micrometer}\n"
+                f"ZYX pixel sizes: {pxl_size_z}, {pxl_size_y}, "
+                f"{pxl_size_x} ({level=})\n"
+                "Hint: As of fractal-tasks-core v0.12, FOV/well ROI "
+                "tables with non-zero origins (e.g. the ones created with "
+                "v0.11) are not supported."
+            )
+
+        # Append ROI indices to to list
+        list_indices.append(indices[:])
+
+    return list_indices
+
+
+
+ +
+ + +
+ + + + +

+ convert_ROIs_from_3D_to_2D(adata, pixel_size_z) + +

+ + +
+ +

TBD

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
adata +
+

TBD

+
+

+ + TYPE: + AnnData + +

+
pixel_size_z +
+

TBD

+
+

+ + TYPE: + float + +

+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
def convert_ROIs_from_3D_to_2D(
+    adata: ad.AnnData,
+    pixel_size_z: float,
+) -> ad.AnnData:
+    """
+    TBD
+
+    Args:
+        adata: TBD
+        pixel_size_z: TBD
+    """
+
+    # Compress a 3D stack of images to a single Z plane,
+    # with thickness equal to pixel_size_z
+    df = adata.to_df()
+    df["len_z_micrometer"] = pixel_size_z
+
+    # Assign dtype explicitly, to avoid
+    # >> UserWarning: X converted to numpy array with dtype float64
+    # when creating AnnData object
+    df = df.astype(np.float32)
+
+    # Create an AnnData object directly from the DataFrame
+    new_adata = ad.AnnData(X=df)
+
+    # Rename rows and columns
+    new_adata.obs_names = adata.obs_names
+    new_adata.var_names = list(map(str, df.columns))
+
+    return new_adata
+
+
+
+ +
+ + +
+ + + + +

+ convert_indices_to_regions(index) + +

+ + +
+ +

Converts index tuples to region tuple

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
index +
+

Tuple containing 6 entries of (z_start, z_end, y_start, +y_end, x_start, x_end).

+
+

+ + TYPE: + list[int] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ region + +
+

tuple of three slices (ZYX)

+
+

+ + TYPE: + tuple[slice, slice, slice] + +

+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
564
+565
+566
+567
+568
+569
+570
+571
+572
+573
+574
+575
+576
+577
+578
+579
+580
+581
def convert_indices_to_regions(
+    index: list[int],
+) -> tuple[slice, slice, slice]:
+    """
+    Converts index tuples to region tuple
+
+    Args:
+        index: Tuple containing 6 entries of (z_start, z_end, y_start,
+            y_end, x_start, x_end).
+
+    Returns:
+        region: tuple of three slices (ZYX)
+    """
+    return (
+        slice(index[0], index[1]),
+        slice(index[2], index[3]),
+        slice(index[4], index[5]),
+    )
+
+
+
+ +
+ + +
+ + + + +

+ empty_bounding_box_table() + +

+ + +
+ +

Construct an empty bounding-box ROI table of given shape.

+

This function mirrors the functionality of array_to_bounding_box_table, +for the specific case where the array includes no label. The advantages of +this function are that:

+
    +
  1. It does not require computing a whole array of zeros;
    2. +
  2. We avoid hardcoding column names in the task functions.
    3. +
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + DataFrame + + +
+

DataFrame with no rows, and with columns corresponding to the output of +array_to_bounding_box_table.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
def empty_bounding_box_table() -> pd.DataFrame:
+    """
+    Construct an empty bounding-box ROI table of given shape.
+
+    This function mirrors the functionality of `array_to_bounding_box_table`,
+    for the specific case where the array includes no label. The advantages of
+    this function are that:
+
+    1. It does not require computing a whole array of zeros;
+    2. We avoid hardcoding column names in the task functions.
+
+    Returns:
+        DataFrame with no rows, and with columns corresponding to the output of
+            `array_to_bounding_box_table`.
+    """
+
+    df_columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+    ]
+    df = pd.DataFrame(columns=[x for x in df_columns] + ["label"])
+    return df
+
+
+
+ +
+ + +
+ + + + +

+ get_image_grid_ROIs(array_shape, pixels_ZYX, grid_YX_shape) + +

+ + +
+ +

Produce a table with ROIS placed on a rectangular grid.

+

The main goal of this ROI grid is to allow processing of smaller subset of +the whole array.

+

In a specific case (that is, if the image array was obtained by stitching +together a set of FOVs placed on a regular grid), the ROIs correspond to +the original FOVs.

+

TODO: make this flexible with respect to the presence/absence of Z.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
array_shape +
+

ZYX shape of the image array.

+
+

+ + TYPE: + tuple[int, int, int] + +

+
pixels_ZYX +
+

ZYX pixel sizes in micrometers.

+
+

+ + TYPE: + list[float] + +

+
grid_YX_shape +
+ +
+

+ + TYPE: + tuple[int, int] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + AnnData + + +
+

An AnnData table with a single ROI.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
688
+689
+690
+691
+692
+693
+694
+695
+696
+697
+698
+699
+700
+701
+702
+703
+704
+705
+706
+707
+708
+709
+710
+711
+712
+713
+714
+715
+716
+717
+718
+719
+720
+721
+722
+723
+724
+725
+726
+727
+728
+729
+730
+731
+732
+733
+734
+735
+736
+737
+738
+739
+740
+741
+742
+743
+744
+745
+746
+747
+748
+749
+750
+751
+752
+753
+754
+755
+756
def get_image_grid_ROIs(
+    array_shape: tuple[int, int, int],
+    pixels_ZYX: list[float],
+    grid_YX_shape: tuple[int, int],
+) -> ad.AnnData:
+    """
+    Produce a table with ROIS placed on a rectangular grid.
+
+    The main goal of this ROI grid is to allow processing of smaller subset of
+    the whole array.
+
+    In a specific case (that is, if the image array was obtained by stitching
+    together a set of FOVs placed on a regular grid), the ROIs correspond to
+    the original FOVs.
+
+    TODO: make this flexible with respect to the presence/absence of Z.
+
+    Args:
+        array_shape: ZYX shape of the image array.
+        pixels_ZYX: ZYX pixel sizes in micrometers.
+        grid_YX_shape:
+
+    Returns:
+        An `AnnData` table with a single ROI.
+    """
+    shape_z, shape_y, shape_x = array_shape[-3:]
+    grid_size_y, grid_size_x = grid_YX_shape[:]
+    X = []
+    obs_names = []
+    counter = 0
+    start_z = 0
+    len_z = shape_z
+
+    # Find minimal len_y that covers [0,shape_y] with grid_size_y intervals
+    len_y = math.ceil(shape_y / grid_size_y)
+    len_x = math.ceil(shape_x / grid_size_x)
+    for ind_y in range(grid_size_y):
+        start_y = ind_y * len_y
+        tmp_len_y = min(shape_y, start_y + len_y) - start_y
+        for ind_x in range(grid_size_x):
+            start_x = ind_x * len_x
+            tmp_len_x = min(shape_x, start_x + len_x) - start_x
+            X.append(
+                [
+                    start_x * pixels_ZYX[2],
+                    start_y * pixels_ZYX[1],
+                    start_z * pixels_ZYX[0],
+                    tmp_len_x * pixels_ZYX[2],
+                    tmp_len_y * pixels_ZYX[1],
+                    len_z * pixels_ZYX[0],
+                    start_x * pixels_ZYX[2],
+                    start_y * pixels_ZYX[1],
+                ]
+            )
+            counter += 1
+            obs_names.append(f"ROI_{counter}")
+    ROI_table = ad.AnnData(X=np.array(X, dtype=np.float32))
+    ROI_table.obs_names = obs_names
+    ROI_table.var_names = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+    ]
+    return ROI_table
+
+
+
+ +
+ + +
+ + + + +

+ get_single_image_ROI(array_shape, pixels_ZYX) + +

+ + +
+ +

Produce a table with a single ROI that covers the whole array

+

TODO: make this flexible with respect to the presence/absence of Z.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
array_shape +
+

ZYX shape of the image array.

+
+

+ + TYPE: + tuple[int, int, int] + +

+
pixels_ZYX +
+

ZYX pixel sizes in micrometers.

+
+

+ + TYPE: + list[float] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + AnnData + + +
+

An AnnData table with a single ROI.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
640
+641
+642
+643
+644
+645
+646
+647
+648
+649
+650
+651
+652
+653
+654
+655
+656
+657
+658
+659
+660
+661
+662
+663
+664
+665
+666
+667
+668
+669
+670
+671
+672
+673
+674
+675
+676
+677
+678
+679
+680
+681
+682
+683
+684
+685
def get_single_image_ROI(
+    array_shape: tuple[int, int, int],
+    pixels_ZYX: list[float],
+) -> ad.AnnData:
+    """
+    Produce a table with a single ROI that covers the whole array
+
+    TODO: make this flexible with respect to the presence/absence of Z.
+
+    Args:
+        array_shape: ZYX shape of the image array.
+        pixels_ZYX: ZYX pixel sizes in micrometers.
+
+    Returns:
+        An `AnnData` table with a single ROI.
+    """
+    shape_z, shape_y, shape_x = array_shape[-3:]
+    ROI_table = ad.AnnData(
+        X=np.array(
+            [
+                [
+                    0.0,
+                    0.0,
+                    0.0,
+                    shape_x * pixels_ZYX[2],
+                    shape_y * pixels_ZYX[1],
+                    shape_z * pixels_ZYX[0],
+                    0.0,
+                    0.0,
+                ],
+            ],
+            dtype=np.float32,
+        )
+    )
+    ROI_table.obs_names = ["image_1"]
+    ROI_table.var_names = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+    ]
+    return ROI_table
+
+
+
+ +
+ + +
+ + + + +

+ is_ROI_table_valid(*, table_path, use_masks) + +

+ + +
+ +

Verify some validity assumptions on a ROI table.

+

This function reflects our current working assumptions (e.g. the presence +of some specific columns); this may change in future versions.

+

If use_masks=True, we verify that the table is suitable to be used as +part of our masked-loading functions (see lib_masked_loading.py); if +these checks fail, use_masks should be set to False upstream in the +parent function.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
table_path +
+

Path of the AnnData ROI table to be checked.

+
+

+ + TYPE: + str + +

+
use_masks +
+

If True, perform some additional checks related to +masked loading.

+
+

+ + TYPE: + bool + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + Optional[bool] + + +
+

Always None if use_masks=False, otherwise return whether the table +is valid for masked loading.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
def is_ROI_table_valid(*, table_path: str, use_masks: bool) -> Optional[bool]:
+    """
+    Verify some validity assumptions on a ROI table.
+
+    This function reflects our current working assumptions (e.g. the presence
+    of some specific columns); this may change in future versions.
+
+    If `use_masks=True`, we verify that the table is suitable to be used as
+    part of our masked-loading functions (see `lib_masked_loading.py`); if
+    these checks fail, `use_masks` should be set to `False` upstream in the
+    parent function.
+
+    Args:
+        table_path: Path of the AnnData ROI table to be checked.
+        use_masks: If `True`, perform some additional checks related to
+            masked loading.
+
+    Returns:
+        Always `None` if `use_masks=False`, otherwise return whether the table
+            is valid for masked loading.
+    """
+
+    table = ad.read_zarr(table_path)
+    are_ROI_table_columns_valid(table=table)
+    if not use_masks:
+        return None
+
+    # Soft constraint: the table can be used for masked loading (if not, return
+    # False)
+    attrs = zarr.group(table_path).attrs
+    logger.info(f"ROI table at {table_path} has attrs: {attrs.asdict()}")
+    valid = set(("type", "region", "instance_key")).issubset(attrs.keys())
+    if valid:
+        valid = valid and attrs["type"] == "ngff:region_table"
+        valid = valid and "path" in attrs["region"].keys()
+    if valid:
+        return True
+    else:
+        return False
+
+
+
+ +
+ + +
+ + + + +

+ is_standard_roi_table(table) + +

+ + +
+ +

True if the name of the table contains one of the standard Fractal tables

+

If a table name is well_ROI_table, FOV_ROI_table or contains either of the +two (e.g. registered_FOV_ROI_table), this function returns True.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
table +
+

table name

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + bool + + +
+

bool of whether it's a standard ROI table

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
618
+619
+620
+621
+622
+623
+624
+625
+626
+627
+628
+629
+630
+631
+632
+633
+634
+635
+636
+637
def is_standard_roi_table(table: str) -> bool:
+    """
+    True if the name of the table contains one of the standard Fractal tables
+
+    If a table name is well_ROI_table, FOV_ROI_table or contains either of the
+    two (e.g. registered_FOV_ROI_table), this function returns True.
+
+    Args:
+        table: table name
+
+    Returns:
+        bool of whether it's a standard ROI table
+
+    """
+    if "well_ROI_table" in table:
+        return True
+    elif "FOV_ROI_table" in table:
+        return True
+    else:
+        return False
+
+
+
+ +
+ + +
+ + + + +

+ load_region(data_zyx, region, compute=True, return_as_3D=False) + +

+ + +
+ +

Load a region from a dask array.

+

Can handle both 2D and 3D dask arrays as input and return them as is or +always as a 3D array.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
data_zyx +
+

Dask array (2D or 3D).

+
+

+ + TYPE: + Array + +

+
region +
+

Region to load, tuple of three slices (ZYX).

+
+

+ + TYPE: + tuple[slice, slice, slice] + +

+
compute +
+

Whether to compute the result. If True, returns a numpy +array. If False, returns a dask array.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
return_as_3D +
+

Whether to return a 3D array, even if the input is 2D.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + Union[Array, ndarray] + + +
+

3D array.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
519
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+545
+546
+547
+548
+549
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+560
+561
def load_region(
+    data_zyx: da.Array,
+    region: tuple[slice, slice, slice],
+    compute: bool = True,
+    return_as_3D: bool = False,
+) -> Union[da.Array, np.ndarray]:
+    """
+    Load a region from a dask array.
+
+    Can handle both 2D and 3D dask arrays as input and return them as is or
+    always as a 3D array.
+
+    Args:
+        data_zyx: Dask array (2D or 3D).
+        region: Region to load, tuple of three slices (ZYX).
+        compute: Whether to compute the result. If `True`, returns a numpy
+            array. If `False`, returns a dask array.
+        return_as_3D: Whether to return a 3D array, even if the input is 2D.
+
+    Returns:
+        3D array.
+    """
+
+    if len(region) != 3:
+        raise ValueError(
+            f"In `load_region`, `region` must have three elements "
+            f"(given: {len(region)})."
+        )
+
+    if len(data_zyx.shape) == 3:
+        img = data_zyx[region]
+    elif len(data_zyx.shape) == 2:
+        img = data_zyx[(region[1], region[2])]
+        if return_as_3D:
+            img = np.expand_dims(img, axis=0)
+    else:
+        raise ValueError(
+            f"Shape {data_zyx.shape} not supported for `load_region`"
+        )
+    if compute:
+        return img.compute()
+    else:
+        return img
+
+
+
+ +
+ + +
+ + + + +

+ prepare_FOV_ROI_table(df, metadata=('time')) + +

+ + +
+ +

Prepare an AnnData table for fields-of-view ROIs.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
df +
+

Input dataframe, possibly prepared through +parse_yokogawa_metadata.

+
+

+ + TYPE: + DataFrame + +

+
metadata +
+

Columns of df to be stored (if present) into AnnData table obs.

+
+

+ + TYPE: + tuple[str, ...] + + + DEFAULT: + ('time') + +

+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
def prepare_FOV_ROI_table(
+    df: pd.DataFrame, metadata: tuple[str, ...] = ("time",)
+) -> ad.AnnData:
+    """
+    Prepare an AnnData table for fields-of-view ROIs.
+
+    Args:
+        df:
+            Input dataframe, possibly prepared through
+            `parse_yokogawa_metadata`.
+        metadata:
+            Columns of `df` to be stored (if present) into AnnData table `obs`.
+    """
+
+    # Make a local copy of the dataframe, to avoid SettingWithCopyWarning
+    df = df.copy()
+
+    # Convert DataFrame index to str, to avoid
+    # >> ImplicitModificationWarning: Transforming to str index
+    # when creating AnnData object.
+    # Do this in the beginning to allow concatenation with e.g. time
+    df.index = df.index.astype(str)
+
+    # Obtain box size in physical units
+    df = df.assign(len_x_micrometer=df.x_pixel * df.pixel_size_x)
+    df = df.assign(len_y_micrometer=df.y_pixel * df.pixel_size_y)
+    df = df.assign(len_z_micrometer=df.z_pixel * df.pixel_size_z)
+
+    # Select only the numeric positional columns needed to define ROIs
+    # (to avoid) casting things like the data column to float32
+    # or to use unnecessary columns like bit_depth
+    positional_columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+    ]
+
+    # Assign dtype explicitly, to avoid
+    # >> UserWarning: X converted to numpy array with dtype float64
+    # when creating AnnData object
+    df_roi = df.loc[:, positional_columns].astype(np.float32)
+
+    # Create an AnnData object directly from the DataFrame
+    adata = ad.AnnData(X=df_roi)
+
+    # Reset origin of the FOV ROI table, so that it matches with the well
+    # origin
+    adata = reset_origin(adata)
+
+    # Save any metadata that is specified to the obs df
+    for col in metadata:
+        if col in df:
+            # Cast all metadata to str.
+            # Reason: AnnData Zarr writers don't support all pandas types.
+            # e.g. pandas.core.arrays.datetimes.DatetimeArray can't be written
+            adata.obs[col] = df[col].astype(str)
+
+    # Rename rows and columns: Maintain FOV indices from the dataframe
+    # (they are already enforced to be unique by Pandas and may contain
+    # information for the user, as they are based on the filenames)
+    adata.obs_names = "FOV_" + adata.obs.index
+    adata.var_names = list(map(str, df_roi.columns))
+
+    return adata
+
+
+
+ +
+ + +
+ + + + +

+ prepare_well_ROI_table(df, metadata=('time')) + +

+ + +
+ +

Prepare an AnnData table with a single well ROI.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
df +
+

Input dataframe, possibly prepared through +parse_yokogawa_metadata.

+
+

+ + TYPE: + DataFrame + +

+
metadata +
+

Columns of df to be stored (if present) into AnnData table obs.

+
+

+ + TYPE: + tuple[str, ...] + + + DEFAULT: + ('time') + +

+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
def prepare_well_ROI_table(
+    df: pd.DataFrame, metadata: tuple[str, ...] = ("time",)
+) -> ad.AnnData:
+    """
+    Prepare an AnnData table with a single well ROI.
+
+    Args:
+        df:
+            Input dataframe, possibly prepared through
+            `parse_yokogawa_metadata`.
+        metadata:
+            Columns of `df` to be stored (if present) into AnnData table `obs`.
+    """
+
+    # Make a local copy of the dataframe, to avoid SettingWithCopyWarning
+    df = df.copy()
+
+    # Convert DataFrame index to str, to avoid
+    # >> ImplicitModificationWarning: Transforming to str index
+    # when creating AnnData object.
+    # Do this in the beginning to allow concatenation with e.g. time
+    df.index = df.index.astype(str)
+
+    # Calculate bounding box extents in physical units
+    for mu in ["x", "y", "z"]:
+        # Obtain per-FOV properties in physical units.
+        # NOTE: a FOV ROI is defined here as the interval [min_micrometer,
+        # max_micrometer], with max_micrometer=min_micrometer+len_micrometer
+        min_micrometer = df[f"{mu}_micrometer"]
+        len_micrometer = df[f"{mu}_pixel"] * df[f"pixel_size_{mu}"]
+        max_micrometer = min_micrometer + len_micrometer
+        # Obtain well bounding box, in physical units
+        min_min_micrometer = min_micrometer.min()
+        max_max_micrometer = max_micrometer.max()
+        df[f"{mu}_micrometer"] = min_min_micrometer
+        df[f"len_{mu}_micrometer"] = max_max_micrometer - min_min_micrometer
+
+    # Select only the numeric positional columns needed to define ROIs
+    # (to avoid) casting things like the data column to float32
+    # or to use unnecessary columns like bit_depth
+    positional_columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+    ]
+
+    # Assign dtype explicitly, to avoid
+    # >> UserWarning: X converted to numpy array with dtype float64
+    # when creating AnnData object
+    df_roi = df.iloc[0:1, :].loc[:, positional_columns].astype(np.float32)
+
+    # Create an AnnData object directly from the DataFrame
+    adata = ad.AnnData(X=df_roi)
+
+    # Reset origin of the single-entry well ROI table
+    adata = reset_origin(adata)
+
+    # Save any metadata that is specified to the obs df
+    for col in metadata:
+        if col in df:
+            # Cast all metadata to str.
+            # Reason: AnnData Zarr writers don't support all pandas types.
+            # e.g. pandas.core.arrays.datetimes.DatetimeArray can't be written
+            adata.obs[col] = df[col].astype(str)
+
+    # Rename rows and columns: Maintain FOV indices from the dataframe
+    # (they are already enforced to be unique by Pandas and may contain
+    # information for the user, as they are based on the filenames)
+    adata.obs_names = "well_" + adata.obs.index
+    adata.var_names = list(map(str, df_roi.columns))
+
+    return adata
+
+
+
+ +
+ + +
+ + + + +

+ reset_origin(ROI_table, x_pos='x_micrometer', y_pos='y_micrometer', z_pos='z_micrometer') + +

+ + +
+ +

Return a copy of a ROI table, with shifted-to-zero origin for some columns.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
ROI_table +
+

Original ROI table.

+
+

+ + TYPE: + AnnData + +

+
x_pos +
+

Name of the column with X position of ROIs.

+
+

+ + TYPE: + str + + + DEFAULT: + 'x_micrometer' + +

+
y_pos +
+

Name of the column with Y position of ROIs.

+
+

+ + TYPE: + str + + + DEFAULT: + 'y_micrometer' + +

+
z_pos +
+

Name of the column with Z position of ROIs.

+
+

+ + TYPE: + str + + + DEFAULT: + 'z_micrometer' + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + AnnData + + +
+

A copy of the ROI_table AnnData table, where values of x_pos, +y_pos and z_pos columns have been shifted by their minimum +values.

+
+
+ +
+ Source code in fractal_tasks_core/lib_regions_of_interest.py + 
584
+585
+586
+587
+588
+589
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+600
+601
+602
+603
+604
+605
+606
+607
+608
+609
+610
+611
+612
+613
+614
+615
def reset_origin(
+    ROI_table: ad.AnnData,
+    x_pos: str = "x_micrometer",
+    y_pos: str = "y_micrometer",
+    z_pos: str = "z_micrometer",
+) -> ad.AnnData:
+    """
+    Return a copy of a ROI table, with shifted-to-zero origin for some columns.
+
+    Args:
+        ROI_table: Original ROI table.
+        x_pos: Name of the column with X position of ROIs.
+        y_pos: Name of the column with Y position of ROIs.
+        z_pos: Name of the column with Z position of ROIs.
+
+    Returns:
+        A copy of the `ROI_table` AnnData table, where values of `x_pos`,
+            `y_pos` and `z_pos` columns have been shifted by their minimum
+            values.
+    """
+    new_table = ROI_table.copy()
+
+    origin_x = min(new_table[:, x_pos].X[:, 0])
+    origin_y = min(new_table[:, y_pos].X[:, 0])
+    origin_z = min(new_table[:, z_pos].X[:, 0])
+
+    for FOV in new_table.obs_names:
+        new_table[FOV, x_pos] = new_table[FOV, x_pos].X[0, 0] - origin_x
+        new_table[FOV, y_pos] = new_table[FOV, y_pos].X[0, 0] - origin_y
+        new_table[FOV, z_pos] = new_table[FOV, z_pos].X[0, 0] - origin_z
+
+    return new_table
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_upscale_array/index.html b/reference/fractal_tasks_core/lib_upscale_array/index.html new file mode 100644 index 000000000..3046cc1ba --- /dev/null +++ b/reference/fractal_tasks_core/lib_upscale_array/index.html @@ -0,0 +1,1778 @@ + + + + + + + + + + + + + + + + + + + + + + lib_upscale_array - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_upscale_array

+ +
+ + + + +
+ +

Function to increase the shape of an array by replicating it.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ convert_region_to_low_res(*, highres_region, lowres_shape, highres_shape) + +

+ + +
+ +

Convert a region defined for a high-resolution array to the corresponding +region for a low-resolution array.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
highres_region +
+

A region of the high-resolution array, defined in a +form like (slice(0, 2), slice(1000, 2000), slice(1000, 2000)).

+
+

+ + TYPE: + tuple[slice, ...] + +

+
highres_shape +
+

The shape of the high-resolution array.

+
+

+ + TYPE: + tuple[int, ...] + +

+
lowres_shape +
+

The shape of the low-resolution array.

+
+

+ + TYPE: + tuple[int, ...] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + tuple[slice, ...] + + +
+

Region for low-resolution array.

+
+
+ +
+ Source code in fractal_tasks_core/lib_upscale_array.py + 
133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
def convert_region_to_low_res(
+    *,
+    highres_region: tuple[slice, ...],
+    lowres_shape: tuple[int, ...],
+    highres_shape: tuple[int, ...],
+) -> tuple[slice, ...]:
+    """
+    Convert a region defined for a high-resolution array to the corresponding
+    region for a low-resolution array.
+
+    Args:
+        highres_region: A region of the high-resolution array, defined in a
+            form like `(slice(0, 2), slice(1000, 2000), slice(1000, 2000))`.
+        highres_shape: The shape of the high-resolution array.
+        lowres_shape: The shape of the low-resolution array.
+
+    Returns:
+        Region for low-resolution array.
+    """
+
+    error_msg = (
+        f"Cannot convert {highres_region=}, "
+        f"given {lowres_shape=} and {highres_shape=}."
+    )
+
+    ndim = len(lowres_shape)
+    if len(highres_shape) != ndim:
+        raise ValueError(f"{error_msg} Dimension mismatch.")
+
+    # Loop over dimensions to construct lowres_region, after some relevant
+    # checks
+    lowres_region = []
+    for ind, lowres_size in enumerate(lowres_shape):
+        # Check that the high-resolution size is not smaller than the
+        # low-resolution size
+        highres_size = highres_shape[ind]
+        if highres_size < lowres_size:
+            raise ValueError(
+                f"{error_msg} High-res size smaller than low-res size."
+            )
+        # Check that sizes are commensurate
+        if highres_size % lowres_size > 0:
+            raise ValueError(
+                f"{error_msg} Incommensurable sizes "
+                f"{highres_size=} and {lowres_size=}."
+            )
+        factor = highres_size // lowres_size
+        # Convert old_slice's start/stop attributes
+        old_slice = highres_region[ind]
+        if old_slice.start % factor > 0 or old_slice.stop % factor > 0:
+            raise ValueError(
+                f"{error_msg} Cannot transform {old_slice=} "
+                f"with {factor=}."
+            )
+        new_slice_start = old_slice.start // factor
+        new_slice_stop = old_slice.stop // factor
+        new_slice_step = None
+        # Covert old_slice's step attribute
+        if old_slice.step:
+            if old_slice.step % factor > 0:
+                raise ValueError(
+                    f"{error_msg} Cannot transform {old_slice=} "
+                    f"with {factor=}."
+                )
+            new_slice_step = old_slice.step // factor
+        # Append new slice
+        lowres_region.append(
+            slice(new_slice_start, new_slice_stop, new_slice_step)
+        )
+
+    return tuple(lowres_region)
+
+
+
+ +
+ + +
+ + + + +

+ upscale_array(*, array, target_shape, axis=None, pad_with_zeros=False, warn_if_inhomogeneous=False) + +

+ + +
+ +

Upscale an array along a given list of axis (through repeated application +of np.repeat), to match a target shape.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
array +
+

The array to be upscaled.

+
+

+ + TYPE: + ndarray + +

+
target_shape +
+

The shape of the rescaled array.

+
+

+ + TYPE: + tuple[int, ...] + +

+
axis +
+

The axis along which to upscale the array (if None, then all +axis are used).

+
+

+ + TYPE: + Optional[Sequence[int]] + + + DEFAULT: + None + +

+
pad_with_zeros +
+

If True, pad the upscaled array with zeros to match +target_shape.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
warn_if_inhomogeneous +
+

If True, raise a warning when the conversion +factors are not identical across all dimensions.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + ndarray + + +
+

The upscaled array, with shape target_shape.

+
+
+ +
+ Source code in fractal_tasks_core/lib_upscale_array.py + 
 22
+ 23
+ 24
+ 25
+ 26
+ 27
+ 28
+ 29
+ 30
+ 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
def upscale_array(
+    *,
+    array: np.ndarray,
+    target_shape: tuple[int, ...],
+    axis: Optional[Sequence[int]] = None,
+    pad_with_zeros: bool = False,
+    warn_if_inhomogeneous: bool = False,
+) -> np.ndarray:
+    """
+    Upscale an array along a given list of axis (through repeated application
+    of `np.repeat`), to match a target shape.
+
+    Args:
+        array: The array to be upscaled.
+        target_shape: The shape of the rescaled array.
+        axis: The axis along which to upscale the array (if `None`, then all
+            axis are used).
+        pad_with_zeros: If `True`, pad the upscaled array with zeros to match
+            `target_shape`.
+        warn_if_inhomogeneous: If `True`, raise a warning when the conversion
+            factors are not identical across all dimensions.
+
+    Returns:
+        The upscaled array, with shape `target_shape`.
+    """
+
+    # Default behavior: use all axis
+    if axis is None:
+        axis = list(range(len(target_shape)))
+
+    array_shape = array.shape
+    info = (
+        f"Trying to upscale from {array_shape=} to {target_shape=}, "
+        f"acting on {axis=}."
+    )
+
+    if len(array_shape) != len(target_shape):
+        raise ValueError(f"{info} Dimensions-number mismatch.")
+    if axis == []:
+        raise ValueError(f"{info} Empty axis list")
+    if min(axis) < 0:
+        raise ValueError(f"{info} Negative axis specification not allowed.")
+
+    # Check that upscale is doable
+    for ind, dim in enumerate(array_shape):
+        # Check that array is not larger than target (downscaling)
+        if dim > target_shape[ind]:
+            raise ValueError(
+                f"{info} {ind}-th array dimension is larger than target."
+            )
+        # Check that all relevant axis are included in axis
+        if dim != target_shape[ind] and ind not in axis:
+            raise ValueError(
+                f"{info} {ind}-th array dimension differs from "
+                f"target, but {ind} is not included in "
+                f"{axis=}."
+            )
+
+    # Compute upscaling factors
+    upscale_factors = {}
+    for ax in axis:
+        if (target_shape[ax] % array_shape[ax]) > 0 and not pad_with_zeros:
+            raise ValueError(
+                "Incommensurable upscale attempt, "
+                f"from {array_shape=} to {target_shape=}."
+            )
+        upscale_factors[ax] = target_shape[ax] // array_shape[ax]
+        # Check that this is not downscaling
+        if upscale_factors[ax] < 1:
+            raise ValueError(info)
+    info = f"{info} Upscale factors: {upscale_factors}"
+
+    # Raise a warning if upscaling is non-homogeneous across all axis
+    if warn_if_inhomogeneous:
+        if len(set(upscale_factors.values())) > 1:
+            warnings.warn(f"{info} (inhomogeneous)")
+
+    # Upscale array, via np.repeat
+    upscaled_array = array
+    for ax in axis:
+        upscaled_array = np.repeat(
+            upscaled_array, upscale_factors[ax], axis=ax
+        )
+
+    # Check that final shape is correct
+    if not upscaled_array.shape == target_shape:
+        if pad_with_zeros:
+            pad_width = []
+            for ax in list(range(len(target_shape))):
+                missing = target_shape[ax] - upscaled_array.shape[ax]
+                if missing < 0 or (missing > 0 and ax not in axis):
+                    raise ValueError(
+                        f"{info} " "Something wrong during zero-padding"
+                    )
+                pad_width.append([0, missing])
+            upscaled_array = np.pad(
+                upscaled_array,
+                pad_width=pad_width,
+                mode="constant",
+                constant_values=0,
+            )
+            logging.warning(f"{info} {upscaled_array.shape=}.")
+            logging.warning(
+                f"Padding upscaled_array with zeros with {pad_width=}"
+            )
+        else:
+            raise ValueError(f"{info} {upscaled_array.shape=}.")
+
+    return upscaled_array
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_write/index.html b/reference/fractal_tasks_core/lib_write/index.html new file mode 100644 index 000000000..93ccfe752 --- /dev/null +++ b/reference/fractal_tasks_core/lib_write/index.html @@ -0,0 +1,2556 @@ + + + + + + + + + + + + + + + + + + + + + + lib_write - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_write

+ +
+ + + + +
+ +

Module with custom wrappers of the Zarr API.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _write_elem_with_overwrite(group, key, elem, *, overwrite, logger=None) + +

+ + +
+ +

Wrap anndata.experimental.write_elem, to include overwrite parameter.

+

See docs for the original function +here.

+

This function writes elem to the sub-group key of group. The +overwrite-related expected behavior is:

+
    +
  • if the sub-group does not exist, create it (independently on + overwrite);
    • +
  • if the sub-group already exists and overwrite=True, overwrite the + sub-group;
    • +
  • if the sub-group already exists and overwrite=False, fail.
    • +
+

Note that this version of the wrapper does not include the original +dataset_kwargs parameter.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
group +
+

The group to write to.

+
+

+ + TYPE: + Group + +

+
key +
+

The key to write to in the group. Note that absolute paths will be +written from the root.

+
+

+ + TYPE: + str + +

+
elem +
+

The element to write. Typically an in-memory object, e.g. an +AnnData, pandas dataframe, scipy sparse matrix, etc.

+
+

+ + TYPE: + Any + +

+
overwrite +
+

If True, overwrite the key sub-group (if present); if False +and key sub-group exists, raise an error.

+
+

+ + TYPE: + bool + +

+
logger +
+

The logger to use (if unset, use logging.getLogger(None))

+
+

+ + TYPE: + Optional[Logger] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + OverwriteNotAllowedError + + +
+

If overwrite=False and the sub-group already exists.

+
+
+ +
+ Source code in fractal_tasks_core/lib_write.py + 
124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
def _write_elem_with_overwrite(
+    group: zarr.Group,
+    key: str,
+    elem: Any,
+    *,
+    overwrite: bool,
+    logger: Optional[logging.Logger] = None,
+) -> None:
+    """
+    Wrap `anndata.experimental.write_elem`, to include `overwrite` parameter.
+
+    See docs for the original function
+    [here](https://anndata.readthedocs.io/en/stable/generated/anndata.experimental.write_elem.html).
+
+    This function writes `elem` to the sub-group `key` of `group`. The
+    `overwrite`-related expected behavior is:
+
+    * if the sub-group does not exist, create it (independently on
+      `overwrite`);
+    * if the sub-group already exists and `overwrite=True`, overwrite the
+      sub-group;
+    * if the sub-group already exists and `overwrite=False`, fail.
+
+    Note that this version of the wrapper does not include the original
+    `dataset_kwargs` parameter.
+
+    Args:
+        group:
+            The group to write to.
+        key:
+            The key to write to in the group. Note that absolute paths will be
+            written from the root.
+        elem:
+            The element to write. Typically an in-memory object, e.g. an
+            AnnData, pandas dataframe, scipy sparse matrix, etc.
+        overwrite:
+            If `True`, overwrite the `key` sub-group (if present); if `False`
+            and `key` sub-group exists, raise an error.
+        logger:
+            The logger to use (if unset, use `logging.getLogger(None)`)
+
+    Raises:
+        OverwriteNotAllowedError:
+            If `overwrite=False` and the sub-group already exists.
+    """
+
+    # Set logger
+    if logger is None:
+        logger = logging.getLogger(None)
+
+    if key in set(group.group_keys()):
+        if not overwrite:
+            error_msg = (
+                f"Sub-group '{key}' of group {group.store.path} "
+                f"already exists, but `{overwrite=}`.\n"
+                "Hint: try setting `overwrite=True`."
+            )
+            logger.error(error_msg)
+            raise OverwriteNotAllowedError(error_msg)
+    write_elem(group, key, elem)
+
+
+
+ +
+ + +
+ + + + +

+ open_zarr_group_with_overwrite(path, *, overwrite, logger=None, **open_group_kwargs) + +

+ + +
+ +

Wrap zarr.open_group and add overwrite argument.

+

This wrapper sets mode="w" for overwrite=True and mode="w-" for +overwrite=False.

+

The expected behavior is

+
    +
  • if the group does not exist, create it (independently on overwrite);
    • +
  • if the group already exists and overwrite=True, replace the group with + an empty one;
    • +
  • if the group already exists and overwrite=False, fail.
    • +
+

From the zarr.open_group +docs:

+
    +
  • mode="r" means read only (must exist);
    • +
  • mode="r+" means read/write (must exist);
    • +
  • mode="a" means read/write (create if doesn’t exist);
    • +
  • mode="w" means create (overwrite if exists);
    • +
  • mode="w-" means create (fail if exists).
    • +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
path +
+

Store or path to directory in file system or name of zip file +(zarr.open_group parameter).

+
+

+ + TYPE: + Union[str, MutableMapping] + +

+
overwrite +
+

Determines the mode parameter of zarr.open_group, which is +"w" (if overwrite=True) or "w-" (if overwrite=False).

+
+

+ + TYPE: + bool + +

+
logger +
+

The logger to use (if unset, use logging.getLogger(None))

+
+

+ + TYPE: + Optional[Logger] + + + DEFAULT: + None + +

+
open_group_kwargs +
+

Keyword arguments of zarr.open_group.

+
+

+ + TYPE: + Any + + + DEFAULT: + {} + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + Group + + +
+

The zarr group.

+
+
+ + + + + + + + + + + + + + + + +
RAISESDESCRIPTION
+ + OverwriteNotAllowedError + + +
+

If overwrite=False and the group already exists.

+
+
+ +
+ Source code in fractal_tasks_core/lib_write.py + 
 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
def open_zarr_group_with_overwrite(
+    path: Union[str, MutableMapping],
+    *,
+    overwrite: bool,
+    logger: Optional[logging.Logger] = None,
+    **open_group_kwargs: Any,
+) -> zarr.Group:
+    """
+    Wrap `zarr.open_group` and add `overwrite` argument.
+
+    This wrapper sets `mode="w"` for `overwrite=True` and `mode="w-"` for
+    `overwrite=False`.
+
+    The expected behavior is
+
+
+    * if the group does not exist, create it (independently on `overwrite`);
+    * if the group already exists and `overwrite=True`, replace the group with
+      an empty one;
+    * if the group already exists and `overwrite=False`, fail.
+
+    From the [`zarr.open_group`
+    docs](https://zarr.readthedocs.io/en/stable/api/hierarchy.html#zarr.hierarchy.open_group):
+
+    * `mode="r"` means read only (must exist);
+    * `mode="r+"` means read/write (must exist);
+    * `mode="a"` means read/write (create if doesn’t exist);
+    * `mode="w"` means create (overwrite if exists);
+    * `mode="w-"` means create (fail if exists).
+
+
+    Args:
+        path:
+            Store or path to directory in file system or name of zip file
+            (`zarr.open_group` parameter).
+        overwrite:
+            Determines the `mode` parameter of `zarr.open_group`, which is
+            `"w"` (if `overwrite=True`) or `"w-"` (if `overwrite=False`).
+        logger:
+            The logger to use (if unset, use `logging.getLogger(None)`)
+        open_group_kwargs:
+            Keyword arguments of `zarr.open_group`.
+
+    Returns:
+        The zarr group.
+
+    Raises:
+        OverwriteNotAllowedError:
+            If `overwrite=False` and the group already exists.
+    """
+
+    # Set logger
+    if logger is None:
+        logger = logging.getLogger(None)
+
+    # Set mode for zarr.open_group
+    if overwrite:
+        new_mode = "w"
+    else:
+        new_mode = "w-"
+
+    # Write log about current status
+    logger.info(f"Start open_zarr_group_with_overwrite ({overwrite=}).")
+    try:
+        # Call `zarr.open_group` with `mode="r"`, which fails for missing group
+        current_group = zarr.open_group(path, mode="r")
+        keys = list(current_group.group_keys())
+        logger.info(f"Zarr group {path} already exists, with {keys=}")
+    except GroupNotFoundError:
+        logger.info(f"Zarr group {path} does not exist yet.")
+
+    # Raise warning if we are overriding an existing value of `mode`
+    if "mode" in open_group_kwargs.keys():
+        mode = open_group_kwargs.pop("mode")
+        logger.warning(
+            f"Overriding {mode=} with {new_mode=}, "
+            "in open_zarr_group_with_overwrite"
+        )
+
+    # Call zarr.open_group
+    try:
+        return zarr.open_group(path, mode=new_mode, **open_group_kwargs)
+    except ContainsGroupError:
+        # Re-raise error with custom message and type
+        error_msg = (
+            f"Cannot create zarr group at {path=} with `{overwrite=}` "
+            "(original error: `zarr.errors.ContainsGroupError`).\n"
+            "Hint: try setting `overwrite=True`."
+        )
+        logger.error(error_msg)
+        raise OverwriteNotAllowedError(error_msg)
+
+
+
+ +
+ + +
+ + + + +

+ prepare_label_group(image_group, label_name, overwrite=False, label_attrs=None, logger=None) + +

+ + +
+ +

Set the stage for writing labels to a zarr group

+

This helper function is similar to write_table, in that it prepares the +appropriate zarr groups (labels and the new-label one) and performs +overwrite-dependent checks. At a difference with write_table, this +function does not actually write the label array to the new zarr group; +such writing operation must take place in the actual task function, since +in fractal-tasks-core it is done sequentially on different regions of the +zarr array.

+

What this function does is:

+
    +
  1. Create the labels group, if needed.
    2. +
  2. If overwrite=False, check that the new label does not exist (either in + zarr attributes or as a zarr sub-group).
    3. +
  3. Update the labels attribute of the image group.
    4. +
  4. If label_attrs is set, include this set of attributes in the + new-label zarr group.
    5. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_group +
+

The group to write to.

+
+

+ + TYPE: + Group + +

+
label_name +
+

The name of the new label.

+
+

+ + TYPE: + str + +

+
overwrite +
+

If False, check that the new label does not exist (either in zarr +attributes or as a zarr sub-group); if True propagate parameter +to create_group method, making it overwrite any existing +sub-group with the given name.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
label_attrs +
+

If set, overwrite label_group attributes with label_attrs key/value +pairs.

+
+

+ + TYPE: + Optional[dict[str, Any]] + + + DEFAULT: + None + +

+
logger +
+

The logger to use (if unset, use logging.getLogger(None)).

+
+

+ + TYPE: + Optional[Logger] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + group + + +
+

Zarr group of the new label.

+
+
+ +
+ Source code in fractal_tasks_core/lib_write.py + 
297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
def prepare_label_group(
+    image_group: zarr.Group,
+    label_name: str,
+    overwrite: bool = False,
+    label_attrs: Optional[dict[str, Any]] = None,
+    logger: Optional[logging.Logger] = None,
+) -> zarr.group:
+    """
+    Set the stage for writing labels to a zarr group
+
+    This helper function is similar to `write_table`, in that it prepares the
+    appropriate zarr groups (`labels` and the new-label one) and performs
+    `overwrite`-dependent checks. At a difference with `write_table`, this
+    function does not actually write the label array to the new zarr group;
+    such writing operation must take place in the actual task function, since
+    in fractal-tasks-core it is done sequentially on different `region`s of the
+    zarr array.
+
+    What this function does is:
+
+    1. Create the `labels` group, if needed.
+    2. If `overwrite=False`, check that the new label does not exist (either in
+       zarr attributes or as a zarr sub-group).
+    3. Update the `labels` attribute of the image group.
+    4. If `label_attrs` is set, include this set of attributes in the
+       new-label zarr group.
+
+    Args:
+        image_group:
+            The group to write to.
+        label_name:
+            The name of the new label.
+        overwrite:
+            If `False`, check that the new label does not exist (either in zarr
+            attributes or as a zarr sub-group); if `True` propagate parameter
+            to `create_group` method, making it overwrite any existing
+            sub-group with the given name.
+        label_attrs:
+            If set, overwrite label_group attributes with label_attrs key/value
+            pairs.
+        logger:
+            The logger to use (if unset, use `logging.getLogger(None)`).
+
+    Returns:
+        Zarr group of the new label.
+    """
+
+    # Set logger
+    if logger is None:
+        logger = logging.getLogger(None)
+
+    # Create labels group (if needed) and extract current_labels
+    if "labels" not in set(image_group.group_keys()):
+        labels_group = image_group.create_group("labels", overwrite=False)
+    else:
+        labels_group = image_group["labels"]
+    current_labels = labels_group.attrs.asdict().get("labels", [])
+
+    # If overwrite=False, check that the new label does not exist (either as a
+    # zarr sub-group or as part of the zarr-group attributes)
+    if not overwrite:
+        if label_name in set(labels_group.group_keys()):
+            error_msg = (
+                f"Sub-group '{label_name}' of group {image_group.store.path} "
+                f"already exists, but `{overwrite=}`.\n"
+                "Hint: try setting `overwrite=True`."
+            )
+            logger.error(error_msg)
+            raise OverwriteNotAllowedError(error_msg)
+        if label_name in current_labels:
+            error_msg = (
+                f"Item '{label_name}' already exists in `labels` attribute of "
+                f"group {image_group.store.path}, but `{overwrite=}`.\n"
+                "Hint: try setting `overwrite=True`."
+            )
+            logger.error(error_msg)
+            raise OverwriteNotAllowedError(error_msg)
+
+    # Update the `labels` metadata of the image group, if needed
+    if label_name not in current_labels:
+        new_labels = current_labels + [label_name]
+        labels_group.attrs["labels"] = new_labels
+
+    # Define new-label group
+    label_group = labels_group.create_group(label_name, overwrite=overwrite)
+
+    # Optionally update attributes of the new-table zarr group
+    if label_attrs is not None:
+        # Overwrite label_group attributes with label_attrs key/value pairs
+        label_group.attrs.put(label_attrs)
+
+    return label_group
+
+
+
+ +
+ + +
+ + + + +

+ write_table(image_group, table_name, table, overwrite=False, table_attrs=None, logger=None) + +

+ + +
+ +

Handle multiple options for writing an AnnData table to a zarr group.

+
    +
  1. Create the tables group, if needed.
    2. +
  2. If overwrite=False, check that the new table does not exist (either in + zarr attributes or as a zarr sub-group).
    3. +
  3. Call the _write_elem_with_overwrite wrapper with the appropriate + overwrite parameter.
    4. +
  4. Update the tables attribute of the image group.
    5. +
  5. If table_attrs is set, include this set of attributes in the + new-table zarr group. One intended usage, within fractal-tasks-core, is + to comply with a proposed change to the OME-NGFF specs + (https://github.com/ome/ngff/pull/64).
    6. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_group +
+

The group to write to.

+
+

+ + TYPE: + Group + +

+
table_name +
+

The name of the new table.

+
+

+ + TYPE: + str + +

+
table +
+

The AnnData table to write.

+
+

+ + TYPE: + AnnData + +

+
overwrite +
+

If False, check that the new table does not exist (either as a +zarr sub-group or as part of the zarr-group attributes). In all +cases, propagate parameter to _write_elem_with_overwrite, to +determine the behavior in case of an existing sub-group named as +table_name.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
table_attrs +
+

If set, overwrite table_group attributes with table_attrs key/value +pairs.

+
+

+ + TYPE: + Optional[dict[str, Any]] + + + DEFAULT: + None + +

+
logger +
+

The logger to use (if unset, use logging.getLogger(None)).

+
+

+ + TYPE: + Optional[Logger] + + + DEFAULT: + None + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + group + + +
+

Zarr group of the new table.

+
+
+ +
+ Source code in fractal_tasks_core/lib_write.py + 
186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
def write_table(
+    image_group: zarr.Group,
+    table_name: str,
+    table: ad.AnnData,
+    overwrite: bool = False,
+    table_attrs: Optional[dict[str, Any]] = None,
+    logger: Optional[logging.Logger] = None,
+) -> zarr.group:
+    """
+    Handle multiple options for writing an AnnData table to a zarr group.
+
+    1. Create the `tables` group, if needed.
+    2. If `overwrite=False`, check that the new table does not exist (either in
+       zarr attributes or as a zarr sub-group).
+    3. Call the `_write_elem_with_overwrite` wrapper with the appropriate
+       `overwrite` parameter.
+    4. Update the `tables` attribute of the image group.
+    5. If `table_attrs` is set, include this set of attributes in the
+       new-table zarr group. One intended usage, within fractal-tasks-core, is
+       to comply with a proposed change to the OME-NGFF specs
+       (https://github.com/ome/ngff/pull/64).
+
+    Args:
+        image_group:
+            The group to write to.
+        table_name:
+            The name of the new table.
+        table:
+            The AnnData table to write.
+        overwrite:
+            If `False`, check that the new table does not exist (either as a
+            zarr sub-group or as part of the zarr-group attributes). In all
+            cases, propagate parameter to `_write_elem_with_overwrite`, to
+            determine the behavior in case of an existing sub-group named as
+            `table_name`.
+        table_attrs:
+            If set, overwrite table_group attributes with table_attrs key/value
+            pairs.
+        logger:
+            The logger to use (if unset, use `logging.getLogger(None)`).
+
+    Returns:
+        Zarr group of the new table.
+    """
+
+    # Set logger
+    if logger is None:
+        logger = logging.getLogger(None)
+
+    # Create tables group (if needed) and extract current_tables
+    if "tables" not in set(image_group.group_keys()):
+        tables_group = image_group.create_group("tables", overwrite=False)
+    else:
+        tables_group = image_group["tables"]
+    current_tables = tables_group.attrs.asdict().get("tables", [])
+
+    # If overwrite=False, check that the new table does not exist (either as a
+    # zarr sub-group or as part of the zarr-group attributes)
+    if not overwrite:
+        if table_name in set(tables_group.group_keys()):
+            error_msg = (
+                f"Sub-group '{table_name}' of group {image_group.store.path} "
+                f"already exists, but `{overwrite=}`.\n"
+                "Hint: try setting `overwrite=True`."
+            )
+            logger.error(error_msg)
+            raise OverwriteNotAllowedError(error_msg)
+        if table_name in current_tables:
+            error_msg = (
+                f"Item '{table_name}' already exists in `tables` attribute of "
+                f"group {image_group.store.path}, but `{overwrite=}`.\n"
+                "Hint: try setting `overwrite=True`."
+            )
+            logger.error(error_msg)
+            raise OverwriteNotAllowedError(error_msg)
+
+    # If it's all OK, proceed and write the table
+    _write_elem_with_overwrite(
+        tables_group,
+        table_name,
+        table,
+        overwrite=overwrite,
+    )
+    table_group = tables_group[table_name]
+
+    # Update the `tables` metadata of the image group, if needed
+    if table_name not in current_tables:
+        new_tables = current_tables + [table_name]
+        tables_group.attrs["tables"] = new_tables
+
+    # Optionally update attributes of the new-table zarr group
+    if table_attrs is not None:
+        if table_attrs.get("type") == "ngff:region_table":
+            # Verify whether we comply with a proposed change to the OME-NGFF
+            # table specs (https://github.com/ome/ngff/pull/64)
+            try:
+                table_attrs["instance_key"]
+                table_attrs["region"]["path"]
+            except KeyError as e:
+                logger.warning(
+                    f"The table_attrs parameter of write_elem has "
+                    "type='ngff:region_table' but does not comply with the "
+                    "proposed table specs. "
+                    f"Original error: KeyError: {str(e)}"
+                )
+        # Update table_group attributes with table_attrs key/value pairs
+        table_group.attrs.update(**table_attrs)
+
+    return table_group
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/lib_zattrs_utils/index.html b/reference/fractal_tasks_core/lib_zattrs_utils/index.html new file mode 100644 index 000000000..51967e099 --- /dev/null +++ b/reference/fractal_tasks_core/lib_zattrs_utils/index.html @@ -0,0 +1,1516 @@ + + + + + + + + + + + + + + + + + + + + + + lib_zattrs_utils - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

lib_zattrs_utils

+ +
+ + + + +
+ +

Helper functions for operations on OME-NGFF metadata.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ get_table_path_dict(input_path, component) + +

+ + +
+ +

Compile dictionary of (table name, table path) key/value pairs.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_path +
+

Path to the parent folder of a plate zarr group (e.g. +/some/path/).

+
+

+ + TYPE: + Path + +

+
component +
+

Path (relative to input_path) to an image zarr group (e.g. +plate.zarr/B/03/0).

+
+

+ + TYPE: + str + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[str, str] + + +
+

Dictionary with table names as keys and table paths as values. If +tables Zarr group is missing, or if it does not have a tables +key, then return an empty dictionary.

+
+
+ +
+ Source code in fractal_tasks_core/lib_zattrs_utils.py + 
 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
def get_table_path_dict(input_path: Path, component: str) -> dict[str, str]:
+    """
+    Compile dictionary of (table name, table path) key/value pairs.
+
+
+    Args:
+        input_path:
+            Path to the parent folder of a plate zarr group (e.g.
+            `/some/path/`).
+        component:
+            Path (relative to `input_path`) to an image zarr group (e.g.
+            `plate.zarr/B/03/0`).
+
+    Returns:
+        Dictionary with table names as keys and table paths as values. If
+            `tables` Zarr group is missing, or if it does not have a `tables`
+            key, then return an empty dictionary.
+    """
+
+    try:
+        tables_group = zarr.open_group(f"{input_path / component}/tables", "r")
+        table_list = tables_group.attrs["tables"]
+    except (zarr.errors.GroupNotFoundError, KeyError):
+        table_list = []
+
+    table_path_dict = {}
+    for table in table_list:
+        table_path_dict[table] = f"{input_path / component}/tables/{table}"
+
+    return table_path_dict
+
+
+
+ +
+ + +
+ + + + +

+ rescale_datasets(*, datasets, coarsening_xy, reference_level, remove_channel_axis=False) + +

+ + +
+ +

Given a set of datasets (as per OME-NGFF specs), update their "scale" +transformations in the YX directions by including a prefactor +(coarsening_xy**reference_level).

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
datasets +
+

list of datasets (as per OME-NGFF specs).

+
+

+ + TYPE: + list[dict] + +

+
coarsening_xy +
+

linear coarsening factor between subsequent levels.

+
+

+ + TYPE: + int + +

+
reference_level +
+

TBD

+
+

+ + TYPE: + int + +

+
remove_channel_axis +
+

If True, remove the first item of all scale +transformations.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ +
+ Source code in fractal_tasks_core/lib_zattrs_utils.py + 
25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
def rescale_datasets(
+    *,
+    datasets: list[dict],
+    coarsening_xy: int,
+    reference_level: int,
+    remove_channel_axis: bool = False,
+) -> list[dict]:
+    """
+    Given a set of datasets (as per OME-NGFF specs), update their "scale"
+    transformations in the YX directions by including a prefactor
+    (coarsening_xy**reference_level).
+
+    Args:
+        datasets: list of datasets (as per OME-NGFF specs).
+        coarsening_xy: linear coarsening factor between subsequent levels.
+        reference_level: TBD
+        remove_channel_axis: If `True`, remove the first item of all `scale`
+            transformations.
+    """
+
+    # Construct rescaled datasets
+    new_datasets = []
+    for ds in datasets:
+        new_ds = {}
+
+        # Copy all keys that are not coordinateTransformations (e.g. path)
+        for key in ds.keys():
+            if key != "coordinateTransformations":
+                new_ds[key] = ds[key]
+
+        # Update coordinateTransformations
+        old_transformations = ds["coordinateTransformations"]
+        new_transformations = []
+        for t in old_transformations:
+            if t["type"] == "scale":
+                new_t: dict[str, Any] = t.copy()
+                # Rescale last two dimensions (that is, Y and X)
+                prefactor = coarsening_xy**reference_level
+                new_t["scale"][-2] = new_t["scale"][-2] * prefactor
+                new_t["scale"][-1] = new_t["scale"][-1] * prefactor
+                if remove_channel_axis:
+                    new_t["scale"].pop(0)
+                new_transformations.append(new_t)
+            else:
+                new_transformations.append(t)
+        new_ds["coordinateTransformations"] = new_transformations
+        new_datasets.append(new_ds)
+
+    return new_datasets
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/_utils/index.html b/reference/fractal_tasks_core/tasks/_utils/index.html new file mode 100644 index 000000000..bf2d2e1d7 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/_utils/index.html @@ -0,0 +1,1465 @@ + + + + + + + + + + + + + + + + + + + + + + _utils - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

_utils

+ +
+ + + + +
+ +

Standard input/output interface for tasks.

+ + + +
+ + + + + + + + +
+ + + + +

+ TaskParameterEncoder + + +

+ + +
+

+ Bases: JSONEncoder

+ + +

Custom JSONEncoder that transforms Path objects to strings.

+ +
+ Source code in fractal_tasks_core/tasks/_utils.py + 
23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
class TaskParameterEncoder(JSONEncoder):
+    """
+    Custom JSONEncoder that transforms Path objects to strings.
+    """
+
+    def default(self, value):
+        """
+        Subclass implementation of `default`, to serialize Path objects as
+        strings.
+        """
+        if isinstance(value, Path):
+            return value.as_posix()
+        return JSONEncoder.default(self, value)
+
+
+ + + +
+ + + + + + + + + + +
+ + + + +

+ default(value) + +

+ + +
+ +

Subclass implementation of default, to serialize Path objects as +strings.

+ +
+ Source code in fractal_tasks_core/tasks/_utils.py + 
28
+29
+30
+31
+32
+33
+34
+35
def default(self, value):
+    """
+    Subclass implementation of `default`, to serialize Path objects as
+    strings.
+    """
+    if isinstance(value, Path):
+        return value.as_posix()
+    return JSONEncoder.default(self, value)
+
+
+
+ +
+ + + +
+ +
+ +
+ + + +
+ + + + +

+ run_fractal_task(*, task_function, logger_name=None) + +

+ + +
+ +

Implement standard task interface and call task_function.

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
task_function +
+

the callable function that runs the task.

+
+

+ + TYPE: + Callable + +

+
logger_name +
+

TBD

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/tasks/_utils.py + 
38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
def run_fractal_task(
+    *,
+    task_function: Callable,
+    logger_name: Optional[str] = None,
+):
+    """
+    Implement standard task interface and call task_function.
+
+    Args:
+        task_function: the callable function that runs the task.
+        logger_name: TBD
+    """
+
+    # Parse `-j` and `--metadata-out` arguments
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-j", "--json", help="Read parameters from json file", required=True
+    )
+    parser.add_argument(
+        "--metadata-out",
+        help="Output file to redirect serialised returned data",
+        required=True,
+    )
+    args = parser.parse_args()
+
+    # Set logger
+    logger = logging.getLogger(logger_name)
+
+    # Preliminary check
+    if Path(args.metadata_out).exists():
+        logger.error(
+            f"Output file {args.metadata_out} already exists. Terminating"
+        )
+        exit(1)
+
+    # Read parameters dictionary
+    with open(args.json, "r") as f:
+        pars = json.load(f)
+
+    # Run task
+    logger.info(f"START {task_function.__name__} task")
+    metadata_update = task_function(**pars)
+    logger.info(f"END {task_function.__name__} task")
+
+    # Write output metadata to file, with custom JSON encoder
+    with open(args.metadata_out, "w") as fout:
+        json.dump(metadata_update, fout, cls=TaskParameterEncoder, indent=2)
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/index.html b/reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/index.html new file mode 100644 index 000000000..70d5ed472 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/index.html @@ -0,0 +1,1888 @@ + + + + + + + + + + + + + + + + + + + + + + apply_registration_to_ROI_tables - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

apply_registration_to_ROI_tables

+ +
+ + + + +
+ +

Applies the multiplexing translation to all ROI tables

+ + + +
+ + + + + + + + + + +
+ + + + +

+ add_zero_translation_columns(ad_table) + +

+ + +
+ +

Add three zero-filled columns (translation_{x,y,z}) to an AnnData table.

+ +
+ Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py + 
179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
def add_zero_translation_columns(ad_table: ad.AnnData):
+    """
+    Add three zero-filled columns (`translation_{x,y,z}`) to an AnnData table.
+    """
+    columns = ["translation_z", "translation_y", "translation_x"]
+    if ad_table.var.index.isin(columns).any().any():
+        raise ValueError(
+            "The roi table already contains translation columns. Did you "
+            "enter a wrong reference cycle?"
+        )
+    df = pd.DataFrame(np.zeros([len(ad_table), 3]), columns=columns)
+    df.index = ad_table.obs.index
+    ad_new = ad.concat([ad_table, ad.AnnData(df)], axis=1)
+    return ad_new
+
+
+
+ +
+ + +
+ + + + +

+ apply_registration_to_ROI_tables(*, input_paths, output_path, component, metadata, roi_table='FOV_ROI_table', reference_cycle=0, new_roi_table=None) + +

+ + +
+ +

Applies pre-calculated registration to ROI tables.

+

Apply pre-calculated registration such that resulting ROIs contain +the consensus align region between all cycles.

+

Parallelization level: well

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
roi_table +
+

Name of the ROI table over which the task loops to +calculate the registration. Examples: FOV_ROI_table => loop over +the field of views, well_ROI_table => process the whole well as +one image.

+
+

+ + TYPE: + str + + + DEFAULT: + 'FOV_ROI_table' + +

+
reference_cycle +
+

Which cycle to register against. Defaults to 0, +which is the first OME-Zarr image in the well, usually the first +cycle that was provided

+
+

+ + TYPE: + int + + + DEFAULT: + 0 + +

+
new_roi_table +
+

Optional name for the new, registered ROI table. If no +name is given, it will default to "registered_" + roi_table

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py + 
 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
@validate_arguments
+def apply_registration_to_ROI_tables(
+    *,
+    # Fractal arguments
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments
+    roi_table: str = "FOV_ROI_table",
+    reference_cycle: int = 0,
+    new_roi_table: Optional[str] = None,
+) -> dict[str, Any]:
+    """
+    Applies pre-calculated registration to ROI tables.
+
+    Apply pre-calculated registration such that resulting ROIs contain
+    the consensus align region between all cycles.
+
+    Parallelization level: well
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        roi_table: Name of the ROI table over which the task loops to
+            calculate the registration. Examples: `FOV_ROI_table` => loop over
+            the field of views, `well_ROI_table` => process the whole well as
+            one image.
+        reference_cycle: Which cycle to register against. Defaults to 0,
+            which is the first OME-Zarr image in the well, usually the first
+            cycle that was provided
+        new_roi_table: Optional name for the new, registered ROI table. If no
+            name is given, it will default to "registered_" + `roi_table`
+
+    """
+    if not new_roi_table:
+        new_roi_table = "registered_" + roi_table
+    logger.info(
+        f"Running for {input_paths=}, {component=}. \n"
+        f"Applyg translation registration to {roi_table=} and storing it as "
+        f"{new_roi_table=}."
+    )
+
+    well_zarr = f"{input_paths[0]}/{component}"
+    ngff_well_meta = load_NgffWellMeta(well_zarr)
+    acquisition_dict = ngff_well_meta.get_acquisition_paths()
+    logger.info(
+        "Calculating common registration for the following cycles: "
+        f"{acquisition_dict}"
+    )
+
+    # TODO: Allow a filter on which acquisitions should get processed?
+
+    # Collect all the ROI tables
+    roi_tables = {}
+    for acq in acquisition_dict.keys():
+        acq_path = acquisition_dict[acq]
+        curr_ROI_table = ad.read_zarr(
+            f"{well_zarr}/{acq_path}/tables/{roi_table}"
+        )
+
+        # For reference_cycle acquisition, handle the fact that it doesn't
+        # have the shifts
+        if acq == reference_cycle:
+            curr_ROI_table = add_zero_translation_columns(curr_ROI_table)
+        # Check for valid ROI tables
+        are_ROI_table_columns_valid(table=curr_ROI_table)
+        translation_columns = [
+            "translation_z",
+            "translation_y",
+            "translation_x",
+        ]
+        if curr_ROI_table.var.index.isin(translation_columns).sum() != 3:
+            raise ValueError(
+                f"Cycle {acq}'s {roi_table} does not contain the "
+                f"translation columns {translation_columns} necessary to use "
+                "this task."
+            )
+        roi_tables[acq] = curr_ROI_table
+
+    # Check that all acquisitions have the same ROIs
+    rois = roi_tables[reference_cycle].obs.index
+    for acq, acq_roi_table in roi_tables.items():
+        if not (acq_roi_table.obs.index == rois).all():
+            raise ValueError(
+                f"Acquisition {acq} does not contain the same ROIs as the "
+                f"reference acquisition {reference_cycle}:\n"
+                f"{acq}: {acq_roi_table.obs.index}\n"
+                f"{reference_cycle}: {rois}"
+            )
+
+    roi_table_dfs = [
+        roi_table.to_df().loc[:, translation_columns]
+        for roi_table in roi_tables.values()
+    ]
+    logger.info("Calculating min & max translation across cycles.")
+    max_df, min_df = calculate_min_max_across_dfs(roi_table_dfs)
+    shifted_rois = {}
+    # Loop over acquisitions
+    for acq in acquisition_dict.keys():
+        shifted_rois[acq] = apply_registration_to_single_ROI_table(
+            roi_tables[acq], max_df, min_df
+        )
+
+        # TODO: Drop translation columns from this table?
+
+        logger.info(
+            f"Write the registered ROI table {new_roi_table} for {acq=}"
+        )
+        # TODO rewrite this block through `write_table` (ref PR #499)
+        # Save the shifted ROI tables as a new tables per acquisition
+        group_tables = zarr.group(f"{well_zarr}/{acq}/tables/")
+        write_elem(group_tables, new_roi_table, shifted_rois[acq])
+        # Update list of available tables & its metadata
+        current_tables = group_tables.attrs.asdict().get("tables") or []
+        new_tables = current_tables + [new_roi_table]
+        group_tables.attrs["tables"] = new_tables
+        group_tables[roi_table].attrs["type"] = "ngff:region_table"
+
+    # TODO: Optionally apply registration to other tables as well?
+    # e.g. to well_ROI_table based on FOV_ROI_table
+    # => out of scope for the initial task, apply registration separately
+    # to each table
+    # Easiest implementation: Apply average shift calculcated here to other
+    # ROIs. From many to 1 (e.g. FOV => well) => average shift, but crop len
+    # From well to many (e.g. well to FOVs) => average shift, crop len by that
+    # amount
+    # Many to many (FOVs to organoids) => tricky because of matching
+
+    return {}
+
+
+
+ +
+ + +
+ + + + +

+ apply_registration_to_single_ROI_table(roi_table, max_df, min_df) + +

+ + +
+ +

Applies the registration to a ROI table

+

Calculates the new position as: p = position + max(shift, 0) - own_shift +Calculates the new len as: l = len - max(shift, 0) + min(shift, 0)

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
roi_table +
+

AnnData table which contains a Fractal ROI table. +Rows are ROIs

+
+

+ + TYPE: + AnnData + +

+
max_df +
+

Max translation shift in z, y, x for each ROI. Rows are ROIs, +columns are translation_z, translation_y, translation_x

+
+

+ + TYPE: + DataFrame + +

+
min_df +
+

Min translation shift in z, y, x for each ROI. Rows are ROIs, +columns are translation_z, translation_y, translation_x

+
+

+ + TYPE: + DataFrame + +

+
+

Returns: + ROI table where all ROIs are registered to the smallest common area + across all cycles.

+ +
+ Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py + 
220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
def apply_registration_to_single_ROI_table(
+    roi_table: ad.AnnData,
+    max_df: pd.DataFrame,
+    min_df: pd.DataFrame,
+) -> ad.AnnData:
+    """
+    Applies the registration to a ROI table
+
+    Calculates the new position as: p = position + max(shift, 0) - own_shift
+    Calculates the new len as: l = len - max(shift, 0) + min(shift, 0)
+
+    Args:
+        roi_table: AnnData table which contains a Fractal ROI table.
+            Rows are ROIs
+        max_df: Max translation shift in z, y, x for each ROI. Rows are ROIs,
+            columns are translation_z, translation_y, translation_x
+        min_df: Min translation shift in z, y, x for each ROI. Rows are ROIs,
+            columns are translation_z, translation_y, translation_x
+    Returns:
+        ROI table where all ROIs are registered to the smallest common area
+        across all cycles.
+    """
+    roi_table = copy.deepcopy(roi_table)
+    rois = roi_table.obs.index
+    if (rois != max_df.index).all() or (rois != min_df.index).all():
+        raise ValueError(
+            "ROI table and max & min translation need to contain the same "
+            f"ROIS, but they were {rois=}, {max_df.index=}, {min_df.index=}"
+        )
+
+    for roi in rois:
+        roi_table[[roi], ["z_micrometer"]] = (
+            roi_table[[roi], ["z_micrometer"]].X
+            + float(max_df.loc[roi, "translation_z"])
+            - roi_table[[roi], ["translation_z"]].X
+        )
+        roi_table[[roi], ["y_micrometer"]] = (
+            roi_table[[roi], ["y_micrometer"]].X
+            + float(max_df.loc[roi, "translation_y"])
+            - roi_table[[roi], ["translation_y"]].X
+        )
+        roi_table[[roi], ["x_micrometer"]] = (
+            roi_table[[roi], ["x_micrometer"]].X
+            + float(max_df.loc[roi, "translation_x"])
+            - roi_table[[roi], ["translation_x"]].X
+        )
+        # This calculation only works if all ROIs are the same size initially!
+        roi_table[[roi], ["len_z_micrometer"]] = (
+            roi_table[[roi], ["len_z_micrometer"]].X
+            - float(max_df.loc[roi, "translation_z"])
+            + float(min_df.loc[roi, "translation_z"])
+        )
+        roi_table[[roi], ["len_y_micrometer"]] = (
+            roi_table[[roi], ["len_y_micrometer"]].X
+            - float(max_df.loc[roi, "translation_y"])
+            + float(min_df.loc[roi, "translation_y"])
+        )
+        roi_table[[roi], ["len_x_micrometer"]] = (
+            roi_table[[roi], ["len_x_micrometer"]].X
+            - float(max_df.loc[roi, "translation_x"])
+            + float(min_df.loc[roi, "translation_x"])
+        )
+    return roi_table
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/apply_registration_to_image/index.html b/reference/fractal_tasks_core/tasks/apply_registration_to_image/index.html new file mode 100644 index 000000000..54ff60505 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/apply_registration_to_image/index.html @@ -0,0 +1,2153 @@ + + + + + + + + + + + + + + + + + + + + + + apply_registration_to_image - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

apply_registration_to_image

+ +
+ + + + +
+ +

Calculates translation for 2D image-based registration

+ + + +
+ + + + + + + + + + +
+ + + + +

+ apply_registration_to_image(*, input_paths, output_path, component, metadata, registered_roi_table, reference_cycle='0', overwrite_input=True) + +

+ + +
+ +

Apply registration to images by using a registered ROI table

+

This task consists of 4 parts:

+
    +
  1. Mask all regions in images that are not available in the +registered ROI table and store each cycle aligned to the +reference_cycle (by looping over ROIs).
    2. +
  2. Do the same for all label images.
    3. +
  3. Copy all tables from the non-aligned image to the aligned image +(currently only works well if the only tables are well & FOV ROI tables +(registered and original). Not implemented for measurement tables and +other ROI tables).
    4. +
  4. Clean up: Delete the old, non-aligned image and rename the new, +aligned image to take over its place.
    5. +
+

Parallelization level: image

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
registered_roi_table +
+

Name of the ROI table which has been registered +and will be applied to mask and shift the images. +Examples: registered_FOV_ROI_table => loop over the field of +views, registered_well_ROI_table => process the whole well as +one image.

+
+

+ + TYPE: + str + +

+
reference_cycle +
+

Which cycle to register against. Defaults to 0, +which is the first OME-Zarr image in the well, usually the first +cycle that was provided

+
+

+ + TYPE: + str + + + DEFAULT: + '0' + +

+
overwrite_input +
+

Whether the old image data should be replaced with the +newly registered image data. Currently only implemented for +overwrite_input=True.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
+ +
+ Source code in fractal_tasks_core/tasks/apply_registration_to_image.py + 
 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
@validate_arguments
+def apply_registration_to_image(
+    *,
+    # Fractal arguments
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments
+    registered_roi_table: str,
+    reference_cycle: str = "0",
+    overwrite_input: bool = True,
+):
+    """
+    Apply registration to images by using a registered ROI table
+
+    This task consists of 4 parts:
+
+    1. Mask all regions in images that are not available in the
+    registered ROI table and store each cycle aligned to the
+    reference_cycle (by looping over ROIs).
+    2. Do the same for all label images.
+    3. Copy all tables from the non-aligned image to the aligned image
+    (currently only works well if the only tables are well & FOV ROI tables
+    (registered and original). Not implemented for measurement tables and
+    other ROI tables).
+    4. Clean up: Delete the old, non-aligned image and rename the new,
+    aligned image to take over its place.
+
+    Parallelization level: image
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        registered_roi_table: Name of the ROI table which has been registered
+            and will be applied to mask and shift the images.
+            Examples: `registered_FOV_ROI_table` => loop over the field of
+            views, `registered_well_ROI_table` => process the whole well as
+            one image.
+        reference_cycle: Which cycle to register against. Defaults to 0,
+            which is the first OME-Zarr image in the well, usually the first
+            cycle that was provided
+        overwrite_input: Whether the old image data should be replaced with the
+            newly registered image data. Currently only implemented for
+            `overwrite_input=True`.
+
+    """
+    logger.info(component)
+    if not overwrite_input:
+        raise NotImplementedError(
+            "This task is only implemented for the overwrite_input version"
+        )
+    logger.info(
+        f"Running `apply_registration_to_image` on {input_paths=}, "
+        f"{component=}, {registered_roi_table=} and {reference_cycle=}. "
+        f"Using {overwrite_input=}"
+    )
+
+    input_path = Path(input_paths[0])
+    new_component = "/".join(
+        component.split("/")[:-1] + [component.split("/")[-1] + "_registered"]
+    )
+    reference_component = "/".join(
+        component.split("/")[:-1] + [reference_cycle]
+    )
+
+    ROI_table_ref = ad.read_zarr(
+        f"{input_path / reference_component}/tables/{registered_roi_table}"
+    )
+    ROI_table_cycle = ad.read_zarr(
+        f"{input_path / component}/tables/{registered_roi_table}"
+    )
+
+    ngff_image_meta = load_NgffImageMeta(str(input_path / component))
+    coarsening_xy = ngff_image_meta.coarsening_xy
+    num_levels = ngff_image_meta.num_levels
+
+    ####################
+    # Process images
+    ####################
+    logger.info("Write the registered Zarr image to disk")
+    write_registered_zarr(
+        input_path=input_path,
+        component=component,
+        new_component=new_component,
+        ROI_table=ROI_table_cycle,
+        ROI_table_ref=ROI_table_ref,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        aggregation_function=np.mean,
+    )
+
+    ####################
+    # Process labels
+    ####################
+    try:
+        labels_group = zarr.open_group(f"{input_path / component}/labels", "r")
+        label_list = labels_group.attrs["labels"]
+    except (zarr.errors.GroupNotFoundError, KeyError):
+        label_list = []
+
+    if label_list:
+        logger.info(f"Processing the label images: {label_list}")
+        labels_group = zarr.group(f"{input_path / new_component}/labels")
+        labels_group.attrs["labels"] = label_list
+
+        for label in label_list:
+            label_component = f"{component}/labels/{label}"
+            label_component_new = f"{new_component}/labels/{label}"
+            write_registered_zarr(
+                input_path=input_path,
+                component=label_component,
+                new_component=label_component_new,
+                ROI_table=ROI_table_cycle,
+                ROI_table_ref=ROI_table_ref,
+                num_levels=num_levels,
+                coarsening_xy=coarsening_xy,
+                aggregation_function=np.max,
+            )
+
+    ####################
+    # Copy tables
+    # 1. Copy all standard ROI tables from cycle 0.
+    # 2. Copy all tables that aren't standard ROI tables from the given cycle
+    ####################
+    table_dict_reference = get_table_path_dict(input_path, reference_component)
+    table_dict_component = get_table_path_dict(input_path, component)
+
+    table_dict = {}
+    # Define which table should get copied:
+    for table in table_dict_reference:
+        if is_standard_roi_table(table):
+            table_dict[table] = table_dict_reference[table]
+    for table in table_dict_component:
+        if not is_standard_roi_table(table):
+            if reference_component != component:
+                logger.warning(
+                    f"{component} contained a table that is not a standard "
+                    "ROI table. The `Apply Registration To Image task` is "
+                    "best used before additional tables are generated. It "
+                    f"will copy the {table} from this cycle without applying "
+                    f"any transformations. This will work well if {table} "
+                    f"contains measurements. But if {table} is a custom ROI "
+                    "table coming from another task, the transformation is "
+                    "not applied and it will not match with the registered "
+                    "image anymore"
+                )
+            table_dict[table] = table_dict_component[table]
+
+    if table_dict:
+        logger.info(f"Processing the tables: {table_dict}")
+        new_tables_group = zarr.group(f"{input_path / new_component}/tables")
+        new_tables_group.attrs["tables"] = list(table_dict.keys())
+
+        for table in table_dict.keys():
+            logger.info(f"Copying table: {table}")
+            # Write the Zarr table
+            curr_table = ad.read_zarr(table_dict[table])
+            write_elem(new_tables_group, table, curr_table)
+            # Get the relevant metadata of the Zarr table & add it
+            # See issue #516 for the need for this workaround
+            try:
+                old_table_group = zarr.open_group(table_dict[table], mode="r")
+            except zarr.errors.GroupNotFoundError:
+                time.sleep(5)
+                old_table_group = zarr.open_group(table_dict[table], mode="r")
+            new_table_group = zarr.open_group(
+                f"{input_path / new_component}/tables/{table}"
+            )
+            new_table_group.attrs.put(old_table_group.attrs.asdict())
+
+    ####################
+    # Clean up Zarr file
+    ####################
+    if overwrite_input:
+        logger.info(
+            "Replace original zarr image with the newly created Zarr image"
+        )
+        # Potential for race conditions: Every cycle reads the
+        # reference cycle, but the reference cycle also gets modified
+        # See issue #516 for the details
+        os.rename(f"{input_path / component}", f"{input_path / component}_tmp")
+        os.rename(f"{input_path / new_component}", f"{input_path / component}")
+        shutil.rmtree(f"{input_path / component}_tmp")
+    else:
+        raise NotImplementedError
+
+
+
+ +
+ + +
+ + + + +

+ write_registered_zarr(input_path, component, new_component, ROI_table, ROI_table_ref, num_levels, coarsening_xy=2, aggregation_function=np.mean) + +

+ + +
+ +

Write registered zarr array based on ROI tables

+

This function loads the image or label data from a zarr array based on the +ROI bounding-box coordinates and stores them into a new zarr array. +The new Zarr array has the same shape as the original array, but will have +0s where the ROI tables don't specify loading of the image data. +The ROIs loaded from list_indices will be written into the +list_indices_ref position, thus performing translational registration if +the two lists of ROI indices vary.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_path +
+

Base folder where the Zarr is stored +(does not contain the Zarr file itself)

+
+

+ + TYPE: + Path + +

+
component +
+

Path to the OME-Zarr image that is processed. For example: +"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1"

+
+

+ + TYPE: + str + +

+
new_component +
+

Path to the new Zarr image that will be written +(also in the input_path folder). For example: +"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1_registered"

+
+

+ + TYPE: + str + +

+
ROI_table +
+

Fractal ROI table for the component

+
+

+ + TYPE: + AnnData + +

+
ROI_table_ref +
+

Fractal ROI table for the reference cycle

+
+

+ + TYPE: + AnnData + +

+
num_levels +
+

Number of pyramid layers to be created (argument of +build_pyramid).

+
+

+ + TYPE: + int + +

+
coarsening_xy +
+

Coarsening factor between pyramid levels

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
aggregation_function +
+

Function to be used when downsampling (argument +of build_pyramid).

+
+

+ + TYPE: + Callable + + + DEFAULT: + mean + +

+
+ +
+ Source code in fractal_tasks_core/tasks/apply_registration_to_image.py + 
246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
def write_registered_zarr(
+    input_path: Path,
+    component: str,
+    new_component: str,
+    ROI_table: ad.AnnData,
+    ROI_table_ref: ad.AnnData,
+    num_levels: int,
+    coarsening_xy: int = 2,
+    aggregation_function: Callable = np.mean,
+):
+    """
+    Write registered zarr array based on ROI tables
+
+    This function loads the image or label data from a zarr array based on the
+    ROI bounding-box coordinates and stores them into a new zarr array.
+    The new Zarr array has the same shape as the original array, but will have
+    0s where the ROI tables don't specify loading of the image data.
+    The ROIs loaded from `list_indices` will be written into the
+    `list_indices_ref` position, thus performing translational registration if
+    the two lists of ROI indices vary.
+
+    Args:
+        input_path: Base folder where the Zarr is stored
+            (does not contain the Zarr file itself)
+        component: Path to the OME-Zarr image that is processed. For example:
+            `"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1"`
+        new_component: Path to the new Zarr image that will be written
+            (also in the input_path folder). For example:
+            `"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1_registered"`
+        ROI_table: Fractal ROI table for the component
+        ROI_table_ref: Fractal ROI table for the reference cycle
+        num_levels: Number of pyramid layers to be created (argument of
+            `build_pyramid`).
+        coarsening_xy: Coarsening factor between pyramid levels
+        aggregation_function: Function to be used when downsampling (argument
+            of `build_pyramid`).
+
+    """
+    # Read pixel sizes from Zarr attributes
+    ngff_image_meta = load_NgffImageMeta(str(input_path / component))
+    pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+
+    # Create list of indices for 3D ROIs
+    list_indices = convert_ROI_table_to_indices(
+        ROI_table,
+        level=0,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=pxl_sizes_zyx,
+    )
+    list_indices_ref = convert_ROI_table_to_indices(
+        ROI_table_ref,
+        level=0,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=pxl_sizes_zyx,
+    )
+
+    old_image_group = zarr.open_group(f"{input_path / component}", mode="r")
+    old_ngff_image_meta = load_NgffImageMeta(str(input_path / component))
+    new_image_group = zarr.group(f"{input_path / new_component}")
+    new_image_group.attrs.put(old_image_group.attrs.asdict())
+
+    # Loop over all channels. For each channel, write full-res image data.
+    data_array = da.from_zarr(old_image_group["0"])
+    # Create dask array with 0s of same shape
+    new_array = da.zeros_like(data_array)
+
+    # TODO: Add sanity checks on the 2 ROI tables:
+    # 1. The number of ROIs need to match
+    # 2. The size of the ROIs need to match
+    # (otherwise, we can't assign them to the reference regions)
+    # ROI_table_ref vs ROI_table_cycle
+    for i, roi_indices in enumerate(list_indices):
+        reference_region = convert_indices_to_regions(list_indices_ref[i])
+        region = convert_indices_to_regions(roi_indices)
+
+        axes_list = old_ngff_image_meta.axes_names
+
+        if axes_list == ["c", "z", "y", "x"]:
+            num_channels = data_array.shape[0]
+            # Loop over channels
+            for ind_ch in range(num_channels):
+                idx = tuple(
+                    [slice(ind_ch, ind_ch + 1)] + list(reference_region)
+                )
+                new_array[idx] = load_region(
+                    data_zyx=data_array[ind_ch], region=region, compute=False
+                )
+        elif axes_list == ["z", "y", "x"]:
+            new_array[reference_region] = load_region(
+                data_zyx=data_array, region=region, compute=False
+            )
+        elif axes_list == ["c", "y", "x"]:
+            # TODO: Implement cyx case (based on looping over xy case)
+            raise NotImplementedError(
+                "`write_registered_zarr` has not been implemented for "
+                f"a zarr with {axes_list=}"
+            )
+        elif axes_list == ["y", "x"]:
+            # TODO: Implement yx case
+            raise NotImplementedError(
+                "`write_registered_zarr` has not been implemented for "
+                f"a zarr with {axes_list=}"
+            )
+        else:
+            raise NotImplementedError(
+                "`write_registered_zarr` has not been implemented for "
+                f"a zarr with {axes_list=}"
+            )
+
+    new_array.to_zarr(
+        f"{input_path / new_component}/0",
+        overwrite=True,
+        dimension_separator="/",
+        write_empty_chunks=False,
+    )
+
+    # Starting from on-disk highest-resolution data, build and write to
+    # disk a pyramid of coarser levels
+    build_pyramid(
+        zarrurl=f"{input_path / new_component}",
+        overwrite=True,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        chunksize=data_array.chunksize,
+        aggregation_function=aggregation_function,
+    )
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/calculate_registration_image_based/index.html b/reference/fractal_tasks_core/tasks/calculate_registration_image_based/index.html new file mode 100644 index 000000000..64abfea7c --- /dev/null +++ b/reference/fractal_tasks_core/tasks/calculate_registration_image_based/index.html @@ -0,0 +1,2166 @@ + + + + + + + + + + + + + + + + + + + + + + calculate_registration_image_based - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

calculate_registration_image_based

+ +
+ + + + +
+ +

Calculates translation for image-based registration

+ + + +
+ + + + + + + + + + +
+ + + + +

+ calculate_physical_shifts(shifts, level, coarsening_xy, full_res_pxl_sizes_zyx) + +

+ + +
+ +

Calculates shifts in physical units based on pixel shifts

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
shifts +
+

array of shifts, zyx or yx

+
+

+ + TYPE: + array + +

+
level +
+

resolution level

+
+

+ + TYPE: + int + +

+
coarsening_xy +
+

coarsening factor between levels

+
+

+ + TYPE: + int + +

+
full_res_pxl_sizes_zyx +
+

pixel sizes in physical units as zyx

+
+

+ + TYPE: + list[float] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ shifts_physical + +
+

shifts in physical units as zyx

+
+

+ + TYPE: + list[float] + +

+
+ +
+ Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py + 
263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
def calculate_physical_shifts(
+    shifts: np.array,
+    level: int,
+    coarsening_xy: int,
+    full_res_pxl_sizes_zyx: list[float],
+) -> list[float]:
+    """
+    Calculates shifts in physical units based on pixel shifts
+
+    Args:
+        shifts: array of shifts, zyx or yx
+        level: resolution level
+        coarsening_xy: coarsening factor between levels
+        full_res_pxl_sizes_zyx: pixel sizes in physical units as zyx
+
+    Returns:
+        shifts_physical: shifts in physical units as zyx
+    """
+
+    curr_pixel_size = np.array(full_res_pxl_sizes_zyx) * coarsening_xy**level
+    if len(shifts) == 3:
+        shifts_physical = shifts * curr_pixel_size
+    elif len(shifts) == 2:
+        shifts_physical = [
+            0,
+            shifts[0] * curr_pixel_size[1],
+            shifts[1] * curr_pixel_size[2],
+        ]
+    else:
+        raise ValueError(
+            f"Wrong input for calculate_physical_shifts ({shifts=})"
+        )
+    return shifts_physical
+
+
+
+ +
+ + +
+ + + + +

+ calculate_registration_image_based(*, input_paths, output_path, component, metadata, wavelength_id, roi_table='FOV_ROI_table', reference_cycle=0, level=2) + +

+ + +
+ +

Calculate registration based on images

+

This task consists of 3 parts:

+
    +
  1. Loading the images of a given ROI (=> loop over ROIs)
    2. +
  2. Calculating the transformation for that ROI
    3. +
  3. Storing the calculated transformation in the ROI table
    4. +
+

Parallelization level: image

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
wavelength_id +
+

Wavelength that will be used for image-based +registration; e.g. A01_C01 for Yokogawa, C01 for MD.

+
+

+ + TYPE: + str + +

+
roi_table +
+

Name of the ROI table over which the task loops to +calculate the registration. Examples: FOV_ROI_table => loop over +the field of views, well_ROI_table => process the whole well as +one image.

+
+

+ + TYPE: + str + + + DEFAULT: + 'FOV_ROI_table' + +

+
reference_cycle +
+

Which cycle to register against. Defaults to 0, +which is the first OME-Zarr image in the well (usually the first +cycle that was provided).

+
+

+ + TYPE: + int + + + DEFAULT: + 0 + +

+
level +
+

Pyramid level of the image to be segmented. Choose 0 to +process at full resolution.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
+ +
+ Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py + 
 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
@validate_arguments
+def calculate_registration_image_based(
+    *,
+    # Fractal arguments
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments
+    wavelength_id: str,
+    roi_table: str = "FOV_ROI_table",
+    reference_cycle: int = 0,
+    level: int = 2,
+) -> dict[str, Any]:
+    """
+    Calculate registration based on images
+
+    This task consists of 3 parts:
+
+    1. Loading the images of a given ROI (=> loop over ROIs)
+    2. Calculating the transformation for that ROI
+    3. Storing the calculated transformation in the ROI table
+
+    Parallelization level: image
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        wavelength_id: Wavelength that will be used for image-based
+            registration; e.g. `A01_C01` for Yokogawa, `C01` for MD.
+        roi_table: Name of the ROI table over which the task loops to
+            calculate the registration. Examples: `FOV_ROI_table` => loop over
+            the field of views, `well_ROI_table` => process the whole well as
+            one image.
+        reference_cycle: Which cycle to register against. Defaults to 0,
+            which is the first OME-Zarr image in the well (usually the first
+            cycle that was provided).
+        level: Pyramid level of the image to be segmented. Choose `0` to
+            process at full resolution.
+
+    """
+    logger.info(
+        f"Running for {input_paths=}, {component=}. \n"
+        f"Calculating translation registration per {roi_table=} for "
+        f"{wavelength_id=}."
+    )
+    # Set OME-Zarr paths
+    zarr_img_cycle_x = Path(input_paths[0]) / component
+
+    # If the task is run for the reference cycle, exit
+    # TODO: Improve the input for this: Can we filter components to not
+    # run for itself?
+    alignment_cycle = zarr_img_cycle_x.name
+    if alignment_cycle == str(reference_cycle):
+        logger.info(
+            "Calculate registration image-based is running for "
+            f"cycle {alignment_cycle}, which is the reference_cycle."
+            "Thus, exiting the task."
+        )
+        return {}
+    else:
+        logger.info(
+            "Calculate registration image-based is running for "
+            f"cycle {alignment_cycle}"
+        )
+
+    zarr_img_ref_cycle = zarr_img_cycle_x.parent / str(reference_cycle)
+
+    # Read some parameters from Zarr metadata
+    ngff_image_meta = load_NgffImageMeta(str(zarr_img_ref_cycle))
+    coarsening_xy = ngff_image_meta.coarsening_xy
+
+    # Get channel_index via wavelength_id.
+    # Intially only allow registration of the same wavelength
+    channel_ref: OmeroChannel = get_channel_from_image_zarr(
+        image_zarr_path=str(zarr_img_ref_cycle),
+        wavelength_id=wavelength_id,
+    )
+    channel_index_ref = channel_ref.index
+
+    channel_align: OmeroChannel = get_channel_from_image_zarr(
+        image_zarr_path=str(zarr_img_cycle_x),
+        wavelength_id=wavelength_id,
+    )
+    channel_index_align = channel_align.index
+
+    # Lazily load zarr array
+    data_reference_zyx = da.from_zarr(f"{zarr_img_ref_cycle}/{level}")[
+        channel_index_ref
+    ]
+    data_alignment_zyx = da.from_zarr(f"{zarr_img_cycle_x}/{level}")[
+        channel_index_align
+    ]
+
+    # Read ROIs
+    ROI_table_ref = ad.read_zarr(f"{zarr_img_ref_cycle}/tables/{roi_table}")
+    ROI_table_x = ad.read_zarr(f"{zarr_img_ref_cycle}/tables/{roi_table}")
+    logger.info(
+        f"Found {len(ROI_table_x)} ROIs in {roi_table=} to be processed."
+    )
+
+    # For each cycle, get the relevant info
+    # TODO: Add additional checks on ROIs?
+    if (ROI_table_ref.obs.index != ROI_table_x.obs.index).all():
+        raise ValueError(
+            "Registration is only implemented for ROIs that match between the "
+            "cycles (e.g. well, FOV ROIs). Here, the ROIs in the reference "
+            "cycles were {ROI_table_ref.obs.index}, but the ROIs in the "
+            "alignment cycle were {ROI_table_x.obs.index}"
+        )
+    # TODO: Make this less restrictive? i.e. could we also run it if different
+    # cycles have different FOVs? But then how do we know which FOVs to match?
+    # If we relax this, downstream assumptions on matching based on order
+    # in the list will break.
+
+    # Read pixel sizes from zarr attributes
+    ngff_image_meta_cycle_x = load_NgffImageMeta(str(zarr_img_cycle_x))
+    pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+    pxl_sizes_zyx_cycle_x = ngff_image_meta_cycle_x.get_pixel_sizes_zyx(
+        level=0
+    )
+
+    if pxl_sizes_zyx != pxl_sizes_zyx_cycle_x:
+        raise ValueError(
+            "Pixel sizes need to be equal between cycles for registration"
+        )
+
+    # Create list of indices for 3D ROIs spanning the entire Z direction
+    list_indices_ref = convert_ROI_table_to_indices(
+        ROI_table_ref,
+        level=level,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(list_indices_ref, roi_table)
+
+    list_indices_cycle_x = convert_ROI_table_to_indices(
+        ROI_table_x,
+        level=level,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(list_indices_cycle_x, roi_table)
+
+    num_ROIs = len(list_indices_ref)
+    compute = True
+    new_shifts = {}
+    for i_ROI in range(num_ROIs):
+        logger.info(
+            f"Now processing ROI {i_ROI+1}/{num_ROIs} "
+            f"for channel {channel_align}."
+        )
+        img_ref = load_region(
+            data_zyx=data_reference_zyx,
+            region=convert_indices_to_regions(list_indices_ref[i_ROI]),
+            compute=compute,
+        )
+        img_cycle_x = load_region(
+            data_zyx=data_alignment_zyx,
+            region=convert_indices_to_regions(list_indices_cycle_x[i_ROI]),
+            compute=compute,
+        )
+
+        ##############
+        #  Calculate the transformation
+        ##############
+        # Basic version (no padding, no internal binning)
+        if img_ref.shape != img_cycle_x.shape:
+            raise NotImplementedError(
+                "This registration is not implemented for ROIs with "
+                "different shapes between cycles"
+            )
+        shifts = phase_cross_correlation(
+            np.squeeze(img_ref), np.squeeze(img_cycle_x)
+        )[0]
+
+        # Registration based on scmultiplex, image-based
+        # shifts, _, _ = calculate_shift(np.squeeze(img_ref),
+        #           np.squeeze(img_cycle_x), bin=binning, binarize=False)
+
+        # TODO: Make this work on label images
+        # (=> different loading) etc.
+
+        ##############
+        # Storing the calculated transformation ###
+        ##############
+        # Store the shift in ROI table
+        # TODO: Store in OME-NGFF transformations: Check SpatialData approach,
+        # per ROI storage?
+
+        # Adapt ROIs for the given ROI table:
+        ROI_name = ROI_table_ref.obs.index[i_ROI]
+        new_shifts[ROI_name] = calculate_physical_shifts(
+            shifts,
+            level=level,
+            coarsening_xy=coarsening_xy,
+            full_res_pxl_sizes_zyx=pxl_sizes_zyx,
+        )
+
+    # Write physical shifts to disk (as part of the ROI table)
+    logger.info(f"Updating the {roi_table=} with translation columns")
+    new_ROI_table = get_ROI_table_with_translation(ROI_table_x, new_shifts)
+    group_tables = zarr.group(f"{zarr_img_cycle_x}/tables/")
+    write_elem(group_tables, roi_table, new_ROI_table)
+    group_tables[roi_table].attrs["type"] = "ngff:region_table"
+
+    return {}
+
+
+
+ +
+ + +
+ + + + +

+ get_ROI_table_with_translation(ROI_table, new_shifts) + +

+ + +
+ +

Adds translation columns to a ROI table

+ + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
ROI_table +
+

Fractal ROI table

+
+

+ + TYPE: + AnnData + +

+
new_shifts +
+

zyx list of shifts

+
+

+ + TYPE: + dict[str, list[float]] + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + AnnData + + +
+

Fractal ROI table with 3 additional columns for calculated translations

+
+
+ +
+ Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py + 
298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
def get_ROI_table_with_translation(
+    ROI_table: ad.AnnData,
+    new_shifts: dict[str, list[float]],
+) -> ad.AnnData:
+    """
+    Adds translation columns to a ROI table
+
+    Args:
+        ROI_table: Fractal ROI table
+        new_shifts: zyx list of shifts
+
+    Returns:
+        Fractal ROI table with 3 additional columns for calculated translations
+    """
+
+    shift_table = pd.DataFrame(new_shifts).T
+    shift_table.columns = ["translation_z", "translation_y", "translation_x"]
+    shift_table = shift_table.rename_axis("FieldIndex")
+    new_roi_table = ROI_table.to_df().merge(
+        shift_table, left_index=True, right_index=True
+    )
+    if len(new_roi_table) != len(ROI_table):
+        raise ValueError(
+            "New ROI table with registration info has a "
+            f"different length ({len(new_roi_table)=}) "
+            f"from the original ROI table ({len(ROI_table)=})"
+        )
+    positional_columns = [
+        "x_micrometer",
+        "y_micrometer",
+        "z_micrometer",
+        "len_x_micrometer",
+        "len_y_micrometer",
+        "len_z_micrometer",
+        "x_micrometer_original",
+        "y_micrometer_original",
+        "translation_z",
+        "translation_y",
+        "translation_x",
+    ]
+    adata = ad.AnnData(
+        X=new_roi_table.loc[:, positional_columns].astype(np.float32)
+    )
+    adata.obs_names = new_roi_table.index
+    adata.var_names = list(map(str, positional_columns))
+    return adata
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/cellpose_segmentation/index.html b/reference/fractal_tasks_core/tasks/cellpose_segmentation/index.html new file mode 100644 index 000000000..df479f765 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/cellpose_segmentation/index.html @@ -0,0 +1,3079 @@ + + + + + + + + + + + + + + + + + + + + + + cellpose_segmentation - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

cellpose_segmentation

+ +
+ + + + +
+ +

Image segmentation via Cellpose library.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ cellpose_segmentation(*, input_paths, output_path, component, metadata, level, channel, channel2=None, input_ROI_table='FOV_ROI_table', output_ROI_table=None, output_label_name=None, use_masks=True, relabeling=True, diameter_level0=30.0, model_type='cyto2', pretrained_model=None, cellprob_threshold=0.0, flow_threshold=0.4, anisotropy=None, min_size=15, augment=False, net_avg=False, use_gpu=True, overwrite=True) + +

+ + +
+ +

Run cellpose segmentation on the ROIs of a single OME-Zarr image.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
level +
+

Pyramid level of the image to be segmented. Choose 0 to +process at full resolution.

+
+

+ + TYPE: + int + +

+
channel +
+

Primary channel for segmentation; requires either +wavelength_id (e.g. A01_C01) or label (e.g. DAPI).

+
+

+ + TYPE: + Channel + +

+
channel2 +
+

Second channel for segmentation (in the same format as +channel). If specified, cellpose runs in dual channel mode. +For dual channel segmentation of cells, the first channel should +contain the membrane marker, the second channel should contain the +nuclear marker.

+
+

+ + TYPE: + Optional[Channel] + + + DEFAULT: + None + +

+
input_ROI_table +
+

Name of the ROI table over which the task loops to +apply Cellpose segmentation. Examples: FOV_ROI_table => loop over +the field of views, organoid_ROI_table => loop over the organoid +ROI table (generated by another task), well_ROI_table => process +the whole well as one image.

+
+

+ + TYPE: + str + + + DEFAULT: + 'FOV_ROI_table' + +

+
output_ROI_table +
+

If provided, a ROI table with that name is created, +which will contain the bounding boxes of the newly segmented +labels. ROI tables should have ROI in their name.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
use_masks +
+

If True, try to use masked loading and fall back to +use_masks=False if the ROI table is not suitable. Masked +loading is relevant when only a subset of the bounding box should +actually be processed (e.g. running within organoid_ROI_table).

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
output_label_name +
+

Name of the output label image (e.g. "organoids").

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
relabeling +
+

If True, apply relabeling so that label values are +unique for all objects in the well.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
diameter_level0 +
+

Expected diameter of the objects that should be +segmented in pixels at level 0. Initial diameter is rescaled using +the level that was selected. The rescaled value is passed as +the diameter to the CellposeModel.eval method.

+
+

+ + TYPE: + float + + + DEFAULT: + 30.0 + +

+
model_type +
+

Parameter of CellposeModel class. Defines which model +should be used. Typical choices are nuclei, cyto, cyto2, etc.

+
+

+ + TYPE: + str + + + DEFAULT: + 'cyto2' + +

+
pretrained_model +
+

Parameter of CellposeModel class (takes +precedence over model_type). Allows you to specify the path of +a custom trained cellpose model.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
cellprob_threshold +
+

Parameter of CellposeModel.eval method. Valid +values between -6 to 6. From Cellpose documentation: "Decrease this +threshold if cellpose is not returning as many ROIs as you’d +expect. Similarly, increase this threshold if cellpose is returning +too ROIs particularly from dim areas."

+
+

+ + TYPE: + float + + + DEFAULT: + 0.0 + +

+
flow_threshold +
+

Parameter of CellposeModel.eval method. Valid +values between 0.0 and 1.0. From Cellpose documentation: "Increase +this threshold if cellpose is not returning as many ROIs as you’d +expect. Similarly, decrease this threshold if cellpose is returning +too many ill-shaped ROIs."

+
+

+ + TYPE: + float + + + DEFAULT: + 0.4 + +

+
anisotropy +
+

Ratio of the pixel sizes along Z and XY axis (ignored if +the image is not three-dimensional). If None, it is inferred from +the OME-NGFF metadata.

+
+

+ + TYPE: + Optional[float] + + + DEFAULT: + None + +

+
min_size +
+

Parameter of CellposeModel class. Minimum size of the +segmented objects (in pixels). Use -1 to turn off the size +filter.

+
+

+ + TYPE: + int + + + DEFAULT: + 15 + +

+
augment +
+

Parameter of CellposeModel class. Whether to use cellpose +augmentation to tile images with overlap.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
net_avg +
+

Parameter of CellposeModel class. Whether to use cellpose +net averaging to run the 4 built-in networks (useful for nuclei, +cyto and cyto2, not sure it works for the others).

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
use_gpu +
+

If False, always use the CPU; if True, use the GPU if +possible (as defined in cellpose.core.use_gpu()) and fall-back +to the CPU otherwise.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
+ +
+ Source code in fractal_tasks_core/tasks/cellpose_segmentation.py + 
147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
+517
+518
+519
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+545
+546
+547
+548
+549
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+560
+561
+562
+563
+564
+565
+566
+567
+568
+569
+570
+571
+572
+573
+574
+575
+576
+577
+578
+579
+580
+581
+582
+583
+584
+585
+586
+587
+588
+589
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+600
+601
+602
+603
+604
+605
+606
+607
+608
+609
+610
+611
+612
+613
+614
+615
+616
+617
+618
+619
+620
+621
+622
+623
+624
+625
+626
+627
+628
+629
+630
+631
+632
+633
+634
+635
+636
+637
+638
+639
+640
+641
+642
+643
+644
+645
+646
+647
+648
+649
+650
+651
+652
+653
+654
@validate_arguments
+def cellpose_segmentation(
+    *,
+    # Fractal arguments
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments
+    level: int,
+    channel: Channel,
+    channel2: Optional[Channel] = None,
+    input_ROI_table: str = "FOV_ROI_table",
+    output_ROI_table: Optional[str] = None,
+    output_label_name: Optional[str] = None,
+    use_masks: bool = True,
+    relabeling: bool = True,
+    # Cellpose-related arguments
+    diameter_level0: float = 30.0,
+    model_type: str = "cyto2",
+    pretrained_model: Optional[str] = None,
+    cellprob_threshold: float = 0.0,
+    flow_threshold: float = 0.4,
+    anisotropy: Optional[float] = None,
+    min_size: int = 15,
+    augment: bool = False,
+    net_avg: bool = False,
+    use_gpu: bool = True,
+    overwrite: bool = True,
+) -> dict[str, Any]:
+    """
+    Run cellpose segmentation on the ROIs of a single OME-Zarr image.
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        level: Pyramid level of the image to be segmented. Choose `0` to
+            process at full resolution.
+        channel: Primary channel for segmentation; requires either
+            `wavelength_id` (e.g. `A01_C01`) or `label` (e.g. `DAPI`).
+        channel2: Second channel for segmentation (in the same format as
+            `channel`). If specified, cellpose runs in dual channel mode.
+            For dual channel segmentation of cells, the first channel should
+            contain the membrane marker, the second channel should contain the
+            nuclear marker.
+        input_ROI_table: Name of the ROI table over which the task loops to
+            apply Cellpose segmentation. Examples: `FOV_ROI_table` => loop over
+            the field of views, `organoid_ROI_table` => loop over the organoid
+            ROI table (generated by another task), `well_ROI_table` => process
+            the whole well as one image.
+        output_ROI_table: If provided, a ROI table with that name is created,
+            which will contain the bounding boxes of the newly segmented
+            labels. ROI tables should have `ROI` in their name.
+        use_masks: If `True`, try to use masked loading and fall back to
+            `use_masks=False` if the ROI table is not suitable. Masked
+            loading is relevant when only a subset of the bounding box should
+            actually be processed (e.g. running within `organoid_ROI_table`).
+        output_label_name: Name of the output label image (e.g. `"organoids"`).
+        relabeling: If `True`, apply relabeling so that label values are
+            unique for all objects in the well.
+        diameter_level0: Expected diameter of the objects that should be
+            segmented in pixels at level 0. Initial diameter is rescaled using
+            the `level` that was selected. The rescaled value is passed as
+            the diameter to the `CellposeModel.eval` method.
+        model_type: Parameter of `CellposeModel` class. Defines which model
+            should be used. Typical choices are `nuclei`, `cyto`, `cyto2`, etc.
+        pretrained_model: Parameter of `CellposeModel` class (takes
+            precedence over `model_type`). Allows you to specify the path of
+            a custom trained cellpose model.
+        cellprob_threshold: Parameter of `CellposeModel.eval` method. Valid
+            values between -6 to 6. From Cellpose documentation: "Decrease this
+            threshold if cellpose is not returning as many ROIs as you’d
+            expect. Similarly, increase this threshold if cellpose is returning
+            too ROIs particularly from dim areas."
+        flow_threshold: Parameter of `CellposeModel.eval` method. Valid
+            values between 0.0 and 1.0. From Cellpose documentation: "Increase
+            this threshold if cellpose is not returning as many ROIs as you’d
+            expect. Similarly, decrease this threshold if cellpose is returning
+            too many ill-shaped ROIs."
+        anisotropy: Ratio of the pixel sizes along Z and XY axis (ignored if
+            the image is not three-dimensional). If `None`, it is inferred from
+            the OME-NGFF metadata.
+        min_size: Parameter of `CellposeModel` class. Minimum size of the
+            segmented objects (in pixels). Use `-1` to turn off the size
+            filter.
+        augment: Parameter of `CellposeModel` class. Whether to use cellpose
+            augmentation to tile images with overlap.
+        net_avg: Parameter of `CellposeModel` class. Whether to use cellpose
+            net averaging to run the 4 built-in networks (useful for `nuclei`,
+            `cyto` and `cyto2`, not sure it works for the others).
+        use_gpu: If `False`, always use the CPU; if `True`, use the GPU if
+            possible (as defined in `cellpose.core.use_gpu()`) and fall-back
+            to the CPU otherwise.
+        overwrite: If `True`, overwrite the task output.
+    """
+
+    # Set input path
+    if len(input_paths) > 1:
+        raise NotImplementedError
+    in_path = Path(input_paths[0])
+    zarrurl = (in_path.resolve() / component).as_posix()
+    logger.info(f"{zarrurl=}")
+
+    # Preliminary checks on Cellpose model
+    if pretrained_model is None:
+        if model_type not in models.MODEL_NAMES:
+            raise ValueError(f"ERROR model_type={model_type} is not allowed.")
+    else:
+        if not os.path.exists(pretrained_model):
+            raise ValueError(f"{pretrained_model=} does not exist.")
+
+    # Read attributes from NGFF metadata
+    ngff_image_meta = load_NgffImageMeta(zarrurl)
+    num_levels = ngff_image_meta.num_levels
+    coarsening_xy = ngff_image_meta.coarsening_xy
+    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+    actual_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=level)
+    logger.info(f"NGFF image has {num_levels=}")
+    logger.info(f"NGFF image has {coarsening_xy=}")
+    logger.info(
+        f"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}"
+    )
+    logger.info(
+        f"NGFF image has level-{level} pixel sizes "
+        f"{actual_res_pxl_sizes_zyx}"
+    )
+
+    plate, well = component.split(".zarr/")
+
+    # Find channel index
+    try:
+        tmp_channel: OmeroChannel = get_channel_from_image_zarr(
+            image_zarr_path=zarrurl,
+            wavelength_id=channel.wavelength_id,
+            label=channel.label,
+        )
+    except ChannelNotFoundError as e:
+        logger.warning(
+            "Channel not found, exit from the task.\n"
+            f"Original error: {str(e)}"
+        )
+        return {}
+    ind_channel = tmp_channel.index
+
+    # Find channel index for second channel, if one is provided
+    if channel2:
+        try:
+            tmp_channel_c2: OmeroChannel = get_channel_from_image_zarr(
+                image_zarr_path=zarrurl,
+                wavelength_id=channel2.wavelength_id,
+                label=channel2.label,
+            )
+        except ChannelNotFoundError as e:
+            logger.warning(
+                f"Second channel with wavelength_id: {channel2.wavelength_id} "
+                f"and label: {channel2.label} not found, exit from the task.\n"
+                f"Original error: {str(e)}"
+            )
+            return {}
+        ind_channel_c2 = tmp_channel_c2.index
+
+    # Set channel label
+    if output_label_name is None:
+        try:
+            channel_label = tmp_channel.label
+            output_label_name = f"label_{channel_label}"
+        except (KeyError, IndexError):
+            output_label_name = f"label_{ind_channel}"
+
+    # Load ZYX data
+    data_zyx = da.from_zarr(f"{zarrurl}/{level}")[ind_channel]
+    logger.info(f"{data_zyx.shape=}")
+    if channel2:
+        data_zyx_c2 = da.from_zarr(f"{zarrurl}/{level}")[ind_channel_c2]
+        logger.info(f"Second channel: {data_zyx_c2.shape=}")
+
+    # Read ROI table
+    ROI_table_path = f"{zarrurl}/tables/{input_ROI_table}"
+    ROI_table = ad.read_zarr(ROI_table_path)
+
+    # Perform some checks on the ROI table
+    valid_ROI_table = is_ROI_table_valid(
+        table_path=ROI_table_path, use_masks=use_masks
+    )
+    if use_masks and not valid_ROI_table:
+        logger.info(
+            f"ROI table at {ROI_table_path} cannot be used for masked "
+            "loading. Set use_masks=False."
+        )
+        use_masks = False
+    logger.info(f"{use_masks=}")
+
+    # Create list of indices for 3D ROIs spanning the entire Z direction
+    list_indices = convert_ROI_table_to_indices(
+        ROI_table,
+        level=level,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(list_indices, input_ROI_table)
+
+    # If we are not planning to use masked loading, fail for overlapping ROIs
+    if not use_masks:
+        overlap = find_overlaps_in_ROI_indices(list_indices)
+        if overlap:
+            raise ValueError(
+                f"ROI indices created from {input_ROI_table} table have "
+                "overlaps, but we are not using masked loading."
+            )
+
+    # Select 2D/3D behavior and set some parameters
+    do_3D = data_zyx.shape[0] > 1 and len(data_zyx.shape) == 3
+    if do_3D:
+        if anisotropy is None:
+            # Compute anisotropy as pixel_size_z/pixel_size_x
+            anisotropy = (
+                actual_res_pxl_sizes_zyx[0] / actual_res_pxl_sizes_zyx[2]
+            )
+        logger.info(f"Anisotropy: {anisotropy}")
+
+    # Rescale datasets (only relevant for level>0)
+    if ngff_image_meta.axes_names[0] != "c":
+        raise ValueError(
+            "Cannot set `remove_channel_axis=True` for multiscale "
+            f"metadata with axes={ngff_image_meta.axes_names}. "
+            'First axis should have name "c".'
+        )
+    new_datasets = rescale_datasets(
+        datasets=[ds.dict() for ds in ngff_image_meta.datasets],
+        coarsening_xy=coarsening_xy,
+        reference_level=level,
+        remove_channel_axis=True,
+    )
+
+    label_attrs = {
+        "image-label": {
+            "version": __OME_NGFF_VERSION__,
+            "source": {"image": "../../"},
+        },
+        "multiscales": [
+            {
+                "name": output_label_name,
+                "version": __OME_NGFF_VERSION__,
+                "axes": [
+                    ax.dict()
+                    for ax in ngff_image_meta.multiscale.axes
+                    if ax.type != "channel"
+                ],
+                "datasets": new_datasets,
+            }
+        ],
+    }
+
+    image_group = zarr.group(zarrurl)
+    label_group = prepare_label_group(
+        image_group,
+        output_label_name,
+        overwrite=overwrite,
+        label_attrs=label_attrs,
+        logger=logger,
+    )
+
+    logger.info(
+        f"Helper function `prepare_label_group` returned {label_group=}"
+    )
+    logger.info(f"Output label path: {zarrurl}/labels/{output_label_name}/0")
+    store = zarr.storage.FSStore(f"{zarrurl}/labels/{output_label_name}/0")
+    label_dtype = np.uint32
+
+    # Ensure that all output shapes & chunks are 3D (for 2D data: (1, y, x))
+    # https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/398
+    shape = data_zyx.shape
+    if len(shape) == 2:
+        shape = (1, *shape)
+    chunks = data_zyx.chunksize
+    if len(chunks) == 2:
+        chunks = (1, *chunks)
+    mask_zarr = zarr.create(
+        shape=shape,
+        chunks=chunks,
+        dtype=label_dtype,
+        store=store,
+        overwrite=False,
+        dimension_separator="/",
+    )
+
+    logger.info(
+        f"mask will have shape {data_zyx.shape} "
+        f"and chunks {data_zyx.chunks}"
+    )
+
+    # Initialize cellpose
+    gpu = use_gpu and cellpose.core.use_gpu()
+    if pretrained_model:
+        model = models.CellposeModel(
+            gpu=gpu, pretrained_model=pretrained_model
+        )
+    else:
+        model = models.CellposeModel(gpu=gpu, model_type=model_type)
+
+    # Initialize other things
+    logger.info(f"Start cellpose_segmentation task for {zarrurl}")
+    logger.info(f"relabeling: {relabeling}")
+    logger.info(f"do_3D: {do_3D}")
+    logger.info(f"use_gpu: {gpu}")
+    logger.info(f"level: {level}")
+    logger.info(f"model_type: {model_type}")
+    logger.info(f"pretrained_model: {pretrained_model}")
+    logger.info(f"anisotropy: {anisotropy}")
+    logger.info("Total well shape/chunks:")
+    logger.info(f"{data_zyx.shape}")
+    logger.info(f"{data_zyx.chunks}")
+    if channel2:
+        logger.info("Dual channel input for cellpose model")
+        logger.info(f"{data_zyx_c2.shape}")
+        logger.info(f"{data_zyx_c2.chunks}")
+
+    # Counters for relabeling
+    if relabeling:
+        num_labels_tot = 0
+
+    # Iterate over ROIs
+    num_ROIs = len(list_indices)
+
+    if output_ROI_table:
+        bbox_dataframe_list = []
+
+    logger.info(f"Now starting loop over {num_ROIs} ROIs")
+    for i_ROI, indices in enumerate(list_indices):
+        # Define region
+        s_z, e_z, s_y, e_y, s_x, e_x = indices[:]
+        region = (
+            slice(s_z, e_z),
+            slice(s_y, e_y),
+            slice(s_x, e_x),
+        )
+        logger.info(f"Now processing ROI {i_ROI+1}/{num_ROIs}")
+
+        # Prepare single-channel or dual-channel input for cellpose
+        if channel2:
+            # Dual channel mode, first channel is the membrane channel
+            img_1 = load_region(
+                data_zyx,
+                region,
+                compute=True,
+                return_as_3D=True,
+            )
+            img_np = np.zeros((2, *img_1.shape))
+            img_np[0, :, :, :] = img_1
+            img_np[1, :, :, :] = load_region(
+                data_zyx_c2,
+                region,
+                compute=True,
+                return_as_3D=True,
+            )
+            channels = [1, 2]
+        else:
+            img_np = np.expand_dims(
+                load_region(data_zyx, region, compute=True, return_as_3D=True),
+                axis=0,
+            )
+            channels = [0, 0]
+
+        # Prepare keyword arguments for segment_ROI function
+        kwargs_segment_ROI = dict(
+            model=model,
+            channels=channels,
+            do_3D=do_3D,
+            anisotropy=anisotropy,
+            label_dtype=label_dtype,
+            diameter=diameter_level0 / coarsening_xy**level,
+            cellprob_threshold=cellprob_threshold,
+            flow_threshold=flow_threshold,
+            min_size=min_size,
+            augment=augment,
+            net_avg=net_avg,
+        )
+
+        # Prepare keyword arguments for preprocessing function
+        preprocessing_kwargs = {}
+        if use_masks:
+            preprocessing_kwargs = dict(
+                region=region,
+                current_label_path=f"{zarrurl}/labels/{output_label_name}/0",
+                ROI_table_path=ROI_table_path,
+                ROI_positional_index=i_ROI,
+            )
+
+        # Call segment_ROI through the masked-loading wrapper, which includes
+        # pre/post-processing functions if needed
+        new_label_img = masked_loading_wrapper(
+            image_array=img_np,
+            function=segment_ROI,
+            kwargs=kwargs_segment_ROI,
+            use_masks=use_masks,
+            preprocessing_kwargs=preprocessing_kwargs,
+        )
+
+        # Shift labels and update relabeling counters
+        if relabeling:
+            num_labels_roi = np.max(new_label_img)
+            new_label_img[new_label_img > 0] += num_labels_tot
+            num_labels_tot += num_labels_roi
+
+            # Write some logs
+            logger.info(f"ROI {indices}, {num_labels_roi=}, {num_labels_tot=}")
+
+            # Check that total number of labels is under control
+            if num_labels_tot > np.iinfo(label_dtype).max:
+                raise ValueError(
+                    "ERROR in re-labeling:"
+                    f"Reached {num_labels_tot} labels, "
+                    f"but dtype={label_dtype}"
+                )
+
+        if output_ROI_table:
+            bbox_df = array_to_bounding_box_table(
+                new_label_img,
+                actual_res_pxl_sizes_zyx,
+                origin_zyx=(s_z, s_y, s_x),
+            )
+
+            bbox_dataframe_list.append(bbox_df)
+
+            overlap_list = []
+            for df in bbox_dataframe_list:
+                overlap_list.extend(
+                    get_overlapping_pairs_3D(df, full_res_pxl_sizes_zyx)
+                )
+            if len(overlap_list) > 0:
+                logger.warning(
+                    f"{len(overlap_list)} bounding-box pairs overlap"
+                )
+
+        # Compute and store 0-th level to disk
+        da.array(new_label_img).to_zarr(
+            url=mask_zarr,
+            region=region,
+            compute=True,
+        )
+
+    logger.info(
+        f"End cellpose_segmentation task for {zarrurl}, "
+        "now building pyramids."
+    )
+
+    # Starting from on-disk highest-resolution data, build and write to disk a
+    # pyramid of coarser levels
+    build_pyramid(
+        zarrurl=f"{zarrurl}/labels/{output_label_name}",
+        overwrite=overwrite,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        chunksize=chunks,
+        aggregation_function=np.max,
+    )
+
+    logger.info("End building pyramids")
+
+    if output_ROI_table:
+        # Handle the case where `bbox_dataframe_list` is empty (typically
+        # because list_indices is also empty)
+        if len(bbox_dataframe_list) == 0:
+            bbox_dataframe_list = [empty_bounding_box_table()]
+        # Concatenate all ROI dataframes
+        df_well = pd.concat(bbox_dataframe_list, axis=0, ignore_index=True)
+        df_well.index = df_well.index.astype(str)
+        # Extract labels and drop them from df_well
+        labels = pd.DataFrame(df_well["label"].astype(str))
+        df_well.drop(labels=["label"], axis=1, inplace=True)
+        # Convert all to float (warning: some would be int, in principle)
+        bbox_dtype = np.float32
+        df_well = df_well.astype(bbox_dtype)
+        # Convert to anndata
+        bbox_table = ad.AnnData(df_well, dtype=bbox_dtype)
+        bbox_table.obs = labels
+
+        # Write to zarr group
+        image_group = zarr.group(f"{in_path}/{component}")
+        logger.info(
+            "Now writing bounding-box ROI table to "
+            f"{in_path}/{component}/tables/{output_ROI_table}"
+        )
+        table_attrs = {
+            "type": "ngff:region_table",
+            "region": {"path": f"../labels/{output_label_name}"},
+            "instance_key": "label",
+        }
+        write_table(
+            image_group,
+            output_ROI_table,
+            bbox_table,
+            overwrite=overwrite,
+            logger=logger,
+            table_attrs=table_attrs,
+        )
+
+    return {}
+
+
+
+ +
+ + +
+ + + + +

+ segment_ROI(x, model=None, do_3D=True, channels=[0, 0], anisotropy=None, diameter=30.0, cellprob_threshold=0.0, flow_threshold=0.4, label_dtype=None, augment=False, net_avg=False, min_size=15) + +

+ + +
+ +

Internal function that runs Cellpose segmentation for a single ROI.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
x +
+

4D numpy array.

+
+

+ + TYPE: + ndarray + +

+
model +
+

An instance of models.CellposeModel.

+
+

+ + TYPE: + CellposeModel + + + DEFAULT: + None + +

+
do_3D +
+

If True, cellpose runs in 3D mode: runs on xy, xz & yz planes, +then averages the flows.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
channels +
+

Which channels to use. If only one channel is provided, [0, +0] should be used. If two channels are provided (the first +dimension of x has length of 2), [1, 2] should be used +(x[0, :, :,:] contains the membrane channel and +x[1, :, :, :] contains the nuclear channel).

+
+

+ + TYPE: + list[int] + + + DEFAULT: + [0, 0] + +

+
anisotropy +
+

Set anisotropy rescaling factor for Z dimension.

+
+

+ + TYPE: + Optional[float] + + + DEFAULT: + None + +

+
diameter +
+

Expected object diameter in pixels for cellpose.

+
+

+ + TYPE: + float + + + DEFAULT: + 30.0 + +

+
cellprob_threshold +
+

Cellpose model parameter.

+
+

+ + TYPE: + float + + + DEFAULT: + 0.0 + +

+
flow_threshold +
+

Cellpose model parameter.

+
+

+ + TYPE: + float + + + DEFAULT: + 0.4 + +

+
label_dtype +
+

Label images are cast into this np.dtype.

+
+

+ + TYPE: + Optional[dtype] + + + DEFAULT: + None + +

+
augment +
+

Whether to use cellpose augmentation to tile images with +overlap.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
net_avg +
+

Whether to use cellpose net averaging to run the 4 built-in +networks (useful for nuclei, cyto and cyto2, not sure it +works for the others).

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
min_size +
+

Minimum size of the segmented objects.

+
+

+ + TYPE: + int + + + DEFAULT: + 15 + +

+
+ +
+ Source code in fractal_tasks_core/tasks/cellpose_segmentation.py + 
 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
def segment_ROI(
+    x: np.ndarray,
+    model: models.CellposeModel = None,
+    do_3D: bool = True,
+    channels: list[int] = [0, 0],
+    anisotropy: Optional[float] = None,
+    diameter: float = 30.0,
+    cellprob_threshold: float = 0.0,
+    flow_threshold: float = 0.4,
+    label_dtype: Optional[np.dtype] = None,
+    augment: bool = False,
+    net_avg: bool = False,
+    min_size: int = 15,
+) -> np.ndarray:
+    """
+    Internal function that runs Cellpose segmentation for a single ROI.
+
+    Args:
+        x: 4D numpy array.
+        model: An instance of `models.CellposeModel`.
+        do_3D: If `True`, cellpose runs in 3D mode: runs on xy, xz & yz planes,
+            then averages the flows.
+        channels: Which channels to use. If only one channel is provided, `[0,
+            0]` should be used. If two channels are provided (the first
+            dimension of `x` has length of 2), `[1, 2]` should be used
+            (`x[0, :, :,:]` contains the membrane channel and
+            `x[1, :, :, :]` contains the nuclear channel).
+        anisotropy: Set anisotropy rescaling factor for Z dimension.
+        diameter: Expected object diameter in pixels for cellpose.
+        cellprob_threshold: Cellpose model parameter.
+        flow_threshold: Cellpose model parameter.
+        label_dtype: Label images are cast into this `np.dtype`.
+        augment: Whether to use cellpose augmentation to tile images with
+            overlap.
+        net_avg: Whether to use cellpose net averaging to run the 4 built-in
+            networks (useful for `nuclei`, `cyto` and `cyto2`, not sure it
+            works for the others).
+        min_size: Minimum size of the segmented objects.
+    """
+
+    # Write some debugging info
+    logger.info(
+        "[segment_ROI] START |"
+        f" x: {type(x)}, {x.shape} |"
+        f" {do_3D=} |"
+        f" {model.diam_mean=} |"
+        f" {diameter=} |"
+        f" {flow_threshold=}"
+    )
+
+    # Actual labeling
+    t0 = time.perf_counter()
+    mask, _, _ = model.eval(
+        x,
+        channels=channels,
+        do_3D=do_3D,
+        net_avg=net_avg,
+        augment=augment,
+        diameter=diameter,
+        anisotropy=anisotropy,
+        cellprob_threshold=cellprob_threshold,
+        flow_threshold=flow_threshold,
+        min_size=min_size,
+    )
+
+    if mask.ndim == 2:
+        # If we get a 2D image, we still return it as a 3D array
+        mask = np.expand_dims(mask, axis=0)
+    t1 = time.perf_counter()
+
+    # Write some debugging info
+    logger.info(
+        "[segment_ROI] END   |"
+        f" Elapsed: {t1-t0:.3f} s |"
+        f" {mask.shape=},"
+        f" {mask.dtype=} (then {label_dtype}),"
+        f" {np.max(mask)=} |"
+        f" {model.diam_mean=} |"
+        f" {diameter=} |"
+        f" {flow_threshold=}"
+    )
+
+    return mask.astype(label_dtype)
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/compress_tif/index.html b/reference/fractal_tasks_core/tasks/compress_tif/index.html new file mode 100644 index 000000000..d94ebb778 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/compress_tif/index.html @@ -0,0 +1,1326 @@ + + + + + + + + + + + + + + + + + + + + + + compress_tif - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

compress_tif

+ +
+ + + + +
+ +

Task to compress tiff images.

+

This task cannot be used in the current form, and it should first be +aligned with the other tasks' structure.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ compress_tif(in_path, out_path, delete_input=False) + +

+ + +
+ +

Compress tiff files.

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
in_path +
+

directory containing the input files.

+
+

+ + TYPE: + str + +

+
out_path +
+

directory containing the output files.

+
+

+ + TYPE: + str + +

+
delete_input +
+

delete input files.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ +
+ Source code in fractal_tasks_core/tasks/compress_tif.py + 
24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
def compress_tif(in_path: str, out_path: str, delete_input: bool = False):
+
+    """
+    Compress tiff files.
+
+    Args:
+        in_path: directory containing the input files.
+        out_path: directory containing the output files.
+        delete_input: delete input files.
+    """
+
+    # Sanitize input/output paths
+    if not in_path.endswith("/"):
+        in_path += "/"
+    if not out_path.endswith("/"):
+        out_path += "/"
+
+    # Create output path, if needed
+    if not os.path.exists(out_path):
+        os.makedirs(out_path)
+
+    num_img_compressed = 0
+    num_img_deleted = 0
+    for filename in glob.glob(in_path + "*.tif"):
+        newfilename = os.path.join(out_path, os.path.basename(filename))
+
+        # Save compressed image
+        with Image.open(filename) as image:
+            image.save(newfilename, format="tiff", compression="tiff_lzw")
+        print(f"Raw:        {filename}\nCompressed: {newfilename}")
+        num_img_compressed += 1
+
+        # Delete raw image, if needed
+        if delete_input:
+            try:
+                os.remove(filename)
+                print(f"Deleted:    {filename}")
+                num_img_deleted += 1
+            except OSError as e:
+                print("ERROR: %s : %s" % (filename, e.strerror))
+
+        print()
+
+    return num_img_compressed, num_img_deleted
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/copy_ome_zarr/index.html b/reference/fractal_tasks_core/tasks/copy_ome_zarr/index.html new file mode 100644 index 000000000..7af04c0c5 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/copy_ome_zarr/index.html @@ -0,0 +1,1710 @@ + + + + + + + + + + + + + + + + + + + + + + copy_ome_zarr - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

copy_ome_zarr

+ +
+ + + + +
+ +

Task that copies the structure of an OME-NGFF zarr array to a new one.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ copy_ome_zarr(*, input_paths, output_path, metadata, project_to_2D=True, suffix='mip', ROI_table_names=('FOV_ROI_table', 'well_ROI_table'), overwrite=False) + +

+ + +
+ +

Duplicate an input zarr structure to a new path.

+

This task copies all the structure, but none of the image data:

+
    +
  • For each plate, create a new zarr group with the same attributes as + the original one.
    • +
  • For each well (in each plate), create a new zarr subgroup with the + same attributes as the original one.
    • +
  • For each image (in each well), create a new zarr subgroup with the + same attributes as the original one.
    • +
  • For each image (in each well), copy the relevant AnnData tables from + the original source.
    • +
+

Note: this task makes use of methods from the Attributes class, see +https://zarr.readthedocs.io/en/stable/api/attrs.html.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Path were the output of this task is stored. Example: +"/some/path/" => puts the new OME-Zarr file in the same folder as +the input OME-Zarr file "/some/new_path" => puts the new OME-Zarr +file into a new folder at /some/new_path. (standard argument for +Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

Dictionary containing metadata about the OME-Zarr. This task +requires the following elements to be present in the metadata: +plate: List of plates +(e.g. ["MyPlate.zarr"]); +well: List of wells in the OME-Zarr plate +(e.g. ["MyPlate.zarr/B/03/MyPlate.zarr/B/05"]); +"image": List of images in the OME-Zarr plate +(e.g. ["MyPlate.zarr/B/03/0", "MyPlate.zarr/B/05/0"]). +standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
project_to_2D +
+

If True, apply a 3D->2D projection to the ROI tables +that are copied to the new OME-Zarr.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
suffix +
+

The suffix that is used to transform plate.zarr into +plate_suffix.zarr. Note that None is not currently supported.

+
+

+ + TYPE: + str + + + DEFAULT: + 'mip' + +

+
ROI_table_names +
+

List of Anndata table names to be copied. Note: +copying non-ROI tables may fail if project_to_2D=True.

+
+

+ + TYPE: + tuple[str, ...] + + + DEFAULT: + ('FOV_ROI_table', 'well_ROI_table') + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[str, Any] + + +
+

An update to the metadata table with new plate, well, image +entries (now with the suffix in the plate name).

+
+
+ +
+ Source code in fractal_tasks_core/tasks/copy_ome_zarr.py + 
 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
@validate_arguments
+def copy_ome_zarr(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    metadata: dict[str, Any],
+    project_to_2D: bool = True,
+    suffix: str = "mip",
+    ROI_table_names: tuple[str, ...] = ("FOV_ROI_table", "well_ROI_table"),
+    overwrite: bool = False,
+) -> dict[str, Any]:
+
+    """
+    Duplicate an input zarr structure to a new path.
+
+    This task copies all the structure, but none of the image data:
+
+    - For each plate, create a new zarr group with the same attributes as
+       the original one.
+    - For each well (in each plate), create a new zarr subgroup with the
+       same attributes as the original one.
+    - For each image (in each well), create a new zarr subgroup with the
+       same attributes as the original one.
+    - For each image (in each well), copy the relevant AnnData tables from
+       the original source.
+
+    Note: this task makes use of methods from the `Attributes` class, see
+    https://zarr.readthedocs.io/en/stable/api/attrs.html.
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: Path were the output of this task is stored. Example:
+            `"/some/path/"` => puts the new OME-Zarr file in the same folder as
+            the input OME-Zarr file `"/some/new_path"` => puts the new OME-Zarr
+            file into a new folder at `/some/new_path`. (standard argument for
+            Fractal tasks, managed by Fractal server).
+        metadata: Dictionary containing metadata about the OME-Zarr. This task
+            requires the following elements to be present in the metadata:
+            `plate`: List of plates
+            (e.g. `["MyPlate.zarr"]`);
+            `well`: List of wells in the OME-Zarr plate
+            (e.g. `["MyPlate.zarr/B/03/MyPlate.zarr/B/05"]`);
+            "image": List of images in the OME-Zarr plate
+            (e.g. `["MyPlate.zarr/B/03/0", "MyPlate.zarr/B/05/0"]`).
+            standard argument for Fractal tasks, managed by Fractal server).
+        project_to_2D: If `True`, apply a 3D->2D projection to the ROI tables
+            that are copied to the new OME-Zarr.
+        suffix: The suffix that is used to transform `plate.zarr` into
+            `plate_suffix.zarr`. Note that `None` is not currently supported.
+        ROI_table_names: List of Anndata table names to be copied. Note:
+            copying non-ROI tables may fail if `project_to_2D=True`.
+        overwrite: If `True`, overwrite the task output.
+
+    Returns:
+        An update to the metadata table with new `plate`, `well`, `image`
+            entries (now with the suffix in the plate name).
+    """
+
+    # Preliminary check
+    if len(input_paths) > 1:
+        raise NotImplementedError
+    if suffix is None:
+        # FIXME create a standard suffix (with timestamp)
+        raise NotImplementedError
+
+    # List all plates
+    in_path = Path(input_paths[0])
+    list_plates = [
+        p.as_posix()
+        for p in Path(in_path).glob("*.zarr")
+        if p.name in metadata["plate"]
+    ]
+    logger.info(f"{list_plates=}")
+
+    meta_update: dict[str, Any] = {"copy_ome_zarr": {}}
+    meta_update["copy_ome_zarr"]["suffix"] = suffix
+    meta_update["copy_ome_zarr"]["sources"] = {}
+
+    # Loop over all plates
+    for zarrurl_old in list_plates:
+        zarrfile = zarrurl_old.split("/")[-1]
+        old_plate_name = zarrfile.split(".zarr")[0]
+        new_plate_name = f"{old_plate_name}_{suffix}"
+        new_plate_dir = Path(output_path).resolve()
+        zarrurl_new = f"{(new_plate_dir / new_plate_name).as_posix()}.zarr"
+        meta_update["copy_ome_zarr"]["sources"][new_plate_name] = zarrurl_old
+
+        logger.info(f"{zarrurl_old=}")
+        logger.info(f"{zarrurl_new=}")
+        logger.info(f"{meta_update=}")
+
+        # Replicate plate attrs
+        old_plate_group = zarr.open_group(zarrurl_old, mode="r")
+        new_plate_group = open_zarr_group_with_overwrite(
+            zarrurl_new, overwrite=overwrite
+        )
+        new_plate_group.attrs.put(old_plate_group.attrs.asdict())
+
+        well_paths = [
+            well["path"] for well in new_plate_group.attrs["plate"]["wells"]
+        ]
+        logger.info(f"{well_paths=}")
+        for well_path in well_paths:
+
+            # Replicate well attrs
+            old_well_group = zarr.open_group(
+                f"{zarrurl_old}/{well_path}", mode="r"
+            )
+            new_well_group = zarr.group(f"{zarrurl_new}/{well_path}")
+            new_well_group.attrs.put(old_well_group.attrs.asdict())
+
+            image_paths = [
+                image["path"]
+                for image in new_well_group.attrs["well"]["images"]
+            ]
+            logger.info(f"{image_paths=}")
+
+            for image_path in image_paths:
+
+                # Replicate image attrs
+                old_image_group = zarr.open_group(
+                    f"{zarrurl_old}/{well_path}/{image_path}", mode="r"
+                )
+                new_image_group = zarr.group(
+                    f"{zarrurl_new}/{well_path}/{image_path}"
+                )
+                new_image_group.attrs.put(old_image_group.attrs.asdict())
+
+                # Extract pixel sizes, if needed
+                if ROI_table_names:
+
+                    new_tables_group = new_image_group.create_group("tables/")
+                    new_tables_group.attrs["tables"] = ROI_table_names
+                    if project_to_2D:
+                        path_image = f"{zarrurl_old}/{well_path}/{image_path}"
+                        ngff_image_meta = load_NgffImageMeta(path_image)
+                        pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(
+                            level=0
+                        )
+                        pxl_size_z = pxl_sizes_zyx[0]
+
+                    # Copy the tables in ROI_table_names
+                    for ROI_table_name in ROI_table_names:
+
+                        logger.info(
+                            f"I will now read {ROI_table_name} from "
+                            f"{zarrurl_old=}, convert it to 2D, and "
+                            "write it back to the new zarr file."
+                        )
+                        ROI_table = ad.read_zarr(
+                            f"{zarrurl_old}/{well_path}/{image_path}/"
+                            f"tables/{ROI_table_name}"
+                        )
+                        # Convert 3D ROIs to 2D
+                        if project_to_2D:
+                            new_ROI_table = convert_ROIs_from_3D_to_2D(
+                                ROI_table, pxl_size_z
+                            )
+                        # Write new table
+                        write_elem(
+                            new_tables_group, ROI_table_name, new_ROI_table
+                        )
+
+    for key in ["plate", "well", "image"]:
+        meta_update[key] = [
+            component.replace(".zarr", f"_{suffix}.zarr")
+            for component in metadata[key]
+        ]
+
+    return meta_update
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/create_ome_zarr/index.html b/reference/fractal_tasks_core/tasks/create_ome_zarr/index.html new file mode 100644 index 000000000..15f666c5b --- /dev/null +++ b/reference/fractal_tasks_core/tasks/create_ome_zarr/index.html @@ -0,0 +1,2248 @@ + + + + + + + + + + + + + + + + + + + + + + create_ome_zarr - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

create_ome_zarr

+ +
+ + + + +
+ +

Create structure for OME-NGFF zarr array.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ create_ome_zarr(*, input_paths, output_path, metadata, allowed_channels, image_glob_patterns=None, num_levels=5, coarsening_xy=2, image_extension='tif', metadata_table_file=None, overwrite=False) + +

+ + +
+ +

Create a OME-NGFF zarr folder, without reading/writing image data.

+

Find plates (for each folder in input_paths):

+
    +
  • glob image files,
    • +
  • parse metadata from image filename to identify plates,
    • +
  • identify populated channels.
    • +
+

Create a zarr folder (for each plate):

+
    +
  • parse mlf metadata,
    • +
  • identify wells and field of view (FOV),
    • +
  • create FOV ZARR,
    • +
  • verify that channels are uniform (i.e., same channels).
    • +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data from +the microscope is stored (as TIF or PNG). Should point to the +parent folder containing the images and the metadata files +MeasurementData.mlf and MeasurementDetail.mrf (if present). +Example: ["/some/path/"]. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Path were the output of this task is stored. +Example: "/some/path/" => puts the new OME-Zarr file in the +"/some/path/". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
allowed_channels +
+

A list of OmeroChannel s, where each channel must +include the wavelength_id attribute and where the +wavelength_id values must be unique across the list.

+
+

+ + TYPE: + list[OmeroChannel] + +

+
image_glob_patterns +
+

If specified, only parse images with filenames +that match with all these patterns. Patterns must be defined as in +https://docs.python.org/3/library/fnmatch.html, Example: +image_glob_pattern=["*_B03_*"] => only process well B03 +image_glob_pattern=["*_C09_*", "*F016*", "*Z[0-5][0-9]C*"] => +only process well C09, field of view 16 and Z planes 0-59.

+
+

+ + TYPE: + Optional[list[str]] + + + DEFAULT: + None + +

+
num_levels +
+

Number of resolution-pyramid levels. If set to 5, there +will be the full-resolution level and 4 levels of +downsampled images.

+
+

+ + TYPE: + int + + + DEFAULT: + 5 + +

+
coarsening_xy +
+

Linear coarsening factor between subsequent levels. +If set to 2, level 1 is 2x downsampled, level 2 is +4x downsampled etc.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
image_extension +
+

Filename extension of images (e.g. "tif" or "png")

+
+

+ + TYPE: + str + + + DEFAULT: + 'tif' + +

+
metadata_table_file +
+

If None, parse Yokogawa metadata from mrf/mlf +files in the input_path folder; else, the full path to a csv file +containing the parsed metadata table.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[str, Any] + + +
+

A metadata dictionary containing important metadata about the OME-Zarr +plate, the images and some parameters required by downstream tasks +(like num_levels).

+
+
+ +
+ Source code in fractal_tasks_core/tasks/create_ome_zarr.py + 
 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
@validate_arguments
+def create_ome_zarr(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    metadata: dict[str, Any],
+    allowed_channels: list[OmeroChannel],
+    image_glob_patterns: Optional[list[str]] = None,
+    num_levels: int = 5,
+    coarsening_xy: int = 2,
+    image_extension: str = "tif",
+    metadata_table_file: Optional[str] = None,
+    overwrite: bool = False,
+) -> dict[str, Any]:
+    """
+    Create a OME-NGFF zarr folder, without reading/writing image data.
+
+    Find plates (for each folder in input_paths):
+
+    - glob image files,
+    - parse metadata from image filename to identify plates,
+    - identify populated channels.
+
+    Create a zarr folder (for each plate):
+
+    - parse mlf metadata,
+    - identify wells and field of view (FOV),
+    - create FOV ZARR,
+    - verify that channels are uniform (i.e., same channels).
+
+    Args:
+        input_paths: List of input paths where the image data from
+            the microscope is stored (as TIF or PNG).  Should point to the
+            parent folder containing the images and the metadata files
+            `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present).
+            Example: `["/some/path/"]`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: Path were the output of this task is stored.
+            Example: "/some/path/" => puts the new OME-Zarr file in the
+            "/some/path/".
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        allowed_channels: A list of `OmeroChannel` s, where each channel must
+            include the `wavelength_id` attribute and where the
+            `wavelength_id` values must be unique across the list.
+        image_glob_patterns: If specified, only parse images with filenames
+            that match with all these patterns. Patterns must be defined as in
+            https://docs.python.org/3/library/fnmatch.html, Example:
+            `image_glob_pattern=["*_B03_*"]` => only process well B03
+            `image_glob_pattern=["*_C09_*", "*F016*", "*Z[0-5][0-9]C*"]` =>
+            only process well C09, field of view 16 and Z planes 0-59.
+        num_levels: Number of resolution-pyramid levels. If set to `5`, there
+            will be the full-resolution level and 4 levels of
+            downsampled images.
+        coarsening_xy: Linear coarsening factor between subsequent levels.
+            If set to `2`, level 1 is 2x downsampled, level 2 is
+            4x downsampled etc.
+        image_extension: Filename extension of images (e.g. `"tif"` or `"png"`)
+        metadata_table_file: If `None`, parse Yokogawa metadata from mrf/mlf
+            files in the input_path folder; else, the full path to a csv file
+            containing the parsed metadata table.
+        overwrite: If `True`, overwrite the task output.
+
+    Returns:
+        A metadata dictionary containing important metadata about the OME-Zarr
+            plate, the images and some parameters required by downstream tasks
+            (like `num_levels`).
+    """
+
+    # Preliminary checks on metadata_table_file
+    if metadata_table_file:
+        if not metadata_table_file.endswith(".csv"):
+            raise ValueError(f"{metadata_table_file=} is not a csv file")
+        if not os.path.isfile(metadata_table_file):
+            raise FileNotFoundError(f"{metadata_table_file=} does not exist")
+
+    # Identify all plates and all channels, across all input folders
+    plates = []
+    actual_wavelength_ids = None
+    dict_plate_paths = {}
+    dict_plate_prefixes: dict[str, Any] = {}
+
+    # Preliminary checks on allowed_channels argument
+    check_unique_wavelength_ids(allowed_channels)
+
+    for in_path_str in input_paths:
+        in_path = Path(in_path_str)
+
+        # Glob image filenames
+        patterns = [f"*.{image_extension}"]
+        if image_glob_patterns:
+            patterns.extend(image_glob_patterns)
+        input_filenames = glob_with_multiple_patterns(
+            folder=in_path_str,
+            patterns=patterns,
+        )
+
+        tmp_wavelength_ids = []
+        tmp_plates = []
+        for fn in input_filenames:
+            try:
+                filename_metadata = parse_filename(Path(fn).name)
+                plate_prefix = filename_metadata["plate_prefix"]
+                plate = filename_metadata["plate"]
+                if plate not in dict_plate_prefixes.keys():
+                    dict_plate_prefixes[plate] = plate_prefix
+                tmp_plates.append(plate)
+                A = filename_metadata["A"]
+                C = filename_metadata["C"]
+                tmp_wavelength_ids.append(f"A{A}_C{C}")
+            except ValueError as e:
+                logger.warning(
+                    f'Skipping "{Path(fn).name}". Original error: ' + str(e)
+                )
+        tmp_plates = sorted(list(set(tmp_plates)))
+        tmp_wavelength_ids = sorted(list(set(tmp_wavelength_ids)))
+
+        info = (
+            "Listing plates/channels:\n"
+            f"Folder:   {in_path_str}\n"
+            f"Patterns: {patterns}\n"
+            f"Plates:   {tmp_plates}\n"
+            f"Channels: {tmp_wavelength_ids}\n"
+        )
+
+        # Check that only one plate is found
+        if len(tmp_plates) > 1:
+            raise ValueError(f"{info}ERROR: {len(tmp_plates)} plates detected")
+        elif len(tmp_plates) == 0:
+            raise ValueError(f"{info}ERROR: No plates detected")
+        plate = tmp_plates[0]
+
+        # If plate already exists in other folder, add suffix
+        if plate in plates:
+            ind = 1
+            new_plate = f"{plate}_{ind}"
+            while new_plate in plates:
+                new_plate = f"{plate}_{ind}"
+                ind += 1
+            logger.info(
+                f"WARNING: {plate} already exists, renaming it as {new_plate}"
+            )
+            plates.append(new_plate)
+            dict_plate_prefixes[new_plate] = dict_plate_prefixes[plate]
+            plate = new_plate
+        else:
+            plates.append(plate)
+
+        # Check that channels are the same as in previous plates
+        if actual_wavelength_ids is None:
+            actual_wavelength_ids = tmp_wavelength_ids[:]
+        else:
+            if actual_wavelength_ids != tmp_wavelength_ids:
+                raise ValueError(
+                    f"ERROR\n{info}\nERROR:"
+                    f" expected channels {actual_wavelength_ids}"
+                )
+
+        # Update dict_plate_paths
+        dict_plate_paths[plate] = in_path
+
+    # Check that all channels are in the allowed_channels
+    allowed_wavelength_ids = [
+        channel.wavelength_id for channel in allowed_channels
+    ]
+    if not set(actual_wavelength_ids).issubset(set(allowed_wavelength_ids)):
+        msg = "ERROR in create_ome_zarr\n"
+        msg += f"actual_wavelength_ids: {actual_wavelength_ids}\n"
+        msg += f"allowed_wavelength_ids: {allowed_wavelength_ids}\n"
+        raise ValueError(msg)
+
+    # Create actual_channels, i.e. a list of the channel dictionaries which are
+    # present
+    actual_channels = [
+        channel
+        for channel in allowed_channels
+        if channel.wavelength_id in actual_wavelength_ids
+    ]
+
+    zarrurls: dict[str, list[str]] = {"plate": [], "well": [], "image": []}
+
+    ################################################################
+    for plate in plates:
+        # Define plate zarr
+        zarrurl = f"{plate}.zarr"
+        in_path = dict_plate_paths[plate]
+        logger.info(f"Creating {zarrurl}")
+        # Call zarr.open_group wrapper, which handles overwrite=True/False
+        group_plate = open_zarr_group_with_overwrite(
+            str(Path(output_path) / zarrurl),
+            overwrite=overwrite,
+        )
+        zarrurls["plate"].append(zarrurl)
+
+        # Obtain FOV-metadata dataframe
+
+        if metadata_table_file is None:
+            mrf_path = f"{in_path}/MeasurementDetail.mrf"
+            mlf_path = f"{in_path}/MeasurementData.mlf"
+
+            site_metadata, number_images_mlf = parse_yokogawa_metadata(
+                mrf_path,
+                mlf_path,
+                filename_patterns=image_glob_patterns,
+            )
+            site_metadata = remove_FOV_overlaps(site_metadata)
+
+        # If a metadata table was passed, load it and use it directly
+        else:
+            logger.warning(
+                "Since a custom metadata table was provided, there will "
+                "be no additional check on the number of image files."
+            )
+            site_metadata = pd.read_csv(metadata_table_file)
+            site_metadata.set_index(["well_id", "FieldIndex"], inplace=True)
+
+        # Extract pixel sizes and bit_depth
+        pixel_size_z = site_metadata["pixel_size_z"][0]
+        pixel_size_y = site_metadata["pixel_size_y"][0]
+        pixel_size_x = site_metadata["pixel_size_x"][0]
+        bit_depth = site_metadata["bit_depth"][0]
+
+        if min(pixel_size_z, pixel_size_y, pixel_size_x) < 1e-9:
+            raise ValueError(pixel_size_z, pixel_size_y, pixel_size_x)
+
+        # Identify all wells
+        plate_prefix = dict_plate_prefixes[plate]
+
+        patterns = [f"{plate_prefix}_*.{image_extension}"]
+        if image_glob_patterns:
+            patterns.extend(image_glob_patterns)
+        plate_images = glob_with_multiple_patterns(
+            folder=str(in_path), patterns=patterns
+        )
+
+        wells = [
+            parse_filename(os.path.basename(fn))["well"] for fn in plate_images
+        ]
+        wells = sorted(list(set(wells)))
+
+        # Verify that all wells have all channels
+        for well in wells:
+            patterns = [f"{plate_prefix}_{well}_*.{image_extension}"]
+            if image_glob_patterns:
+                patterns.extend(image_glob_patterns)
+            well_images = glob_with_multiple_patterns(
+                folder=str(in_path), patterns=patterns
+            )
+
+            # Check number of images matches with expected one
+            if metadata_table_file is None:
+                num_images_glob = len(well_images)
+                num_images_expected = number_images_mlf[well]
+                if num_images_glob != num_images_expected:
+                    raise ValueError(
+                        f"Wrong number of images for {well=}\n"
+                        f"Expected {num_images_expected} (from mlf file)\n"
+                        f"Found {num_images_glob} files\n"
+                        "Other parameters:\n"
+                        f"  {image_extension=}\n"
+                        f"  {image_glob_patterns=}"
+                    )
+
+            well_wavelength_ids = []
+            for fpath in well_images:
+                try:
+                    filename_metadata = parse_filename(os.path.basename(fpath))
+                    well_wavelength_ids.append(
+                        f"A{filename_metadata['A']}_C{filename_metadata['C']}"
+                    )
+                except IndexError:
+                    logger.info(f"Skipping {fpath}")
+            well_wavelength_ids = sorted(list(set(well_wavelength_ids)))
+            if well_wavelength_ids != actual_wavelength_ids:
+                raise ValueError(
+                    f"ERROR: well {well} in plate {plate} (prefix: "
+                    f"{plate_prefix}) has missing channels.\n"
+                    f"Expected: {actual_channels}\n"
+                    f"Found: {well_wavelength_ids}.\n"
+                )
+
+        well_rows_columns = [
+            ind for ind in sorted([(n[0], n[1:]) for n in wells])
+        ]
+        row_list = [
+            well_row_column[0] for well_row_column in well_rows_columns
+        ]
+        col_list = [
+            well_row_column[1] for well_row_column in well_rows_columns
+        ]
+        row_list = sorted(list(set(row_list)))
+        col_list = sorted(list(set(col_list)))
+
+        group_plate.attrs["plate"] = {
+            "acquisitions": [{"id": 0, "name": plate}],
+            "columns": [{"name": col} for col in col_list],
+            "rows": [{"name": row} for row in row_list],
+            "wells": [
+                {
+                    "path": well_row_column[0] + "/" + well_row_column[1],
+                    "rowIndex": row_list.index(well_row_column[0]),
+                    "columnIndex": col_list.index(well_row_column[1]),
+                }
+                for well_row_column in well_rows_columns
+            ],
+        }
+
+        for row, column in well_rows_columns:
+
+            group_well = group_plate.create_group(f"{row}/{column}/")
+
+            group_well.attrs["well"] = {
+                "images": [{"path": "0"}],
+                "version": __OME_NGFF_VERSION__,
+            }
+
+            group_image = group_well.create_group("0/")  # noqa: F841
+            zarrurls["well"].append(f"{plate}.zarr/{row}/{column}/")
+            zarrurls["image"].append(f"{plate}.zarr/{row}/{column}/0/")
+
+            group_image.attrs["multiscales"] = [
+                {
+                    "version": __OME_NGFF_VERSION__,
+                    "axes": [
+                        {"name": "c", "type": "channel"},
+                        {
+                            "name": "z",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                        {
+                            "name": "y",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                        {
+                            "name": "x",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                    ],
+                    "datasets": [
+                        {
+                            "path": f"{ind_level}",
+                            "coordinateTransformations": [
+                                {
+                                    "type": "scale",
+                                    "scale": [
+                                        1,
+                                        pixel_size_z,
+                                        pixel_size_y
+                                        * coarsening_xy**ind_level,
+                                        pixel_size_x
+                                        * coarsening_xy**ind_level,
+                                    ],
+                                }
+                            ],
+                        }
+                        for ind_level in range(num_levels)
+                    ],
+                }
+            ]
+
+            group_image.attrs["omero"] = {
+                "id": 1,  # FIXME does this depend on the plate number?
+                "name": "TBD",
+                "version": __OME_NGFF_VERSION__,
+                "channels": define_omero_channels(
+                    channels=actual_channels, bit_depth=bit_depth
+                ),
+            }
+
+            # Prepare AnnData tables for FOV/well ROIs
+            well_id = row + column
+            FOV_ROIs_table = prepare_FOV_ROI_table(site_metadata.loc[well_id])
+            well_ROIs_table = prepare_well_ROI_table(
+                site_metadata.loc[well_id]
+            )
+
+            # Write AnnData tables into the `tables` zarr group
+            write_table(
+                group_image,
+                "FOV_ROI_table",
+                FOV_ROIs_table,
+                overwrite=overwrite,
+                logger=logger,
+            )
+            write_table(
+                group_image,
+                "well_ROI_table",
+                well_ROIs_table,
+                overwrite=overwrite,
+                logger=logger,
+            )
+
+    # Check that the different images in each well have unique channel labels.
+    # Since we currently merge all fields of view in the same image, this check
+    # is useless. It should remain there to catch an error in case we switch
+    # back to one-image-per-field-of-view mode
+    for well_path in zarrurls["well"]:
+        check_well_channel_labels(
+            well_zarr_path=str(Path(output_path) / well_path)
+        )
+
+    metadata_update = dict(
+        plate=zarrurls["plate"],
+        well=zarrurls["well"],
+        image=zarrurls["image"],
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        image_extension=image_extension,
+        image_glob_patterns=image_glob_patterns,
+        original_paths=input_paths[:],
+    )
+    return metadata_update
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/index.html b/reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/index.html new file mode 100644 index 000000000..c8a8719aa --- /dev/null +++ b/reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/index.html @@ -0,0 +1,2331 @@ + + + + + + + + + + + + + + + + + + + + + + create_ome_zarr_multiplex - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

create_ome_zarr_multiplex

+ +
+ + + + +
+ +

Create OME-NGFF zarr group, for multiplexing dataset.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ create_ome_zarr_multiplex(*, input_paths, output_path, metadata, allowed_channels, image_glob_patterns=None, num_levels=5, coarsening_xy=2, image_extension='tif', metadata_table_files=None, overwrite=False) + +

+ + +
+ +

Create OME-NGFF structure and metadata to host a multiplexing dataset.

+

This task takes a set of image folders (i.e. different acquisition cycles) +and build the internal structure and metadata of a OME-NGFF zarr group, +without actually loading/writing the image data.

+

Each element in input_paths should be treated as a different acquisition.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data from the +microscope is stored (as TIF or PNG). Each element of the list is +treated as another cycle of the multiplexing data, the cycles are +ordered by their order in this list. Should point to the parent +folder containing the images and the metadata files +MeasurementData.mlf and MeasurementDetail.mrf (if present). +Example: ["/path/cycle1/", "/path/cycle2/"]. (standard argument +for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Path were the output of this task is stored. +Example: "/some/path/" => puts the new OME-Zarr file in the +/some/path/. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
allowed_channels +
+

A dictionary of lists of OmeroChannels, where +each channel must include the wavelength_id attribute and where +the wavelength_id values must be unique across each list. +Dictionary keys represent channel indices ("0","1",..).

+
+

+ + TYPE: + dict[str, list[OmeroChannel]] + +

+
image_glob_patterns +
+

If specified, only parse images with filenames +that match with all these patterns. Patterns must be defined as in +https://docs.python.org/3/library/fnmatch.html, Example: +image_glob_pattern=["*_B03_*"] => only process well B03 +image_glob_pattern=["*_C09_*", "*F016*", "*Z[0-5][0-9]C*"] => +only process well C09, field of view 16 and Z planes 0-59.

+
+

+ + TYPE: + Optional[list[str]] + + + DEFAULT: + None + +

+
num_levels +
+

Number of resolution-pyramid levels. If set to 5, there +will be the full-resolution level and 4 levels of downsampled +images.

+
+

+ + TYPE: + int + + + DEFAULT: + 5 + +

+
coarsening_xy +
+

Linear coarsening factor between subsequent levels. +If set to 2, level 1 is 2x downsampled, level 2 is 4x downsampled +etc.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
image_extension +
+

Filename extension of images +(e.g. "tif" or "png").

+
+

+ + TYPE: + str + + + DEFAULT: + 'tif' + +

+
metadata_table_files +
+

If None, parse Yokogawa metadata from mrf/mlf +files in the input_path folder; else, a dictionary of key-value +pairs like (acquisition, path) with acquisition a string +and path pointing to a csv file containing the parsed metadata +table.

+
+

+ + TYPE: + Optional[dict[str, str]] + + + DEFAULT: + None + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ + + + + + + + + + + + + + + + +
RETURNSDESCRIPTION
+ + dict[str, Any] + + +
+

A metadata dictionary containing important metadata about the OME-Zarr +plate, the images and some parameters required by downstream tasks +(like num_levels).

+
+
+ +
+ Source code in fractal_tasks_core/tasks/create_ome_zarr_multiplex.py + 
 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
@validate_arguments
+def create_ome_zarr_multiplex(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    metadata: dict[str, Any],
+    allowed_channels: dict[str, list[OmeroChannel]],
+    image_glob_patterns: Optional[list[str]] = None,
+    num_levels: int = 5,
+    coarsening_xy: int = 2,
+    image_extension: str = "tif",
+    metadata_table_files: Optional[dict[str, str]] = None,
+    overwrite: bool = False,
+) -> dict[str, Any]:
+    """
+    Create OME-NGFF structure and metadata to host a multiplexing dataset.
+
+    This task takes a set of image folders (i.e. different acquisition cycles)
+    and build the internal structure and metadata of a OME-NGFF zarr group,
+    without actually loading/writing the image data.
+
+    Each element in input_paths should be treated as a different acquisition.
+
+    Args:
+        input_paths: List of input paths where the image data from the
+            microscope is stored (as TIF or PNG).  Each element of the list is
+            treated as another cycle of the multiplexing data, the cycles are
+            ordered by their order in this list.  Should point to the parent
+            folder containing the images and the metadata files
+            `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present).
+            Example: `["/path/cycle1/", "/path/cycle2/"]`. (standard argument
+            for Fractal tasks, managed by Fractal server).
+        output_path: Path were the output of this task is stored.
+            Example: `"/some/path/"` => puts the new OME-Zarr file in the
+            `/some/path/`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        allowed_channels: A dictionary of lists of `OmeroChannel`s, where
+            each channel must include the `wavelength_id` attribute and where
+            the `wavelength_id` values must be unique across each list.
+            Dictionary keys represent channel indices (`"0","1",..`).
+        image_glob_patterns: If specified, only parse images with filenames
+            that match with all these patterns. Patterns must be defined as in
+            https://docs.python.org/3/library/fnmatch.html, Example:
+            `image_glob_pattern=["*_B03_*"]` => only process well B03
+            `image_glob_pattern=["*_C09_*", "*F016*", "*Z[0-5][0-9]C*"]` =>
+            only process well C09, field of view 16 and Z planes 0-59.
+        num_levels: Number of resolution-pyramid levels. If set to `5`, there
+            will be the full-resolution level and 4 levels of downsampled
+            images.
+        coarsening_xy: Linear coarsening factor between subsequent levels.
+            If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled
+            etc.
+        image_extension: Filename extension of images
+            (e.g. `"tif"` or `"png"`).
+        metadata_table_files: If `None`, parse Yokogawa metadata from mrf/mlf
+            files in the input_path folder; else, a dictionary of key-value
+            pairs like `(acquisition, path)` with `acquisition` a string
+            and `path` pointing to a csv file containing the parsed metadata
+            table.
+        overwrite: If `True`, overwrite the task output.
+
+    Returns:
+        A metadata dictionary containing important metadata about the OME-Zarr
+            plate, the images and some parameters required by downstream tasks
+            (like `num_levels`).
+    """
+
+    if metadata_table_files:
+
+        # Checks on the dict:
+        # 1. Acquisitions as keys (same as keys of allowed_channels)
+        # 2. Files end with ".csv"
+        # 3. Files exist.
+        if set(allowed_channels.keys()) != set(metadata_table_files.keys()):
+            raise ValueError(
+                "Mismatch in acquisition keys between "
+                f"{allowed_channels.keys()=} and "
+                f"{metadata_table_files.keys()=}"
+            )
+        for f in metadata_table_files.values():
+            if not f.endswith(".csv"):
+                raise ValueError(
+                    f"{f} (in metadata_table_file) is not a csv file."
+                )
+            if not os.path.isfile(f):
+                raise ValueError(
+                    f"{f} (in metadata_table_file) does not exist."
+                )
+
+    # Preliminary checks on allowed_channels
+    # Note that in metadata the keys of dictionary arguments should be
+    # strings (and not integers), so that they can be read from a JSON file
+    for key, _channels in allowed_channels.items():
+        if not isinstance(key, str):
+            raise ValueError(f"{allowed_channels=} has non-string keys")
+        check_unique_wavelength_ids(_channels)
+
+    # Identify all plates and all channels, per input folders
+    dict_acquisitions: dict = {}
+
+    for ind_in_path, in_path_str in enumerate(input_paths):
+        acquisition = str(ind_in_path)
+        in_path = Path(in_path_str)
+        dict_acquisitions[acquisition] = {}
+
+        actual_wavelength_ids = []
+        plates = []
+        plate_prefixes = []
+
+        # Loop over all images
+        patterns = [f"*.{image_extension}"]
+        if image_glob_patterns:
+            patterns.extend(image_glob_patterns)
+        input_filenames = glob_with_multiple_patterns(
+            folder=in_path_str,
+            patterns=patterns,
+        )
+        for fn in input_filenames:
+            try:
+                filename_metadata = parse_filename(Path(fn).name)
+                plate = filename_metadata["plate"]
+                plates.append(plate)
+                plate_prefix = filename_metadata["plate_prefix"]
+                plate_prefixes.append(plate_prefix)
+                A = filename_metadata["A"]
+                C = filename_metadata["C"]
+                actual_wavelength_ids.append(f"A{A}_C{C}")
+            except ValueError as e:
+                logger.warning(
+                    f'Skipping "{Path(fn).name}". Original error: ' + str(e)
+                )
+        plates = sorted(list(set(plates)))
+        actual_wavelength_ids = sorted(list(set(actual_wavelength_ids)))
+
+        info = (
+            "Listing all plates/channels:\n"
+            f"Patterns: {patterns}\n"
+            f"Plates:   {plates}\n"
+            f"Actual wavelength IDs: {actual_wavelength_ids}\n"
+        )
+
+        # Check that a folder includes a single plate
+        if len(plates) > 1:
+            raise ValueError(f"{info}ERROR: {len(plates)} plates detected")
+        elif len(plates) == 0:
+            raise ValueError(f"{info}ERROR: No plates detected")
+        original_plate = plates[0]
+        plate_prefix = plate_prefixes[0]
+
+        # Replace plate with the one of acquisition 0, if needed
+        if int(acquisition) > 0:
+            plate = dict_acquisitions["0"]["plate"]
+            logger.warning(
+                f"For {acquisition=}, we replace {original_plate=} with "
+                f"{plate=} (the one for acquisition 0)"
+            )
+
+        # Check that all channels are in the allowed_channels
+        allowed_wavelength_ids = [
+            c.wavelength_id for c in allowed_channels[acquisition]
+        ]
+        if not set(actual_wavelength_ids).issubset(
+            set(allowed_wavelength_ids)
+        ):
+            msg = "ERROR in create_ome_zarr\n"
+            msg += f"actual_wavelength_ids: {actual_wavelength_ids}\n"
+            msg += f"allowed_wavelength_ids: {allowed_wavelength_ids}\n"
+            raise ValueError(msg)
+
+        # Create actual_channels, i.e. a list of the channel dictionaries which
+        # are present
+        actual_channels = [
+            channel
+            for channel in allowed_channels[acquisition]
+            if channel.wavelength_id in actual_wavelength_ids
+        ]
+
+        logger.info(f"plate: {plate}")
+        logger.info(f"actual_channels: {actual_channels}")
+
+        dict_acquisitions[acquisition] = {}
+        dict_acquisitions[acquisition]["plate"] = plate
+        dict_acquisitions[acquisition]["original_plate"] = original_plate
+        dict_acquisitions[acquisition]["plate_prefix"] = plate_prefix
+        dict_acquisitions[acquisition]["image_folder"] = in_path
+        dict_acquisitions[acquisition]["original_paths"] = [in_path]
+        dict_acquisitions[acquisition]["actual_channels"] = actual_channels
+        dict_acquisitions[acquisition][
+            "actual_wavelength_ids"
+        ] = actual_wavelength_ids
+
+    acquisitions = sorted(list(dict_acquisitions.keys()))
+    current_plates = [item["plate"] for item in dict_acquisitions.values()]
+    if len(set(current_plates)) > 1:
+        raise ValueError(f"{current_plates=}")
+    plate = current_plates[0]
+
+    zarrurl = dict_acquisitions[acquisitions[0]]["plate"] + ".zarr"
+    full_zarrurl = str(Path(output_path) / zarrurl)
+    logger.info(f"Creating {full_zarrurl=}")
+    # Call zarr.open_group wrapper, which handles overwrite=True/False
+    group_plate = open_zarr_group_with_overwrite(
+        full_zarrurl, overwrite=overwrite
+    )
+    group_plate.attrs["plate"] = {
+        "acquisitions": [
+            {
+                "id": int(acquisition),
+                "name": dict_acquisitions[acquisition]["original_plate"],
+            }
+            for acquisition in acquisitions
+        ]
+    }
+
+    zarrurls: dict[str, list[str]] = {"well": [], "image": []}
+    zarrurls["plate"] = [f"{plate}.zarr"]
+
+    ################################################################
+    logging.info(f"{acquisitions=}")
+
+    for acquisition in acquisitions:
+
+        # Define plate zarr
+        image_folder = dict_acquisitions[acquisition]["image_folder"]
+        logger.info(f"Looking at {image_folder=}")
+
+        # Obtain FOV-metadata dataframe
+        if metadata_table_files is None:
+            mrf_path = f"{image_folder}/MeasurementDetail.mrf"
+            mlf_path = f"{image_folder}/MeasurementData.mlf"
+            site_metadata, total_files = parse_yokogawa_metadata(
+                mrf_path, mlf_path, filename_patterns=image_glob_patterns
+            )
+            site_metadata = remove_FOV_overlaps(site_metadata)
+        else:
+            site_metadata = pd.read_csv(metadata_table_files[acquisition])
+            site_metadata.set_index(["well_id", "FieldIndex"], inplace=True)
+
+        # Extract pixel sizes and bit_depth
+        pixel_size_z = site_metadata["pixel_size_z"][0]
+        pixel_size_y = site_metadata["pixel_size_y"][0]
+        pixel_size_x = site_metadata["pixel_size_x"][0]
+        bit_depth = site_metadata["bit_depth"][0]
+
+        if min(pixel_size_z, pixel_size_y, pixel_size_x) < 1e-9:
+            raise ValueError(pixel_size_z, pixel_size_y, pixel_size_x)
+
+        # Identify all wells
+        plate_prefix = dict_acquisitions[acquisition]["plate_prefix"]
+        patterns = [f"{plate_prefix}_*.{image_extension}"]
+        if image_glob_patterns:
+            patterns.extend(image_glob_patterns)
+        plate_images = glob_with_multiple_patterns(
+            folder=str(image_folder),
+            patterns=patterns,
+        )
+
+        wells = [
+            parse_filename(os.path.basename(fn))["well"] for fn in plate_images
+        ]
+        wells = sorted(list(set(wells)))
+        logger.info(f"{wells=}")
+
+        # Verify that all wells have all channels
+        actual_channels = dict_acquisitions[acquisition]["actual_channels"]
+        for well in wells:
+            patterns = [f"{plate_prefix}_{well}_*.{image_extension}"]
+            if image_glob_patterns:
+                patterns.extend(image_glob_patterns)
+            well_images = glob_with_multiple_patterns(
+                folder=str(image_folder),
+                patterns=patterns,
+            )
+
+            well_wavelength_ids = []
+            for fpath in well_images:
+                try:
+                    filename_metadata = parse_filename(os.path.basename(fpath))
+                    A = filename_metadata["A"]
+                    C = filename_metadata["C"]
+                    well_wavelength_ids.append(f"A{A}_C{C}")
+                except IndexError:
+                    logger.info(f"Skipping {fpath}")
+            well_wavelength_ids = sorted(list(set(well_wavelength_ids)))
+            actual_wavelength_ids = dict_acquisitions[acquisition][
+                "actual_wavelength_ids"
+            ]
+            if well_wavelength_ids != actual_wavelength_ids:
+                raise ValueError(
+                    f"ERROR: well {well} in plate {plate} (prefix: "
+                    f"{plate_prefix}) has missing channels.\n"
+                    f"Expected: {actual_wavelength_ids}\n"
+                    f"Found: {well_wavelength_ids}.\n"
+                )
+
+        well_rows_columns = [
+            ind for ind in sorted([(n[0], n[1:]) for n in wells])
+        ]
+        row_list = [
+            well_row_column[0] for well_row_column in well_rows_columns
+        ]
+        col_list = [
+            well_row_column[1] for well_row_column in well_rows_columns
+        ]
+        row_list = sorted(list(set(row_list)))
+        col_list = sorted(list(set(col_list)))
+
+        plate_attrs = group_plate.attrs["plate"]
+        plate_attrs["columns"] = [{"name": col} for col in col_list]
+        plate_attrs["rows"] = [{"name": row} for row in row_list]
+        plate_attrs["wells"] = [
+            {
+                "path": well_row_column[0] + "/" + well_row_column[1],
+                "rowIndex": row_list.index(well_row_column[0]),
+                "columnIndex": col_list.index(well_row_column[1]),
+            }
+            for well_row_column in well_rows_columns
+        ]
+        group_plate.attrs["plate"] = plate_attrs
+
+        for row, column in well_rows_columns:
+
+            try:
+                group_well = group_plate.create_group(f"{row}/{column}/")
+                logging.info(f"Created new group_well at {row}/{column}/")
+                group_well.attrs["well"] = {
+                    "images": [
+                        {
+                            "path": f"{acquisition}",
+                            "acquisition": int(acquisition),
+                        }
+                    ],
+                    "version": __OME_NGFF_VERSION__,
+                }
+                zarrurls["well"].append(f"{plate}.zarr/{row}/{column}")
+            except ContainsGroupError:
+                group_well = zarr.open_group(
+                    f"{full_zarrurl}/{row}/{column}/", mode="r+"
+                )
+                logging.info(
+                    f"Loaded group_well from {full_zarrurl}/{row}/{column}"
+                )
+                current_images = group_well.attrs["well"]["images"] + [
+                    {"path": f"{acquisition}", "acquisition": int(acquisition)}
+                ]
+                group_well.attrs["well"] = dict(
+                    images=current_images,
+                    version=group_well.attrs["well"]["version"],
+                )
+
+            group_image = group_well.create_group(
+                f"{acquisition}/"
+            )  # noqa: F841
+            logging.info(f"Created image group {row}/{column}/{acquisition}")
+            image = f"{plate}.zarr/{row}/{column}/{acquisition}"
+            zarrurls["image"].append(image)
+
+            group_image.attrs["multiscales"] = [
+                {
+                    "version": __OME_NGFF_VERSION__,
+                    "axes": [
+                        {"name": "c", "type": "channel"},
+                        {
+                            "name": "z",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                        {
+                            "name": "y",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                        {
+                            "name": "x",
+                            "type": "space",
+                            "unit": "micrometer",
+                        },
+                    ],
+                    "datasets": [
+                        {
+                            "path": f"{ind_level}",
+                            "coordinateTransformations": [
+                                {
+                                    "type": "scale",
+                                    "scale": [
+                                        1,
+                                        pixel_size_z,
+                                        pixel_size_y
+                                        * coarsening_xy**ind_level,
+                                        pixel_size_x
+                                        * coarsening_xy**ind_level,
+                                    ],
+                                }
+                            ],
+                        }
+                        for ind_level in range(num_levels)
+                    ],
+                }
+            ]
+
+            group_image.attrs["omero"] = {
+                "id": 1,  # FIXME does this depend on the plate number?
+                "name": "TBD",
+                "version": __OME_NGFF_VERSION__,
+                "channels": define_omero_channels(
+                    channels=actual_channels,
+                    bit_depth=bit_depth,
+                    label_prefix=acquisition,
+                ),
+            }
+
+            # Prepare AnnData tables for FOV/well ROIs
+            well_id = row + column
+            FOV_ROIs_table = prepare_FOV_ROI_table(site_metadata.loc[well_id])
+            well_ROIs_table = prepare_well_ROI_table(
+                site_metadata.loc[well_id]
+            )
+
+            # Write AnnData tables into the `tables` zarr group
+            write_table(
+                group_image,
+                "FOV_ROI_table",
+                FOV_ROIs_table,
+                overwrite=overwrite,
+                logger=logger,
+            )
+            write_table(
+                group_image,
+                "well_ROI_table",
+                well_ROIs_table,
+                overwrite=overwrite,
+                logger=logger,
+            )
+
+    # Check that the different images (e.g. different cycles) in the each well
+    # have unique labels
+    for well_path in zarrurls["well"]:
+        check_well_channel_labels(
+            well_zarr_path=str(Path(output_path) / well_path)
+        )
+
+    original_paths = {
+        acquisition: dict_acquisitions[acquisition]["original_paths"]
+        for acquisition in acquisitions
+    }
+
+    metadata_update = dict(
+        plate=zarrurls["plate"],
+        well=zarrurls["well"],
+        image=zarrurls["image"],
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        original_paths=original_paths,
+        image_extension=image_extension,
+        image_glob_patterns=image_glob_patterns,
+    )
+    return metadata_update
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/illumination_correction/index.html b/reference/fractal_tasks_core/tasks/illumination_correction/index.html new file mode 100644 index 000000000..4d8be5029 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/illumination_correction/index.html @@ -0,0 +1,1964 @@ + + + + + + + + + + + + + + + + + + + + + + illumination_correction - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

illumination_correction

+ +
+ + + + +
+ +

Apply illumination correction to all fields of view.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ correct(img_stack, corr_img, background=110) + +

+ + +
+ +

Corrects a stack of images, using a given illumination profile (e.g. bright +in the center of the image, dim outside).

+ + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
img_stack +
+

4D numpy array (czyx), with dummy size along c.

+
+

+ + TYPE: + ndarray + +

+
corr_img +
+

2D numpy array (yx)

+
+

+ + TYPE: + ndarray + +

+
background +
+

Background value that is subtracted from the image before +the illumination correction is applied.

+
+

+ + TYPE: + int + + + DEFAULT: + 110 + +

+
+ +
+ Source code in fractal_tasks_core/tasks/illumination_correction.py + 
42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
def correct(
+    img_stack: np.ndarray,
+    corr_img: np.ndarray,
+    background: int = 110,
+):
+    """
+    Corrects a stack of images, using a given illumination profile (e.g. bright
+    in the center of the image, dim outside).
+
+    Args:
+        img_stack: 4D numpy array (czyx), with dummy size along c.
+        corr_img: 2D numpy array (yx)
+        background: Background value that is subtracted from the image before
+            the illumination correction is applied.
+    """
+
+    logger.info(f"Start correct, {img_stack.shape}")
+
+    # Check shapes
+    if corr_img.shape != img_stack.shape[2:] or img_stack.shape[0] != 1:
+        raise ValueError(
+            "Error in illumination_correction:\n"
+            f"{img_stack.shape=}\n{corr_img.shape=}"
+        )
+
+    # Store info about dtype
+    dtype = img_stack.dtype
+    dtype_max = np.iinfo(dtype).max
+
+    # Background subtraction
+    img_stack[img_stack <= background] = 0
+    img_stack[img_stack > background] -= background
+
+    #  Apply the normalized correction matrix (requires a float array)
+    # img_stack = img_stack.astype(np.float64)
+    new_img_stack = img_stack / (corr_img / np.max(corr_img))[None, None, :, :]
+
+    # Handle edge case: corrected image may have values beyond the limit of
+    # the encoding, e.g. beyond 65535 for 16bit images. This clips values
+    # that surpass this limit and triggers a warning
+    if np.sum(new_img_stack > dtype_max) > 0:
+        warnings.warn(
+            "Illumination correction created values beyond the max range of "
+            f"the current image type. These have been clipped to {dtype_max=}."
+        )
+        new_img_stack[new_img_stack > dtype_max] = dtype_max
+
+    logger.info("End correct")
+
+    # Cast back to original dtype and return
+    return new_img_stack.astype(dtype)
+
+
+
+ +
+ + +
+ + + + +

+ illumination_correction(*, input_paths, output_path, component, metadata, illumination_profiles_folder, dict_corr, background=110, overwrite_input=True, new_component=None) + +

+ + +
+ +

Applies illumination correction to the images in the OME-Zarr.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. Example: +["/some/path/"]. This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Path were the output of this task is stored. Examples: +"/some/path/" => puts the new OME-Zarr file in the same folder as +the input OME-Zarr file; "/some/new_path" => puts the new +OME-Zarr file into a new folder at /some/new_path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
illumination_profiles_folder +
+

Path of folder of illumination profiles.

+
+

+ + TYPE: + str + +

+
dict_corr +
+

Dictionary where keys match the wavelength_id attributes +of existing channels (e.g. A01_C01 ) and values are the +filenames of the corresponding illumination profiles.

+
+

+ + TYPE: + dict[str, str] + +

+
background +
+

Background value that is subtracted from the image before +the illumination correction is applied. Set it to 0 if you don't +want any background subtraction.

+
+

+ + TYPE: + int + + + DEFAULT: + 110 + +

+
overwrite_input +
+

If True, the results of this task will overwrite the input image +data. In the current version, overwrite_input=False is not +implemented.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
new_component +
+

Not implemented yet. This is not implemented well in +Fractal server at the moment, it's unclear how a user would specify +fitting new components. If the results shall not overwrite the +input data and the output path is the same as the input path, a new +component needs to be provided. +Example: myplate_new_name.zarr/B/03/0/.

+
+

+ + TYPE: + Optional[str] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/tasks/illumination_correction.py + 
 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
@validate_arguments
+def illumination_correction(
+    *,
+    # Standard arguments
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments
+    illumination_profiles_folder: str,
+    dict_corr: dict[str, str],
+    background: int = 110,
+    overwrite_input: bool = True,
+    new_component: Optional[str] = None,
+) -> dict[str, Any]:
+
+    """
+    Applies illumination correction to the images in the OME-Zarr.
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file. Example:
+            `["/some/path/"]`. This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: Path were the output of this task is stored. Examples:
+            `"/some/path/"` => puts the new OME-Zarr file in the same folder as
+            the input OME-Zarr file; `"/some/new_path"` => puts the new
+            OME-Zarr file into a new folder at `/some/new_path`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        illumination_profiles_folder: Path of folder of illumination profiles.
+        dict_corr: Dictionary where keys match the `wavelength_id` attributes
+            of existing channels (e.g.  `A01_C01` ) and values are the
+            filenames of the corresponding illumination profiles.
+        background: Background value that is subtracted from the image before
+            the illumination correction is applied. Set it to `0` if you don't
+            want any background subtraction.
+        overwrite_input:
+            If `True`, the results of this task will overwrite the input image
+            data. In the current version, `overwrite_input=False` is not
+            implemented.
+        new_component: Not implemented yet. This is not implemented well in
+            Fractal server at the moment, it's unclear how a user would specify
+            fitting new components. If the results shall not overwrite the
+            input data and the output path is the same as the input path, a new
+            component needs to be provided.
+            Example: `myplate_new_name.zarr/B/03/0/`.
+    """
+
+    # Preliminary checks
+    if len(input_paths) > 1:
+        raise NotImplementedError
+    if (overwrite_input and new_component is not None) or (
+        new_component is None and not overwrite_input
+    ):
+        raise ValueError(f"{overwrite_input=}, but {new_component=}")
+
+    if not overwrite_input:
+        msg = (
+            "We still have to harmonize illumination_correction("
+            "overwrite_input=False) with replicate_zarr_structure(..., "
+            "suffix=..)"
+        )
+        raise NotImplementedError(msg)
+
+    # Defione old/new zarrurls
+    plate, well = component.split(".zarr/")
+    in_path = Path(input_paths[0])
+    zarrurl_old = (in_path / component).as_posix()
+    if overwrite_input:
+        zarrurl_new = zarrurl_old
+    else:
+        new_plate, new_well = new_component.split(".zarr/")
+        if new_well != well:
+            raise ValueError(f"{well=}, {new_well=}")
+        zarrurl_new = (Path(output_path) / new_component).as_posix()
+
+    t_start = time.perf_counter()
+    logger.info("Start illumination_correction")
+    logger.info(f"  {overwrite_input=}")
+    logger.info(f"  {zarrurl_old=}")
+    logger.info(f"  {zarrurl_new=}")
+
+    # Read attributes from NGFF metadata
+    ngff_image_meta = load_NgffImageMeta(zarrurl_old)
+    num_levels = ngff_image_meta.num_levels
+    coarsening_xy = ngff_image_meta.coarsening_xy
+    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+    logger.info(f"NGFF image has {num_levels=}")
+    logger.info(f"NGFF image has {coarsening_xy=}")
+    logger.info(
+        f"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}"
+    )
+
+    # Read channels from .zattrs
+    channels: list[OmeroChannel] = get_omero_channel_list(
+        image_zarr_path=zarrurl_old
+    )
+    num_channels = len(channels)
+
+    # Read FOV ROIs
+    FOV_ROI_table = ad.read_zarr(f"{zarrurl_old}/tables/FOV_ROI_table")
+
+    # Create list of indices for 3D FOVs spanning the entire Z direction
+    list_indices = convert_ROI_table_to_indices(
+        FOV_ROI_table,
+        level=0,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(list_indices, "FOV_ROI_table")
+
+    # Extract image size from FOV-ROI indices. Note: this works at level=0,
+    # where FOVs should all be of the exact same size (in pixels)
+    ref_img_size = None
+    for indices in list_indices:
+        img_size = (indices[3] - indices[2], indices[5] - indices[4])
+        if ref_img_size is None:
+            ref_img_size = img_size
+        else:
+            if img_size != ref_img_size:
+                raise ValueError(
+                    "ERROR: inconsistent image sizes in list_indices"
+                )
+    img_size_y, img_size_x = img_size[:]
+
+    # Assemble dictionary of matrices and check their shapes
+    corrections = {}
+    for channel in channels:
+        wavelength_id = channel.wavelength_id
+        corrections[wavelength_id] = imread(
+            (
+                Path(illumination_profiles_folder) / dict_corr[wavelength_id]
+            ).as_posix()
+        )
+        if corrections[wavelength_id].shape != (img_size_y, img_size_x):
+            raise ValueError(
+                "Error in illumination_correction, "
+                "correction matrix has wrong shape."
+            )
+
+    # Lazily load highest-res level from original zarr array
+    data_czyx = da.from_zarr(f"{zarrurl_old}/0")
+
+    # Create zarr for output
+    if overwrite_input:
+        fov_path = zarrurl_old
+        new_zarr = zarr.open(f"{zarrurl_old}/0")
+    else:
+        fov_path = zarrurl_new
+        new_zarr = zarr.create(
+            shape=data_czyx.shape,
+            chunks=data_czyx.chunksize,
+            dtype=data_czyx.dtype,
+            store=zarr.storage.FSStore(f"{zarrurl_new}/0"),
+            overwrite=False,
+            dimension_separator="/",
+        )
+
+    # Iterate over FOV ROIs
+    num_ROIs = len(list_indices)
+    for i_c, channel in enumerate(channels):
+        for i_ROI, indices in enumerate(list_indices):
+            # Define region
+            s_z, e_z, s_y, e_y, s_x, e_x = indices[:]
+            region = (
+                slice(i_c, i_c + 1),
+                slice(s_z, e_z),
+                slice(s_y, e_y),
+                slice(s_x, e_x),
+            )
+            logger.info(
+                f"Now processing ROI {i_ROI+1}/{num_ROIs} "
+                f"for channel {i_c+1}/{num_channels}"
+            )
+            # Execute illumination correction
+            corrected_fov = correct(
+                data_czyx[region].compute(),
+                corrections[channel.wavelength_id],
+                background=background,
+            )
+            # Write to disk
+            da.array(corrected_fov).to_zarr(
+                url=new_zarr,
+                region=region,
+                compute=True,
+            )
+
+    # Starting from on-disk highest-resolution data, build and write to disk a
+    # pyramid of coarser levels
+    build_pyramid(
+        zarrurl=fov_path,
+        overwrite=True,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        chunksize=data_czyx.chunksize,
+    )
+
+    t_end = time.perf_counter()
+    logger.info(f"End illumination_correction, elapsed: {t_end-t_start}")
+
+    return {}
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/import_ome_zarr/index.html b/reference/fractal_tasks_core/tasks/import_ome_zarr/index.html new file mode 100644 index 000000000..012cbfdd7 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/import_ome_zarr/index.html @@ -0,0 +1,1983 @@ + + + + + + + + + + + + + + + + + + + + + + import_ome_zarr - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

import_ome_zarr

+ +
+ + + + +
+ +

Task to import an existing OME-Zarr.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ _process_single_image(image_path, add_image_ROI_table, add_grid_ROI_table, update_omero_metadata, *, grid_YX_shape=None, overwrite=False) + +

+ + +
+ +

Validate OME-NGFF metadata and optionally generate ROI tables.

+

This task:

+
    +
  1. Validates OME-NGFF image metadata, via NgffImageMeta;
    2. +
  2. Optionally generates and writes two ROI tables;
    3. +
  3. Optionally update OME-NGFF omero metadata.
    4. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
image_path +
+

Absolute path to the image Zarr group.

+
+

+ + TYPE: + str + +

+
add_image_ROI_table +
+

Whether to add a image_ROI_table table +(argument propagated from import_ome_zarr).

+
+

+ + TYPE: + bool + +

+
add_grid_ROI_table +
+

Whether to add a grid_ROI_table table (argument +propagated from import_ome_zarr).

+
+

+ + TYPE: + bool + +

+
update_omero_metadata +
+

Whether to update Omero-channels metadata +(argument propagated from import_ome_zarr).

+
+

+ + TYPE: + bool + +

+
grid_YX_shape +
+

YX shape of the ROI grid (it must be not None, if +add_grid_ROI_table=True.

+
+

+ + TYPE: + Optional[tuple[int, int]] + + + DEFAULT: + None + +

+
+ +
+ Source code in fractal_tasks_core/tasks/import_ome_zarr.py + 
 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
def _process_single_image(
+    image_path: str,
+    add_image_ROI_table: bool,
+    add_grid_ROI_table: bool,
+    update_omero_metadata: bool,
+    *,
+    grid_YX_shape: Optional[tuple[int, int]] = None,
+    overwrite: bool = False,
+) -> None:
+    """
+    Validate OME-NGFF metadata and optionally generate ROI tables.
+
+    This task:
+
+    1. Validates OME-NGFF image metadata, via `NgffImageMeta`;
+    2. Optionally generates and writes two ROI tables;
+    3. Optionally update OME-NGFF omero metadata.
+
+    Args:
+        image_path: Absolute path to the image Zarr group.
+        add_image_ROI_table: Whether to add a `image_ROI_table` table
+            (argument propagated from `import_ome_zarr`).
+        add_grid_ROI_table: Whether to add a `grid_ROI_table` table (argument
+            propagated from `import_ome_zarr`).
+        update_omero_metadata: Whether to update Omero-channels metadata
+            (argument propagated from `import_ome_zarr`).
+        grid_YX_shape: YX shape of the ROI grid (it must be not `None`, if
+            `add_grid_ROI_table=True`.
+    """
+
+    # Note from zarr docs: `r+` means read/write (must exist)
+    image_group = zarr.open_group(image_path, mode="r+")
+    image_meta = NgffImageMeta(**image_group.attrs.asdict())
+
+    # Preliminary checks
+    if not (add_image_ROI_table or add_grid_ROI_table):
+        return
+    if add_grid_ROI_table and (grid_YX_shape is None):
+        raise ValueError(
+            f"_process_single_image called with {add_grid_ROI_table=}, "
+            f"but {grid_YX_shape=}."
+        )
+
+    pixels_ZYX = image_meta.get_pixel_sizes_zyx(level=0)
+
+    # Read zarr array
+    dataset_subpath = image_meta.datasets[0].path
+    array = da.from_zarr(f"{image_path}/{dataset_subpath}")
+
+    # Prepare image_ROI_table and write it into the zarr group
+    if add_image_ROI_table:
+        image_ROI_table = get_single_image_ROI(array.shape, pixels_ZYX)
+        write_table(
+            image_group,
+            "image_ROI_table",
+            image_ROI_table,
+            overwrite=overwrite,
+            logger=logger,
+        )
+
+    # Prepare grid_ROI_table and write it into the zarr group
+    if add_grid_ROI_table:
+        grid_ROI_table = get_image_grid_ROIs(
+            array.shape,
+            pixels_ZYX,
+            grid_YX_shape,
+        )
+        write_table(
+            image_group,
+            "grid_ROI_table",
+            grid_ROI_table,
+            overwrite=overwrite,
+            logger=logger,
+        )
+
+    # Update Omero-channels metadata
+    if update_omero_metadata:
+        # Extract number of channels from zarr array
+        try:
+            channel_axis_index = image_meta.axes_names.index("c")
+        except ValueError:
+            logger.error(f"Existing axes: {image_meta.axes_names}")
+            msg = (
+                "OME-Zarrs with no channel axis are not currently "
+                "supported in fractal-tasks-core. Upcoming flexibility "
+                "improvements are tracked in https://github.com/"
+                "fractal-analytics-platform/fractal-tasks-core/issues/150."
+            )
+            logger.error(msg)
+            raise NotImplementedError(msg)
+        logger.info(f"Existing axes: {image_meta.axes_names}")
+        logger.info(f"Channel-axis index: {channel_axis_index}")
+        num_channels_zarr = array.shape[channel_axis_index]
+        logger.info(
+            f"{num_channels_zarr} channel(s) found in Zarr array "
+            f"at {image_path}/{dataset_subpath}"
+        )
+        # Update or create omero channels metadata
+        old_omero = image_group.attrs.get("omero", {})
+        old_channels = old_omero.get("channels", [])
+        if len(old_channels) > 0:
+            logger.info(
+                f"{len(old_channels)} channel(s) found in NGFF omero metadata"
+            )
+            if len(old_channels) != num_channels_zarr:
+                error_msg = (
+                    "Channels-number mismatch: Number of channels in the "
+                    f"zarr array ({num_channels_zarr}) differs from number "
+                    "of channels listed in NGFF omero metadata "
+                    f"({len(old_channels)})."
+                )
+                logging.error(error_msg)
+                raise ValueError(error_msg)
+        else:
+            old_channels = [{} for ind in range(num_channels_zarr)]
+        new_channels = update_omero_channels(old_channels)
+        new_omero = old_omero.copy()
+        new_omero["channels"] = new_channels
+        image_group.attrs.update(omero=new_omero)
+
+
+
+ +
+ + +
+ + + + +

+ import_ome_zarr(*, input_paths, output_path, metadata, zarr_name, add_image_ROI_table=True, add_grid_ROI_table=True, grid_y_shape=2, grid_x_shape=2, update_omero_metadata=True, overwrite=False) + +

+ + +
+ +

Import an OME-Zarr into Fractal.

+

The current version of this task:

+
    +
  1. Creates the appropriate components-related metadata, needed for + processing an existing OME-Zarr through Fractal.
    2. +
  2. Optionally adds new ROI tables to the existing OME-Zarr.
    3. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

A length-one list with the parent folder of the OME-Zarr +to be imported; e.g. input_paths=["/somewhere"], if the OME-Zarr +path is /somewhere/array.zarr. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Not used in this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

Not used in this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
zarr_name +
+

The OME-Zarr name, without its parent folder; e.g. +zarr_name="array.zarr", if the OME-Zarr path is +/somewhere/array.zarr.

+
+

+ + TYPE: + str + +

+
add_image_ROI_table +
+

Whether to add a image_ROI_table table to each +image, with a single ROI covering the whole image.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
add_grid_ROI_table +
+

Whether to add a grid_ROI_table table to each +image, with the image split into a rectangular grid of ROIs.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
grid_y_shape +
+

Y shape of the ROI grid in grid_ROI_table.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
grid_x_shape +
+

X shape of the ROI grid in grid_ROI_table.

+
+

+ + TYPE: + int + + + DEFAULT: + 2 + +

+
update_omero_metadata +
+

Whether to update Omero-channels metadata, to +make them Fractal-compatible.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
overwrite +
+

Whether new ROI tables (added when add_image_ROI_table +and/or add_grid_ROI_table are True) can overwite existing ones.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ +
+ Source code in fractal_tasks_core/tasks/import_ome_zarr.py + 
155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
@validate_arguments
+def import_ome_zarr(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    metadata: dict[str, Any],
+    zarr_name: str,
+    add_image_ROI_table: bool = True,
+    add_grid_ROI_table: bool = True,
+    grid_y_shape: int = 2,
+    grid_x_shape: int = 2,
+    update_omero_metadata: bool = True,
+    overwrite: bool = False,
+) -> dict[str, Any]:
+    """
+    Import an OME-Zarr into Fractal.
+
+    The current version of this task:
+
+    1. Creates the appropriate components-related metadata, needed for
+       processing an existing OME-Zarr through Fractal.
+    2. Optionally adds new ROI tables to the existing OME-Zarr.
+
+    Args:
+        input_paths: A length-one list with the parent folder of the OME-Zarr
+            to be imported; e.g. `input_paths=["/somewhere"]`, if the OME-Zarr
+            path is `/somewhere/array.zarr`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: Not used in this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: Not used in this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        zarr_name: The OME-Zarr name, without its parent folder; e.g.
+            `zarr_name="array.zarr"`, if the OME-Zarr path is
+            `/somewhere/array.zarr`.
+        add_image_ROI_table: Whether to add a `image_ROI_table` table to each
+            image, with a single ROI covering the whole image.
+        add_grid_ROI_table: Whether to add a `grid_ROI_table` table to each
+            image, with the image split into a rectangular grid of ROIs.
+        grid_y_shape: Y shape of the ROI grid in `grid_ROI_table`.
+        grid_x_shape: X shape of the ROI grid in `grid_ROI_table`.
+        update_omero_metadata: Whether to update Omero-channels metadata, to
+            make them Fractal-compatible.
+        overwrite: Whether new ROI tables (added when `add_image_ROI_table`
+            and/or `add_grid_ROI_table` are `True`) can overwite existing ones.
+    """
+
+    # Preliminary checks
+    if len(input_paths) > 1:
+        raise NotImplementedError
+
+    zarr_path = str(Path(input_paths[0]) / zarr_name)
+    logger.info(f"Zarr path: {zarr_path}")
+
+    zarrurls: dict = dict(plate=[], well=[], image=[])
+
+    root_group = zarr.open_group(zarr_path, mode="r")
+    ngff_type = detect_ome_ngff_type(root_group)
+    grid_YX_shape = (grid_y_shape, grid_x_shape)
+
+    if ngff_type == "plate":
+        zarrurls["plate"].append(zarr_name)
+        for well in root_group.attrs["plate"]["wells"]:
+            well_path = well["path"]
+            zarrurls["well"].append(f"{zarr_name}/{well_path}")
+
+            well_group = zarr.open_group(zarr_path, path=well_path, mode="r")
+            for image in well_group.attrs["well"]["images"]:
+                image_path = image["path"]
+                zarrurls["image"].append(
+                    f"{zarr_name}/{well_path}/{image_path}"
+                )
+                _process_single_image(
+                    f"{zarr_path}/{well_path}/{image_path}",
+                    add_image_ROI_table,
+                    add_grid_ROI_table,
+                    update_omero_metadata,
+                    grid_YX_shape=grid_YX_shape,
+                    overwrite=overwrite,
+                )
+    elif ngff_type == "well":
+        zarrurls["well"].append(zarr_name)
+        logger.warning(
+            "Only OME-Zarr for plates are fully supported in Fractal; "
+            f"e.g. the current one ({ngff_type=}) cannot be "
+            "processed via the `maximum_intensity_projection` task."
+        )
+        for image in root_group.attrs["well"]["images"]:
+            image_path = image["path"]
+            zarrurls["image"].append(f"{zarr_name}/{image_path}")
+            _process_single_image(
+                f"{zarr_path}/{image_path}",
+                add_image_ROI_table,
+                add_grid_ROI_table,
+                update_omero_metadata,
+                grid_YX_shape=grid_YX_shape,
+                overwrite=overwrite,
+            )
+    elif ngff_type == "image":
+        zarrurls["image"].append(zarr_name)
+        logger.warning(
+            "Only OME-Zarr for plates are fully supported in Fractal; "
+            f"e.g. the current one ({ngff_type=}) cannot be "
+            "processed via the `maximum_intensity_projection` task."
+        )
+        _process_single_image(
+            zarr_path,
+            add_image_ROI_table,
+            add_grid_ROI_table,
+            update_omero_metadata,
+            grid_YX_shape=grid_YX_shape,
+            overwrite=overwrite,
+        )
+
+    # Remove zarrurls keys pointing to empty lists
+    clean_zarrurls = {
+        key: value for key, value in zarrurls.items() if len(value) > 0
+    }
+
+    return clean_zarrurls
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/index.html b/reference/fractal_tasks_core/tasks/index.html new file mode 100644 index 000000000..0ab323b17 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/index.html @@ -0,0 +1,1103 @@ + + + + + + + + + + + + + + + + + + + + + + tasks - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

tasks

+ +
+ + + + +
+ +

Tasks subpackage (requires installation extra fractal-tasks).

+ + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/maximum_intensity_projection/index.html b/reference/fractal_tasks_core/tasks/maximum_intensity_projection/index.html new file mode 100644 index 000000000..3256e3cd3 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/maximum_intensity_projection/index.html @@ -0,0 +1,1465 @@ + + + + + + + + + + + + + + + + + + + + + + maximum_intensity_projection - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

maximum_intensity_projection

+ +
+ + + + +
+ +

Task for 3D->2D maximum-intensity projection.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ maximum_intensity_projection(*, input_paths, output_path, component, metadata, overwrite=False) + +

+ + +
+ +

Perform maximum-intensity projection along Z axis.

+

Note: this task stores the output in a new zarr file.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

This parameter is not used by this task. +This task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Path were the output of this task is stored. +Example: "/some/path/" => puts the new OME-Zarr file in that +folder. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that +is processed. Component is typically changed by the copy_ome_zarr +task before, to point to a new mip Zarr file. +Example: "some_plate_mip.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

Dictionary containing metadata about the OME-Zarr. +This task requires the key copy_ome_zarr to be present in the +metadata (as defined in copy_ome_zarr task). +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ +
+ Source code in fractal_tasks_core/tasks/maximum_intensity_projection.py + 
 31
+ 32
+ 33
+ 34
+ 35
+ 36
+ 37
+ 38
+ 39
+ 40
+ 41
+ 42
+ 43
+ 44
+ 45
+ 46
+ 47
+ 48
+ 49
+ 50
+ 51
+ 52
+ 53
+ 54
+ 55
+ 56
+ 57
+ 58
+ 59
+ 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
@validate_arguments
+def maximum_intensity_projection(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    overwrite: bool = False,
+) -> dict[str, Any]:
+    """
+    Perform maximum-intensity projection along Z axis.
+
+    Note: this task stores the output in a new zarr file.
+
+    Args:
+        input_paths: This parameter is not used by this task.
+            This task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: Path were the output of this task is stored.
+            Example: `"/some/path/"` => puts the new OME-Zarr file in that
+            folder.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that
+            is processed. Component is typically changed by the `copy_ome_zarr`
+            task before, to point to a new mip Zarr file.
+            Example: `"some_plate_mip.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: Dictionary containing metadata about the OME-Zarr.
+            This task requires the key `copy_ome_zarr` to be present in the
+            metadata (as defined in `copy_ome_zarr` task).
+            (standard argument for Fractal tasks, managed by Fractal server).
+        overwrite: If `True`, overwrite the task output.
+    """
+
+    # Preliminary checks
+    if len(input_paths) > 1:
+        raise NotImplementedError
+
+    plate, well = component.split(".zarr/")
+    zarrurl_old = metadata["copy_ome_zarr"]["sources"][plate] + "/" + well
+    clean_output_path = Path(output_path).resolve()
+    zarrurl_new = (clean_output_path / component).as_posix()
+    logger.info(f"{zarrurl_old=}")
+    logger.info(f"{zarrurl_new=}")
+
+    # Read some parameters from metadata
+    ngff_image = load_NgffImageMeta(zarrurl_old)
+    num_levels = ngff_image.num_levels
+    coarsening_xy = ngff_image.coarsening_xy
+
+    # Load 0-th level
+    data_czyx = da.from_zarr(zarrurl_old + "/0")
+    num_channels = data_czyx.shape[0]
+    chunksize_y = data_czyx.chunksize[-2]
+    chunksize_x = data_czyx.chunksize[-1]
+    logger.info(f"{num_channels=}")
+    logger.info(f"{chunksize_y=}")
+    logger.info(f"{chunksize_x=}")
+    # Loop over channels
+    accumulate_chl = []
+    for ind_ch in range(num_channels):
+        # Perform MIP for each channel of level 0
+        mip_yx = da.stack([da.max(data_czyx[ind_ch], axis=0)], axis=0)
+        accumulate_chl.append(mip_yx)
+    accumulated_array = da.stack(accumulate_chl, axis=0)
+
+    # Write to disk (triggering execution)
+    try:
+        accumulated_array.to_zarr(
+            f"{zarrurl_new}/0",
+            overwrite=overwrite,
+            dimension_separator="/",
+            write_empty_chunks=False,
+        )
+    except ContainsArrayError as e:
+        error_msg = (
+            f"Cannot write array to zarr group at '{zarrurl_new}/0', "
+            f"with {overwrite=} (original error: {str(e)}).\n"
+            "Hint: try setting overwrite=True."
+        )
+        logger.error(error_msg)
+        raise OverwriteNotAllowedError(error_msg)
+
+    # Starting from on-disk highest-resolution data, build and write to disk a
+    # pyramid of coarser levels
+    build_pyramid(
+        zarrurl=zarrurl_new,
+        overwrite=overwrite,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        chunksize=(1, 1, chunksize_y, chunksize_x),
+    )
+
+    return {}
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/napari_workflows_wrapper/index.html b/reference/fractal_tasks_core/tasks/napari_workflows_wrapper/index.html new file mode 100644 index 000000000..df0efb1a2 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/napari_workflows_wrapper/index.html @@ -0,0 +1,2653 @@ + + + + + + + + + + + + + + + + + + + + + + napari_workflows_wrapper - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

napari_workflows_wrapper

+ +
+ + + + +
+ +

Wrapper of napari-workflows.

+ + + +
+ + + + + + + + +
+ + + + +

+ OutOfTaskScopeError + + +

+ + +
+

+ Bases: NotImplementedError

+ + +

Encapsulates features that are out-of-scope for the current wrapper task.

+ +
+ Source code in fractal_tasks_core/tasks/napari_workflows_wrapper.py + 
52
+53
+54
+55
+56
+57
class OutOfTaskScopeError(NotImplementedError):
+    """
+    Encapsulates features that are out-of-scope for the current wrapper task.
+    """
+
+    pass
+
+
+ +
+ +
+ + + +
+ + + + +

+ napari_workflows_wrapper(*, input_paths, output_path, component, metadata, workflow_file, input_specs, output_specs, input_ROI_table='FOV_ROI_table', level=0, relabeling=True, expected_dimensions=3, overwrite=True) + +

+ + +
+ +

Run a napari-workflow on the ROIs of a single OME-NGFF image.

+

This task takes images and labels and runs a napari-workflow on them that +can produce a label and tables as output.

+

Examples of allowed entries for input_specs and output_specs:

+
input_specs = {
+    "in_1": {"type": "image", "channel": {"wavelength_id": "A01_C02"}},
+    "in_2": {"type": "image", "channel": {"label": "DAPI"}},
+    "in_3": {"type": "label", "label_name": "label_DAPI"},
+}
+
+output_specs = {
+    "out_1": {"type": "label", "label_name": "label_DAPI_new"},
+    "out_2": {"type": "dataframe", "table_name": "measurements"},
+}
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the image data is stored as +OME-Zarrs. Should point to the parent folder containing one or many +OME-Zarr files, not the actual OME-Zarr file. +Example: ["/some/path/"]. +his task only supports a single input path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. +Example: "some_plate.zarr/B/03/0". +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

This parameter is not used by this task. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
workflow_file +
+

Absolute path to napari-workflows YAML file

+
+

+ + TYPE: + str + +

+
input_specs +
+

A dictionary of NapariWorkflowsInput values.

+
+

+ + TYPE: + dict[str, NapariWorkflowsInput] + +

+
output_specs +
+

A dictionary of NapariWorkflowsOutput values.

+
+

+ + TYPE: + dict[str, NapariWorkflowsOutput] + +

+
input_ROI_table +
+

Name of the ROI table over which the task loops to +apply napari workflows. +Examples: +FOV_ROI_table +=> loop over the field of views; +organoid_ROI_table +=> loop over the organoid ROI table (generated by another task); +well_ROI_table +=> process the whole well as one image.

+
+

+ + TYPE: + str + + + DEFAULT: + 'FOV_ROI_table' + +

+
level +
+

Pyramid level of the image to be used as input for +napari-workflows. Choose 0 to process at full resolution. +Levels > 0 are currently only supported for workflows that only +have intensity images as input and only produce a label images as +output.

+
+

+ + TYPE: + int + + + DEFAULT: + 0 + +

+
relabeling +
+

If True, apply relabeling so that label values are +unique across all ROIs in the well.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
expected_dimensions +
+

Expected dimensions (either 2 or 3). Useful +when loading 2D images that are stored in a 3D array with shape +(1, size_x, size_y) [which is the default way Fractal stores 2D +images], but you want to make sure the napari workflow gets a 2D +array to process. Also useful to set to 2 when loading a 2D +OME-Zarr that is saved as (size_x, size_y).

+
+

+ + TYPE: + int + + + DEFAULT: + 3 + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + True + +

+
+ +
+ Source code in fractal_tasks_core/tasks/napari_workflows_wrapper.py + 
 60
+ 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+386
+387
+388
+389
+390
+391
+392
+393
+394
+395
+396
+397
+398
+399
+400
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
+517
+518
+519
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+545
+546
+547
+548
+549
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+560
+561
+562
+563
+564
+565
+566
+567
+568
+569
+570
+571
+572
+573
+574
+575
+576
+577
+578
+579
+580
+581
+582
+583
+584
+585
+586
+587
+588
+589
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+600
+601
+602
+603
+604
+605
+606
+607
+608
+609
+610
+611
+612
+613
+614
+615
+616
+617
+618
+619
+620
+621
+622
+623
+624
+625
+626
+627
+628
+629
+630
+631
+632
+633
+634
+635
+636
+637
+638
+639
+640
+641
+642
+643
+644
+645
+646
+647
+648
+649
@validate_arguments
+def napari_workflows_wrapper(
+    *,
+    # Default arguments for fractal tasks:
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    # Task-specific arguments:
+    workflow_file: str,
+    input_specs: dict[str, NapariWorkflowsInput],
+    output_specs: dict[str, NapariWorkflowsOutput],
+    input_ROI_table: str = "FOV_ROI_table",
+    level: int = 0,
+    relabeling: bool = True,
+    expected_dimensions: int = 3,
+    overwrite: bool = True,
+):
+    """
+    Run a napari-workflow on the ROIs of a single OME-NGFF image.
+
+    This task takes images and labels and runs a napari-workflow on them that
+    can produce a label and tables as output.
+
+    Examples of allowed entries for `input_specs` and `output_specs`:
+
+    ```
+    input_specs = {
+        "in_1": {"type": "image", "channel": {"wavelength_id": "A01_C02"}},
+        "in_2": {"type": "image", "channel": {"label": "DAPI"}},
+        "in_3": {"type": "label", "label_name": "label_DAPI"},
+    }
+
+    output_specs = {
+        "out_1": {"type": "label", "label_name": "label_DAPI_new"},
+        "out_2": {"type": "dataframe", "table_name": "measurements"},
+    }
+    ```
+
+    Args:
+        input_paths: List of input paths where the image data is stored as
+            OME-Zarrs. Should point to the parent folder containing one or many
+            OME-Zarr files, not the actual OME-Zarr file.
+            Example: `["/some/path/"]`.
+            his task only supports a single input path.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        output_path: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed.
+            Example: `"some_plate.zarr/B/03/0"`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: This parameter is not used by this task.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        workflow_file: Absolute path to napari-workflows YAML file
+        input_specs: A dictionary of `NapariWorkflowsInput` values.
+        output_specs: A dictionary of `NapariWorkflowsOutput` values.
+        input_ROI_table: Name of the ROI table over which the task loops to
+            apply napari workflows.
+            Examples:
+            `FOV_ROI_table`
+            => loop over the field of views;
+            `organoid_ROI_table`
+            => loop over the organoid ROI table (generated by another task);
+            `well_ROI_table`
+            => process the whole well as one image.
+        level: Pyramid level of the image to be used as input for
+            napari-workflows. Choose `0` to process at full resolution.
+            Levels > 0 are currently only supported for workflows that only
+            have intensity images as input and only produce a label images as
+            output.
+        relabeling: If `True`, apply relabeling so that label values are
+            unique across all ROIs in the well.
+        expected_dimensions: Expected dimensions (either `2` or `3`). Useful
+            when loading 2D images that are stored in a 3D array with shape
+            `(1, size_x, size_y)` [which is the default way Fractal stores 2D
+            images], but you want to make sure the napari workflow gets a 2D
+            array to process. Also useful to set to `2` when loading a 2D
+            OME-Zarr that is saved as `(size_x, size_y)`.
+        overwrite: If `True`, overwrite the task output.
+    """
+    wf: napari_workflows.Worfklow = load_workflow(workflow_file)
+    logger.info(f"Loaded workflow from {workflow_file}")
+
+    # Validation of input/output specs
+    if not (set(wf.leafs()) <= set(output_specs.keys())):
+        msg = f"Some item of {wf.leafs()=} is not part of {output_specs=}."
+        logger.warning(msg)
+    if not (set(wf.roots()) <= set(input_specs.keys())):
+        msg = f"Some item of {wf.roots()=} is not part of {input_specs=}."
+        logger.error(msg)
+        raise ValueError(msg)
+    list_outputs = sorted(output_specs.keys())
+
+    # Characterization of workflow and scope restriction
+    input_types = [in_params.type for (name, in_params) in input_specs.items()]
+    output_types = [
+        out_params.type for (name, out_params) in output_specs.items()
+    ]
+    are_inputs_all_images = set(input_types) == {"image"}
+    are_outputs_all_labels = set(output_types) == {"label"}
+    are_outputs_all_dataframes = set(output_types) == {"dataframe"}
+    is_labeling_workflow = are_inputs_all_images and are_outputs_all_labels
+    is_measurement_only_workflow = are_outputs_all_dataframes
+    # Level-related constraint
+    logger.info(f"This workflow acts at {level=}")
+    logger.info(
+        f"Is the current workflow a labeling one? {is_labeling_workflow}"
+    )
+    if level > 0 and not is_labeling_workflow:
+        msg = (
+            f"{level=}>0 is currently only accepted for labeling workflows, "
+            "i.e. those going from image(s) to label(s)"
+        )
+        logger.error(msg)
+        raise OutOfTaskScopeError(msg)
+    # Relabeling-related (soft) constraint
+    if is_measurement_only_workflow and relabeling:
+        logger.warning(
+            "This is a measurement-output-only workflow, setting "
+            "relabeling=False."
+        )
+        relabeling = False
+    if relabeling:
+        max_label_for_relabeling = 0
+
+    # Pre-processing of task inputs
+    if len(input_paths) > 1:
+        raise NotImplementedError(
+            "We currently only support a single input path"
+        )
+    in_path = Path(input_paths[0]).as_posix()
+    label_dtype = np.uint32
+
+    # Read ROI table
+    zarrurl = f"{in_path}/{component}"
+    ROI_table = ad.read_zarr(f"{in_path}/{component}/tables/{input_ROI_table}")
+
+    # Load image metadata
+    ngff_image_meta = load_NgffImageMeta(zarrurl)
+    num_levels = ngff_image_meta.num_levels
+    coarsening_xy = ngff_image_meta.coarsening_xy
+
+    # Read pixel sizes from zattrs file
+    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+
+    # Create list of indices for 3D FOVs spanning the entire Z direction
+    list_indices = convert_ROI_table_to_indices(
+        ROI_table,
+        level=level,
+        coarsening_xy=coarsening_xy,
+        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(list_indices, input_ROI_table)
+    num_ROIs = len(list_indices)
+    logger.info(
+        f"Completed reading ROI table {input_ROI_table},"
+        f" found {num_ROIs} ROIs."
+    )
+
+    # Input preparation: "image" type
+    image_inputs = [
+        (name, in_params)
+        for (name, in_params) in input_specs.items()
+        if in_params.type == "image"
+    ]
+    input_image_arrays = {}
+    if image_inputs:
+        img_array = da.from_zarr(f"{in_path}/{component}/{level}")
+        # Loop over image inputs and assign corresponding channel of the image
+        for (name, params) in image_inputs:
+            channel = get_channel_from_image_zarr(
+                image_zarr_path=f"{in_path}/{component}",
+                wavelength_id=params.channel.wavelength_id,
+                label=params.channel.label,
+            )
+            channel_index = channel.index
+            input_image_arrays[name] = img_array[channel_index]
+
+            # Handle dimensions
+            shape = input_image_arrays[name].shape
+            if expected_dimensions == 3 and shape[0] == 1:
+                logger.warning(
+                    f"Input {name} has shape {shape} "
+                    f"but {expected_dimensions=}"
+                )
+            if expected_dimensions == 2:
+                if len(shape) == 2:
+                    # We already load the data as a 2D array
+                    pass
+                elif shape[0] == 1:
+                    input_image_arrays[name] = input_image_arrays[name][
+                        0, :, :
+                    ]
+                else:
+                    msg = (
+                        f"Input {name} has shape {shape} "
+                        f"but {expected_dimensions=}"
+                    )
+                    logger.error(msg)
+                    raise ValueError(msg)
+            logger.info(f"Prepared input with {name=} and {params=}")
+        logger.info(f"{input_image_arrays=}")
+
+    # Input preparation: "label" type
+    label_inputs = [
+        (name, in_params)
+        for (name, in_params) in input_specs.items()
+        if in_params.type == "label"
+    ]
+    if label_inputs:
+        # Set target_shape for upscaling labels
+        if not image_inputs:
+            logger.warning(
+                f"{len(label_inputs)=} but num_image_inputs=0. "
+                "Label array(s) will not be upscaled."
+            )
+            upscale_labels = False
+        else:
+            target_shape = list(input_image_arrays.values())[0].shape
+            upscale_labels = True
+        # Loop over label inputs and load corresponding (upscaled) image
+        input_label_arrays = {}
+        for (name, params) in label_inputs:
+            label_name = params.label_name
+            label_array_raw = da.from_zarr(
+                f"{in_path}/{component}/labels/{label_name}/{level}"
+            )
+            input_label_arrays[name] = label_array_raw
+
+            # Handle dimensions
+            shape = input_label_arrays[name].shape
+            if expected_dimensions == 3 and shape[0] == 1:
+                logger.warning(
+                    f"Input {name} has shape {shape} "
+                    f"but {expected_dimensions=}"
+                )
+            if expected_dimensions == 2:
+                if len(shape) == 2:
+                    # We already load the data as a 2D array
+                    pass
+                elif shape[0] == 1:
+                    input_label_arrays[name] = input_label_arrays[name][
+                        0, :, :
+                    ]
+                else:
+                    msg = (
+                        f"Input {name} has shape {shape} "
+                        f"but {expected_dimensions=}"
+                    )
+                    logger.error(msg)
+                    raise ValueError(msg)
+
+            if upscale_labels:
+                # Check that dimensionality matches the image
+                if len(input_label_arrays[name].shape) != len(target_shape):
+                    raise ValueError(
+                        f"Label {name} has shape "
+                        f"{input_label_arrays[name].shape}. "
+                        "But the corresponding image has shape "
+                        f"{target_shape}. Those dimensionalities do not "
+                        f"match. Is {expected_dimensions=} the correct "
+                        "setting?"
+                    )
+                if expected_dimensions == 3:
+                    upscaling_axes = [1, 2]
+                else:
+                    upscaling_axes = [0, 1]
+                input_label_arrays[name] = upscale_array(
+                    array=input_label_arrays[name],
+                    target_shape=target_shape,
+                    axis=upscaling_axes,
+                    pad_with_zeros=True,
+                )
+
+            logger.info(f"Prepared input with {name=} and {params=}")
+        logger.info(f"{input_label_arrays=}")
+
+    # Output preparation: "label" type
+    label_outputs = [
+        (name, out_params)
+        for (name, out_params) in output_specs.items()
+        if out_params.type == "label"
+    ]
+    if label_outputs:
+        # Preliminary scope checks
+        if len(label_outputs) > 1:
+            raise OutOfTaskScopeError(
+                "Multiple label outputs would break label-inputs-only "
+                f"workflows (found {len(label_outputs)=})."
+            )
+        if len(label_outputs) > 1 and relabeling:
+            raise OutOfTaskScopeError(
+                "Multiple label outputs would break relabeling in labeling+"
+                f"measurement workflows (found {len(label_outputs)=})."
+            )
+
+        # We only support two cases:
+        # 1. If there exist some input images, then use the first one to
+        #    determine output-label array properties
+        # 2. If there are no input images, but there are input labels, then (A)
+        #    re-load the pixel sizes and re-build ROI indices, and (B) use the
+        #    first input label to determine output-label array properties
+        if image_inputs:
+            reference_array = list(input_image_arrays.values())[0]
+        elif label_inputs:
+            reference_array = list(input_label_arrays.values())[0]
+            # Re-load pixel size, matching to the correct level
+            input_label_name = label_inputs[0][1].label_name
+            ngff_label_image_meta = load_NgffImageMeta(
+                f"{in_path}/{component}/labels/{input_label_name}"
+            )
+            full_res_pxl_sizes_zyx = ngff_label_image_meta.get_pixel_sizes_zyx(
+                level=0
+            )
+            # Create list of indices for 3D FOVs spanning the whole Z direction
+            list_indices = convert_ROI_table_to_indices(
+                ROI_table,
+                level=level,
+                coarsening_xy=coarsening_xy,
+                full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+            )
+            check_valid_ROI_indices(list_indices, input_ROI_table)
+            num_ROIs = len(list_indices)
+            logger.info(
+                f"Re-create ROI indices from ROI table {input_ROI_table}, "
+                f"using {full_res_pxl_sizes_zyx=}. "
+                "This is necessary because label-input-only workflows may "
+                "have label inputs that are at a different resolution and "
+                "are not upscaled."
+            )
+        else:
+            msg = (
+                "Missing image_inputs and label_inputs, we cannot assign"
+                " label output properties"
+            )
+            raise OutOfTaskScopeError(msg)
+
+        # Extract label properties from reference_array, and make sure they are
+        # for three dimensions
+        label_shape = reference_array.shape
+        label_chunksize = reference_array.chunksize
+        if len(label_shape) == 2 and len(label_chunksize) == 2:
+            if expected_dimensions == 3:
+                raise ValueError(
+                    f"Something wrong: {label_shape=} but "
+                    f"{expected_dimensions=}"
+                )
+            label_shape = (1, label_shape[0], label_shape[1])
+            label_chunksize = (1, label_chunksize[0], label_chunksize[1])
+        logger.info(f"{label_shape=}")
+        logger.info(f"{label_chunksize=}")
+
+        # Loop over label outputs and (1) set zattrs, (2) create zarr group
+        output_label_zarr_groups: dict[str, Any] = {}
+        for (name, out_params) in label_outputs:
+
+            # (1a) Rescale OME-NGFF datasets (relevant for level>0)
+            if not ngff_image_meta.multiscale.axes[0].name == "c":
+                raise ValueError(
+                    "Cannot set `remove_channel_axis=True` for multiscale "
+                    f"metadata with axes={ngff_image_meta.multiscale.axes}. "
+                    'First axis should have name "c".'
+                )
+            new_datasets = rescale_datasets(
+                datasets=[
+                    ds.dict() for ds in ngff_image_meta.multiscale.datasets
+                ],
+                coarsening_xy=coarsening_xy,
+                reference_level=level,
+                remove_channel_axis=True,
+            )
+
+            # (1b) Prepare attrs for label group
+            label_name = out_params.label_name
+            label_attrs = {
+                "image-label": {
+                    "version": __OME_NGFF_VERSION__,
+                    "source": {"image": "../../"},
+                },
+                "multiscales": [
+                    {
+                        "name": label_name,
+                        "version": __OME_NGFF_VERSION__,
+                        "axes": [
+                            ax.dict()
+                            for ax in ngff_image_meta.multiscale.axes
+                            if ax.type != "channel"
+                        ],
+                        "datasets": new_datasets,
+                    }
+                ],
+            }
+
+            # (2) Prepare label group
+            zarrurl = f"{in_path}/{component}"
+            image_group = zarr.group(zarrurl)
+            label_group = prepare_label_group(
+                image_group,
+                label_name,
+                overwrite=overwrite,
+                label_attrs=label_attrs,
+                logger=logger,
+            )
+            logger.info(
+                "Helper function `prepare_label_group` returned "
+                f"{label_group=}"
+            )
+
+            # (3) Create zarr group at level=0
+            store = zarr.storage.FSStore(
+                f"{in_path}/{component}/labels/{label_name}/0"
+            )
+            mask_zarr = zarr.create(
+                shape=label_shape,
+                chunks=label_chunksize,
+                dtype=label_dtype,
+                store=store,
+                overwrite=overwrite,
+                dimension_separator="/",
+            )
+            output_label_zarr_groups[name] = mask_zarr
+            logger.info(f"Prepared output with {name=} and {out_params=}")
+        logger.info(f"{output_label_zarr_groups=}")
+
+    # Output preparation: "dataframe" type
+    dataframe_outputs = [
+        (name, out_params)
+        for (name, out_params) in output_specs.items()
+        if out_params.type == "dataframe"
+    ]
+    output_dataframe_lists: dict[str, list] = {}
+    for (name, out_params) in dataframe_outputs:
+        output_dataframe_lists[name] = []
+        logger.info(f"Prepared output with {name=} and {out_params=}")
+        logger.info(f"{output_dataframe_lists=}")
+
+    #####
+
+    for i_ROI, indices in enumerate(list_indices):
+        s_z, e_z, s_y, e_y, s_x, e_x = indices[:]
+        region = (slice(s_z, e_z), slice(s_y, e_y), slice(s_x, e_x))
+
+        logger.info(f"ROI {i_ROI+1}/{num_ROIs}: {region=}")
+
+        # Always re-load napari worfklow
+        wf = load_workflow(workflow_file)
+
+        # Set inputs
+        for input_name in input_specs.keys():
+            input_type = input_specs[input_name].type
+
+            if input_type == "image":
+                wf.set(
+                    input_name,
+                    load_region(
+                        input_image_arrays[input_name],
+                        region,
+                        compute=True,
+                        return_as_3D=False,
+                    ),
+                )
+            elif input_type == "label":
+                wf.set(
+                    input_name,
+                    load_region(
+                        input_label_arrays[input_name],
+                        region,
+                        compute=True,
+                        return_as_3D=False,
+                    ),
+                )
+
+        # Get outputs
+        outputs = wf.get(list_outputs)
+
+        # Iterate first over dataframe outputs (to use the correct
+        # max_label_for_relabeling, if needed)
+        for ind_output, output_name in enumerate(list_outputs):
+            if output_specs[output_name].type != "dataframe":
+                continue
+            df = outputs[ind_output]
+            if relabeling:
+                df["label"] += max_label_for_relabeling
+                logger.info(
+                    f'ROI {i_ROI+1}/{num_ROIs}: Relabeling "{name}" dataframe'
+                    "output, with {max_label_for_relabeling=}"
+                )
+
+            # Append the new-ROI dataframe to the all-ROIs list
+            output_dataframe_lists[output_name].append(df)
+
+        # After all dataframe outputs, iterate over label outputs (which
+        # actually can be only 0 or 1)
+        for ind_output, output_name in enumerate(list_outputs):
+            if output_specs[output_name].type != "label":
+                continue
+            mask = outputs[ind_output]
+
+            # Check dimensions
+            if len(mask.shape) != expected_dimensions:
+                msg = (
+                    f"Output {output_name} has shape {mask.shape} "
+                    f"but {expected_dimensions=}"
+                )
+                logger.error(msg)
+                raise ValueError(msg)
+            elif expected_dimensions == 2:
+                mask = np.expand_dims(mask, axis=0)
+
+            # Sanity check: issue warning for non-consecutive labels
+            unique_labels = np.unique(mask)
+            num_unique_labels_in_this_ROI = len(unique_labels)
+            if np.min(unique_labels) == 0:
+                num_unique_labels_in_this_ROI -= 1
+            num_labels_in_this_ROI = int(np.max(mask))
+            if num_labels_in_this_ROI != num_unique_labels_in_this_ROI:
+                logger.warning(
+                    f'ROI {i_ROI+1}/{num_ROIs}: "{name}" label output has'
+                    f"non-consecutive labels: {num_labels_in_this_ROI=} but"
+                    f"{num_unique_labels_in_this_ROI=}"
+                )
+
+            if relabeling:
+                mask[mask > 0] += max_label_for_relabeling
+                logger.info(
+                    f'ROI {i_ROI+1}/{num_ROIs}: Relabeling "{name}" label '
+                    f"output, with {max_label_for_relabeling=}"
+                )
+                max_label_for_relabeling += num_labels_in_this_ROI
+                logger.info(
+                    f"ROI {i_ROI+1}/{num_ROIs}: label-number update with "
+                    f"{num_labels_in_this_ROI=}; "
+                    f"new {max_label_for_relabeling=}"
+                )
+
+            da.array(mask).to_zarr(
+                url=output_label_zarr_groups[output_name],
+                region=region,
+                compute=True,
+                overwrite=overwrite,
+            )
+        logger.info(f"ROI {i_ROI+1}/{num_ROIs}: output handling complete")
+
+    # Output handling: "dataframe" type (for each output, concatenate ROI
+    # dataframes, clean up, and store in a AnnData table on-disk)
+    for (name, out_params) in dataframe_outputs:
+        table_name = out_params.table_name
+        # Concatenate all FOV dataframes
+        list_dfs = output_dataframe_lists[name]
+        if len(list_dfs) == 0:
+            measurement_table = ad.AnnData()
+        else:
+            df_well = pd.concat(list_dfs, axis=0, ignore_index=True)
+            # Extract labels and drop them from df_well
+            labels = pd.DataFrame(df_well["label"].astype(str))
+            df_well.drop(labels=["label"], axis=1, inplace=True)
+            # Convert all to float (warning: some would be int, in principle)
+            measurement_dtype = np.float32
+            df_well = df_well.astype(measurement_dtype)
+            df_well.index = df_well.index.map(str)
+            # Convert to anndata
+            measurement_table = ad.AnnData(df_well, dtype=measurement_dtype)
+            measurement_table.obs = labels
+
+        # Write to zarr group
+        image_group = zarr.group(f"{in_path}/{component}")
+        # TODO: should we include any table_attrs? (ref issue #333)
+        write_table(
+            image_group,
+            table_name,
+            measurement_table,
+            overwrite=overwrite,
+            logger=logger,
+        )
+
+    # Output handling: "label" type (for each output, build and write to disk
+    # pyramid of coarser levels)
+    for (name, out_params) in label_outputs:
+        label_name = out_params.label_name
+        build_pyramid(
+            zarrurl=f"{zarrurl}/labels/{label_name}",
+            overwrite=overwrite,
+            num_levels=num_levels,
+            coarsening_xy=coarsening_xy,
+            chunksize=label_chunksize,
+            aggregation_function=np.max,
+        )
+
+    return {}
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/index.html b/reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/index.html new file mode 100644 index 000000000..41b25a6e4 --- /dev/null +++ b/reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/index.html @@ -0,0 +1,1792 @@ + + + + + + + + + + + + + + + + + + + + + + yokogawa_to_ome_zarr - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

yokogawa_to_ome_zarr

+ +
+ + + + +
+ +

Task that writes image data to an existing OME-NGFF zarr array.

+ + + +
+ + + + + + + + + + +
+ + + + +

+ sort_fun(filename) + +

+ + +
+ +

Takes a string (filename of a Yokogawa image), extract site and +z-index metadata and returns them as a list of integers.

+ + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
filename +
+

Name of the image file.

+
+

+ + TYPE: + str + +

+
+ +
+ Source code in fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py + 
46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
def sort_fun(filename: str) -> list[int]:
+    """
+    Takes a string (filename of a Yokogawa image), extract site and
+    z-index metadata and returns them as a list of integers.
+
+    Args:
+        filename: Name of the image file.
+    """
+
+    filename_metadata = parse_filename(filename)
+    site = int(filename_metadata["F"])
+    z_index = int(filename_metadata["Z"])
+    return [site, z_index]
+
+
+
+ +
+ + +
+ + + + +

+ yokogawa_to_ome_zarr(*, input_paths, output_path, component, metadata, overwrite=False) + +

+ + +
+ +

Convert Yokogawa output (png, tif) to zarr file.

+

This task is typically run after Create OME-Zarr or +Create OME-Zarr Multiplexing and populates the empty OME-Zarr files that +were prepared.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PARAMETER DESCRIPTION
input_paths +
+

List of input paths where the OME-Zarrs. Should point to +the parent folder containing one or many OME-Zarr files, not the +actual OME-Zarr file. Example: ["/some/path/"]. +This task only supports a single input path. +(standard argument for Fractal tasks, +managed by Fractal server).

+
+

+ + TYPE: + Sequence[str] + +

+
output_path +
+

Unclear. Should be the same as input_path. +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
component +
+

Path to the OME-Zarr image in the OME-Zarr plate that is +processed. Example: "some_plate.zarr/B/03/0" +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + str + +

+
metadata +
+

Dictionary containing metadata about the OME-Zarr. This task +requires the following elements to be present in the metadata. +original_paths: +list of paths that correspond to the input_paths of the +create_ome_zarr task (=> where the microscopy image are stored); +num_levels (int): +number of pyramid levels in the image (this determines how many +pyramid levels are built for the segmentation); +coarsening_xy (int): +coarsening factor in XY of the downsampling when building the +pyramid; +image_extension: +filename extension of images (e.g. "tif" or "png"); +image_glob_patterns: +parameter of create_ome_zarr task (if specified, only parse +images with filenames that match with all these patterns). +(standard argument for Fractal tasks, managed by Fractal server).

+
+

+ + TYPE: + dict[str, Any] + +

+
overwrite +
+

If True, overwrite the task output.

+
+

+ + TYPE: + bool + + + DEFAULT: + False + +

+
+ +
+ Source code in fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py + 
 61
+ 62
+ 63
+ 64
+ 65
+ 66
+ 67
+ 68
+ 69
+ 70
+ 71
+ 72
+ 73
+ 74
+ 75
+ 76
+ 77
+ 78
+ 79
+ 80
+ 81
+ 82
+ 83
+ 84
+ 85
+ 86
+ 87
+ 88
+ 89
+ 90
+ 91
+ 92
+ 93
+ 94
+ 95
+ 96
+ 97
+ 98
+ 99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
@validate_arguments
+def yokogawa_to_ome_zarr(
+    *,
+    input_paths: Sequence[str],
+    output_path: str,
+    component: str,
+    metadata: dict[str, Any],
+    overwrite: bool = False,
+):
+    """
+    Convert Yokogawa output (png, tif) to zarr file.
+
+    This task is typically run after Create OME-Zarr or
+    Create OME-Zarr Multiplexing and populates the empty OME-Zarr files that
+    were prepared.
+
+    Args:
+        input_paths: List of input paths where the OME-Zarrs. Should point to
+            the parent folder containing one or many OME-Zarr files, not the
+            actual OME-Zarr file. Example: `["/some/path/"]`.
+            This task only supports a single input path.
+            (standard argument for Fractal tasks,
+            managed by Fractal server).
+        output_path: Unclear. Should be the same as `input_path`.
+            (standard argument for Fractal tasks, managed by Fractal server).
+        component: Path to the OME-Zarr image in the OME-Zarr plate that is
+            processed. Example: `"some_plate.zarr/B/03/0"`
+            (standard argument for Fractal tasks, managed by Fractal server).
+        metadata: Dictionary containing metadata about the OME-Zarr. This task
+            requires the following elements to be present in the metadata.
+            `original_paths`:
+            list of paths that correspond to the `input_paths` of the
+            `create_ome_zarr` task (=> where the microscopy image are stored);
+            `num_levels (int)`:
+            number of pyramid levels in the image (this determines how many
+            pyramid levels are built for the segmentation);
+            `coarsening_xy (int)`:
+            coarsening factor in XY of the downsampling when building the
+            pyramid;
+            `image_extension`:
+            filename extension of images (e.g. `"tif"` or `"png"`);
+            `image_glob_patterns`:
+            parameter of `create_ome_zarr` task (if specified, only parse
+            images with filenames that match with all these patterns).
+            (standard argument for Fractal tasks, managed by Fractal server).
+        overwrite: If `True`, overwrite the task output.
+    """
+
+    # Preliminary checks
+    if len(input_paths) > 1:
+        raise NotImplementedError
+    zarrurl = Path(input_paths[0]).as_posix() + f"/{component}"
+
+    # Read attributes from NGFF metadata
+    ngff_image_meta = load_NgffImageMeta(zarrurl)
+    num_levels = ngff_image_meta.num_levels
+    coarsening_xy = ngff_image_meta.coarsening_xy
+    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)
+    logger.info(f"NGFF image has {num_levels=}")
+    logger.info(f"NGFF image has {coarsening_xy=}")
+    logger.info(
+        f"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}"
+    )
+
+    parameters = get_parameters_from_metadata(
+        keys=[
+            "original_paths",
+            "image_extension",
+            "image_glob_patterns",
+        ],
+        metadata=metadata,
+        # FIXME: Why rely on output_path here, when we use the input path for
+        # the zarr_url? That just means that different input & output paths
+        # don't work, no?
+        image_zarr_path=(Path(output_path) / component),
+    )
+    original_path_list = parameters["original_paths"]
+    image_extension = parameters["image_extension"]
+    image_glob_patterns = parameters["image_glob_patterns"]
+
+    channels: list[OmeroChannel] = get_omero_channel_list(
+        image_zarr_path=zarrurl
+    )
+    wavelength_ids = [c.wavelength_id for c in channels]
+
+    in_path = Path(original_path_list[0])
+
+    # Define well
+    component_split = component.split("/")
+    well_row = component_split[1]
+    well_column = component_split[2]
+    well_ID = well_row + well_column
+
+    # Read useful information from ROI table
+    adata = read_zarr(f"{zarrurl}/tables/FOV_ROI_table")
+    fov_indices = convert_ROI_table_to_indices(
+        adata,
+        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(fov_indices, "FOV_ROI_table")
+    adata_well = read_zarr(f"{zarrurl}/tables/well_ROI_table")
+    well_indices = convert_ROI_table_to_indices(
+        adata_well,
+        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,
+    )
+    check_valid_ROI_indices(well_indices, "well_ROI_table")
+    if len(well_indices) > 1:
+        raise ValueError(f"Something wrong with {well_indices=}")
+
+    # FIXME: Put back the choice of columns by name? Not here..
+
+    max_z = well_indices[0][1]
+    max_y = well_indices[0][3]
+    max_x = well_indices[0][5]
+
+    # Load a single image, to retrieve useful information
+    patterns = [f"*_{well_ID}_*.{image_extension}"]
+    if image_glob_patterns:
+        patterns.extend(image_glob_patterns)
+    tmp_images = glob_with_multiple_patterns(
+        folder=str(in_path),
+        patterns=patterns,
+    )
+    sample = imread(tmp_images.pop())
+
+    # Initialize zarr
+    chunksize = (1, 1, sample.shape[1], sample.shape[2])
+    try:
+        canvas_zarr = zarr.create(
+            shape=(len(wavelength_ids), max_z, max_y, max_x),
+            chunks=chunksize,
+            dtype=sample.dtype,
+            store=zarr.storage.FSStore(zarrurl + "/0"),
+            overwrite=overwrite,
+            dimension_separator="/",
+        )
+    except ContainsArrayError as e:
+        error_msg = (
+            f"Cannot create a zarr group at '{zarrurl}/0', "
+            f"with {overwrite=} (original error: {str(e)}).\n"
+            "Hint: try setting overwrite=True."
+        )
+        logger.error(error_msg)
+        raise OverwriteNotAllowedError(error_msg)
+
+    # Loop over channels
+    for i_c, wavelength_id in enumerate(wavelength_ids):
+        A, C = wavelength_id.split("_")
+
+        patterns = [f"*_{well_ID}_*{A}*{C}*.{image_extension}"]
+        if image_glob_patterns:
+            patterns.extend(image_glob_patterns)
+        filenames_set = glob_with_multiple_patterns(
+            folder=str(in_path),
+            patterns=patterns,
+        )
+        filenames = sorted(list(filenames_set), key=sort_fun)
+        if len(filenames) == 0:
+            raise ValueError(
+                "Error in yokogawa_to_ome_zarr: len(filenames)=0.\n"
+                f"  in_path: {in_path}\n"
+                f"  image_extension: {image_extension}\n"
+                f"  well_ID: {well_ID}\n"
+                f"  wavelength_id: {wavelength_id},\n"
+                f"  patterns: {patterns}"
+            )
+        # Loop over 3D FOV ROIs
+        for indices in fov_indices:
+            s_z, e_z, s_y, e_y, s_x, e_x = indices[:]
+            region = (
+                slice(i_c, i_c + 1),
+                slice(s_z, e_z),
+                slice(s_y, e_y),
+                slice(s_x, e_x),
+            )
+            FOV_3D = da.concatenate(
+                [imread(img) for img in filenames[:e_z]],
+            )
+            FOV_4D = da.expand_dims(FOV_3D, axis=0)
+            filenames = filenames[e_z:]
+            da.array(FOV_4D).to_zarr(
+                url=canvas_zarr,
+                region=region,
+                compute=True,
+            )
+
+    # Starting from on-disk highest-resolution data, build and write to disk a
+    # pyramid of coarser levels
+    build_pyramid(
+        zarrurl=zarrurl,
+        overwrite=overwrite,
+        num_levels=num_levels,
+        coarsening_xy=coarsening_xy,
+        chunksize=chunksize,
+    )
+
+    # Deprecated: Delete images (optional)
+    # if delete_input:
+    #     for f in filenames:
+    #         try:
+    #             os.remove(f)
+    #         except OSError as e:
+    #             logging.info("Error: %s : %s" % (f, e.strerror))
+
+    return {}
+
+
+
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/search/search_index.json b/search/search_index.json new file mode 100644 index 000000000..20364dc21 --- /dev/null +++ b/search/search_index.json @@ -0,0 +1 @@ +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Welcome to Fractal Tasks Core's documentation!","text":"

Fractal is a framework to process high content imaging data at scale and prepare it for interactive visualization.

This project is under active development \ud83d\udd28. If you need help or found a bug, open an issue here.

Fractal provides distributed workflows that convert TBs of image data into OME-Zar files. The platform then processes the 3D image data by applying tasks like illumination correction, maximum intensity projection, 3D segmentation using cellpose and measurements using napari workflows. The pyramidal OME-Zarr files enable interactive visualization in the napari viewer.

The fractal-tasks-core package contains the python tasks that parse Yokogawa CV7000 images into OME-Zarr and process OME-Zarr files. Find more information about Fractal in general and the other repositories at this link. All tasks are written as Python functions and are optimized for usage in Fractal workflows, but they can also be used as standalone functions to parse data or process OME-Zarr files. We heavily use regions of interest (ROIs) in our OME-Zarr files to store the positions of field of views. ROIs are saved as AnnData tables following this spec proposal. We save wells as large Zarr arrays instead of a collection of arrays for each field of view (see details here).

Here is an example of the interactive visualization in napari using the newly-proposed async loading in NAP4 and the napari-ome-zarr plugin:

"},{"location":"#available-tasks","title":"Available tasks","text":"

Currently, the following tasks are available:

  • Create Zarr Structure: Task to generate the zarr structure based on Yokogawa metadata files
  • Yokogawa to Zarr: Parses the Yokogawa CV7000 image data and saves it to the Zarr file
  • Illumination Correction: Applies an illumination correction based on a flatfield image & subtracts a background from the image.
  • Image Labeling (& Image Labeling Whole Well): Applies a cellpose network to the image of a single ROI or the whole well. cellpose parameters can be tuned for optimal performance.
  • Maximum Intensity Projection: Creates a maximum intensity projection of the whole plate.
  • Measurement: Make some standard measurements (intensity & morphology) using napari workflows, saving results to AnnData tables.

Some additional tasks are currently being worked on and some older tasks are still present in the fractal_tasks_core folder. See the package page for the detailed description of all tasks.

"},{"location":"#contributors","title":"Contributors","text":"

Fractal was conceived in the Liberali Lab at the Friedrich Miescher Institute for Biomedical Research and in the Pelkmans Lab at the University of Zurich (both in Switzerland). The project lead is with @gusqgm & @jluethi. The core development is done under contract by eXact lab S.r.l..

"},{"location":"#license","title":"License","text":"

Fractal is released according to a BSD 3-Clause License, see LICENSE.

"},{"location":"changelog/","title":"Changelog","text":"

Note: Numbers like (#123) point to closed Pull Requests on the fractal-tasks-core repository.

"},{"location":"changelog/#0130","title":"0.13.0","text":"
  • Tasks:
    • New task and helper functions:
      • Introduce import_ome_zarr task (#557, #579).
      • Introduce get_single_image_ROI and get_image_grid_ROIs (#557).
      • Introduce detect_ome_ngff_type (#557).
      • Introduce update_omero_channels (#579).
    • Make maximum_intensity_projection independent from ROI tables (#557).
    • Make Cellpose task work when input_ROI_table is empty (#566).
    • Fix bug of missing attributes in ROI-table Zarr group (#573).
  • Dependencies:
    • Restrict Pillow version to <10.1 (#571).
    • Support AnnData 0.10 (#574).
  • Testing:
    • Align with new Zenodo API (#568).
    • Use ubuntu-22 for GitHub CI (#576).
"},{"location":"changelog/#0122","title":"0.12.2","text":"
  • Relax check_valid_ROI_indices to support search-first scenario (#555).
  • Do not install docs dependencies in GitHub CI (#551).
"},{"location":"changelog/#0121","title":"0.12.1","text":"
  • Make Channel.window attribute optional in lib_ngff.py (#548).
  • Automate procedure for publishing package to PyPI (#545).
"},{"location":"changelog/#0120","title":"0.12.0","text":"

This release includes work on Pydantic models for NGFF specs and on ROI tables.

  • NGFF Pydantic models:
    • Introduce Pydantic models for NGFF metadata in lib_ngff.py (#528).
    • Extract num_levels and coarsening_xy parameters from NGFF objects, rather than from metadata task input (#528).
    • Transform several lib_zattrs_utils.py functions (get_axes_names, extract_zyx_pixel_sizes and get_acquisition_paths) into lib_ngff.py methods (#528).
    • Load Zarr attributes from groups, rather than from .zattrs files (#528).
  • Regions of interest:
    • Set FOV_ROI_table and well_ROI_table ZYX origin to zero (#524).
    • Remove heuristics to determine whether to reset origin, in cellpose_segmentation task (#524).
    • Remove obsolete reset_origin argument from convert_ROI_table_to_indices function (#524).
    • Remove redundant reset_origin call from apply_registration_to_ROI_tables task (#524).
    • Add check on non-negative ROI indices (#534).
    • Add check on ROI indices not starting at (0,0,0), to highlight v0.12/v0.11 incompatibility (#534).
    • Fix bug in creation of bounding-box ROIs when cellpose_segmentation loops of FOVs (#524).
    • Update type of metadata parameter of prepare_FOV_ROI_table and prepare_well_ROI_table functions (#524).
    • Fix reset_origin so that it returns an updated copy of its input (#524).
  • Dependencies:
    • Relax fsspec<=2023.6 constraint into fsspec!=2023.9.0 (#539).
"},{"location":"changelog/#0110","title":"0.11.0","text":"
  • Tasks:
    • (major) Introduce new tasks for registration of multiplexing cycles: calculate_registration_image_based, apply_registration_to_ROI_tables, apply_registration_to_image (#487).
    • (major) Introduce new overwrite argument for tasks create_ome_zarr, create_ome_zarr_multiplex, yokogawa_to_ome_zarr, copy_ome_zarr, maximum_intensity_projection, cellpose_segmentation, napari_workflows_wrapper (#499).
    • (major) Rename illumination_correction parameter from overwrite to overwrite_input (#499).
    • Fix plate-selection bug in copy_ome_zarr task (#513).
    • Fix bug in definition of metadata[\"plate\"] in create_ome_zarr_multiplex task (#513).
    • Introduce new helper functions write_table, prepare_label_group and open_zarr_group_with_overwrite (#499).
    • Introduce new helper functions are_ROI_table_columns_valid, convert_indices_to_regions, reset_origin, is_standard_roi_table, get_acquisition_paths, get_table_path_dict, get_axes_names, add_zero_translation_columns, calculate_min_max_across_dfs, apply_registration_to_single_ROI_table, write_registered_zarr, calculate_physical_shifts, get_ROI_table_with_translation (#487).
  • Testing:
    • Add tests for overwrite-related task behaviors (#499).
    • Introduce mock-up of napari_skimage_regionprops package, for testing of napari_workflows_wrapper task (#499).
  • Dependencies:
    • Require fsspec version to be <=2023.6 (#509).
"},{"location":"changelog/#0101","title":"0.10.1","text":"
  • Tasks:
    • Improve validation for OmeroChannel.color field (#488).
    • Include image-label/source/image OME-NGFF attribute when creating labels (#478).
    • Update default values for tolerance (tol) in lib_ROI_overlaps.py functions (#466).
  • Development tools:
    • Include docs_info and docs_link attributes in manifest tasks (#486).
    • Rename and revamp scripts to update/check the manifest (#486).
    • Improve logging and error-handling in tools for args-schema creation (#469).
  • Documentation:
    • Convert docstrings to Google style (#473, #479).
    • Switch from sphinx to mkdocs for documentation (#479).
    • Update generic type hints (#462, #479).
    • Align examples to recent package version, and mention them in the documentation (#470).
  • Testing:
    • Improve coverage of core library (#459, #467, #468).
    • Update Zenodo datasets used in tests (#454).
    • Run tests both for the poetry-installed and pip-installed package (#455).
  • Dependencies:
    • Relax numpy required version to <2 (#477).
    • Relax dask required version to >=2023.1.0 (#455).
    • Relax zarr required version to >=2.13.6,<3 (#455).
    • Relax pandas required version to >=1.2.0,<2 (#455).
    • Relax Pillow required version to >=9.1.1,<10.0.0 (#455).
    • Full update of poetry.lock file (mutiple PRs, e.g. #472).
    • Include requests and wget in the dev poetry dependency group (#455).
"},{"location":"changelog/#0100","title":"0.10.0","text":"
  • Restructure the package and repository:
    • Move tasks to tasks subpackage (#390)
    • Create new dev subpackage (#384).
    • Make tasks-related dependencies optional, and installable via fractal-tasks extra (#390).
    • Remove tools package extra (#384), and split the subpackage content into lib_ROI_overlaps and examples (#390).
  • (major) Modify task arguments
    • Add Pydantic model lib_channels.OmeroChannel (#410, #422);
    • Add Pydantic model tasks._input_models.Channel (#422);
    • Add Pydantic model tasks._input_models.NapariWorkflowsInput (#422);
    • Add Pydantic model tasks._input_models.NapariWorkflowsOutput (#422);
    • Move all Pydantic models to main package (#438).
    • Modify arguments of illumination_correction task (#431);
    • Modify arguments of create_ome_zarr and create_ome_zarr_multiplex (#433).
    • Modify argument default for ROI_table_names, in copy_ome_zarr (#449).
    • Remove the delete option from yokogawa to ome zarr (#443).
    • Reorder task inputs (#451).
  • JSON Schemas for task arguments:
    • Add JSON Schemas for task arguments in the package manifest (#369, #384).
    • Add JSON Schemas for attributes of custom task-argument Pydantic models (#436).
    • Make schema-generation tools more general, when handling custom Pydantic models (#445).
    • Include titles for custom-model-typed arguments and argument attributes (#447).
    • Remove TaskArguments models and switch to Pydantic V1 validate_arguments (#369).
    • Make coercing&validating task arguments required, rather than optional (#408).
    • Remove default_args from manifest (#379, #393).
  • Other:
    • Make pydantic dependency required for running tasks, and pin it to V1 (#408).
    • Remove legacy executor definitions from manifest (#361).
    • Add GitHub action for testing pip install with/without fractal-tasks extra (#390).
    • Remove sqlmodel from dev dependencies (#374).
    • Relax constraint on torch version, from ==1.12.1 to <=2.0.0 (#406).
    • Review task docstrings and improve documentation (#413, #416).
    • Update anndata dependency requirements (from ^0.8.0 to >=0.8.0,<=0.9.1), and replace anndata.experimental.write_elem with anndata._io.specs.write_elem (#428).
"},{"location":"changelog/#094","title":"0.9.4","text":"
  • Relax constraint on scikit-image version, by only requiring a version >=0.19 (#367).
"},{"location":"changelog/#093","title":"0.9.3","text":"
  • For labeling tasks (cellpose_segmentation or napari_worfklows_wrapper), allow empty ROI tables as input or output (#365).
  • Relax constraint related to the presence of channels in create_zarr_structure_multiplex task (#365).
"},{"location":"changelog/#092","title":"0.9.2","text":"
  • Increase memory requirements for some tasks in manifest (#363).
"},{"location":"changelog/#091","title":"0.9.1","text":"
  • Add use_gpu argument for cellpose_segmentation task (#350).
  • Add dummy return object to napari-workflows task (#359).
  • Include memory/cpu/gpu requirements in manifest, in view of new fractal-server SLURM backend (#360).
"},{"location":"changelog/#090","title":"0.9.0","text":"
  • Introduce a module for masked loading of ROIs, and update the cellpose_segmentation task accordingly (#306).
  • Rename task arguments: ROI_table_name->input_ROI_table and bounding_box_ROI_table_name->output_ROI_table (#306).
  • Implement part of the proposed table support in OME-NGFF specs, both for the tables zarr group and then for each table subgroup (#306).
  • Rename module: lib_remove_FOV_overlaps.py->lib_ROI_overlaps.py (#306).
  • Add new functions to existing modules: lib_regions_of_interest.convert_region_to_low_res, lib_ROI_overlaps.find_overlaps_in_ROI_indices (#306).
"},{"location":"changelog/#081","title":"0.8.1","text":"
  • Disable bugged validation of model_type argument in cellpose_segmentation (#344).
  • Raise an error if the user provides an unexpected argument to a task (#337); this applies to the case of running a task as a script, with a pydantic model for task-argument validation.
"},{"location":"changelog/#080","title":"0.8.0","text":"
  • (major) Update task interface: remove filename extension from input_paths and output_path for all tasks, and add new arguments (image_extension,image_glob_pattern) to create_ome_zarr task (#323).
  • Implement logic for handling image_glob_patterns argument, both when globbing images and in Yokogawa metadata parsing (#326).
  • Fix minor bugs in task arguments (#329).
"},{"location":"changelog/#075","title":"0.7.5","text":"
  • Update cellpose_segmentation defaults and parse additional parameters (#316).
  • Add dual-channel input for cellpose_segmentation task (#315).
"},{"location":"changelog/#074","title":"0.7.4","text":"
  • Add tests for python 3.10 (#309).
  • Drop support for python 3.8 (#319).
  • Update task interface: use string arguments instead of pathlib.Path, and only set defaults in function call signatures (#303).
"},{"location":"changelog/#073","title":"0.7.3","text":"
  • Add reset_origin argument to convert_ROI_table_to_indices (#305).
  • Do not overwrite existing labels in cellpose_segmentation task (#308).
"},{"location":"changelog/#072","title":"0.7.2","text":"
  • Remove pyqt5-related dependencies (#288).
"},{"location":"changelog/#071","title":"0.7.1","text":"

Missing

"},{"location":"changelog/#070","title":"0.7.0","text":"
  • Replace dask.array.core.get_mapper() with zarr.storage.FSStore() (#282).
  • Pin dask version to >=2023.1.0, <2023.2.
  • Pin zarr version to >=2.13.6, <2.14.
  • Pin numpy version to >=1.23.5,<1.24.
  • Pin cellpose version to >=2.2,<2.3.
"},{"location":"changelog/#065","title":"0.6.5","text":"
  • Remove FOV overlaps with more flexibility (#265).
"},{"location":"changelog/#064","title":"0.6.4","text":"
  • Created tools submodule and installation extra (#262).
"},{"location":"changelog/#063","title":"0.6.3","text":"
  • Added napari dependency, pinned to 0.4.16 version.
  • Fixed type-hinting bug in task to create multiplexing OME-Zarr structure (#258).
"},{"location":"changelog/#062","title":"0.6.2","text":"
  • Support passing a pre-made metadata table to tasks creating the OME-Zarr structure (#252).
"},{"location":"changelog/#061","title":"0.6.1","text":"
  • Add option for padding an array with zeros in upscale_array (#251).
  • Simplified imagecodecs and PyQt5 dependencies (#248).
"},{"location":"changelog/#060","title":"0.6.0","text":"
  • (major) Refactor of how to address channels (#239).
  • Fix bug in well ROI table (#245).
"},{"location":"changelog/#051","title":"0.5.1","text":"
  • Fix sorting of image files when number of Z planes passes 100 (#237).
"},{"location":"changelog/#050","title":"0.5.0","text":"
  • (major) Deprecate measurement task (#235).
  • (major) Use more uniform names for tasks, both in python modules and manifest (#235).
  • Remove deprecated manifest from __init__.py (#233).
"},{"location":"changelog/#046","title":"0.4.6","text":"
  • Skip image files if filename is not parsable (#219).
  • Preserve order of input_paths for multiplexing subfolders (#222).
  • Major refactor of replicate_zarr_structure, also enabling support for zarr files with multiple images (#223).
"},{"location":"changelog/#045","title":"0.4.5","text":"
  • Replace Cellpose wrapper with CellposeModel, to support pretrained_model argument (#218).
  • Update cellpose version (it was pinned to 2.0, in previous versions) (#218).
  • Pin torch dependency to version 1.12.1, to support CUDA version 10.2 (#218).
"},{"location":"changelog/#044","title":"0.4.4","text":"

Missing due to releasing error.

"},{"location":"changelog/#043","title":"0.4.3","text":"
  • In create_zarr_structure_multiplex, always use/require strings for acquisition field (#217).
"},{"location":"changelog/#042","title":"0.4.2","text":"
  • Bugfixes
"},{"location":"changelog/#041","title":"0.4.1","text":"
  • Only use strings as keys of channel_parameters (in create_zarr_structure_multiplex).
"},{"location":"changelog/#040","title":"0.4.0","text":"
  • (major) Rename well to image (both in metadata list and in manifest) and add an actual well field (#210).
  • Add create_ome_zarr_multiplexing, and adapt yokogawa_to_zarr (#210).
  • Relax constraint about outputs in napari_worfklows_wrapper (#209).
"},{"location":"changelog/#034","title":"0.3.4","text":"
  • Always log START/END times for each task (#204).
  • Add label_name argument to cellpose_segmentation (#207).
  • Add pretrained_model argument to cellpose_segmentation (#207).
"},{"location":"changelog/#033","title":"0.3.3","text":"
  • Added napari_worfklows_wrapper to manifest.
"},{"location":"changelog/#032","title":"0.3.2","text":"
  • Compute bounding boxes of labels, in cellpose_segmentation (#192).
  • Parse image filenames in a more robust way (#191).
  • Update manifest, moving parallelization_level and executor to meta attribute.
"},{"location":"changelog/#031","title":"0.3.1","text":"
  • Fix executable fields in manifest.
  • Remove graphviz dependency.
"},{"location":"changelog/#030","title":"0.3.0","text":"
  • Conform to Fractal v1, through new task manifest (#162) and standard input/output interface (#155, #157).
  • Add several type hints (#148) and validate them in the standard task interface (#175).
  • Update napari_worfklows_wrapper: pyramid level for labeling worfklows (#148), label-only inputs (#163, #171), relabeling (#167), 2D/3D handling (#166).
  • Deprecate dummy and dummy_fail tasks.
"},{"location":"changelog/#026","title":"0.2.6","text":"
  • Setup sphinx docs, to be built and hosted on https://fractal-tasks-core.readthedocs.io; include some preliminary updates of docstrings (#143).
  • Dependency cleanup via deptry (#144).
"},{"location":"changelog/#025","title":"0.2.5","text":"
  • Add napari_workflows_wrapper task (#141).
  • Add lib_upscale_array.py module (#141).
"},{"location":"changelog/#024","title":"0.2.4","text":"
  • Major updates to metadata_parsing.py (#136).
"},{"location":"custom_task/","title":"How to write a Fractal-compatible custom task","text":"

The fractal-tasks-core repository is the reference implementation for Fractal tasks and for Fractal task packages, but the Fractal platform can also be used to execute custom tasks. This page lists the Fractal-compatibility requirements, for a single custom task or for a task package.

Note: these specifications evolve frequently, see e.g. discussion at https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/151.

A Fractal task is mainly formed by two components:

  1. A set of metadata, which are stored in the task table of the database of a fractal-server instance, see Task metadata.
  2. An executable command, which can take some specific command-line arguments (see Command-line interface); the standard example is a Python script.

In the following we explain what are the Fractal-compatibility requirements for a single task, and then for a task package.

"},{"location":"custom_task/#single-custom-task","title":"Single custom task","text":"

We describe how to define the multiple aspects of a task, and provide a Full task example.

"},{"location":"custom_task/#task-metadata","title":"Task metadata","text":"

Each task must be associated to some metadata, so that it can be used in Fractal. The full specification is here, and the required attributes are:

  • name: the task name, e.g. \"Create OME-Zarr structure\";
  • command: a command that can be executed from the command line;
  • input_type: this can be any string (typical examples: \"image\" or \"zarr\"); the special value \"Any\" means that Fractal won't perform any check of the input_type when applying the task to a dataset.
  • output_type: same logic as input_type.
  • source: this is meant to be as close as possible to unique task identifier; for custom tasks, it can be anything (e.g. \"my_task\"), but for task that are collected automatically from a package (see Task package this attribute will have a very specific form (e.g. \"pip_remote:fractal_tasks_core:0.10.0:fractal-tasks::convert_yokogawa_to_ome-zarr\").
  • meta: a JSON object (similar to a Python dictionary) with some additional information, see Task meta-parameters.

There are multiple ways to get the appropriate metadata into the database, including a POST request to the fractal-server API (see Tasks section in the fractal-server API documentation) or the automated addition of a whole set of tasks through specific API endpoints (see Task package).

"},{"location":"custom_task/#command-line-interface","title":"Command-line interface","text":"

Some examples of task commands may look like

  • python3 /some/path/my_task.py,
  • /some/absolute/path/python3.10 /some/other/absolute/path/my_task.py,
  • /some/path/my_executable_task.py,
  • any other executable command (not necessarily based on Python).

Given a task command, Fractal will add two additional command-line arguments to it:

  • -j /some/path/input-arguments.json
  • --metadata-out /some/path/output-metadata-update.json

Therefore the task command must accept these additional command-line arguments. If the task is a Python script, this can be achieved easily by using the run_fractal_task function - which is available as part of fractal_tasks_core.tasks._utils.

"},{"location":"custom_task/#task-meta-parameters","title":"Task meta-parameters","text":"

The meta attribute of tasks (see the corresponding item in Task metadata) is where we specify some requirements on how the task should be run. This notably includes:

  • If the task has to be run in parallel (e.g. over multiple wells of an OME-Zarr dataset), then meta should include a key-value pair like {\"parallelization_level\": \"well\"}. If the parallelization_level key is missing, the task is considered as non-parallel.
  • If Fractal is configured to run on a SLURM cluster, meta may include additional information on the SLRUM requirements (more info on the Fractal SLURM backend here).
"},{"location":"custom_task/#task-input-parameters","title":"Task input parameters","text":"

When a task is run via Fractal, its input parameters (i.e. the ones in the file specified via the -j command-line otion) will always include a set of keyword arguments with specific names:

  • input_paths
  • output_path
  • metadata
  • component (only for parallel tasks)
"},{"location":"custom_task/#task-output","title":"Task output","text":"

The only task output which will be visible to Fractal is what goes in the output metadata-update file (i.e. the one specified through the --metadata-out command-line option). Note that this only holds for non-parallel tasks, while (for the moment) Fractal fully ignores the output of parallel tasks.

IMPORTANT: This means that each task must always write any output to disk, before ending.

"},{"location":"custom_task/#advanced-features","title":"Advanced features","text":"

The description of other advanced features is not yet available in this page.

  1. Also other attributes of the Task metadata exist, and they would be recognized by other Fractal components (e.g. fractal-server or fractal-web). These include JSON Schemas for input parameters and additional documentation-related attributes.
  2. In fractal-tasks-core, we use pydantic v1 to fully coerce and validate the input parameters into a set of given types.
"},{"location":"custom_task/#full-task-example","title":"Full task example","text":"

Here we describe a simplified example of a Fractal-compatible Python task (for more realistic examples see the fractal-task-core tasks folder).

The script /some/path/my_task.py may look like 

# Import a helper function from fractal_tasks_core\nfrom fractal_tasks_core.tasks._utils import run_fractal_task\n\ndef my_task_function(\n    # Reserved Fractal arguments\n    input_paths,\n    output_path,\n    metadata,\n    # Task-specific arguments\n    argument_A,\n    argument_B = \"default_B_value\",\n):\n    # Do something, based on the task parameters\n    print(\"Here we go, we are in `my_task_function`\")\n    with open(f\"{output_path}/output.txt\", \"w\") as f:\n        f.write(f\"argument_A={argument_A}\\n\")\n        f.write(f\"argument_B={argument_B}\\n\")\n    # Compile the output metadata update and return\n    output_metadata_update = {\"nothing\": \"to add\"}\n    return output_metadata_update\n\n# Thi block is executed when running the Python script directly\nif __name__ == \"__main__\":\n    run_fractal_task(task_function=my_task_function)\n
where we use run_fractal_task so that we don't have to take care of the command-line arguments.

Some valid metadata attributes for this task would be: 

name=\"My Task\"\ncommand=\"python3 /some/path/my_task.py\"\ninput_type=\"Any\"\noutput_type=\"Any\"\nsource=\"my_custom_task\"\nmeta={}\n

Note that this was an example of a non-parallel tasks; to have a parallel one, we would also need to:

  1. Set meta={\"parallelization_level\": \"something\"};
  2. Include component in the input arguments of my_task_function.
"},{"location":"custom_task/#task-package","title":"Task package","text":"

Given a set of Python scripts corresponding to Fractal tasks, it is useful to combine them into a single Python package, using the standard tools or other options (e.g. for fractal-tasks-core we use poetry).

"},{"location":"custom_task/#reasons","title":"Reasons","text":"

Creating a package is often a good practice, for reasons unrelated to Fractal:

  1. It makes it simple to assign a global version to the package, and to host it on a public index like PyPI;
  2. It may reduce code duplication:
    • The scripts may have a shared set of external dependencies, which are defined in a single place for a package.
    • The scripts may import functions from a shared set of auxiliary Python modules, which can be included in the package.

Moreover, having a single package also streamlines some Fractal-related operations. Given the package MyTasks (available on PyPI, or locally), the Fractal platform offers a feature that automatically:

  1. Downloads the wheel file of package MyTasks (if it's on a public index, rather than a local file);
  2. Creates a Python virtual environment (venv) which is specific for a given version of the MyTasks package, and installs the MyTasks package in that venv;
  3. Populates all the corresponding entries in the task database table with the appropriate Task metadata, which are extracted from the package manifest.

This feature is currently exposed in the /api/v1/task/collect/pip/ endpoint of fractal-server (see API documentation).

"},{"location":"custom_task/#requirements","title":"Requirements","text":"

To be compatible with Fractal, a task package must satisfy some additional requirements:

  • The package is built as a a wheel file, and can be installed via pip.
  • The __FRACTAL_MANIFEST__.json file is bundled in the package, in its root folder. If you are using poetry, no special operation is needed. If you are using a setup.cfg file, see this comment.
  • Include JSON Schemas. The tools in fractal_tasks_core.dev are used to generate JSON Schema's for the input parameters of each task in fractal-tasks-core. They are meant to be flexible and re-usable to perform the same operation on an independent package, but they are not thoroughly documented/tested for more general use; feel free to open an issue if something is not clear.
  • Include additional task metadata like docs_info or docs_link, which will be displayed in the Fractal web-client. Note: this feature is not yet implemented.

The ones in the list are the main requirements; if you hit unexpected behaviors, also have a look at https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/151 or open a new issue.

"},{"location":"development/","title":"Development","text":""},{"location":"development/#setting-up-environment","title":"Setting up environment","text":"

We use poetry v1.5 to manage the development environment and the dependencies. A simple way to install it is pipx install poetry==1.5, or you can look at the installation section here.

Running:

poetry install [--with dev] [--with docs]\n

will take care of installing all the dependencies in a separate environment, optionally installing also the dependencies for developement and to build the documentation.

"},{"location":"development/#testing","title":"Testing","text":"

We use pytest for unit and integration testing of Fractal. If you installed the development dependencies, you may run the test suite by invoking: 

poetry run pytest\n

The tests files are in the tests folder of the repository, and they are also run through GitHub Actions; both the main fractal_tasks_core tests (in tests/) and the fractal_tasks_core.tasks tests (in tests/tasks/) are run with Python 3.9, 3.10 and 3.11.

"},{"location":"development/#documentation","title":"Documentation","text":"

The documentations is built with mkdocs. To build the documentation locally, setup a development python environment (e.g. with poetry install --with docs) and then run one of these commands: 

poetry run mkdocs serve --config-file mkdocs.yml  # serves the docs at http://127.0.0.1:8000\npoetry run mkdocs build --config-file mkdocs.yml  # creates a build in the `site` folder\n

"},{"location":"development/#mypy","title":"Mypy","text":"

We do not enforce strict mypy compliance, but we do run it as part of a specific GitHub Action. You can run mypy locally for instance as: 

poetry run mypy --package fractal_tasks_core --ignore-missing-imports --warn-redundant-casts --warn-unused-ignores --warn-unreachable --pretty\n

"},{"location":"development/#how-to-release","title":"How to release","text":"

Preliminary check-list:

  1. The main branch is checked out.
  2. You reviewed dependencies and dev dependencies and the lock file is up to date with pyproject.toml (it is useful to have a look at the output of deptry . -v, where deptry is already installed as part of the dev dependencies - NOTE: deptry should be installed independently, e.g. via pipx install deptry).
  3. The current HEAD of the main branch passes all the tests (note: make sure that you are using the poetry-installed local package).
  4. Update changelog. First look at the list of commits since the last tag, via: 
    git log --pretty=\"[%cs] %h - %s\" `git tag --sort version:refname | tail -n 1`..HEAD\n
    then add the upcoming release to docs/source/changelog.rst with the main information about it, using standard categories like \"New features\", \"Fixes\" and \"Other changes\", and including PR numbers when relevant. Commit docs/source/changelog.rst and push.
  5. If appropriate (e.g. if you added some new task arguments, or if you modified some of their descriptions), update the JSON Schemas in the manifest via: 
    poetry run python fractal_tasks_core/dev/update_manifest.py\n

Actual release

  1. Use: 
    poetry run bumpver update --[tag-num|patch|minor] --tag-commit --commit --dry\n
    to test updating the version bump.
  2. If the previous step looks good, use: 
    poetry run bumpver update --[tag-num|patch|minor] --tag-commit --commit\n
    to actually bump the version. This will trigger a dedicated GitHub action to build the new package and publish it to PyPI.
"},{"location":"install/","title":"How to install","text":"

The fractal_tasks_core Python package is hosted on PyPI (https://pypi.org/project/fractal-tasks-core), and can be installed via pip. It includes three (sub)packages:

  1. The main fractal_tasks_core package: a set of helper functions to be used in the Fractal tasks (and possibly in other independent packages).
  2. The fractal_tasks_core.tasks subpackage: a set of standard Fractal tasks.
  3. The fractal_tasks_core.dev subpackage: a set of developement tools (mostly related to creation of JSON Schemas for task arguments).
"},{"location":"install/#minimal-installation","title":"Minimal installation","text":"

The minimal installation command is 

pip install fractal-tasks-core\n
which only installs the dependencies necessary for the main package and for the dev subpackage.

"},{"location":"install/#full-installation","title":"Full installation","text":"

In order to also use the tasks subpackage, the additional extra fractal-tasks must be included, as in 

pip install fractal-tasks-core[fractal-tasks]\n
Warning: This command installs heavier dependencies (e.g. torch).

"},{"location":"manifest/","title":"Task manifest","text":"

To be imported automatically within Fractal, tasks must be included in a standard manifest. Here is the one for fractal-task-core's tasks.

{\n\"args_schema_version\": \"pydantic_v1\",\n\"has_args_schema\": true,\n\"has_args_schemas\": true,\n\"manifest_version\": \"1\",\n\"task_list\": [\n{\n\"args_schema\": {\n\"title\": \"CreateOmeZarr\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data from the microscope is stored (as TIF or PNG). Should point to the parent folder containing the images and the metadata files `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present). Example: `[\\\"/some/path/\\\"]`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Path were the output of this task is stored. Example: \\\"/some/path/\\\" => puts the new OME-Zarr file in the \\\"/some/path/\\\". (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"allowed_channels\": {\n\"title\": \"Allowed Channels\",\n\"type\": \"array\",\n\"items\": {\n\"$ref\": \"#/definitions/OmeroChannel\"\n},\n\"description\": \"A list of `OmeroChannel` s, where each channel must include the `wavelength_id` attribute and where the `wavelength_id` values must be unique across the list.\"\n},\n\"image_glob_patterns\": {\n\"title\": \"Image Glob Patterns\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: `image_glob_pattern=[\\\"*_B03_*\\\"]` => only process well B03 `image_glob_pattern=[\\\"*_C09_*\\\", \\\"*F016*\\\", \\\"*Z[0-5][0-9]C*\\\"]` => only process well C09, field of view 16 and Z planes 0-59.\"\n},\n\"num_levels\": {\n\"title\": \"Num Levels\",\n\"default\": 5,\n\"type\": \"integer\",\n\"description\": \"Number of resolution-pyramid levels. If set to `5`, there will be the full-resolution level and 4 levels of downsampled images.\"\n},\n\"coarsening_xy\": {\n\"title\": \"Coarsening Xy\",\n\"default\": 2,\n\"type\": \"integer\",\n\"description\": \"Linear coarsening factor between subsequent levels. If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled etc.\"\n},\n\"image_extension\": {\n\"title\": \"Image Extension\",\n\"default\": \"tif\",\n\"type\": \"string\",\n\"description\": \"Filename extension of images (e.g. `\\\"tif\\\"` or `\\\"png\\\"`)\"\n},\n\"metadata_table_file\": {\n\"title\": \"Metadata Table File\",\n\"type\": \"string\",\n\"description\": \"If `None`, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, the full path to a csv file containing the parsed metadata table.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"metadata\",\n\"allowed_channels\"\n],\n\"additionalProperties\": false,\n\"definitions\": {\n\"Window\": {\n\"title\": \"Window\",\n\"description\": \"Custom class for Omero-channel window, based on OME-NGFF v0.4.\",\n\"type\": \"object\",\n\"properties\": {\n\"min\": {\n\"title\": \"Min\",\n\"type\": \"integer\",\n\"description\": \"Do not change. It will be set to `0` by default.\"\n},\n\"max\": {\n\"title\": \"Max\",\n\"type\": \"integer\",\n\"description\": \"Do not change. It will be set according to bit-depth of the images by default (e.g. 65535 for 16 bit images).\"\n},\n\"start\": {\n\"title\": \"Start\",\n\"type\": \"integer\",\n\"description\": \"Lower-bound rescaling value for visualization.\"\n},\n\"end\": {\n\"title\": \"End\",\n\"type\": \"integer\",\n\"description\": \"Upper-bound rescaling value for visualization.\"\n}\n},\n\"required\": [\n\"start\",\n\"end\"\n]\n},\n\"OmeroChannel\": {\n\"title\": \"OmeroChannel\",\n\"description\": \"Custom class for Omero channels, based on OME-NGFF v0.4.\",\n\"type\": \"object\",\n\"properties\": {\n\"wavelength_id\": {\n\"title\": \"Wavelength Id\",\n\"type\": \"string\",\n\"description\": \"Unique ID for the channel wavelength, e.g. `A01_C01`.\"\n},\n\"index\": {\n\"title\": \"Index\",\n\"type\": \"integer\",\n\"description\": \"Do not change. For internal use only.\"\n},\n\"label\": {\n\"title\": \"Label\",\n\"type\": \"string\",\n\"description\": \"Name of the channel.\"\n},\n\"window\": {\n\"$ref\": \"#/definitions/Window\",\n\"title\": \"Window\",\n\"description\": \"Optional `Window` object to set default display settings for napari.\"\n},\n\"color\": {\n\"title\": \"Color\",\n\"type\": \"string\",\n\"description\": \"Optional hex colormap to display the channel in napari (it must be of length 6, e.g. `00FFFF`).\"\n},\n\"active\": {\n\"title\": \"Active\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Should this channel be shown in the viewer?\"\n},\n\"coefficient\": {\n\"title\": \"Coefficient\",\n\"default\": 1,\n\"type\": \"integer\",\n\"description\": \"Do not change. Omero-channel attribute.\"\n},\n\"inverted\": {\n\"title\": \"Inverted\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"Do not change. Omero-channel attribute.\"\n}\n},\n\"required\": [\n\"wavelength_id\"\n]\n}\n}\n},\n\"executable\": \"tasks/create_ome_zarr.py\",\n\"input_type\": \"image\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000\n},\n\"name\": \"Create OME-Zarr structure\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Create a OME-NGFF zarr folder, without reading/writing image data.\\n\\nFind plates (for each folder in input_paths):\\n\\n- glob image files,\\n- parse metadata from image filename to identify plates,\\n- identify populated channels.\\n\\nCreate a zarr folder (for each plate):\\n\\n- parse mlf metadata,\\n- identify wells and field of view (FOV),\\n- create FOV ZARR,\\n- verify that channels are uniform (i.e., same channels).\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/create_ome_zarr/#fractal_tasks_core.tasks.create_ome_zarr.create_ome_zarr\"\n},\n{\n\"args_schema\": {\n\"title\": \"YokogawaToOmeZarr\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Unclear. Should be the same as `input_path`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"` (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata. `original_paths`: list of paths that correspond to the `input_paths` of the `create_ome_zarr` task (=> where the microscopy image are stored); `num_levels (int)`: number of pyramid levels in the image (this determines how many pyramid levels are built for the segmentation); `coarsening_xy (int)`: coarsening factor in XY of the downsampling when building the pyramid; `image_extension`: filename extension of images (e.g. `\\\"tif\\\"` or `\\\"png\\\"`); `image_glob_patterns`: parameter of `create_ome_zarr` task (if specified, only parse images with filenames that match with all these patterns). (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\"\n],\n\"additionalProperties\": false\n},\n\"executable\": \"tasks/yokogawa_to_ome_zarr.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Convert Yokogawa to OME-Zarr\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Convert Yokogawa output (png, tif) to zarr file.\\n\\nThis task is typically run after Create OME-Zarr or\\nCreate OME-Zarr Multiplexing and populates the empty OME-Zarr files that\\nwere prepared.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/#fractal_tasks_core.tasks.yokogawa_to_ome_zarr.yokogawa_to_ome_zarr\"\n},\n{\n\"args_schema\": {\n\"title\": \"CopyOmeZarr\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Path were the output of this task is stored. Example: `\\\"/some/path/\\\"` => puts the new OME-Zarr file in the same folder as the input OME-Zarr file `\\\"/some/new_path\\\"` => puts the new OME-Zarr file into a new folder at `/some/new_path`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata: `plate`: List of plates (e.g. `[\\\"MyPlate.zarr\\\"]`); `well`: List of wells in the OME-Zarr plate (e.g. `[\\\"MyPlate.zarr/B/03/MyPlate.zarr/B/05\\\"]`); \\\"image\\\": List of images in the OME-Zarr plate (e.g. `[\\\"MyPlate.zarr/B/03/0\\\", \\\"MyPlate.zarr/B/05/0\\\"]`). standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"project_to_2D\": {\n\"title\": \"Project To 2D\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, apply a 3D->2D projection to the ROI tables that are copied to the new OME-Zarr.\"\n},\n\"suffix\": {\n\"title\": \"Suffix\",\n\"default\": \"mip\",\n\"type\": \"string\",\n\"description\": \"The suffix that is used to transform `plate.zarr` into `plate_suffix.zarr`. Note that `None` is not currently supported.\"\n},\n\"ROI_table_names\": {\n\"title\": \"Roi Table Names\",\n\"default\": [\n\"FOV_ROI_table\",\n\"well_ROI_table\"\n],\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of Anndata table names to be copied. Note: copying non-ROI tables may fail if `project_to_2D=True`.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"metadata\"\n],\n\"additionalProperties\": false\n},\n\"executable\": \"tasks/copy_ome_zarr.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 1000\n},\n\"name\": \"Copy OME-Zarr structure\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Duplicate an input zarr structure to a new path.\\n\\nThis task copies all the structure, but none of the image data:\\n\\n- For each plate, create a new zarr group with the same attributes as\\n   the original one.\\n- For each well (in each plate), create a new zarr subgroup with the\\n   same attributes as the original one.\\n- For each image (in each well), create a new zarr subgroup with the\\n   same attributes as the original one.\\n- For each image (in each well), copy the relevant AnnData tables from\\n   the original source.\\n\\nNote: this task makes use of methods from the `Attributes` class, see\\nhttps://zarr.readthedocs.io/en/stable/api/attrs.html.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/copy_ome_zarr/#fractal_tasks_core.tasks.copy_ome_zarr.copy_ome_zarr\"\n},\n{\n\"args_schema\": {\n\"title\": \"MaximumIntensityProjection\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"This parameter is not used by this task. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Path were the output of this task is stored. Example: `\\\"/some/path/\\\"` => puts the new OME-Zarr file in that folder. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Component is typically changed by the `copy_ome_zarr` task before, to point to a new mip Zarr file. Example: `\\\"some_plate_mip.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"Dictionary containing metadata about the OME-Zarr. This task requires the key `copy_ome_zarr` to be present in the metadata (as defined in `copy_ome_zarr` task). (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\"\n],\n\"additionalProperties\": false\n},\n\"executable\": \"tasks/maximum_intensity_projection.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Maximum Intensity Projection\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Perform maximum-intensity projection along Z axis.\\n\\nNote: this task stores the output in a new zarr file.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/maximum_intensity_projection/#fractal_tasks_core.tasks.maximum_intensity_projection.maximum_intensity_projection\"\n},\n{\n\"args_schema\": {\n\"title\": \"CellposeSegmentation\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"level\": {\n\"title\": \"Level\",\n\"type\": \"integer\",\n\"description\": \"Pyramid level of the image to be segmented. Choose `0` to process at full resolution.\"\n},\n\"channel\": {\n\"$ref\": \"#/definitions/Channel\",\n\"title\": \"Channel\",\n\"description\": \"Primary channel for segmentation; requires either `wavelength_id` (e.g. `A01_C01`) or `label` (e.g. `DAPI`).\"\n},\n\"channel2\": {\n\"$ref\": \"#/definitions/Channel\",\n\"title\": \"Channel2\",\n\"description\": \"Second channel for segmentation (in the same format as `channel`). If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker.\"\n},\n\"input_ROI_table\": {\n\"title\": \"Input Roi Table\",\n\"default\": \"FOV_ROI_table\",\n\"type\": \"string\",\n\"description\": \"Name of the ROI table over which the task loops to apply Cellpose segmentation. Examples: `FOV_ROI_table` => loop over the field of views, `organoid_ROI_table` => loop over the organoid ROI table (generated by another task), `well_ROI_table` => process the whole well as one image.\"\n},\n\"output_ROI_table\": {\n\"title\": \"Output Roi Table\",\n\"type\": \"string\",\n\"description\": \"If provided, a ROI table with that name is created, which will contain the bounding boxes of the newly segmented labels. ROI tables should have `ROI` in their name.\"\n},\n\"output_label_name\": {\n\"title\": \"Output Label Name\",\n\"type\": \"string\",\n\"description\": \"Name of the output label image (e.g. `\\\"organoids\\\"`).\"\n},\n\"use_masks\": {\n\"title\": \"Use Masks\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, try to use masked loading and fall back to `use_masks=False` if the ROI table is not suitable. Masked loading is relevant when only a subset of the bounding box should actually be processed (e.g. running within `organoid_ROI_table`).\"\n},\n\"relabeling\": {\n\"title\": \"Relabeling\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, apply relabeling so that label values are unique for all objects in the well.\"\n},\n\"diameter_level0\": {\n\"title\": \"Diameter Level0\",\n\"default\": 30.0,\n\"type\": \"number\",\n\"description\": \"Expected diameter of the objects that should be segmented in pixels at level 0. Initial diameter is rescaled using the `level` that was selected. The rescaled value is passed as the diameter to the `CellposeModel.eval` method.\"\n},\n\"model_type\": {\n\"title\": \"Model Type\",\n\"default\": \"cyto2\",\n\"type\": \"string\",\n\"description\": \"Parameter of `CellposeModel` class. Defines which model should be used. Typical choices are `nuclei`, `cyto`, `cyto2`, etc.\"\n},\n\"pretrained_model\": {\n\"title\": \"Pretrained Model\",\n\"type\": \"string\",\n\"description\": \"Parameter of `CellposeModel` class (takes precedence over `model_type`). Allows you to specify the path of a custom trained cellpose model.\"\n},\n\"cellprob_threshold\": {\n\"title\": \"Cellprob Threshold\",\n\"default\": 0.0,\n\"type\": \"number\",\n\"description\": \"Parameter of `CellposeModel.eval` method. Valid values between -6 to 6. From Cellpose documentation: \\\"Decrease this threshold if cellpose is not returning as many ROIs as you\\u2019d expect. Similarly, increase this threshold if cellpose is returning too ROIs particularly from dim areas.\\\"\"\n},\n\"flow_threshold\": {\n\"title\": \"Flow Threshold\",\n\"default\": 0.4,\n\"type\": \"number\",\n\"description\": \"Parameter of `CellposeModel.eval` method. Valid values between 0.0 and 1.0. From Cellpose documentation: \\\"Increase this threshold if cellpose is not returning as many ROIs as you\\u2019d expect. Similarly, decrease this threshold if cellpose is returning too many ill-shaped ROIs.\\\"\"\n},\n\"anisotropy\": {\n\"title\": \"Anisotropy\",\n\"type\": \"number\",\n\"description\": \"Ratio of the pixel sizes along Z and XY axis (ignored if the image is not three-dimensional). If `None`, it is inferred from the OME-NGFF metadata.\"\n},\n\"min_size\": {\n\"title\": \"Min Size\",\n\"default\": 15,\n\"type\": \"integer\",\n\"description\": \"Parameter of `CellposeModel` class. Minimum size of the segmented objects (in pixels). Use `-1` to turn off the size filter.\"\n},\n\"augment\": {\n\"title\": \"Augment\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"Parameter of `CellposeModel` class. Whether to use cellpose augmentation to tile images with overlap.\"\n},\n\"net_avg\": {\n\"title\": \"Net Avg\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"Parameter of `CellposeModel` class. Whether to use cellpose net averaging to run the 4 built-in networks (useful for `nuclei`, `cyto` and `cyto2`, not sure it works for the others).\"\n},\n\"use_gpu\": {\n\"title\": \"Use Gpu\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `False`, always use the CPU; if `True`, use the GPU if possible (as defined in `cellpose.core.use_gpu()`) and fall-back to the CPU otherwise.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\",\n\"level\",\n\"channel\"\n],\n\"additionalProperties\": false,\n\"definitions\": {\n\"Channel\": {\n\"title\": \"Channel\",\n\"description\": \"A channel which is specified by either `wavelength_id` or `label`.\",\n\"type\": \"object\",\n\"properties\": {\n\"wavelength_id\": {\n\"title\": \"Wavelength Id\",\n\"type\": \"string\",\n\"description\": \"Unique ID for the channel wavelength, e.g. `A01_C01`.\"\n},\n\"label\": {\n\"title\": \"Label\",\n\"type\": \"string\",\n\"description\": \"Name of the channel.\"\n}\n}\n}\n}\n},\n\"executable\": \"tasks/cellpose_segmentation.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 4,\n\"mem\": 16000,\n\"needs_gpu\": true,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Cellpose Segmentation\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Run cellpose segmentation on the ROIs of a single OME-Zarr image.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/cellpose_segmentation/#fractal_tasks_core.tasks.cellpose_segmentation.cellpose_segmentation\"\n},\n{\n\"args_schema\": {\n\"title\": \"IlluminationCorrection\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Path were the output of this task is stored. Examples: `\\\"/some/path/\\\"` => puts the new OME-Zarr file in the same folder as the input OME-Zarr file; `\\\"/some/new_path\\\"` => puts the new OME-Zarr file into a new folder at `/some/new_path`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"illumination_profiles_folder\": {\n\"title\": \"Illumination Profiles Folder\",\n\"type\": \"string\",\n\"description\": \"Path of folder of illumination profiles.\"\n},\n\"dict_corr\": {\n\"title\": \"Dict Corr\",\n\"type\": \"object\",\n\"additionalProperties\": {\n\"type\": \"string\"\n},\n\"description\": \"Dictionary where keys match the `wavelength_id` attributes of existing channels (e.g. `A01_C01` ) and values are the filenames of the corresponding illumination profiles.\"\n},\n\"background\": {\n\"title\": \"Background\",\n\"default\": 110,\n\"type\": \"integer\",\n\"description\": \"Background value that is subtracted from the image before the illumination correction is applied. Set it to `0` if you don't want any background subtraction.\"\n},\n\"overwrite_input\": {\n\"title\": \"Overwrite Input\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, the results of this task will overwrite the input image data. In the current version, `overwrite_input=False` is not implemented.\"\n},\n\"new_component\": {\n\"title\": \"New Component\",\n\"type\": \"string\",\n\"description\": \"Not implemented yet. This is not implemented well in Fractal server at the moment, it's unclear how a user would specify fitting new components. If the results shall not overwrite the input data and the output path is the same as the input path, a new component needs to be provided. Example: `myplate_new_name.zarr/B/03/0/`.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\",\n\"illumination_profiles_folder\",\n\"dict_corr\"\n],\n\"additionalProperties\": false\n},\n\"executable\": \"tasks/illumination_correction.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Illumination correction\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Applies illumination correction to the images in the OME-Zarr.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/illumination_correction/#fractal_tasks_core.tasks.illumination_correction.illumination_correction\"\n},\n{\n\"args_schema\": {\n\"title\": \"NapariWorkflowsWrapper\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. his task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"workflow_file\": {\n\"title\": \"Workflow File\",\n\"type\": \"string\",\n\"description\": \"Absolute path to napari-workflows YAML file\"\n},\n\"input_specs\": {\n\"title\": \"Input Specs\",\n\"type\": \"object\",\n\"additionalProperties\": {\n\"$ref\": \"#/definitions/NapariWorkflowsInput\"\n},\n\"description\": \"A dictionary of `NapariWorkflowsInput` values.\"\n},\n\"output_specs\": {\n\"title\": \"Output Specs\",\n\"type\": \"object\",\n\"additionalProperties\": {\n\"$ref\": \"#/definitions/NapariWorkflowsOutput\"\n},\n\"description\": \"A dictionary of `NapariWorkflowsOutput` values.\"\n},\n\"input_ROI_table\": {\n\"title\": \"Input Roi Table\",\n\"default\": \"FOV_ROI_table\",\n\"type\": \"string\",\n\"description\": \"Name of the ROI table over which the task loops to apply napari workflows. Examples: `FOV_ROI_table` => loop over the field of views; `organoid_ROI_table` => loop over the organoid ROI table (generated by another task); `well_ROI_table` => process the whole well as one image.\"\n},\n\"level\": {\n\"title\": \"Level\",\n\"default\": 0,\n\"type\": \"integer\",\n\"description\": \"Pyramid level of the image to be used as input for napari-workflows. Choose `0` to process at full resolution. Levels > 0 are currently only supported for workflows that only have intensity images as input and only produce a label images as output.\"\n},\n\"relabeling\": {\n\"title\": \"Relabeling\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, apply relabeling so that label values are unique across all ROIs in the well.\"\n},\n\"expected_dimensions\": {\n\"title\": \"Expected Dimensions\",\n\"default\": 3,\n\"type\": \"integer\",\n\"description\": \"Expected dimensions (either `2` or `3`). Useful when loading 2D images that are stored in a 3D array with shape `(1, size_x, size_y)` [which is the default way Fractal stores 2D images], but you want to make sure the napari workflow gets a 2D array to process. Also useful to set to `2` when loading a 2D OME-Zarr that is saved as `(size_x, size_y)`.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\",\n\"workflow_file\",\n\"input_specs\",\n\"output_specs\"\n],\n\"additionalProperties\": false,\n\"definitions\": {\n\"Channel\": {\n\"title\": \"Channel\",\n\"description\": \"A channel which is specified by either `wavelength_id` or `label`.\",\n\"type\": \"object\",\n\"properties\": {\n\"wavelength_id\": {\n\"title\": \"Wavelength Id\",\n\"type\": \"string\",\n\"description\": \"Unique ID for the channel wavelength, e.g. `A01_C01`.\"\n},\n\"label\": {\n\"title\": \"Label\",\n\"type\": \"string\",\n\"description\": \"Name of the channel.\"\n}\n}\n},\n\"NapariWorkflowsInput\": {\n\"title\": \"NapariWorkflowsInput\",\n\"description\": \"A value of the `input_specs` argument in `napari_workflows_wrapper`.\",\n\"type\": \"object\",\n\"properties\": {\n\"type\": {\n\"title\": \"Type\",\n\"enum\": [\n\"image\",\n\"label\"\n],\n\"type\": \"string\",\n\"description\": \"Input type (either `image` or `label`).\"\n},\n\"label_name\": {\n\"title\": \"Label Name\",\n\"type\": \"string\",\n\"description\": \"Label name (for label inputs only).\"\n},\n\"channel\": {\n\"$ref\": \"#/definitions/Channel\",\n\"title\": \"Channel\",\n\"description\": \"Channel object (for image inputs only).\"\n}\n},\n\"required\": [\n\"type\"\n]\n},\n\"NapariWorkflowsOutput\": {\n\"title\": \"NapariWorkflowsOutput\",\n\"description\": \"A value of the `output_specs` argument in `napari_workflows_wrapper`.\",\n\"type\": \"object\",\n\"properties\": {\n\"type\": {\n\"title\": \"Type\",\n\"enum\": [\n\"label\",\n\"dataframe\"\n],\n\"type\": \"string\",\n\"description\": \"Output type (either `label` or `dataframe`).\"\n},\n\"label_name\": {\n\"title\": \"Label Name\",\n\"type\": \"string\",\n\"description\": \"Label name (for label outputs only).\"\n},\n\"table_name\": {\n\"title\": \"Table Name\",\n\"type\": \"string\",\n\"description\": \"Table name (for dataframe outputs only).\"\n}\n},\n\"required\": [\n\"type\"\n]\n}\n}\n},\n\"executable\": \"tasks/napari_workflows_wrapper.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 8,\n\"mem\": 32000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Napari workflows wrapper\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Run a napari-workflow on the ROIs of a single OME-NGFF image.\\n\\nThis task takes images and labels and runs a napari-workflow on them that\\ncan produce a label and tables as output.\\n\\nExamples of allowed entries for `input_specs` and `output_specs`:\\n\\n```\\ninput_specs = {\\n    \\\"in_1\\\": {\\\"type\\\": \\\"image\\\", \\\"channel\\\": {\\\"wavelength_id\\\": \\\"A01_C02\\\"}},\\n    \\\"in_2\\\": {\\\"type\\\": \\\"image\\\", \\\"channel\\\": {\\\"label\\\": \\\"DAPI\\\"}},\\n    \\\"in_3\\\": {\\\"type\\\": \\\"label\\\", \\\"label_name\\\": \\\"label_DAPI\\\"},\\n}\\n\\noutput_specs = {\\n    \\\"out_1\\\": {\\\"type\\\": \\\"label\\\", \\\"label_name\\\": \\\"label_DAPI_new\\\"},\\n    \\\"out_2\\\": {\\\"type\\\": \\\"dataframe\\\", \\\"table_name\\\": \\\"measurements\\\"},\\n}\\n```\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/napari_workflows_wrapper/#fractal_tasks_core.tasks.napari_workflows_wrapper.napari_workflows_wrapper\"\n},\n{\n\"args_schema\": {\n\"title\": \"CreateOmeZarrMultiplex\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data from the microscope is stored (as TIF or PNG). Each element of the list is treated as another cycle of the multiplexing data, the cycles are ordered by their order in this list. Should point to the parent folder containing the images and the metadata files `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present). Example: `[\\\"/path/cycle1/\\\", \\\"/path/cycle2/\\\"]`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Path were the output of this task is stored. Example: `\\\"/some/path/\\\"` => puts the new OME-Zarr file in the `/some/path/`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"allowed_channels\": {\n\"title\": \"Allowed Channels\",\n\"type\": \"object\",\n\"additionalProperties\": {\n\"type\": \"array\",\n\"items\": {\n\"$ref\": \"#/definitions/OmeroChannel\"\n}\n},\n\"description\": \"A dictionary of lists of `OmeroChannel`s, where each channel must include the `wavelength_id` attribute and where the `wavelength_id` values must be unique across each list. Dictionary keys represent channel indices (`\\\"0\\\",\\\"1\\\",..`).\"\n},\n\"image_glob_patterns\": {\n\"title\": \"Image Glob Patterns\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: `image_glob_pattern=[\\\"*_B03_*\\\"]` => only process well B03 `image_glob_pattern=[\\\"*_C09_*\\\", \\\"*F016*\\\", \\\"*Z[0-5][0-9]C*\\\"]` => only process well C09, field of view 16 and Z planes 0-59.\"\n},\n\"num_levels\": {\n\"title\": \"Num Levels\",\n\"default\": 5,\n\"type\": \"integer\",\n\"description\": \"Number of resolution-pyramid levels. If set to `5`, there will be the full-resolution level and 4 levels of downsampled images.\"\n},\n\"coarsening_xy\": {\n\"title\": \"Coarsening Xy\",\n\"default\": 2,\n\"type\": \"integer\",\n\"description\": \"Linear coarsening factor between subsequent levels. If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled etc.\"\n},\n\"image_extension\": {\n\"title\": \"Image Extension\",\n\"default\": \"tif\",\n\"type\": \"string\",\n\"description\": \"Filename extension of images (e.g. `\\\"tif\\\"` or `\\\"png\\\"`).\"\n},\n\"metadata_table_files\": {\n\"title\": \"Metadata Table Files\",\n\"type\": \"object\",\n\"additionalProperties\": {\n\"type\": \"string\"\n},\n\"description\": \"If `None`, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, a dictionary of key-value pairs like `(acquisition, path)` with `acquisition` a string and `path` pointing to a csv file containing the parsed metadata table.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"If `True`, overwrite the task output.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"metadata\",\n\"allowed_channels\"\n],\n\"additionalProperties\": false,\n\"definitions\": {\n\"Window\": {\n\"title\": \"Window\",\n\"description\": \"Custom class for Omero-channel window, based on OME-NGFF v0.4.\",\n\"type\": \"object\",\n\"properties\": {\n\"min\": {\n\"title\": \"Min\",\n\"type\": \"integer\",\n\"description\": \"Do not change. It will be set to `0` by default.\"\n},\n\"max\": {\n\"title\": \"Max\",\n\"type\": \"integer\",\n\"description\": \"Do not change. It will be set according to bit-depth of the images by default (e.g. 65535 for 16 bit images).\"\n},\n\"start\": {\n\"title\": \"Start\",\n\"type\": \"integer\",\n\"description\": \"Lower-bound rescaling value for visualization.\"\n},\n\"end\": {\n\"title\": \"End\",\n\"type\": \"integer\",\n\"description\": \"Upper-bound rescaling value for visualization.\"\n}\n},\n\"required\": [\n\"start\",\n\"end\"\n]\n},\n\"OmeroChannel\": {\n\"title\": \"OmeroChannel\",\n\"description\": \"Custom class for Omero channels, based on OME-NGFF v0.4.\",\n\"type\": \"object\",\n\"properties\": {\n\"wavelength_id\": {\n\"title\": \"Wavelength Id\",\n\"type\": \"string\",\n\"description\": \"Unique ID for the channel wavelength, e.g. `A01_C01`.\"\n},\n\"index\": {\n\"title\": \"Index\",\n\"type\": \"integer\",\n\"description\": \"Do not change. For internal use only.\"\n},\n\"label\": {\n\"title\": \"Label\",\n\"type\": \"string\",\n\"description\": \"Name of the channel.\"\n},\n\"window\": {\n\"$ref\": \"#/definitions/Window\",\n\"title\": \"Window\",\n\"description\": \"Optional `Window` object to set default display settings for napari.\"\n},\n\"color\": {\n\"title\": \"Color\",\n\"type\": \"string\",\n\"description\": \"Optional hex colormap to display the channel in napari (it must be of length 6, e.g. `00FFFF`).\"\n},\n\"active\": {\n\"title\": \"Active\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Should this channel be shown in the viewer?\"\n},\n\"coefficient\": {\n\"title\": \"Coefficient\",\n\"default\": 1,\n\"type\": \"integer\",\n\"description\": \"Do not change. Omero-channel attribute.\"\n},\n\"inverted\": {\n\"title\": \"Inverted\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"Do not change. Omero-channel attribute.\"\n}\n},\n\"required\": [\n\"wavelength_id\"\n]\n}\n}\n},\n\"executable\": \"tasks/create_ome_zarr_multiplex.py\",\n\"input_type\": \"image\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000\n},\n\"name\": \"Create OME-ZARR structure (multiplexing)\",\n\"output_type\": \"zarr\",\n\"docs_info\": \"Create OME-NGFF structure and metadata to host a multiplexing dataset.\\n\\nThis task takes a set of image folders (i.e. different acquisition cycles)\\nand build the internal structure and metadata of a OME-NGFF zarr group,\\nwithout actually loading/writing the image data.\\n\\nEach element in input_paths should be treated as a different acquisition.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/#fractal_tasks_core.tasks.create_ome_zarr_multiplex.create_ome_zarr_multiplex\"\n},\n{\n\"executable\": \"tasks/calculate_registration_image_based.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 8000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Calculate registration (image-based)\",\n\"output_type\": \"zarr\",\n\"args_schema\": {\n\"title\": \"CalculateRegistrationImageBased\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"wavelength_id\": {\n\"title\": \"Wavelength Id\",\n\"type\": \"string\",\n\"description\": \"Wavelength that will be used for image-based registration; e.g. `A01_C01` for Yokogawa, `C01` for MD.\"\n},\n\"roi_table\": {\n\"title\": \"Roi Table\",\n\"default\": \"FOV_ROI_table\",\n\"type\": \"string\",\n\"description\": \"Name of the ROI table over which the task loops to calculate the registration. Examples: `FOV_ROI_table` => loop over the field of views, `well_ROI_table` => process the whole well as one image.\"\n},\n\"reference_cycle\": {\n\"title\": \"Reference Cycle\",\n\"default\": 0,\n\"type\": \"integer\",\n\"description\": \"Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well (usually the first cycle that was provided).\"\n},\n\"level\": {\n\"title\": \"Level\",\n\"default\": 2,\n\"type\": \"integer\",\n\"description\": \"Pyramid level of the image to be segmented. Choose `0` to process at full resolution.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\",\n\"wavelength_id\"\n],\n\"additionalProperties\": false\n},\n\"docs_info\": \"Calculate registration based on images\\n\\nThis task consists of 3 parts:\\n\\n1. Loading the images of a given ROI (=> loop over ROIs)\\n2. Calculating the transformation for that ROI\\n3. Storing the calculated transformation in the ROI table\\n\\nParallelization level: image\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/calculate_registration_image_based/#fractal_tasks_core.tasks.calculate_registration_image_based.calculate_registration_image_based\"\n},\n{\n\"executable\": \"tasks/apply_registration_to_ROI_tables.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 1000,\n\"parallelization_level\": \"well\"\n},\n\"name\": \"Apply Registration to ROI Tables\",\n\"output_type\": \"zarr\",\n\"args_schema\": {\n\"title\": \"ApplyRegistrationToRoiTables\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"roi_table\": {\n\"title\": \"Roi Table\",\n\"default\": \"FOV_ROI_table\",\n\"type\": \"string\",\n\"description\": \"Name of the ROI table over which the task loops to calculate the registration. Examples: `FOV_ROI_table` => loop over the field of views, `well_ROI_table` => process the whole well as one image.\"\n},\n\"reference_cycle\": {\n\"title\": \"Reference Cycle\",\n\"default\": 0,\n\"type\": \"integer\",\n\"description\": \"Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided\"\n},\n\"new_roi_table\": {\n\"title\": \"New Roi Table\",\n\"type\": \"string\",\n\"description\": \"Optional name for the new, registered ROI table. If no name is given, it will default to \\\"registered_\\\" + `roi_table`\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\"\n],\n\"additionalProperties\": false\n},\n\"docs_info\": \"Applies pre-calculated registration to ROI tables.\\n\\nApply pre-calculated registration such that resulting ROIs contain\\nthe consensus align region between all cycles.\\n\\nParallelization level: well\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/#fractal_tasks_core.tasks.apply_registration_to_ROI_tables.apply_registration_to_ROI_tables\"\n},\n{\n\"executable\": \"tasks/apply_registration_to_image.py\",\n\"input_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000,\n\"parallelization_level\": \"image\"\n},\n\"name\": \"Apply Registration to Image\",\n\"output_type\": \"zarr\",\n\"args_schema\": {\n\"title\": \"ApplyRegistrationToImage\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: `[\\\"/some/path/\\\"]`. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"component\": {\n\"title\": \"Component\",\n\"type\": \"string\",\n\"description\": \"Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: `\\\"some_plate.zarr/B/03/0\\\"`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"registered_roi_table\": {\n\"title\": \"Registered Roi Table\",\n\"type\": \"string\",\n\"description\": \"Name of the ROI table which has been registered and will be applied to mask and shift the images. Examples: `registered_FOV_ROI_table` => loop over the field of views, `registered_well_ROI_table` => process the whole well as one image.\"\n},\n\"reference_cycle\": {\n\"title\": \"Reference Cycle\",\n\"default\": \"0\",\n\"type\": \"string\",\n\"description\": \"Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided\"\n},\n\"overwrite_input\": {\n\"title\": \"Overwrite Input\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Whether the old image data should be replaced with the newly registered image data. Currently only implemented for `overwrite_input=True`.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"component\",\n\"metadata\",\n\"registered_roi_table\"\n],\n\"additionalProperties\": false\n},\n\"docs_info\": \"Apply registration to images by using a registered ROI table\\n\\nThis task consists of 4 parts:\\n\\n1. Mask all regions in images that are not available in the\\nregistered ROI table and store each cycle aligned to the\\nreference_cycle (by looping over ROIs).\\n2. Do the same for all label images.\\n3. Copy all tables from the non-aligned image to the aligned image\\n(currently only works well if the only tables are well & FOV ROI tables\\n(registered and original). Not implemented for measurement tables and\\nother ROI tables).\\n4. Clean up: Delete the old, non-aligned image and rename the new,\\naligned image to take over its place.\\n\\nParallelization level: image\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/apply_registration_to_image/#fractal_tasks_core.tasks.apply_registration_to_image.apply_registration_to_image\"\n},\n{\n\"name\": \"Import OME-Zarr\",\n\"executable\": \"tasks/import_ome_zarr.py\",\n\"input_type\": \"zarr\",\n\"output_type\": \"zarr\",\n\"meta\": {\n\"cpus_per_task\": 1,\n\"mem\": 4000\n},\n\"args_schema\": {\n\"title\": \"ImportOmeZarr\",\n\"type\": \"object\",\n\"properties\": {\n\"input_paths\": {\n\"title\": \"Input Paths\",\n\"type\": \"array\",\n\"items\": {\n\"type\": \"string\"\n},\n\"description\": \"A length-one list with the parent folder of the OME-Zarr to be imported; e.g. `input_paths=[\\\"/somewhere\\\"]`, if the OME-Zarr path is `/somewhere/array.zarr`. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"output_path\": {\n\"title\": \"Output Path\",\n\"type\": \"string\",\n\"description\": \"Not used in this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"metadata\": {\n\"title\": \"Metadata\",\n\"type\": \"object\",\n\"description\": \"Not used in this task. (standard argument for Fractal tasks, managed by Fractal server).\"\n},\n\"zarr_name\": {\n\"title\": \"Zarr Name\",\n\"type\": \"string\",\n\"description\": \"The OME-Zarr name, without its parent folder; e.g. `zarr_name=\\\"array.zarr\\\"`, if the OME-Zarr path is `/somewhere/array.zarr`.\"\n},\n\"add_image_ROI_table\": {\n\"title\": \"Add Image Roi Table\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Whether to add a `image_ROI_table` table to each image, with a single ROI covering the whole image.\"\n},\n\"add_grid_ROI_table\": {\n\"title\": \"Add Grid Roi Table\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Whether to add a `grid_ROI_table` table to each image, with the image split into a rectangular grid of ROIs.\"\n},\n\"grid_y_shape\": {\n\"title\": \"Grid Y Shape\",\n\"default\": 2,\n\"type\": \"integer\",\n\"description\": \"Y shape of the ROI grid in `grid_ROI_table`.\"\n},\n\"grid_x_shape\": {\n\"title\": \"Grid X Shape\",\n\"default\": 2,\n\"type\": \"integer\",\n\"description\": \"X shape of the ROI grid in `grid_ROI_table`.\"\n},\n\"update_omero_metadata\": {\n\"title\": \"Update Omero Metadata\",\n\"default\": true,\n\"type\": \"boolean\",\n\"description\": \"Whether to update Omero-channels metadata, to make them Fractal-compatible.\"\n},\n\"overwrite\": {\n\"title\": \"Overwrite\",\n\"default\": false,\n\"type\": \"boolean\",\n\"description\": \"Whether new ROI tables (added when `add_image_ROI_table` and/or `add_grid_ROI_table` are `True`) can overwite existing ones.\"\n}\n},\n\"required\": [\n\"input_paths\",\n\"output_path\",\n\"metadata\",\n\"zarr_name\"\n],\n\"additionalProperties\": false\n},\n\"docs_info\": \"Import an OME-Zarr into Fractal.\\n\\nThe current version of this task:\\n\\n1. Creates the appropriate components-related metadata, needed for\\n   processing an existing OME-Zarr through Fractal.\\n2. Optionally adds new ROI tables to the existing OME-Zarr.\",\n\"docs_link\": \"https://fractal-analytics-platform.github.io/fractal-tasks-core/reference/fractal_tasks_core/tasks/import_ome_zarr/#fractal_tasks_core.tasks.import_ome_zarr.import_ome_zarr\"\n}\n]\n}\n
"},{"location":"tasks_in_fractal/","title":"Run tasks within the Fractal platform","text":"

Thanks to the package manifest and to their structure, the tasks in fractal_tasks_core.tasks can be run within the Fractal platform; this consists in a backend server which can be accessed by one of the two available clients (a command-line client and a web-client).

The fractal-demos repository lists a set of relevant examples, including:

  • How to set up a fractal-server instance;
  • How to set up a fractal command-line client;
  • How to use the command-line client to submit a series of typical workflows (based on fractal-tasks-core tasks) to Fractal; see folders from 01 to 10 in the examples folder.
"},{"location":"tasks_in_scripts/","title":"Run tasks from Python scripts","text":"

The fractal-tasks-core GitHub repository includes an examples folder, listing a few examples of how to run fractal-tasks-core tasks from a standard Python script (instead of using the Fractal platform).

What follows is the content of examples/README.md:

"},{"location":"tasks_in_scripts/#examples","title":"Examples","text":"

This examples folder offers a few examples of how to run fractal-tasks-core tasks as part of a Python script.

"},{"location":"tasks_in_scripts/#notes","title":"Notes","text":"
  • This folder is not always kept up-to-date. If you encounter any unexpected problem, please open a new issue on the fractal-tasks-core GitHub repository.
  • Examples from 01 to 09 are currently aligned with fractal-tasks-core 0.10.0.
"},{"location":"tasks_in_scripts/#general-instructions","title":"General instructions","text":"

The following instructions are valid for all examples; check the specific README.md files in each folder for more specific details.

  1. Set up the correct environment via 

    pip install fractal-tasks-core[fractal-tasks]\n
    (note: this can be done e.g. from a venv or from a conda environment).

  2. Download the example data from Zenodo, if necessary, via 

    pip install zenodo-get\n./fetch_test_data_from_zenodo.sh\n

  3. Enter one of the example folders, remove the tmp_out temporary output folder (if present), and run one of the run_workflow Python scripts.

  4. View the output OME-Zarr in the tmp_out folder with napari, which can be installed via pip install napari[pyqt5] napari-ome-zarr.

"},{"location":"reference/fractal_tasks_core/","title":"Index","text":""},{"location":"reference/fractal_tasks_core/SUMMARY/","title":"SUMMARY","text":"
  • tasks
    • _utils
    • apply_registration_to_image
    • apply_registration_to_ROI_tables
    • calculate_registration_image_based
    • cellpose_segmentation
    • compress_tif
    • copy_ome_zarr
    • create_ome_zarr
    • create_ome_zarr_multiplex
    • illumination_correction
    • import_ome_zarr
    • maximum_intensity_projection
    • napari_workflows_wrapper
    • yokogawa_to_ome_zarr
  • dev
    • check_manifest
    • lib_args_schemas
    • lib_descriptions
    • lib_signature_constraints
    • lib_task_docs
    • lib_titles
    • update_manifest
  • lib_channels
  • lib_glob
  • lib_input_models
  • lib_masked_loading
  • lib_metadata_parsing
  • lib_ngff
  • lib_parse_filename_metadata
  • lib_pyramid_creation
  • lib_read_fractal_metadata
  • lib_regions_of_interest
  • lib_ROI_overlaps
  • lib_upscale_array
  • lib_write
  • lib_zattrs_utils
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/","title":"lib_ROI_overlaps","text":"

Functions to identify and remove overlaps between regions of interest.

"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps._is_overlapping_1D_int","title":"_is_overlapping_1D_int(line1, line2)","text":"

Given two integer intervals, find whether they overlap

This is the same as is_overlapping_1D (based on https://stackoverflow.com/a/70023212/19085332), for integer-valued intervals.

PARAMETER DESCRIPTION line1

The boundaries of the first interval , written as [x_min, x_max].

TYPE: Sequence[int]

line2

The boundaries of the second interval , written as [x_min, x_max].

TYPE: Sequence[int]

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def _is_overlapping_1D_int(\n    line1: Sequence[int],\n    line2: Sequence[int],\n) -> bool:\n\"\"\"\n    Given two integer intervals, find whether they overlap\n\n    This is the same as `is_overlapping_1D` (based on\n    https://stackoverflow.com/a/70023212/19085332), for integer-valued\n    intervals.\n\n    Args:\n        line1: The boundaries of the first interval , written as\n            `[x_min, x_max]`.\n        line2: The boundaries of the second interval , written as\n            `[x_min, x_max]`.\n    \"\"\"\n    return line1[0] < line2[1] and line2[0] < line1[1]\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps._is_overlapping_3D_int","title":"_is_overlapping_3D_int(box1, box2)","text":"

Given two three-dimensional integer boxes, find whether they overlap.

This is the same as is_overlapping_3D (based on https://stackoverflow.com/a/70023212/19085332), for integer-valued boxes.

PARAMETER DESCRIPTION box1

The boundaries of the first box, written as [x_min, y_min, z_min, x_max, y_max, z_max].

TYPE: list[int]

box2

The boundaries of the second box, written as [x_min, y_min, z_min, x_max, y_max, z_max].

TYPE: list[int]

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def _is_overlapping_3D_int(box1: list[int], box2: list[int]) -> bool:\n\"\"\"\n    Given two three-dimensional integer boxes, find whether they overlap.\n\n    This is the same as is_overlapping_3D (based on\n    https://stackoverflow.com/a/70023212/19085332), for integer-valued\n    boxes.\n\n    Args:\n        box1: The boundaries of the first box, written as\n            `[x_min, y_min, z_min, x_max, y_max, z_max]`.\n        box2: The boundaries of the second box, written as\n            `[x_min, y_min, z_min, x_max, y_max, z_max]`.\n    \"\"\"\n    overlap_x = _is_overlapping_1D_int([box1[0], box1[3]], [box2[0], box2[3]])\n    overlap_y = _is_overlapping_1D_int([box1[1], box1[4]], [box2[1], box2[4]])\n    overlap_z = _is_overlapping_1D_int([box1[2], box1[5]], [box2[2], box2[5]])\n    return overlap_x and overlap_y and overlap_z\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.apply_shift_in_one_direction","title":"apply_shift_in_one_direction(tmp_df_well, line_1, line_2, mu, tol=1e-10)","text":"

TBD

PARAMETER DESCRIPTION tmp_df_well

TBD

TYPE: DataFrame

line_1

TBD

TYPE: Sequence[float]

line_2

TBD

TYPE: Sequence[float]

mu

TBD

TYPE: str

tol

TBD

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def apply_shift_in_one_direction(\n    tmp_df_well: pd.DataFrame,\n    line_1: Sequence[float],\n    line_2: Sequence[float],\n    mu: str,\n    tol: float = 1e-10,\n):\n\"\"\"\n    TBD\n\n    Args:\n        tmp_df_well: TBD\n        line_1: TBD\n        line_2: TBD\n        mu: TBD\n        tol: TBD\n    \"\"\"\n    min_1, max_1 = line_1[:]\n    min_2, max_2 = line_2[:]\n    min_max = min(max_1, max_2)\n    max_min = max(min_1, min_2)\n    shift = min_max - max_min\n    logging.debug(f\"{mu}-shifting by {shift=}\")\n    ind = tmp_df_well.loc[:, f\"{mu}min\"] >= max_min - tol\n    if not (shift > 0.0 and ind.to_numpy().max() > 0):\n        raise ValueError(\n            \"Something wrong in apply_shift_in_one_direction\\n\"\n            f\"{mu=}\\n{shift=}\\n{ind.to_numpy()=}\"\n        )\n    tmp_df_well.loc[ind, f\"{mu}min\"] += shift\n    tmp_df_well.loc[ind, f\"{mu}max\"] += shift\n    tmp_df_well.loc[ind, f\"{mu}_micrometer\"] += shift\n    return tmp_df_well\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.check_well_for_FOV_overlap","title":"check_well_for_FOV_overlap(site_metadata, selected_well, plotting_function, tol=1e-10)","text":"

This function is currently only used in tests and examples.

The plotting_function parameter is exposed so that other tools (see examples in this repository) may use it to show the FOV ROIs.

PARAMETER DESCRIPTION site_metadata

TBD

TYPE: DataFrame

selected_well

TBD

TYPE: str

plotting_function

TBD

TYPE: Callable

tol

TBD

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def check_well_for_FOV_overlap(\n    site_metadata: pd.DataFrame,\n    selected_well: str,\n    plotting_function: Callable,\n    tol: float = 1e-10,\n):\n\"\"\"\n    This function is currently only used in tests and examples.\n\n    The `plotting_function` parameter is exposed so that other tools (see\n    examples in this repository) may use it to show the FOV ROIs.\n\n    Args:\n        site_metadata: TBD\n        selected_well: TBD\n        plotting_function: TBD\n        tol: TBD\n    \"\"\"\n\n    df = site_metadata.loc[selected_well].copy()\n    df[\"xmin\"] = df[\"x_micrometer\"]\n    df[\"ymin\"] = df[\"y_micrometer\"]\n    df[\"xmax\"] = df[\"x_micrometer\"] + df[\"pixel_size_x\"] * df[\"x_pixel\"]\n    df[\"ymax\"] = df[\"y_micrometer\"] + df[\"pixel_size_y\"] * df[\"y_pixel\"]\n\n    xmin = list(df.loc[:, \"xmin\"])\n    ymin = list(df.loc[:, \"ymin\"])\n    xmax = list(df.loc[:, \"xmax\"])\n    ymax = list(df.loc[:, \"ymax\"])\n    num_lines = len(xmin)\n\n    list_overlapping_FOVs = []\n    for line_1 in range(num_lines):\n        min_x_1, max_x_1 = [a[line_1] for a in [xmin, xmax]]\n        min_y_1, max_y_1 = [a[line_1] for a in [ymin, ymax]]\n        for line_2 in range(line_1):\n            min_x_2, max_x_2 = [a[line_2] for a in [xmin, xmax]]\n            min_y_2, max_y_2 = [a[line_2] for a in [ymin, ymax]]\n            overlap = is_overlapping_2D(\n                (min_x_1, min_y_1, max_x_1, max_y_1),\n                (min_x_2, min_y_2, max_x_2, max_y_2),\n                tol=tol,\n            )\n            if overlap:\n                list_overlapping_FOVs.append(line_1)\n                list_overlapping_FOVs.append(line_2)\n\n    # Call plotting_function\n    plotting_function(\n        xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well\n    )\n\n    if len(list_overlapping_FOVs) > 0:\n        # Increase values by one to switch from index to the label plotted\n        return {selected_well: [x + 1 for x in list_overlapping_FOVs]}\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.find_overlaps_in_ROI_indices","title":"find_overlaps_in_ROI_indices(list_indices)","text":"

Given a list of integer ROI indices, find whether there are overlaps.

PARAMETER DESCRIPTION list_indices

List of ROI indices, where each element in the list should look like [start_z, end_z, start_y, end_y, start_x, end_x].

TYPE: list[list[int]]

RETURNS DESCRIPTION Optional[tuple[int, int]]

None if no overlap was detected, otherwise a tuple with the positional indices of a pair of overlapping ROIs.

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def find_overlaps_in_ROI_indices(\n    list_indices: list[list[int]],\n) -> Optional[tuple[int, int]]:\n\"\"\"\n    Given a list of integer ROI indices, find whether there are overlaps.\n\n    Args:\n        list_indices: List of ROI indices, where each element in the list\n            should look like\n            `[start_z, end_z, start_y, end_y, start_x, end_x]`.\n\n    Returns:\n        `None` if no overlap was detected, otherwise a tuple with the\n            positional indices of a pair of overlapping ROIs.\n    \"\"\"\n\n    for ind_1, ROI_1 in enumerate(list_indices):\n        s_z, e_z, s_y, e_y, s_x, e_x = ROI_1[:]\n        box_1 = [s_x, s_y, s_z, e_x, e_y, e_z]\n        for ind_2 in range(ind_1):\n            ROI_2 = list_indices[ind_2]\n            s_z, e_z, s_y, e_y, s_x, e_x = ROI_2[:]\n            box_2 = [s_x, s_y, s_z, e_x, e_y, e_z]\n            if _is_overlapping_3D_int(box_1, box_2):\n                return (ind_1, ind_2)\n    return None\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.get_overlapping_pair","title":"get_overlapping_pair(tmp_df, tol=1e-10)","text":"

Finds the indices for the next overlapping FOVs pair.

Note: the returned indices are positional indices, starting from 0.

PARAMETER DESCRIPTION tmp_df

Dataframe with columns [\"xmin\", \"ymin\", \"xmax\", \"ymax\"].

TYPE: DataFrame

tol

Finite tolerance for floating-point comparisons.

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def get_overlapping_pair(\n    tmp_df: pd.DataFrame, tol: float = 1e-10\n) -> Union[tuple[int, int], bool]:\n\"\"\"\n    Finds the indices for the next overlapping FOVs pair.\n\n    Note: the returned indices are positional indices, starting from 0.\n\n    Args:\n        tmp_df: Dataframe with columns `[\"xmin\", \"ymin\", \"xmax\", \"ymax\"]`.\n        tol: Finite tolerance for floating-point comparisons.\n    \"\"\"\n\n    num_lines = len(tmp_df.index)\n    for pos_ind_1 in range(num_lines):\n        for pos_ind_2 in range(pos_ind_1):\n            if is_overlapping_2D(\n                tmp_df.iloc[pos_ind_1], tmp_df.iloc[pos_ind_2], tol=tol\n            ):\n                return (pos_ind_1, pos_ind_2)\n    return False\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.get_overlapping_pairs_3D","title":"get_overlapping_pairs_3D(tmp_df, full_res_pxl_sizes_zyx)","text":"

Finds the indices for the all overlapping FOVs pair, in three dimensions.

Note: the returned indices are positional indices, starting from 0.

PARAMETER DESCRIPTION tmp_df

Dataframe with columns {x,y,z}_micrometer and len_{x,y,z}_micrometer.

TYPE: DataFrame

full_res_pxl_sizes_zyx

TBD

TYPE: Sequence[float]

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def get_overlapping_pairs_3D(\n    tmp_df: pd.DataFrame,\n    full_res_pxl_sizes_zyx: Sequence[float],\n):\n\"\"\"\n    Finds the indices for the all overlapping FOVs pair, in three dimensions.\n\n    Note: the returned indices are positional indices, starting from 0.\n\n    Args:\n        tmp_df: Dataframe with columns `{x,y,z}_micrometer` and\n            `len_{x,y,z}_micrometer`.\n        full_res_pxl_sizes_zyx: TBD\n    \"\"\"\n\n    tol = 1e-10\n    if tol > min(full_res_pxl_sizes_zyx) / 1e3:\n        raise ValueError(f\"{tol=} but {full_res_pxl_sizes_zyx=}\")\n\n    new_tmp_df = tmp_df.copy()\n\n    new_tmp_df[\"x_micrometer_max\"] = (\n        new_tmp_df[\"x_micrometer\"] + new_tmp_df[\"len_x_micrometer\"]\n    )\n    new_tmp_df[\"y_micrometer_max\"] = (\n        new_tmp_df[\"y_micrometer\"] + new_tmp_df[\"len_y_micrometer\"]\n    )\n    new_tmp_df[\"z_micrometer_max\"] = (\n        new_tmp_df[\"z_micrometer\"] + new_tmp_df[\"len_z_micrometer\"]\n    )\n    # Remove columns which are not necessary for overlap checks\n    list_columns = [\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n        \"label\",\n    ]\n    new_tmp_df.drop(labels=list_columns, axis=1, inplace=True)\n\n    # Loop over all pairs, and construct list of overlapping ones\n    num_lines = len(new_tmp_df.index)\n    overlapping_list = []\n    for pos_ind_1 in range(num_lines):\n        for pos_ind_2 in range(pos_ind_1):\n            overlap = is_overlapping_3D(\n                new_tmp_df.iloc[pos_ind_1], new_tmp_df.iloc[pos_ind_2], tol=tol\n            )\n            if overlap:\n                overlapping_list.append((pos_ind_1, pos_ind_2))\n    return overlapping_list\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.is_overlapping_1D","title":"is_overlapping_1D(line1, line2, tol=1e-10)","text":"

Given two intervals, finds whether they overlap.

This is based on https://stackoverflow.com/a/70023212/19085332, and we additionally use a finite tolerance for floating-point comparisons.

PARAMETER DESCRIPTION line1

The boundaries of the first interval, written as [x_min, x_max].

TYPE: Sequence[float]

line2

The boundaries of the second interval, written as [x_min, x_max].

TYPE: Sequence[float]

tol

Finite tolerance for floating-point comparisons.

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def is_overlapping_1D(\n    line1: Sequence[float], line2: Sequence[float], tol: float = 1e-10\n) -> bool:\n\"\"\"\n    Given two intervals, finds whether they overlap.\n\n    This is based on https://stackoverflow.com/a/70023212/19085332, and we\n    additionally use a finite tolerance for floating-point comparisons.\n\n    Args:\n        line1: The boundaries of the first interval, written as\n            `[x_min, x_max]`.\n        line2: The boundaries of the second interval, written as\n            `[x_min, x_max]`.\n        tol: Finite tolerance for floating-point comparisons.\n    \"\"\"\n    return line1[0] <= line2[1] - tol and line2[0] <= line1[1] - tol\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.is_overlapping_2D","title":"is_overlapping_2D(box1, box2, tol=1e-10)","text":"

Given two rectangular boxes, finds whether they overlap.

This is based on https://stackoverflow.com/a/70023212/19085332, and we additionally use a finite tolerance for floating-point comparisons.

PARAMETER DESCRIPTION box1

The boundaries of the first rectangle, written as [x_min, y_min, x_max, y_max].

TYPE: Sequence[float]

box2

The boundaries of the second rectangle, written as [x_min, y_min, x_max, y_max].

TYPE: Sequence[float]

tol

Finite tolerance for floating-point comparisons.

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def is_overlapping_2D(\n    box1: Sequence[float], box2: Sequence[float], tol: float = 1e-10\n) -> bool:\n\"\"\"\n    Given two rectangular boxes, finds whether they overlap.\n\n    This is based on https://stackoverflow.com/a/70023212/19085332, and we\n    additionally use a finite tolerance for floating-point comparisons.\n\n    Args:\n        box1: The boundaries of the first rectangle, written as\n            `[x_min, y_min, x_max, y_max]`.\n        box2: The boundaries of the second rectangle, written as\n            `[x_min, y_min, x_max, y_max]`.\n        tol: Finite tolerance for floating-point comparisons.\n    \"\"\"\n    overlap_x = is_overlapping_1D(\n        [box1[0], box1[2]], [box2[0], box2[2]], tol=tol\n    )\n    overlap_y = is_overlapping_1D(\n        [box1[1], box1[3]], [box2[1], box2[3]], tol=tol\n    )\n    return overlap_x and overlap_y\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.is_overlapping_3D","title":"is_overlapping_3D(box1, box2, tol=1e-10)","text":"

Given two three-dimensional boxes, finds whether they overlap.

This is based on https://stackoverflow.com/a/70023212/19085332, and we additionally use a finite tolerance for floating-point comparisons.

PARAMETER DESCRIPTION box1

The boundaries of the first box, written as [x_min, y_min, z_min, x_max, y_max, z_max].

TYPE: Sequence[float]

box2

The boundaries of the second box, written as [x_min, y_min, z_min, x_max, y_max, z_max].

TYPE: Sequence[float]

tol

Finite tolerance for floating-point comparisons.

TYPE: float DEFAULT: 1e-10

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def is_overlapping_3D(\n    box1: Sequence[float], box2: Sequence[float], tol: float = 1e-10\n) -> bool:\n\"\"\"\n    Given two three-dimensional boxes, finds whether they overlap.\n\n    This is based on https://stackoverflow.com/a/70023212/19085332, and we\n    additionally use a finite tolerance for floating-point comparisons.\n\n    Args:\n        box1: The boundaries of the first box, written as\n            `[x_min, y_min, z_min, x_max, y_max, z_max]`.\n        box2: The boundaries of the second box, written as\n            `[x_min, y_min, z_min, x_max, y_max, z_max]`.\n        tol: Finite tolerance for floating-point comparisons.\n    \"\"\"\n\n    overlap_x = is_overlapping_1D(\n        [box1[0], box1[3]], [box2[0], box2[3]], tol=tol\n    )\n    overlap_y = is_overlapping_1D(\n        [box1[1], box1[4]], [box2[1], box2[4]], tol=tol\n    )\n    overlap_z = is_overlapping_1D(\n        [box1[2], box1[5]], [box2[2], box2[5]], tol=tol\n    )\n    return overlap_x and overlap_y and overlap_z\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.remove_FOV_overlaps","title":"remove_FOV_overlaps(df)","text":"

Given a metadata dataframe, shift its columns to remove FOV overlaps.

PARAMETER DESCRIPTION df

Metadata dataframe.

TYPE: DataFrame

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def remove_FOV_overlaps(df: pd.DataFrame):\n\"\"\"\n    Given a metadata dataframe, shift its columns to remove FOV overlaps.\n\n    Args:\n        df: Metadata dataframe.\n    \"\"\"\n\n    # Set tolerance (this should be much smaller than pixel size or expected\n    # round-offs), and maximum number of iterations in constraint solver\n    tol = 1e-10\n    max_iterations = 200\n\n    # Create a local copy of the dataframe\n    df = df.copy()\n\n    # Create temporary columns (to streamline overlap removals), which are\n    # then removed at the end of the remove_FOV_overlaps function\n    df[\"xmin\"] = df[\"x_micrometer\"]\n    df[\"ymin\"] = df[\"y_micrometer\"]\n    df[\"xmax\"] = df[\"x_micrometer\"] + df[\"pixel_size_x\"] * df[\"x_pixel\"]\n    df[\"ymax\"] = df[\"y_micrometer\"] + df[\"pixel_size_y\"] * df[\"y_pixel\"]\n    list_columns = [\"xmin\", \"ymin\", \"xmax\", \"ymax\"]\n\n    # Create columns with the original positions (not to be removed)\n    df[\"x_micrometer_original\"] = df[\"x_micrometer\"]\n    df[\"y_micrometer_original\"] = df[\"y_micrometer\"]\n\n    # Check that tolerance is much smaller than pixel sizes\n    min_pixel_size = df[[\"pixel_size_x\", \"pixel_size_y\"]].min().min()\n    if tol > min_pixel_size / 1e3:\n        raise ValueError(\n            f\"In remove_FOV_overlaps, {tol=} but {min_pixel_size=}\"\n        )\n\n    # Loop over wells\n    wells = sorted(list(set([ind[0] for ind in df.index])))\n    for well in wells:\n\n        logger.info(f\"removing FOV overlaps for {well=}\")\n        df_well = df.loc[well].copy()\n\n        # NOTE: these are positional indices (i.e. starting from 0)\n        pair_pos_indices = get_overlapping_pair(df_well[list_columns], tol=tol)\n\n        # Keep going until there are no overlaps, or until iteration reaches\n        # max_iterations\n        iteration = 0\n        while pair_pos_indices:\n            iteration += 1\n\n            # Identify overlapping FOVs\n            pos_ind_1, pos_ind_2 = pair_pos_indices\n            fov_id_1 = df_well.index[pos_ind_1]\n            fov_id_2 = df_well.index[pos_ind_2]\n            xmin_1, ymin_1, xmax_1, ymax_1 = df_well[list_columns].iloc[\n                pos_ind_1\n            ]\n            xmin_2, ymin_2, xmax_2, ymax_2 = df_well[list_columns].iloc[\n                pos_ind_2\n            ]\n            logger.debug(\n                f\"{well=}, {iteration=}, removing overlap between\"\n                f\" {fov_id_1=} and {fov_id_2=}\"\n            )\n\n            # Check what kind of overlap is there (X, Y, or XY)\n            is_x_equal = abs(xmin_1 - xmin_2) < tol and (xmax_1 - xmax_2) < tol\n            is_y_equal = abs(ymin_1 - ymin_2) < tol and (ymax_1 - ymax_2) < tol\n            is_x_overlap = is_overlapping_1D(\n                [xmin_1, xmax_1], [xmin_2, xmax_2], tol=tol\n            )\n            is_y_overlap = is_overlapping_1D(\n                [ymin_1, ymax_1], [ymin_2, ymax_2], tol=tol\n            )\n\n            if is_x_equal and is_y_overlap:\n                # Y overlap\n                df_well = apply_shift_in_one_direction(\n                    df_well,\n                    [ymin_1, ymax_1],\n                    [ymin_2, ymax_2],\n                    mu=\"y\",\n                    tol=tol,\n                )\n            elif is_y_equal and is_x_overlap:\n                # X overlap\n                df_well = apply_shift_in_one_direction(\n                    df_well,\n                    [xmin_1, xmax_1],\n                    [xmin_2, xmax_2],\n                    mu=\"x\",\n                    tol=tol,\n                )\n            elif not (is_x_equal or is_y_equal) and (\n                is_x_overlap and is_y_overlap\n            ):\n                # XY overlap\n                df_well = apply_shift_in_one_direction(\n                    df_well,\n                    [xmin_1, xmax_1],\n                    [xmin_2, xmax_2],\n                    mu=\"x\",\n                    tol=tol,\n                )\n                df_well = apply_shift_in_one_direction(\n                    df_well,\n                    [ymin_1, ymax_1],\n                    [ymin_2, ymax_2],\n                    mu=\"y\",\n                    tol=tol,\n                )\n            else:\n                raise ValueError(\n                    \"Trying to remove overlap which is not there.\"\n                )\n\n            # Look for next overlapping FOV pair\n            pair_pos_indices = get_overlapping_pair(\n                df_well[list_columns], tol=tol\n            )\n\n            # Enforce maximum number of iterations\n            if iteration >= max_iterations:\n                raise ValueError(f\"Reached {max_iterations=} for {well=}\")\n\n        # Note: using df.loc[well] = df_well leads to a NaN dataframe, see\n        # for instance https://stackoverflow.com/a/28432733/19085332\n        df.loc[well, :] = df_well.values\n\n    # Remove temporary columns that were added only as part of this function\n    df.drop(list_columns, axis=1, inplace=True)\n\n    return df\n
"},{"location":"reference/fractal_tasks_core/lib_ROI_overlaps/#fractal_tasks_core.lib_ROI_overlaps.run_overlap_check","title":"run_overlap_check(site_metadata, tol=1e-10, plotting_function=None)","text":"

Run an overlap check over all wells and optionally plots overlaps.

This function is currently only used in tests and examples.

The plotting_function parameter is exposed so that other tools (see examples in this repository) may use it to show the FOV ROIs. Its arguments are: [xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well].

PARAMETER DESCRIPTION site_metadata

TBD

TYPE: DataFrame

tol

TBD

TYPE: float DEFAULT: 1e-10

plotting_function

TBD

TYPE: Optional[Callable] DEFAULT: None

Source code in fractal_tasks_core/lib_ROI_overlaps.py 
def run_overlap_check(\n    site_metadata: pd.DataFrame,\n    tol: float = 1e-10,\n    plotting_function: Optional[Callable] = None,\n):\n\"\"\"\n    Run an overlap check over all wells and optionally plots overlaps.\n\n    This function is currently only used in tests and examples.\n\n    The `plotting_function` parameter is exposed so that other tools (see\n    examples in this repository) may use it to show the FOV ROIs. Its arguments\n    are: `[xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well]`.\n\n    Args:\n        site_metadata: TBD\n        tol: TBD\n        plotting_function: TBD\n    \"\"\"\n\n    if plotting_function is None:\n\n        def plotting_function(\n            xmin, xmax, ymin, ymax, list_overlapping_FOVs, selected_well\n        ):\n            pass\n\n    wells = site_metadata.index.unique(level=\"well_id\")\n    overlapping_FOVs = []\n    for selected_well in wells:\n        overlap_curr_well = check_well_for_FOV_overlap(\n            site_metadata,\n            selected_well=selected_well,\n            tol=tol,\n            plotting_function=plotting_function,\n        )\n        if overlap_curr_well:\n            print(selected_well)\n            overlapping_FOVs.append(overlap_curr_well)\n\n    return overlapping_FOVs\n
"},{"location":"reference/fractal_tasks_core/lib_channels/","title":"lib_channels","text":"

Helper functions to address channels via OME-NGFF/OMERO metadata.

"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.ChannelNotFoundError","title":"ChannelNotFoundError","text":"

Bases: ValueError

Custom error for when get_channel_from_list fails, that can be captured and handled upstream if needed.

Source code in fractal_tasks_core/lib_channels.py 
class ChannelNotFoundError(ValueError):\n\"\"\"\n    Custom error for when `get_channel_from_list` fails,\n    that can be captured and handled upstream if needed.\n    \"\"\"\n\n    pass\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.OmeroChannel","title":"OmeroChannel","text":"

Bases: BaseModel

Custom class for Omero channels, based on OME-NGFF v0.4.

ATTRIBUTE DESCRIPTION wavelength_id

Unique ID for the channel wavelength, e.g. A01_C01.

TYPE: str

index

Do not change. For internal use only.

TYPE: Optional[int]

label

Name of the channel.

TYPE: Optional[str]

window

Optional Window object to set default display settings for napari.

TYPE: Optional[Window]

color

Optional hex colormap to display the channel in napari (it must be of length 6, e.g. 00FFFF).

TYPE: Optional[str]

active

Should this channel be shown in the viewer?

TYPE: bool

coefficient

Do not change. Omero-channel attribute.

TYPE: int

inverted

Do not change. Omero-channel attribute.

TYPE: bool

Source code in fractal_tasks_core/lib_channels.py 
class OmeroChannel(BaseModel):\n\"\"\"\n    Custom class for Omero channels, based on OME-NGFF v0.4.\n\n    Attributes:\n        wavelength_id: Unique ID for the channel wavelength, e.g. `A01_C01`.\n        index: Do not change. For internal use only.\n        label: Name of the channel.\n        window: Optional `Window` object to set default display settings for\n            napari.\n        color: Optional hex colormap to display the channel in napari (it\n            must be of length 6, e.g. `00FFFF`).\n        active: Should this channel be shown in the viewer?\n        coefficient: Do not change. Omero-channel attribute.\n        inverted: Do not change. Omero-channel attribute.\n    \"\"\"\n\n    # Custom\n\n    wavelength_id: str\n    index: Optional[int]\n\n    # From OME-NGFF v0.4 transitional metadata\n\n    label: Optional[str]\n    window: Optional[Window]\n    color: Optional[str]\n    active: bool = True\n    coefficient: int = 1\n    inverted: bool = False\n\n    @validator(\"color\", always=True)\n    def valid_hex_color(cls, v, values):\n\"\"\"\n        Check that `color` is made of exactly six elements which are letters\n        (a-f or A-F) or digits (0-9).\n        \"\"\"\n        if v is None:\n            return v\n        if len(v) != 6:\n            raise ValueError(f'color must have length 6 (given: \"{v}\")')\n        allowed_characters = \"abcdefABCDEF0123456789\"\n        for character in v:\n            if character not in allowed_characters:\n                raise ValueError(\n                    \"color must only include characters from \"\n                    f'\"{allowed_characters}\" (given: \"{v}\")'\n                )\n        return v\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.OmeroChannel.valid_hex_color","title":"valid_hex_color(v, values)","text":"

Check that color is made of exactly six elements which are letters (a-f or A-F) or digits (0-9).

Source code in fractal_tasks_core/lib_channels.py 
@validator(\"color\", always=True)\ndef valid_hex_color(cls, v, values):\n\"\"\"\n    Check that `color` is made of exactly six elements which are letters\n    (a-f or A-F) or digits (0-9).\n    \"\"\"\n    if v is None:\n        return v\n    if len(v) != 6:\n        raise ValueError(f'color must have length 6 (given: \"{v}\")')\n    allowed_characters = \"abcdefABCDEF0123456789\"\n    for character in v:\n        if character not in allowed_characters:\n            raise ValueError(\n                \"color must only include characters from \"\n                f'\"{allowed_characters}\" (given: \"{v}\")'\n            )\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.Window","title":"Window","text":"

Bases: BaseModel

Custom class for Omero-channel window, based on OME-NGFF v0.4.

ATTRIBUTE DESCRIPTION min

Do not change. It will be set to 0 by default.

TYPE: Optional[int]

max

Do not change. It will be set according to bit-depth of the images by default (e.g. 65535 for 16 bit images).

TYPE: Optional[int]

start

Lower-bound rescaling value for visualization.

TYPE: int

end

Upper-bound rescaling value for visualization.

TYPE: int

Source code in fractal_tasks_core/lib_channels.py 
class Window(BaseModel):\n\"\"\"\n    Custom class for Omero-channel window, based on OME-NGFF v0.4.\n\n    Attributes:\n        min: Do not change. It will be set to `0` by default.\n        max:\n            Do not change. It will be set according to bit-depth of the images\n            by default (e.g. 65535 for 16 bit images).\n        start: Lower-bound rescaling value for visualization.\n        end: Upper-bound rescaling value for visualization.\n    \"\"\"\n\n    min: Optional[int]\n    max: Optional[int]\n    start: int\n    end: int\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels._get_new_unique_value","title":"_get_new_unique_value(value, existing_values)","text":"

Produce a string value that is not present in a given list

Append _1, _2, ... to a given string, if needed, until finding a value which is not already present in existing_values.

PARAMETER DESCRIPTION value

The first guess for the new value

TYPE: str

existing_values

The list of existing values

TYPE: list[str]

RETURNS DESCRIPTION str

A string value which is not present in existing_values

Source code in fractal_tasks_core/lib_channels.py 
def _get_new_unique_value(\n    value: str,\n    existing_values: list[str],\n) -> str:\n\"\"\"\n    Produce a string value that is not present in a given list\n\n    Append `_1`, `_2`, ... to a given string, if needed, until finding a value\n    which is not already present in `existing_values`.\n\n    Args:\n        value: The first guess for the new value\n        existing_values: The list of existing values\n\n    Returns:\n        A string value which is not present in `existing_values`\n    \"\"\"\n    counter = 1\n    new_value = value\n    while new_value in existing_values:\n        new_value = f\"{value}-{counter}\"\n        counter += 1\n    return new_value\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.check_unique_wavelength_ids","title":"check_unique_wavelength_ids(channels)","text":"

Check that the wavelength_id attributes of a channel list are unique.

PARAMETER DESCRIPTION channels

TBD

TYPE: list[OmeroChannel]

Source code in fractal_tasks_core/lib_channels.py 
def check_unique_wavelength_ids(channels: list[OmeroChannel]):\n\"\"\"\n    Check that the `wavelength_id` attributes of a channel list are unique.\n\n    Args:\n        channels: TBD\n    \"\"\"\n    wavelength_ids = [c.wavelength_id for c in channels]\n    if len(set(wavelength_ids)) < len(wavelength_ids):\n        raise ValueError(\n            f\"Non-unique wavelength_id's in {wavelength_ids}\\n\" f\"{channels=}\"\n        )\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.check_well_channel_labels","title":"check_well_channel_labels(*, well_zarr_path)","text":"

Check that the channel labels for a well are unique.

First identify the channel-labels list for each image in the well, then compare lists and verify their intersection is empty.

PARAMETER DESCRIPTION well_zarr_path

path to an OME-NGFF well zarr group.

TYPE: str

Source code in fractal_tasks_core/lib_channels.py 
def check_well_channel_labels(*, well_zarr_path: str) -> None:\n\"\"\"\n    Check that the channel labels for a well are unique.\n\n    First identify the channel-labels list for each image in the well, then\n    compare lists and verify their intersection is empty.\n\n    Args:\n        well_zarr_path: path to an OME-NGFF well zarr group.\n    \"\"\"\n\n    # Iterate over all images (multiplexing cycles, multi-FOVs, ...)\n    group = zarr.open_group(well_zarr_path, mode=\"r+\")\n    image_paths = [image[\"path\"] for image in group.attrs[\"well\"][\"images\"]]\n    list_of_channel_lists = []\n    for image_path in image_paths:\n        channels = get_omero_channel_list(\n            image_zarr_path=f\"{well_zarr_path}/{image_path}\"\n        )\n        list_of_channel_lists.append(channels[:])\n\n    # For each pair of channel-labels lists, verify they do not overlap\n    for ind_1, channels_1 in enumerate(list_of_channel_lists):\n        labels_1 = set([c.label for c in channels_1])\n        for ind_2 in range(ind_1):\n            channels_2 = list_of_channel_lists[ind_2]\n            labels_2 = set([c.label for c in channels_2])\n            intersection = labels_1 & labels_2\n            if intersection:\n                hint = (\n                    \"Are you parsing fields of view into separate OME-Zarr \"\n                    \"images? This could lead to non-unique channel labels, \"\n                    \"and then could be the reason of the error\"\n                )\n                raise ValueError(\n                    \"Non-unique channel labels\\n\"\n                    f\"{labels_1=}\\n{labels_2=}\\n{hint}\"\n                )\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.define_omero_channels","title":"define_omero_channels(*, channels, bit_depth, label_prefix=None)","text":"

Update a channel list to use it in the OMERO/channels metadata.

Given a list of channel dictionaries, update each one of them by: 1. Adding a label (if missing); 2. Adding a set of OMERO-specific attributes; 3. Discarding all other attributes.

The new_channels output can be used in the attrs[\"omero\"][\"channels\"] attribute of an image group.

PARAMETER DESCRIPTION channels

A list of channel dictionaries (each one must include the wavelength_id key).

TYPE: list[OmeroChannel]

bit_depth

bit depth.

TYPE: int

label_prefix

TBD

TYPE: Optional[str] DEFAULT: None

RETURNS DESCRIPTION list[dict[str, Union[str, int, bool, dict[str, int]]]]

new_channels, a new list of consistent channel dictionaries that can be written to OMERO metadata.

Source code in fractal_tasks_core/lib_channels.py 
def define_omero_channels(\n    *,\n    channels: list[OmeroChannel],\n    bit_depth: int,\n    label_prefix: Optional[str] = None,\n) -> list[dict[str, Union[str, int, bool, dict[str, int]]]]:\n\"\"\"\n    Update a channel list to use it in the OMERO/channels metadata.\n\n    Given a list of channel dictionaries, update each one of them by:\n        1. Adding a label (if missing);\n        2. Adding a set of OMERO-specific attributes;\n        3. Discarding all other attributes.\n\n    The `new_channels` output can be used in the `attrs[\"omero\"][\"channels\"]`\n    attribute of an image group.\n\n    Args:\n        channels: A list of channel dictionaries (each one must include the\n            `wavelength_id` key).\n        bit_depth: bit depth.\n        label_prefix: TBD\n\n    Returns:\n        `new_channels`, a new list of consistent channel dictionaries that\n            can be written to OMERO metadata.\n    \"\"\"\n\n    new_channels = [c.copy(deep=True) for c in channels]\n    default_colors = [\"00FFFF\", \"FF00FF\", \"FFFF00\"]\n\n    for channel in new_channels:\n        wavelength_id = channel.wavelength_id\n\n        # If channel.label is None, set it to a default value\n        if channel.label is None:\n            default_label = wavelength_id\n            if label_prefix:\n                default_label = f\"{label_prefix}_{default_label}\"\n            logging.warning(\n                f\"Missing label for {channel=}, using {default_label=}\"\n            )\n            channel.label = default_label\n\n        # If channel.color is None, set it to a default value (use the default\n        # ones for the first three channels, or gray otherwise)\n        if channel.color is None:\n            try:\n                channel.color = default_colors.pop()\n            except IndexError:\n                channel.color = \"808080\"\n\n        # Set channel.window attribute\n        if channel.window:\n            channel.window.min = 0\n            channel.window.max = 2**bit_depth - 1\n\n    # Check that channel labels are unique for this image\n    labels = [c.label for c in new_channels]\n    if len(set(labels)) < len(labels):\n        raise ValueError(f\"Non-unique labels in {new_channels=}\")\n\n    new_channels_dictionaries = [\n        c.dict(exclude={\"index\"}, exclude_unset=True) for c in new_channels\n    ]\n\n    return new_channels_dictionaries\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.get_channel_from_image_zarr","title":"get_channel_from_image_zarr(*, image_zarr_path, label=None, wavelength_id=None)","text":"

Extract a channel from OME-NGFF zarr attributes.

This is a helper function that combines get_omero_channel_list with get_channel_from_list.

PARAMETER DESCRIPTION image_zarr_path

Path to an OME-NGFF image zarr group.

TYPE: str

label

label attribute of the channel to be extracted.

TYPE: Optional[str] DEFAULT: None

wavelength_id

wavelength_id attribute of the channel to be extracted.

TYPE: Optional[str] DEFAULT: None

RETURNS DESCRIPTION OmeroChannel

A single channel dictionary.

Source code in fractal_tasks_core/lib_channels.py 
def get_channel_from_image_zarr(\n    *,\n    image_zarr_path: str,\n    label: Optional[str] = None,\n    wavelength_id: Optional[str] = None,\n) -> OmeroChannel:\n\"\"\"\n    Extract a channel from OME-NGFF zarr attributes.\n\n    This is a helper function that combines `get_omero_channel_list` with\n    `get_channel_from_list`.\n\n    Args:\n        image_zarr_path: Path to an OME-NGFF image zarr group.\n        label: `label` attribute of the channel to be extracted.\n        wavelength_id: `wavelength_id` attribute of the channel to be\n            extracted.\n\n    Returns:\n        A single channel dictionary.\n    \"\"\"\n    omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path)\n    channel = get_channel_from_list(\n        channels=omero_channels, label=label, wavelength_id=wavelength_id\n    )\n    return channel\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.get_channel_from_list","title":"get_channel_from_list(*, channels, label=None, wavelength_id=None)","text":"

Find matching channel in a list.

Find the channel that has the required values of label and/or wavelength_id, and identify its positional index (which also corresponds to its index in the zarr array).

PARAMETER DESCRIPTION channels

A list of channel dictionary, where each channel includes (at least) the label and wavelength_id keys.

TYPE: list[OmeroChannel]

label

The label to look for in the list of channels.

TYPE: Optional[str] DEFAULT: None

wavelength_id

The wavelength_id to look for in the list of channels.

TYPE: Optional[str] DEFAULT: None

RETURNS DESCRIPTION OmeroChannel

A single channel dictionary.

Source code in fractal_tasks_core/lib_channels.py 
def get_channel_from_list(\n    *,\n    channels: list[OmeroChannel],\n    label: Optional[str] = None,\n    wavelength_id: Optional[str] = None,\n) -> OmeroChannel:\n\"\"\"\n    Find matching channel in a list.\n\n    Find the channel that has the required values of `label` and/or\n    `wavelength_id`, and identify its positional index (which also\n    corresponds to its index in the zarr array).\n\n    Args:\n        channels: A list of channel dictionary, where each channel includes (at\n            least) the `label` and `wavelength_id` keys.\n        label: The label to look for in the list of channels.\n        wavelength_id: The wavelength_id to look for in the list of channels.\n\n    Returns:\n        A single channel dictionary.\n    \"\"\"\n\n    # Identify matching channels\n    if label:\n        if wavelength_id:\n            # Both label and wavelength_id are specified\n            matching_channels = [\n                c\n                for c in channels\n                if (c.label == label and c.wavelength_id == wavelength_id)\n            ]\n        else:\n            # Only label is specified\n            matching_channels = [c for c in channels if c.label == label]\n    else:\n        if wavelength_id:\n            # Only wavelength_id is specified\n            matching_channels = [\n                c for c in channels if c.wavelength_id == wavelength_id\n            ]\n        else:\n            # Neither label or wavelength_id are specified\n            raise ValueError(\n                \"get_channel requires at least one in {label,wavelength_id} \"\n                \"arguments\"\n            )\n\n    # Verify that there is one and only one matching channel\n    if len(matching_channels) == 0:\n        required_match = [f\"{label=}\", f\"{wavelength_id=}\"]\n        required_match_string = \" and \".join(\n            [x for x in required_match if \"None\" not in x]\n        )\n        raise ChannelNotFoundError(\n            f\"ChannelNotFoundError: No channel found in {channels}\"\n            f\" for {required_match_string}\"\n        )\n    if len(matching_channels) > 1:\n        raise ValueError(f\"Inconsistent set of channels: {channels}\")\n\n    channel = matching_channels[0]\n    channel.index = channels.index(channel)\n    return channel\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.get_omero_channel_list","title":"get_omero_channel_list(*, image_zarr_path)","text":"

Extract the list of channels from OME-NGFF zarr attributes.

PARAMETER DESCRIPTION image_zarr_path

Path to an OME-NGFF image zarr group.

TYPE: str

RETURNS DESCRIPTION list[OmeroChannel]

A list of channel dictionaries.

Source code in fractal_tasks_core/lib_channels.py 
def get_omero_channel_list(*, image_zarr_path: str) -> list[OmeroChannel]:\n\"\"\"\n    Extract the list of channels from OME-NGFF zarr attributes.\n\n    Args:\n        image_zarr_path: Path to an OME-NGFF image zarr group.\n\n    Returns:\n        A list of channel dictionaries.\n    \"\"\"\n    group = zarr.open_group(image_zarr_path, mode=\"r+\")\n    channels_dicts = group.attrs[\"omero\"][\"channels\"]\n    channels = [OmeroChannel(**c) for c in channels_dicts]\n    return channels\n
"},{"location":"reference/fractal_tasks_core/lib_channels/#fractal_tasks_core.lib_channels.update_omero_channels","title":"update_omero_channels(old_channels)","text":"

Make an existing list of Omero channels Fractal-compatible

The output channels all have keys label, wavelength_id and color; the wavelength_id values are unique across the channel list.

See https://ngff.openmicroscopy.org/0.4/index.html#omero-md for the definition of NGFF Omero metadata.

PARAMETER DESCRIPTION old_channels

Existing list of Omero-channel dictionaries

TYPE: list[dict[str, Any]]

RETURNS DESCRIPTION list[dict[str, Any]]

New list of Fractal-compatible Omero-channel dictionaries

Source code in fractal_tasks_core/lib_channels.py 
def update_omero_channels(\n    old_channels: list[dict[str, Any]]\n) -> list[dict[str, Any]]:\n\"\"\"\n    Make an existing list of Omero channels Fractal-compatible\n\n    The output channels all have keys `label`, `wavelength_id` and `color`;\n    the `wavelength_id` values are unique across the channel list.\n\n    See https://ngff.openmicroscopy.org/0.4/index.html#omero-md for the\n    definition of NGFF Omero metadata.\n\n    Args:\n        old_channels: Existing list of Omero-channel dictionaries\n\n    Returns:\n        New list of Fractal-compatible Omero-channel dictionaries\n    \"\"\"\n    new_channels = deepcopy(old_channels)\n    existing_wavelength_ids: list[str] = []\n    handled_channels = []\n\n    default_colors = [\"00FFFF\", \"FF00FF\", \"FFFF00\"]\n\n    def _get_next_color() -> str:\n        try:\n            return default_colors.pop(0)\n        except IndexError:\n            return \"808080\"\n\n    # Channels that contain the key \"wavelength_id\"\n    for ind, old_channel in enumerate(old_channels):\n        if \"wavelength_id\" in old_channel.keys():\n            handled_channels.append(ind)\n            existing_wavelength_ids.append(old_channel[\"wavelength_id\"])\n            new_channel = old_channel.copy()\n            try:\n                label = old_channel[\"label\"]\n            except KeyError:\n                label = str(ind + 1)\n            new_channel[\"label\"] = label\n            if \"color\" not in old_channel:\n                new_channel[\"color\"] = _get_next_color()\n            new_channels[ind] = new_channel\n\n    # Channels that contain the key \"label\" but do not contain the key\n    # \"wavelength_id\"\n    for ind, old_channel in enumerate(old_channels):\n        if ind in handled_channels:\n            continue\n        if \"label\" not in old_channel.keys():\n            continue\n        handled_channels.append(ind)\n        label = old_channel[\"label\"]\n        wavelength_id = _get_new_unique_value(\n            label,\n            existing_wavelength_ids,\n        )\n        existing_wavelength_ids.append(wavelength_id)\n        new_channel = old_channel.copy()\n        new_channel[\"wavelength_id\"] = wavelength_id\n        if \"color\" not in old_channel:\n            new_channel[\"color\"] = _get_next_color()\n        new_channels[ind] = new_channel\n\n    # Channels that do not contain the key \"label\" nor the key \"wavelength_id\"\n    # NOTE: these channels must be treated last, as they have lower priority\n    # w.r.t. existing \"wavelength_id\" or \"label\" values\n    for ind, old_channel in enumerate(old_channels):\n        if ind in handled_channels:\n            continue\n        label = str(ind + 1)\n        wavelength_id = _get_new_unique_value(\n            label,\n            existing_wavelength_ids,\n        )\n        existing_wavelength_ids.append(wavelength_id)\n        new_channel = old_channel.copy()\n        new_channel[\"label\"] = label\n        new_channel[\"wavelength_id\"] = wavelength_id\n        if \"color\" not in old_channel:\n            new_channel[\"color\"] = _get_next_color()\n        new_channels[ind] = new_channel\n\n    # Log old/new values of label, wavelength_id and color\n    for ind, old_channel in enumerate(old_channels):\n        label = old_channel.get(\"label\")\n        color = old_channel.get(\"color\")\n        wavelength_id = old_channel.get(\"wavelength_id\")\n        old_attributes = (\n            f\"Old attributes: {label=}, {wavelength_id=}, {color=}\"\n        )\n        label = new_channels[ind][\"label\"]\n        wavelength_id = new_channels[ind][\"wavelength_id\"]\n        color = new_channels[ind][\"color\"]\n        new_attributes = (\n            f\"New attributes: {label=}, {wavelength_id=}, {color=}\"\n        )\n        logging.info(\n            \"Omero channel update:\\n\"\n            f\"    {old_attributes}\\n\"\n            f\"    {new_attributes}\"\n        )\n\n    return new_channels\n
"},{"location":"reference/fractal_tasks_core/lib_glob/","title":"lib_glob","text":"

Auxiliary functions related to globbing (i.e. listing some items from a directory).

"},{"location":"reference/fractal_tasks_core/lib_glob/#fractal_tasks_core.lib_glob.glob_with_multiple_patterns","title":"glob_with_multiple_patterns(*, folder, patterns=None)","text":"

List all the items (files and folders) in a given folder that simultaneously match a series of glob patterns.

PARAMETER DESCRIPTION folder

Base folder where items will be searched.

TYPE: str

patterns

If specified, the list of patterns (defined as in https://docs.python.org/3/library/fnmatch.html) that item names will match with.

TYPE: Sequence[str] DEFAULT: None

Source code in fractal_tasks_core/lib_glob.py 
def glob_with_multiple_patterns(\n    *,\n    folder: str,\n    patterns: Sequence[str] = None,\n) -> set[str]:\n\"\"\"\n    List all the items (files and folders) in a given folder that\n    simultaneously match a series of glob patterns.\n\n    Args:\n        folder: Base folder where items will be searched.\n        patterns: If specified, the list of patterns (defined as in\n            https://docs.python.org/3/library/fnmatch.html) that item\n            names will match with.\n    \"\"\"\n\n    # Sanitize base-folder path\n    if folder.endswith(\"/\"):\n        actual_folder = folder[:-1]\n    else:\n        actual_folder = folder[:]\n\n    # If not pattern is specified, look for *all* items in the base folder\n    if not patterns:\n        patterns = [\"*\"]\n\n    # Combine multiple glob searches (via set intersection)\n    logging.info(f\"[glob_with_multiple_patterns] {patterns=}\")\n    items = None\n    for pattern in patterns:\n        new_matches = glob(f\"{actual_folder}/{pattern}\")\n        if items is None:\n            items = set(new_matches)\n        else:\n            items = items.intersection(new_matches)\n    items = items or set()\n    logging.info(f\"[glob_with_multiple_patterns] Found {len(items)} items\")\n\n    return items\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/","title":"lib_input_models","text":"

Pydantic models for some task parameters.

"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.Channel","title":"Channel","text":"

Bases: BaseModel

A channel which is specified by either wavelength_id or label.

ATTRIBUTE DESCRIPTION wavelength_id

Unique ID for the channel wavelength, e.g. A01_C01.

TYPE: Optional[str]

label

Name of the channel.

TYPE: Optional[str]

Source code in fractal_tasks_core/lib_input_models.py 
class Channel(BaseModel):\n\"\"\"\n    A channel which is specified by either `wavelength_id` or `label`.\n\n    Attributes:\n        wavelength_id: Unique ID for the channel wavelength, e.g. `A01_C01`.\n        label: Name of the channel.\n    \"\"\"\n\n    wavelength_id: Optional[str] = None\n    label: Optional[str] = None\n\n    @validator(\"label\", always=True)\n    def mutually_exclusive_channel_attributes(cls, v, values):\n\"\"\"\n        Check that either `label` or `wavelength_id` is set.\n        \"\"\"\n        wavelength_id = values.get(\"wavelength_id\")\n        label = v\n        if wavelength_id and v:\n            raise ValueError(\n                \"`wavelength_id` and `label` cannot be both set \"\n                f\"(given {wavelength_id=} and {label=}).\"\n            )\n        if wavelength_id is None and v is None:\n            raise ValueError(\n                \"`wavelength_id` and `label` cannot be both `None`\"\n            )\n        return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.Channel.mutually_exclusive_channel_attributes","title":"mutually_exclusive_channel_attributes(v, values)","text":"

Check that either label or wavelength_id is set.

Source code in fractal_tasks_core/lib_input_models.py 
@validator(\"label\", always=True)\ndef mutually_exclusive_channel_attributes(cls, v, values):\n\"\"\"\n    Check that either `label` or `wavelength_id` is set.\n    \"\"\"\n    wavelength_id = values.get(\"wavelength_id\")\n    label = v\n    if wavelength_id and v:\n        raise ValueError(\n            \"`wavelength_id` and `label` cannot be both set \"\n            f\"(given {wavelength_id=} and {label=}).\"\n        )\n    if wavelength_id is None and v is None:\n        raise ValueError(\n            \"`wavelength_id` and `label` cannot be both `None`\"\n        )\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsInput","title":"NapariWorkflowsInput","text":"

Bases: BaseModel

A value of the input_specs argument in napari_workflows_wrapper.

ATTRIBUTE DESCRIPTION type

Input type (either image or label).

TYPE: Literal['image', 'label']

label_name

Label name (for label inputs only).

TYPE: Optional[str]

channel

Channel object (for image inputs only).

TYPE: Optional[Channel]

Source code in fractal_tasks_core/lib_input_models.py 
class NapariWorkflowsInput(BaseModel):\n\"\"\"\n    A value of the `input_specs` argument in `napari_workflows_wrapper`.\n\n    Attributes:\n        type: Input type (either `image` or `label`).\n        label_name: Label name (for label inputs only).\n        channel: Channel object (for image inputs only).\n    \"\"\"\n\n    type: Literal[\"image\", \"label\"]\n    label_name: Optional[str]\n    channel: Optional[Channel]\n\n    @validator(\"label_name\", always=True)\n    def label_name_is_present(cls, v, values):\n\"\"\"\n        Check that label inputs have `label_name` set.\n        \"\"\"\n        _type = values.get(\"type\")\n        if _type == \"label\" and not v:\n            raise ValueError(\n                f\"Input item has type={_type} but label_name={v}.\"\n            )\n        return v\n\n    @validator(\"channel\", always=True)\n    def channel_is_present(cls, v, values):\n\"\"\"\n        Check that image inputs have `channel` set.\n        \"\"\"\n        _type = values.get(\"type\")\n        if _type == \"image\" and not v:\n            raise ValueError(f\"Input item has type={_type} but channel={v}.\")\n        return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsInput.channel_is_present","title":"channel_is_present(v, values)","text":"

Check that image inputs have channel set.

Source code in fractal_tasks_core/lib_input_models.py 
@validator(\"channel\", always=True)\ndef channel_is_present(cls, v, values):\n\"\"\"\n    Check that image inputs have `channel` set.\n    \"\"\"\n    _type = values.get(\"type\")\n    if _type == \"image\" and not v:\n        raise ValueError(f\"Input item has type={_type} but channel={v}.\")\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsInput.label_name_is_present","title":"label_name_is_present(v, values)","text":"

Check that label inputs have label_name set.

Source code in fractal_tasks_core/lib_input_models.py 
@validator(\"label_name\", always=True)\ndef label_name_is_present(cls, v, values):\n\"\"\"\n    Check that label inputs have `label_name` set.\n    \"\"\"\n    _type = values.get(\"type\")\n    if _type == \"label\" and not v:\n        raise ValueError(\n            f\"Input item has type={_type} but label_name={v}.\"\n        )\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsOutput","title":"NapariWorkflowsOutput","text":"

Bases: BaseModel

A value of the output_specs argument in napari_workflows_wrapper.

ATTRIBUTE DESCRIPTION type

Output type (either label or dataframe).

TYPE: Literal['label', 'dataframe']

label_name

Label name (for label outputs only).

TYPE: Optional[str]

table_name

Table name (for dataframe outputs only).

TYPE: Optional[str]

Source code in fractal_tasks_core/lib_input_models.py 
class NapariWorkflowsOutput(BaseModel):\n\"\"\"\n    A value of the `output_specs` argument in `napari_workflows_wrapper`.\n\n    Attributes:\n        type: Output type (either `label` or `dataframe`).\n        label_name: Label name (for label outputs only).\n        table_name: Table name (for dataframe outputs only).\n    \"\"\"\n\n    type: Literal[\"label\", \"dataframe\"]\n    label_name: Optional[str] = None\n    table_name: Optional[str] = None\n\n    @validator(\"label_name\", always=True)\n    def label_name_only_for_label_type(cls, v, values):\n\"\"\"\n        Check that label_name is set only for label outputs.\n        \"\"\"\n        _type = values.get(\"type\")\n        if (_type == \"label\" and (not v)) or (_type != \"label\" and v):\n            raise ValueError(\n                f\"Output item has type={_type} but label_name={v}.\"\n            )\n        return v\n\n    @validator(\"table_name\", always=True)\n    def table_name_only_for_dataframe_type(cls, v, values):\n\"\"\"\n        Check that table_name is set only for dataframe outputs.\n        \"\"\"\n        _type = values.get(\"type\")\n        if (_type == \"dataframe\" and (not v)) or (_type != \"dataframe\" and v):\n            raise ValueError(\n                f\"Output item has type={_type} but table_name={v}.\"\n            )\n        return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsOutput.label_name_only_for_label_type","title":"label_name_only_for_label_type(v, values)","text":"

Check that label_name is set only for label outputs.

Source code in fractal_tasks_core/lib_input_models.py 
@validator(\"label_name\", always=True)\ndef label_name_only_for_label_type(cls, v, values):\n\"\"\"\n    Check that label_name is set only for label outputs.\n    \"\"\"\n    _type = values.get(\"type\")\n    if (_type == \"label\" and (not v)) or (_type != \"label\" and v):\n        raise ValueError(\n            f\"Output item has type={_type} but label_name={v}.\"\n        )\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_input_models/#fractal_tasks_core.lib_input_models.NapariWorkflowsOutput.table_name_only_for_dataframe_type","title":"table_name_only_for_dataframe_type(v, values)","text":"

Check that table_name is set only for dataframe outputs.

Source code in fractal_tasks_core/lib_input_models.py 
@validator(\"table_name\", always=True)\ndef table_name_only_for_dataframe_type(cls, v, values):\n\"\"\"\n    Check that table_name is set only for dataframe outputs.\n    \"\"\"\n    _type = values.get(\"type\")\n    if (_type == \"dataframe\" and (not v)) or (_type != \"dataframe\" and v):\n        raise ValueError(\n            f\"Output item has type={_type} but table_name={v}.\"\n        )\n    return v\n
"},{"location":"reference/fractal_tasks_core/lib_masked_loading/","title":"lib_masked_loading","text":"

Functions to use masked loading of ROIs before/after processing.

"},{"location":"reference/fractal_tasks_core/lib_masked_loading/#fractal_tasks_core.lib_masked_loading._postprocess_output","title":"_postprocess_output(*, modified_array, original_array, background)","text":"

Postprocess cellpose output, mainly to restore its original background.

NOTE: The pre/post-processing functions and the masked_loading_wrapper are currently meant to work as part of the cellpose_segmentation task, with the plan of then making them more flexible; see https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.

PARAMETER DESCRIPTION modified_array

The 3D (ZYX) array with the correct object data and wrong background data.

TYPE: ndarray

original_array

The 3D (ZYX) array with the wrong object data and correct background data.

TYPE: ndarray

background

The 3D (ZYX) boolean array that defines the background.

TYPE: ndarray

RETURNS DESCRIPTION ndarray

The postprocessed array.

Source code in fractal_tasks_core/lib_masked_loading.py 
def _postprocess_output(\n    *,\n    modified_array: np.ndarray,\n    original_array: np.ndarray,\n    background: np.ndarray,\n) -> np.ndarray:\n\"\"\"\n    Postprocess cellpose output, mainly to restore its original background.\n\n    **NOTE**: The pre/post-processing functions and the\n    masked_loading_wrapper are currently meant to work as part of the\n    cellpose_segmentation task, with the plan of then making them more\n    flexible; see\n    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.\n\n    Args:\n        modified_array: The 3D (ZYX) array with the correct object data and\n            wrong background data.\n        original_array: The 3D (ZYX) array with the wrong object data and\n            correct background data.\n        background: The 3D (ZYX) boolean array that defines the background.\n\n    Returns:\n        The postprocessed array.\n    \"\"\"\n    # Restore background\n    modified_array[background] = original_array[background]\n    return modified_array\n
"},{"location":"reference/fractal_tasks_core/lib_masked_loading/#fractal_tasks_core.lib_masked_loading._preprocess_input","title":"_preprocess_input(image_array, *, region, current_label_path, ROI_table_path, ROI_positional_index)","text":"

Preprocess a four-dimensional cellpose input.

This involves :

  • Loading the masking label array for the appropriate ROI;
  • Extracting the appropriate label value from the ROI_table.obs dataframe;
  • Constructing the background mask, where the masking label matches with a specific label value;
  • Setting the background of image_array to 0;
  • Loading the array which will be needed in postprocessing to restore background.

NOTE 1: This function relies on a change to OME-NGFF table specs (https://github.com/ome/ngff/pull/64) which is still in-progress.

NOTE 2: The pre/post-processing functions and the masked_loading_wrapper are currently meant to work as part of the cellpose_segmentation task, with the plan of then making them more flexible; see https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.

Naming of variables refers to a two-steps labeling, as in \"first identify organoids, then look for nuclei inside each organoid\") :

  • \"masking\" refers to the labels that are used to identify the object vs background (e.g. the organoid labels); these labels already exist.
  • \"current\" refers to the labels that are currently being computed in the cellpose_segmentation task, e.g. the nuclear labels.
PARAMETER DESCRIPTION image_array

The 4D CZYX array with image data for a specific ROI.

TYPE: ndarray

region

The ZYX indices of the ROI, in a form like (slice(0, 1), slice(1000, 2000), slice(1000, 2000)).

TYPE: tuple[slice, ...]

current_label_path

Path to the image used as current label, in a form like /somewhere/plate.zarr/A/01/0/labels/nuclei_in_organoids/0.

TYPE: str

ROI_table_path

Path of the AnnData table for the masking-label ROIs; this is used (together with ROI_positional_index) to extract label_value.

TYPE: str

ROI_positional_index

Index of the current ROI, which is used to extract label_value from ROI_table_obs.

TYPE: int

Returns: A tuple with three arrays: the preprocessed image array, the background mask, the current label.

Source code in fractal_tasks_core/lib_masked_loading.py 
def _preprocess_input(\n    image_array: np.ndarray,\n    *,\n    region: tuple[slice, ...],\n    current_label_path: str,\n    ROI_table_path: str,\n    ROI_positional_index: int,\n) -> tuple[np.ndarray, np.ndarray, np.ndarray]:\n\"\"\"\n    Preprocess a four-dimensional cellpose input.\n\n    This involves :\n\n    - Loading the masking label array for the appropriate ROI;\n    - Extracting the appropriate label value from the `ROI_table.obs`\n      dataframe;\n    - Constructing the background mask, where the masking label matches with a\n      specific label value;\n    - Setting the background of `image_array` to `0`;\n    - Loading the array which will be needed in postprocessing to restore\n      background.\n\n    **NOTE 1**: This function relies on a change to OME-NGFF table specs\n    (https://github.com/ome/ngff/pull/64) which is still in-progress.\n\n    **NOTE 2**: The pre/post-processing functions and the\n    masked_loading_wrapper are currently meant to work as part of the\n    cellpose_segmentation task, with the plan of then making them more\n    flexible; see\n    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/340.\n\n    Naming of variables refers to a two-steps labeling, as in \"first identify\n    organoids, then look for nuclei inside each organoid\") :\n\n    - `\"masking\"` refers to the labels that are used to identify the object\n      vs background (e.g. the organoid labels); these labels already exist.\n    - `\"current\"` refers to the labels that are currently being computed in\n      the `cellpose_segmentation` task, e.g. the nuclear labels.\n\n    Args:\n        image_array: The 4D CZYX array with image data for a specific ROI.\n        region: The ZYX indices of the ROI, in a form like\n            `(slice(0, 1), slice(1000, 2000), slice(1000, 2000))`.\n        current_label_path: Path to the image used as current label, in a form\n            like `/somewhere/plate.zarr/A/01/0/labels/nuclei_in_organoids/0`.\n        ROI_table_path: Path of the AnnData table for the masking-label ROIs;\n            this is used (together with `ROI_positional_index`) to extract\n            `label_value`.\n        ROI_positional_index: Index of the current ROI, which is used to\n            extract `label_value` from `ROI_table_obs`.\n    Returns:\n        A tuple with three arrays: the preprocessed image array, the background\n            mask, the current label.\n    \"\"\"\n\n    logger.info(f\"[_preprocess_input] {image_array.shape=}\")\n    logger.info(f\"[_preprocess_input] {region=}\")\n\n    # Check that image data are 4D (CZYX) - FIXME issue 340\n    if not image_array.ndim == 4:\n        raise ValueError(\n            \"_preprocess_input requires a 4D \"\n            f\"image_array argument, but {image_array.shape=}\"\n        )\n\n    # Load the ROI table and its metadata attributes\n    ROI_table = ad.read_zarr(ROI_table_path)\n    attrs = zarr.group(ROI_table_path).attrs\n    logger.info(f\"[_preprocess_input] {ROI_table_path=}\")\n    logger.info(f\"[_preprocess_input] {attrs.asdict()=}\")\n    if not attrs[\"type\"] == \"ngff:region_table\":\n        raise ValueError(\"Wrong attributes for {ROI_table_path}:\\n{attrs}\")\n    label_relative_path = attrs[\"region\"][\"path\"]\n    column_name = attrs[\"instance_key\"]\n\n    # Check that ROI_table.obs has the right column and extract label_value\n    if column_name not in ROI_table.obs.columns:\n        raise ValueError(\n            'In _preprocess_input, \"{column_name}\" '\n            f\" missing in {ROI_table.obs.columns=}\"\n        )\n    label_value = int(ROI_table.obs[column_name][ROI_positional_index])\n\n    # Load masking-label array (lazily)\n    masking_label_path = str(\n        Path(ROI_table_path).parent / label_relative_path / \"0\"\n    )\n    logger.info(f\"{masking_label_path=}\")\n    masking_label_array = da.from_zarr(masking_label_path)\n    logger.info(\n        f\"[_preprocess_input] {masking_label_path=}, \"\n        f\"{masking_label_array.shape=}\"\n    )\n\n    # Load current-label array (lazily)\n    current_label_array = da.from_zarr(current_label_path)\n    logger.info(\n        f\"[_preprocess_input] {current_label_path=}, \"\n        f\"{current_label_array.shape=}\"\n    )\n\n    # Load ROI data for current label array\n    current_label_region = current_label_array[region].compute()\n\n    # Load ROI data for masking label array, with or without upscaling\n    if masking_label_array.shape != current_label_array.shape:\n        logger.info(\"Upscaling of masking label is needed\")\n        lowres_region = convert_region_to_low_res(\n            highres_region=region,\n            highres_shape=current_label_array.shape,\n            lowres_shape=masking_label_array.shape,\n        )\n        masking_label_region = masking_label_array[lowres_region].compute()\n        masking_label_region = upscale_array(\n            array=masking_label_region,\n            target_shape=current_label_region.shape,\n        )\n    else:\n        masking_label_region = masking_label_array[region].compute()\n\n    # Check that all shapes match\n    shapes = (\n        masking_label_region.shape,\n        current_label_region.shape,\n        image_array.shape[1:],\n    )\n    if len(set(shapes)) > 1:\n        raise ValueError(\n            \"Shape mismatch:\\n\"\n            f\"{current_label_region.shape=}\\n\"\n            f\"{masking_label_region.shape=}\\n\"\n            f\"{image_array.shape=}\"\n        )\n\n    # Compute background mask\n    background_3D = masking_label_region != label_value\n    if (masking_label_region == label_value).sum() == 0:\n        raise ValueError(\n            f\"Label {label_value} is not present in the extracted ROI\"\n        )\n\n    # Set image background to zero\n    background_4D = np.expand_dims(background_3D, axis=0)\n    image_array[background_4D] = 0\n\n    return (image_array, background_3D, current_label_region)\n
"},{"location":"reference/fractal_tasks_core/lib_masked_loading/#fractal_tasks_core.lib_masked_loading.masked_loading_wrapper","title":"masked_loading_wrapper(*, function, image_array, kwargs=None, use_masks, preprocessing_kwargs=None)","text":"

Wrap a function with some pre/post-processing functions

PARAMETER DESCRIPTION function

The callable function to be wrapped.

TYPE: Callable

image_array

The image array to be preprocessed and then used as positional argument for function.

TYPE: ndarray

kwargs

Keyword arguments for function.

TYPE: Optional[dict] DEFAULT: None

use_masks

If False, the wrapper only calls function(*args, **kwargs).

TYPE: bool

preprocessing_kwargs

Keyword arguments for the preprocessing function (see call signature of _preprocess_input()).

TYPE: Optional[dict] DEFAULT: None

Source code in fractal_tasks_core/lib_masked_loading.py 
def masked_loading_wrapper(\n    *,\n    function: Callable,\n    image_array: np.ndarray,\n    kwargs: Optional[dict] = None,\n    use_masks: bool,\n    preprocessing_kwargs: Optional[dict] = None,\n):\n\"\"\"\n    Wrap a function with some pre/post-processing functions\n\n    Args:\n        function: The callable function to be wrapped.\n        image_array: The image array to be preprocessed and then used as\n            positional argument for `function`.\n        kwargs: Keyword arguments for `function`.\n        use_masks: If `False`, the wrapper only calls\n            `function(*args, **kwargs)`.\n        preprocessing_kwargs: Keyword arguments for the preprocessing function\n            (see call signature of `_preprocess_input()`).\n    \"\"\"\n    # Optional preprocessing\n    if use_masks:\n        preprocessing_kwargs = preprocessing_kwargs or {}\n        (\n            image_array,\n            background_3D,\n            current_label_region,\n        ) = _preprocess_input(image_array, **preprocessing_kwargs)\n    # Run function\n    kwargs = kwargs or {}\n    new_label_img = function(image_array, **kwargs)\n    # Optional postprocessing\n    if use_masks:\n        new_label_img = _postprocess_output(\n            modified_array=new_label_img,\n            original_array=current_label_region,\n            background=background_3D,\n        )\n    return new_label_img\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/","title":"lib_metadata_parsing","text":"

Functions to create a metadata dataframe from Yokogawa files.

"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.calculate_steps","title":"calculate_steps(site_series)","text":"

TBD

PARAMETER DESCRIPTION site_series

TBD

TYPE: Series

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def calculate_steps(site_series: pd.Series):\n\"\"\"\n    TBD\n\n    Args:\n        site_series: TBD\n    \"\"\"\n\n    # site_series is the z_micrometer series for a given site of a given\n    # channel. This function calculates the step size in Z\n\n    # First diff is always NaN because there is nothing to compare it to\n    steps = site_series.diff().dropna().astype(float)\n    if not np.allclose(steps.iloc[0], np.array(steps)):\n        raise NotImplementedError(\n            \"When parsing the Yokogawa mlf file, some sites \"\n            \"had varying step size in Z. \"\n            \"That is not supported for the OME-Zarr parsing\"\n        )\n    return steps.mean()\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.check_group_consistency","title":"check_group_consistency(grouped_df, message='')","text":"

TBD

PARAMETER DESCRIPTION grouped_df

TBD

TYPE: DataFrame

message

TBD

TYPE: str DEFAULT: ''

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def check_group_consistency(grouped_df: pd.DataFrame, message: str = \"\"):\n\"\"\"\n    TBD\n\n    Args:\n        grouped_df: TBD\n        message: TBD\n    \"\"\"\n\n    # Check consistency in grouped df for multi-index, multi-column dataframes\n    # raises an exception if there is variability\n    diff_df = grouped_df.max() - grouped_df.min()\n    if not np.isclose(np.sum(np.sum(diff_df)), 0.0):\n        raise ValueError(\n            \"During metadata parsing, a consistency check failed: \\n\"\n            f\"{message}\\n\"\n            f\"Difference dataframe: \\n{diff_df}\"\n        )\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.get_earliest_time_per_site","title":"get_earliest_time_per_site(mlf_frame)","text":"

TBD

PARAMETER DESCRIPTION mlf_frame

TBD

TYPE: DataFrame

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def get_earliest_time_per_site(mlf_frame: pd.DataFrame) -> pd.DataFrame:\n\"\"\"\n    TBD\n\n    Args:\n        mlf_frame: TBD\n    \"\"\"\n\n    # Get the time information per site\n    # Because a site will contain time information for each plane\n    # of each channel, we just return the earliest time infromation\n    # per site.\n    return pd.to_datetime(\n        mlf_frame.groupby([\"well_id\", \"FieldIndex\"]).min()[\"Time\"], utc=True\n    )\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.get_z_steps","title":"get_z_steps(mlf_frame)","text":"

TBD

PARAMETER DESCRIPTION mlf_frame

TBD

TYPE: DataFrame

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def get_z_steps(mlf_frame: pd.DataFrame) -> pd.DataFrame:\n\"\"\"\n    TBD\n\n    Args:\n        mlf_frame: TBD\n    \"\"\"\n\n    # Process mlf_frame to extract Z information (pixel size & steps).\n    # Run checks on consistencies & return site-based z step dataframe\n    # Group by well, field & channel\n    grouped_sites_z = (\n        mlf_frame.loc[\n            :,\n            [\"well_id\", \"FieldIndex\", \"ActionIndex\", \"Ch\", \"Z\"],\n        ]\n        .set_index([\"well_id\", \"FieldIndex\", \"ActionIndex\", \"Ch\"])\n        .groupby(level=[0, 1, 2, 3])\n    )\n\n    # If there is only 1 Z step, set the Z spacing to the count of planes => 1\n    if grouped_sites_z.count()[\"Z\"].max() == 1:\n        z_data = grouped_sites_z.count().groupby([\"well_id\", \"FieldIndex\"])\n    else:\n        # Group the whole site (combine channels), because Z steps need to be\n        # consistent between channels for OME-Zarr.\n        z_data = grouped_sites_z.apply(calculate_steps).groupby(\n            [\"well_id\", \"FieldIndex\"]\n        )\n\n    check_group_consistency(\n        z_data, message=\"Comparing Z steps between channels\"\n    )\n\n    # Ensure that channels have the same number of z planes and\n    # reduce it to one value.\n    # Only check if there is more than one channel available\n    if any(\n        grouped_sites_z.count().groupby([\"well_id\", \"FieldIndex\"]).count() > 1\n    ):\n        check_group_consistency(\n            grouped_sites_z.count().groupby([\"well_id\", \"FieldIndex\"]),\n            message=\"Checking number of Z steps between channels\",\n        )\n\n    z_steps = (\n        grouped_sites_z.count()\n        .groupby([\"well_id\", \"FieldIndex\"])\n        .mean()\n        .astype(int)\n    )\n\n    # Combine the two dataframes\n    z_frame = pd.concat([z_data.mean(), z_steps], axis=1)\n    z_frame.columns = [\"pixel_size_z\", \"z_pixel\"]\n    return z_frame\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.parse_yokogawa_metadata","title":"parse_yokogawa_metadata(mrf_path, mlf_path, *, filename_patterns=None)","text":"

Parse Yokogawa CV7000 metadata files and prepare site-level metadata.

PARAMETER DESCRIPTION mrf_path

Full path to MeasurementDetail.mrf metadata file.

TYPE: Union[str, Path]

mlf_path

Full path to MeasurementData.mlf metadata file.

TYPE: Union[str, Path]

filename_patterns

List of patterns to filter the image filenames in the mlf metadata table. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html

TYPE: Optional[list[str]] DEFAULT: None

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def parse_yokogawa_metadata(\n    mrf_path: Union[str, Path],\n    mlf_path: Union[str, Path],\n    *,\n    filename_patterns: Optional[list[str]] = None,\n) -> tuple[pd.DataFrame, dict[str, int]]:\n\"\"\"\n    Parse Yokogawa CV7000 metadata files and prepare site-level metadata.\n\n    Args:\n        mrf_path: Full path to MeasurementDetail.mrf metadata file.\n        mlf_path: Full path to MeasurementData.mlf metadata file.\n        filename_patterns:\n            List of patterns to filter the image filenames in the mlf metadata\n            table. Patterns must be defined as in\n            https://docs.python.org/3/library/fnmatch.html\n    \"\"\"\n\n    # Convert paths to strings\n    mrf_str = Path(mrf_path).as_posix()\n    mlf_str = Path(mlf_path).as_posix()\n\n    mrf_frame, mlf_frame, error_count = read_metadata_files(\n        mrf_str, mlf_str, filename_patterns\n    )\n\n    # Aggregate information from the mlf file\n    per_site_parameters = [\"X\", \"Y\"]\n\n    grouping_params = [\"well_id\", \"FieldIndex\"]\n    grouped_sites = mlf_frame.loc[\n        :, grouping_params + per_site_parameters\n    ].groupby(by=grouping_params)\n\n    check_group_consistency(grouped_sites, message=\"X & Y stage positions\")\n    site_metadata = grouped_sites.mean()\n    site_metadata.columns = [\"x_micrometer\", \"y_micrometer\"]\n    site_metadata[\"z_micrometer\"] = 0\n\n    site_metadata = pd.concat(\n        [\n            site_metadata,\n            get_z_steps(mlf_frame),\n            get_earliest_time_per_site(mlf_frame),\n        ],\n        axis=1,\n    )\n\n    # Aggregate information from the mrf file\n    mrf_columns = [\n        \"horiz_pixel_dim\",\n        \"vert_pixel_dim\",\n        \"horiz_pixels\",\n        \"vert_pixels\",\n        \"bit_depth\",\n    ]\n    check_group_consistency(\n        mrf_frame.loc[:, mrf_columns], message=\"Image dimensions\"\n    )\n    site_metadata[\"pixel_size_x\"] = mrf_frame.loc[:, \"horiz_pixel_dim\"].max()\n    site_metadata[\"pixel_size_y\"] = mrf_frame.loc[:, \"vert_pixel_dim\"].max()\n    site_metadata[\"x_pixel\"] = int(mrf_frame.loc[:, \"horiz_pixels\"].max())\n    site_metadata[\"y_pixel\"] = int(mrf_frame.loc[:, \"vert_pixels\"].max())\n    site_metadata[\"bit_depth\"] = int(mrf_frame.loc[:, \"bit_depth\"].max())\n\n    if error_count > 0:\n        logger.info(\n            f\"There were {error_count} ERR entries in the metadatafile. \"\n            f\"Still succesfully parsed {len(site_metadata)} sites. \"\n        )\n\n    # Compute expected number of image files for each well\n    list_of_wells = set(site_metadata.index.get_level_values(\"well_id\"))\n    number_of_files = {}\n    for this_well_id in list_of_wells:\n        num_images = (mlf_frame.well_id == this_well_id).sum()\n        logger.info(\n            f\"Expected number of images for well {this_well_id}: {num_images}\"\n        )\n        number_of_files[this_well_id] = num_images\n    # Check that the sum of per-well file numbers correspond to the total\n    # file number\n    if not sum(number_of_files.values()) == len(mlf_frame):\n        raise ValueError(\n            \"Error while counting the number of image files per well.\\n\"\n            f\"{len(mlf_frame)=}\\n\"\n            f\"{number_of_files=}\"\n        )\n\n    return site_metadata, number_of_files\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.read_metadata_files","title":"read_metadata_files(mrf_path, mlf_path, filename_patterns=None)","text":"

TBD

PARAMETER DESCRIPTION mrf_path

Full path to MeasurementDetail.mrf metadata file.

TYPE: str

mlf_path

Full path to MeasurementData.mlf metadata file.

TYPE: str

filename_patterns

List of patterns to filter the image filenames in the mlf metadata table. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html.

TYPE: Optional[list[str]] DEFAULT: None

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def read_metadata_files(\n    mrf_path: str,\n    mlf_path: str,\n    filename_patterns: Optional[list[str]] = None,\n) -> tuple[pd.DataFrame, pd.DataFrame, int]:\n\"\"\"\n    TBD\n\n    Args:\n        mrf_path: Full path to MeasurementDetail.mrf metadata file.\n        mlf_path: Full path to MeasurementData.mlf metadata file.\n        filename_patterns: List of patterns to filter the image filenames in\n            the mlf metadata table. Patterns must be defined as in\n            https://docs.python.org/3/library/fnmatch.html.\n    \"\"\"\n\n    # parsing of mrf & mlf files are based on the\n    # yokogawa_image_collection_task v0.5 in drogon, written by Dario Vischi.\n    # https://github.com/fmi-basel/job-system-workflows/blob/00bbf34448972d27f258a2c28245dd96180e8229/src/gliberal_workflows/tasks/yokogawa_image_collection_task/versions/version_0_5.py  # noqa\n    # Now modified for Fractal use\n\n    mrf_frame = read_mrf_file(mrf_path)\n    # TODO: filter_position & filter_wheel_position are parsed, but not\n    # processed further. Figure out how to save them as relevant metadata for\n    # use e.g. during illumination correction\n\n    mlf_frame, error_count = read_mlf_file(mlf_path, filename_patterns)\n    # TODO: Time points are parsed as part of the mlf_frame, but currently not\n    # processed further. Once we tackle time-resolved data, parse from here.\n\n    return mrf_frame, mlf_frame, error_count\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.read_mlf_file","title":"read_mlf_file(mlf_path, filename_patterns=None)","text":"

TBD

PARAMETER DESCRIPTION mlf_path

Full path to MeasurementData.mlf metadata file.

TYPE: str

filename_patterns

List of patterns to filter the image filenames in the mlf metadata table. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html.

TYPE: Optional[list[str]] DEFAULT: None

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def read_mlf_file(\n    mlf_path: str,\n    filename_patterns: Optional[list[str]] = None,\n) -> tuple[pd.DataFrame, int]:\n\"\"\"\n    TBD\n\n    Args:\n        mlf_path: Full path to MeasurementData.mlf metadata file.\n        filename_patterns: List of patterns to filter the image filenames in\n            the mlf metadata table. Patterns must be defined as in\n            https://docs.python.org/3/library/fnmatch.html.\n    \"\"\"\n\n    # Load the whole MeasurementData.mlf file\n    mlf_frame_raw = pd.read_xml(mlf_path)\n\n    # Remove all rows that do not match the given patterns\n    logger.info(\n        f\"Read {mlf_path}, and apply following patterns to \"\n        f\"image filenames: {filename_patterns}\"\n    )\n    if filename_patterns:\n        filenames = mlf_frame_raw.MeasurementRecord\n        keep_row = None\n        for pattern in filename_patterns:\n            actual_pattern = fnmatch.translate(pattern)\n            new_matches = filenames.str.fullmatch(actual_pattern)\n            if new_matches.sum() == 0:\n                raise ValueError(\n                    f\"In {mlf_path} there is no image filename \"\n                    f'matching \"{actual_pattern}\".'\n                )\n            if keep_row is None:\n                keep_row = new_matches.copy()\n            else:\n                keep_row = keep_row & new_matches\n        if keep_row.sum() == 0:\n            raise ValueError(\n                f\"In {mlf_path} there is no image filename \"\n                f\"matching {filename_patterns}.\"\n            )\n        mlf_frame_matching = mlf_frame_raw[keep_row.values].copy()\n    else:\n        mlf_frame_matching = mlf_frame_raw.copy()\n\n    # Create a well ID column\n    row_str = [chr(x) for x in (mlf_frame_matching[\"Row\"] + 64)]\n    mlf_frame_matching[\"well_id\"] = [\n        f\"{a}{b:02}\" for a, b in zip(row_str, mlf_frame_matching[\"Column\"])\n    ]\n\n    # Flip Y axis to align to image coordinate system\n    mlf_frame_matching[\"Y\"] = -mlf_frame_matching[\"Y\"]\n\n    # Compute number or errors\n    error_count = (mlf_frame_matching[\"Type\"] == \"ERR\").sum()\n\n    # We're only interested in the image metadata\n    mlf_frame = mlf_frame_matching[mlf_frame_matching[\"Type\"] == \"IMG\"]\n\n    return mlf_frame, error_count\n
"},{"location":"reference/fractal_tasks_core/lib_metadata_parsing/#fractal_tasks_core.lib_metadata_parsing.read_mrf_file","title":"read_mrf_file(mrf_path)","text":"

TBD

PARAMETER DESCRIPTION mrf_path

Full path to MeasurementDetail.mrf metadata file.

TYPE: str

Source code in fractal_tasks_core/lib_metadata_parsing.py 
def read_mrf_file(mrf_path: str):\n\"\"\"\n    TBD\n\n    Args:\n        mrf_path: Full path to MeasurementDetail.mrf metadata file.\n    \"\"\"\n\n    # Prepare mrf dataframe\n    mrf_columns = [\n        \"channel_id\",\n        \"horiz_pixel_dim\",\n        \"vert_pixel_dim\",\n        \"camera_no\",\n        \"bit_depth\",\n        \"horiz_pixels\",\n        \"vert_pixels\",\n        \"filter_wheel_position\",\n        \"filter_position\",\n        \"shading_corr_src\",\n    ]\n    mrf_frame = pd.DataFrame(columns=mrf_columns)\n\n    mrf_xml = ElementTree.parse(mrf_path).getroot()\n    # Read mrf file\n    ns = {\"bts\": \"http://www.yokogawa.co.jp/BTS/BTSSchema/1.0\"}\n    for channel in mrf_xml.findall(\"bts:MeasurementChannel\", namespaces=ns):\n        mrf_frame.loc[channel.get(\"{%s}Ch\" % ns[\"bts\"])] = [\n            channel.get(\"{%s}Ch\" % ns[\"bts\"]),\n            float(channel.get(\"{%s}HorizontalPixelDimension\" % ns[\"bts\"])),\n            float(channel.get(\"{%s}VerticalPixelDimension\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}CameraNumber\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}InputBitDepth\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}HorizontalPixels\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}VerticalPixels\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}FilterWheelPosition\" % ns[\"bts\"])),\n            int(channel.get(\"{%s}FilterPosition\" % ns[\"bts\"])),\n            channel.get(\"{%s}ShadingCorrectionSource\" % ns[\"bts\"]),\n        ]\n\n    return mrf_frame\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/","title":"lib_ngff","text":"

Pydantic models related to OME-NGFF 0.4 specs.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Axis","title":"Axis","text":"

Bases: BaseModel

Model for an element of Multiscale.axes.

See https://ngff.openmicroscopy.org/0.4/#axes-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Axis(BaseModel):\n\"\"\"\n    Model for an element of `Multiscale.axes`.\n\n    See https://ngff.openmicroscopy.org/0.4/#axes-md.\n    \"\"\"\n\n    name: str\n    type: Optional[str] = None\n    unit: Optional[str] = None\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Channel","title":"Channel","text":"

Bases: BaseModel

Model for an element of Omero.channels.

See https://ngff.openmicroscopy.org/0.4/#omero-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Channel(BaseModel):\n\"\"\"\n    Model for an element of `Omero.channels`.\n\n    See https://ngff.openmicroscopy.org/0.4/#omero-md.\n    \"\"\"\n\n    window: Optional[Window] = None\n    label: Optional[str] = None\n    family: Optional[str] = None\n    color: str\n    active: Optional[bool] = None\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Dataset","title":"Dataset","text":"

Bases: BaseModel

Model for an element of Multiscale.datasets.

See https://ngff.openmicroscopy.org/0.4/#multiscale-md

Source code in fractal_tasks_core/lib_ngff.py 
class Dataset(BaseModel):\n\"\"\"\n    Model for an element of `Multiscale.datasets`.\n\n    See https://ngff.openmicroscopy.org/0.4/#multiscale-md\n    \"\"\"\n\n    path: str\n    coordinateTransformations: list[\n        Union[\n            ScaleCoordinateTransformation, TranslationCoordinateTransformation\n        ]\n    ] = Field(..., min_items=1)\n\n    @property\n    def scale_transformation(self) -> ScaleCoordinateTransformation:\n\"\"\"\n        Extract the unique scale transformation, or fail otherwise.\n        \"\"\"\n        _transformations = [\n            t for t in self.coordinateTransformations if t.type == \"scale\"\n        ]\n        if len(_transformations) == 0:\n            raise ValueError(\n                \"Missing scale transformation in dataset.\\n\"\n                \"Current coordinateTransformations:\\n\"\n                f\"{self.coordinateTransformations}\"\n            )\n        elif len(_transformations) > 1:\n            raise ValueError(\n                \"More than one scale transformation in dataset.\\n\"\n                \"Current coordinateTransformations:\\n\"\n                f\"{self.coordinateTransformations}\"\n            )\n        else:\n            return _transformations[0]\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Dataset.scale_transformation","title":"scale_transformation: ScaleCoordinateTransformation property","text":"

Extract the unique scale transformation, or fail otherwise.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.ImageInWell","title":"ImageInWell","text":"

Bases: BaseModel

Model for an element of Well.images.

Note 1: The NGFF image is defined in a different model (NgffImageMeta), while the Image model only refere to an item of Well.images.

Note 2: We deviate from NGFF specs, since we allow path to be an arbitrary string. TODO: include a check like constr(regex=r'^[A-Za-z0-9]+$'), through a Pydantic validator.

See https://ngff.openmicroscopy.org/0.4/#well-md.

Source code in fractal_tasks_core/lib_ngff.py 
class ImageInWell(BaseModel):\n\"\"\"\n    Model for an element of `Well.images`.\n\n    **Note 1:** The NGFF image is defined in a different model\n    (`NgffImageMeta`), while the `Image` model only refere to an item of\n    `Well.images`.\n\n    **Note 2:** We deviate from NGFF specs, since we allow `path` to be an\n    arbitrary string.\n    TODO: include a check like `constr(regex=r'^[A-Za-z0-9]+$')`, through a\n    Pydantic validator.\n\n    See https://ngff.openmicroscopy.org/0.4/#well-md.\n    \"\"\"\n\n    acquisition: Optional[int] = Field(\n        None, description=\"A unique identifier within the context of the plate\"\n    )\n    path: str = Field(\n        ..., description=\"The path for this field of view subgroup\"\n    )\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Multiscale","title":"Multiscale","text":"

Bases: BaseModel

Model for an element of NgffImageMeta.multiscales.

See https://ngff.openmicroscopy.org/0.4/#multiscale-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Multiscale(BaseModel):\n\"\"\"\n    Model for an element of `NgffImageMeta.multiscales`.\n\n    See https://ngff.openmicroscopy.org/0.4/#multiscale-md.\n    \"\"\"\n\n    name: Optional[str] = None\n    datasets: list[Dataset] = Field(..., min_items=1)\n    version: Optional[str] = None\n    axes: list[Axis] = Field(..., max_items=5, min_items=2, unique_items=True)\n    coordinateTransformations: Optional[\n        list[\n            Union[\n                ScaleCoordinateTransformation,\n                TranslationCoordinateTransformation,\n            ]\n        ]\n    ] = None\n\n    @validator(\"coordinateTransformations\", always=True)\n    def _no_global_coordinateTransformations(cls, v):\n\"\"\"\n        Fail if Multiscale has a (global) coordinateTransformations attribute.\n        \"\"\"\n        if v is not None:\n            raise NotImplementedError(\n                \"Global coordinateTransformations at the multiscales \"\n                \"level are not currently supported in the fractal-tasks-core \"\n                \"model for the NGFF multiscale.\"\n            )\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Multiscale._no_global_coordinateTransformations","title":"_no_global_coordinateTransformations(v)","text":"

Fail if Multiscale has a (global) coordinateTransformations attribute.

Source code in fractal_tasks_core/lib_ngff.py 
@validator(\"coordinateTransformations\", always=True)\ndef _no_global_coordinateTransformations(cls, v):\n\"\"\"\n    Fail if Multiscale has a (global) coordinateTransformations attribute.\n    \"\"\"\n    if v is not None:\n        raise NotImplementedError(\n            \"Global coordinateTransformations at the multiscales \"\n            \"level are not currently supported in the fractal-tasks-core \"\n            \"model for the NGFF multiscale.\"\n        )\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta","title":"NgffImageMeta","text":"

Bases: BaseModel

Model for the metadata of a NGFF image.

See https://ngff.openmicroscopy.org/0.4/#image-layout.

Source code in fractal_tasks_core/lib_ngff.py 
class NgffImageMeta(BaseModel):\n\"\"\"\n    Model for the metadata of a NGFF image.\n\n    See https://ngff.openmicroscopy.org/0.4/#image-layout.\n    \"\"\"\n\n    multiscales: list[Multiscale] = Field(\n        ...,\n        description=\"The multiscale datasets for this image\",\n        min_items=1,\n        unique_items=True,\n    )\n    omero: Optional[Omero] = None\n\n    @property\n    def multiscale(self) -> Multiscale:\n\"\"\"\n        The single element of `self.multiscales`.\n\n        Raises:\n            NotImplementedError:\n                If there are no multiscales or more than one.\n        \"\"\"\n        if len(self.multiscales) > 1:\n            raise NotImplementedError(\n                \"Only images with one multiscale are supported \"\n                f\"(given: {len(self.multiscales)}\"\n            )\n        return self.multiscales[0]\n\n    @property\n    def datasets(self) -> list[Dataset]:\n\"\"\"\n        The `datasets` attribute of `self.multiscale`.\n        \"\"\"\n        return self.multiscale.datasets\n\n    @property\n    def num_levels(self) -> int:\n        return len(self.datasets)\n\n    @property\n    def axes_names(self) -> list[str]:\n\"\"\"\n        List of axes names.\n        \"\"\"\n        return [ax.name for ax in self.multiscale.axes]\n\n    @property\n    def pixel_sizes_zyx(self) -> list[list[float]]:\n\"\"\"\n        Pixel sizes extracted from scale transformations of datasets.\n\n        Raises:\n            ValueError:\n                If pixel sizes are below a given threshold (1e-9).\n        \"\"\"\n        x_index = self.axes_names.index(\"x\")\n        y_index = self.axes_names.index(\"y\")\n        try:\n            z_index = self.axes_names.index(\"z\")\n        except ValueError:\n            z_index = None\n            logging.warning(\n                f\"Z axis is not present (axes: {self.axes_names}), and Z pixel\"\n                \" size is set to 1. This may work, by accident, but it is \"\n                \"not fully supported.\"\n            )\n        _pixel_sizes_zyx = []\n        for level in range(self.num_levels):\n            scale = self.datasets[level].scale_transformation.scale\n            pixel_size_x = scale[x_index]\n            pixel_size_y = scale[y_index]\n            if z_index is not None:\n                pixel_size_z = scale[z_index]\n            else:\n                pixel_size_z = 1.0\n            _pixel_sizes_zyx.append([pixel_size_z, pixel_size_y, pixel_size_x])\n            if min(_pixel_sizes_zyx[-1]) < 1e-9:\n                raise ValueError(\n                    f\"Pixel sizes at level {level} are too small: \"\n                    f\"{_pixel_sizes_zyx[-1]}\"\n                )\n\n        return _pixel_sizes_zyx\n\n    def get_pixel_sizes_zyx(self, *, level: int = 0) -> list[float]:\n        return self.pixel_sizes_zyx[level]\n\n    @property\n    def coarsening_xy(self) -> int:\n\"\"\"\n        Linear coarsening factor in the YX plane.\n\n        We only support coarsening factors that are homogeneous (both in the\n        X/Y directions and across pyramid levels).\n\n        Raises:\n            NotImplementedError:\n                If coarsening ratios are not homogeneous.\n        \"\"\"\n        current_ratio = None\n        for ind in range(1, self.num_levels):\n            ratio_x = round(\n                self.pixel_sizes_zyx[ind][2] / self.pixel_sizes_zyx[ind - 1][2]\n            )\n            ratio_y = round(\n                self.pixel_sizes_zyx[ind][1] / self.pixel_sizes_zyx[ind - 1][1]\n            )\n            if ratio_x != ratio_y:\n                raise NotImplementedError(\n                    \"Inhomogeneous coarsening in X/Y directions \"\n                    \"is not supported.\\n\"\n                    f\"ZYX pixel sizes:\\n {self.pixel_sizes_zyx}\"\n                )\n            if current_ratio is None:\n                current_ratio = ratio_x\n            else:\n                if current_ratio != ratio_x:\n                    raise NotImplementedError(\n                        \"Inhomogeneous coarsening across levels \"\n                        \"is not supported.\\n\"\n                        f\"ZYX pixel sizes:\\n {self.pixel_sizes_zyx}\"\n                    )\n\n        return current_ratio\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta.axes_names","title":"axes_names: list[str] property","text":"

List of axes names.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta.coarsening_xy","title":"coarsening_xy: int property","text":"

Linear coarsening factor in the YX plane.

We only support coarsening factors that are homogeneous (both in the X/Y directions and across pyramid levels).

RAISES DESCRIPTION NotImplementedError

If coarsening ratios are not homogeneous.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta.datasets","title":"datasets: list[Dataset] property","text":"

The datasets attribute of self.multiscale.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta.multiscale","title":"multiscale: Multiscale property","text":"

The single element of self.multiscales.

RAISES DESCRIPTION NotImplementedError

If there are no multiscales or more than one.

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffImageMeta.pixel_sizes_zyx","title":"pixel_sizes_zyx: list[list[float]] property","text":"

Pixel sizes extracted from scale transformations of datasets.

RAISES DESCRIPTION ValueError

If pixel sizes are below a given threshold (1e-9).

"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffWellMeta","title":"NgffWellMeta","text":"

Bases: BaseModel

Model for the metadata of a NGFF well.

See https://ngff.openmicroscopy.org/0.4/#well-md.

Source code in fractal_tasks_core/lib_ngff.py 
class NgffWellMeta(BaseModel):\n\"\"\"\n    Model for the metadata of a NGFF well.\n\n    See https://ngff.openmicroscopy.org/0.4/#well-md.\n    \"\"\"\n\n    well: Optional[Well] = None\n\n    def get_acquisition_paths(self) -> dict[int, str]:\n\"\"\"\n        Create mapping from acquisition indices to corresponding paths.\n\n        Runs on the well zarr attributes and loads the relative paths in the\n        well.\n\n        Returns:\n            Dictionary with `(acquisition index: image path)` key/value pairs.\n\n        Raises:\n            ValueError:\n                If an element of `self.well.images` has no `acquisition`\n                    attribute.\n            NotImplementedError:\n                If acquisitions are not unique.\n        \"\"\"\n        acquisition_dict = {}\n        for image in self.well.images:\n            if image.acquisition is None:\n                raise ValueError(\n                    \"Cannot get acquisition paths for Zarr files without \"\n                    \"'acquisition' metadata at the well level\"\n                )\n            if image.acquisition in acquisition_dict:\n                raise NotImplementedError(\n                    \"The `NgffWellMeta.get_acquisition_paths` method (in \"\n                    \"fractal-tasks-core) does not support wells with \"\n                    \"multiple images of the same acquisition.\"\n                )\n            acquisition_dict[image.acquisition] = image.path\n        return acquisition_dict\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.NgffWellMeta.get_acquisition_paths","title":"get_acquisition_paths()","text":"

Create mapping from acquisition indices to corresponding paths.

Runs on the well zarr attributes and loads the relative paths in the well.

RETURNS DESCRIPTION dict[int, str]

Dictionary with (acquisition index: image path) key/value pairs.

RAISES DESCRIPTION ValueError

If an element of self.well.images has no acquisition attribute.

NotImplementedError

If acquisitions are not unique.

Source code in fractal_tasks_core/lib_ngff.py 
def get_acquisition_paths(self) -> dict[int, str]:\n\"\"\"\n    Create mapping from acquisition indices to corresponding paths.\n\n    Runs on the well zarr attributes and loads the relative paths in the\n    well.\n\n    Returns:\n        Dictionary with `(acquisition index: image path)` key/value pairs.\n\n    Raises:\n        ValueError:\n            If an element of `self.well.images` has no `acquisition`\n                attribute.\n        NotImplementedError:\n            If acquisitions are not unique.\n    \"\"\"\n    acquisition_dict = {}\n    for image in self.well.images:\n        if image.acquisition is None:\n            raise ValueError(\n                \"Cannot get acquisition paths for Zarr files without \"\n                \"'acquisition' metadata at the well level\"\n            )\n        if image.acquisition in acquisition_dict:\n            raise NotImplementedError(\n                \"The `NgffWellMeta.get_acquisition_paths` method (in \"\n                \"fractal-tasks-core) does not support wells with \"\n                \"multiple images of the same acquisition.\"\n            )\n        acquisition_dict[image.acquisition] = image.path\n    return acquisition_dict\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Omero","title":"Omero","text":"

Bases: BaseModel

Model for NgffImageMeta.omero.

See https://ngff.openmicroscopy.org/0.4/#omero-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Omero(BaseModel):\n\"\"\"\n    Model for `NgffImageMeta.omero`.\n\n    See https://ngff.openmicroscopy.org/0.4/#omero-md.\n    \"\"\"\n\n    channels: list[Channel]\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.ScaleCoordinateTransformation","title":"ScaleCoordinateTransformation","text":"

Bases: BaseModel

Model for a scale transformation.

This corresponds to scale-type elements of Dataset.coordinateTransformations or Multiscale.coordinateTransformations. See https://ngff.openmicroscopy.org/0.4/#trafo-md

Source code in fractal_tasks_core/lib_ngff.py 
class ScaleCoordinateTransformation(BaseModel):\n\"\"\"\n    Model for a scale transformation.\n\n    This corresponds to scale-type elements of\n    `Dataset.coordinateTransformations` or\n    `Multiscale.coordinateTransformations`.\n    See https://ngff.openmicroscopy.org/0.4/#trafo-md\n    \"\"\"\n\n    type: Literal[\"scale\"]\n    scale: list[float] = Field(..., min_items=2)\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.TranslationCoordinateTransformation","title":"TranslationCoordinateTransformation","text":"

Bases: BaseModel

Model for a translation transformation.

This corresponds to translation-type elements of Dataset.coordinateTransformations or Multiscale.coordinateTransformations. See https://ngff.openmicroscopy.org/0.4/#trafo-md

Source code in fractal_tasks_core/lib_ngff.py 
class TranslationCoordinateTransformation(BaseModel):\n\"\"\"\n    Model for a translation transformation.\n\n    This corresponds to translation-type elements of\n    `Dataset.coordinateTransformations` or\n    `Multiscale.coordinateTransformations`.\n    See https://ngff.openmicroscopy.org/0.4/#trafo-md\n    \"\"\"\n\n    type: Literal[\"translation\"]\n    translation: list[float] = Field(..., min_items=2)\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Well","title":"Well","text":"

Bases: BaseModel

Model for NgffWellMeta.well.

See https://ngff.openmicroscopy.org/0.4/#well-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Well(BaseModel):\n\"\"\"\n    Model for `NgffWellMeta.well`.\n\n    See https://ngff.openmicroscopy.org/0.4/#well-md.\n    \"\"\"\n\n    images: list[ImageInWell] = Field(\n        ...,\n        description=\"The images included in this well\",\n        min_items=1,\n        unique_items=True,\n    )\n    version: Optional[str] = Field(\n        None, description=\"The version of the specification\"\n    )\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.Window","title":"Window","text":"

Bases: BaseModel

Model for Channel.window.

Note that we deviate by NGFF specs by making start and end optional. See https://ngff.openmicroscopy.org/0.4/#omero-md.

Source code in fractal_tasks_core/lib_ngff.py 
class Window(BaseModel):\n\"\"\"\n    Model for `Channel.window`.\n\n    Note that we deviate by NGFF specs by making `start` and `end` optional.\n    See https://ngff.openmicroscopy.org/0.4/#omero-md.\n    \"\"\"\n\n    max: float\n    min: float\n    start: Optional[float] = None\n    end: Optional[float] = None\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.detect_ome_ngff_type","title":"detect_ome_ngff_type(group)","text":"

Given a Zarr group, find whether it is an OME-NGFF plate, well or image.

PARAMETER DESCRIPTION group

Zarr group

TYPE: Group

RETURNS DESCRIPTION str

The detected OME-NGFF type (plate, well or image).

Source code in fractal_tasks_core/lib_ngff.py 
def detect_ome_ngff_type(group: zarr.Group) -> str:\n\"\"\"\n    Given a Zarr group, find whether it is an OME-NGFF plate, well or image.\n\n    Args:\n        group: Zarr group\n\n    Returns:\n        The detected OME-NGFF type (`plate`, `well` or `image`).\n    \"\"\"\n    attrs = group.attrs.asdict()\n    if \"plate\" in attrs.keys():\n        ngff_type = \"plate\"\n    elif \"well\" in attrs.keys():\n        ngff_type = \"well\"\n    elif \"multiscales\" in attrs.keys():\n        ngff_type = \"image\"\n    else:\n        error_msg = (\n            \"Zarr group at cannot be identified as one \"\n            \"of OME-NGFF plate/well/image groups.\"\n        )\n        logger.error(error_msg)\n        raise ValueError(error_msg)\n    logger.info(f\"Zarr group identified as OME-NGFF {ngff_type}.\")\n    return ngff_type\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.load_NgffImageMeta","title":"load_NgffImageMeta(zarr_path)","text":"

Load the attributes of a zarr group and cast them to NgffImageMeta.

PARAMETER DESCRIPTION zarr_path

Path to the zarr group.

TYPE: str

RETURNS DESCRIPTION NgffImageMeta

A new NgffImageMeta object.

Source code in fractal_tasks_core/lib_ngff.py 
def load_NgffImageMeta(zarr_path: str) -> NgffImageMeta:\n\"\"\"\n    Load the attributes of a zarr group and cast them to `NgffImageMeta`.\n\n    Args:\n        zarr_path: Path to the zarr group.\n\n    Returns:\n        A new `NgffImageMeta` object.\n    \"\"\"\n    zarr_group = zarr.open_group(zarr_path, mode=\"r\")\n    zarr_attrs = zarr_group.attrs.asdict()\n    try:\n        return NgffImageMeta(**zarr_attrs)\n    except Exception as e:\n        logging.error(\n            f\"Contents of {zarr_path} cannot be cast to NgffImageMeta.\\n\"\n            f\"Original error:\\n{str(e)}\"\n        )\n        raise e\n
"},{"location":"reference/fractal_tasks_core/lib_ngff/#fractal_tasks_core.lib_ngff.load_NgffWellMeta","title":"load_NgffWellMeta(zarr_path)","text":"

Load the attributes of a zarr group and cast them to NgffWellMeta.

PARAMETER DESCRIPTION zarr_path

Path to the zarr group.

TYPE: str

RETURNS DESCRIPTION NgffWellMeta

A new NgffWellMeta object.

Source code in fractal_tasks_core/lib_ngff.py 
def load_NgffWellMeta(zarr_path: str) -> NgffWellMeta:\n\"\"\"\n    Load the attributes of a zarr group and cast them to `NgffWellMeta`.\n\n    Args:\n        zarr_path: Path to the zarr group.\n\n    Returns:\n        A new `NgffWellMeta` object.\n    \"\"\"\n    zarr_group = zarr.open_group(zarr_path, mode=\"r\")\n    zarr_attrs = zarr_group.attrs.asdict()\n    try:\n        return NgffWellMeta(**zarr_attrs)\n    except Exception as e:\n        logging.error(\n            f\"Contents of {zarr_path} cannot be cast to NgffWellMeta.\\n\"\n            f\"Original error:\\n{str(e)}\"\n        )\n        raise e\n
"},{"location":"reference/fractal_tasks_core/lib_parse_filename_metadata/","title":"lib_parse_filename_metadata","text":"

Extract metadata from image filename.

"},{"location":"reference/fractal_tasks_core/lib_parse_filename_metadata/#fractal_tasks_core.lib_parse_filename_metadata._get_plate_name","title":"_get_plate_name(plate_prefix)","text":"

Two kinds of plate_prefix values are handled in a special way:

  1. Filenames from FMI, with successful barcode reading: 210305NAR005AAN_210416_164828 with plate name 210305NAR005AAN;
  2. Filenames from FMI, with failed barcode reading: yymmdd_hhmmss_210416_164828 with plate name RS{yymmddhhmmss}.

For all non-matching filenames, plate name is plate_prefix.

PARAMETER DESCRIPTION plate_prefix

TBD

TYPE: str

Source code in fractal_tasks_core/lib_parse_filename_metadata.py 
def _get_plate_name(plate_prefix: str) -> str:\n\"\"\"\n    Two kinds of plate_prefix values are handled in a special way:\n\n    1. Filenames from FMI, with successful barcode reading:\n       `210305NAR005AAN_210416_164828` with plate name `210305NAR005AAN`;\n    2. Filenames from FMI, with failed barcode reading:\n       `yymmdd_hhmmss_210416_164828` with plate name `RS{yymmddhhmmss}`.\n\n    For all non-matching filenames, plate name is `plate_prefix`.\n\n    Args:\n        plate_prefix: TBD\n    \"\"\"\n\n    fields = plate_prefix.split(\"_\")\n\n    # FMI (successful barcode reading)\n    if (\n        len(fields) == 3\n        and len(fields[1]) == 6\n        and len(fields[2]) == 6\n        and fields[1].isdigit()\n        and fields[2].isdigit()\n    ):\n        barcode, img_date, img_time = fields[:]\n        plate = barcode\n    # FMI (failed barcode reading)\n    elif (\n        len(fields) == 4\n        and len(fields[0]) == 6\n        and len(fields[1]) == 6\n        and len(fields[2]) == 6\n        and len(fields[3]) == 6\n        and fields[0].isdigit()\n        and fields[1].isdigit()\n        and fields[2].isdigit()\n        and fields[3].isdigit()\n    ):\n        scan_date, scan_time, img_date, img_time = fields[:]\n        plate = f\"RS{scan_date + scan_time}\"\n    # All non-matching cases\n    else:\n        plate = plate_prefix\n\n    return plate\n
"},{"location":"reference/fractal_tasks_core/lib_parse_filename_metadata/#fractal_tasks_core.lib_parse_filename_metadata.parse_filename","title":"parse_filename(filename)","text":"

Parse image metadata from filename.

PARAMETER DESCRIPTION filename

Name of the image.

TYPE: str

RETURNS DESCRIPTION dict[str, str]

Metadata dictionary.

Source code in fractal_tasks_core/lib_parse_filename_metadata.py 
def parse_filename(filename: str) -> dict[str, str]:\n\"\"\"\n    Parse image metadata from filename.\n\n    Args:\n        filename: Name of the image.\n\n    Returns:\n        Metadata dictionary.\n    \"\"\"\n\n    # Remove extension and folder from filename\n    filename = Path(filename).with_suffix(\"\").name\n\n    output = {}\n\n    # Split filename into plate_prefix + well + TFLAZC\n    filename_fields = filename.split(\"_\")\n    if len(filename_fields) < 3:\n        raise ValueError(f\"{filename} not valid\")\n    output[\"plate_prefix\"] = \"_\".join(filename_fields[:-2])\n    output[\"plate\"] = _get_plate_name(output[\"plate_prefix\"])\n\n    # Assign well\n    output[\"well\"] = filename_fields[-2]\n\n    # Assign TFLAZC\n    TFLAZC = filename_fields[-1]\n    metadata = re.split(r\"([0-9]+)\", TFLAZC)\n    if metadata[-1] != \"\" or len(metadata) != 13:\n        raise ValueError(f\"Something wrong with {filename=}, {TFLAZC=}\")\n    # Remove 13-th (and last) element of the metadata list (an empty string)\n    metadata = metadata[:-1]\n    # Fill output dictionary\n    for ind, key in enumerate(metadata[::2]):\n        value = metadata[2 * ind + 1]\n        if key.isdigit() or not value.isdigit():\n            raise ValueError(\n                f\"Something wrong with {filename=}, for {key=} {value=}\"\n            )\n        output[key] = value\n    return output\n
"},{"location":"reference/fractal_tasks_core/lib_pyramid_creation/","title":"lib_pyramid_creation","text":"

Construct and write pyramid of lower-resolution levels.

"},{"location":"reference/fractal_tasks_core/lib_pyramid_creation/#fractal_tasks_core.lib_pyramid_creation.build_pyramid","title":"build_pyramid(*, zarrurl, overwrite=False, num_levels=2, coarsening_xy=2, chunksize=None, aggregation_function=None)","text":"

Starting from on-disk highest-resolution data, build and write to disk a pyramid with (num_levels - 1) coarsened levels. This function works for 2D, 3D or 4D arrays.

PARAMETER DESCRIPTION zarrurl

Path of the zarr group, which must already include level 0 (e.g. \"some/path/plate.zarr/B/03/0\").

TYPE: Union[str, Path]

overwrite

Whether to overwrite existing pyramid levels.

TYPE: bool DEFAULT: False

num_levels

Total number of pyramid levels (including 0).

TYPE: int DEFAULT: 2

coarsening_xy

Linear coarsening factor between subsequent levels.

TYPE: int DEFAULT: 2

chunksize

Shape of a single chunk.

TYPE: Optional[Sequence[int]] DEFAULT: None

aggregation_function

Function to be used when downsampling.

TYPE: Optional[Callable] DEFAULT: None

Source code in fractal_tasks_core/lib_pyramid_creation.py 
def build_pyramid(\n    *,\n    zarrurl: Union[str, pathlib.Path],\n    overwrite: bool = False,\n    num_levels: int = 2,\n    coarsening_xy: int = 2,\n    chunksize: Optional[Sequence[int]] = None,\n    aggregation_function: Optional[Callable] = None,\n) -> None:\n\n\"\"\"\n    Starting from on-disk highest-resolution data, build and write to disk a\n    pyramid with `(num_levels - 1)` coarsened levels.\n    This function works for 2D, 3D or 4D arrays.\n\n    Args:\n        zarrurl: Path of the zarr group, which must already include level 0\n            (e.g. `\"some/path/plate.zarr/B/03/0\"`).\n        overwrite: Whether to overwrite existing pyramid levels.\n        num_levels: Total number of pyramid levels (including 0).\n        coarsening_xy: Linear coarsening factor between subsequent levels.\n        chunksize: Shape of a single chunk.\n        aggregation_function: Function to be used when downsampling.\n    \"\"\"\n\n    # Clean up zarrurl\n    zarrurl = str(pathlib.Path(zarrurl))  # FIXME\n    zarrurl_highres = f\"{zarrurl}/0\"\n    logger.info(f\"[build_pyramid] High-resolution path: {zarrurl_highres}\")\n\n    # Lazily load highest-resolution data\n    data_highres = da.from_zarr(zarrurl_highres)\n    logger.info(f\"[build_pyramid] High-resolution data: {str(data_highres)}\")\n\n    # Check the number of axes and identify YX dimensions\n    ndims = len(data_highres.shape)\n    if ndims not in [2, 3, 4]:\n        raise ValueError(f\"{data_highres.shape=}, ndims not in [2,3,4]\")\n    y_axis = ndims - 2\n    x_axis = ndims - 1\n\n    # Set aggregation_function\n    if aggregation_function is None:\n        aggregation_function = np.mean\n\n    # Compute and write lower-resolution levels\n    previous_level = data_highres\n    for ind_level in range(1, num_levels):\n        # Verify that coarsening is doable\n        if min(previous_level.shape[-2:]) < coarsening_xy:\n            raise ValueError(\n                f\"ERROR: at {ind_level}-th level, \"\n                f\"coarsening_xy={coarsening_xy} \"\n                f\"but previous level has shape {previous_level.shape}\"\n            )\n        # Apply coarsening\n        newlevel = da.coarsen(\n            aggregation_function,\n            previous_level,\n            {y_axis: coarsening_xy, x_axis: coarsening_xy},\n            trim_excess=True,\n        ).astype(data_highres.dtype)\n\n        # Apply rechunking\n        if chunksize is None:\n            newlevel_rechunked = newlevel\n        else:\n            newlevel_rechunked = newlevel.rechunk(chunksize)\n        logger.info(\n            f\"[build_pyramid] Level {ind_level} data: \"\n            f\"{str(newlevel_rechunked)}\"\n        )\n\n        # Write zarr and store output (useful to construct next level)\n        previous_level = newlevel_rechunked.to_zarr(\n            zarrurl,\n            component=f\"{ind_level}\",\n            overwrite=overwrite,\n            compute=True,\n            return_stored=True,\n            write_empty_chunks=False,\n            dimension_separator=\"/\",\n        )\n
"},{"location":"reference/fractal_tasks_core/lib_read_fractal_metadata/","title":"lib_read_fractal_metadata","text":"

Helper functions to extract parameters from metadata dictionary.

"},{"location":"reference/fractal_tasks_core/lib_read_fractal_metadata/#fractal_tasks_core.lib_read_fractal_metadata.find_omengff_acquisition","title":"find_omengff_acquisition(image_zarr_path)","text":"

Discover the acquisition index based on OME-NGFF metadata.

Given the path to a zarr image folder (e.g. /path/plate.zarr/B/03/0), extract the acquisition index from the .zattrs file of the parent folder (i.e. at the well level), or return None if acquisition is not specified.

Notes:

  1. For non-multiplexing datasets, acquisition is not a required information in the metadata. If it is not there, this function returns None.
  2. This function fails if we use an image that does not belong to an OME-NGFF well.
PARAMETER DESCRIPTION image_zarr_path

Full path to an OME-NGFF image folder.

TYPE: Path

Source code in fractal_tasks_core/lib_read_fractal_metadata.py 
def find_omengff_acquisition(image_zarr_path: Path) -> Union[int, None]:\n\"\"\"\n    Discover the acquisition index based on OME-NGFF metadata.\n\n    Given the path to a zarr image folder (e.g. `/path/plate.zarr/B/03/0`),\n    extract the acquisition index from the `.zattrs` file of the parent\n    folder (i.e. at the well level), or return `None` if acquisition is not\n    specified.\n\n    Notes:\n\n    1. For non-multiplexing datasets, acquisition is not a required\n       information in the metadata. If it is not there, this function\n       returns `None`.\n    2. This function fails if we use an image that does not belong to\n       an OME-NGFF well.\n\n    Args:\n        image_zarr_path: Full path to an OME-NGFF image folder.\n    \"\"\"\n\n    # Identify well path and attrs\n    well_zarr_path = image_zarr_path.parent\n    if not (well_zarr_path / \".zattrs\").exists():\n        raise ValueError(\n            f\"{str(well_zarr_path)} must be an OME-NGFF well \"\n            \"folder, but it does not include a .zattrs file.\"\n        )\n    well_group = zarr.open_group(str(well_zarr_path))\n    attrs_images = well_group.attrs[\"well\"][\"images\"]\n\n    # Loook for the acquisition of the current image (if any)\n    acquisition = None\n    for img_dict in attrs_images:\n        if (\n            img_dict[\"path\"] == image_zarr_path.name\n            and \"acquisition\" in img_dict.keys()\n        ):\n            acquisition = img_dict[\"acquisition\"]\n            break\n\n    return acquisition\n
"},{"location":"reference/fractal_tasks_core/lib_read_fractal_metadata/#fractal_tasks_core.lib_read_fractal_metadata.get_parameters_from_metadata","title":"get_parameters_from_metadata(*, keys, metadata, image_zarr_path)","text":"

Flexibly extract parameters from metadata dictionary

This covers both parameters which are acquisition-specific (if the image belongs to an OME-NGFF array and its acquisition is specified) or simply available in the dictionary. The two cases are handled as: 

metadata[acquisition][\"some_parameter\"]  # acquisition available\nmetadata[\"some_parameter\"]               # acquisition not available\n

PARAMETER DESCRIPTION keys

list of required parameters.

TYPE: Sequence[str]

metadata

metadata dictionary.

TYPE: Dict[str, Any]

image_zarr_path

full path to image, e.g. /path/plate.zarr/B/03/0.

TYPE: Path

Source code in fractal_tasks_core/lib_read_fractal_metadata.py 
def get_parameters_from_metadata(\n    *,\n    keys: Sequence[str],\n    metadata: Dict[str, Any],\n    image_zarr_path: Path,\n) -> Dict[str, Any]:\n\"\"\"\n    Flexibly extract parameters from metadata dictionary\n\n    This covers both parameters which are acquisition-specific (if the image\n    belongs to an OME-NGFF array and its acquisition is specified) or simply\n    available in the dictionary.\n    The two cases are handled as:\n    ```\n    metadata[acquisition][\"some_parameter\"]  # acquisition available\n    metadata[\"some_parameter\"]               # acquisition not available\n    ```\n\n    Args:\n        keys: list of required parameters.\n        metadata: metadata dictionary.\n        image_zarr_path: full path to image, e.g. `/path/plate.zarr/B/03/0`.\n    \"\"\"\n\n    parameters = {}\n    acquisition = find_omengff_acquisition(image_zarr_path)\n    if acquisition is not None:\n        parameters[\"acquisition\"] = acquisition\n\n    for key in keys:\n        if acquisition is None:\n            parameter = metadata[key]\n        else:\n            try:\n                parameter = metadata[key][str(acquisition)]\n            except TypeError:\n                parameter = metadata[key]\n            except KeyError:\n                parameter = metadata[key]\n        parameters[key] = parameter\n    return parameters\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/","title":"lib_regions_of_interest","text":"

Functions to handle regions of interests (via pandas and AnnData).

"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.are_ROI_table_columns_valid","title":"are_ROI_table_columns_valid(*, table)","text":"

Verify some validity assumptions on a ROI table.

This function reflects our current working assumptions (e.g. the presence of some specific columns); this may change in future versions.

PARAMETER DESCRIPTION table

AnnData table to be checked

TYPE: AnnData

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def are_ROI_table_columns_valid(*, table: ad.AnnData) -> None:\n\"\"\"\n    Verify some validity assumptions on a ROI table.\n\n    This function reflects our current working assumptions (e.g. the presence\n    of some specific columns); this may change in future versions.\n\n    Args:\n        table: AnnData table to be checked\n    \"\"\"\n\n    # Hard constraint: table columns must include some expected ones\n    columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n    ]\n    for column in columns:\n        if column not in table.var_names:\n            raise ValueError(f\"Column {column} is not present in ROI table\")\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.array_to_bounding_box_table","title":"array_to_bounding_box_table(mask_array, pxl_sizes_zyx, origin_zyx=(0, 0, 0))","text":"

Construct bounding-box ROI table for a mask array.

PARAMETER DESCRIPTION mask_array

Original array to construct bounding boxes.

TYPE: ndarray

pxl_sizes_zyx

Physical-unit pixel ZYX sizes.

TYPE: list[float]

origin_zyx

Shift ROI origin by this amount of ZYX pixels.

TYPE: tuple[int, int, int] DEFAULT: (0, 0, 0)

RETURNS DESCRIPTION DataFrame

DataFrame with each line representing the bounding-box ROI that corresponds to a unique value of mask_array. ROI properties are expressed in physical units (with columns defined as elsewhere this module - see e.g. prepare_well_ROI_table), and positions are optionally shifted (if origin_zyx is set). An additional column label keeps track of the mask_array value corresponding to each ROI.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def array_to_bounding_box_table(\n    mask_array: np.ndarray,\n    pxl_sizes_zyx: list[float],\n    origin_zyx: tuple[int, int, int] = (0, 0, 0),\n) -> pd.DataFrame:\n\"\"\"\n    Construct bounding-box ROI table for a mask array.\n\n    Args:\n        mask_array: Original array to construct bounding boxes.\n        pxl_sizes_zyx: Physical-unit pixel ZYX sizes.\n        origin_zyx: Shift ROI origin by this amount of ZYX pixels.\n\n    Returns:\n        DataFrame with each line representing the bounding-box ROI that\n            corresponds to a unique value of `mask_array`. ROI properties are\n            expressed in physical units (with columns defined as elsewhere this\n            module - see e.g. `prepare_well_ROI_table`), and positions are\n            optionally shifted (if `origin_zyx` is set). An additional column\n            `label` keeps track of the `mask_array` value corresponding to each\n            ROI.\n    \"\"\"\n\n    pxl_sizes_zyx_array = np.array(pxl_sizes_zyx)\n    z_origin, y_origin, x_origin = origin_zyx[:]\n\n    labels = np.unique(mask_array)\n    labels = labels[labels > 0]\n    elem_list = []\n    for label in labels:\n\n        # Compute bounding box\n        label_match = np.where(mask_array == label)\n        zmin, ymin, xmin = np.min(label_match, axis=1) * pxl_sizes_zyx_array\n        zmax, ymax, xmax = (\n            np.max(label_match, axis=1) + 1\n        ) * pxl_sizes_zyx_array\n\n        # Compute bounding-box edges\n        length_x = xmax - xmin\n        length_y = ymax - ymin\n        length_z = zmax - zmin\n\n        # Shift origin\n        zmin += z_origin * pxl_sizes_zyx[0]\n        ymin += y_origin * pxl_sizes_zyx[1]\n        xmin += x_origin * pxl_sizes_zyx[2]\n\n        elem_list.append((xmin, ymin, zmin, length_x, length_y, length_z))\n\n    df_columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n    ]\n\n    if len(elem_list) == 0:\n        df = pd.DataFrame(columns=[x for x in df_columns] + [\"label\"])\n    else:\n        df = pd.DataFrame(np.array(elem_list), columns=df_columns)\n        df[\"label\"] = labels\n\n    return df\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.check_valid_ROI_indices","title":"check_valid_ROI_indices(list_indices, ROI_table_name)","text":"

Check that list of indices has zero origin on each axis.

See fractal-tasks-core issues #530 and #554.

This helper function is meant to provide informative error messages when ROI tables created with fractal-tasks-core up to v0.11 are used in v0.12. This function will be deprecated and removed as soon as the v0.11/v0.12 transition advances.

Note that only FOV_ROI_table and well_ROI_table have to fulfill this constraint, while ROI tables obtained through segmentation may have arbitrary (non-negative) indices.

PARAMETER DESCRIPTION list_indices

Output of convert_ROI_table_to_indices; each item is like [start_z, end_z, start_y, end_y, start_x, end_x].

TYPE: list[list[int]]

ROI_table_name

Name of the ROI table.

TYPE: str

RAISES DESCRIPTION ValueError

If the table name is FOV_ROI_table or well_ROI_table and the minimum value of start_x, start_y and start_z are not all zero.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def check_valid_ROI_indices(\n    list_indices: list[list[int]],\n    ROI_table_name: str,\n) -> None:\n\"\"\"\n    Check that list of indices has zero origin on each axis.\n\n    See fractal-tasks-core issues #530 and #554.\n\n    This helper function is meant to provide informative error messages when\n    ROI tables created with fractal-tasks-core up to v0.11 are used in v0.12.\n    This function will be deprecated and removed as soon as the v0.11/v0.12\n    transition advances.\n\n    Note that only `FOV_ROI_table` and `well_ROI_table` have to fulfill this\n    constraint, while ROI tables obtained through segmentation may have\n    arbitrary (non-negative) indices.\n\n    Args:\n        list_indices:\n            Output of `convert_ROI_table_to_indices`; each item is like\n            `[start_z, end_z, start_y, end_y, start_x, end_x]`.\n        ROI_table_name: Name of the ROI table.\n\n    Raises:\n        ValueError:\n            If the table name is `FOV_ROI_table` or `well_ROI_table` and the\n                minimum value of `start_x`, `start_y` and `start_z` are not all\n                zero.\n    \"\"\"\n    if ROI_table_name not in [\"FOV_ROI_table\", \"well_ROI_table\"]:\n        # This validation function only applies to the FOV/well ROI tables\n        # generated with fractal-tasks-core\n        return\n\n    # Find minimum index along ZYX\n    min_start_z = min(item[0] for item in list_indices)\n    min_start_y = min(item[2] for item in list_indices)\n    min_start_x = min(item[4] for item in list_indices)\n\n    # Check that minimum indices are all zero\n    for (ind, min_index) in enumerate((min_start_z, min_start_y, min_start_x)):\n        if min_index != 0:\n            axis = [\"Z\", \"Y\", \"X\"][ind]\n            raise ValueError(\n                f\"{axis} component of ROI indices for table `{ROI_table_name}`\"\n                f\" do not start with 0, but with {min_index}.\\n\"\n                \"Hint: As of fractal-tasks-core v0.12, FOV/well ROI \"\n                \"tables with non-zero origins (e.g. the ones created with \"\n                \"v0.11) are not supported.\"\n            )\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.convert_ROI_table_to_indices","title":"convert_ROI_table_to_indices(ROI, full_res_pxl_sizes_zyx, level=0, coarsening_xy=2, cols_xyz_pos=['x_micrometer', 'y_micrometer', 'z_micrometer'], cols_xyz_len=['len_x_micrometer', 'len_y_micrometer', 'len_z_micrometer'])","text":"

Convert a ROI AnnData table into integer array indices.

PARAMETER DESCRIPTION ROI

AnnData table with list of ROIs.

TYPE: AnnData

full_res_pxl_sizes_zyx

Physical-unit pixel ZYX sizes at the full-resolution pyramid level.

TYPE: Sequence[float]

level

Pyramid level.

TYPE: int DEFAULT: 0

coarsening_xy

Linear coarsening factor in the YX plane.

TYPE: int DEFAULT: 2

cols_xyz_pos

Column names for XYZ ROI positions.

TYPE: Sequence[str] DEFAULT: ['x_micrometer', 'y_micrometer', 'z_micrometer']

cols_xyz_len

Column names for XYZ ROI edges.

TYPE: Sequence[str] DEFAULT: ['len_x_micrometer', 'len_y_micrometer', 'len_z_micrometer']

RAISES DESCRIPTION ValueError

If any of the array indices is negative.

RETURNS DESCRIPTION list[list[int]]

Nested list of indices. The main list has one item per ROI. Each ROI item is a list of six integers as in [start_z, end_z, start_y, end_y, start_x, end_x]. The array-index interval for a given ROI is start_x:end_x along X, and so on for Y and Z.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def convert_ROI_table_to_indices(\n    ROI: ad.AnnData,\n    full_res_pxl_sizes_zyx: Sequence[float],\n    level: int = 0,\n    coarsening_xy: int = 2,\n    cols_xyz_pos: Sequence[str] = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n    ],\n    cols_xyz_len: Sequence[str] = [\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n    ],\n) -> list[list[int]]:\n\"\"\"\n    Convert a ROI AnnData table into integer array indices.\n\n    Args:\n        ROI: AnnData table with list of ROIs.\n        full_res_pxl_sizes_zyx:\n            Physical-unit pixel ZYX sizes at the full-resolution pyramid level.\n        level: Pyramid level.\n        coarsening_xy: Linear coarsening factor in the YX plane.\n        cols_xyz_pos: Column names for XYZ ROI positions.\n        cols_xyz_len: Column names for XYZ ROI edges.\n\n    Raises:\n        ValueError:\n            If any of the array indices is negative.\n\n    Returns:\n        Nested list of indices. The main list has one item per ROI. Each ROI\n            item is a list of six integers as in `[start_z, end_z, start_y,\n            end_y, start_x, end_x]`. The array-index interval for a given ROI\n            is `start_x:end_x` along X, and so on for Y and Z.\n    \"\"\"\n    # Handle empty ROI table\n    if len(ROI) == 0:\n        return []\n\n    # Set pyramid-level pixel sizes\n    pxl_size_z, pxl_size_y, pxl_size_x = full_res_pxl_sizes_zyx\n    prefactor = coarsening_xy**level\n    pxl_size_x *= prefactor\n    pxl_size_y *= prefactor\n\n    x_pos, y_pos, z_pos = cols_xyz_pos[:]\n    x_len, y_len, z_len = cols_xyz_len[:]\n\n    list_indices = []\n    for ROI_name in ROI.obs_names:\n\n        # Extract data from anndata table\n        x_micrometer = ROI[ROI_name, x_pos].X[0, 0]\n        y_micrometer = ROI[ROI_name, y_pos].X[0, 0]\n        z_micrometer = ROI[ROI_name, z_pos].X[0, 0]\n        len_x_micrometer = ROI[ROI_name, x_len].X[0, 0]\n        len_y_micrometer = ROI[ROI_name, y_len].X[0, 0]\n        len_z_micrometer = ROI[ROI_name, z_len].X[0, 0]\n\n        # Identify indices along the three dimensions\n        start_x = x_micrometer / pxl_size_x\n        end_x = (x_micrometer + len_x_micrometer) / pxl_size_x\n        start_y = y_micrometer / pxl_size_y\n        end_y = (y_micrometer + len_y_micrometer) / pxl_size_y\n        start_z = z_micrometer / pxl_size_z\n        end_z = (z_micrometer + len_z_micrometer) / pxl_size_z\n        indices = [start_z, end_z, start_y, end_y, start_x, end_x]\n\n        # Round indices to lower integer\n        indices = list(map(round, indices))\n\n        # Fail for negative indices\n        if min(indices) < 0:\n            raise ValueError(\n                f\"ROI {ROI_name} converted into negative array indices.\\n\"\n                f\"ZYX position: {z_micrometer}, {y_micrometer}, \"\n                f\"{x_micrometer}\\n\"\n                f\"ZYX pixel sizes: {pxl_size_z}, {pxl_size_y}, \"\n                f\"{pxl_size_x} ({level=})\\n\"\n                \"Hint: As of fractal-tasks-core v0.12, FOV/well ROI \"\n                \"tables with non-zero origins (e.g. the ones created with \"\n                \"v0.11) are not supported.\"\n            )\n\n        # Append ROI indices to to list\n        list_indices.append(indices[:])\n\n    return list_indices\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.convert_ROIs_from_3D_to_2D","title":"convert_ROIs_from_3D_to_2D(adata, pixel_size_z)","text":"

TBD

PARAMETER DESCRIPTION adata

TBD

TYPE: AnnData

pixel_size_z

TBD

TYPE: float

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def convert_ROIs_from_3D_to_2D(\n    adata: ad.AnnData,\n    pixel_size_z: float,\n) -> ad.AnnData:\n\"\"\"\n    TBD\n\n    Args:\n        adata: TBD\n        pixel_size_z: TBD\n    \"\"\"\n\n    # Compress a 3D stack of images to a single Z plane,\n    # with thickness equal to pixel_size_z\n    df = adata.to_df()\n    df[\"len_z_micrometer\"] = pixel_size_z\n\n    # Assign dtype explicitly, to avoid\n    # >> UserWarning: X converted to numpy array with dtype float64\n    # when creating AnnData object\n    df = df.astype(np.float32)\n\n    # Create an AnnData object directly from the DataFrame\n    new_adata = ad.AnnData(X=df)\n\n    # Rename rows and columns\n    new_adata.obs_names = adata.obs_names\n    new_adata.var_names = list(map(str, df.columns))\n\n    return new_adata\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.convert_indices_to_regions","title":"convert_indices_to_regions(index)","text":"

Converts index tuples to region tuple

PARAMETER DESCRIPTION index

Tuple containing 6 entries of (z_start, z_end, y_start, y_end, x_start, x_end).

TYPE: list[int]

RETURNS DESCRIPTION region

tuple of three slices (ZYX)

TYPE: tuple[slice, slice, slice]

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def convert_indices_to_regions(\n    index: list[int],\n) -> tuple[slice, slice, slice]:\n\"\"\"\n    Converts index tuples to region tuple\n\n    Args:\n        index: Tuple containing 6 entries of (z_start, z_end, y_start,\n            y_end, x_start, x_end).\n\n    Returns:\n        region: tuple of three slices (ZYX)\n    \"\"\"\n    return (\n        slice(index[0], index[1]),\n        slice(index[2], index[3]),\n        slice(index[4], index[5]),\n    )\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.empty_bounding_box_table","title":"empty_bounding_box_table()","text":"

Construct an empty bounding-box ROI table of given shape.

This function mirrors the functionality of array_to_bounding_box_table, for the specific case where the array includes no label. The advantages of this function are that:

  1. It does not require computing a whole array of zeros;
  2. We avoid hardcoding column names in the task functions.
RETURNS DESCRIPTION DataFrame

DataFrame with no rows, and with columns corresponding to the output of array_to_bounding_box_table.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def empty_bounding_box_table() -> pd.DataFrame:\n\"\"\"\n    Construct an empty bounding-box ROI table of given shape.\n\n    This function mirrors the functionality of `array_to_bounding_box_table`,\n    for the specific case where the array includes no label. The advantages of\n    this function are that:\n\n    1. It does not require computing a whole array of zeros;\n    2. We avoid hardcoding column names in the task functions.\n\n    Returns:\n        DataFrame with no rows, and with columns corresponding to the output of\n            `array_to_bounding_box_table`.\n    \"\"\"\n\n    df_columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n    ]\n    df = pd.DataFrame(columns=[x for x in df_columns] + [\"label\"])\n    return df\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.get_image_grid_ROIs","title":"get_image_grid_ROIs(array_shape, pixels_ZYX, grid_YX_shape)","text":"

Produce a table with ROIS placed on a rectangular grid.

The main goal of this ROI grid is to allow processing of smaller subset of the whole array.

In a specific case (that is, if the image array was obtained by stitching together a set of FOVs placed on a regular grid), the ROIs correspond to the original FOVs.

TODO: make this flexible with respect to the presence/absence of Z.

PARAMETER DESCRIPTION array_shape

ZYX shape of the image array.

TYPE: tuple[int, int, int]

pixels_ZYX

ZYX pixel sizes in micrometers.

TYPE: list[float]

grid_YX_shape

TYPE: tuple[int, int]

RETURNS DESCRIPTION AnnData

An AnnData table with a single ROI.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def get_image_grid_ROIs(\n    array_shape: tuple[int, int, int],\n    pixels_ZYX: list[float],\n    grid_YX_shape: tuple[int, int],\n) -> ad.AnnData:\n\"\"\"\n    Produce a table with ROIS placed on a rectangular grid.\n\n    The main goal of this ROI grid is to allow processing of smaller subset of\n    the whole array.\n\n    In a specific case (that is, if the image array was obtained by stitching\n    together a set of FOVs placed on a regular grid), the ROIs correspond to\n    the original FOVs.\n\n    TODO: make this flexible with respect to the presence/absence of Z.\n\n    Args:\n        array_shape: ZYX shape of the image array.\n        pixels_ZYX: ZYX pixel sizes in micrometers.\n        grid_YX_shape:\n\n    Returns:\n        An `AnnData` table with a single ROI.\n    \"\"\"\n    shape_z, shape_y, shape_x = array_shape[-3:]\n    grid_size_y, grid_size_x = grid_YX_shape[:]\n    X = []\n    obs_names = []\n    counter = 0\n    start_z = 0\n    len_z = shape_z\n\n    # Find minimal len_y that covers [0,shape_y] with grid_size_y intervals\n    len_y = math.ceil(shape_y / grid_size_y)\n    len_x = math.ceil(shape_x / grid_size_x)\n    for ind_y in range(grid_size_y):\n        start_y = ind_y * len_y\n        tmp_len_y = min(shape_y, start_y + len_y) - start_y\n        for ind_x in range(grid_size_x):\n            start_x = ind_x * len_x\n            tmp_len_x = min(shape_x, start_x + len_x) - start_x\n            X.append(\n                [\n                    start_x * pixels_ZYX[2],\n                    start_y * pixels_ZYX[1],\n                    start_z * pixels_ZYX[0],\n                    tmp_len_x * pixels_ZYX[2],\n                    tmp_len_y * pixels_ZYX[1],\n                    len_z * pixels_ZYX[0],\n                    start_x * pixels_ZYX[2],\n                    start_y * pixels_ZYX[1],\n                ]\n            )\n            counter += 1\n            obs_names.append(f\"ROI_{counter}\")\n    ROI_table = ad.AnnData(X=np.array(X, dtype=np.float32))\n    ROI_table.obs_names = obs_names\n    ROI_table.var_names = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n        \"x_micrometer_original\",\n        \"y_micrometer_original\",\n    ]\n    return ROI_table\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.get_single_image_ROI","title":"get_single_image_ROI(array_shape, pixels_ZYX)","text":"

Produce a table with a single ROI that covers the whole array

TODO: make this flexible with respect to the presence/absence of Z.

PARAMETER DESCRIPTION array_shape

ZYX shape of the image array.

TYPE: tuple[int, int, int]

pixels_ZYX

ZYX pixel sizes in micrometers.

TYPE: list[float]

RETURNS DESCRIPTION AnnData

An AnnData table with a single ROI.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def get_single_image_ROI(\n    array_shape: tuple[int, int, int],\n    pixels_ZYX: list[float],\n) -> ad.AnnData:\n\"\"\"\n    Produce a table with a single ROI that covers the whole array\n\n    TODO: make this flexible with respect to the presence/absence of Z.\n\n    Args:\n        array_shape: ZYX shape of the image array.\n        pixels_ZYX: ZYX pixel sizes in micrometers.\n\n    Returns:\n        An `AnnData` table with a single ROI.\n    \"\"\"\n    shape_z, shape_y, shape_x = array_shape[-3:]\n    ROI_table = ad.AnnData(\n        X=np.array(\n            [\n                [\n                    0.0,\n                    0.0,\n                    0.0,\n                    shape_x * pixels_ZYX[2],\n                    shape_y * pixels_ZYX[1],\n                    shape_z * pixels_ZYX[0],\n                    0.0,\n                    0.0,\n                ],\n            ],\n            dtype=np.float32,\n        )\n    )\n    ROI_table.obs_names = [\"image_1\"]\n    ROI_table.var_names = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n        \"x_micrometer_original\",\n        \"y_micrometer_original\",\n    ]\n    return ROI_table\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.is_ROI_table_valid","title":"is_ROI_table_valid(*, table_path, use_masks)","text":"

Verify some validity assumptions on a ROI table.

This function reflects our current working assumptions (e.g. the presence of some specific columns); this may change in future versions.

If use_masks=True, we verify that the table is suitable to be used as part of our masked-loading functions (see lib_masked_loading.py); if these checks fail, use_masks should be set to False upstream in the parent function.

PARAMETER DESCRIPTION table_path

Path of the AnnData ROI table to be checked.

TYPE: str

use_masks

If True, perform some additional checks related to masked loading.

TYPE: bool

RETURNS DESCRIPTION Optional[bool]

Always None if use_masks=False, otherwise return whether the table is valid for masked loading.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def is_ROI_table_valid(*, table_path: str, use_masks: bool) -> Optional[bool]:\n\"\"\"\n    Verify some validity assumptions on a ROI table.\n\n    This function reflects our current working assumptions (e.g. the presence\n    of some specific columns); this may change in future versions.\n\n    If `use_masks=True`, we verify that the table is suitable to be used as\n    part of our masked-loading functions (see `lib_masked_loading.py`); if\n    these checks fail, `use_masks` should be set to `False` upstream in the\n    parent function.\n\n    Args:\n        table_path: Path of the AnnData ROI table to be checked.\n        use_masks: If `True`, perform some additional checks related to\n            masked loading.\n\n    Returns:\n        Always `None` if `use_masks=False`, otherwise return whether the table\n            is valid for masked loading.\n    \"\"\"\n\n    table = ad.read_zarr(table_path)\n    are_ROI_table_columns_valid(table=table)\n    if not use_masks:\n        return None\n\n    # Soft constraint: the table can be used for masked loading (if not, return\n    # False)\n    attrs = zarr.group(table_path).attrs\n    logger.info(f\"ROI table at {table_path} has attrs: {attrs.asdict()}\")\n    valid = set((\"type\", \"region\", \"instance_key\")).issubset(attrs.keys())\n    if valid:\n        valid = valid and attrs[\"type\"] == \"ngff:region_table\"\n        valid = valid and \"path\" in attrs[\"region\"].keys()\n    if valid:\n        return True\n    else:\n        return False\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.is_standard_roi_table","title":"is_standard_roi_table(table)","text":"

True if the name of the table contains one of the standard Fractal tables

If a table name is well_ROI_table, FOV_ROI_table or contains either of the two (e.g. registered_FOV_ROI_table), this function returns True.

PARAMETER DESCRIPTION table

table name

TYPE: str

RETURNS DESCRIPTION bool

bool of whether it's a standard ROI table

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def is_standard_roi_table(table: str) -> bool:\n\"\"\"\n    True if the name of the table contains one of the standard Fractal tables\n\n    If a table name is well_ROI_table, FOV_ROI_table or contains either of the\n    two (e.g. registered_FOV_ROI_table), this function returns True.\n\n    Args:\n        table: table name\n\n    Returns:\n        bool of whether it's a standard ROI table\n\n    \"\"\"\n    if \"well_ROI_table\" in table:\n        return True\n    elif \"FOV_ROI_table\" in table:\n        return True\n    else:\n        return False\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.load_region","title":"load_region(data_zyx, region, compute=True, return_as_3D=False)","text":"

Load a region from a dask array.

Can handle both 2D and 3D dask arrays as input and return them as is or always as a 3D array.

PARAMETER DESCRIPTION data_zyx

Dask array (2D or 3D).

TYPE: Array

region

Region to load, tuple of three slices (ZYX).

TYPE: tuple[slice, slice, slice]

compute

Whether to compute the result. If True, returns a numpy array. If False, returns a dask array.

TYPE: bool DEFAULT: True

return_as_3D

Whether to return a 3D array, even if the input is 2D.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION Union[Array, ndarray]

3D array.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def load_region(\n    data_zyx: da.Array,\n    region: tuple[slice, slice, slice],\n    compute: bool = True,\n    return_as_3D: bool = False,\n) -> Union[da.Array, np.ndarray]:\n\"\"\"\n    Load a region from a dask array.\n\n    Can handle both 2D and 3D dask arrays as input and return them as is or\n    always as a 3D array.\n\n    Args:\n        data_zyx: Dask array (2D or 3D).\n        region: Region to load, tuple of three slices (ZYX).\n        compute: Whether to compute the result. If `True`, returns a numpy\n            array. If `False`, returns a dask array.\n        return_as_3D: Whether to return a 3D array, even if the input is 2D.\n\n    Returns:\n        3D array.\n    \"\"\"\n\n    if len(region) != 3:\n        raise ValueError(\n            f\"In `load_region`, `region` must have three elements \"\n            f\"(given: {len(region)}).\"\n        )\n\n    if len(data_zyx.shape) == 3:\n        img = data_zyx[region]\n    elif len(data_zyx.shape) == 2:\n        img = data_zyx[(region[1], region[2])]\n        if return_as_3D:\n            img = np.expand_dims(img, axis=0)\n    else:\n        raise ValueError(\n            f\"Shape {data_zyx.shape} not supported for `load_region`\"\n        )\n    if compute:\n        return img.compute()\n    else:\n        return img\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.prepare_FOV_ROI_table","title":"prepare_FOV_ROI_table(df, metadata=('time'))","text":"

Prepare an AnnData table for fields-of-view ROIs.

PARAMETER DESCRIPTION df

Input dataframe, possibly prepared through parse_yokogawa_metadata.

TYPE: DataFrame

metadata

Columns of df to be stored (if present) into AnnData table obs.

TYPE: tuple[str, ...] DEFAULT: ('time')

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def prepare_FOV_ROI_table(\n    df: pd.DataFrame, metadata: tuple[str, ...] = (\"time\",)\n) -> ad.AnnData:\n\"\"\"\n    Prepare an AnnData table for fields-of-view ROIs.\n\n    Args:\n        df:\n            Input dataframe, possibly prepared through\n            `parse_yokogawa_metadata`.\n        metadata:\n            Columns of `df` to be stored (if present) into AnnData table `obs`.\n    \"\"\"\n\n    # Make a local copy of the dataframe, to avoid SettingWithCopyWarning\n    df = df.copy()\n\n    # Convert DataFrame index to str, to avoid\n    # >> ImplicitModificationWarning: Transforming to str index\n    # when creating AnnData object.\n    # Do this in the beginning to allow concatenation with e.g. time\n    df.index = df.index.astype(str)\n\n    # Obtain box size in physical units\n    df = df.assign(len_x_micrometer=df.x_pixel * df.pixel_size_x)\n    df = df.assign(len_y_micrometer=df.y_pixel * df.pixel_size_y)\n    df = df.assign(len_z_micrometer=df.z_pixel * df.pixel_size_z)\n\n    # Select only the numeric positional columns needed to define ROIs\n    # (to avoid) casting things like the data column to float32\n    # or to use unnecessary columns like bit_depth\n    positional_columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n        \"x_micrometer_original\",\n        \"y_micrometer_original\",\n    ]\n\n    # Assign dtype explicitly, to avoid\n    # >> UserWarning: X converted to numpy array with dtype float64\n    # when creating AnnData object\n    df_roi = df.loc[:, positional_columns].astype(np.float32)\n\n    # Create an AnnData object directly from the DataFrame\n    adata = ad.AnnData(X=df_roi)\n\n    # Reset origin of the FOV ROI table, so that it matches with the well\n    # origin\n    adata = reset_origin(adata)\n\n    # Save any metadata that is specified to the obs df\n    for col in metadata:\n        if col in df:\n            # Cast all metadata to str.\n            # Reason: AnnData Zarr writers don't support all pandas types.\n            # e.g. pandas.core.arrays.datetimes.DatetimeArray can't be written\n            adata.obs[col] = df[col].astype(str)\n\n    # Rename rows and columns: Maintain FOV indices from the dataframe\n    # (they are already enforced to be unique by Pandas and may contain\n    # information for the user, as they are based on the filenames)\n    adata.obs_names = \"FOV_\" + adata.obs.index\n    adata.var_names = list(map(str, df_roi.columns))\n\n    return adata\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.prepare_well_ROI_table","title":"prepare_well_ROI_table(df, metadata=('time'))","text":"

Prepare an AnnData table with a single well ROI.

PARAMETER DESCRIPTION df

Input dataframe, possibly prepared through parse_yokogawa_metadata.

TYPE: DataFrame

metadata

Columns of df to be stored (if present) into AnnData table obs.

TYPE: tuple[str, ...] DEFAULT: ('time')

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def prepare_well_ROI_table(\n    df: pd.DataFrame, metadata: tuple[str, ...] = (\"time\",)\n) -> ad.AnnData:\n\"\"\"\n    Prepare an AnnData table with a single well ROI.\n\n    Args:\n        df:\n            Input dataframe, possibly prepared through\n            `parse_yokogawa_metadata`.\n        metadata:\n            Columns of `df` to be stored (if present) into AnnData table `obs`.\n    \"\"\"\n\n    # Make a local copy of the dataframe, to avoid SettingWithCopyWarning\n    df = df.copy()\n\n    # Convert DataFrame index to str, to avoid\n    # >> ImplicitModificationWarning: Transforming to str index\n    # when creating AnnData object.\n    # Do this in the beginning to allow concatenation with e.g. time\n    df.index = df.index.astype(str)\n\n    # Calculate bounding box extents in physical units\n    for mu in [\"x\", \"y\", \"z\"]:\n        # Obtain per-FOV properties in physical units.\n        # NOTE: a FOV ROI is defined here as the interval [min_micrometer,\n        # max_micrometer], with max_micrometer=min_micrometer+len_micrometer\n        min_micrometer = df[f\"{mu}_micrometer\"]\n        len_micrometer = df[f\"{mu}_pixel\"] * df[f\"pixel_size_{mu}\"]\n        max_micrometer = min_micrometer + len_micrometer\n        # Obtain well bounding box, in physical units\n        min_min_micrometer = min_micrometer.min()\n        max_max_micrometer = max_micrometer.max()\n        df[f\"{mu}_micrometer\"] = min_min_micrometer\n        df[f\"len_{mu}_micrometer\"] = max_max_micrometer - min_min_micrometer\n\n    # Select only the numeric positional columns needed to define ROIs\n    # (to avoid) casting things like the data column to float32\n    # or to use unnecessary columns like bit_depth\n    positional_columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n    ]\n\n    # Assign dtype explicitly, to avoid\n    # >> UserWarning: X converted to numpy array with dtype float64\n    # when creating AnnData object\n    df_roi = df.iloc[0:1, :].loc[:, positional_columns].astype(np.float32)\n\n    # Create an AnnData object directly from the DataFrame\n    adata = ad.AnnData(X=df_roi)\n\n    # Reset origin of the single-entry well ROI table\n    adata = reset_origin(adata)\n\n    # Save any metadata that is specified to the obs df\n    for col in metadata:\n        if col in df:\n            # Cast all metadata to str.\n            # Reason: AnnData Zarr writers don't support all pandas types.\n            # e.g. pandas.core.arrays.datetimes.DatetimeArray can't be written\n            adata.obs[col] = df[col].astype(str)\n\n    # Rename rows and columns: Maintain FOV indices from the dataframe\n    # (they are already enforced to be unique by Pandas and may contain\n    # information for the user, as they are based on the filenames)\n    adata.obs_names = \"well_\" + adata.obs.index\n    adata.var_names = list(map(str, df_roi.columns))\n\n    return adata\n
"},{"location":"reference/fractal_tasks_core/lib_regions_of_interest/#fractal_tasks_core.lib_regions_of_interest.reset_origin","title":"reset_origin(ROI_table, x_pos='x_micrometer', y_pos='y_micrometer', z_pos='z_micrometer')","text":"

Return a copy of a ROI table, with shifted-to-zero origin for some columns.

PARAMETER DESCRIPTION ROI_table

Original ROI table.

TYPE: AnnData

x_pos

Name of the column with X position of ROIs.

TYPE: str DEFAULT: 'x_micrometer'

y_pos

Name of the column with Y position of ROIs.

TYPE: str DEFAULT: 'y_micrometer'

z_pos

Name of the column with Z position of ROIs.

TYPE: str DEFAULT: 'z_micrometer'

RETURNS DESCRIPTION AnnData

A copy of the ROI_table AnnData table, where values of x_pos, y_pos and z_pos columns have been shifted by their minimum values.

Source code in fractal_tasks_core/lib_regions_of_interest.py 
def reset_origin(\n    ROI_table: ad.AnnData,\n    x_pos: str = \"x_micrometer\",\n    y_pos: str = \"y_micrometer\",\n    z_pos: str = \"z_micrometer\",\n) -> ad.AnnData:\n\"\"\"\n    Return a copy of a ROI table, with shifted-to-zero origin for some columns.\n\n    Args:\n        ROI_table: Original ROI table.\n        x_pos: Name of the column with X position of ROIs.\n        y_pos: Name of the column with Y position of ROIs.\n        z_pos: Name of the column with Z position of ROIs.\n\n    Returns:\n        A copy of the `ROI_table` AnnData table, where values of `x_pos`,\n            `y_pos` and `z_pos` columns have been shifted by their minimum\n            values.\n    \"\"\"\n    new_table = ROI_table.copy()\n\n    origin_x = min(new_table[:, x_pos].X[:, 0])\n    origin_y = min(new_table[:, y_pos].X[:, 0])\n    origin_z = min(new_table[:, z_pos].X[:, 0])\n\n    for FOV in new_table.obs_names:\n        new_table[FOV, x_pos] = new_table[FOV, x_pos].X[0, 0] - origin_x\n        new_table[FOV, y_pos] = new_table[FOV, y_pos].X[0, 0] - origin_y\n        new_table[FOV, z_pos] = new_table[FOV, z_pos].X[0, 0] - origin_z\n\n    return new_table\n
"},{"location":"reference/fractal_tasks_core/lib_upscale_array/","title":"lib_upscale_array","text":"

Function to increase the shape of an array by replicating it.

"},{"location":"reference/fractal_tasks_core/lib_upscale_array/#fractal_tasks_core.lib_upscale_array.convert_region_to_low_res","title":"convert_region_to_low_res(*, highres_region, lowres_shape, highres_shape)","text":"

Convert a region defined for a high-resolution array to the corresponding region for a low-resolution array.

PARAMETER DESCRIPTION highres_region

A region of the high-resolution array, defined in a form like (slice(0, 2), slice(1000, 2000), slice(1000, 2000)).

TYPE: tuple[slice, ...]

highres_shape

The shape of the high-resolution array.

TYPE: tuple[int, ...]

lowres_shape

The shape of the low-resolution array.

TYPE: tuple[int, ...]

RETURNS DESCRIPTION tuple[slice, ...]

Region for low-resolution array.

Source code in fractal_tasks_core/lib_upscale_array.py 
def convert_region_to_low_res(\n    *,\n    highres_region: tuple[slice, ...],\n    lowres_shape: tuple[int, ...],\n    highres_shape: tuple[int, ...],\n) -> tuple[slice, ...]:\n\"\"\"\n    Convert a region defined for a high-resolution array to the corresponding\n    region for a low-resolution array.\n\n    Args:\n        highres_region: A region of the high-resolution array, defined in a\n            form like `(slice(0, 2), slice(1000, 2000), slice(1000, 2000))`.\n        highres_shape: The shape of the high-resolution array.\n        lowres_shape: The shape of the low-resolution array.\n\n    Returns:\n        Region for low-resolution array.\n    \"\"\"\n\n    error_msg = (\n        f\"Cannot convert {highres_region=}, \"\n        f\"given {lowres_shape=} and {highres_shape=}.\"\n    )\n\n    ndim = len(lowres_shape)\n    if len(highres_shape) != ndim:\n        raise ValueError(f\"{error_msg} Dimension mismatch.\")\n\n    # Loop over dimensions to construct lowres_region, after some relevant\n    # checks\n    lowres_region = []\n    for ind, lowres_size in enumerate(lowres_shape):\n        # Check that the high-resolution size is not smaller than the\n        # low-resolution size\n        highres_size = highres_shape[ind]\n        if highres_size < lowres_size:\n            raise ValueError(\n                f\"{error_msg} High-res size smaller than low-res size.\"\n            )\n        # Check that sizes are commensurate\n        if highres_size % lowres_size > 0:\n            raise ValueError(\n                f\"{error_msg} Incommensurable sizes \"\n                f\"{highres_size=} and {lowres_size=}.\"\n            )\n        factor = highres_size // lowres_size\n        # Convert old_slice's start/stop attributes\n        old_slice = highres_region[ind]\n        if old_slice.start % factor > 0 or old_slice.stop % factor > 0:\n            raise ValueError(\n                f\"{error_msg} Cannot transform {old_slice=} \"\n                f\"with {factor=}.\"\n            )\n        new_slice_start = old_slice.start // factor\n        new_slice_stop = old_slice.stop // factor\n        new_slice_step = None\n        # Covert old_slice's step attribute\n        if old_slice.step:\n            if old_slice.step % factor > 0:\n                raise ValueError(\n                    f\"{error_msg} Cannot transform {old_slice=} \"\n                    f\"with {factor=}.\"\n                )\n            new_slice_step = old_slice.step // factor\n        # Append new slice\n        lowres_region.append(\n            slice(new_slice_start, new_slice_stop, new_slice_step)\n        )\n\n    return tuple(lowres_region)\n
"},{"location":"reference/fractal_tasks_core/lib_upscale_array/#fractal_tasks_core.lib_upscale_array.upscale_array","title":"upscale_array(*, array, target_shape, axis=None, pad_with_zeros=False, warn_if_inhomogeneous=False)","text":"

Upscale an array along a given list of axis (through repeated application of np.repeat), to match a target shape.

PARAMETER DESCRIPTION array

The array to be upscaled.

TYPE: ndarray

target_shape

The shape of the rescaled array.

TYPE: tuple[int, ...]

axis

The axis along which to upscale the array (if None, then all axis are used).

TYPE: Optional[Sequence[int]] DEFAULT: None

pad_with_zeros

If True, pad the upscaled array with zeros to match target_shape.

TYPE: bool DEFAULT: False

warn_if_inhomogeneous

If True, raise a warning when the conversion factors are not identical across all dimensions.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION ndarray

The upscaled array, with shape target_shape.

Source code in fractal_tasks_core/lib_upscale_array.py 
def upscale_array(\n    *,\n    array: np.ndarray,\n    target_shape: tuple[int, ...],\n    axis: Optional[Sequence[int]] = None,\n    pad_with_zeros: bool = False,\n    warn_if_inhomogeneous: bool = False,\n) -> np.ndarray:\n\"\"\"\n    Upscale an array along a given list of axis (through repeated application\n    of `np.repeat`), to match a target shape.\n\n    Args:\n        array: The array to be upscaled.\n        target_shape: The shape of the rescaled array.\n        axis: The axis along which to upscale the array (if `None`, then all\n            axis are used).\n        pad_with_zeros: If `True`, pad the upscaled array with zeros to match\n            `target_shape`.\n        warn_if_inhomogeneous: If `True`, raise a warning when the conversion\n            factors are not identical across all dimensions.\n\n    Returns:\n        The upscaled array, with shape `target_shape`.\n    \"\"\"\n\n    # Default behavior: use all axis\n    if axis is None:\n        axis = list(range(len(target_shape)))\n\n    array_shape = array.shape\n    info = (\n        f\"Trying to upscale from {array_shape=} to {target_shape=}, \"\n        f\"acting on {axis=}.\"\n    )\n\n    if len(array_shape) != len(target_shape):\n        raise ValueError(f\"{info} Dimensions-number mismatch.\")\n    if axis == []:\n        raise ValueError(f\"{info} Empty axis list\")\n    if min(axis) < 0:\n        raise ValueError(f\"{info} Negative axis specification not allowed.\")\n\n    # Check that upscale is doable\n    for ind, dim in enumerate(array_shape):\n        # Check that array is not larger than target (downscaling)\n        if dim > target_shape[ind]:\n            raise ValueError(\n                f\"{info} {ind}-th array dimension is larger than target.\"\n            )\n        # Check that all relevant axis are included in axis\n        if dim != target_shape[ind] and ind not in axis:\n            raise ValueError(\n                f\"{info} {ind}-th array dimension differs from \"\n                f\"target, but {ind} is not included in \"\n                f\"{axis=}.\"\n            )\n\n    # Compute upscaling factors\n    upscale_factors = {}\n    for ax in axis:\n        if (target_shape[ax] % array_shape[ax]) > 0 and not pad_with_zeros:\n            raise ValueError(\n                \"Incommensurable upscale attempt, \"\n                f\"from {array_shape=} to {target_shape=}.\"\n            )\n        upscale_factors[ax] = target_shape[ax] // array_shape[ax]\n        # Check that this is not downscaling\n        if upscale_factors[ax] < 1:\n            raise ValueError(info)\n    info = f\"{info} Upscale factors: {upscale_factors}\"\n\n    # Raise a warning if upscaling is non-homogeneous across all axis\n    if warn_if_inhomogeneous:\n        if len(set(upscale_factors.values())) > 1:\n            warnings.warn(f\"{info} (inhomogeneous)\")\n\n    # Upscale array, via np.repeat\n    upscaled_array = array\n    for ax in axis:\n        upscaled_array = np.repeat(\n            upscaled_array, upscale_factors[ax], axis=ax\n        )\n\n    # Check that final shape is correct\n    if not upscaled_array.shape == target_shape:\n        if pad_with_zeros:\n            pad_width = []\n            for ax in list(range(len(target_shape))):\n                missing = target_shape[ax] - upscaled_array.shape[ax]\n                if missing < 0 or (missing > 0 and ax not in axis):\n                    raise ValueError(\n                        f\"{info} \" \"Something wrong during zero-padding\"\n                    )\n                pad_width.append([0, missing])\n            upscaled_array = np.pad(\n                upscaled_array,\n                pad_width=pad_width,\n                mode=\"constant\",\n                constant_values=0,\n            )\n            logging.warning(f\"{info} {upscaled_array.shape=}.\")\n            logging.warning(\n                f\"Padding upscaled_array with zeros with {pad_width=}\"\n            )\n        else:\n            raise ValueError(f\"{info} {upscaled_array.shape=}.\")\n\n    return upscaled_array\n
"},{"location":"reference/fractal_tasks_core/lib_write/","title":"lib_write","text":"

Module with custom wrappers of the Zarr API.

"},{"location":"reference/fractal_tasks_core/lib_write/#fractal_tasks_core.lib_write._write_elem_with_overwrite","title":"_write_elem_with_overwrite(group, key, elem, *, overwrite, logger=None)","text":"

Wrap anndata.experimental.write_elem, to include overwrite parameter.

See docs for the original function here.

This function writes elem to the sub-group key of group. The overwrite-related expected behavior is:

  • if the sub-group does not exist, create it (independently on overwrite);
  • if the sub-group already exists and overwrite=True, overwrite the sub-group;
  • if the sub-group already exists and overwrite=False, fail.

Note that this version of the wrapper does not include the original dataset_kwargs parameter.

PARAMETER DESCRIPTION group

The group to write to.

TYPE: Group

key

The key to write to in the group. Note that absolute paths will be written from the root.

TYPE: str

elem

The element to write. Typically an in-memory object, e.g. an AnnData, pandas dataframe, scipy sparse matrix, etc.

TYPE: Any

overwrite

If True, overwrite the key sub-group (if present); if False and key sub-group exists, raise an error.

TYPE: bool

logger

The logger to use (if unset, use logging.getLogger(None))

TYPE: Optional[Logger] DEFAULT: None

RAISES DESCRIPTION OverwriteNotAllowedError

If overwrite=False and the sub-group already exists.

Source code in fractal_tasks_core/lib_write.py 
def _write_elem_with_overwrite(\n    group: zarr.Group,\n    key: str,\n    elem: Any,\n    *,\n    overwrite: bool,\n    logger: Optional[logging.Logger] = None,\n) -> None:\n\"\"\"\n    Wrap `anndata.experimental.write_elem`, to include `overwrite` parameter.\n\n    See docs for the original function\n    [here](https://anndata.readthedocs.io/en/stable/generated/anndata.experimental.write_elem.html).\n\n    This function writes `elem` to the sub-group `key` of `group`. The\n    `overwrite`-related expected behavior is:\n\n    * if the sub-group does not exist, create it (independently on\n      `overwrite`);\n    * if the sub-group already exists and `overwrite=True`, overwrite the\n      sub-group;\n    * if the sub-group already exists and `overwrite=False`, fail.\n\n    Note that this version of the wrapper does not include the original\n    `dataset_kwargs` parameter.\n\n    Args:\n        group:\n            The group to write to.\n        key:\n            The key to write to in the group. Note that absolute paths will be\n            written from the root.\n        elem:\n            The element to write. Typically an in-memory object, e.g. an\n            AnnData, pandas dataframe, scipy sparse matrix, etc.\n        overwrite:\n            If `True`, overwrite the `key` sub-group (if present); if `False`\n            and `key` sub-group exists, raise an error.\n        logger:\n            The logger to use (if unset, use `logging.getLogger(None)`)\n\n    Raises:\n        OverwriteNotAllowedError:\n            If `overwrite=False` and the sub-group already exists.\n    \"\"\"\n\n    # Set logger\n    if logger is None:\n        logger = logging.getLogger(None)\n\n    if key in set(group.group_keys()):\n        if not overwrite:\n            error_msg = (\n                f\"Sub-group '{key}' of group {group.store.path} \"\n                f\"already exists, but `{overwrite=}`.\\n\"\n                \"Hint: try setting `overwrite=True`.\"\n            )\n            logger.error(error_msg)\n            raise OverwriteNotAllowedError(error_msg)\n    write_elem(group, key, elem)\n
"},{"location":"reference/fractal_tasks_core/lib_write/#fractal_tasks_core.lib_write.open_zarr_group_with_overwrite","title":"open_zarr_group_with_overwrite(path, *, overwrite, logger=None, **open_group_kwargs)","text":"

Wrap zarr.open_group and add overwrite argument.

This wrapper sets mode=\"w\" for overwrite=True and mode=\"w-\" for overwrite=False.

The expected behavior is

  • if the group does not exist, create it (independently on overwrite);
  • if the group already exists and overwrite=True, replace the group with an empty one;
  • if the group already exists and overwrite=False, fail.

From the zarr.open_group docs:

  • mode=\"r\" means read only (must exist);
  • mode=\"r+\" means read/write (must exist);
  • mode=\"a\" means read/write (create if doesn\u2019t exist);
  • mode=\"w\" means create (overwrite if exists);
  • mode=\"w-\" means create (fail if exists).
PARAMETER DESCRIPTION path

Store or path to directory in file system or name of zip file (zarr.open_group parameter).

TYPE: Union[str, MutableMapping]

overwrite

Determines the mode parameter of zarr.open_group, which is \"w\" (if overwrite=True) or \"w-\" (if overwrite=False).

TYPE: bool

logger

The logger to use (if unset, use logging.getLogger(None))

TYPE: Optional[Logger] DEFAULT: None

open_group_kwargs

Keyword arguments of zarr.open_group.

TYPE: Any DEFAULT: {}

RETURNS DESCRIPTION Group

The zarr group.

RAISES DESCRIPTION OverwriteNotAllowedError

If overwrite=False and the group already exists.

Source code in fractal_tasks_core/lib_write.py 
def open_zarr_group_with_overwrite(\n    path: Union[str, MutableMapping],\n    *,\n    overwrite: bool,\n    logger: Optional[logging.Logger] = None,\n    **open_group_kwargs: Any,\n) -> zarr.Group:\n\"\"\"\n    Wrap `zarr.open_group` and add `overwrite` argument.\n\n    This wrapper sets `mode=\"w\"` for `overwrite=True` and `mode=\"w-\"` for\n    `overwrite=False`.\n\n    The expected behavior is\n\n\n    * if the group does not exist, create it (independently on `overwrite`);\n    * if the group already exists and `overwrite=True`, replace the group with\n      an empty one;\n    * if the group already exists and `overwrite=False`, fail.\n\n    From the [`zarr.open_group`\n    docs](https://zarr.readthedocs.io/en/stable/api/hierarchy.html#zarr.hierarchy.open_group):\n\n    * `mode=\"r\"` means read only (must exist);\n    * `mode=\"r+\"` means read/write (must exist);\n    * `mode=\"a\"` means read/write (create if doesn\u2019t exist);\n    * `mode=\"w\"` means create (overwrite if exists);\n    * `mode=\"w-\"` means create (fail if exists).\n\n\n    Args:\n        path:\n            Store or path to directory in file system or name of zip file\n            (`zarr.open_group` parameter).\n        overwrite:\n            Determines the `mode` parameter of `zarr.open_group`, which is\n            `\"w\"` (if `overwrite=True`) or `\"w-\"` (if `overwrite=False`).\n        logger:\n            The logger to use (if unset, use `logging.getLogger(None)`)\n        open_group_kwargs:\n            Keyword arguments of `zarr.open_group`.\n\n    Returns:\n        The zarr group.\n\n    Raises:\n        OverwriteNotAllowedError:\n            If `overwrite=False` and the group already exists.\n    \"\"\"\n\n    # Set logger\n    if logger is None:\n        logger = logging.getLogger(None)\n\n    # Set mode for zarr.open_group\n    if overwrite:\n        new_mode = \"w\"\n    else:\n        new_mode = \"w-\"\n\n    # Write log about current status\n    logger.info(f\"Start open_zarr_group_with_overwrite ({overwrite=}).\")\n    try:\n        # Call `zarr.open_group` with `mode=\"r\"`, which fails for missing group\n        current_group = zarr.open_group(path, mode=\"r\")\n        keys = list(current_group.group_keys())\n        logger.info(f\"Zarr group {path} already exists, with {keys=}\")\n    except GroupNotFoundError:\n        logger.info(f\"Zarr group {path} does not exist yet.\")\n\n    # Raise warning if we are overriding an existing value of `mode`\n    if \"mode\" in open_group_kwargs.keys():\n        mode = open_group_kwargs.pop(\"mode\")\n        logger.warning(\n            f\"Overriding {mode=} with {new_mode=}, \"\n            \"in open_zarr_group_with_overwrite\"\n        )\n\n    # Call zarr.open_group\n    try:\n        return zarr.open_group(path, mode=new_mode, **open_group_kwargs)\n    except ContainsGroupError:\n        # Re-raise error with custom message and type\n        error_msg = (\n            f\"Cannot create zarr group at {path=} with `{overwrite=}` \"\n            \"(original error: `zarr.errors.ContainsGroupError`).\\n\"\n            \"Hint: try setting `overwrite=True`.\"\n        )\n        logger.error(error_msg)\n        raise OverwriteNotAllowedError(error_msg)\n
"},{"location":"reference/fractal_tasks_core/lib_write/#fractal_tasks_core.lib_write.prepare_label_group","title":"prepare_label_group(image_group, label_name, overwrite=False, label_attrs=None, logger=None)","text":"

Set the stage for writing labels to a zarr group

This helper function is similar to write_table, in that it prepares the appropriate zarr groups (labels and the new-label one) and performs overwrite-dependent checks. At a difference with write_table, this function does not actually write the label array to the new zarr group; such writing operation must take place in the actual task function, since in fractal-tasks-core it is done sequentially on different regions of the zarr array.

What this function does is:

  1. Create the labels group, if needed.
  2. If overwrite=False, check that the new label does not exist (either in zarr attributes or as a zarr sub-group).
  3. Update the labels attribute of the image group.
  4. If label_attrs is set, include this set of attributes in the new-label zarr group.
PARAMETER DESCRIPTION image_group

The group to write to.

TYPE: Group

label_name

The name of the new label.

TYPE: str

overwrite

If False, check that the new label does not exist (either in zarr attributes or as a zarr sub-group); if True propagate parameter to create_group method, making it overwrite any existing sub-group with the given name.

TYPE: bool DEFAULT: False

label_attrs

If set, overwrite label_group attributes with label_attrs key/value pairs.

TYPE: Optional[dict[str, Any]] DEFAULT: None

logger

The logger to use (if unset, use logging.getLogger(None)).

TYPE: Optional[Logger] DEFAULT: None

RETURNS DESCRIPTION group

Zarr group of the new label.

Source code in fractal_tasks_core/lib_write.py 
def prepare_label_group(\n    image_group: zarr.Group,\n    label_name: str,\n    overwrite: bool = False,\n    label_attrs: Optional[dict[str, Any]] = None,\n    logger: Optional[logging.Logger] = None,\n) -> zarr.group:\n\"\"\"\n    Set the stage for writing labels to a zarr group\n\n    This helper function is similar to `write_table`, in that it prepares the\n    appropriate zarr groups (`labels` and the new-label one) and performs\n    `overwrite`-dependent checks. At a difference with `write_table`, this\n    function does not actually write the label array to the new zarr group;\n    such writing operation must take place in the actual task function, since\n    in fractal-tasks-core it is done sequentially on different `region`s of the\n    zarr array.\n\n    What this function does is:\n\n    1. Create the `labels` group, if needed.\n    2. If `overwrite=False`, check that the new label does not exist (either in\n       zarr attributes or as a zarr sub-group).\n    3. Update the `labels` attribute of the image group.\n    4. If `label_attrs` is set, include this set of attributes in the\n       new-label zarr group.\n\n    Args:\n        image_group:\n            The group to write to.\n        label_name:\n            The name of the new label.\n        overwrite:\n            If `False`, check that the new label does not exist (either in zarr\n            attributes or as a zarr sub-group); if `True` propagate parameter\n            to `create_group` method, making it overwrite any existing\n            sub-group with the given name.\n        label_attrs:\n            If set, overwrite label_group attributes with label_attrs key/value\n            pairs.\n        logger:\n            The logger to use (if unset, use `logging.getLogger(None)`).\n\n    Returns:\n        Zarr group of the new label.\n    \"\"\"\n\n    # Set logger\n    if logger is None:\n        logger = logging.getLogger(None)\n\n    # Create labels group (if needed) and extract current_labels\n    if \"labels\" not in set(image_group.group_keys()):\n        labels_group = image_group.create_group(\"labels\", overwrite=False)\n    else:\n        labels_group = image_group[\"labels\"]\n    current_labels = labels_group.attrs.asdict().get(\"labels\", [])\n\n    # If overwrite=False, check that the new label does not exist (either as a\n    # zarr sub-group or as part of the zarr-group attributes)\n    if not overwrite:\n        if label_name in set(labels_group.group_keys()):\n            error_msg = (\n                f\"Sub-group '{label_name}' of group {image_group.store.path} \"\n                f\"already exists, but `{overwrite=}`.\\n\"\n                \"Hint: try setting `overwrite=True`.\"\n            )\n            logger.error(error_msg)\n            raise OverwriteNotAllowedError(error_msg)\n        if label_name in current_labels:\n            error_msg = (\n                f\"Item '{label_name}' already exists in `labels` attribute of \"\n                f\"group {image_group.store.path}, but `{overwrite=}`.\\n\"\n                \"Hint: try setting `overwrite=True`.\"\n            )\n            logger.error(error_msg)\n            raise OverwriteNotAllowedError(error_msg)\n\n    # Update the `labels` metadata of the image group, if needed\n    if label_name not in current_labels:\n        new_labels = current_labels + [label_name]\n        labels_group.attrs[\"labels\"] = new_labels\n\n    # Define new-label group\n    label_group = labels_group.create_group(label_name, overwrite=overwrite)\n\n    # Optionally update attributes of the new-table zarr group\n    if label_attrs is not None:\n        # Overwrite label_group attributes with label_attrs key/value pairs\n        label_group.attrs.put(label_attrs)\n\n    return label_group\n
"},{"location":"reference/fractal_tasks_core/lib_write/#fractal_tasks_core.lib_write.write_table","title":"write_table(image_group, table_name, table, overwrite=False, table_attrs=None, logger=None)","text":"

Handle multiple options for writing an AnnData table to a zarr group.

  1. Create the tables group, if needed.
  2. If overwrite=False, check that the new table does not exist (either in zarr attributes or as a zarr sub-group).
  3. Call the _write_elem_with_overwrite wrapper with the appropriate overwrite parameter.
  4. Update the tables attribute of the image group.
  5. If table_attrs is set, include this set of attributes in the new-table zarr group. One intended usage, within fractal-tasks-core, is to comply with a proposed change to the OME-NGFF specs (https://github.com/ome/ngff/pull/64).
PARAMETER DESCRIPTION image_group

The group to write to.

TYPE: Group

table_name

The name of the new table.

TYPE: str

table

The AnnData table to write.

TYPE: AnnData

overwrite

If False, check that the new table does not exist (either as a zarr sub-group or as part of the zarr-group attributes). In all cases, propagate parameter to _write_elem_with_overwrite, to determine the behavior in case of an existing sub-group named as table_name.

TYPE: bool DEFAULT: False

table_attrs

If set, overwrite table_group attributes with table_attrs key/value pairs.

TYPE: Optional[dict[str, Any]] DEFAULT: None

logger

The logger to use (if unset, use logging.getLogger(None)).

TYPE: Optional[Logger] DEFAULT: None

RETURNS DESCRIPTION group

Zarr group of the new table.

Source code in fractal_tasks_core/lib_write.py 
def write_table(\n    image_group: zarr.Group,\n    table_name: str,\n    table: ad.AnnData,\n    overwrite: bool = False,\n    table_attrs: Optional[dict[str, Any]] = None,\n    logger: Optional[logging.Logger] = None,\n) -> zarr.group:\n\"\"\"\n    Handle multiple options for writing an AnnData table to a zarr group.\n\n    1. Create the `tables` group, if needed.\n    2. If `overwrite=False`, check that the new table does not exist (either in\n       zarr attributes or as a zarr sub-group).\n    3. Call the `_write_elem_with_overwrite` wrapper with the appropriate\n       `overwrite` parameter.\n    4. Update the `tables` attribute of the image group.\n    5. If `table_attrs` is set, include this set of attributes in the\n       new-table zarr group. One intended usage, within fractal-tasks-core, is\n       to comply with a proposed change to the OME-NGFF specs\n       (https://github.com/ome/ngff/pull/64).\n\n    Args:\n        image_group:\n            The group to write to.\n        table_name:\n            The name of the new table.\n        table:\n            The AnnData table to write.\n        overwrite:\n            If `False`, check that the new table does not exist (either as a\n            zarr sub-group or as part of the zarr-group attributes). In all\n            cases, propagate parameter to `_write_elem_with_overwrite`, to\n            determine the behavior in case of an existing sub-group named as\n            `table_name`.\n        table_attrs:\n            If set, overwrite table_group attributes with table_attrs key/value\n            pairs.\n        logger:\n            The logger to use (if unset, use `logging.getLogger(None)`).\n\n    Returns:\n        Zarr group of the new table.\n    \"\"\"\n\n    # Set logger\n    if logger is None:\n        logger = logging.getLogger(None)\n\n    # Create tables group (if needed) and extract current_tables\n    if \"tables\" not in set(image_group.group_keys()):\n        tables_group = image_group.create_group(\"tables\", overwrite=False)\n    else:\n        tables_group = image_group[\"tables\"]\n    current_tables = tables_group.attrs.asdict().get(\"tables\", [])\n\n    # If overwrite=False, check that the new table does not exist (either as a\n    # zarr sub-group or as part of the zarr-group attributes)\n    if not overwrite:\n        if table_name in set(tables_group.group_keys()):\n            error_msg = (\n                f\"Sub-group '{table_name}' of group {image_group.store.path} \"\n                f\"already exists, but `{overwrite=}`.\\n\"\n                \"Hint: try setting `overwrite=True`.\"\n            )\n            logger.error(error_msg)\n            raise OverwriteNotAllowedError(error_msg)\n        if table_name in current_tables:\n            error_msg = (\n                f\"Item '{table_name}' already exists in `tables` attribute of \"\n                f\"group {image_group.store.path}, but `{overwrite=}`.\\n\"\n                \"Hint: try setting `overwrite=True`.\"\n            )\n            logger.error(error_msg)\n            raise OverwriteNotAllowedError(error_msg)\n\n    # If it's all OK, proceed and write the table\n    _write_elem_with_overwrite(\n        tables_group,\n        table_name,\n        table,\n        overwrite=overwrite,\n    )\n    table_group = tables_group[table_name]\n\n    # Update the `tables` metadata of the image group, if needed\n    if table_name not in current_tables:\n        new_tables = current_tables + [table_name]\n        tables_group.attrs[\"tables\"] = new_tables\n\n    # Optionally update attributes of the new-table zarr group\n    if table_attrs is not None:\n        if table_attrs.get(\"type\") == \"ngff:region_table\":\n            # Verify whether we comply with a proposed change to the OME-NGFF\n            # table specs (https://github.com/ome/ngff/pull/64)\n            try:\n                table_attrs[\"instance_key\"]\n                table_attrs[\"region\"][\"path\"]\n            except KeyError as e:\n                logger.warning(\n                    f\"The table_attrs parameter of write_elem has \"\n                    \"type='ngff:region_table' but does not comply with the \"\n                    \"proposed table specs. \"\n                    f\"Original error: KeyError: {str(e)}\"\n                )\n        # Update table_group attributes with table_attrs key/value pairs\n        table_group.attrs.update(**table_attrs)\n\n    return table_group\n
"},{"location":"reference/fractal_tasks_core/lib_zattrs_utils/","title":"lib_zattrs_utils","text":"

Helper functions for operations on OME-NGFF metadata.

"},{"location":"reference/fractal_tasks_core/lib_zattrs_utils/#fractal_tasks_core.lib_zattrs_utils.get_table_path_dict","title":"get_table_path_dict(input_path, component)","text":"

Compile dictionary of (table name, table path) key/value pairs.

PARAMETER DESCRIPTION input_path

Path to the parent folder of a plate zarr group (e.g. /some/path/).

TYPE: Path

component

Path (relative to input_path) to an image zarr group (e.g. plate.zarr/B/03/0).

TYPE: str

RETURNS DESCRIPTION dict[str, str]

Dictionary with table names as keys and table paths as values. If tables Zarr group is missing, or if it does not have a tables key, then return an empty dictionary.

Source code in fractal_tasks_core/lib_zattrs_utils.py 
def get_table_path_dict(input_path: Path, component: str) -> dict[str, str]:\n\"\"\"\n    Compile dictionary of (table name, table path) key/value pairs.\n\n\n    Args:\n        input_path:\n            Path to the parent folder of a plate zarr group (e.g.\n            `/some/path/`).\n        component:\n            Path (relative to `input_path`) to an image zarr group (e.g.\n            `plate.zarr/B/03/0`).\n\n    Returns:\n        Dictionary with table names as keys and table paths as values. If\n            `tables` Zarr group is missing, or if it does not have a `tables`\n            key, then return an empty dictionary.\n    \"\"\"\n\n    try:\n        tables_group = zarr.open_group(f\"{input_path / component}/tables\", \"r\")\n        table_list = tables_group.attrs[\"tables\"]\n    except (zarr.errors.GroupNotFoundError, KeyError):\n        table_list = []\n\n    table_path_dict = {}\n    for table in table_list:\n        table_path_dict[table] = f\"{input_path / component}/tables/{table}\"\n\n    return table_path_dict\n
"},{"location":"reference/fractal_tasks_core/lib_zattrs_utils/#fractal_tasks_core.lib_zattrs_utils.rescale_datasets","title":"rescale_datasets(*, datasets, coarsening_xy, reference_level, remove_channel_axis=False)","text":"

Given a set of datasets (as per OME-NGFF specs), update their \"scale\" transformations in the YX directions by including a prefactor (coarsening_xy**reference_level).

PARAMETER DESCRIPTION datasets

list of datasets (as per OME-NGFF specs).

TYPE: list[dict]

coarsening_xy

linear coarsening factor between subsequent levels.

TYPE: int

reference_level

TBD

TYPE: int

remove_channel_axis

If True, remove the first item of all scale transformations.

TYPE: bool DEFAULT: False

Source code in fractal_tasks_core/lib_zattrs_utils.py 
def rescale_datasets(\n    *,\n    datasets: list[dict],\n    coarsening_xy: int,\n    reference_level: int,\n    remove_channel_axis: bool = False,\n) -> list[dict]:\n\"\"\"\n    Given a set of datasets (as per OME-NGFF specs), update their \"scale\"\n    transformations in the YX directions by including a prefactor\n    (coarsening_xy**reference_level).\n\n    Args:\n        datasets: list of datasets (as per OME-NGFF specs).\n        coarsening_xy: linear coarsening factor between subsequent levels.\n        reference_level: TBD\n        remove_channel_axis: If `True`, remove the first item of all `scale`\n            transformations.\n    \"\"\"\n\n    # Construct rescaled datasets\n    new_datasets = []\n    for ds in datasets:\n        new_ds = {}\n\n        # Copy all keys that are not coordinateTransformations (e.g. path)\n        for key in ds.keys():\n            if key != \"coordinateTransformations\":\n                new_ds[key] = ds[key]\n\n        # Update coordinateTransformations\n        old_transformations = ds[\"coordinateTransformations\"]\n        new_transformations = []\n        for t in old_transformations:\n            if t[\"type\"] == \"scale\":\n                new_t: dict[str, Any] = t.copy()\n                # Rescale last two dimensions (that is, Y and X)\n                prefactor = coarsening_xy**reference_level\n                new_t[\"scale\"][-2] = new_t[\"scale\"][-2] * prefactor\n                new_t[\"scale\"][-1] = new_t[\"scale\"][-1] * prefactor\n                if remove_channel_axis:\n                    new_t[\"scale\"].pop(0)\n                new_transformations.append(new_t)\n            else:\n                new_transformations.append(t)\n        new_ds[\"coordinateTransformations\"] = new_transformations\n        new_datasets.append(new_ds)\n\n    return new_datasets\n
"},{"location":"reference/fractal_tasks_core/dev/","title":"dev","text":"

Development-tools subpackage (e.g. for the creation of JSON Schemas for task parameters).

"},{"location":"reference/fractal_tasks_core/dev/check_manifest/","title":"check_manifest","text":"

Script to check that JSON schemas for task arguments (as reported in the package manfest) are up-to-date.

"},{"location":"reference/fractal_tasks_core/dev/check_manifest/#fractal_tasks_core.dev.check_manifest._compare_dicts","title":"_compare_dicts(old, new, path=[])","text":"

Provide more informative comparison of two (possibly nested) dictionaries.

PARAMETER DESCRIPTION old

TBD

TYPE: dict[str, Any]

new

TBD

TYPE: dict[str, Any]

path

TBD

TYPE: list[str] DEFAULT: []

Source code in fractal_tasks_core/dev/check_manifest.py 
def _compare_dicts(\n    old: dict[str, Any], new: dict[str, Any], path: list[str] = []\n):\n\"\"\"\n    Provide more informative comparison of two (possibly nested) dictionaries.\n\n    Args:\n        old: TBD\n        new: TBD\n        path: TBD\n    \"\"\"\n    path_str = \"/\".join(path)\n    keys_old = set(old.keys())\n    keys_new = set(new.keys())\n    if not keys_old == keys_new:\n        msg = (\n            \"\\n\\n\"\n            f\"Dictionaries at path {path_str} have different keys:\\n\\n\"\n            f\"OLD KEYS:\\n{keys_old}\\n\\n\"\n            f\"NEW KEYS:\\n{keys_new}\\n\\n\"\n        )\n        raise ValueError(msg)\n    for key, value_old in old.items():\n        value_new = new[key]\n        if type(value_old) != type(value_new):\n            msg = (\n                \"\\n\\n\"\n                f\"Values at path {path_str}/{key} \"\n                \"have different types:\\n\\n\"\n                f\"OLD TYPE:\\n{type(value_old)}\\n\\n\"\n                f\"NEW TYPE:\\n{type(value_new)}\\n\\n\"\n            )\n            raise ValueError(msg)\n        if isinstance(value_old, dict):\n            _compare_dicts(value_old, value_new, path=path + [key])\n        else:\n            if value_old != value_new:\n                msg = (\n                    \"\\n\\n\"\n                    f\"Values at path {path_str}/{key} \"\n                    \"are different:\\n\\n\"\n                    f\"OLD VALUE:\\n{value_old}\\n\\n\"\n                    f\"NEW VALUE:\\n{value_new}\\n\\n\"\n                )\n                raise ValueError(msg)\n
"},{"location":"reference/fractal_tasks_core/dev/lib_args_schemas/","title":"lib_args_schemas","text":"

Helper functions to handle JSON schemas for task arguments.

"},{"location":"reference/fractal_tasks_core/dev/lib_args_schemas/#fractal_tasks_core.dev.lib_args_schemas._remove_args_kwargs_properties","title":"_remove_args_kwargs_properties(old_schema)","text":"

Remove args and kwargs schema properties.

Pydantic v1 automatically includes args and kwargs properties in JSON Schemas generated via ValidatedFunction(task_function, config=None).model.schema(), with some default (empty) values -- see see https://github.com/pydantic/pydantic/blob/1.10.X-fixes/pydantic/decorator.py.

Verify that these properties match with their expected default values, and then remove them from the schema.

PARAMETER DESCRIPTION old_schema

TBD

TYPE: _Schema

Source code in fractal_tasks_core/dev/lib_args_schemas.py 
def _remove_args_kwargs_properties(old_schema: _Schema) -> _Schema:\n\"\"\"\n    Remove `args` and `kwargs` schema properties.\n\n    Pydantic v1 automatically includes `args` and `kwargs` properties in\n    JSON Schemas generated via `ValidatedFunction(task_function,\n    config=None).model.schema()`, with some default (empty) values -- see see\n    https://github.com/pydantic/pydantic/blob/1.10.X-fixes/pydantic/decorator.py.\n\n    Verify that these properties match with their expected default values, and\n    then remove them from the schema.\n\n    Args:\n        old_schema: TBD\n    \"\"\"\n    new_schema = old_schema.copy()\n    args_property = new_schema[\"properties\"].pop(\"args\")\n    kwargs_property = new_schema[\"properties\"].pop(\"kwargs\")\n    expected_args_property = {\"title\": \"Args\", \"type\": \"array\", \"items\": {}}\n    expected_kwargs_property = {\"title\": \"Kwargs\", \"type\": \"object\"}\n    if args_property != expected_args_property:\n        raise ValueError(\n            f\"{args_property=}\\ndiffers from\\n{expected_args_property=}\"\n        )\n    if kwargs_property != expected_kwargs_property:\n        raise ValueError(\n            f\"{kwargs_property=}\\ndiffers from\\n\"\n            f\"{expected_kwargs_property=}\"\n        )\n    logging.info(\"[_remove_args_kwargs_properties] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_args_schemas/#fractal_tasks_core.dev.lib_args_schemas._remove_attributes_from_descriptions","title":"_remove_attributes_from_descriptions(old_schema)","text":"

Keeps only the description part of the docstrings: e.g from 

'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'\n'\\n'\n'Attributes:\\n'\n'min: Do not change. It will be set to `0` by default.\\n'\n'max: Do not change. It will be set according to bitdepth of the images\\n'\n'    by default (e.g. 65535 for 16 bit images).\\n'\n'start: Lower-bound rescaling value for visualization.\\n'\n'end: Upper-bound rescaling value for visualization.'\n
to 'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'.

PARAMETER DESCRIPTION old_schema

TBD

TYPE: _Schema

Source code in fractal_tasks_core/dev/lib_args_schemas.py 
def _remove_attributes_from_descriptions(old_schema: _Schema) -> _Schema:\n\"\"\"\n    Keeps only the description part of the docstrings: e.g from\n    ```\n    'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\\\n'\n    '\\\\n'\n    'Attributes:\\\\n'\n    'min: Do not change. It will be set to `0` by default.\\\\n'\n    'max: Do not change. It will be set according to bitdepth of the images\\\\n'\n    '    by default (e.g. 65535 for 16 bit images).\\\\n'\n    'start: Lower-bound rescaling value for visualization.\\\\n'\n    'end: Upper-bound rescaling value for visualization.'\n    ```\n    to `'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\\\n'`.\n\n    Args:\n        old_schema: TBD\n    \"\"\"\n    new_schema = old_schema.copy()\n    if \"definitions\" in new_schema:\n        for name, definition in new_schema[\"definitions\"].items():\n            parsed_docstring = docparse(definition[\"description\"])\n            new_schema[\"definitions\"][name][\n                \"description\"\n            ] = parsed_docstring.short_description\n    logging.info(\"[_remove_attributes_from_descriptions] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_args_schemas/#fractal_tasks_core.dev.lib_args_schemas._remove_pydantic_internals","title":"_remove_pydantic_internals(old_schema)","text":"

Remove schema properties that are only used internally by Pydantic V1.

PARAMETER DESCRIPTION old_schema

TBD

TYPE: _Schema

Source code in fractal_tasks_core/dev/lib_args_schemas.py 
def _remove_pydantic_internals(old_schema: _Schema) -> _Schema:\n\"\"\"\n    Remove schema properties that are only used internally by Pydantic V1.\n\n    Args:\n        old_schema: TBD\n    \"\"\"\n    new_schema = old_schema.copy()\n    for key in (\n        V_POSITIONAL_ONLY_NAME,\n        V_DUPLICATE_KWARGS,\n        ALT_V_ARGS,\n        ALT_V_KWARGS,\n    ):\n        new_schema[\"properties\"].pop(key, None)\n    logging.info(\"[_remove_pydantic_internals] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_args_schemas/#fractal_tasks_core.dev.lib_args_schemas.create_schema_for_single_task","title":"create_schema_for_single_task(executable, package='fractal_tasks_core', custom_pydantic_models=None)","text":"

Main function to create a JSON Schema of task arguments

Source code in fractal_tasks_core/dev/lib_args_schemas.py 
def create_schema_for_single_task(\n    executable: str,\n    package: str = \"fractal_tasks_core\",\n    custom_pydantic_models: Optional[list[tuple[str, str, str]]] = None,\n) -> _Schema:\n\"\"\"\n    Main function to create a JSON Schema of task arguments\n    \"\"\"\n\n    logging.info(\"[create_schema_for_single_task] START\")\n\n    # Extract the function name. Note: this could be made more general, but for\n    # the moment we assume the function has the same name as the module)\n    function_name = Path(executable).with_suffix(\"\").name\n    logging.info(f\"[create_schema_for_single_task] {function_name=}\")\n\n    # Extract function from module\n    task_function = _extract_function(\n        package_name=package,\n        module_relative_path=executable,\n        function_name=function_name,\n    )\n\n    logging.info(f\"[create_schema_for_single_task] {task_function=}\")\n\n    # Validate function signature against some custom constraints\n    _validate_function_signature(task_function)\n\n    # Create and clean up schema\n    vf = ValidatedFunction(task_function, config=None)\n    schema = vf.model.schema()\n    schema = _remove_args_kwargs_properties(schema)\n    schema = _remove_pydantic_internals(schema)\n    schema = _remove_attributes_from_descriptions(schema)\n\n    # Include titles for custom-model-typed arguments\n    schema = _include_titles(schema)\n\n    # Include descriptions of function arguments\n    function_args_descriptions = _get_function_args_descriptions(\n        package_name=package,\n        module_relative_path=executable,\n        function_name=function_name,\n    )\n    schema = _insert_function_args_descriptions(\n        schema=schema, descriptions=function_args_descriptions\n    )\n\n    # Merge lists of fractal-tasks-core and user-provided Pydantic models\n    user_provided_models = custom_pydantic_models or []\n    pydantic_models = FRACTAL_TASKS_CORE_PYDANTIC_MODELS + user_provided_models\n\n    # Check that model names are unique\n    pydantic_models_names = [item[2] for item in pydantic_models]\n    duplicate_class_names = [\n        name\n        for name, count in Counter(pydantic_models_names).items()\n        if count > 1\n    ]\n    if duplicate_class_names:\n        pydantic_models_str = \"  \" + \"\\n  \".join(map(str, pydantic_models))\n        raise ValueError(\n            \"Cannot parse docstrings for models with non-unique names \"\n            f\"{duplicate_class_names}, in\\n{pydantic_models_str}\"\n        )\n\n    # Extract model-attribute descriptions and insert them into schema\n    for package_name, module_relative_path, class_name in pydantic_models:\n        attrs_descriptions = _get_class_attrs_descriptions(\n            package_name=package_name,\n            module_relative_path=module_relative_path,\n            class_name=class_name,\n        )\n        schema = _insert_class_attrs_descriptions(\n            schema=schema,\n            class_name=class_name,\n            descriptions=attrs_descriptions,\n        )\n\n    logging.info(\"[create_schema_for_single_task] END\")\n    return schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/","title":"lib_descriptions","text":""},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._get_class_attrs_descriptions","title":"_get_class_attrs_descriptions(package_name, module_relative_path, class_name)","text":"

Extract attribute descriptions from a class.

PARAMETER DESCRIPTION package_name

Example fractal_tasks_core.

TYPE: str

module_relative_path

Example lib_channels.py.

TYPE: str

class_name

Example OmeroChannel.

TYPE: str

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _get_class_attrs_descriptions(\n    package_name: str, module_relative_path: str, class_name: str\n) -> dict[str, str]:\n\"\"\"\n    Extract attribute descriptions from a class.\n\n    Args:\n        package_name: Example `fractal_tasks_core`.\n        module_relative_path: Example `lib_channels.py`.\n        class_name: Example `OmeroChannel`.\n    \"\"\"\n\n    if not module_relative_path.endswith(\".py\"):\n        raise ValueError(f\"Module {module_relative_path} must end with '.py'\")\n\n    # Get the class ast.ClassDef object\n    package_path = Path(import_module(package_name).__file__).parent\n    module_path = package_path / module_relative_path\n    tree = ast.parse(module_path.read_text())\n    try:\n        _class = next(\n            c\n            for c in ast.walk(tree)\n            if (isinstance(c, ast.ClassDef) and c.name == class_name)\n        )\n    except StopIteration:\n        raise RuntimeError(\n            f\"Cannot find {class_name=} for {package_name=} \"\n            f\"and {module_relative_path=}\"\n        )\n    docstring = ast.get_docstring(_class)\n    parsed_docstring = docparse(docstring)\n    descriptions = {\n        x.arg_name: _sanitize_description(x.description)\n        if x.description\n        else \"Missing description\"\n        for x in parsed_docstring.params\n    }\n    logging.info(f\"[_get_class_attrs_descriptions] END ({class_name=})\")\n    return descriptions\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._get_function_args_descriptions","title":"_get_function_args_descriptions(package_name, module_relative_path, function_name)","text":"

Extract argument descriptions from a function.

PARAMETER DESCRIPTION package_name

Example fractal_tasks_core.

TYPE: str

module_relative_path

Example tasks/create_ome_zarr.py.

TYPE: str

function_name

Example create_ome_zarr.

TYPE: str

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _get_function_args_descriptions(\n    package_name: str, module_relative_path: str, function_name: str\n) -> dict[str, str]:\n\"\"\"\n    Extract argument descriptions from a function.\n\n    Args:\n        package_name: Example `fractal_tasks_core`.\n        module_relative_path: Example `tasks/create_ome_zarr.py`.\n        function_name: Example `create_ome_zarr`.\n    \"\"\"\n\n    # Extract docstring from ast.FunctionDef\n    docstring = _get_function_docstring(\n        package_name, module_relative_path, function_name\n    )\n\n    # Parse docstring (via docstring_parser) and prepare output\n    parsed_docstring = docparse(docstring)\n    descriptions = {\n        param.arg_name: _sanitize_description(param.description)\n        for param in parsed_docstring.params\n    }\n    logging.info(f\"[_get_function_args_descriptions] END ({function_name=})\")\n    return descriptions\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._get_function_docstring","title":"_get_function_docstring(package_name, module_relative_path, function_name)","text":"

Extract docstring from a function.

PARAMETER DESCRIPTION package_name

Example fractal_tasks_core.

TYPE: str

module_relative_path

Example tasks/create_ome_zarr.py.

TYPE: str

function_name

Example create_ome_zarr.

TYPE: str

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _get_function_docstring(\n    package_name: str, module_relative_path: str, function_name: str\n) -> str:\n\"\"\"\n    Extract docstring from a function.\n\n    Args:\n        package_name: Example `fractal_tasks_core`.\n        module_relative_path: Example `tasks/create_ome_zarr.py`.\n        function_name: Example `create_ome_zarr`.\n    \"\"\"\n\n    if not module_relative_path.endswith(\".py\"):\n        raise ValueError(f\"Module {module_relative_path} must end with '.py'\")\n\n    # Get the function ast.FunctionDef object\n    package_path = Path(import_module(package_name).__file__).parent\n    module_path = package_path / module_relative_path\n    tree = ast.parse(module_path.read_text())\n    _function = next(\n        f\n        for f in ast.walk(tree)\n        if (isinstance(f, ast.FunctionDef) and f.name == function_name)\n    )\n\n    # Extract docstring from ast.FunctionDef\n    return ast.get_docstring(_function)\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._insert_class_attrs_descriptions","title":"_insert_class_attrs_descriptions(*, schema, class_name, descriptions)","text":"

Merge the descriptions obtained via _get_attributes_models_descriptions into the class_name definition, within an existing JSON Schema

PARAMETER DESCRIPTION schema

TBD

TYPE: dict

class_name

TBD

TYPE: str

descriptions

TBD

TYPE: dict

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _insert_class_attrs_descriptions(\n    *, schema: dict, class_name: str, descriptions: dict\n):\n\"\"\"\n    Merge the descriptions obtained via `_get_attributes_models_descriptions`\n    into the `class_name` definition, within an existing JSON Schema\n\n    Args:\n        schema: TBD\n        class_name: TBD\n        descriptions: TBD\n    \"\"\"\n    new_schema = schema.copy()\n    if \"definitions\" not in schema:\n        return new_schema\n    else:\n        new_definitions = schema[\"definitions\"].copy()\n    # Loop over existing definitions\n    for name, definition in schema[\"definitions\"].items():\n        if name == class_name:\n            for prop in definition[\"properties\"]:\n                if \"description\" in new_definitions[name][\"properties\"][prop]:\n                    raise ValueError(\n                        f\"Property {name}.{prop} already has description\"\n                    )\n                else:\n                    new_definitions[name][\"properties\"][prop][\n                        \"description\"\n                    ] = descriptions[prop]\n    new_schema[\"definitions\"] = new_definitions\n    logging.info(\"[_insert_class_attrs_descriptions] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._insert_function_args_descriptions","title":"_insert_function_args_descriptions(*, schema, descriptions)","text":"

Merge the descriptions obtained via _get_args_descriptions into the properties of an existing JSON Schema.

PARAMETER DESCRIPTION schema

TBD

TYPE: dict

descriptions

TBD

TYPE: dict

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _insert_function_args_descriptions(*, schema: dict, descriptions: dict):\n\"\"\"\n    Merge the descriptions obtained via `_get_args_descriptions` into the\n    properties of an existing JSON Schema.\n\n    Args:\n        schema: TBD\n        descriptions: TBD\n    \"\"\"\n    new_schema = schema.copy()\n    new_properties = schema[\"properties\"].copy()\n    for key, value in schema[\"properties\"].items():\n        if \"description\" in value:\n            raise ValueError(\"Property already has description\")\n        else:\n            if key in descriptions:\n                value[\"description\"] = descriptions[key]\n            else:\n                value[\"description\"] = \"Missing description\"\n            new_properties[key] = value\n    new_schema[\"properties\"] = new_properties\n    logging.info(\"[_insert_function_args_descriptions] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_descriptions/#fractal_tasks_core.dev.lib_descriptions._sanitize_description","title":"_sanitize_description(string)","text":"

Sanitize a description string.

This is a provisional helper function that replaces newlines with spaces and reduces multiple contiguous whitespace characters to a single one. Future iterations of the docstrings format/parsing may render this function not-needed or obsolete.

PARAMETER DESCRIPTION string

TBD

TYPE: str

Source code in fractal_tasks_core/dev/lib_descriptions.py 
def _sanitize_description(string: str) -> str:\n\"\"\"\n    Sanitize a description string.\n\n    This is a provisional helper function that replaces newlines with spaces\n    and reduces multiple contiguous whitespace characters to a single one.\n    Future iterations of the docstrings format/parsing may render this function\n    not-needed or obsolete.\n\n    Args:\n        string: TBD\n    \"\"\"\n    # Replace newline with space\n    new_string = string.replace(\"\\n\", \" \")\n    # Replace N-whitespace characterss with a single one\n    while \"  \" in new_string:\n        new_string = new_string.replace(\"  \", \" \")\n    return new_string\n
"},{"location":"reference/fractal_tasks_core/dev/lib_signature_constraints/","title":"lib_signature_constraints","text":""},{"location":"reference/fractal_tasks_core/dev/lib_signature_constraints/#fractal_tasks_core.dev.lib_signature_constraints._extract_function","title":"_extract_function(module_relative_path, function_name, package_name='fractal_tasks_core')","text":"

Extract function from a module with the same name.

PARAMETER DESCRIPTION package_name

Example fractal_tasks_core.

TYPE: str DEFAULT: 'fractal_tasks_core'

module_relative_path

Example tasks/create_ome_zarr.py.

TYPE: str

function_name

Example create_ome_zarr.

TYPE: str

Source code in fractal_tasks_core/dev/lib_signature_constraints.py 
def _extract_function(\n    module_relative_path: str,\n    function_name: str,\n    package_name: str = \"fractal_tasks_core\",\n) -> Callable:\n\"\"\"\n    Extract function from a module with the same name.\n\n    Args:\n        package_name: Example `fractal_tasks_core`.\n        module_relative_path: Example `tasks/create_ome_zarr.py`.\n        function_name: Example `create_ome_zarr`.\n    \"\"\"\n    if not module_relative_path.endswith(\".py\"):\n        raise ValueError(f\"{module_relative_path=} must end with '.py'\")\n    module_relative_path_no_py = str(\n        Path(module_relative_path).with_suffix(\"\")\n    )\n    module_relative_path_dots = module_relative_path_no_py.replace(\"/\", \".\")\n    module = import_module(f\"{package_name}.{module_relative_path_dots}\")\n    task_function = getattr(module, function_name)\n    return task_function\n
"},{"location":"reference/fractal_tasks_core/dev/lib_signature_constraints/#fractal_tasks_core.dev.lib_signature_constraints._validate_function_signature","title":"_validate_function_signature(function)","text":"

Validate the function signature.

Implement a set of checks for type hints that do not play well with the creation of JSON Schema, see https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/399.

PARAMETER DESCRIPTION function

TBD

TYPE: Callable

Source code in fractal_tasks_core/dev/lib_signature_constraints.py 
def _validate_function_signature(function: Callable):\n\"\"\"\n    Validate the function signature.\n\n    Implement a set of checks for type hints that do not play well with the\n    creation of JSON Schema, see\n    https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/399.\n\n    Args:\n        function: TBD\n    \"\"\"\n    sig = signature(function)\n    for param in sig.parameters.values():\n\n        # CASE 1: Check that name is not forbidden\n        if param.name in FORBIDDEN_PARAM_NAMES:\n            raise ValueError(\n                f\"Function {function} has argument with name {param.name}\"\n            )\n\n        # CASE 2: Raise an error for unions\n        if str(param.annotation).startswith((\"typing.Union[\", \"Union[\")):\n            raise ValueError(\"typing.Union is not supported\")\n\n        # CASE 3: Raise an error for \"|\"\n        if \"|\" in str(param.annotation):\n            raise ValueError('Use of \"|\" in type hints is not supported')\n\n        # CASE 4: Raise an error for optional parameter with given (non-None)\n        # default, e.g. Optional[str] = \"asd\"\n        is_annotation_optional = str(param.annotation).startswith(\n            (\"typing.Optional[\", \"Optional[\")\n        )\n        default_given = (param.default is not None) and (\n            param.default != inspect._empty\n        )\n        if default_given and is_annotation_optional:\n            raise ValueError(\"Optional parameter has non-None default value\")\n\n    logging.info(\"[_validate_function_signature] END\")\n    return sig\n
"},{"location":"reference/fractal_tasks_core/dev/lib_task_docs/","title":"lib_task_docs","text":""},{"location":"reference/fractal_tasks_core/dev/lib_task_docs/#fractal_tasks_core.dev.lib_task_docs._get_function_description","title":"_get_function_description(package_name, module_relative_path, function_name)","text":"

Extract function description from its docstring.

PARAMETER DESCRIPTION package_name

Example fractal_tasks_core.

TYPE: str

module_relative_path

Example tasks/create_ome_zarr.py.

TYPE: str

function_name

Example create_ome_zarr.

TYPE: str

Source code in fractal_tasks_core/dev/lib_task_docs.py 
def _get_function_description(\n    package_name: str, module_relative_path: str, function_name: str\n) -> str:\n\"\"\"\n    Extract function description from its docstring.\n\n    Args:\n        package_name: Example `fractal_tasks_core`.\n        module_relative_path: Example `tasks/create_ome_zarr.py`.\n        function_name: Example `create_ome_zarr`.\n    \"\"\"\n    # Extract docstring from ast.FunctionDef\n    docstring = _get_function_docstring(\n        package_name, module_relative_path, function_name\n    )\n    # Parse docstring (via docstring_parser)\n    parsed_docstring = docparse(docstring)\n    # Combine short/long descriptions (if present)\n    short_description = parsed_docstring.short_description\n    long_description = parsed_docstring.long_description\n    items = []\n    if short_description:\n        items.append(short_description)\n    if long_description:\n        items.append(long_description)\n    if items:\n        if parsed_docstring.blank_after_short_description:\n            return \"\\n\\n\".join(items)\n        else:\n            return \"\\n\".join(items)\n    else:\n        return \"\"\n
"},{"location":"reference/fractal_tasks_core/dev/lib_task_docs/#fractal_tasks_core.dev.lib_task_docs.create_docs_info","title":"create_docs_info(executable, package='fractal_tasks_core')","text":"

Return task description based on function docstring.

Source code in fractal_tasks_core/dev/lib_task_docs.py 
def create_docs_info(\n    executable: str,\n    package: str = \"fractal_tasks_core\",\n) -> str:\n\"\"\"\n    Return task description based on function docstring.\n    \"\"\"\n    logging.info(\"[create_docs_info] START\")\n    # Extract the function name. Note: this could be made more general, but for\n    # the moment we assume the function has the same name as the module)\n    function_name = Path(executable).with_suffix(\"\").name\n    logging.info(f\"[create_docs_info] {function_name=}\")\n    # Get function description\n    docs_info = _get_function_description(\n        package_name=package,\n        module_relative_path=executable,\n        function_name=function_name,\n    )\n    logging.info(\"[create_docs_info] END\")\n    return docs_info\n
"},{"location":"reference/fractal_tasks_core/dev/lib_task_docs/#fractal_tasks_core.dev.lib_task_docs.create_docs_link","title":"create_docs_link(executable)","text":"

Return link to docs page for a fractal_tasks_core task.

Source code in fractal_tasks_core/dev/lib_task_docs.py 
def create_docs_link(executable: str) -> str:\n\"\"\"\n    Return link to docs page for a fractal_tasks_core task.\n    \"\"\"\n    logging.info(\"[create_docs_link] START\")\n\n    # Extract the function name. Note: this could be made more general, but for\n    # the moment we assume the function has the same name as the module)\n    function_name = Path(executable).with_suffix(\"\").name\n    logging.info(f\"[create_docs_link] {function_name=}\")\n    # Define docs_link\n    docs_link = (\n        \"https://fractal-analytics-platform.github.io/fractal-tasks-core/\"\n        f\"reference/fractal_tasks_core/tasks/{function_name}/\"\n        f\"#fractal_tasks_core.tasks.{function_name}.{function_name}\"\n    )\n    logging.info(\"[create_docs_link] END\")\n    return docs_link\n
"},{"location":"reference/fractal_tasks_core/dev/lib_titles/","title":"lib_titles","text":"

Module to include titles in JSON Schema properties.

"},{"location":"reference/fractal_tasks_core/dev/lib_titles/#fractal_tasks_core.dev.lib_titles._include_titles","title":"_include_titles(schema)","text":"

Include property titles, when missing.

This handles both:

  • first-level JSON Schema properties (corresponding to task arguments);
  • properties of JSON Schema definitions (corresponding to task-argument attributes).
PARAMETER DESCRIPTION schema

TBD

TYPE: _Schema

Source code in fractal_tasks_core/dev/lib_titles.py 
def _include_titles(schema: _Schema) -> _Schema:\n\"\"\"\n    Include property titles, when missing.\n\n    This handles both:\n\n    - first-level JSON Schema properties (corresponding to task\n        arguments);\n    - properties of JSON Schema definitions (corresponding to\n        task-argument attributes).\n\n    Args:\n        schema: TBD\n    \"\"\"\n    new_schema = schema.copy()\n\n    # Update first-level properties (that is, task arguments)\n    new_properties = _include_titles_for_properties(schema[\"properties\"])\n    new_schema[\"properties\"] = new_properties\n\n    # Update properties of definitions\n    if \"definitions\" in schema.keys():\n        new_definitions = schema[\"definitions\"].copy()\n        for def_name, def_schema in new_definitions.items():\n            new_properties = _include_titles_for_properties(\n                def_schema[\"properties\"]\n            )\n            new_definitions[def_name][\"properties\"] = new_properties\n        new_schema[\"definitions\"] = new_definitions\n\n    logging.info(\"[_include_titles] END\")\n    return new_schema\n
"},{"location":"reference/fractal_tasks_core/dev/lib_titles/#fractal_tasks_core.dev.lib_titles._include_titles_for_properties","title":"_include_titles_for_properties(properties)","text":"

Scan through properties of a JSON Schema, and set their title when it is missing.

The title is set to name.title(), where title is a standard string method - see https://docs.python.org/3/library/stdtypes.html#str.title.

PARAMETER DESCRIPTION properties

TBD

TYPE: dict[str, dict]

Source code in fractal_tasks_core/dev/lib_titles.py 
def _include_titles_for_properties(\n    properties: dict[str, dict]\n) -> dict[str, dict]:\n\"\"\"\n    Scan through properties of a JSON Schema, and set their title when it is\n    missing.\n\n    The title is set to `name.title()`, where `title` is a standard string\n    method - see https://docs.python.org/3/library/stdtypes.html#str.title.\n\n    Args:\n        properties: TBD\n    \"\"\"\n    new_properties = properties.copy()\n    for prop_name, prop in properties.items():\n        if \"title\" not in prop.keys():\n            new_prop = prop.copy()\n            new_prop[\"title\"] = prop_name.title()\n            new_properties[prop_name] = new_prop\n    return new_properties\n
"},{"location":"reference/fractal_tasks_core/dev/update_manifest/","title":"update_manifest","text":"

Script to generate JSON schemas for task arguments afresh, and write them to the package manifest.

"},{"location":"reference/fractal_tasks_core/tasks/","title":"tasks","text":"

Tasks subpackage (requires installation extra fractal-tasks).

"},{"location":"reference/fractal_tasks_core/tasks/_utils/","title":"_utils","text":"

Standard input/output interface for tasks.

"},{"location":"reference/fractal_tasks_core/tasks/_utils/#fractal_tasks_core.tasks._utils.TaskParameterEncoder","title":"TaskParameterEncoder","text":"

Bases: JSONEncoder

Custom JSONEncoder that transforms Path objects to strings.

Source code in fractal_tasks_core/tasks/_utils.py 
class TaskParameterEncoder(JSONEncoder):\n\"\"\"\n    Custom JSONEncoder that transforms Path objects to strings.\n    \"\"\"\n\n    def default(self, value):\n\"\"\"\n        Subclass implementation of `default`, to serialize Path objects as\n        strings.\n        \"\"\"\n        if isinstance(value, Path):\n            return value.as_posix()\n        return JSONEncoder.default(self, value)\n
"},{"location":"reference/fractal_tasks_core/tasks/_utils/#fractal_tasks_core.tasks._utils.TaskParameterEncoder.default","title":"default(value)","text":"

Subclass implementation of default, to serialize Path objects as strings.

Source code in fractal_tasks_core/tasks/_utils.py 
def default(self, value):\n\"\"\"\n    Subclass implementation of `default`, to serialize Path objects as\n    strings.\n    \"\"\"\n    if isinstance(value, Path):\n        return value.as_posix()\n    return JSONEncoder.default(self, value)\n
"},{"location":"reference/fractal_tasks_core/tasks/_utils/#fractal_tasks_core.tasks._utils.run_fractal_task","title":"run_fractal_task(*, task_function, logger_name=None)","text":"

Implement standard task interface and call task_function.

PARAMETER DESCRIPTION task_function

the callable function that runs the task.

TYPE: Callable

logger_name

TBD

TYPE: Optional[str] DEFAULT: None

Source code in fractal_tasks_core/tasks/_utils.py 
def run_fractal_task(\n    *,\n    task_function: Callable,\n    logger_name: Optional[str] = None,\n):\n\"\"\"\n    Implement standard task interface and call task_function.\n\n    Args:\n        task_function: the callable function that runs the task.\n        logger_name: TBD\n    \"\"\"\n\n    # Parse `-j` and `--metadata-out` arguments\n    parser = ArgumentParser()\n    parser.add_argument(\n        \"-j\", \"--json\", help=\"Read parameters from json file\", required=True\n    )\n    parser.add_argument(\n        \"--metadata-out\",\n        help=\"Output file to redirect serialised returned data\",\n        required=True,\n    )\n    args = parser.parse_args()\n\n    # Set logger\n    logger = logging.getLogger(logger_name)\n\n    # Preliminary check\n    if Path(args.metadata_out).exists():\n        logger.error(\n            f\"Output file {args.metadata_out} already exists. Terminating\"\n        )\n        exit(1)\n\n    # Read parameters dictionary\n    with open(args.json, \"r\") as f:\n        pars = json.load(f)\n\n    # Run task\n    logger.info(f\"START {task_function.__name__} task\")\n    metadata_update = task_function(**pars)\n    logger.info(f\"END {task_function.__name__} task\")\n\n    # Write output metadata to file, with custom JSON encoder\n    with open(args.metadata_out, \"w\") as fout:\n        json.dump(metadata_update, fout, cls=TaskParameterEncoder, indent=2)\n
"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/","title":"apply_registration_to_ROI_tables","text":"

Applies the multiplexing translation to all ROI tables

"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/#fractal_tasks_core.tasks.apply_registration_to_ROI_tables.add_zero_translation_columns","title":"add_zero_translation_columns(ad_table)","text":"

Add three zero-filled columns (translation_{x,y,z}) to an AnnData table.

Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py 
def add_zero_translation_columns(ad_table: ad.AnnData):\n\"\"\"\n    Add three zero-filled columns (`translation_{x,y,z}`) to an AnnData table.\n    \"\"\"\n    columns = [\"translation_z\", \"translation_y\", \"translation_x\"]\n    if ad_table.var.index.isin(columns).any().any():\n        raise ValueError(\n            \"The roi table already contains translation columns. Did you \"\n            \"enter a wrong reference cycle?\"\n        )\n    df = pd.DataFrame(np.zeros([len(ad_table), 3]), columns=columns)\n    df.index = ad_table.obs.index\n    ad_new = ad.concat([ad_table, ad.AnnData(df)], axis=1)\n    return ad_new\n
"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/#fractal_tasks_core.tasks.apply_registration_to_ROI_tables.apply_registration_to_ROI_tables","title":"apply_registration_to_ROI_tables(*, input_paths, output_path, component, metadata, roi_table='FOV_ROI_table', reference_cycle=0, new_roi_table=None)","text":"

Applies pre-calculated registration to ROI tables.

Apply pre-calculated registration such that resulting ROIs contain the consensus align region between all cycles.

Parallelization level: well

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

roi_table

Name of the ROI table over which the task loops to calculate the registration. Examples: FOV_ROI_table => loop over the field of views, well_ROI_table => process the whole well as one image.

TYPE: str DEFAULT: 'FOV_ROI_table'

reference_cycle

Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided

TYPE: int DEFAULT: 0

new_roi_table

Optional name for the new, registered ROI table. If no name is given, it will default to \"registered_\" + roi_table

TYPE: Optional[str] DEFAULT: None

Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py 
@validate_arguments\ndef apply_registration_to_ROI_tables(\n    *,\n    # Fractal arguments\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments\n    roi_table: str = \"FOV_ROI_table\",\n    reference_cycle: int = 0,\n    new_roi_table: Optional[str] = None,\n) -> dict[str, Any]:\n\"\"\"\n    Applies pre-calculated registration to ROI tables.\n\n    Apply pre-calculated registration such that resulting ROIs contain\n    the consensus align region between all cycles.\n\n    Parallelization level: well\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        roi_table: Name of the ROI table over which the task loops to\n            calculate the registration. Examples: `FOV_ROI_table` => loop over\n            the field of views, `well_ROI_table` => process the whole well as\n            one image.\n        reference_cycle: Which cycle to register against. Defaults to 0,\n            which is the first OME-Zarr image in the well, usually the first\n            cycle that was provided\n        new_roi_table: Optional name for the new, registered ROI table. If no\n            name is given, it will default to \"registered_\" + `roi_table`\n\n    \"\"\"\n    if not new_roi_table:\n        new_roi_table = \"registered_\" + roi_table\n    logger.info(\n        f\"Running for {input_paths=}, {component=}. \\n\"\n        f\"Applyg translation registration to {roi_table=} and storing it as \"\n        f\"{new_roi_table=}.\"\n    )\n\n    well_zarr = f\"{input_paths[0]}/{component}\"\n    ngff_well_meta = load_NgffWellMeta(well_zarr)\n    acquisition_dict = ngff_well_meta.get_acquisition_paths()\n    logger.info(\n        \"Calculating common registration for the following cycles: \"\n        f\"{acquisition_dict}\"\n    )\n\n    # TODO: Allow a filter on which acquisitions should get processed?\n\n    # Collect all the ROI tables\n    roi_tables = {}\n    for acq in acquisition_dict.keys():\n        acq_path = acquisition_dict[acq]\n        curr_ROI_table = ad.read_zarr(\n            f\"{well_zarr}/{acq_path}/tables/{roi_table}\"\n        )\n\n        # For reference_cycle acquisition, handle the fact that it doesn't\n        # have the shifts\n        if acq == reference_cycle:\n            curr_ROI_table = add_zero_translation_columns(curr_ROI_table)\n        # Check for valid ROI tables\n        are_ROI_table_columns_valid(table=curr_ROI_table)\n        translation_columns = [\n            \"translation_z\",\n            \"translation_y\",\n            \"translation_x\",\n        ]\n        if curr_ROI_table.var.index.isin(translation_columns).sum() != 3:\n            raise ValueError(\n                f\"Cycle {acq}'s {roi_table} does not contain the \"\n                f\"translation columns {translation_columns} necessary to use \"\n                \"this task.\"\n            )\n        roi_tables[acq] = curr_ROI_table\n\n    # Check that all acquisitions have the same ROIs\n    rois = roi_tables[reference_cycle].obs.index\n    for acq, acq_roi_table in roi_tables.items():\n        if not (acq_roi_table.obs.index == rois).all():\n            raise ValueError(\n                f\"Acquisition {acq} does not contain the same ROIs as the \"\n                f\"reference acquisition {reference_cycle}:\\n\"\n                f\"{acq}: {acq_roi_table.obs.index}\\n\"\n                f\"{reference_cycle}: {rois}\"\n            )\n\n    roi_table_dfs = [\n        roi_table.to_df().loc[:, translation_columns]\n        for roi_table in roi_tables.values()\n    ]\n    logger.info(\"Calculating min & max translation across cycles.\")\n    max_df, min_df = calculate_min_max_across_dfs(roi_table_dfs)\n    shifted_rois = {}\n    # Loop over acquisitions\n    for acq in acquisition_dict.keys():\n        shifted_rois[acq] = apply_registration_to_single_ROI_table(\n            roi_tables[acq], max_df, min_df\n        )\n\n        # TODO: Drop translation columns from this table?\n\n        logger.info(\n            f\"Write the registered ROI table {new_roi_table} for {acq=}\"\n        )\n        # TODO rewrite this block through `write_table` (ref PR #499)\n        # Save the shifted ROI tables as a new tables per acquisition\n        group_tables = zarr.group(f\"{well_zarr}/{acq}/tables/\")\n        write_elem(group_tables, new_roi_table, shifted_rois[acq])\n        # Update list of available tables & its metadata\n        current_tables = group_tables.attrs.asdict().get(\"tables\") or []\n        new_tables = current_tables + [new_roi_table]\n        group_tables.attrs[\"tables\"] = new_tables\n        group_tables[roi_table].attrs[\"type\"] = \"ngff:region_table\"\n\n    # TODO: Optionally apply registration to other tables as well?\n    # e.g. to well_ROI_table based on FOV_ROI_table\n    # => out of scope for the initial task, apply registration separately\n    # to each table\n    # Easiest implementation: Apply average shift calculcated here to other\n    # ROIs. From many to 1 (e.g. FOV => well) => average shift, but crop len\n    # From well to many (e.g. well to FOVs) => average shift, crop len by that\n    # amount\n    # Many to many (FOVs to organoids) => tricky because of matching\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_ROI_tables/#fractal_tasks_core.tasks.apply_registration_to_ROI_tables.apply_registration_to_single_ROI_table","title":"apply_registration_to_single_ROI_table(roi_table, max_df, min_df)","text":"

Applies the registration to a ROI table

Calculates the new position as: p = position + max(shift, 0) - own_shift Calculates the new len as: l = len - max(shift, 0) + min(shift, 0)

PARAMETER DESCRIPTION roi_table

AnnData table which contains a Fractal ROI table. Rows are ROIs

TYPE: AnnData

max_df

Max translation shift in z, y, x for each ROI. Rows are ROIs, columns are translation_z, translation_y, translation_x

TYPE: DataFrame

min_df

Min translation shift in z, y, x for each ROI. Rows are ROIs, columns are translation_z, translation_y, translation_x

TYPE: DataFrame

Returns: ROI table where all ROIs are registered to the smallest common area across all cycles.

Source code in fractal_tasks_core/tasks/apply_registration_to_ROI_tables.py 
def apply_registration_to_single_ROI_table(\n    roi_table: ad.AnnData,\n    max_df: pd.DataFrame,\n    min_df: pd.DataFrame,\n) -> ad.AnnData:\n\"\"\"\n    Applies the registration to a ROI table\n\n    Calculates the new position as: p = position + max(shift, 0) - own_shift\n    Calculates the new len as: l = len - max(shift, 0) + min(shift, 0)\n\n    Args:\n        roi_table: AnnData table which contains a Fractal ROI table.\n            Rows are ROIs\n        max_df: Max translation shift in z, y, x for each ROI. Rows are ROIs,\n            columns are translation_z, translation_y, translation_x\n        min_df: Min translation shift in z, y, x for each ROI. Rows are ROIs,\n            columns are translation_z, translation_y, translation_x\n    Returns:\n        ROI table where all ROIs are registered to the smallest common area\n        across all cycles.\n    \"\"\"\n    roi_table = copy.deepcopy(roi_table)\n    rois = roi_table.obs.index\n    if (rois != max_df.index).all() or (rois != min_df.index).all():\n        raise ValueError(\n            \"ROI table and max & min translation need to contain the same \"\n            f\"ROIS, but they were {rois=}, {max_df.index=}, {min_df.index=}\"\n        )\n\n    for roi in rois:\n        roi_table[[roi], [\"z_micrometer\"]] = (\n            roi_table[[roi], [\"z_micrometer\"]].X\n            + float(max_df.loc[roi, \"translation_z\"])\n            - roi_table[[roi], [\"translation_z\"]].X\n        )\n        roi_table[[roi], [\"y_micrometer\"]] = (\n            roi_table[[roi], [\"y_micrometer\"]].X\n            + float(max_df.loc[roi, \"translation_y\"])\n            - roi_table[[roi], [\"translation_y\"]].X\n        )\n        roi_table[[roi], [\"x_micrometer\"]] = (\n            roi_table[[roi], [\"x_micrometer\"]].X\n            + float(max_df.loc[roi, \"translation_x\"])\n            - roi_table[[roi], [\"translation_x\"]].X\n        )\n        # This calculation only works if all ROIs are the same size initially!\n        roi_table[[roi], [\"len_z_micrometer\"]] = (\n            roi_table[[roi], [\"len_z_micrometer\"]].X\n            - float(max_df.loc[roi, \"translation_z\"])\n            + float(min_df.loc[roi, \"translation_z\"])\n        )\n        roi_table[[roi], [\"len_y_micrometer\"]] = (\n            roi_table[[roi], [\"len_y_micrometer\"]].X\n            - float(max_df.loc[roi, \"translation_y\"])\n            + float(min_df.loc[roi, \"translation_y\"])\n        )\n        roi_table[[roi], [\"len_x_micrometer\"]] = (\n            roi_table[[roi], [\"len_x_micrometer\"]].X\n            - float(max_df.loc[roi, \"translation_x\"])\n            + float(min_df.loc[roi, \"translation_x\"])\n        )\n    return roi_table\n
"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_image/","title":"apply_registration_to_image","text":"

Calculates translation for 2D image-based registration

"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_image/#fractal_tasks_core.tasks.apply_registration_to_image.apply_registration_to_image","title":"apply_registration_to_image(*, input_paths, output_path, component, metadata, registered_roi_table, reference_cycle='0', overwrite_input=True)","text":"

Apply registration to images by using a registered ROI table

This task consists of 4 parts:

  1. Mask all regions in images that are not available in the registered ROI table and store each cycle aligned to the reference_cycle (by looping over ROIs).
  2. Do the same for all label images.
  3. Copy all tables from the non-aligned image to the aligned image (currently only works well if the only tables are well & FOV ROI tables (registered and original). Not implemented for measurement tables and other ROI tables).
  4. Clean up: Delete the old, non-aligned image and rename the new, aligned image to take over its place.

Parallelization level: image

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

registered_roi_table

Name of the ROI table which has been registered and will be applied to mask and shift the images. Examples: registered_FOV_ROI_table => loop over the field of views, registered_well_ROI_table => process the whole well as one image.

TYPE: str

reference_cycle

Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well, usually the first cycle that was provided

TYPE: str DEFAULT: '0'

overwrite_input

Whether the old image data should be replaced with the newly registered image data. Currently only implemented for overwrite_input=True.

TYPE: bool DEFAULT: True

Source code in fractal_tasks_core/tasks/apply_registration_to_image.py 
@validate_arguments\ndef apply_registration_to_image(\n    *,\n    # Fractal arguments\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments\n    registered_roi_table: str,\n    reference_cycle: str = \"0\",\n    overwrite_input: bool = True,\n):\n\"\"\"\n    Apply registration to images by using a registered ROI table\n\n    This task consists of 4 parts:\n\n    1. Mask all regions in images that are not available in the\n    registered ROI table and store each cycle aligned to the\n    reference_cycle (by looping over ROIs).\n    2. Do the same for all label images.\n    3. Copy all tables from the non-aligned image to the aligned image\n    (currently only works well if the only tables are well & FOV ROI tables\n    (registered and original). Not implemented for measurement tables and\n    other ROI tables).\n    4. Clean up: Delete the old, non-aligned image and rename the new,\n    aligned image to take over its place.\n\n    Parallelization level: image\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        registered_roi_table: Name of the ROI table which has been registered\n            and will be applied to mask and shift the images.\n            Examples: `registered_FOV_ROI_table` => loop over the field of\n            views, `registered_well_ROI_table` => process the whole well as\n            one image.\n        reference_cycle: Which cycle to register against. Defaults to 0,\n            which is the first OME-Zarr image in the well, usually the first\n            cycle that was provided\n        overwrite_input: Whether the old image data should be replaced with the\n            newly registered image data. Currently only implemented for\n            `overwrite_input=True`.\n\n    \"\"\"\n    logger.info(component)\n    if not overwrite_input:\n        raise NotImplementedError(\n            \"This task is only implemented for the overwrite_input version\"\n        )\n    logger.info(\n        f\"Running `apply_registration_to_image` on {input_paths=}, \"\n        f\"{component=}, {registered_roi_table=} and {reference_cycle=}. \"\n        f\"Using {overwrite_input=}\"\n    )\n\n    input_path = Path(input_paths[0])\n    new_component = \"/\".join(\n        component.split(\"/\")[:-1] + [component.split(\"/\")[-1] + \"_registered\"]\n    )\n    reference_component = \"/\".join(\n        component.split(\"/\")[:-1] + [reference_cycle]\n    )\n\n    ROI_table_ref = ad.read_zarr(\n        f\"{input_path / reference_component}/tables/{registered_roi_table}\"\n    )\n    ROI_table_cycle = ad.read_zarr(\n        f\"{input_path / component}/tables/{registered_roi_table}\"\n    )\n\n    ngff_image_meta = load_NgffImageMeta(str(input_path / component))\n    coarsening_xy = ngff_image_meta.coarsening_xy\n    num_levels = ngff_image_meta.num_levels\n\n    ####################\n    # Process images\n    ####################\n    logger.info(\"Write the registered Zarr image to disk\")\n    write_registered_zarr(\n        input_path=input_path,\n        component=component,\n        new_component=new_component,\n        ROI_table=ROI_table_cycle,\n        ROI_table_ref=ROI_table_ref,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        aggregation_function=np.mean,\n    )\n\n    ####################\n    # Process labels\n    ####################\n    try:\n        labels_group = zarr.open_group(f\"{input_path / component}/labels\", \"r\")\n        label_list = labels_group.attrs[\"labels\"]\n    except (zarr.errors.GroupNotFoundError, KeyError):\n        label_list = []\n\n    if label_list:\n        logger.info(f\"Processing the label images: {label_list}\")\n        labels_group = zarr.group(f\"{input_path / new_component}/labels\")\n        labels_group.attrs[\"labels\"] = label_list\n\n        for label in label_list:\n            label_component = f\"{component}/labels/{label}\"\n            label_component_new = f\"{new_component}/labels/{label}\"\n            write_registered_zarr(\n                input_path=input_path,\n                component=label_component,\n                new_component=label_component_new,\n                ROI_table=ROI_table_cycle,\n                ROI_table_ref=ROI_table_ref,\n                num_levels=num_levels,\n                coarsening_xy=coarsening_xy,\n                aggregation_function=np.max,\n            )\n\n    ####################\n    # Copy tables\n    # 1. Copy all standard ROI tables from cycle 0.\n    # 2. Copy all tables that aren't standard ROI tables from the given cycle\n    ####################\n    table_dict_reference = get_table_path_dict(input_path, reference_component)\n    table_dict_component = get_table_path_dict(input_path, component)\n\n    table_dict = {}\n    # Define which table should get copied:\n    for table in table_dict_reference:\n        if is_standard_roi_table(table):\n            table_dict[table] = table_dict_reference[table]\n    for table in table_dict_component:\n        if not is_standard_roi_table(table):\n            if reference_component != component:\n                logger.warning(\n                    f\"{component} contained a table that is not a standard \"\n                    \"ROI table. The `Apply Registration To Image task` is \"\n                    \"best used before additional tables are generated. It \"\n                    f\"will copy the {table} from this cycle without applying \"\n                    f\"any transformations. This will work well if {table} \"\n                    f\"contains measurements. But if {table} is a custom ROI \"\n                    \"table coming from another task, the transformation is \"\n                    \"not applied and it will not match with the registered \"\n                    \"image anymore\"\n                )\n            table_dict[table] = table_dict_component[table]\n\n    if table_dict:\n        logger.info(f\"Processing the tables: {table_dict}\")\n        new_tables_group = zarr.group(f\"{input_path / new_component}/tables\")\n        new_tables_group.attrs[\"tables\"] = list(table_dict.keys())\n\n        for table in table_dict.keys():\n            logger.info(f\"Copying table: {table}\")\n            # Write the Zarr table\n            curr_table = ad.read_zarr(table_dict[table])\n            write_elem(new_tables_group, table, curr_table)\n            # Get the relevant metadata of the Zarr table & add it\n            # See issue #516 for the need for this workaround\n            try:\n                old_table_group = zarr.open_group(table_dict[table], mode=\"r\")\n            except zarr.errors.GroupNotFoundError:\n                time.sleep(5)\n                old_table_group = zarr.open_group(table_dict[table], mode=\"r\")\n            new_table_group = zarr.open_group(\n                f\"{input_path / new_component}/tables/{table}\"\n            )\n            new_table_group.attrs.put(old_table_group.attrs.asdict())\n\n    ####################\n    # Clean up Zarr file\n    ####################\n    if overwrite_input:\n        logger.info(\n            \"Replace original zarr image with the newly created Zarr image\"\n        )\n        # Potential for race conditions: Every cycle reads the\n        # reference cycle, but the reference cycle also gets modified\n        # See issue #516 for the details\n        os.rename(f\"{input_path / component}\", f\"{input_path / component}_tmp\")\n        os.rename(f\"{input_path / new_component}\", f\"{input_path / component}\")\n        shutil.rmtree(f\"{input_path / component}_tmp\")\n    else:\n        raise NotImplementedError\n
"},{"location":"reference/fractal_tasks_core/tasks/apply_registration_to_image/#fractal_tasks_core.tasks.apply_registration_to_image.write_registered_zarr","title":"write_registered_zarr(input_path, component, new_component, ROI_table, ROI_table_ref, num_levels, coarsening_xy=2, aggregation_function=np.mean)","text":"

Write registered zarr array based on ROI tables

This function loads the image or label data from a zarr array based on the ROI bounding-box coordinates and stores them into a new zarr array. The new Zarr array has the same shape as the original array, but will have 0s where the ROI tables don't specify loading of the image data. The ROIs loaded from list_indices will be written into the list_indices_ref position, thus performing translational registration if the two lists of ROI indices vary.

PARAMETER DESCRIPTION input_path

Base folder where the Zarr is stored (does not contain the Zarr file itself)

TYPE: Path

component

Path to the OME-Zarr image that is processed. For example: \"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1\"

TYPE: str

new_component

Path to the new Zarr image that will be written (also in the input_path folder). For example: \"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1_registered\"

TYPE: str

ROI_table

Fractal ROI table for the component

TYPE: AnnData

ROI_table_ref

Fractal ROI table for the reference cycle

TYPE: AnnData

num_levels

Number of pyramid layers to be created (argument of build_pyramid).

TYPE: int

coarsening_xy

Coarsening factor between pyramid levels

TYPE: int DEFAULT: 2

aggregation_function

Function to be used when downsampling (argument of build_pyramid).

TYPE: Callable DEFAULT: mean

Source code in fractal_tasks_core/tasks/apply_registration_to_image.py 
def write_registered_zarr(\n    input_path: Path,\n    component: str,\n    new_component: str,\n    ROI_table: ad.AnnData,\n    ROI_table_ref: ad.AnnData,\n    num_levels: int,\n    coarsening_xy: int = 2,\n    aggregation_function: Callable = np.mean,\n):\n\"\"\"\n    Write registered zarr array based on ROI tables\n\n    This function loads the image or label data from a zarr array based on the\n    ROI bounding-box coordinates and stores them into a new zarr array.\n    The new Zarr array has the same shape as the original array, but will have\n    0s where the ROI tables don't specify loading of the image data.\n    The ROIs loaded from `list_indices` will be written into the\n    `list_indices_ref` position, thus performing translational registration if\n    the two lists of ROI indices vary.\n\n    Args:\n        input_path: Base folder where the Zarr is stored\n            (does not contain the Zarr file itself)\n        component: Path to the OME-Zarr image that is processed. For example:\n            `\"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1\"`\n        new_component: Path to the new Zarr image that will be written\n            (also in the input_path folder). For example:\n            `\"20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr/B/03/1_registered\"`\n        ROI_table: Fractal ROI table for the component\n        ROI_table_ref: Fractal ROI table for the reference cycle\n        num_levels: Number of pyramid layers to be created (argument of\n            `build_pyramid`).\n        coarsening_xy: Coarsening factor between pyramid levels\n        aggregation_function: Function to be used when downsampling (argument\n            of `build_pyramid`).\n\n    \"\"\"\n    # Read pixel sizes from Zarr attributes\n    ngff_image_meta = load_NgffImageMeta(str(input_path / component))\n    pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n\n    # Create list of indices for 3D ROIs\n    list_indices = convert_ROI_table_to_indices(\n        ROI_table,\n        level=0,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=pxl_sizes_zyx,\n    )\n    list_indices_ref = convert_ROI_table_to_indices(\n        ROI_table_ref,\n        level=0,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=pxl_sizes_zyx,\n    )\n\n    old_image_group = zarr.open_group(f\"{input_path / component}\", mode=\"r\")\n    old_ngff_image_meta = load_NgffImageMeta(str(input_path / component))\n    new_image_group = zarr.group(f\"{input_path / new_component}\")\n    new_image_group.attrs.put(old_image_group.attrs.asdict())\n\n    # Loop over all channels. For each channel, write full-res image data.\n    data_array = da.from_zarr(old_image_group[\"0\"])\n    # Create dask array with 0s of same shape\n    new_array = da.zeros_like(data_array)\n\n    # TODO: Add sanity checks on the 2 ROI tables:\n    # 1. The number of ROIs need to match\n    # 2. The size of the ROIs need to match\n    # (otherwise, we can't assign them to the reference regions)\n    # ROI_table_ref vs ROI_table_cycle\n    for i, roi_indices in enumerate(list_indices):\n        reference_region = convert_indices_to_regions(list_indices_ref[i])\n        region = convert_indices_to_regions(roi_indices)\n\n        axes_list = old_ngff_image_meta.axes_names\n\n        if axes_list == [\"c\", \"z\", \"y\", \"x\"]:\n            num_channels = data_array.shape[0]\n            # Loop over channels\n            for ind_ch in range(num_channels):\n                idx = tuple(\n                    [slice(ind_ch, ind_ch + 1)] + list(reference_region)\n                )\n                new_array[idx] = load_region(\n                    data_zyx=data_array[ind_ch], region=region, compute=False\n                )\n        elif axes_list == [\"z\", \"y\", \"x\"]:\n            new_array[reference_region] = load_region(\n                data_zyx=data_array, region=region, compute=False\n            )\n        elif axes_list == [\"c\", \"y\", \"x\"]:\n            # TODO: Implement cyx case (based on looping over xy case)\n            raise NotImplementedError(\n                \"`write_registered_zarr` has not been implemented for \"\n                f\"a zarr with {axes_list=}\"\n            )\n        elif axes_list == [\"y\", \"x\"]:\n            # TODO: Implement yx case\n            raise NotImplementedError(\n                \"`write_registered_zarr` has not been implemented for \"\n                f\"a zarr with {axes_list=}\"\n            )\n        else:\n            raise NotImplementedError(\n                \"`write_registered_zarr` has not been implemented for \"\n                f\"a zarr with {axes_list=}\"\n            )\n\n    new_array.to_zarr(\n        f\"{input_path / new_component}/0\",\n        overwrite=True,\n        dimension_separator=\"/\",\n        write_empty_chunks=False,\n    )\n\n    # Starting from on-disk highest-resolution data, build and write to\n    # disk a pyramid of coarser levels\n    build_pyramid(\n        zarrurl=f\"{input_path / new_component}\",\n        overwrite=True,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        chunksize=data_array.chunksize,\n        aggregation_function=aggregation_function,\n    )\n
"},{"location":"reference/fractal_tasks_core/tasks/calculate_registration_image_based/","title":"calculate_registration_image_based","text":"

Calculates translation for image-based registration

"},{"location":"reference/fractal_tasks_core/tasks/calculate_registration_image_based/#fractal_tasks_core.tasks.calculate_registration_image_based.calculate_physical_shifts","title":"calculate_physical_shifts(shifts, level, coarsening_xy, full_res_pxl_sizes_zyx)","text":"

Calculates shifts in physical units based on pixel shifts

PARAMETER DESCRIPTION shifts

array of shifts, zyx or yx

TYPE: array

level

resolution level

TYPE: int

coarsening_xy

coarsening factor between levels

TYPE: int

full_res_pxl_sizes_zyx

pixel sizes in physical units as zyx

TYPE: list[float]

RETURNS DESCRIPTION shifts_physical

shifts in physical units as zyx

TYPE: list[float]

Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py 
def calculate_physical_shifts(\n    shifts: np.array,\n    level: int,\n    coarsening_xy: int,\n    full_res_pxl_sizes_zyx: list[float],\n) -> list[float]:\n\"\"\"\n    Calculates shifts in physical units based on pixel shifts\n\n    Args:\n        shifts: array of shifts, zyx or yx\n        level: resolution level\n        coarsening_xy: coarsening factor between levels\n        full_res_pxl_sizes_zyx: pixel sizes in physical units as zyx\n\n    Returns:\n        shifts_physical: shifts in physical units as zyx\n    \"\"\"\n\n    curr_pixel_size = np.array(full_res_pxl_sizes_zyx) * coarsening_xy**level\n    if len(shifts) == 3:\n        shifts_physical = shifts * curr_pixel_size\n    elif len(shifts) == 2:\n        shifts_physical = [\n            0,\n            shifts[0] * curr_pixel_size[1],\n            shifts[1] * curr_pixel_size[2],\n        ]\n    else:\n        raise ValueError(\n            f\"Wrong input for calculate_physical_shifts ({shifts=})\"\n        )\n    return shifts_physical\n
"},{"location":"reference/fractal_tasks_core/tasks/calculate_registration_image_based/#fractal_tasks_core.tasks.calculate_registration_image_based.calculate_registration_image_based","title":"calculate_registration_image_based(*, input_paths, output_path, component, metadata, wavelength_id, roi_table='FOV_ROI_table', reference_cycle=0, level=2)","text":"

Calculate registration based on images

This task consists of 3 parts:

  1. Loading the images of a given ROI (=> loop over ROIs)
  2. Calculating the transformation for that ROI
  3. Storing the calculated transformation in the ROI table

Parallelization level: image

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

wavelength_id

Wavelength that will be used for image-based registration; e.g. A01_C01 for Yokogawa, C01 for MD.

TYPE: str

roi_table

Name of the ROI table over which the task loops to calculate the registration. Examples: FOV_ROI_table => loop over the field of views, well_ROI_table => process the whole well as one image.

TYPE: str DEFAULT: 'FOV_ROI_table'

reference_cycle

Which cycle to register against. Defaults to 0, which is the first OME-Zarr image in the well (usually the first cycle that was provided).

TYPE: int DEFAULT: 0

level

Pyramid level of the image to be segmented. Choose 0 to process at full resolution.

TYPE: int DEFAULT: 2

Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py 
@validate_arguments\ndef calculate_registration_image_based(\n    *,\n    # Fractal arguments\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments\n    wavelength_id: str,\n    roi_table: str = \"FOV_ROI_table\",\n    reference_cycle: int = 0,\n    level: int = 2,\n) -> dict[str, Any]:\n\"\"\"\n    Calculate registration based on images\n\n    This task consists of 3 parts:\n\n    1. Loading the images of a given ROI (=> loop over ROIs)\n    2. Calculating the transformation for that ROI\n    3. Storing the calculated transformation in the ROI table\n\n    Parallelization level: image\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        wavelength_id: Wavelength that will be used for image-based\n            registration; e.g. `A01_C01` for Yokogawa, `C01` for MD.\n        roi_table: Name of the ROI table over which the task loops to\n            calculate the registration. Examples: `FOV_ROI_table` => loop over\n            the field of views, `well_ROI_table` => process the whole well as\n            one image.\n        reference_cycle: Which cycle to register against. Defaults to 0,\n            which is the first OME-Zarr image in the well (usually the first\n            cycle that was provided).\n        level: Pyramid level of the image to be segmented. Choose `0` to\n            process at full resolution.\n\n    \"\"\"\n    logger.info(\n        f\"Running for {input_paths=}, {component=}. \\n\"\n        f\"Calculating translation registration per {roi_table=} for \"\n        f\"{wavelength_id=}.\"\n    )\n    # Set OME-Zarr paths\n    zarr_img_cycle_x = Path(input_paths[0]) / component\n\n    # If the task is run for the reference cycle, exit\n    # TODO: Improve the input for this: Can we filter components to not\n    # run for itself?\n    alignment_cycle = zarr_img_cycle_x.name\n    if alignment_cycle == str(reference_cycle):\n        logger.info(\n            \"Calculate registration image-based is running for \"\n            f\"cycle {alignment_cycle}, which is the reference_cycle.\"\n            \"Thus, exiting the task.\"\n        )\n        return {}\n    else:\n        logger.info(\n            \"Calculate registration image-based is running for \"\n            f\"cycle {alignment_cycle}\"\n        )\n\n    zarr_img_ref_cycle = zarr_img_cycle_x.parent / str(reference_cycle)\n\n    # Read some parameters from Zarr metadata\n    ngff_image_meta = load_NgffImageMeta(str(zarr_img_ref_cycle))\n    coarsening_xy = ngff_image_meta.coarsening_xy\n\n    # Get channel_index via wavelength_id.\n    # Intially only allow registration of the same wavelength\n    channel_ref: OmeroChannel = get_channel_from_image_zarr(\n        image_zarr_path=str(zarr_img_ref_cycle),\n        wavelength_id=wavelength_id,\n    )\n    channel_index_ref = channel_ref.index\n\n    channel_align: OmeroChannel = get_channel_from_image_zarr(\n        image_zarr_path=str(zarr_img_cycle_x),\n        wavelength_id=wavelength_id,\n    )\n    channel_index_align = channel_align.index\n\n    # Lazily load zarr array\n    data_reference_zyx = da.from_zarr(f\"{zarr_img_ref_cycle}/{level}\")[\n        channel_index_ref\n    ]\n    data_alignment_zyx = da.from_zarr(f\"{zarr_img_cycle_x}/{level}\")[\n        channel_index_align\n    ]\n\n    # Read ROIs\n    ROI_table_ref = ad.read_zarr(f\"{zarr_img_ref_cycle}/tables/{roi_table}\")\n    ROI_table_x = ad.read_zarr(f\"{zarr_img_ref_cycle}/tables/{roi_table}\")\n    logger.info(\n        f\"Found {len(ROI_table_x)} ROIs in {roi_table=} to be processed.\"\n    )\n\n    # For each cycle, get the relevant info\n    # TODO: Add additional checks on ROIs?\n    if (ROI_table_ref.obs.index != ROI_table_x.obs.index).all():\n        raise ValueError(\n            \"Registration is only implemented for ROIs that match between the \"\n            \"cycles (e.g. well, FOV ROIs). Here, the ROIs in the reference \"\n            \"cycles were {ROI_table_ref.obs.index}, but the ROIs in the \"\n            \"alignment cycle were {ROI_table_x.obs.index}\"\n        )\n    # TODO: Make this less restrictive? i.e. could we also run it if different\n    # cycles have different FOVs? But then how do we know which FOVs to match?\n    # If we relax this, downstream assumptions on matching based on order\n    # in the list will break.\n\n    # Read pixel sizes from zarr attributes\n    ngff_image_meta_cycle_x = load_NgffImageMeta(str(zarr_img_cycle_x))\n    pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n    pxl_sizes_zyx_cycle_x = ngff_image_meta_cycle_x.get_pixel_sizes_zyx(\n        level=0\n    )\n\n    if pxl_sizes_zyx != pxl_sizes_zyx_cycle_x:\n        raise ValueError(\n            \"Pixel sizes need to be equal between cycles for registration\"\n        )\n\n    # Create list of indices for 3D ROIs spanning the entire Z direction\n    list_indices_ref = convert_ROI_table_to_indices(\n        ROI_table_ref,\n        level=level,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(list_indices_ref, roi_table)\n\n    list_indices_cycle_x = convert_ROI_table_to_indices(\n        ROI_table_x,\n        level=level,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(list_indices_cycle_x, roi_table)\n\n    num_ROIs = len(list_indices_ref)\n    compute = True\n    new_shifts = {}\n    for i_ROI in range(num_ROIs):\n        logger.info(\n            f\"Now processing ROI {i_ROI+1}/{num_ROIs} \"\n            f\"for channel {channel_align}.\"\n        )\n        img_ref = load_region(\n            data_zyx=data_reference_zyx,\n            region=convert_indices_to_regions(list_indices_ref[i_ROI]),\n            compute=compute,\n        )\n        img_cycle_x = load_region(\n            data_zyx=data_alignment_zyx,\n            region=convert_indices_to_regions(list_indices_cycle_x[i_ROI]),\n            compute=compute,\n        )\n\n        ##############\n        #  Calculate the transformation\n        ##############\n        # Basic version (no padding, no internal binning)\n        if img_ref.shape != img_cycle_x.shape:\n            raise NotImplementedError(\n                \"This registration is not implemented for ROIs with \"\n                \"different shapes between cycles\"\n            )\n        shifts = phase_cross_correlation(\n            np.squeeze(img_ref), np.squeeze(img_cycle_x)\n        )[0]\n\n        # Registration based on scmultiplex, image-based\n        # shifts, _, _ = calculate_shift(np.squeeze(img_ref),\n        #           np.squeeze(img_cycle_x), bin=binning, binarize=False)\n\n        # TODO: Make this work on label images\n        # (=> different loading) etc.\n\n        ##############\n        # Storing the calculated transformation ###\n        ##############\n        # Store the shift in ROI table\n        # TODO: Store in OME-NGFF transformations: Check SpatialData approach,\n        # per ROI storage?\n\n        # Adapt ROIs for the given ROI table:\n        ROI_name = ROI_table_ref.obs.index[i_ROI]\n        new_shifts[ROI_name] = calculate_physical_shifts(\n            shifts,\n            level=level,\n            coarsening_xy=coarsening_xy,\n            full_res_pxl_sizes_zyx=pxl_sizes_zyx,\n        )\n\n    # Write physical shifts to disk (as part of the ROI table)\n    logger.info(f\"Updating the {roi_table=} with translation columns\")\n    new_ROI_table = get_ROI_table_with_translation(ROI_table_x, new_shifts)\n    group_tables = zarr.group(f\"{zarr_img_cycle_x}/tables/\")\n    write_elem(group_tables, roi_table, new_ROI_table)\n    group_tables[roi_table].attrs[\"type\"] = \"ngff:region_table\"\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/calculate_registration_image_based/#fractal_tasks_core.tasks.calculate_registration_image_based.get_ROI_table_with_translation","title":"get_ROI_table_with_translation(ROI_table, new_shifts)","text":"

Adds translation columns to a ROI table

PARAMETER DESCRIPTION ROI_table

Fractal ROI table

TYPE: AnnData

new_shifts

zyx list of shifts

TYPE: dict[str, list[float]]

RETURNS DESCRIPTION AnnData

Fractal ROI table with 3 additional columns for calculated translations

Source code in fractal_tasks_core/tasks/calculate_registration_image_based.py 
def get_ROI_table_with_translation(\n    ROI_table: ad.AnnData,\n    new_shifts: dict[str, list[float]],\n) -> ad.AnnData:\n\"\"\"\n    Adds translation columns to a ROI table\n\n    Args:\n        ROI_table: Fractal ROI table\n        new_shifts: zyx list of shifts\n\n    Returns:\n        Fractal ROI table with 3 additional columns for calculated translations\n    \"\"\"\n\n    shift_table = pd.DataFrame(new_shifts).T\n    shift_table.columns = [\"translation_z\", \"translation_y\", \"translation_x\"]\n    shift_table = shift_table.rename_axis(\"FieldIndex\")\n    new_roi_table = ROI_table.to_df().merge(\n        shift_table, left_index=True, right_index=True\n    )\n    if len(new_roi_table) != len(ROI_table):\n        raise ValueError(\n            \"New ROI table with registration info has a \"\n            f\"different length ({len(new_roi_table)=}) \"\n            f\"from the original ROI table ({len(ROI_table)=})\"\n        )\n    positional_columns = [\n        \"x_micrometer\",\n        \"y_micrometer\",\n        \"z_micrometer\",\n        \"len_x_micrometer\",\n        \"len_y_micrometer\",\n        \"len_z_micrometer\",\n        \"x_micrometer_original\",\n        \"y_micrometer_original\",\n        \"translation_z\",\n        \"translation_y\",\n        \"translation_x\",\n    ]\n    adata = ad.AnnData(\n        X=new_roi_table.loc[:, positional_columns].astype(np.float32)\n    )\n    adata.obs_names = new_roi_table.index\n    adata.var_names = list(map(str, positional_columns))\n    return adata\n
"},{"location":"reference/fractal_tasks_core/tasks/cellpose_segmentation/","title":"cellpose_segmentation","text":"

Image segmentation via Cellpose library.

"},{"location":"reference/fractal_tasks_core/tasks/cellpose_segmentation/#fractal_tasks_core.tasks.cellpose_segmentation.cellpose_segmentation","title":"cellpose_segmentation(*, input_paths, output_path, component, metadata, level, channel, channel2=None, input_ROI_table='FOV_ROI_table', output_ROI_table=None, output_label_name=None, use_masks=True, relabeling=True, diameter_level0=30.0, model_type='cyto2', pretrained_model=None, cellprob_threshold=0.0, flow_threshold=0.4, anisotropy=None, min_size=15, augment=False, net_avg=False, use_gpu=True, overwrite=True)","text":"

Run cellpose segmentation on the ROIs of a single OME-Zarr image.

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

level

Pyramid level of the image to be segmented. Choose 0 to process at full resolution.

TYPE: int

channel

Primary channel for segmentation; requires either wavelength_id (e.g. A01_C01) or label (e.g. DAPI).

TYPE: Channel

channel2

Second channel for segmentation (in the same format as channel). If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker.

TYPE: Optional[Channel] DEFAULT: None

input_ROI_table

Name of the ROI table over which the task loops to apply Cellpose segmentation. Examples: FOV_ROI_table => loop over the field of views, organoid_ROI_table => loop over the organoid ROI table (generated by another task), well_ROI_table => process the whole well as one image.

TYPE: str DEFAULT: 'FOV_ROI_table'

output_ROI_table

If provided, a ROI table with that name is created, which will contain the bounding boxes of the newly segmented labels. ROI tables should have ROI in their name.

TYPE: Optional[str] DEFAULT: None

use_masks

If True, try to use masked loading and fall back to use_masks=False if the ROI table is not suitable. Masked loading is relevant when only a subset of the bounding box should actually be processed (e.g. running within organoid_ROI_table).

TYPE: bool DEFAULT: True

output_label_name

Name of the output label image (e.g. \"organoids\").

TYPE: Optional[str] DEFAULT: None

relabeling

If True, apply relabeling so that label values are unique for all objects in the well.

TYPE: bool DEFAULT: True

diameter_level0

Expected diameter of the objects that should be segmented in pixels at level 0. Initial diameter is rescaled using the level that was selected. The rescaled value is passed as the diameter to the CellposeModel.eval method.

TYPE: float DEFAULT: 30.0

model_type

Parameter of CellposeModel class. Defines which model should be used. Typical choices are nuclei, cyto, cyto2, etc.

TYPE: str DEFAULT: 'cyto2'

pretrained_model

Parameter of CellposeModel class (takes precedence over model_type). Allows you to specify the path of a custom trained cellpose model.

TYPE: Optional[str] DEFAULT: None

cellprob_threshold

Parameter of CellposeModel.eval method. Valid values between -6 to 6. From Cellpose documentation: \"Decrease this threshold if cellpose is not returning as many ROIs as you\u2019d expect. Similarly, increase this threshold if cellpose is returning too ROIs particularly from dim areas.\"

TYPE: float DEFAULT: 0.0

flow_threshold

Parameter of CellposeModel.eval method. Valid values between 0.0 and 1.0. From Cellpose documentation: \"Increase this threshold if cellpose is not returning as many ROIs as you\u2019d expect. Similarly, decrease this threshold if cellpose is returning too many ill-shaped ROIs.\"

TYPE: float DEFAULT: 0.4

anisotropy

Ratio of the pixel sizes along Z and XY axis (ignored if the image is not three-dimensional). If None, it is inferred from the OME-NGFF metadata.

TYPE: Optional[float] DEFAULT: None

min_size

Parameter of CellposeModel class. Minimum size of the segmented objects (in pixels). Use -1 to turn off the size filter.

TYPE: int DEFAULT: 15

augment

Parameter of CellposeModel class. Whether to use cellpose augmentation to tile images with overlap.

TYPE: bool DEFAULT: False

net_avg

Parameter of CellposeModel class. Whether to use cellpose net averaging to run the 4 built-in networks (useful for nuclei, cyto and cyto2, not sure it works for the others).

TYPE: bool DEFAULT: False

use_gpu

If False, always use the CPU; if True, use the GPU if possible (as defined in cellpose.core.use_gpu()) and fall-back to the CPU otherwise.

TYPE: bool DEFAULT: True

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: True

Source code in fractal_tasks_core/tasks/cellpose_segmentation.py 
@validate_arguments\ndef cellpose_segmentation(\n    *,\n    # Fractal arguments\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments\n    level: int,\n    channel: Channel,\n    channel2: Optional[Channel] = None,\n    input_ROI_table: str = \"FOV_ROI_table\",\n    output_ROI_table: Optional[str] = None,\n    output_label_name: Optional[str] = None,\n    use_masks: bool = True,\n    relabeling: bool = True,\n    # Cellpose-related arguments\n    diameter_level0: float = 30.0,\n    model_type: str = \"cyto2\",\n    pretrained_model: Optional[str] = None,\n    cellprob_threshold: float = 0.0,\n    flow_threshold: float = 0.4,\n    anisotropy: Optional[float] = None,\n    min_size: int = 15,\n    augment: bool = False,\n    net_avg: bool = False,\n    use_gpu: bool = True,\n    overwrite: bool = True,\n) -> dict[str, Any]:\n\"\"\"\n    Run cellpose segmentation on the ROIs of a single OME-Zarr image.\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        level: Pyramid level of the image to be segmented. Choose `0` to\n            process at full resolution.\n        channel: Primary channel for segmentation; requires either\n            `wavelength_id` (e.g. `A01_C01`) or `label` (e.g. `DAPI`).\n        channel2: Second channel for segmentation (in the same format as\n            `channel`). If specified, cellpose runs in dual channel mode.\n            For dual channel segmentation of cells, the first channel should\n            contain the membrane marker, the second channel should contain the\n            nuclear marker.\n        input_ROI_table: Name of the ROI table over which the task loops to\n            apply Cellpose segmentation. Examples: `FOV_ROI_table` => loop over\n            the field of views, `organoid_ROI_table` => loop over the organoid\n            ROI table (generated by another task), `well_ROI_table` => process\n            the whole well as one image.\n        output_ROI_table: If provided, a ROI table with that name is created,\n            which will contain the bounding boxes of the newly segmented\n            labels. ROI tables should have `ROI` in their name.\n        use_masks: If `True`, try to use masked loading and fall back to\n            `use_masks=False` if the ROI table is not suitable. Masked\n            loading is relevant when only a subset of the bounding box should\n            actually be processed (e.g. running within `organoid_ROI_table`).\n        output_label_name: Name of the output label image (e.g. `\"organoids\"`).\n        relabeling: If `True`, apply relabeling so that label values are\n            unique for all objects in the well.\n        diameter_level0: Expected diameter of the objects that should be\n            segmented in pixels at level 0. Initial diameter is rescaled using\n            the `level` that was selected. The rescaled value is passed as\n            the diameter to the `CellposeModel.eval` method.\n        model_type: Parameter of `CellposeModel` class. Defines which model\n            should be used. Typical choices are `nuclei`, `cyto`, `cyto2`, etc.\n        pretrained_model: Parameter of `CellposeModel` class (takes\n            precedence over `model_type`). Allows you to specify the path of\n            a custom trained cellpose model.\n        cellprob_threshold: Parameter of `CellposeModel.eval` method. Valid\n            values between -6 to 6. From Cellpose documentation: \"Decrease this\n            threshold if cellpose is not returning as many ROIs as you\u2019d\n            expect. Similarly, increase this threshold if cellpose is returning\n            too ROIs particularly from dim areas.\"\n        flow_threshold: Parameter of `CellposeModel.eval` method. Valid\n            values between 0.0 and 1.0. From Cellpose documentation: \"Increase\n            this threshold if cellpose is not returning as many ROIs as you\u2019d\n            expect. Similarly, decrease this threshold if cellpose is returning\n            too many ill-shaped ROIs.\"\n        anisotropy: Ratio of the pixel sizes along Z and XY axis (ignored if\n            the image is not three-dimensional). If `None`, it is inferred from\n            the OME-NGFF metadata.\n        min_size: Parameter of `CellposeModel` class. Minimum size of the\n            segmented objects (in pixels). Use `-1` to turn off the size\n            filter.\n        augment: Parameter of `CellposeModel` class. Whether to use cellpose\n            augmentation to tile images with overlap.\n        net_avg: Parameter of `CellposeModel` class. Whether to use cellpose\n            net averaging to run the 4 built-in networks (useful for `nuclei`,\n            `cyto` and `cyto2`, not sure it works for the others).\n        use_gpu: If `False`, always use the CPU; if `True`, use the GPU if\n            possible (as defined in `cellpose.core.use_gpu()`) and fall-back\n            to the CPU otherwise.\n        overwrite: If `True`, overwrite the task output.\n    \"\"\"\n\n    # Set input path\n    if len(input_paths) > 1:\n        raise NotImplementedError\n    in_path = Path(input_paths[0])\n    zarrurl = (in_path.resolve() / component).as_posix()\n    logger.info(f\"{zarrurl=}\")\n\n    # Preliminary checks on Cellpose model\n    if pretrained_model is None:\n        if model_type not in models.MODEL_NAMES:\n            raise ValueError(f\"ERROR model_type={model_type} is not allowed.\")\n    else:\n        if not os.path.exists(pretrained_model):\n            raise ValueError(f\"{pretrained_model=} does not exist.\")\n\n    # Read attributes from NGFF metadata\n    ngff_image_meta = load_NgffImageMeta(zarrurl)\n    num_levels = ngff_image_meta.num_levels\n    coarsening_xy = ngff_image_meta.coarsening_xy\n    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n    actual_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=level)\n    logger.info(f\"NGFF image has {num_levels=}\")\n    logger.info(f\"NGFF image has {coarsening_xy=}\")\n    logger.info(\n        f\"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}\"\n    )\n    logger.info(\n        f\"NGFF image has level-{level} pixel sizes \"\n        f\"{actual_res_pxl_sizes_zyx}\"\n    )\n\n    plate, well = component.split(\".zarr/\")\n\n    # Find channel index\n    try:\n        tmp_channel: OmeroChannel = get_channel_from_image_zarr(\n            image_zarr_path=zarrurl,\n            wavelength_id=channel.wavelength_id,\n            label=channel.label,\n        )\n    except ChannelNotFoundError as e:\n        logger.warning(\n            \"Channel not found, exit from the task.\\n\"\n            f\"Original error: {str(e)}\"\n        )\n        return {}\n    ind_channel = tmp_channel.index\n\n    # Find channel index for second channel, if one is provided\n    if channel2:\n        try:\n            tmp_channel_c2: OmeroChannel = get_channel_from_image_zarr(\n                image_zarr_path=zarrurl,\n                wavelength_id=channel2.wavelength_id,\n                label=channel2.label,\n            )\n        except ChannelNotFoundError as e:\n            logger.warning(\n                f\"Second channel with wavelength_id: {channel2.wavelength_id} \"\n                f\"and label: {channel2.label} not found, exit from the task.\\n\"\n                f\"Original error: {str(e)}\"\n            )\n            return {}\n        ind_channel_c2 = tmp_channel_c2.index\n\n    # Set channel label\n    if output_label_name is None:\n        try:\n            channel_label = tmp_channel.label\n            output_label_name = f\"label_{channel_label}\"\n        except (KeyError, IndexError):\n            output_label_name = f\"label_{ind_channel}\"\n\n    # Load ZYX data\n    data_zyx = da.from_zarr(f\"{zarrurl}/{level}\")[ind_channel]\n    logger.info(f\"{data_zyx.shape=}\")\n    if channel2:\n        data_zyx_c2 = da.from_zarr(f\"{zarrurl}/{level}\")[ind_channel_c2]\n        logger.info(f\"Second channel: {data_zyx_c2.shape=}\")\n\n    # Read ROI table\n    ROI_table_path = f\"{zarrurl}/tables/{input_ROI_table}\"\n    ROI_table = ad.read_zarr(ROI_table_path)\n\n    # Perform some checks on the ROI table\n    valid_ROI_table = is_ROI_table_valid(\n        table_path=ROI_table_path, use_masks=use_masks\n    )\n    if use_masks and not valid_ROI_table:\n        logger.info(\n            f\"ROI table at {ROI_table_path} cannot be used for masked \"\n            \"loading. Set use_masks=False.\"\n        )\n        use_masks = False\n    logger.info(f\"{use_masks=}\")\n\n    # Create list of indices for 3D ROIs spanning the entire Z direction\n    list_indices = convert_ROI_table_to_indices(\n        ROI_table,\n        level=level,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(list_indices, input_ROI_table)\n\n    # If we are not planning to use masked loading, fail for overlapping ROIs\n    if not use_masks:\n        overlap = find_overlaps_in_ROI_indices(list_indices)\n        if overlap:\n            raise ValueError(\n                f\"ROI indices created from {input_ROI_table} table have \"\n                \"overlaps, but we are not using masked loading.\"\n            )\n\n    # Select 2D/3D behavior and set some parameters\n    do_3D = data_zyx.shape[0] > 1 and len(data_zyx.shape) == 3\n    if do_3D:\n        if anisotropy is None:\n            # Compute anisotropy as pixel_size_z/pixel_size_x\n            anisotropy = (\n                actual_res_pxl_sizes_zyx[0] / actual_res_pxl_sizes_zyx[2]\n            )\n        logger.info(f\"Anisotropy: {anisotropy}\")\n\n    # Rescale datasets (only relevant for level>0)\n    if ngff_image_meta.axes_names[0] != \"c\":\n        raise ValueError(\n            \"Cannot set `remove_channel_axis=True` for multiscale \"\n            f\"metadata with axes={ngff_image_meta.axes_names}. \"\n            'First axis should have name \"c\".'\n        )\n    new_datasets = rescale_datasets(\n        datasets=[ds.dict() for ds in ngff_image_meta.datasets],\n        coarsening_xy=coarsening_xy,\n        reference_level=level,\n        remove_channel_axis=True,\n    )\n\n    label_attrs = {\n        \"image-label\": {\n            \"version\": __OME_NGFF_VERSION__,\n            \"source\": {\"image\": \"../../\"},\n        },\n        \"multiscales\": [\n            {\n                \"name\": output_label_name,\n                \"version\": __OME_NGFF_VERSION__,\n                \"axes\": [\n                    ax.dict()\n                    for ax in ngff_image_meta.multiscale.axes\n                    if ax.type != \"channel\"\n                ],\n                \"datasets\": new_datasets,\n            }\n        ],\n    }\n\n    image_group = zarr.group(zarrurl)\n    label_group = prepare_label_group(\n        image_group,\n        output_label_name,\n        overwrite=overwrite,\n        label_attrs=label_attrs,\n        logger=logger,\n    )\n\n    logger.info(\n        f\"Helper function `prepare_label_group` returned {label_group=}\"\n    )\n    logger.info(f\"Output label path: {zarrurl}/labels/{output_label_name}/0\")\n    store = zarr.storage.FSStore(f\"{zarrurl}/labels/{output_label_name}/0\")\n    label_dtype = np.uint32\n\n    # Ensure that all output shapes & chunks are 3D (for 2D data: (1, y, x))\n    # https://github.com/fractal-analytics-platform/fractal-tasks-core/issues/398\n    shape = data_zyx.shape\n    if len(shape) == 2:\n        shape = (1, *shape)\n    chunks = data_zyx.chunksize\n    if len(chunks) == 2:\n        chunks = (1, *chunks)\n    mask_zarr = zarr.create(\n        shape=shape,\n        chunks=chunks,\n        dtype=label_dtype,\n        store=store,\n        overwrite=False,\n        dimension_separator=\"/\",\n    )\n\n    logger.info(\n        f\"mask will have shape {data_zyx.shape} \"\n        f\"and chunks {data_zyx.chunks}\"\n    )\n\n    # Initialize cellpose\n    gpu = use_gpu and cellpose.core.use_gpu()\n    if pretrained_model:\n        model = models.CellposeModel(\n            gpu=gpu, pretrained_model=pretrained_model\n        )\n    else:\n        model = models.CellposeModel(gpu=gpu, model_type=model_type)\n\n    # Initialize other things\n    logger.info(f\"Start cellpose_segmentation task for {zarrurl}\")\n    logger.info(f\"relabeling: {relabeling}\")\n    logger.info(f\"do_3D: {do_3D}\")\n    logger.info(f\"use_gpu: {gpu}\")\n    logger.info(f\"level: {level}\")\n    logger.info(f\"model_type: {model_type}\")\n    logger.info(f\"pretrained_model: {pretrained_model}\")\n    logger.info(f\"anisotropy: {anisotropy}\")\n    logger.info(\"Total well shape/chunks:\")\n    logger.info(f\"{data_zyx.shape}\")\n    logger.info(f\"{data_zyx.chunks}\")\n    if channel2:\n        logger.info(\"Dual channel input for cellpose model\")\n        logger.info(f\"{data_zyx_c2.shape}\")\n        logger.info(f\"{data_zyx_c2.chunks}\")\n\n    # Counters for relabeling\n    if relabeling:\n        num_labels_tot = 0\n\n    # Iterate over ROIs\n    num_ROIs = len(list_indices)\n\n    if output_ROI_table:\n        bbox_dataframe_list = []\n\n    logger.info(f\"Now starting loop over {num_ROIs} ROIs\")\n    for i_ROI, indices in enumerate(list_indices):\n        # Define region\n        s_z, e_z, s_y, e_y, s_x, e_x = indices[:]\n        region = (\n            slice(s_z, e_z),\n            slice(s_y, e_y),\n            slice(s_x, e_x),\n        )\n        logger.info(f\"Now processing ROI {i_ROI+1}/{num_ROIs}\")\n\n        # Prepare single-channel or dual-channel input for cellpose\n        if channel2:\n            # Dual channel mode, first channel is the membrane channel\n            img_1 = load_region(\n                data_zyx,\n                region,\n                compute=True,\n                return_as_3D=True,\n            )\n            img_np = np.zeros((2, *img_1.shape))\n            img_np[0, :, :, :] = img_1\n            img_np[1, :, :, :] = load_region(\n                data_zyx_c2,\n                region,\n                compute=True,\n                return_as_3D=True,\n            )\n            channels = [1, 2]\n        else:\n            img_np = np.expand_dims(\n                load_region(data_zyx, region, compute=True, return_as_3D=True),\n                axis=0,\n            )\n            channels = [0, 0]\n\n        # Prepare keyword arguments for segment_ROI function\n        kwargs_segment_ROI = dict(\n            model=model,\n            channels=channels,\n            do_3D=do_3D,\n            anisotropy=anisotropy,\n            label_dtype=label_dtype,\n            diameter=diameter_level0 / coarsening_xy**level,\n            cellprob_threshold=cellprob_threshold,\n            flow_threshold=flow_threshold,\n            min_size=min_size,\n            augment=augment,\n            net_avg=net_avg,\n        )\n\n        # Prepare keyword arguments for preprocessing function\n        preprocessing_kwargs = {}\n        if use_masks:\n            preprocessing_kwargs = dict(\n                region=region,\n                current_label_path=f\"{zarrurl}/labels/{output_label_name}/0\",\n                ROI_table_path=ROI_table_path,\n                ROI_positional_index=i_ROI,\n            )\n\n        # Call segment_ROI through the masked-loading wrapper, which includes\n        # pre/post-processing functions if needed\n        new_label_img = masked_loading_wrapper(\n            image_array=img_np,\n            function=segment_ROI,\n            kwargs=kwargs_segment_ROI,\n            use_masks=use_masks,\n            preprocessing_kwargs=preprocessing_kwargs,\n        )\n\n        # Shift labels and update relabeling counters\n        if relabeling:\n            num_labels_roi = np.max(new_label_img)\n            new_label_img[new_label_img > 0] += num_labels_tot\n            num_labels_tot += num_labels_roi\n\n            # Write some logs\n            logger.info(f\"ROI {indices}, {num_labels_roi=}, {num_labels_tot=}\")\n\n            # Check that total number of labels is under control\n            if num_labels_tot > np.iinfo(label_dtype).max:\n                raise ValueError(\n                    \"ERROR in re-labeling:\"\n                    f\"Reached {num_labels_tot} labels, \"\n                    f\"but dtype={label_dtype}\"\n                )\n\n        if output_ROI_table:\n            bbox_df = array_to_bounding_box_table(\n                new_label_img,\n                actual_res_pxl_sizes_zyx,\n                origin_zyx=(s_z, s_y, s_x),\n            )\n\n            bbox_dataframe_list.append(bbox_df)\n\n            overlap_list = []\n            for df in bbox_dataframe_list:\n                overlap_list.extend(\n                    get_overlapping_pairs_3D(df, full_res_pxl_sizes_zyx)\n                )\n            if len(overlap_list) > 0:\n                logger.warning(\n                    f\"{len(overlap_list)} bounding-box pairs overlap\"\n                )\n\n        # Compute and store 0-th level to disk\n        da.array(new_label_img).to_zarr(\n            url=mask_zarr,\n            region=region,\n            compute=True,\n        )\n\n    logger.info(\n        f\"End cellpose_segmentation task for {zarrurl}, \"\n        \"now building pyramids.\"\n    )\n\n    # Starting from on-disk highest-resolution data, build and write to disk a\n    # pyramid of coarser levels\n    build_pyramid(\n        zarrurl=f\"{zarrurl}/labels/{output_label_name}\",\n        overwrite=overwrite,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        chunksize=chunks,\n        aggregation_function=np.max,\n    )\n\n    logger.info(\"End building pyramids\")\n\n    if output_ROI_table:\n        # Handle the case where `bbox_dataframe_list` is empty (typically\n        # because list_indices is also empty)\n        if len(bbox_dataframe_list) == 0:\n            bbox_dataframe_list = [empty_bounding_box_table()]\n        # Concatenate all ROI dataframes\n        df_well = pd.concat(bbox_dataframe_list, axis=0, ignore_index=True)\n        df_well.index = df_well.index.astype(str)\n        # Extract labels and drop them from df_well\n        labels = pd.DataFrame(df_well[\"label\"].astype(str))\n        df_well.drop(labels=[\"label\"], axis=1, inplace=True)\n        # Convert all to float (warning: some would be int, in principle)\n        bbox_dtype = np.float32\n        df_well = df_well.astype(bbox_dtype)\n        # Convert to anndata\n        bbox_table = ad.AnnData(df_well, dtype=bbox_dtype)\n        bbox_table.obs = labels\n\n        # Write to zarr group\n        image_group = zarr.group(f\"{in_path}/{component}\")\n        logger.info(\n            \"Now writing bounding-box ROI table to \"\n            f\"{in_path}/{component}/tables/{output_ROI_table}\"\n        )\n        table_attrs = {\n            \"type\": \"ngff:region_table\",\n            \"region\": {\"path\": f\"../labels/{output_label_name}\"},\n            \"instance_key\": \"label\",\n        }\n        write_table(\n            image_group,\n            output_ROI_table,\n            bbox_table,\n            overwrite=overwrite,\n            logger=logger,\n            table_attrs=table_attrs,\n        )\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/cellpose_segmentation/#fractal_tasks_core.tasks.cellpose_segmentation.segment_ROI","title":"segment_ROI(x, model=None, do_3D=True, channels=[0, 0], anisotropy=None, diameter=30.0, cellprob_threshold=0.0, flow_threshold=0.4, label_dtype=None, augment=False, net_avg=False, min_size=15)","text":"

Internal function that runs Cellpose segmentation for a single ROI.

PARAMETER DESCRIPTION x

4D numpy array.

TYPE: ndarray

model

An instance of models.CellposeModel.

TYPE: CellposeModel DEFAULT: None

do_3D

If True, cellpose runs in 3D mode: runs on xy, xz & yz planes, then averages the flows.

TYPE: bool DEFAULT: True

channels

Which channels to use. If only one channel is provided, [0, 0] should be used. If two channels are provided (the first dimension of x has length of 2), [1, 2] should be used (x[0, :, :,:] contains the membrane channel and x[1, :, :, :] contains the nuclear channel).

TYPE: list[int] DEFAULT: [0, 0]

anisotropy

Set anisotropy rescaling factor for Z dimension.

TYPE: Optional[float] DEFAULT: None

diameter

Expected object diameter in pixels for cellpose.

TYPE: float DEFAULT: 30.0

cellprob_threshold

Cellpose model parameter.

TYPE: float DEFAULT: 0.0

flow_threshold

Cellpose model parameter.

TYPE: float DEFAULT: 0.4

label_dtype

Label images are cast into this np.dtype.

TYPE: Optional[dtype] DEFAULT: None

augment

Whether to use cellpose augmentation to tile images with overlap.

TYPE: bool DEFAULT: False

net_avg

Whether to use cellpose net averaging to run the 4 built-in networks (useful for nuclei, cyto and cyto2, not sure it works for the others).

TYPE: bool DEFAULT: False

min_size

Minimum size of the segmented objects.

TYPE: int DEFAULT: 15

Source code in fractal_tasks_core/tasks/cellpose_segmentation.py 
def segment_ROI(\n    x: np.ndarray,\n    model: models.CellposeModel = None,\n    do_3D: bool = True,\n    channels: list[int] = [0, 0],\n    anisotropy: Optional[float] = None,\n    diameter: float = 30.0,\n    cellprob_threshold: float = 0.0,\n    flow_threshold: float = 0.4,\n    label_dtype: Optional[np.dtype] = None,\n    augment: bool = False,\n    net_avg: bool = False,\n    min_size: int = 15,\n) -> np.ndarray:\n\"\"\"\n    Internal function that runs Cellpose segmentation for a single ROI.\n\n    Args:\n        x: 4D numpy array.\n        model: An instance of `models.CellposeModel`.\n        do_3D: If `True`, cellpose runs in 3D mode: runs on xy, xz & yz planes,\n            then averages the flows.\n        channels: Which channels to use. If only one channel is provided, `[0,\n            0]` should be used. If two channels are provided (the first\n            dimension of `x` has length of 2), `[1, 2]` should be used\n            (`x[0, :, :,:]` contains the membrane channel and\n            `x[1, :, :, :]` contains the nuclear channel).\n        anisotropy: Set anisotropy rescaling factor for Z dimension.\n        diameter: Expected object diameter in pixels for cellpose.\n        cellprob_threshold: Cellpose model parameter.\n        flow_threshold: Cellpose model parameter.\n        label_dtype: Label images are cast into this `np.dtype`.\n        augment: Whether to use cellpose augmentation to tile images with\n            overlap.\n        net_avg: Whether to use cellpose net averaging to run the 4 built-in\n            networks (useful for `nuclei`, `cyto` and `cyto2`, not sure it\n            works for the others).\n        min_size: Minimum size of the segmented objects.\n    \"\"\"\n\n    # Write some debugging info\n    logger.info(\n        \"[segment_ROI] START |\"\n        f\" x: {type(x)}, {x.shape} |\"\n        f\" {do_3D=} |\"\n        f\" {model.diam_mean=} |\"\n        f\" {diameter=} |\"\n        f\" {flow_threshold=}\"\n    )\n\n    # Actual labeling\n    t0 = time.perf_counter()\n    mask, _, _ = model.eval(\n        x,\n        channels=channels,\n        do_3D=do_3D,\n        net_avg=net_avg,\n        augment=augment,\n        diameter=diameter,\n        anisotropy=anisotropy,\n        cellprob_threshold=cellprob_threshold,\n        flow_threshold=flow_threshold,\n        min_size=min_size,\n    )\n\n    if mask.ndim == 2:\n        # If we get a 2D image, we still return it as a 3D array\n        mask = np.expand_dims(mask, axis=0)\n    t1 = time.perf_counter()\n\n    # Write some debugging info\n    logger.info(\n        \"[segment_ROI] END   |\"\n        f\" Elapsed: {t1-t0:.3f} s |\"\n        f\" {mask.shape=},\"\n        f\" {mask.dtype=} (then {label_dtype}),\"\n        f\" {np.max(mask)=} |\"\n        f\" {model.diam_mean=} |\"\n        f\" {diameter=} |\"\n        f\" {flow_threshold=}\"\n    )\n\n    return mask.astype(label_dtype)\n
"},{"location":"reference/fractal_tasks_core/tasks/compress_tif/","title":"compress_tif","text":"

Task to compress tiff images.

This task cannot be used in the current form, and it should first be aligned with the other tasks' structure.

"},{"location":"reference/fractal_tasks_core/tasks/compress_tif/#fractal_tasks_core.tasks.compress_tif.compress_tif","title":"compress_tif(in_path, out_path, delete_input=False)","text":"

Compress tiff files.

PARAMETER DESCRIPTION in_path

directory containing the input files.

TYPE: str

out_path

directory containing the output files.

TYPE: str

delete_input

delete input files.

TYPE: bool DEFAULT: False

Source code in fractal_tasks_core/tasks/compress_tif.py 
def compress_tif(in_path: str, out_path: str, delete_input: bool = False):\n\n\"\"\"\n    Compress tiff files.\n\n    Args:\n        in_path: directory containing the input files.\n        out_path: directory containing the output files.\n        delete_input: delete input files.\n    \"\"\"\n\n    # Sanitize input/output paths\n    if not in_path.endswith(\"/\"):\n        in_path += \"/\"\n    if not out_path.endswith(\"/\"):\n        out_path += \"/\"\n\n    # Create output path, if needed\n    if not os.path.exists(out_path):\n        os.makedirs(out_path)\n\n    num_img_compressed = 0\n    num_img_deleted = 0\n    for filename in glob.glob(in_path + \"*.tif\"):\n        newfilename = os.path.join(out_path, os.path.basename(filename))\n\n        # Save compressed image\n        with Image.open(filename) as image:\n            image.save(newfilename, format=\"tiff\", compression=\"tiff_lzw\")\n        print(f\"Raw:        {filename}\\nCompressed: {newfilename}\")\n        num_img_compressed += 1\n\n        # Delete raw image, if needed\n        if delete_input:\n            try:\n                os.remove(filename)\n                print(f\"Deleted:    {filename}\")\n                num_img_deleted += 1\n            except OSError as e:\n                print(\"ERROR: %s : %s\" % (filename, e.strerror))\n\n        print()\n\n    return num_img_compressed, num_img_deleted\n
"},{"location":"reference/fractal_tasks_core/tasks/copy_ome_zarr/","title":"copy_ome_zarr","text":"

Task that copies the structure of an OME-NGFF zarr array to a new one.

"},{"location":"reference/fractal_tasks_core/tasks/copy_ome_zarr/#fractal_tasks_core.tasks.copy_ome_zarr.copy_ome_zarr","title":"copy_ome_zarr(*, input_paths, output_path, metadata, project_to_2D=True, suffix='mip', ROI_table_names=('FOV_ROI_table', 'well_ROI_table'), overwrite=False)","text":"

Duplicate an input zarr structure to a new path.

This task copies all the structure, but none of the image data:

  • For each plate, create a new zarr group with the same attributes as the original one.
  • For each well (in each plate), create a new zarr subgroup with the same attributes as the original one.
  • For each image (in each well), create a new zarr subgroup with the same attributes as the original one.
  • For each image (in each well), copy the relevant AnnData tables from the original source.

Note: this task makes use of methods from the Attributes class, see https://zarr.readthedocs.io/en/stable/api/attrs.html.

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Path were the output of this task is stored. Example: \"/some/path/\" => puts the new OME-Zarr file in the same folder as the input OME-Zarr file \"/some/new_path\" => puts the new OME-Zarr file into a new folder at /some/new_path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata: plate: List of plates (e.g. [\"MyPlate.zarr\"]); well: List of wells in the OME-Zarr plate (e.g. [\"MyPlate.zarr/B/03/MyPlate.zarr/B/05\"]); \"image\": List of images in the OME-Zarr plate (e.g. [\"MyPlate.zarr/B/03/0\", \"MyPlate.zarr/B/05/0\"]). standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

project_to_2D

If True, apply a 3D->2D projection to the ROI tables that are copied to the new OME-Zarr.

TYPE: bool DEFAULT: True

suffix

The suffix that is used to transform plate.zarr into plate_suffix.zarr. Note that None is not currently supported.

TYPE: str DEFAULT: 'mip'

ROI_table_names

List of Anndata table names to be copied. Note: copying non-ROI tables may fail if project_to_2D=True.

TYPE: tuple[str, ...] DEFAULT: ('FOV_ROI_table', 'well_ROI_table')

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION dict[str, Any]

An update to the metadata table with new plate, well, image entries (now with the suffix in the plate name).

Source code in fractal_tasks_core/tasks/copy_ome_zarr.py 
@validate_arguments\ndef copy_ome_zarr(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    metadata: dict[str, Any],\n    project_to_2D: bool = True,\n    suffix: str = \"mip\",\n    ROI_table_names: tuple[str, ...] = (\"FOV_ROI_table\", \"well_ROI_table\"),\n    overwrite: bool = False,\n) -> dict[str, Any]:\n\n\"\"\"\n    Duplicate an input zarr structure to a new path.\n\n    This task copies all the structure, but none of the image data:\n\n    - For each plate, create a new zarr group with the same attributes as\n       the original one.\n    - For each well (in each plate), create a new zarr subgroup with the\n       same attributes as the original one.\n    - For each image (in each well), create a new zarr subgroup with the\n       same attributes as the original one.\n    - For each image (in each well), copy the relevant AnnData tables from\n       the original source.\n\n    Note: this task makes use of methods from the `Attributes` class, see\n    https://zarr.readthedocs.io/en/stable/api/attrs.html.\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: Path were the output of this task is stored. Example:\n            `\"/some/path/\"` => puts the new OME-Zarr file in the same folder as\n            the input OME-Zarr file `\"/some/new_path\"` => puts the new OME-Zarr\n            file into a new folder at `/some/new_path`. (standard argument for\n            Fractal tasks, managed by Fractal server).\n        metadata: Dictionary containing metadata about the OME-Zarr. This task\n            requires the following elements to be present in the metadata:\n            `plate`: List of plates\n            (e.g. `[\"MyPlate.zarr\"]`);\n            `well`: List of wells in the OME-Zarr plate\n            (e.g. `[\"MyPlate.zarr/B/03/MyPlate.zarr/B/05\"]`);\n            \"image\": List of images in the OME-Zarr plate\n            (e.g. `[\"MyPlate.zarr/B/03/0\", \"MyPlate.zarr/B/05/0\"]`).\n            standard argument for Fractal tasks, managed by Fractal server).\n        project_to_2D: If `True`, apply a 3D->2D projection to the ROI tables\n            that are copied to the new OME-Zarr.\n        suffix: The suffix that is used to transform `plate.zarr` into\n            `plate_suffix.zarr`. Note that `None` is not currently supported.\n        ROI_table_names: List of Anndata table names to be copied. Note:\n            copying non-ROI tables may fail if `project_to_2D=True`.\n        overwrite: If `True`, overwrite the task output.\n\n    Returns:\n        An update to the metadata table with new `plate`, `well`, `image`\n            entries (now with the suffix in the plate name).\n    \"\"\"\n\n    # Preliminary check\n    if len(input_paths) > 1:\n        raise NotImplementedError\n    if suffix is None:\n        # FIXME create a standard suffix (with timestamp)\n        raise NotImplementedError\n\n    # List all plates\n    in_path = Path(input_paths[0])\n    list_plates = [\n        p.as_posix()\n        for p in Path(in_path).glob(\"*.zarr\")\n        if p.name in metadata[\"plate\"]\n    ]\n    logger.info(f\"{list_plates=}\")\n\n    meta_update: dict[str, Any] = {\"copy_ome_zarr\": {}}\n    meta_update[\"copy_ome_zarr\"][\"suffix\"] = suffix\n    meta_update[\"copy_ome_zarr\"][\"sources\"] = {}\n\n    # Loop over all plates\n    for zarrurl_old in list_plates:\n        zarrfile = zarrurl_old.split(\"/\")[-1]\n        old_plate_name = zarrfile.split(\".zarr\")[0]\n        new_plate_name = f\"{old_plate_name}_{suffix}\"\n        new_plate_dir = Path(output_path).resolve()\n        zarrurl_new = f\"{(new_plate_dir / new_plate_name).as_posix()}.zarr\"\n        meta_update[\"copy_ome_zarr\"][\"sources\"][new_plate_name] = zarrurl_old\n\n        logger.info(f\"{zarrurl_old=}\")\n        logger.info(f\"{zarrurl_new=}\")\n        logger.info(f\"{meta_update=}\")\n\n        # Replicate plate attrs\n        old_plate_group = zarr.open_group(zarrurl_old, mode=\"r\")\n        new_plate_group = open_zarr_group_with_overwrite(\n            zarrurl_new, overwrite=overwrite\n        )\n        new_plate_group.attrs.put(old_plate_group.attrs.asdict())\n\n        well_paths = [\n            well[\"path\"] for well in new_plate_group.attrs[\"plate\"][\"wells\"]\n        ]\n        logger.info(f\"{well_paths=}\")\n        for well_path in well_paths:\n\n            # Replicate well attrs\n            old_well_group = zarr.open_group(\n                f\"{zarrurl_old}/{well_path}\", mode=\"r\"\n            )\n            new_well_group = zarr.group(f\"{zarrurl_new}/{well_path}\")\n            new_well_group.attrs.put(old_well_group.attrs.asdict())\n\n            image_paths = [\n                image[\"path\"]\n                for image in new_well_group.attrs[\"well\"][\"images\"]\n            ]\n            logger.info(f\"{image_paths=}\")\n\n            for image_path in image_paths:\n\n                # Replicate image attrs\n                old_image_group = zarr.open_group(\n                    f\"{zarrurl_old}/{well_path}/{image_path}\", mode=\"r\"\n                )\n                new_image_group = zarr.group(\n                    f\"{zarrurl_new}/{well_path}/{image_path}\"\n                )\n                new_image_group.attrs.put(old_image_group.attrs.asdict())\n\n                # Extract pixel sizes, if needed\n                if ROI_table_names:\n\n                    new_tables_group = new_image_group.create_group(\"tables/\")\n                    new_tables_group.attrs[\"tables\"] = ROI_table_names\n                    if project_to_2D:\n                        path_image = f\"{zarrurl_old}/{well_path}/{image_path}\"\n                        ngff_image_meta = load_NgffImageMeta(path_image)\n                        pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(\n                            level=0\n                        )\n                        pxl_size_z = pxl_sizes_zyx[0]\n\n                    # Copy the tables in ROI_table_names\n                    for ROI_table_name in ROI_table_names:\n\n                        logger.info(\n                            f\"I will now read {ROI_table_name} from \"\n                            f\"{zarrurl_old=}, convert it to 2D, and \"\n                            \"write it back to the new zarr file.\"\n                        )\n                        ROI_table = ad.read_zarr(\n                            f\"{zarrurl_old}/{well_path}/{image_path}/\"\n                            f\"tables/{ROI_table_name}\"\n                        )\n                        # Convert 3D ROIs to 2D\n                        if project_to_2D:\n                            new_ROI_table = convert_ROIs_from_3D_to_2D(\n                                ROI_table, pxl_size_z\n                            )\n                        # Write new table\n                        write_elem(\n                            new_tables_group, ROI_table_name, new_ROI_table\n                        )\n\n    for key in [\"plate\", \"well\", \"image\"]:\n        meta_update[key] = [\n            component.replace(\".zarr\", f\"_{suffix}.zarr\")\n            for component in metadata[key]\n        ]\n\n    return meta_update\n
"},{"location":"reference/fractal_tasks_core/tasks/create_ome_zarr/","title":"create_ome_zarr","text":"

Create structure for OME-NGFF zarr array.

"},{"location":"reference/fractal_tasks_core/tasks/create_ome_zarr/#fractal_tasks_core.tasks.create_ome_zarr.create_ome_zarr","title":"create_ome_zarr(*, input_paths, output_path, metadata, allowed_channels, image_glob_patterns=None, num_levels=5, coarsening_xy=2, image_extension='tif', metadata_table_file=None, overwrite=False)","text":"

Create a OME-NGFF zarr folder, without reading/writing image data.

Find plates (for each folder in input_paths):

  • glob image files,
  • parse metadata from image filename to identify plates,
  • identify populated channels.

Create a zarr folder (for each plate):

  • parse mlf metadata,
  • identify wells and field of view (FOV),
  • create FOV ZARR,
  • verify that channels are uniform (i.e., same channels).
PARAMETER DESCRIPTION input_paths

List of input paths where the image data from the microscope is stored (as TIF or PNG). Should point to the parent folder containing the images and the metadata files MeasurementData.mlf and MeasurementDetail.mrf (if present). Example: [\"/some/path/\"]. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Path were the output of this task is stored. Example: \"/some/path/\" => puts the new OME-Zarr file in the \"/some/path/\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

allowed_channels

A list of OmeroChannel s, where each channel must include the wavelength_id attribute and where the wavelength_id values must be unique across the list.

TYPE: list[OmeroChannel]

image_glob_patterns

If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: image_glob_pattern=[\"*_B03_*\"] => only process well B03 image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"] => only process well C09, field of view 16 and Z planes 0-59.

TYPE: Optional[list[str]] DEFAULT: None

num_levels

Number of resolution-pyramid levels. If set to 5, there will be the full-resolution level and 4 levels of downsampled images.

TYPE: int DEFAULT: 5

coarsening_xy

Linear coarsening factor between subsequent levels. If set to 2, level 1 is 2x downsampled, level 2 is 4x downsampled etc.

TYPE: int DEFAULT: 2

image_extension

Filename extension of images (e.g. \"tif\" or \"png\")

TYPE: str DEFAULT: 'tif'

metadata_table_file

If None, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, the full path to a csv file containing the parsed metadata table.

TYPE: Optional[str] DEFAULT: None

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION dict[str, Any]

A metadata dictionary containing important metadata about the OME-Zarr plate, the images and some parameters required by downstream tasks (like num_levels).

Source code in fractal_tasks_core/tasks/create_ome_zarr.py 
@validate_arguments\ndef create_ome_zarr(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    metadata: dict[str, Any],\n    allowed_channels: list[OmeroChannel],\n    image_glob_patterns: Optional[list[str]] = None,\n    num_levels: int = 5,\n    coarsening_xy: int = 2,\n    image_extension: str = \"tif\",\n    metadata_table_file: Optional[str] = None,\n    overwrite: bool = False,\n) -> dict[str, Any]:\n\"\"\"\n    Create a OME-NGFF zarr folder, without reading/writing image data.\n\n    Find plates (for each folder in input_paths):\n\n    - glob image files,\n    - parse metadata from image filename to identify plates,\n    - identify populated channels.\n\n    Create a zarr folder (for each plate):\n\n    - parse mlf metadata,\n    - identify wells and field of view (FOV),\n    - create FOV ZARR,\n    - verify that channels are uniform (i.e., same channels).\n\n    Args:\n        input_paths: List of input paths where the image data from\n            the microscope is stored (as TIF or PNG).  Should point to the\n            parent folder containing the images and the metadata files\n            `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present).\n            Example: `[\"/some/path/\"]`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: Path were the output of this task is stored.\n            Example: \"/some/path/\" => puts the new OME-Zarr file in the\n            \"/some/path/\".\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        allowed_channels: A list of `OmeroChannel` s, where each channel must\n            include the `wavelength_id` attribute and where the\n            `wavelength_id` values must be unique across the list.\n        image_glob_patterns: If specified, only parse images with filenames\n            that match with all these patterns. Patterns must be defined as in\n            https://docs.python.org/3/library/fnmatch.html, Example:\n            `image_glob_pattern=[\"*_B03_*\"]` => only process well B03\n            `image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"]` =>\n            only process well C09, field of view 16 and Z planes 0-59.\n        num_levels: Number of resolution-pyramid levels. If set to `5`, there\n            will be the full-resolution level and 4 levels of\n            downsampled images.\n        coarsening_xy: Linear coarsening factor between subsequent levels.\n            If set to `2`, level 1 is 2x downsampled, level 2 is\n            4x downsampled etc.\n        image_extension: Filename extension of images (e.g. `\"tif\"` or `\"png\"`)\n        metadata_table_file: If `None`, parse Yokogawa metadata from mrf/mlf\n            files in the input_path folder; else, the full path to a csv file\n            containing the parsed metadata table.\n        overwrite: If `True`, overwrite the task output.\n\n    Returns:\n        A metadata dictionary containing important metadata about the OME-Zarr\n            plate, the images and some parameters required by downstream tasks\n            (like `num_levels`).\n    \"\"\"\n\n    # Preliminary checks on metadata_table_file\n    if metadata_table_file:\n        if not metadata_table_file.endswith(\".csv\"):\n            raise ValueError(f\"{metadata_table_file=} is not a csv file\")\n        if not os.path.isfile(metadata_table_file):\n            raise FileNotFoundError(f\"{metadata_table_file=} does not exist\")\n\n    # Identify all plates and all channels, across all input folders\n    plates = []\n    actual_wavelength_ids = None\n    dict_plate_paths = {}\n    dict_plate_prefixes: dict[str, Any] = {}\n\n    # Preliminary checks on allowed_channels argument\n    check_unique_wavelength_ids(allowed_channels)\n\n    for in_path_str in input_paths:\n        in_path = Path(in_path_str)\n\n        # Glob image filenames\n        patterns = [f\"*.{image_extension}\"]\n        if image_glob_patterns:\n            patterns.extend(image_glob_patterns)\n        input_filenames = glob_with_multiple_patterns(\n            folder=in_path_str,\n            patterns=patterns,\n        )\n\n        tmp_wavelength_ids = []\n        tmp_plates = []\n        for fn in input_filenames:\n            try:\n                filename_metadata = parse_filename(Path(fn).name)\n                plate_prefix = filename_metadata[\"plate_prefix\"]\n                plate = filename_metadata[\"plate\"]\n                if plate not in dict_plate_prefixes.keys():\n                    dict_plate_prefixes[plate] = plate_prefix\n                tmp_plates.append(plate)\n                A = filename_metadata[\"A\"]\n                C = filename_metadata[\"C\"]\n                tmp_wavelength_ids.append(f\"A{A}_C{C}\")\n            except ValueError as e:\n                logger.warning(\n                    f'Skipping \"{Path(fn).name}\". Original error: ' + str(e)\n                )\n        tmp_plates = sorted(list(set(tmp_plates)))\n        tmp_wavelength_ids = sorted(list(set(tmp_wavelength_ids)))\n\n        info = (\n            \"Listing plates/channels:\\n\"\n            f\"Folder:   {in_path_str}\\n\"\n            f\"Patterns: {patterns}\\n\"\n            f\"Plates:   {tmp_plates}\\n\"\n            f\"Channels: {tmp_wavelength_ids}\\n\"\n        )\n\n        # Check that only one plate is found\n        if len(tmp_plates) > 1:\n            raise ValueError(f\"{info}ERROR: {len(tmp_plates)} plates detected\")\n        elif len(tmp_plates) == 0:\n            raise ValueError(f\"{info}ERROR: No plates detected\")\n        plate = tmp_plates[0]\n\n        # If plate already exists in other folder, add suffix\n        if plate in plates:\n            ind = 1\n            new_plate = f\"{plate}_{ind}\"\n            while new_plate in plates:\n                new_plate = f\"{plate}_{ind}\"\n                ind += 1\n            logger.info(\n                f\"WARNING: {plate} already exists, renaming it as {new_plate}\"\n            )\n            plates.append(new_plate)\n            dict_plate_prefixes[new_plate] = dict_plate_prefixes[plate]\n            plate = new_plate\n        else:\n            plates.append(plate)\n\n        # Check that channels are the same as in previous plates\n        if actual_wavelength_ids is None:\n            actual_wavelength_ids = tmp_wavelength_ids[:]\n        else:\n            if actual_wavelength_ids != tmp_wavelength_ids:\n                raise ValueError(\n                    f\"ERROR\\n{info}\\nERROR:\"\n                    f\" expected channels {actual_wavelength_ids}\"\n                )\n\n        # Update dict_plate_paths\n        dict_plate_paths[plate] = in_path\n\n    # Check that all channels are in the allowed_channels\n    allowed_wavelength_ids = [\n        channel.wavelength_id for channel in allowed_channels\n    ]\n    if not set(actual_wavelength_ids).issubset(set(allowed_wavelength_ids)):\n        msg = \"ERROR in create_ome_zarr\\n\"\n        msg += f\"actual_wavelength_ids: {actual_wavelength_ids}\\n\"\n        msg += f\"allowed_wavelength_ids: {allowed_wavelength_ids}\\n\"\n        raise ValueError(msg)\n\n    # Create actual_channels, i.e. a list of the channel dictionaries which are\n    # present\n    actual_channels = [\n        channel\n        for channel in allowed_channels\n        if channel.wavelength_id in actual_wavelength_ids\n    ]\n\n    zarrurls: dict[str, list[str]] = {\"plate\": [], \"well\": [], \"image\": []}\n\n    ################################################################\n    for plate in plates:\n        # Define plate zarr\n        zarrurl = f\"{plate}.zarr\"\n        in_path = dict_plate_paths[plate]\n        logger.info(f\"Creating {zarrurl}\")\n        # Call zarr.open_group wrapper, which handles overwrite=True/False\n        group_plate = open_zarr_group_with_overwrite(\n            str(Path(output_path) / zarrurl),\n            overwrite=overwrite,\n        )\n        zarrurls[\"plate\"].append(zarrurl)\n\n        # Obtain FOV-metadata dataframe\n\n        if metadata_table_file is None:\n            mrf_path = f\"{in_path}/MeasurementDetail.mrf\"\n            mlf_path = f\"{in_path}/MeasurementData.mlf\"\n\n            site_metadata, number_images_mlf = parse_yokogawa_metadata(\n                mrf_path,\n                mlf_path,\n                filename_patterns=image_glob_patterns,\n            )\n            site_metadata = remove_FOV_overlaps(site_metadata)\n\n        # If a metadata table was passed, load it and use it directly\n        else:\n            logger.warning(\n                \"Since a custom metadata table was provided, there will \"\n                \"be no additional check on the number of image files.\"\n            )\n            site_metadata = pd.read_csv(metadata_table_file)\n            site_metadata.set_index([\"well_id\", \"FieldIndex\"], inplace=True)\n\n        # Extract pixel sizes and bit_depth\n        pixel_size_z = site_metadata[\"pixel_size_z\"][0]\n        pixel_size_y = site_metadata[\"pixel_size_y\"][0]\n        pixel_size_x = site_metadata[\"pixel_size_x\"][0]\n        bit_depth = site_metadata[\"bit_depth\"][0]\n\n        if min(pixel_size_z, pixel_size_y, pixel_size_x) < 1e-9:\n            raise ValueError(pixel_size_z, pixel_size_y, pixel_size_x)\n\n        # Identify all wells\n        plate_prefix = dict_plate_prefixes[plate]\n\n        patterns = [f\"{plate_prefix}_*.{image_extension}\"]\n        if image_glob_patterns:\n            patterns.extend(image_glob_patterns)\n        plate_images = glob_with_multiple_patterns(\n            folder=str(in_path), patterns=patterns\n        )\n\n        wells = [\n            parse_filename(os.path.basename(fn))[\"well\"] for fn in plate_images\n        ]\n        wells = sorted(list(set(wells)))\n\n        # Verify that all wells have all channels\n        for well in wells:\n            patterns = [f\"{plate_prefix}_{well}_*.{image_extension}\"]\n            if image_glob_patterns:\n                patterns.extend(image_glob_patterns)\n            well_images = glob_with_multiple_patterns(\n                folder=str(in_path), patterns=patterns\n            )\n\n            # Check number of images matches with expected one\n            if metadata_table_file is None:\n                num_images_glob = len(well_images)\n                num_images_expected = number_images_mlf[well]\n                if num_images_glob != num_images_expected:\n                    raise ValueError(\n                        f\"Wrong number of images for {well=}\\n\"\n                        f\"Expected {num_images_expected} (from mlf file)\\n\"\n                        f\"Found {num_images_glob} files\\n\"\n                        \"Other parameters:\\n\"\n                        f\"  {image_extension=}\\n\"\n                        f\"  {image_glob_patterns=}\"\n                    )\n\n            well_wavelength_ids = []\n            for fpath in well_images:\n                try:\n                    filename_metadata = parse_filename(os.path.basename(fpath))\n                    well_wavelength_ids.append(\n                        f\"A{filename_metadata['A']}_C{filename_metadata['C']}\"\n                    )\n                except IndexError:\n                    logger.info(f\"Skipping {fpath}\")\n            well_wavelength_ids = sorted(list(set(well_wavelength_ids)))\n            if well_wavelength_ids != actual_wavelength_ids:\n                raise ValueError(\n                    f\"ERROR: well {well} in plate {plate} (prefix: \"\n                    f\"{plate_prefix}) has missing channels.\\n\"\n                    f\"Expected: {actual_channels}\\n\"\n                    f\"Found: {well_wavelength_ids}.\\n\"\n                )\n\n        well_rows_columns = [\n            ind for ind in sorted([(n[0], n[1:]) for n in wells])\n        ]\n        row_list = [\n            well_row_column[0] for well_row_column in well_rows_columns\n        ]\n        col_list = [\n            well_row_column[1] for well_row_column in well_rows_columns\n        ]\n        row_list = sorted(list(set(row_list)))\n        col_list = sorted(list(set(col_list)))\n\n        group_plate.attrs[\"plate\"] = {\n            \"acquisitions\": [{\"id\": 0, \"name\": plate}],\n            \"columns\": [{\"name\": col} for col in col_list],\n            \"rows\": [{\"name\": row} for row in row_list],\n            \"wells\": [\n                {\n                    \"path\": well_row_column[0] + \"/\" + well_row_column[1],\n                    \"rowIndex\": row_list.index(well_row_column[0]),\n                    \"columnIndex\": col_list.index(well_row_column[1]),\n                }\n                for well_row_column in well_rows_columns\n            ],\n        }\n\n        for row, column in well_rows_columns:\n\n            group_well = group_plate.create_group(f\"{row}/{column}/\")\n\n            group_well.attrs[\"well\"] = {\n                \"images\": [{\"path\": \"0\"}],\n                \"version\": __OME_NGFF_VERSION__,\n            }\n\n            group_image = group_well.create_group(\"0/\")  # noqa: F841\n            zarrurls[\"well\"].append(f\"{plate}.zarr/{row}/{column}/\")\n            zarrurls[\"image\"].append(f\"{plate}.zarr/{row}/{column}/0/\")\n\n            group_image.attrs[\"multiscales\"] = [\n                {\n                    \"version\": __OME_NGFF_VERSION__,\n                    \"axes\": [\n                        {\"name\": \"c\", \"type\": \"channel\"},\n                        {\n                            \"name\": \"z\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                        {\n                            \"name\": \"y\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                        {\n                            \"name\": \"x\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                    ],\n                    \"datasets\": [\n                        {\n                            \"path\": f\"{ind_level}\",\n                            \"coordinateTransformations\": [\n                                {\n                                    \"type\": \"scale\",\n                                    \"scale\": [\n                                        1,\n                                        pixel_size_z,\n                                        pixel_size_y\n                                        * coarsening_xy**ind_level,\n                                        pixel_size_x\n                                        * coarsening_xy**ind_level,\n                                    ],\n                                }\n                            ],\n                        }\n                        for ind_level in range(num_levels)\n                    ],\n                }\n            ]\n\n            group_image.attrs[\"omero\"] = {\n                \"id\": 1,  # FIXME does this depend on the plate number?\n                \"name\": \"TBD\",\n                \"version\": __OME_NGFF_VERSION__,\n                \"channels\": define_omero_channels(\n                    channels=actual_channels, bit_depth=bit_depth\n                ),\n            }\n\n            # Prepare AnnData tables for FOV/well ROIs\n            well_id = row + column\n            FOV_ROIs_table = prepare_FOV_ROI_table(site_metadata.loc[well_id])\n            well_ROIs_table = prepare_well_ROI_table(\n                site_metadata.loc[well_id]\n            )\n\n            # Write AnnData tables into the `tables` zarr group\n            write_table(\n                group_image,\n                \"FOV_ROI_table\",\n                FOV_ROIs_table,\n                overwrite=overwrite,\n                logger=logger,\n            )\n            write_table(\n                group_image,\n                \"well_ROI_table\",\n                well_ROIs_table,\n                overwrite=overwrite,\n                logger=logger,\n            )\n\n    # Check that the different images in each well have unique channel labels.\n    # Since we currently merge all fields of view in the same image, this check\n    # is useless. It should remain there to catch an error in case we switch\n    # back to one-image-per-field-of-view mode\n    for well_path in zarrurls[\"well\"]:\n        check_well_channel_labels(\n            well_zarr_path=str(Path(output_path) / well_path)\n        )\n\n    metadata_update = dict(\n        plate=zarrurls[\"plate\"],\n        well=zarrurls[\"well\"],\n        image=zarrurls[\"image\"],\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        image_extension=image_extension,\n        image_glob_patterns=image_glob_patterns,\n        original_paths=input_paths[:],\n    )\n    return metadata_update\n
"},{"location":"reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/","title":"create_ome_zarr_multiplex","text":"

Create OME-NGFF zarr group, for multiplexing dataset.

"},{"location":"reference/fractal_tasks_core/tasks/create_ome_zarr_multiplex/#fractal_tasks_core.tasks.create_ome_zarr_multiplex.create_ome_zarr_multiplex","title":"create_ome_zarr_multiplex(*, input_paths, output_path, metadata, allowed_channels, image_glob_patterns=None, num_levels=5, coarsening_xy=2, image_extension='tif', metadata_table_files=None, overwrite=False)","text":"

Create OME-NGFF structure and metadata to host a multiplexing dataset.

This task takes a set of image folders (i.e. different acquisition cycles) and build the internal structure and metadata of a OME-NGFF zarr group, without actually loading/writing the image data.

Each element in input_paths should be treated as a different acquisition.

PARAMETER DESCRIPTION input_paths

List of input paths where the image data from the microscope is stored (as TIF or PNG). Each element of the list is treated as another cycle of the multiplexing data, the cycles are ordered by their order in this list. Should point to the parent folder containing the images and the metadata files MeasurementData.mlf and MeasurementDetail.mrf (if present). Example: [\"/path/cycle1/\", \"/path/cycle2/\"]. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Path were the output of this task is stored. Example: \"/some/path/\" => puts the new OME-Zarr file in the /some/path/. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

allowed_channels

A dictionary of lists of OmeroChannels, where each channel must include the wavelength_id attribute and where the wavelength_id values must be unique across each list. Dictionary keys represent channel indices (\"0\",\"1\",..).

TYPE: dict[str, list[OmeroChannel]]

image_glob_patterns

If specified, only parse images with filenames that match with all these patterns. Patterns must be defined as in https://docs.python.org/3/library/fnmatch.html, Example: image_glob_pattern=[\"*_B03_*\"] => only process well B03 image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"] => only process well C09, field of view 16 and Z planes 0-59.

TYPE: Optional[list[str]] DEFAULT: None

num_levels

Number of resolution-pyramid levels. If set to 5, there will be the full-resolution level and 4 levels of downsampled images.

TYPE: int DEFAULT: 5

coarsening_xy

Linear coarsening factor between subsequent levels. If set to 2, level 1 is 2x downsampled, level 2 is 4x downsampled etc.

TYPE: int DEFAULT: 2

image_extension

Filename extension of images (e.g. \"tif\" or \"png\").

TYPE: str DEFAULT: 'tif'

metadata_table_files

If None, parse Yokogawa metadata from mrf/mlf files in the input_path folder; else, a dictionary of key-value pairs like (acquisition, path) with acquisition a string and path pointing to a csv file containing the parsed metadata table.

TYPE: Optional[dict[str, str]] DEFAULT: None

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION dict[str, Any]

A metadata dictionary containing important metadata about the OME-Zarr plate, the images and some parameters required by downstream tasks (like num_levels).

Source code in fractal_tasks_core/tasks/create_ome_zarr_multiplex.py 
@validate_arguments\ndef create_ome_zarr_multiplex(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    metadata: dict[str, Any],\n    allowed_channels: dict[str, list[OmeroChannel]],\n    image_glob_patterns: Optional[list[str]] = None,\n    num_levels: int = 5,\n    coarsening_xy: int = 2,\n    image_extension: str = \"tif\",\n    metadata_table_files: Optional[dict[str, str]] = None,\n    overwrite: bool = False,\n) -> dict[str, Any]:\n\"\"\"\n    Create OME-NGFF structure and metadata to host a multiplexing dataset.\n\n    This task takes a set of image folders (i.e. different acquisition cycles)\n    and build the internal structure and metadata of a OME-NGFF zarr group,\n    without actually loading/writing the image data.\n\n    Each element in input_paths should be treated as a different acquisition.\n\n    Args:\n        input_paths: List of input paths where the image data from the\n            microscope is stored (as TIF or PNG).  Each element of the list is\n            treated as another cycle of the multiplexing data, the cycles are\n            ordered by their order in this list.  Should point to the parent\n            folder containing the images and the metadata files\n            `MeasurementData.mlf` and `MeasurementDetail.mrf` (if present).\n            Example: `[\"/path/cycle1/\", \"/path/cycle2/\"]`. (standard argument\n            for Fractal tasks, managed by Fractal server).\n        output_path: Path were the output of this task is stored.\n            Example: `\"/some/path/\"` => puts the new OME-Zarr file in the\n            `/some/path/`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        allowed_channels: A dictionary of lists of `OmeroChannel`s, where\n            each channel must include the `wavelength_id` attribute and where\n            the `wavelength_id` values must be unique across each list.\n            Dictionary keys represent channel indices (`\"0\",\"1\",..`).\n        image_glob_patterns: If specified, only parse images with filenames\n            that match with all these patterns. Patterns must be defined as in\n            https://docs.python.org/3/library/fnmatch.html, Example:\n            `image_glob_pattern=[\"*_B03_*\"]` => only process well B03\n            `image_glob_pattern=[\"*_C09_*\", \"*F016*\", \"*Z[0-5][0-9]C*\"]` =>\n            only process well C09, field of view 16 and Z planes 0-59.\n        num_levels: Number of resolution-pyramid levels. If set to `5`, there\n            will be the full-resolution level and 4 levels of downsampled\n            images.\n        coarsening_xy: Linear coarsening factor between subsequent levels.\n            If set to `2`, level 1 is 2x downsampled, level 2 is 4x downsampled\n            etc.\n        image_extension: Filename extension of images\n            (e.g. `\"tif\"` or `\"png\"`).\n        metadata_table_files: If `None`, parse Yokogawa metadata from mrf/mlf\n            files in the input_path folder; else, a dictionary of key-value\n            pairs like `(acquisition, path)` with `acquisition` a string\n            and `path` pointing to a csv file containing the parsed metadata\n            table.\n        overwrite: If `True`, overwrite the task output.\n\n    Returns:\n        A metadata dictionary containing important metadata about the OME-Zarr\n            plate, the images and some parameters required by downstream tasks\n            (like `num_levels`).\n    \"\"\"\n\n    if metadata_table_files:\n\n        # Checks on the dict:\n        # 1. Acquisitions as keys (same as keys of allowed_channels)\n        # 2. Files end with \".csv\"\n        # 3. Files exist.\n        if set(allowed_channels.keys()) != set(metadata_table_files.keys()):\n            raise ValueError(\n                \"Mismatch in acquisition keys between \"\n                f\"{allowed_channels.keys()=} and \"\n                f\"{metadata_table_files.keys()=}\"\n            )\n        for f in metadata_table_files.values():\n            if not f.endswith(\".csv\"):\n                raise ValueError(\n                    f\"{f} (in metadata_table_file) is not a csv file.\"\n                )\n            if not os.path.isfile(f):\n                raise ValueError(\n                    f\"{f} (in metadata_table_file) does not exist.\"\n                )\n\n    # Preliminary checks on allowed_channels\n    # Note that in metadata the keys of dictionary arguments should be\n    # strings (and not integers), so that they can be read from a JSON file\n    for key, _channels in allowed_channels.items():\n        if not isinstance(key, str):\n            raise ValueError(f\"{allowed_channels=} has non-string keys\")\n        check_unique_wavelength_ids(_channels)\n\n    # Identify all plates and all channels, per input folders\n    dict_acquisitions: dict = {}\n\n    for ind_in_path, in_path_str in enumerate(input_paths):\n        acquisition = str(ind_in_path)\n        in_path = Path(in_path_str)\n        dict_acquisitions[acquisition] = {}\n\n        actual_wavelength_ids = []\n        plates = []\n        plate_prefixes = []\n\n        # Loop over all images\n        patterns = [f\"*.{image_extension}\"]\n        if image_glob_patterns:\n            patterns.extend(image_glob_patterns)\n        input_filenames = glob_with_multiple_patterns(\n            folder=in_path_str,\n            patterns=patterns,\n        )\n        for fn in input_filenames:\n            try:\n                filename_metadata = parse_filename(Path(fn).name)\n                plate = filename_metadata[\"plate\"]\n                plates.append(plate)\n                plate_prefix = filename_metadata[\"plate_prefix\"]\n                plate_prefixes.append(plate_prefix)\n                A = filename_metadata[\"A\"]\n                C = filename_metadata[\"C\"]\n                actual_wavelength_ids.append(f\"A{A}_C{C}\")\n            except ValueError as e:\n                logger.warning(\n                    f'Skipping \"{Path(fn).name}\". Original error: ' + str(e)\n                )\n        plates = sorted(list(set(plates)))\n        actual_wavelength_ids = sorted(list(set(actual_wavelength_ids)))\n\n        info = (\n            \"Listing all plates/channels:\\n\"\n            f\"Patterns: {patterns}\\n\"\n            f\"Plates:   {plates}\\n\"\n            f\"Actual wavelength IDs: {actual_wavelength_ids}\\n\"\n        )\n\n        # Check that a folder includes a single plate\n        if len(plates) > 1:\n            raise ValueError(f\"{info}ERROR: {len(plates)} plates detected\")\n        elif len(plates) == 0:\n            raise ValueError(f\"{info}ERROR: No plates detected\")\n        original_plate = plates[0]\n        plate_prefix = plate_prefixes[0]\n\n        # Replace plate with the one of acquisition 0, if needed\n        if int(acquisition) > 0:\n            plate = dict_acquisitions[\"0\"][\"plate\"]\n            logger.warning(\n                f\"For {acquisition=}, we replace {original_plate=} with \"\n                f\"{plate=} (the one for acquisition 0)\"\n            )\n\n        # Check that all channels are in the allowed_channels\n        allowed_wavelength_ids = [\n            c.wavelength_id for c in allowed_channels[acquisition]\n        ]\n        if not set(actual_wavelength_ids).issubset(\n            set(allowed_wavelength_ids)\n        ):\n            msg = \"ERROR in create_ome_zarr\\n\"\n            msg += f\"actual_wavelength_ids: {actual_wavelength_ids}\\n\"\n            msg += f\"allowed_wavelength_ids: {allowed_wavelength_ids}\\n\"\n            raise ValueError(msg)\n\n        # Create actual_channels, i.e. a list of the channel dictionaries which\n        # are present\n        actual_channels = [\n            channel\n            for channel in allowed_channels[acquisition]\n            if channel.wavelength_id in actual_wavelength_ids\n        ]\n\n        logger.info(f\"plate: {plate}\")\n        logger.info(f\"actual_channels: {actual_channels}\")\n\n        dict_acquisitions[acquisition] = {}\n        dict_acquisitions[acquisition][\"plate\"] = plate\n        dict_acquisitions[acquisition][\"original_plate\"] = original_plate\n        dict_acquisitions[acquisition][\"plate_prefix\"] = plate_prefix\n        dict_acquisitions[acquisition][\"image_folder\"] = in_path\n        dict_acquisitions[acquisition][\"original_paths\"] = [in_path]\n        dict_acquisitions[acquisition][\"actual_channels\"] = actual_channels\n        dict_acquisitions[acquisition][\n            \"actual_wavelength_ids\"\n        ] = actual_wavelength_ids\n\n    acquisitions = sorted(list(dict_acquisitions.keys()))\n    current_plates = [item[\"plate\"] for item in dict_acquisitions.values()]\n    if len(set(current_plates)) > 1:\n        raise ValueError(f\"{current_plates=}\")\n    plate = current_plates[0]\n\n    zarrurl = dict_acquisitions[acquisitions[0]][\"plate\"] + \".zarr\"\n    full_zarrurl = str(Path(output_path) / zarrurl)\n    logger.info(f\"Creating {full_zarrurl=}\")\n    # Call zarr.open_group wrapper, which handles overwrite=True/False\n    group_plate = open_zarr_group_with_overwrite(\n        full_zarrurl, overwrite=overwrite\n    )\n    group_plate.attrs[\"plate\"] = {\n        \"acquisitions\": [\n            {\n                \"id\": int(acquisition),\n                \"name\": dict_acquisitions[acquisition][\"original_plate\"],\n            }\n            for acquisition in acquisitions\n        ]\n    }\n\n    zarrurls: dict[str, list[str]] = {\"well\": [], \"image\": []}\n    zarrurls[\"plate\"] = [f\"{plate}.zarr\"]\n\n    ################################################################\n    logging.info(f\"{acquisitions=}\")\n\n    for acquisition in acquisitions:\n\n        # Define plate zarr\n        image_folder = dict_acquisitions[acquisition][\"image_folder\"]\n        logger.info(f\"Looking at {image_folder=}\")\n\n        # Obtain FOV-metadata dataframe\n        if metadata_table_files is None:\n            mrf_path = f\"{image_folder}/MeasurementDetail.mrf\"\n            mlf_path = f\"{image_folder}/MeasurementData.mlf\"\n            site_metadata, total_files = parse_yokogawa_metadata(\n                mrf_path, mlf_path, filename_patterns=image_glob_patterns\n            )\n            site_metadata = remove_FOV_overlaps(site_metadata)\n        else:\n            site_metadata = pd.read_csv(metadata_table_files[acquisition])\n            site_metadata.set_index([\"well_id\", \"FieldIndex\"], inplace=True)\n\n        # Extract pixel sizes and bit_depth\n        pixel_size_z = site_metadata[\"pixel_size_z\"][0]\n        pixel_size_y = site_metadata[\"pixel_size_y\"][0]\n        pixel_size_x = site_metadata[\"pixel_size_x\"][0]\n        bit_depth = site_metadata[\"bit_depth\"][0]\n\n        if min(pixel_size_z, pixel_size_y, pixel_size_x) < 1e-9:\n            raise ValueError(pixel_size_z, pixel_size_y, pixel_size_x)\n\n        # Identify all wells\n        plate_prefix = dict_acquisitions[acquisition][\"plate_prefix\"]\n        patterns = [f\"{plate_prefix}_*.{image_extension}\"]\n        if image_glob_patterns:\n            patterns.extend(image_glob_patterns)\n        plate_images = glob_with_multiple_patterns(\n            folder=str(image_folder),\n            patterns=patterns,\n        )\n\n        wells = [\n            parse_filename(os.path.basename(fn))[\"well\"] for fn in plate_images\n        ]\n        wells = sorted(list(set(wells)))\n        logger.info(f\"{wells=}\")\n\n        # Verify that all wells have all channels\n        actual_channels = dict_acquisitions[acquisition][\"actual_channels\"]\n        for well in wells:\n            patterns = [f\"{plate_prefix}_{well}_*.{image_extension}\"]\n            if image_glob_patterns:\n                patterns.extend(image_glob_patterns)\n            well_images = glob_with_multiple_patterns(\n                folder=str(image_folder),\n                patterns=patterns,\n            )\n\n            well_wavelength_ids = []\n            for fpath in well_images:\n                try:\n                    filename_metadata = parse_filename(os.path.basename(fpath))\n                    A = filename_metadata[\"A\"]\n                    C = filename_metadata[\"C\"]\n                    well_wavelength_ids.append(f\"A{A}_C{C}\")\n                except IndexError:\n                    logger.info(f\"Skipping {fpath}\")\n            well_wavelength_ids = sorted(list(set(well_wavelength_ids)))\n            actual_wavelength_ids = dict_acquisitions[acquisition][\n                \"actual_wavelength_ids\"\n            ]\n            if well_wavelength_ids != actual_wavelength_ids:\n                raise ValueError(\n                    f\"ERROR: well {well} in plate {plate} (prefix: \"\n                    f\"{plate_prefix}) has missing channels.\\n\"\n                    f\"Expected: {actual_wavelength_ids}\\n\"\n                    f\"Found: {well_wavelength_ids}.\\n\"\n                )\n\n        well_rows_columns = [\n            ind for ind in sorted([(n[0], n[1:]) for n in wells])\n        ]\n        row_list = [\n            well_row_column[0] for well_row_column in well_rows_columns\n        ]\n        col_list = [\n            well_row_column[1] for well_row_column in well_rows_columns\n        ]\n        row_list = sorted(list(set(row_list)))\n        col_list = sorted(list(set(col_list)))\n\n        plate_attrs = group_plate.attrs[\"plate\"]\n        plate_attrs[\"columns\"] = [{\"name\": col} for col in col_list]\n        plate_attrs[\"rows\"] = [{\"name\": row} for row in row_list]\n        plate_attrs[\"wells\"] = [\n            {\n                \"path\": well_row_column[0] + \"/\" + well_row_column[1],\n                \"rowIndex\": row_list.index(well_row_column[0]),\n                \"columnIndex\": col_list.index(well_row_column[1]),\n            }\n            for well_row_column in well_rows_columns\n        ]\n        group_plate.attrs[\"plate\"] = plate_attrs\n\n        for row, column in well_rows_columns:\n\n            try:\n                group_well = group_plate.create_group(f\"{row}/{column}/\")\n                logging.info(f\"Created new group_well at {row}/{column}/\")\n                group_well.attrs[\"well\"] = {\n                    \"images\": [\n                        {\n                            \"path\": f\"{acquisition}\",\n                            \"acquisition\": int(acquisition),\n                        }\n                    ],\n                    \"version\": __OME_NGFF_VERSION__,\n                }\n                zarrurls[\"well\"].append(f\"{plate}.zarr/{row}/{column}\")\n            except ContainsGroupError:\n                group_well = zarr.open_group(\n                    f\"{full_zarrurl}/{row}/{column}/\", mode=\"r+\"\n                )\n                logging.info(\n                    f\"Loaded group_well from {full_zarrurl}/{row}/{column}\"\n                )\n                current_images = group_well.attrs[\"well\"][\"images\"] + [\n                    {\"path\": f\"{acquisition}\", \"acquisition\": int(acquisition)}\n                ]\n                group_well.attrs[\"well\"] = dict(\n                    images=current_images,\n                    version=group_well.attrs[\"well\"][\"version\"],\n                )\n\n            group_image = group_well.create_group(\n                f\"{acquisition}/\"\n            )  # noqa: F841\n            logging.info(f\"Created image group {row}/{column}/{acquisition}\")\n            image = f\"{plate}.zarr/{row}/{column}/{acquisition}\"\n            zarrurls[\"image\"].append(image)\n\n            group_image.attrs[\"multiscales\"] = [\n                {\n                    \"version\": __OME_NGFF_VERSION__,\n                    \"axes\": [\n                        {\"name\": \"c\", \"type\": \"channel\"},\n                        {\n                            \"name\": \"z\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                        {\n                            \"name\": \"y\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                        {\n                            \"name\": \"x\",\n                            \"type\": \"space\",\n                            \"unit\": \"micrometer\",\n                        },\n                    ],\n                    \"datasets\": [\n                        {\n                            \"path\": f\"{ind_level}\",\n                            \"coordinateTransformations\": [\n                                {\n                                    \"type\": \"scale\",\n                                    \"scale\": [\n                                        1,\n                                        pixel_size_z,\n                                        pixel_size_y\n                                        * coarsening_xy**ind_level,\n                                        pixel_size_x\n                                        * coarsening_xy**ind_level,\n                                    ],\n                                }\n                            ],\n                        }\n                        for ind_level in range(num_levels)\n                    ],\n                }\n            ]\n\n            group_image.attrs[\"omero\"] = {\n                \"id\": 1,  # FIXME does this depend on the plate number?\n                \"name\": \"TBD\",\n                \"version\": __OME_NGFF_VERSION__,\n                \"channels\": define_omero_channels(\n                    channels=actual_channels,\n                    bit_depth=bit_depth,\n                    label_prefix=acquisition,\n                ),\n            }\n\n            # Prepare AnnData tables for FOV/well ROIs\n            well_id = row + column\n            FOV_ROIs_table = prepare_FOV_ROI_table(site_metadata.loc[well_id])\n            well_ROIs_table = prepare_well_ROI_table(\n                site_metadata.loc[well_id]\n            )\n\n            # Write AnnData tables into the `tables` zarr group\n            write_table(\n                group_image,\n                \"FOV_ROI_table\",\n                FOV_ROIs_table,\n                overwrite=overwrite,\n                logger=logger,\n            )\n            write_table(\n                group_image,\n                \"well_ROI_table\",\n                well_ROIs_table,\n                overwrite=overwrite,\n                logger=logger,\n            )\n\n    # Check that the different images (e.g. different cycles) in the each well\n    # have unique labels\n    for well_path in zarrurls[\"well\"]:\n        check_well_channel_labels(\n            well_zarr_path=str(Path(output_path) / well_path)\n        )\n\n    original_paths = {\n        acquisition: dict_acquisitions[acquisition][\"original_paths\"]\n        for acquisition in acquisitions\n    }\n\n    metadata_update = dict(\n        plate=zarrurls[\"plate\"],\n        well=zarrurls[\"well\"],\n        image=zarrurls[\"image\"],\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        original_paths=original_paths,\n        image_extension=image_extension,\n        image_glob_patterns=image_glob_patterns,\n    )\n    return metadata_update\n
"},{"location":"reference/fractal_tasks_core/tasks/illumination_correction/","title":"illumination_correction","text":"

Apply illumination correction to all fields of view.

"},{"location":"reference/fractal_tasks_core/tasks/illumination_correction/#fractal_tasks_core.tasks.illumination_correction.correct","title":"correct(img_stack, corr_img, background=110)","text":"

Corrects a stack of images, using a given illumination profile (e.g. bright in the center of the image, dim outside).

PARAMETER DESCRIPTION img_stack

4D numpy array (czyx), with dummy size along c.

TYPE: ndarray

corr_img

2D numpy array (yx)

TYPE: ndarray

background

Background value that is subtracted from the image before the illumination correction is applied.

TYPE: int DEFAULT: 110

Source code in fractal_tasks_core/tasks/illumination_correction.py 
def correct(\n    img_stack: np.ndarray,\n    corr_img: np.ndarray,\n    background: int = 110,\n):\n\"\"\"\n    Corrects a stack of images, using a given illumination profile (e.g. bright\n    in the center of the image, dim outside).\n\n    Args:\n        img_stack: 4D numpy array (czyx), with dummy size along c.\n        corr_img: 2D numpy array (yx)\n        background: Background value that is subtracted from the image before\n            the illumination correction is applied.\n    \"\"\"\n\n    logger.info(f\"Start correct, {img_stack.shape}\")\n\n    # Check shapes\n    if corr_img.shape != img_stack.shape[2:] or img_stack.shape[0] != 1:\n        raise ValueError(\n            \"Error in illumination_correction:\\n\"\n            f\"{img_stack.shape=}\\n{corr_img.shape=}\"\n        )\n\n    # Store info about dtype\n    dtype = img_stack.dtype\n    dtype_max = np.iinfo(dtype).max\n\n    # Background subtraction\n    img_stack[img_stack <= background] = 0\n    img_stack[img_stack > background] -= background\n\n    #  Apply the normalized correction matrix (requires a float array)\n    # img_stack = img_stack.astype(np.float64)\n    new_img_stack = img_stack / (corr_img / np.max(corr_img))[None, None, :, :]\n\n    # Handle edge case: corrected image may have values beyond the limit of\n    # the encoding, e.g. beyond 65535 for 16bit images. This clips values\n    # that surpass this limit and triggers a warning\n    if np.sum(new_img_stack > dtype_max) > 0:\n        warnings.warn(\n            \"Illumination correction created values beyond the max range of \"\n            f\"the current image type. These have been clipped to {dtype_max=}.\"\n        )\n        new_img_stack[new_img_stack > dtype_max] = dtype_max\n\n    logger.info(\"End correct\")\n\n    # Cast back to original dtype and return\n    return new_img_stack.astype(dtype)\n
"},{"location":"reference/fractal_tasks_core/tasks/illumination_correction/#fractal_tasks_core.tasks.illumination_correction.illumination_correction","title":"illumination_correction(*, input_paths, output_path, component, metadata, illumination_profiles_folder, dict_corr, background=110, overwrite_input=True, new_component=None)","text":"

Applies illumination correction to the images in the OME-Zarr.

PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Path were the output of this task is stored. Examples: \"/some/path/\" => puts the new OME-Zarr file in the same folder as the input OME-Zarr file; \"/some/new_path\" => puts the new OME-Zarr file into a new folder at /some/new_path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

illumination_profiles_folder

Path of folder of illumination profiles.

TYPE: str

dict_corr

Dictionary where keys match the wavelength_id attributes of existing channels (e.g. A01_C01 ) and values are the filenames of the corresponding illumination profiles.

TYPE: dict[str, str]

background

Background value that is subtracted from the image before the illumination correction is applied. Set it to 0 if you don't want any background subtraction.

TYPE: int DEFAULT: 110

overwrite_input

If True, the results of this task will overwrite the input image data. In the current version, overwrite_input=False is not implemented.

TYPE: bool DEFAULT: True

new_component

Not implemented yet. This is not implemented well in Fractal server at the moment, it's unclear how a user would specify fitting new components. If the results shall not overwrite the input data and the output path is the same as the input path, a new component needs to be provided. Example: myplate_new_name.zarr/B/03/0/.

TYPE: Optional[str] DEFAULT: None

Source code in fractal_tasks_core/tasks/illumination_correction.py 
@validate_arguments\ndef illumination_correction(\n    *,\n    # Standard arguments\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments\n    illumination_profiles_folder: str,\n    dict_corr: dict[str, str],\n    background: int = 110,\n    overwrite_input: bool = True,\n    new_component: Optional[str] = None,\n) -> dict[str, Any]:\n\n\"\"\"\n    Applies illumination correction to the images in the OME-Zarr.\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file. Example:\n            `[\"/some/path/\"]`. This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: Path were the output of this task is stored. Examples:\n            `\"/some/path/\"` => puts the new OME-Zarr file in the same folder as\n            the input OME-Zarr file; `\"/some/new_path\"` => puts the new\n            OME-Zarr file into a new folder at `/some/new_path`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        illumination_profiles_folder: Path of folder of illumination profiles.\n        dict_corr: Dictionary where keys match the `wavelength_id` attributes\n            of existing channels (e.g.  `A01_C01` ) and values are the\n            filenames of the corresponding illumination profiles.\n        background: Background value that is subtracted from the image before\n            the illumination correction is applied. Set it to `0` if you don't\n            want any background subtraction.\n        overwrite_input:\n            If `True`, the results of this task will overwrite the input image\n            data. In the current version, `overwrite_input=False` is not\n            implemented.\n        new_component: Not implemented yet. This is not implemented well in\n            Fractal server at the moment, it's unclear how a user would specify\n            fitting new components. If the results shall not overwrite the\n            input data and the output path is the same as the input path, a new\n            component needs to be provided.\n            Example: `myplate_new_name.zarr/B/03/0/`.\n    \"\"\"\n\n    # Preliminary checks\n    if len(input_paths) > 1:\n        raise NotImplementedError\n    if (overwrite_input and new_component is not None) or (\n        new_component is None and not overwrite_input\n    ):\n        raise ValueError(f\"{overwrite_input=}, but {new_component=}\")\n\n    if not overwrite_input:\n        msg = (\n            \"We still have to harmonize illumination_correction(\"\n            \"overwrite_input=False) with replicate_zarr_structure(..., \"\n            \"suffix=..)\"\n        )\n        raise NotImplementedError(msg)\n\n    # Defione old/new zarrurls\n    plate, well = component.split(\".zarr/\")\n    in_path = Path(input_paths[0])\n    zarrurl_old = (in_path / component).as_posix()\n    if overwrite_input:\n        zarrurl_new = zarrurl_old\n    else:\n        new_plate, new_well = new_component.split(\".zarr/\")\n        if new_well != well:\n            raise ValueError(f\"{well=}, {new_well=}\")\n        zarrurl_new = (Path(output_path) / new_component).as_posix()\n\n    t_start = time.perf_counter()\n    logger.info(\"Start illumination_correction\")\n    logger.info(f\"  {overwrite_input=}\")\n    logger.info(f\"  {zarrurl_old=}\")\n    logger.info(f\"  {zarrurl_new=}\")\n\n    # Read attributes from NGFF metadata\n    ngff_image_meta = load_NgffImageMeta(zarrurl_old)\n    num_levels = ngff_image_meta.num_levels\n    coarsening_xy = ngff_image_meta.coarsening_xy\n    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n    logger.info(f\"NGFF image has {num_levels=}\")\n    logger.info(f\"NGFF image has {coarsening_xy=}\")\n    logger.info(\n        f\"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}\"\n    )\n\n    # Read channels from .zattrs\n    channels: list[OmeroChannel] = get_omero_channel_list(\n        image_zarr_path=zarrurl_old\n    )\n    num_channels = len(channels)\n\n    # Read FOV ROIs\n    FOV_ROI_table = ad.read_zarr(f\"{zarrurl_old}/tables/FOV_ROI_table\")\n\n    # Create list of indices for 3D FOVs spanning the entire Z direction\n    list_indices = convert_ROI_table_to_indices(\n        FOV_ROI_table,\n        level=0,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(list_indices, \"FOV_ROI_table\")\n\n    # Extract image size from FOV-ROI indices. Note: this works at level=0,\n    # where FOVs should all be of the exact same size (in pixels)\n    ref_img_size = None\n    for indices in list_indices:\n        img_size = (indices[3] - indices[2], indices[5] - indices[4])\n        if ref_img_size is None:\n            ref_img_size = img_size\n        else:\n            if img_size != ref_img_size:\n                raise ValueError(\n                    \"ERROR: inconsistent image sizes in list_indices\"\n                )\n    img_size_y, img_size_x = img_size[:]\n\n    # Assemble dictionary of matrices and check their shapes\n    corrections = {}\n    for channel in channels:\n        wavelength_id = channel.wavelength_id\n        corrections[wavelength_id] = imread(\n            (\n                Path(illumination_profiles_folder) / dict_corr[wavelength_id]\n            ).as_posix()\n        )\n        if corrections[wavelength_id].shape != (img_size_y, img_size_x):\n            raise ValueError(\n                \"Error in illumination_correction, \"\n                \"correction matrix has wrong shape.\"\n            )\n\n    # Lazily load highest-res level from original zarr array\n    data_czyx = da.from_zarr(f\"{zarrurl_old}/0\")\n\n    # Create zarr for output\n    if overwrite_input:\n        fov_path = zarrurl_old\n        new_zarr = zarr.open(f\"{zarrurl_old}/0\")\n    else:\n        fov_path = zarrurl_new\n        new_zarr = zarr.create(\n            shape=data_czyx.shape,\n            chunks=data_czyx.chunksize,\n            dtype=data_czyx.dtype,\n            store=zarr.storage.FSStore(f\"{zarrurl_new}/0\"),\n            overwrite=False,\n            dimension_separator=\"/\",\n        )\n\n    # Iterate over FOV ROIs\n    num_ROIs = len(list_indices)\n    for i_c, channel in enumerate(channels):\n        for i_ROI, indices in enumerate(list_indices):\n            # Define region\n            s_z, e_z, s_y, e_y, s_x, e_x = indices[:]\n            region = (\n                slice(i_c, i_c + 1),\n                slice(s_z, e_z),\n                slice(s_y, e_y),\n                slice(s_x, e_x),\n            )\n            logger.info(\n                f\"Now processing ROI {i_ROI+1}/{num_ROIs} \"\n                f\"for channel {i_c+1}/{num_channels}\"\n            )\n            # Execute illumination correction\n            corrected_fov = correct(\n                data_czyx[region].compute(),\n                corrections[channel.wavelength_id],\n                background=background,\n            )\n            # Write to disk\n            da.array(corrected_fov).to_zarr(\n                url=new_zarr,\n                region=region,\n                compute=True,\n            )\n\n    # Starting from on-disk highest-resolution data, build and write to disk a\n    # pyramid of coarser levels\n    build_pyramid(\n        zarrurl=fov_path,\n        overwrite=True,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        chunksize=data_czyx.chunksize,\n    )\n\n    t_end = time.perf_counter()\n    logger.info(f\"End illumination_correction, elapsed: {t_end-t_start}\")\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/import_ome_zarr/","title":"import_ome_zarr","text":"

Task to import an existing OME-Zarr.

"},{"location":"reference/fractal_tasks_core/tasks/import_ome_zarr/#fractal_tasks_core.tasks.import_ome_zarr._process_single_image","title":"_process_single_image(image_path, add_image_ROI_table, add_grid_ROI_table, update_omero_metadata, *, grid_YX_shape=None, overwrite=False)","text":"

Validate OME-NGFF metadata and optionally generate ROI tables.

This task:

  1. Validates OME-NGFF image metadata, via NgffImageMeta;
  2. Optionally generates and writes two ROI tables;
  3. Optionally update OME-NGFF omero metadata.
PARAMETER DESCRIPTION image_path

Absolute path to the image Zarr group.

TYPE: str

add_image_ROI_table

Whether to add a image_ROI_table table (argument propagated from import_ome_zarr).

TYPE: bool

add_grid_ROI_table

Whether to add a grid_ROI_table table (argument propagated from import_ome_zarr).

TYPE: bool

update_omero_metadata

Whether to update Omero-channels metadata (argument propagated from import_ome_zarr).

TYPE: bool

grid_YX_shape

YX shape of the ROI grid (it must be not None, if add_grid_ROI_table=True.

TYPE: Optional[tuple[int, int]] DEFAULT: None

Source code in fractal_tasks_core/tasks/import_ome_zarr.py 
def _process_single_image(\n    image_path: str,\n    add_image_ROI_table: bool,\n    add_grid_ROI_table: bool,\n    update_omero_metadata: bool,\n    *,\n    grid_YX_shape: Optional[tuple[int, int]] = None,\n    overwrite: bool = False,\n) -> None:\n\"\"\"\n    Validate OME-NGFF metadata and optionally generate ROI tables.\n\n    This task:\n\n    1. Validates OME-NGFF image metadata, via `NgffImageMeta`;\n    2. Optionally generates and writes two ROI tables;\n    3. Optionally update OME-NGFF omero metadata.\n\n    Args:\n        image_path: Absolute path to the image Zarr group.\n        add_image_ROI_table: Whether to add a `image_ROI_table` table\n            (argument propagated from `import_ome_zarr`).\n        add_grid_ROI_table: Whether to add a `grid_ROI_table` table (argument\n            propagated from `import_ome_zarr`).\n        update_omero_metadata: Whether to update Omero-channels metadata\n            (argument propagated from `import_ome_zarr`).\n        grid_YX_shape: YX shape of the ROI grid (it must be not `None`, if\n            `add_grid_ROI_table=True`.\n    \"\"\"\n\n    # Note from zarr docs: `r+` means read/write (must exist)\n    image_group = zarr.open_group(image_path, mode=\"r+\")\n    image_meta = NgffImageMeta(**image_group.attrs.asdict())\n\n    # Preliminary checks\n    if not (add_image_ROI_table or add_grid_ROI_table):\n        return\n    if add_grid_ROI_table and (grid_YX_shape is None):\n        raise ValueError(\n            f\"_process_single_image called with {add_grid_ROI_table=}, \"\n            f\"but {grid_YX_shape=}.\"\n        )\n\n    pixels_ZYX = image_meta.get_pixel_sizes_zyx(level=0)\n\n    # Read zarr array\n    dataset_subpath = image_meta.datasets[0].path\n    array = da.from_zarr(f\"{image_path}/{dataset_subpath}\")\n\n    # Prepare image_ROI_table and write it into the zarr group\n    if add_image_ROI_table:\n        image_ROI_table = get_single_image_ROI(array.shape, pixels_ZYX)\n        write_table(\n            image_group,\n            \"image_ROI_table\",\n            image_ROI_table,\n            overwrite=overwrite,\n            logger=logger,\n        )\n\n    # Prepare grid_ROI_table and write it into the zarr group\n    if add_grid_ROI_table:\n        grid_ROI_table = get_image_grid_ROIs(\n            array.shape,\n            pixels_ZYX,\n            grid_YX_shape,\n        )\n        write_table(\n            image_group,\n            \"grid_ROI_table\",\n            grid_ROI_table,\n            overwrite=overwrite,\n            logger=logger,\n        )\n\n    # Update Omero-channels metadata\n    if update_omero_metadata:\n        # Extract number of channels from zarr array\n        try:\n            channel_axis_index = image_meta.axes_names.index(\"c\")\n        except ValueError:\n            logger.error(f\"Existing axes: {image_meta.axes_names}\")\n            msg = (\n                \"OME-Zarrs with no channel axis are not currently \"\n                \"supported in fractal-tasks-core. Upcoming flexibility \"\n                \"improvements are tracked in https://github.com/\"\n                \"fractal-analytics-platform/fractal-tasks-core/issues/150.\"\n            )\n            logger.error(msg)\n            raise NotImplementedError(msg)\n        logger.info(f\"Existing axes: {image_meta.axes_names}\")\n        logger.info(f\"Channel-axis index: {channel_axis_index}\")\n        num_channels_zarr = array.shape[channel_axis_index]\n        logger.info(\n            f\"{num_channels_zarr} channel(s) found in Zarr array \"\n            f\"at {image_path}/{dataset_subpath}\"\n        )\n        # Update or create omero channels metadata\n        old_omero = image_group.attrs.get(\"omero\", {})\n        old_channels = old_omero.get(\"channels\", [])\n        if len(old_channels) > 0:\n            logger.info(\n                f\"{len(old_channels)} channel(s) found in NGFF omero metadata\"\n            )\n            if len(old_channels) != num_channels_zarr:\n                error_msg = (\n                    \"Channels-number mismatch: Number of channels in the \"\n                    f\"zarr array ({num_channels_zarr}) differs from number \"\n                    \"of channels listed in NGFF omero metadata \"\n                    f\"({len(old_channels)}).\"\n                )\n                logging.error(error_msg)\n                raise ValueError(error_msg)\n        else:\n            old_channels = [{} for ind in range(num_channels_zarr)]\n        new_channels = update_omero_channels(old_channels)\n        new_omero = old_omero.copy()\n        new_omero[\"channels\"] = new_channels\n        image_group.attrs.update(omero=new_omero)\n
"},{"location":"reference/fractal_tasks_core/tasks/import_ome_zarr/#fractal_tasks_core.tasks.import_ome_zarr.import_ome_zarr","title":"import_ome_zarr(*, input_paths, output_path, metadata, zarr_name, add_image_ROI_table=True, add_grid_ROI_table=True, grid_y_shape=2, grid_x_shape=2, update_omero_metadata=True, overwrite=False)","text":"

Import an OME-Zarr into Fractal.

The current version of this task:

  1. Creates the appropriate components-related metadata, needed for processing an existing OME-Zarr through Fractal.
  2. Optionally adds new ROI tables to the existing OME-Zarr.
PARAMETER DESCRIPTION input_paths

A length-one list with the parent folder of the OME-Zarr to be imported; e.g. input_paths=[\"/somewhere\"], if the OME-Zarr path is /somewhere/array.zarr. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Not used in this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

Not used in this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

zarr_name

The OME-Zarr name, without its parent folder; e.g. zarr_name=\"array.zarr\", if the OME-Zarr path is /somewhere/array.zarr.

TYPE: str

add_image_ROI_table

Whether to add a image_ROI_table table to each image, with a single ROI covering the whole image.

TYPE: bool DEFAULT: True

add_grid_ROI_table

Whether to add a grid_ROI_table table to each image, with the image split into a rectangular grid of ROIs.

TYPE: bool DEFAULT: True

grid_y_shape

Y shape of the ROI grid in grid_ROI_table.

TYPE: int DEFAULT: 2

grid_x_shape

X shape of the ROI grid in grid_ROI_table.

TYPE: int DEFAULT: 2

update_omero_metadata

Whether to update Omero-channels metadata, to make them Fractal-compatible.

TYPE: bool DEFAULT: True

overwrite

Whether new ROI tables (added when add_image_ROI_table and/or add_grid_ROI_table are True) can overwite existing ones.

TYPE: bool DEFAULT: False

Source code in fractal_tasks_core/tasks/import_ome_zarr.py 
@validate_arguments\ndef import_ome_zarr(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    metadata: dict[str, Any],\n    zarr_name: str,\n    add_image_ROI_table: bool = True,\n    add_grid_ROI_table: bool = True,\n    grid_y_shape: int = 2,\n    grid_x_shape: int = 2,\n    update_omero_metadata: bool = True,\n    overwrite: bool = False,\n) -> dict[str, Any]:\n\"\"\"\n    Import an OME-Zarr into Fractal.\n\n    The current version of this task:\n\n    1. Creates the appropriate components-related metadata, needed for\n       processing an existing OME-Zarr through Fractal.\n    2. Optionally adds new ROI tables to the existing OME-Zarr.\n\n    Args:\n        input_paths: A length-one list with the parent folder of the OME-Zarr\n            to be imported; e.g. `input_paths=[\"/somewhere\"]`, if the OME-Zarr\n            path is `/somewhere/array.zarr`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: Not used in this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: Not used in this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        zarr_name: The OME-Zarr name, without its parent folder; e.g.\n            `zarr_name=\"array.zarr\"`, if the OME-Zarr path is\n            `/somewhere/array.zarr`.\n        add_image_ROI_table: Whether to add a `image_ROI_table` table to each\n            image, with a single ROI covering the whole image.\n        add_grid_ROI_table: Whether to add a `grid_ROI_table` table to each\n            image, with the image split into a rectangular grid of ROIs.\n        grid_y_shape: Y shape of the ROI grid in `grid_ROI_table`.\n        grid_x_shape: X shape of the ROI grid in `grid_ROI_table`.\n        update_omero_metadata: Whether to update Omero-channels metadata, to\n            make them Fractal-compatible.\n        overwrite: Whether new ROI tables (added when `add_image_ROI_table`\n            and/or `add_grid_ROI_table` are `True`) can overwite existing ones.\n    \"\"\"\n\n    # Preliminary checks\n    if len(input_paths) > 1:\n        raise NotImplementedError\n\n    zarr_path = str(Path(input_paths[0]) / zarr_name)\n    logger.info(f\"Zarr path: {zarr_path}\")\n\n    zarrurls: dict = dict(plate=[], well=[], image=[])\n\n    root_group = zarr.open_group(zarr_path, mode=\"r\")\n    ngff_type = detect_ome_ngff_type(root_group)\n    grid_YX_shape = (grid_y_shape, grid_x_shape)\n\n    if ngff_type == \"plate\":\n        zarrurls[\"plate\"].append(zarr_name)\n        for well in root_group.attrs[\"plate\"][\"wells\"]:\n            well_path = well[\"path\"]\n            zarrurls[\"well\"].append(f\"{zarr_name}/{well_path}\")\n\n            well_group = zarr.open_group(zarr_path, path=well_path, mode=\"r\")\n            for image in well_group.attrs[\"well\"][\"images\"]:\n                image_path = image[\"path\"]\n                zarrurls[\"image\"].append(\n                    f\"{zarr_name}/{well_path}/{image_path}\"\n                )\n                _process_single_image(\n                    f\"{zarr_path}/{well_path}/{image_path}\",\n                    add_image_ROI_table,\n                    add_grid_ROI_table,\n                    update_omero_metadata,\n                    grid_YX_shape=grid_YX_shape,\n                    overwrite=overwrite,\n                )\n    elif ngff_type == \"well\":\n        zarrurls[\"well\"].append(zarr_name)\n        logger.warning(\n            \"Only OME-Zarr for plates are fully supported in Fractal; \"\n            f\"e.g. the current one ({ngff_type=}) cannot be \"\n            \"processed via the `maximum_intensity_projection` task.\"\n        )\n        for image in root_group.attrs[\"well\"][\"images\"]:\n            image_path = image[\"path\"]\n            zarrurls[\"image\"].append(f\"{zarr_name}/{image_path}\")\n            _process_single_image(\n                f\"{zarr_path}/{image_path}\",\n                add_image_ROI_table,\n                add_grid_ROI_table,\n                update_omero_metadata,\n                grid_YX_shape=grid_YX_shape,\n                overwrite=overwrite,\n            )\n    elif ngff_type == \"image\":\n        zarrurls[\"image\"].append(zarr_name)\n        logger.warning(\n            \"Only OME-Zarr for plates are fully supported in Fractal; \"\n            f\"e.g. the current one ({ngff_type=}) cannot be \"\n            \"processed via the `maximum_intensity_projection` task.\"\n        )\n        _process_single_image(\n            zarr_path,\n            add_image_ROI_table,\n            add_grid_ROI_table,\n            update_omero_metadata,\n            grid_YX_shape=grid_YX_shape,\n            overwrite=overwrite,\n        )\n\n    # Remove zarrurls keys pointing to empty lists\n    clean_zarrurls = {\n        key: value for key, value in zarrurls.items() if len(value) > 0\n    }\n\n    return clean_zarrurls\n
"},{"location":"reference/fractal_tasks_core/tasks/maximum_intensity_projection/","title":"maximum_intensity_projection","text":"

Task for 3D->2D maximum-intensity projection.

"},{"location":"reference/fractal_tasks_core/tasks/maximum_intensity_projection/#fractal_tasks_core.tasks.maximum_intensity_projection.maximum_intensity_projection","title":"maximum_intensity_projection(*, input_paths, output_path, component, metadata, overwrite=False)","text":"

Perform maximum-intensity projection along Z axis.

Note: this task stores the output in a new zarr file.

PARAMETER DESCRIPTION input_paths

This parameter is not used by this task. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Path were the output of this task is stored. Example: \"/some/path/\" => puts the new OME-Zarr file in that folder. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Component is typically changed by the copy_ome_zarr task before, to point to a new mip Zarr file. Example: \"some_plate_mip.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

Dictionary containing metadata about the OME-Zarr. This task requires the key copy_ome_zarr to be present in the metadata (as defined in copy_ome_zarr task). (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: False

Source code in fractal_tasks_core/tasks/maximum_intensity_projection.py 
@validate_arguments\ndef maximum_intensity_projection(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    overwrite: bool = False,\n) -> dict[str, Any]:\n\"\"\"\n    Perform maximum-intensity projection along Z axis.\n\n    Note: this task stores the output in a new zarr file.\n\n    Args:\n        input_paths: This parameter is not used by this task.\n            This task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: Path were the output of this task is stored.\n            Example: `\"/some/path/\"` => puts the new OME-Zarr file in that\n            folder.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that\n            is processed. Component is typically changed by the `copy_ome_zarr`\n            task before, to point to a new mip Zarr file.\n            Example: `\"some_plate_mip.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: Dictionary containing metadata about the OME-Zarr.\n            This task requires the key `copy_ome_zarr` to be present in the\n            metadata (as defined in `copy_ome_zarr` task).\n            (standard argument for Fractal tasks, managed by Fractal server).\n        overwrite: If `True`, overwrite the task output.\n    \"\"\"\n\n    # Preliminary checks\n    if len(input_paths) > 1:\n        raise NotImplementedError\n\n    plate, well = component.split(\".zarr/\")\n    zarrurl_old = metadata[\"copy_ome_zarr\"][\"sources\"][plate] + \"/\" + well\n    clean_output_path = Path(output_path).resolve()\n    zarrurl_new = (clean_output_path / component).as_posix()\n    logger.info(f\"{zarrurl_old=}\")\n    logger.info(f\"{zarrurl_new=}\")\n\n    # Read some parameters from metadata\n    ngff_image = load_NgffImageMeta(zarrurl_old)\n    num_levels = ngff_image.num_levels\n    coarsening_xy = ngff_image.coarsening_xy\n\n    # Load 0-th level\n    data_czyx = da.from_zarr(zarrurl_old + \"/0\")\n    num_channels = data_czyx.shape[0]\n    chunksize_y = data_czyx.chunksize[-2]\n    chunksize_x = data_czyx.chunksize[-1]\n    logger.info(f\"{num_channels=}\")\n    logger.info(f\"{chunksize_y=}\")\n    logger.info(f\"{chunksize_x=}\")\n    # Loop over channels\n    accumulate_chl = []\n    for ind_ch in range(num_channels):\n        # Perform MIP for each channel of level 0\n        mip_yx = da.stack([da.max(data_czyx[ind_ch], axis=0)], axis=0)\n        accumulate_chl.append(mip_yx)\n    accumulated_array = da.stack(accumulate_chl, axis=0)\n\n    # Write to disk (triggering execution)\n    try:\n        accumulated_array.to_zarr(\n            f\"{zarrurl_new}/0\",\n            overwrite=overwrite,\n            dimension_separator=\"/\",\n            write_empty_chunks=False,\n        )\n    except ContainsArrayError as e:\n        error_msg = (\n            f\"Cannot write array to zarr group at '{zarrurl_new}/0', \"\n            f\"with {overwrite=} (original error: {str(e)}).\\n\"\n            \"Hint: try setting overwrite=True.\"\n        )\n        logger.error(error_msg)\n        raise OverwriteNotAllowedError(error_msg)\n\n    # Starting from on-disk highest-resolution data, build and write to disk a\n    # pyramid of coarser levels\n    build_pyramid(\n        zarrurl=zarrurl_new,\n        overwrite=overwrite,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        chunksize=(1, 1, chunksize_y, chunksize_x),\n    )\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/napari_workflows_wrapper/","title":"napari_workflows_wrapper","text":"

Wrapper of napari-workflows.

"},{"location":"reference/fractal_tasks_core/tasks/napari_workflows_wrapper/#fractal_tasks_core.tasks.napari_workflows_wrapper.OutOfTaskScopeError","title":"OutOfTaskScopeError","text":"

Bases: NotImplementedError

Encapsulates features that are out-of-scope for the current wrapper task.

Source code in fractal_tasks_core/tasks/napari_workflows_wrapper.py 
class OutOfTaskScopeError(NotImplementedError):\n\"\"\"\n    Encapsulates features that are out-of-scope for the current wrapper task.\n    \"\"\"\n\n    pass\n
"},{"location":"reference/fractal_tasks_core/tasks/napari_workflows_wrapper/#fractal_tasks_core.tasks.napari_workflows_wrapper.napari_workflows_wrapper","title":"napari_workflows_wrapper(*, input_paths, output_path, component, metadata, workflow_file, input_specs, output_specs, input_ROI_table='FOV_ROI_table', level=0, relabeling=True, expected_dimensions=3, overwrite=True)","text":"

Run a napari-workflow on the ROIs of a single OME-NGFF image.

This task takes images and labels and runs a napari-workflow on them that can produce a label and tables as output.

Examples of allowed entries for input_specs and output_specs:

input_specs = {\n    \"in_1\": {\"type\": \"image\", \"channel\": {\"wavelength_id\": \"A01_C02\"}},\n    \"in_2\": {\"type\": \"image\", \"channel\": {\"label\": \"DAPI\"}},\n    \"in_3\": {\"type\": \"label\", \"label_name\": \"label_DAPI\"},\n}\n\noutput_specs = {\n    \"out_1\": {\"type\": \"label\", \"label_name\": \"label_DAPI_new\"},\n    \"out_2\": {\"type\": \"dataframe\", \"table_name\": \"measurements\"},\n}\n
PARAMETER DESCRIPTION input_paths

List of input paths where the image data is stored as OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. his task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\". (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

This parameter is not used by this task. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

workflow_file

Absolute path to napari-workflows YAML file

TYPE: str

input_specs

A dictionary of NapariWorkflowsInput values.

TYPE: dict[str, NapariWorkflowsInput]

output_specs

A dictionary of NapariWorkflowsOutput values.

TYPE: dict[str, NapariWorkflowsOutput]

input_ROI_table

Name of the ROI table over which the task loops to apply napari workflows. Examples: FOV_ROI_table => loop over the field of views; organoid_ROI_table => loop over the organoid ROI table (generated by another task); well_ROI_table => process the whole well as one image.

TYPE: str DEFAULT: 'FOV_ROI_table'

level

Pyramid level of the image to be used as input for napari-workflows. Choose 0 to process at full resolution. Levels > 0 are currently only supported for workflows that only have intensity images as input and only produce a label images as output.

TYPE: int DEFAULT: 0

relabeling

If True, apply relabeling so that label values are unique across all ROIs in the well.

TYPE: bool DEFAULT: True

expected_dimensions

Expected dimensions (either 2 or 3). Useful when loading 2D images that are stored in a 3D array with shape (1, size_x, size_y) [which is the default way Fractal stores 2D images], but you want to make sure the napari workflow gets a 2D array to process. Also useful to set to 2 when loading a 2D OME-Zarr that is saved as (size_x, size_y).

TYPE: int DEFAULT: 3

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: True

Source code in fractal_tasks_core/tasks/napari_workflows_wrapper.py 
@validate_arguments\ndef napari_workflows_wrapper(\n    *,\n    # Default arguments for fractal tasks:\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    # Task-specific arguments:\n    workflow_file: str,\n    input_specs: dict[str, NapariWorkflowsInput],\n    output_specs: dict[str, NapariWorkflowsOutput],\n    input_ROI_table: str = \"FOV_ROI_table\",\n    level: int = 0,\n    relabeling: bool = True,\n    expected_dimensions: int = 3,\n    overwrite: bool = True,\n):\n\"\"\"\n    Run a napari-workflow on the ROIs of a single OME-NGFF image.\n\n    This task takes images and labels and runs a napari-workflow on them that\n    can produce a label and tables as output.\n\n    Examples of allowed entries for `input_specs` and `output_specs`:\n\n    ```\n    input_specs = {\n        \"in_1\": {\"type\": \"image\", \"channel\": {\"wavelength_id\": \"A01_C02\"}},\n        \"in_2\": {\"type\": \"image\", \"channel\": {\"label\": \"DAPI\"}},\n        \"in_3\": {\"type\": \"label\", \"label_name\": \"label_DAPI\"},\n    }\n\n    output_specs = {\n        \"out_1\": {\"type\": \"label\", \"label_name\": \"label_DAPI_new\"},\n        \"out_2\": {\"type\": \"dataframe\", \"table_name\": \"measurements\"},\n    }\n    ```\n\n    Args:\n        input_paths: List of input paths where the image data is stored as\n            OME-Zarrs. Should point to the parent folder containing one or many\n            OME-Zarr files, not the actual OME-Zarr file.\n            Example: `[\"/some/path/\"]`.\n            his task only supports a single input path.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        output_path: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed.\n            Example: `\"some_plate.zarr/B/03/0\"`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: This parameter is not used by this task.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        workflow_file: Absolute path to napari-workflows YAML file\n        input_specs: A dictionary of `NapariWorkflowsInput` values.\n        output_specs: A dictionary of `NapariWorkflowsOutput` values.\n        input_ROI_table: Name of the ROI table over which the task loops to\n            apply napari workflows.\n            Examples:\n            `FOV_ROI_table`\n            => loop over the field of views;\n            `organoid_ROI_table`\n            => loop over the organoid ROI table (generated by another task);\n            `well_ROI_table`\n            => process the whole well as one image.\n        level: Pyramid level of the image to be used as input for\n            napari-workflows. Choose `0` to process at full resolution.\n            Levels > 0 are currently only supported for workflows that only\n            have intensity images as input and only produce a label images as\n            output.\n        relabeling: If `True`, apply relabeling so that label values are\n            unique across all ROIs in the well.\n        expected_dimensions: Expected dimensions (either `2` or `3`). Useful\n            when loading 2D images that are stored in a 3D array with shape\n            `(1, size_x, size_y)` [which is the default way Fractal stores 2D\n            images], but you want to make sure the napari workflow gets a 2D\n            array to process. Also useful to set to `2` when loading a 2D\n            OME-Zarr that is saved as `(size_x, size_y)`.\n        overwrite: If `True`, overwrite the task output.\n    \"\"\"\n    wf: napari_workflows.Worfklow = load_workflow(workflow_file)\n    logger.info(f\"Loaded workflow from {workflow_file}\")\n\n    # Validation of input/output specs\n    if not (set(wf.leafs()) <= set(output_specs.keys())):\n        msg = f\"Some item of {wf.leafs()=} is not part of {output_specs=}.\"\n        logger.warning(msg)\n    if not (set(wf.roots()) <= set(input_specs.keys())):\n        msg = f\"Some item of {wf.roots()=} is not part of {input_specs=}.\"\n        logger.error(msg)\n        raise ValueError(msg)\n    list_outputs = sorted(output_specs.keys())\n\n    # Characterization of workflow and scope restriction\n    input_types = [in_params.type for (name, in_params) in input_specs.items()]\n    output_types = [\n        out_params.type for (name, out_params) in output_specs.items()\n    ]\n    are_inputs_all_images = set(input_types) == {\"image\"}\n    are_outputs_all_labels = set(output_types) == {\"label\"}\n    are_outputs_all_dataframes = set(output_types) == {\"dataframe\"}\n    is_labeling_workflow = are_inputs_all_images and are_outputs_all_labels\n    is_measurement_only_workflow = are_outputs_all_dataframes\n    # Level-related constraint\n    logger.info(f\"This workflow acts at {level=}\")\n    logger.info(\n        f\"Is the current workflow a labeling one? {is_labeling_workflow}\"\n    )\n    if level > 0 and not is_labeling_workflow:\n        msg = (\n            f\"{level=}>0 is currently only accepted for labeling workflows, \"\n            \"i.e. those going from image(s) to label(s)\"\n        )\n        logger.error(msg)\n        raise OutOfTaskScopeError(msg)\n    # Relabeling-related (soft) constraint\n    if is_measurement_only_workflow and relabeling:\n        logger.warning(\n            \"This is a measurement-output-only workflow, setting \"\n            \"relabeling=False.\"\n        )\n        relabeling = False\n    if relabeling:\n        max_label_for_relabeling = 0\n\n    # Pre-processing of task inputs\n    if len(input_paths) > 1:\n        raise NotImplementedError(\n            \"We currently only support a single input path\"\n        )\n    in_path = Path(input_paths[0]).as_posix()\n    label_dtype = np.uint32\n\n    # Read ROI table\n    zarrurl = f\"{in_path}/{component}\"\n    ROI_table = ad.read_zarr(f\"{in_path}/{component}/tables/{input_ROI_table}\")\n\n    # Load image metadata\n    ngff_image_meta = load_NgffImageMeta(zarrurl)\n    num_levels = ngff_image_meta.num_levels\n    coarsening_xy = ngff_image_meta.coarsening_xy\n\n    # Read pixel sizes from zattrs file\n    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n\n    # Create list of indices for 3D FOVs spanning the entire Z direction\n    list_indices = convert_ROI_table_to_indices(\n        ROI_table,\n        level=level,\n        coarsening_xy=coarsening_xy,\n        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(list_indices, input_ROI_table)\n    num_ROIs = len(list_indices)\n    logger.info(\n        f\"Completed reading ROI table {input_ROI_table},\"\n        f\" found {num_ROIs} ROIs.\"\n    )\n\n    # Input preparation: \"image\" type\n    image_inputs = [\n        (name, in_params)\n        for (name, in_params) in input_specs.items()\n        if in_params.type == \"image\"\n    ]\n    input_image_arrays = {}\n    if image_inputs:\n        img_array = da.from_zarr(f\"{in_path}/{component}/{level}\")\n        # Loop over image inputs and assign corresponding channel of the image\n        for (name, params) in image_inputs:\n            channel = get_channel_from_image_zarr(\n                image_zarr_path=f\"{in_path}/{component}\",\n                wavelength_id=params.channel.wavelength_id,\n                label=params.channel.label,\n            )\n            channel_index = channel.index\n            input_image_arrays[name] = img_array[channel_index]\n\n            # Handle dimensions\n            shape = input_image_arrays[name].shape\n            if expected_dimensions == 3 and shape[0] == 1:\n                logger.warning(\n                    f\"Input {name} has shape {shape} \"\n                    f\"but {expected_dimensions=}\"\n                )\n            if expected_dimensions == 2:\n                if len(shape) == 2:\n                    # We already load the data as a 2D array\n                    pass\n                elif shape[0] == 1:\n                    input_image_arrays[name] = input_image_arrays[name][\n                        0, :, :\n                    ]\n                else:\n                    msg = (\n                        f\"Input {name} has shape {shape} \"\n                        f\"but {expected_dimensions=}\"\n                    )\n                    logger.error(msg)\n                    raise ValueError(msg)\n            logger.info(f\"Prepared input with {name=} and {params=}\")\n        logger.info(f\"{input_image_arrays=}\")\n\n    # Input preparation: \"label\" type\n    label_inputs = [\n        (name, in_params)\n        for (name, in_params) in input_specs.items()\n        if in_params.type == \"label\"\n    ]\n    if label_inputs:\n        # Set target_shape for upscaling labels\n        if not image_inputs:\n            logger.warning(\n                f\"{len(label_inputs)=} but num_image_inputs=0. \"\n                \"Label array(s) will not be upscaled.\"\n            )\n            upscale_labels = False\n        else:\n            target_shape = list(input_image_arrays.values())[0].shape\n            upscale_labels = True\n        # Loop over label inputs and load corresponding (upscaled) image\n        input_label_arrays = {}\n        for (name, params) in label_inputs:\n            label_name = params.label_name\n            label_array_raw = da.from_zarr(\n                f\"{in_path}/{component}/labels/{label_name}/{level}\"\n            )\n            input_label_arrays[name] = label_array_raw\n\n            # Handle dimensions\n            shape = input_label_arrays[name].shape\n            if expected_dimensions == 3 and shape[0] == 1:\n                logger.warning(\n                    f\"Input {name} has shape {shape} \"\n                    f\"but {expected_dimensions=}\"\n                )\n            if expected_dimensions == 2:\n                if len(shape) == 2:\n                    # We already load the data as a 2D array\n                    pass\n                elif shape[0] == 1:\n                    input_label_arrays[name] = input_label_arrays[name][\n                        0, :, :\n                    ]\n                else:\n                    msg = (\n                        f\"Input {name} has shape {shape} \"\n                        f\"but {expected_dimensions=}\"\n                    )\n                    logger.error(msg)\n                    raise ValueError(msg)\n\n            if upscale_labels:\n                # Check that dimensionality matches the image\n                if len(input_label_arrays[name].shape) != len(target_shape):\n                    raise ValueError(\n                        f\"Label {name} has shape \"\n                        f\"{input_label_arrays[name].shape}. \"\n                        \"But the corresponding image has shape \"\n                        f\"{target_shape}. Those dimensionalities do not \"\n                        f\"match. Is {expected_dimensions=} the correct \"\n                        \"setting?\"\n                    )\n                if expected_dimensions == 3:\n                    upscaling_axes = [1, 2]\n                else:\n                    upscaling_axes = [0, 1]\n                input_label_arrays[name] = upscale_array(\n                    array=input_label_arrays[name],\n                    target_shape=target_shape,\n                    axis=upscaling_axes,\n                    pad_with_zeros=True,\n                )\n\n            logger.info(f\"Prepared input with {name=} and {params=}\")\n        logger.info(f\"{input_label_arrays=}\")\n\n    # Output preparation: \"label\" type\n    label_outputs = [\n        (name, out_params)\n        for (name, out_params) in output_specs.items()\n        if out_params.type == \"label\"\n    ]\n    if label_outputs:\n        # Preliminary scope checks\n        if len(label_outputs) > 1:\n            raise OutOfTaskScopeError(\n                \"Multiple label outputs would break label-inputs-only \"\n                f\"workflows (found {len(label_outputs)=}).\"\n            )\n        if len(label_outputs) > 1 and relabeling:\n            raise OutOfTaskScopeError(\n                \"Multiple label outputs would break relabeling in labeling+\"\n                f\"measurement workflows (found {len(label_outputs)=}).\"\n            )\n\n        # We only support two cases:\n        # 1. If there exist some input images, then use the first one to\n        #    determine output-label array properties\n        # 2. If there are no input images, but there are input labels, then (A)\n        #    re-load the pixel sizes and re-build ROI indices, and (B) use the\n        #    first input label to determine output-label array properties\n        if image_inputs:\n            reference_array = list(input_image_arrays.values())[0]\n        elif label_inputs:\n            reference_array = list(input_label_arrays.values())[0]\n            # Re-load pixel size, matching to the correct level\n            input_label_name = label_inputs[0][1].label_name\n            ngff_label_image_meta = load_NgffImageMeta(\n                f\"{in_path}/{component}/labels/{input_label_name}\"\n            )\n            full_res_pxl_sizes_zyx = ngff_label_image_meta.get_pixel_sizes_zyx(\n                level=0\n            )\n            # Create list of indices for 3D FOVs spanning the whole Z direction\n            list_indices = convert_ROI_table_to_indices(\n                ROI_table,\n                level=level,\n                coarsening_xy=coarsening_xy,\n                full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n            )\n            check_valid_ROI_indices(list_indices, input_ROI_table)\n            num_ROIs = len(list_indices)\n            logger.info(\n                f\"Re-create ROI indices from ROI table {input_ROI_table}, \"\n                f\"using {full_res_pxl_sizes_zyx=}. \"\n                \"This is necessary because label-input-only workflows may \"\n                \"have label inputs that are at a different resolution and \"\n                \"are not upscaled.\"\n            )\n        else:\n            msg = (\n                \"Missing image_inputs and label_inputs, we cannot assign\"\n                \" label output properties\"\n            )\n            raise OutOfTaskScopeError(msg)\n\n        # Extract label properties from reference_array, and make sure they are\n        # for three dimensions\n        label_shape = reference_array.shape\n        label_chunksize = reference_array.chunksize\n        if len(label_shape) == 2 and len(label_chunksize) == 2:\n            if expected_dimensions == 3:\n                raise ValueError(\n                    f\"Something wrong: {label_shape=} but \"\n                    f\"{expected_dimensions=}\"\n                )\n            label_shape = (1, label_shape[0], label_shape[1])\n            label_chunksize = (1, label_chunksize[0], label_chunksize[1])\n        logger.info(f\"{label_shape=}\")\n        logger.info(f\"{label_chunksize=}\")\n\n        # Loop over label outputs and (1) set zattrs, (2) create zarr group\n        output_label_zarr_groups: dict[str, Any] = {}\n        for (name, out_params) in label_outputs:\n\n            # (1a) Rescale OME-NGFF datasets (relevant for level>0)\n            if not ngff_image_meta.multiscale.axes[0].name == \"c\":\n                raise ValueError(\n                    \"Cannot set `remove_channel_axis=True` for multiscale \"\n                    f\"metadata with axes={ngff_image_meta.multiscale.axes}. \"\n                    'First axis should have name \"c\".'\n                )\n            new_datasets = rescale_datasets(\n                datasets=[\n                    ds.dict() for ds in ngff_image_meta.multiscale.datasets\n                ],\n                coarsening_xy=coarsening_xy,\n                reference_level=level,\n                remove_channel_axis=True,\n            )\n\n            # (1b) Prepare attrs for label group\n            label_name = out_params.label_name\n            label_attrs = {\n                \"image-label\": {\n                    \"version\": __OME_NGFF_VERSION__,\n                    \"source\": {\"image\": \"../../\"},\n                },\n                \"multiscales\": [\n                    {\n                        \"name\": label_name,\n                        \"version\": __OME_NGFF_VERSION__,\n                        \"axes\": [\n                            ax.dict()\n                            for ax in ngff_image_meta.multiscale.axes\n                            if ax.type != \"channel\"\n                        ],\n                        \"datasets\": new_datasets,\n                    }\n                ],\n            }\n\n            # (2) Prepare label group\n            zarrurl = f\"{in_path}/{component}\"\n            image_group = zarr.group(zarrurl)\n            label_group = prepare_label_group(\n                image_group,\n                label_name,\n                overwrite=overwrite,\n                label_attrs=label_attrs,\n                logger=logger,\n            )\n            logger.info(\n                \"Helper function `prepare_label_group` returned \"\n                f\"{label_group=}\"\n            )\n\n            # (3) Create zarr group at level=0\n            store = zarr.storage.FSStore(\n                f\"{in_path}/{component}/labels/{label_name}/0\"\n            )\n            mask_zarr = zarr.create(\n                shape=label_shape,\n                chunks=label_chunksize,\n                dtype=label_dtype,\n                store=store,\n                overwrite=overwrite,\n                dimension_separator=\"/\",\n            )\n            output_label_zarr_groups[name] = mask_zarr\n            logger.info(f\"Prepared output with {name=} and {out_params=}\")\n        logger.info(f\"{output_label_zarr_groups=}\")\n\n    # Output preparation: \"dataframe\" type\n    dataframe_outputs = [\n        (name, out_params)\n        for (name, out_params) in output_specs.items()\n        if out_params.type == \"dataframe\"\n    ]\n    output_dataframe_lists: dict[str, list] = {}\n    for (name, out_params) in dataframe_outputs:\n        output_dataframe_lists[name] = []\n        logger.info(f\"Prepared output with {name=} and {out_params=}\")\n        logger.info(f\"{output_dataframe_lists=}\")\n\n    #####\n\n    for i_ROI, indices in enumerate(list_indices):\n        s_z, e_z, s_y, e_y, s_x, e_x = indices[:]\n        region = (slice(s_z, e_z), slice(s_y, e_y), slice(s_x, e_x))\n\n        logger.info(f\"ROI {i_ROI+1}/{num_ROIs}: {region=}\")\n\n        # Always re-load napari worfklow\n        wf = load_workflow(workflow_file)\n\n        # Set inputs\n        for input_name in input_specs.keys():\n            input_type = input_specs[input_name].type\n\n            if input_type == \"image\":\n                wf.set(\n                    input_name,\n                    load_region(\n                        input_image_arrays[input_name],\n                        region,\n                        compute=True,\n                        return_as_3D=False,\n                    ),\n                )\n            elif input_type == \"label\":\n                wf.set(\n                    input_name,\n                    load_region(\n                        input_label_arrays[input_name],\n                        region,\n                        compute=True,\n                        return_as_3D=False,\n                    ),\n                )\n\n        # Get outputs\n        outputs = wf.get(list_outputs)\n\n        # Iterate first over dataframe outputs (to use the correct\n        # max_label_for_relabeling, if needed)\n        for ind_output, output_name in enumerate(list_outputs):\n            if output_specs[output_name].type != \"dataframe\":\n                continue\n            df = outputs[ind_output]\n            if relabeling:\n                df[\"label\"] += max_label_for_relabeling\n                logger.info(\n                    f'ROI {i_ROI+1}/{num_ROIs}: Relabeling \"{name}\" dataframe'\n                    \"output, with {max_label_for_relabeling=}\"\n                )\n\n            # Append the new-ROI dataframe to the all-ROIs list\n            output_dataframe_lists[output_name].append(df)\n\n        # After all dataframe outputs, iterate over label outputs (which\n        # actually can be only 0 or 1)\n        for ind_output, output_name in enumerate(list_outputs):\n            if output_specs[output_name].type != \"label\":\n                continue\n            mask = outputs[ind_output]\n\n            # Check dimensions\n            if len(mask.shape) != expected_dimensions:\n                msg = (\n                    f\"Output {output_name} has shape {mask.shape} \"\n                    f\"but {expected_dimensions=}\"\n                )\n                logger.error(msg)\n                raise ValueError(msg)\n            elif expected_dimensions == 2:\n                mask = np.expand_dims(mask, axis=0)\n\n            # Sanity check: issue warning for non-consecutive labels\n            unique_labels = np.unique(mask)\n            num_unique_labels_in_this_ROI = len(unique_labels)\n            if np.min(unique_labels) == 0:\n                num_unique_labels_in_this_ROI -= 1\n            num_labels_in_this_ROI = int(np.max(mask))\n            if num_labels_in_this_ROI != num_unique_labels_in_this_ROI:\n                logger.warning(\n                    f'ROI {i_ROI+1}/{num_ROIs}: \"{name}\" label output has'\n                    f\"non-consecutive labels: {num_labels_in_this_ROI=} but\"\n                    f\"{num_unique_labels_in_this_ROI=}\"\n                )\n\n            if relabeling:\n                mask[mask > 0] += max_label_for_relabeling\n                logger.info(\n                    f'ROI {i_ROI+1}/{num_ROIs}: Relabeling \"{name}\" label '\n                    f\"output, with {max_label_for_relabeling=}\"\n                )\n                max_label_for_relabeling += num_labels_in_this_ROI\n                logger.info(\n                    f\"ROI {i_ROI+1}/{num_ROIs}: label-number update with \"\n                    f\"{num_labels_in_this_ROI=}; \"\n                    f\"new {max_label_for_relabeling=}\"\n                )\n\n            da.array(mask).to_zarr(\n                url=output_label_zarr_groups[output_name],\n                region=region,\n                compute=True,\n                overwrite=overwrite,\n            )\n        logger.info(f\"ROI {i_ROI+1}/{num_ROIs}: output handling complete\")\n\n    # Output handling: \"dataframe\" type (for each output, concatenate ROI\n    # dataframes, clean up, and store in a AnnData table on-disk)\n    for (name, out_params) in dataframe_outputs:\n        table_name = out_params.table_name\n        # Concatenate all FOV dataframes\n        list_dfs = output_dataframe_lists[name]\n        if len(list_dfs) == 0:\n            measurement_table = ad.AnnData()\n        else:\n            df_well = pd.concat(list_dfs, axis=0, ignore_index=True)\n            # Extract labels and drop them from df_well\n            labels = pd.DataFrame(df_well[\"label\"].astype(str))\n            df_well.drop(labels=[\"label\"], axis=1, inplace=True)\n            # Convert all to float (warning: some would be int, in principle)\n            measurement_dtype = np.float32\n            df_well = df_well.astype(measurement_dtype)\n            df_well.index = df_well.index.map(str)\n            # Convert to anndata\n            measurement_table = ad.AnnData(df_well, dtype=measurement_dtype)\n            measurement_table.obs = labels\n\n        # Write to zarr group\n        image_group = zarr.group(f\"{in_path}/{component}\")\n        # TODO: should we include any table_attrs? (ref issue #333)\n        write_table(\n            image_group,\n            table_name,\n            measurement_table,\n            overwrite=overwrite,\n            logger=logger,\n        )\n\n    # Output handling: \"label\" type (for each output, build and write to disk\n    # pyramid of coarser levels)\n    for (name, out_params) in label_outputs:\n        label_name = out_params.label_name\n        build_pyramid(\n            zarrurl=f\"{zarrurl}/labels/{label_name}\",\n            overwrite=overwrite,\n            num_levels=num_levels,\n            coarsening_xy=coarsening_xy,\n            chunksize=label_chunksize,\n            aggregation_function=np.max,\n        )\n\n    return {}\n
"},{"location":"reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/","title":"yokogawa_to_ome_zarr","text":"

Task that writes image data to an existing OME-NGFF zarr array.

"},{"location":"reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/#fractal_tasks_core.tasks.yokogawa_to_ome_zarr.sort_fun","title":"sort_fun(filename)","text":"

Takes a string (filename of a Yokogawa image), extract site and z-index metadata and returns them as a list of integers.

PARAMETER DESCRIPTION filename

Name of the image file.

TYPE: str

Source code in fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py 
def sort_fun(filename: str) -> list[int]:\n\"\"\"\n    Takes a string (filename of a Yokogawa image), extract site and\n    z-index metadata and returns them as a list of integers.\n\n    Args:\n        filename: Name of the image file.\n    \"\"\"\n\n    filename_metadata = parse_filename(filename)\n    site = int(filename_metadata[\"F\"])\n    z_index = int(filename_metadata[\"Z\"])\n    return [site, z_index]\n
"},{"location":"reference/fractal_tasks_core/tasks/yokogawa_to_ome_zarr/#fractal_tasks_core.tasks.yokogawa_to_ome_zarr.yokogawa_to_ome_zarr","title":"yokogawa_to_ome_zarr(*, input_paths, output_path, component, metadata, overwrite=False)","text":"

Convert Yokogawa output (png, tif) to zarr file.

This task is typically run after Create OME-Zarr or Create OME-Zarr Multiplexing and populates the empty OME-Zarr files that were prepared.

PARAMETER DESCRIPTION input_paths

List of input paths where the OME-Zarrs. Should point to the parent folder containing one or many OME-Zarr files, not the actual OME-Zarr file. Example: [\"/some/path/\"]. This task only supports a single input path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: Sequence[str]

output_path

Unclear. Should be the same as input_path. (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

component

Path to the OME-Zarr image in the OME-Zarr plate that is processed. Example: \"some_plate.zarr/B/03/0\" (standard argument for Fractal tasks, managed by Fractal server).

TYPE: str

metadata

Dictionary containing metadata about the OME-Zarr. This task requires the following elements to be present in the metadata. original_paths: list of paths that correspond to the input_paths of the create_ome_zarr task (=> where the microscopy image are stored); num_levels (int): number of pyramid levels in the image (this determines how many pyramid levels are built for the segmentation); coarsening_xy (int): coarsening factor in XY of the downsampling when building the pyramid; image_extension: filename extension of images (e.g. \"tif\" or \"png\"); image_glob_patterns: parameter of create_ome_zarr task (if specified, only parse images with filenames that match with all these patterns). (standard argument for Fractal tasks, managed by Fractal server).

TYPE: dict[str, Any]

overwrite

If True, overwrite the task output.

TYPE: bool DEFAULT: False

Source code in fractal_tasks_core/tasks/yokogawa_to_ome_zarr.py 
@validate_arguments\ndef yokogawa_to_ome_zarr(\n    *,\n    input_paths: Sequence[str],\n    output_path: str,\n    component: str,\n    metadata: dict[str, Any],\n    overwrite: bool = False,\n):\n\"\"\"\n    Convert Yokogawa output (png, tif) to zarr file.\n\n    This task is typically run after Create OME-Zarr or\n    Create OME-Zarr Multiplexing and populates the empty OME-Zarr files that\n    were prepared.\n\n    Args:\n        input_paths: List of input paths where the OME-Zarrs. Should point to\n            the parent folder containing one or many OME-Zarr files, not the\n            actual OME-Zarr file. Example: `[\"/some/path/\"]`.\n            This task only supports a single input path.\n            (standard argument for Fractal tasks,\n            managed by Fractal server).\n        output_path: Unclear. Should be the same as `input_path`.\n            (standard argument for Fractal tasks, managed by Fractal server).\n        component: Path to the OME-Zarr image in the OME-Zarr plate that is\n            processed. Example: `\"some_plate.zarr/B/03/0\"`\n            (standard argument for Fractal tasks, managed by Fractal server).\n        metadata: Dictionary containing metadata about the OME-Zarr. This task\n            requires the following elements to be present in the metadata.\n            `original_paths`:\n            list of paths that correspond to the `input_paths` of the\n            `create_ome_zarr` task (=> where the microscopy image are stored);\n            `num_levels (int)`:\n            number of pyramid levels in the image (this determines how many\n            pyramid levels are built for the segmentation);\n            `coarsening_xy (int)`:\n            coarsening factor in XY of the downsampling when building the\n            pyramid;\n            `image_extension`:\n            filename extension of images (e.g. `\"tif\"` or `\"png\"`);\n            `image_glob_patterns`:\n            parameter of `create_ome_zarr` task (if specified, only parse\n            images with filenames that match with all these patterns).\n            (standard argument for Fractal tasks, managed by Fractal server).\n        overwrite: If `True`, overwrite the task output.\n    \"\"\"\n\n    # Preliminary checks\n    if len(input_paths) > 1:\n        raise NotImplementedError\n    zarrurl = Path(input_paths[0]).as_posix() + f\"/{component}\"\n\n    # Read attributes from NGFF metadata\n    ngff_image_meta = load_NgffImageMeta(zarrurl)\n    num_levels = ngff_image_meta.num_levels\n    coarsening_xy = ngff_image_meta.coarsening_xy\n    full_res_pxl_sizes_zyx = ngff_image_meta.get_pixel_sizes_zyx(level=0)\n    logger.info(f\"NGFF image has {num_levels=}\")\n    logger.info(f\"NGFF image has {coarsening_xy=}\")\n    logger.info(\n        f\"NGFF image has full-res pixel sizes {full_res_pxl_sizes_zyx}\"\n    )\n\n    parameters = get_parameters_from_metadata(\n        keys=[\n            \"original_paths\",\n            \"image_extension\",\n            \"image_glob_patterns\",\n        ],\n        metadata=metadata,\n        # FIXME: Why rely on output_path here, when we use the input path for\n        # the zarr_url? That just means that different input & output paths\n        # don't work, no?\n        image_zarr_path=(Path(output_path) / component),\n    )\n    original_path_list = parameters[\"original_paths\"]\n    image_extension = parameters[\"image_extension\"]\n    image_glob_patterns = parameters[\"image_glob_patterns\"]\n\n    channels: list[OmeroChannel] = get_omero_channel_list(\n        image_zarr_path=zarrurl\n    )\n    wavelength_ids = [c.wavelength_id for c in channels]\n\n    in_path = Path(original_path_list[0])\n\n    # Define well\n    component_split = component.split(\"/\")\n    well_row = component_split[1]\n    well_column = component_split[2]\n    well_ID = well_row + well_column\n\n    # Read useful information from ROI table\n    adata = read_zarr(f\"{zarrurl}/tables/FOV_ROI_table\")\n    fov_indices = convert_ROI_table_to_indices(\n        adata,\n        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(fov_indices, \"FOV_ROI_table\")\n    adata_well = read_zarr(f\"{zarrurl}/tables/well_ROI_table\")\n    well_indices = convert_ROI_table_to_indices(\n        adata_well,\n        full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx,\n    )\n    check_valid_ROI_indices(well_indices, \"well_ROI_table\")\n    if len(well_indices) > 1:\n        raise ValueError(f\"Something wrong with {well_indices=}\")\n\n    # FIXME: Put back the choice of columns by name? Not here..\n\n    max_z = well_indices[0][1]\n    max_y = well_indices[0][3]\n    max_x = well_indices[0][5]\n\n    # Load a single image, to retrieve useful information\n    patterns = [f\"*_{well_ID}_*.{image_extension}\"]\n    if image_glob_patterns:\n        patterns.extend(image_glob_patterns)\n    tmp_images = glob_with_multiple_patterns(\n        folder=str(in_path),\n        patterns=patterns,\n    )\n    sample = imread(tmp_images.pop())\n\n    # Initialize zarr\n    chunksize = (1, 1, sample.shape[1], sample.shape[2])\n    try:\n        canvas_zarr = zarr.create(\n            shape=(len(wavelength_ids), max_z, max_y, max_x),\n            chunks=chunksize,\n            dtype=sample.dtype,\n            store=zarr.storage.FSStore(zarrurl + \"/0\"),\n            overwrite=overwrite,\n            dimension_separator=\"/\",\n        )\n    except ContainsArrayError as e:\n        error_msg = (\n            f\"Cannot create a zarr group at '{zarrurl}/0', \"\n            f\"with {overwrite=} (original error: {str(e)}).\\n\"\n            \"Hint: try setting overwrite=True.\"\n        )\n        logger.error(error_msg)\n        raise OverwriteNotAllowedError(error_msg)\n\n    # Loop over channels\n    for i_c, wavelength_id in enumerate(wavelength_ids):\n        A, C = wavelength_id.split(\"_\")\n\n        patterns = [f\"*_{well_ID}_*{A}*{C}*.{image_extension}\"]\n        if image_glob_patterns:\n            patterns.extend(image_glob_patterns)\n        filenames_set = glob_with_multiple_patterns(\n            folder=str(in_path),\n            patterns=patterns,\n        )\n        filenames = sorted(list(filenames_set), key=sort_fun)\n        if len(filenames) == 0:\n            raise ValueError(\n                \"Error in yokogawa_to_ome_zarr: len(filenames)=0.\\n\"\n                f\"  in_path: {in_path}\\n\"\n                f\"  image_extension: {image_extension}\\n\"\n                f\"  well_ID: {well_ID}\\n\"\n                f\"  wavelength_id: {wavelength_id},\\n\"\n                f\"  patterns: {patterns}\"\n            )\n        # Loop over 3D FOV ROIs\n        for indices in fov_indices:\n            s_z, e_z, s_y, e_y, s_x, e_x = indices[:]\n            region = (\n                slice(i_c, i_c + 1),\n                slice(s_z, e_z),\n                slice(s_y, e_y),\n                slice(s_x, e_x),\n            )\n            FOV_3D = da.concatenate(\n                [imread(img) for img in filenames[:e_z]],\n            )\n            FOV_4D = da.expand_dims(FOV_3D, axis=0)\n            filenames = filenames[e_z:]\n            da.array(FOV_4D).to_zarr(\n                url=canvas_zarr,\n                region=region,\n                compute=True,\n            )\n\n    # Starting from on-disk highest-resolution data, build and write to disk a\n    # pyramid of coarser levels\n    build_pyramid(\n        zarrurl=zarrurl,\n        overwrite=overwrite,\n        num_levels=num_levels,\n        coarsening_xy=coarsening_xy,\n        chunksize=chunksize,\n    )\n\n    # Deprecated: Delete images (optional)\n    # if delete_input:\n    #     for f in filenames:\n    #         try:\n    #             os.remove(f)\n    #         except OSError as e:\n    #             logging.info(\"Error: %s : %s\" % (f, e.strerror))\n\n    return {}\n
"}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 000000000..0f8724efd --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..bd82a09443510fceeea842d792b4a52a2df117db GIT binary patch literal 127 zcmV-_0D%7=iwFo6Au?qG|8r?{Wo=<_E_iKh04<9_3V)_WXo8&M?ytk3HC}0~zlG)Vu + + + + + + + + + + + + + + + + + + + + Run tasks in Fractal - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Run tasks within the Fractal platform

+

Thanks to the package manifest and to their structure, the +tasks in fractal_tasks_core.tasks can be run within the Fractal +platform; this consists in a +backend server +which can be accessed by one of the two available clients (a command-line +client and a +web-client).

+

The fractal-demos repository lists a set of relevant examples, including:

+ + + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/tasks_in_scripts/index.html b/tasks_in_scripts/index.html new file mode 100644 index 000000000..2a2d81903 --- /dev/null +++ b/tasks_in_scripts/index.html @@ -0,0 +1,1108 @@ + + + + + + + + + + + + + + + + + + + + + + Run tasks in Python scripts - Fractal Tasks Core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Run tasks from Python scripts

+

The fractal-tasks-core GitHub repository includes an examples folder, listing a few examples of how to run fractal-tasks-core tasks from a standard Python script (instead of using the Fractal platform).

+

What follows is the content of examples/README.md:

+ +

Examples

+

This examples folder offers a few examples of how to run fractal-tasks-core +tasks as part of a Python script.

+

Notes

+ +

General instructions

+

The following instructions are valid for all examples; check the specific +README.md files in each folder for more specific details.

+
    +
  1. +

    Set up the correct environment via +

    pip install fractal-tasks-core[fractal-tasks]
+
    +(note: this can be done e.g. from a venv or from a conda environment).

    +
    2. +
  2. +

    Download the example data from Zenodo, if necessary, via +

    pip install zenodo-get
+./fetch_test_data_from_zenodo.sh
+

    +
    3. +
  3. +

    Enter one of the example folders, remove the tmp_out temporary output + folder (if present), and run one of the run_workflow Python scripts.

    +
    4. +
  4. +

    View the output OME-Zarr in the tmp_out folder with + napari, which can be installed via pip install + napari[pyqt5] napari-ome-zarr.

    +
    5. +
+ + + + + + + +
+
+ + +
+ +
+ +
+ +
+
+
+ +
+ © Copyright 2022 +Friedrich Miescher Institute for Biomedical Research and +University of Zurich +(see +LICENSE +). + +
+ + + Made with + + Material for MkDocs + + +
+ +
+
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file