Laion5Bblogpost.html

<html>

<head>
    <meta content="text/html; charset=UTF-8" http-equiv="content-type">
    <style type="text/css">
        .lst-kix_w64dywk42787-5>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-7>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-4>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-8>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-1>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-3>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-2>li:before {
            content: "+  "
        }

        .lst-kix_w64dywk42787-6>li:before {
            content: "+  "
        }

        .lst-kix_eiazrjwerwj4-5>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-5
        }

        ol.lst-kix_eiazrjwerwj4-3.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-3 0
        }

        .lst-kix_a5e5o2yyrzau-5>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-5
        }

        .lst-kix_mcx9mwl64ou5-1>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-1
        }

        ul.lst-kix_bxk20eq5dbuj-8 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-2 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-3 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-0 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-1 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-6 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-7 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-4 {
            list-style-type: none
        }

        ul.lst-kix_bxk20eq5dbuj-5 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-8 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-8 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-4.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-4 0
        }

        ul.lst-kix_uk2bl0dl1nvh-1 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-0 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-3 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-2 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-5 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-4 {
            list-style-type: none
        }

        ul.lst-kix_uk2bl0dl1nvh-7 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-8.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-8 0
        }

        ul.lst-kix_uk2bl0dl1nvh-6 {
            list-style-type: none
        }

        .lst-kix_w64dywk42787-0>li:before {
            content: "+  "
        }

        ul.lst-kix_7t2fnfiik8pc-3 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-2 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-1 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-0 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-7 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-6 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-5 {
            list-style-type: none
        }

        ul.lst-kix_7t2fnfiik8pc-4 {
            list-style-type: none
        }

        .lst-kix_vcjwjha01w8n-7>li:before {
            content: "-  "
        }

        .lst-kix_jkaumvtzuajs-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_jkaumvtzuajs-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_jkaumvtzuajs-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_vcjwjha01w8n-5>li:before {
            content: "-  "
        }

        ul.lst-kix_2l2t7l68oh17-7 {
            list-style-type: none
        }

        ul.lst-kix_2l2t7l68oh17-8 {
            list-style-type: none
        }

        .lst-kix_cvb2fa7g0inw-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_2l2t7l68oh17-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_vcjwjha01w8n-3>li:before {
            content: "-  "
        }

        ul.lst-kix_2l2t7l68oh17-0 {
            list-style-type: none
        }

        ul.lst-kix_2l2t7l68oh17-1 {
            list-style-type: none
        }

        .lst-kix_jkaumvtzuajs-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_2l2t7l68oh17-2 {
            list-style-type: none
        }

        ul.lst-kix_2l2t7l68oh17-3 {
            list-style-type: none
        }

        .lst-kix_vcjwjha01w8n-1>li:before {
            content: "-  "
        }

        ul.lst-kix_2l2t7l68oh17-4 {
            list-style-type: none
        }

        .lst-kix_cvb2fa7g0inw-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_2l2t7l68oh17-5 {
            list-style-type: none
        }

        .lst-kix_2l2t7l68oh17-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_2l2t7l68oh17-6 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_eiazrjwerwj4-8>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-8
        }

        .lst-kix_8mezmq2lvg5s-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_wh3bw2uekp0m-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_cvb2fa7g0inw-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_cvb2fa7g0inw-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_2l2t7l68oh17-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_2l2t7l68oh17-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_wh3bw2uekp0m-6>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_a5e5o2yyrzau-7.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-7 0
        }

        .lst-kix_cvb2fa7g0inw-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_2l2t7l68oh17-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_7t2fnfiik8pc-7>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_qgdrxcrwev1n-8 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-7 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_wh3bw2uekp0m-8>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_qgdrxcrwev1n-0 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_qgdrxcrwev1n-2 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-1 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-4 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-3 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-6 {
            list-style-type: none
        }

        ul.lst-kix_qgdrxcrwev1n-5 {
            list-style-type: none
        }

        .lst-kix_q9ohf0mp8zzu-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_q9ohf0mp8zzu-6>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_eiazrjwerwj4-0.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-0 0
        }

        .lst-kix_q9ohf0mp8zzu-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_q9ohf0mp8zzu-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_q9ohf0mp8zzu-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_eiazrjwerwj4-1>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-1
        }

        .lst-kix_vumiwzrrem62-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_g9u3sdyi5sn-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_vumiwzrrem62-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_kw79sbq53lji-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_g9u3sdyi5sn-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_7t2fnfiik8pc-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_vumiwzrrem62-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_6qd5emnhfh12-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_6qd5emnhfh12-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_vumiwzrrem62-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_7t2fnfiik8pc-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_kw79sbq53lji-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_6qd5emnhfh12-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_6qd5emnhfh12-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_kw79sbq53lji-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_kw79sbq53lji-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_g9u3sdyi5sn-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_7t2fnfiik8pc-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_kw79sbq53lji-2>li:before {
            content: "\0025a0  "
        }

        ol.lst-kix_mcx9mwl64ou5-1.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-1 0
        }

        .lst-kix_mcx9mwl64ou5-8>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-8
        }

        .lst-kix_g9u3sdyi5sn-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_g9u3sdyi5sn-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_peqe1kx757sw-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_peqe1kx757sw-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_peqe1kx757sw-8>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_vcjwjha01w8n-6 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-7 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-8 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-2 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-3 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-4 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-4>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-4
        }

        ul.lst-kix_vcjwjha01w8n-5 {
            list-style-type: none
        }

        ul.lst-kix_kw79sbq53lji-8 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-0 {
            list-style-type: none
        }

        ul.lst-kix_vcjwjha01w8n-1 {
            list-style-type: none
        }

        .lst-kix_317oy7bf1j1g-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_a5e5o2yyrzau-2>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-2
        }

        .lst-kix_qgdrxcrwev1n-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_v0ratkyi1f8x-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_317oy7bf1j1g-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_vumiwzrrem62-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_qgdrxcrwev1n-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_4afi4pjkpfna-5 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-2.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-2 0
        }

        ul.lst-kix_4afi4pjkpfna-6 {
            list-style-type: none
        }

        ul.lst-kix_4afi4pjkpfna-3 {
            list-style-type: none
        }

        .lst-kix_v0ratkyi1f8x-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_317oy7bf1j1g-4>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_4afi4pjkpfna-4 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_4afi4pjkpfna-7 {
            list-style-type: none
        }

        ul.lst-kix_4afi4pjkpfna-8 {
            list-style-type: none
        }

        .lst-kix_peqe1kx757sw-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_v0ratkyi1f8x-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_peqe1kx757sw-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_4afi4pjkpfna-1 {
            list-style-type: none
        }

        .lst-kix_v0ratkyi1f8x-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_317oy7bf1j1g-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_4afi4pjkpfna-2 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-0.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-0 0
        }

        ul.lst-kix_4afi4pjkpfna-0 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-1>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-1, lower-latin) ". "
        }

        ul.lst-kix_v0ratkyi1f8x-0 {
            list-style-type: none
        }

        ul.lst-kix_v0ratkyi1f8x-1 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_a5e5o2yyrzau-0>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-0, decimal) ". "
        }

        ol.lst-kix_a5e5o2yyrzau-6.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-6 0
        }

        ul.lst-kix_v0ratkyi1f8x-2 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-0>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-0
        }

        .lst-kix_uk2bl0dl1nvh-7>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_v0ratkyi1f8x-3 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-1>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-1, lower-latin) ". "
        }

        ul.lst-kix_v0ratkyi1f8x-8 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-0>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-0, decimal) ". "
        }

        .lst-kix_a5e5o2yyrzau-4>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-4, lower-latin) ". "
        }

        .lst-kix_v0ratkyi1f8x-0>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_v0ratkyi1f8x-4 {
            list-style-type: none
        }

        ul.lst-kix_v0ratkyi1f8x-5 {
            list-style-type: none
        }

        ul.lst-kix_v0ratkyi1f8x-6 {
            list-style-type: none
        }

        .lst-kix_uk2bl0dl1nvh-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_v0ratkyi1f8x-7 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-5 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-6 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-7 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-8 {
            list-style-type: none
        }

        .lst-kix_uk2bl0dl1nvh-3>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_mcx9mwl64ou5-1 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-2 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-3 {
            list-style-type: none
        }

        .lst-kix_eiazrjwerwj4-0>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-0
        }

        ol.lst-kix_mcx9mwl64ou5-4 {
            list-style-type: none
        }

        .lst-kix_uk2bl0dl1nvh-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_wh3bw2uekp0m-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_wh3bw2uekp0m-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_mcx9mwl64ou5-0>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-0
        }

        .lst-kix_mcx9mwl64ou5-8>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-8, lower-roman) ". "
        }

        .lst-kix_lp9fh9kx7cvx-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_lp9fh9kx7cvx-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-5>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-5, lower-roman) ". "
        }

        ul.lst-kix_peqe1kx757sw-2 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-1 {
            list-style-type: none
        }

        .lst-kix_mcx9mwl64ou5-4>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-4, lower-latin) ". "
        }

        .lst-kix_a5e5o2yyrzau-8>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-8, lower-roman) ". "
        }

        ul.lst-kix_peqe1kx757sw-4 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-3 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-6 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-5 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-8 {
            list-style-type: none
        }

        ul.lst-kix_peqe1kx757sw-7 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-5>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-5, lower-roman) ". "
        }

        ol.lst-kix_a5e5o2yyrzau-5.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-5 0
        }

        .lst-kix_lp9fh9kx7cvx-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_lp9fh9kx7cvx-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_peqe1kx757sw-0 {
            list-style-type: none
        }

        .lst-kix_bxk20eq5dbuj-4>li:before {
            content: "\0025cb  "
        }

        ol.lst-kix_a5e5o2yyrzau-1.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-1 0
        }

        .lst-kix_bxk20eq5dbuj-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_bxk20eq5dbuj-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_jkaumvtzuajs-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_jkaumvtzuajs-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_4afi4pjkpfna-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_vcjwjha01w8n-6>li:before {
            content: "-  "
        }

        .lst-kix_7taahgizfshk-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_1ahkn1ua0ys7-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_2l2t7l68oh17-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_cvb2fa7g0inw-3>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_a5e5o2yyrzau-4.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-4 0
        }

        .lst-kix_jkaumvtzuajs-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_4afi4pjkpfna-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_vcjwjha01w8n-2>li:before {
            content: "-  "
        }

        ul.lst-kix_8mezmq2lvg5s-2 {
            list-style-type: none
        }

        .lst-kix_1ahkn1ua0ys7-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_8mezmq2lvg5s-3 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-0>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_8mezmq2lvg5s-4 {
            list-style-type: none
        }

        .lst-kix_2l2t7l68oh17-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_8mezmq2lvg5s-5 {
            list-style-type: none
        }

        ul.lst-kix_8mezmq2lvg5s-6 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-6.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-6 0
        }

        .lst-kix_wh3bw2uekp0m-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_7taahgizfshk-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_8mezmq2lvg5s-7 {
            list-style-type: none
        }

        ul.lst-kix_8mezmq2lvg5s-8 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-7>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-7
        }

        .lst-kix_cvb2fa7g0inw-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_eiazrjwerwj4-4>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-4, lower-latin) ". "
        }

        ul.lst-kix_8mezmq2lvg5s-0 {
            list-style-type: none
        }

        ul.lst-kix_8mezmq2lvg5s-1 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_7t2fnfiik8pc-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_8mezmq2lvg5s-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_eiazrjwerwj4-8>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-8, lower-roman) ". "
        }

        .lst-kix_vumiwzrrem62-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_q9ohf0mp8zzu-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_q9ohf0mp8zzu-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_oat2jjdhxffc-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_317oy7bf1j1g-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_oat2jjdhxffc-4>li:before {
            content: "\0025cb  "
        }

        ol.lst-kix_mcx9mwl64ou5-8.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-8 0
        }

        .lst-kix_g9u3sdyi5sn-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_qgdrxcrwev1n-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_6qd5emnhfh12-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_7t2fnfiik8pc-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_vumiwzrrem62-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_eiazrjwerwj4-0>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-0, decimal) ". "
        }

        ul.lst-kix_kw79sbq53lji-4 {
            list-style-type: none
        }

        .lst-kix_6qd5emnhfh12-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_kw79sbq53lji-5 {
            list-style-type: none
        }

        .lst-kix_kw79sbq53lji-7>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_kw79sbq53lji-6 {
            list-style-type: none
        }

        ul.lst-kix_kw79sbq53lji-7 {
            list-style-type: none
        }

        .lst-kix_7t2fnfiik8pc-0>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_kw79sbq53lji-0 {
            list-style-type: none
        }

        ul.lst-kix_kw79sbq53lji-1 {
            list-style-type: none
        }

        ul.lst-kix_kw79sbq53lji-2 {
            list-style-type: none
        }

        ul.lst-kix_kw79sbq53lji-3 {
            list-style-type: none
        }

        .lst-kix_kw79sbq53lji-3>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_a5e5o2yyrzau-2.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-2 0
        }

        .lst-kix_mcx9mwl64ou5-5>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-5
        }

        .lst-kix_oat2jjdhxffc-0>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_mcx9mwl64ou5-7.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-7 0
        }

        .lst-kix_g9u3sdyi5sn-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_a5e5o2yyrzau-3>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-3
        }

        .lst-kix_eiazrjwerwj4-3>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-3
        }

        ol.lst-kix_a5e5o2yyrzau-3.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-3 0
        }

        ol.lst-kix_mcx9mwl64ou5-5.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-5 0
        }

        ol.lst-kix_a5e5o2yyrzau-6 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-7 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-4 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-5 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-8 {
            list-style-type: none
        }

        .lst-kix_joualahdnmon-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_joualahdnmon-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-3>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-3
        }

        .lst-kix_joualahdnmon-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_1ahkn1ua0ys7-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_joualahdnmon-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_joualahdnmon-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_joualahdnmon-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_joualahdnmon-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_joualahdnmon-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_joualahdnmon-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_bxk20eq5dbuj-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_1ahkn1ua0ys7-1 {
            list-style-type: none
        }

        .lst-kix_4afi4pjkpfna-0>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_1ahkn1ua0ys7-0 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-0.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-0 0
        }

        .lst-kix_4afi4pjkpfna-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-7>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-7
        }

        .lst-kix_bxk20eq5dbuj-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_bxk20eq5dbuj-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_lp9fh9kx7cvx-2 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-3 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-0 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-1 {
            list-style-type: none
        }

        .lst-kix_4afi4pjkpfna-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_7taahgizfshk-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_4afi4pjkpfna-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_1ahkn1ua0ys7-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_4afi4pjkpfna-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_1ahkn1ua0ys7-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_1ahkn1ua0ys7-8 {
            list-style-type: none
        }

        ul.lst-kix_1ahkn1ua0ys7-7 {
            list-style-type: none
        }

        ul.lst-kix_1ahkn1ua0ys7-6 {
            list-style-type: none
        }

        ol.lst-kix_mcx9mwl64ou5-3.start {
            counter-reset: lst-ctn-kix_mcx9mwl64ou5-3 0
        }

        ul.lst-kix_1ahkn1ua0ys7-5 {
            list-style-type: none
        }

        ul.lst-kix_1ahkn1ua0ys7-4 {
            list-style-type: none
        }

        ul.lst-kix_1ahkn1ua0ys7-3 {
            list-style-type: none
        }

        ul.lst-kix_1ahkn1ua0ys7-2 {
            list-style-type: none
        }

        .lst-kix_1ahkn1ua0ys7-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_7taahgizfshk-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_eiazrjwerwj4-1>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-1, lower-latin) ". "
        }

        ol.lst-kix_mcx9mwl64ou5-0 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-2 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-3 {
            list-style-type: none
        }

        .lst-kix_1ahkn1ua0ys7-5>li:before {
            content: "\0025a0  "
        }

        ol.lst-kix_a5e5o2yyrzau-0 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-1 {
            list-style-type: none
        }

        .lst-kix_7taahgizfshk-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_eiazrjwerwj4-5>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-5, lower-roman) ". "
        }

        .lst-kix_7taahgizfshk-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_eiazrjwerwj4-3>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-3, decimal) ". "
        }

        ul.lst-kix_w64dywk42787-2 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-1 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-0 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-6 {
            list-style-type: none
        }

        .lst-kix_bxk20eq5dbuj-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_w64dywk42787-5 {
            list-style-type: none
        }

        .lst-kix_7taahgizfshk-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_eiazrjwerwj4-7>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-7, lower-latin) ". "
        }

        ul.lst-kix_w64dywk42787-4 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-3 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-8 {
            list-style-type: none
        }

        ul.lst-kix_w64dywk42787-7 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-0 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-8 {
            list-style-type: none
        }

        .lst-kix_eiazrjwerwj4-7>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-7
        }

        ul.lst-kix_wh3bw2uekp0m-4 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-5 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-6 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-7 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-0 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-1 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-1 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-0 {
            list-style-type: none
        }

        ul.lst-kix_wh3bw2uekp0m-2 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-3 {
            list-style-type: none
        }

        .lst-kix_oat2jjdhxffc-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_oat2jjdhxffc-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_wh3bw2uekp0m-3 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-2 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-5 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-4 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-7 {
            list-style-type: none
        }

        .lst-kix_qgdrxcrwev1n-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_qgdrxcrwev1n-8>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_6qd5emnhfh12-6 {
            list-style-type: none
        }

        ul.lst-kix_6qd5emnhfh12-8 {
            list-style-type: none
        }

        .lst-kix_oat2jjdhxffc-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_qgdrxcrwev1n-6>li:before {
            content: "\0025cf  "
        }

        ol.lst-kix_eiazrjwerwj4-8 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-7 {
            list-style-type: none
        }

        .lst-kix_oat2jjdhxffc-1>li:before {
            content: "\0025cb  "
        }

        ol.lst-kix_eiazrjwerwj4-6 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-5 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-6 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-0 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-7 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-4 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-5 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-4 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-3 {
            list-style-type: none
        }

        ul.lst-kix_lp9fh9kx7cvx-8 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-2 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-1 {
            list-style-type: none
        }

        ol.lst-kix_a5e5o2yyrzau-8.start {
            counter-reset: lst-ctn-kix_a5e5o2yyrzau-8 0
        }

        .lst-kix_peqe1kx757sw-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_eiazrjwerwj4-4>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-4
        }

        .lst-kix_lp9fh9kx7cvx-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_mcx9mwl64ou5-2>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-2
        }

        .lst-kix_lp9fh9kx7cvx-7>li:before {
            content: "\0025cb  "
        }

        ol.lst-kix_eiazrjwerwj4-6.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-6 0
        }

        ul.lst-kix_7taahgizfshk-2 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-3 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-0 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-1 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-6 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-7 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-4 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-5 {
            list-style-type: none
        }

        ul.lst-kix_7taahgizfshk-8 {
            list-style-type: none
        }

        .lst-kix_qgdrxcrwev1n-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_eiazrjwerwj4-2>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-2
        }

        .lst-kix_qgdrxcrwev1n-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_v0ratkyi1f8x-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_317oy7bf1j1g-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_fk7u4doo34jn-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_v0ratkyi1f8x-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_317oy7bf1j1g-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_peqe1kx757sw-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_peqe1kx757sw-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_joualahdnmon-3 {
            list-style-type: none
        }

        .lst-kix_317oy7bf1j1g-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_joualahdnmon-4 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-1 {
            list-style-type: none
        }

        .lst-kix_317oy7bf1j1g-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_peqe1kx757sw-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_joualahdnmon-2 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-7 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-8 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-5 {
            list-style-type: none
        }

        ul.lst-kix_joualahdnmon-6 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_fk7u4doo34jn-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_a5e5o2yyrzau-2>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-2, lower-roman) ". "
        }

        .lst-kix_a5e5o2yyrzau-6>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-6
        }

        .lst-kix_mcx9mwl64ou5-4>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-4
        }

        .lst-kix_uk2bl0dl1nvh-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_vumiwzrrem62-8 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-7.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-7 0
        }

        ul.lst-kix_vumiwzrrem62-7 {
            list-style-type: none
        }

        .lst-kix_uk2bl0dl1nvh-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_uk2bl0dl1nvh-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_fk7u4doo34jn-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_v0ratkyi1f8x-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_vumiwzrrem62-4 {
            list-style-type: none
        }

        ul.lst-kix_vumiwzrrem62-3 {
            list-style-type: none
        }

        .lst-kix_fk7u4doo34jn-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_v0ratkyi1f8x-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_vumiwzrrem62-6 {
            list-style-type: none
        }

        ul.lst-kix_vumiwzrrem62-5 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-3>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-3, decimal) ". "
        }

        ul.lst-kix_vumiwzrrem62-0 {
            list-style-type: none
        }

        ul.lst-kix_vumiwzrrem62-2 {
            list-style-type: none
        }

        ul.lst-kix_vumiwzrrem62-1 {
            list-style-type: none
        }

        .lst-kix_uk2bl0dl1nvh-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_eiazrjwerwj4-6>li {
            counter-increment: lst-ctn-kix_eiazrjwerwj4-6
        }

        .lst-kix_uk2bl0dl1nvh-1>li:before {
            content: "\0025cb  "
        }

        ol.lst-kix_eiazrjwerwj4-1.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-1 0
        }

        .lst-kix_lp9fh9kx7cvx-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_wh3bw2uekp0m-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_lp9fh9kx7cvx-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_lp9fh9kx7cvx-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_mcx9mwl64ou5-7>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-7, lower-latin) ". "
        }

        .lst-kix_mcx9mwl64ou5-6>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-6, decimal) ". "
        }

        .lst-kix_mcx9mwl64ou5-2>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-2, lower-roman) ". "
        }

        .lst-kix_a5e5o2yyrzau-6>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-6, decimal) ". "
        }

        .lst-kix_mcx9mwl64ou5-3>li:before {
            content: ""counter(lst-ctn-kix_mcx9mwl64ou5-3, decimal) ". "
        }

        .lst-kix_a5e5o2yyrzau-7>li:before {
            content: ""counter(lst-ctn-kix_a5e5o2yyrzau-7, lower-latin) ". "
        }

        .lst-kix_jkaumvtzuajs-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_cvb2fa7g0inw-0 {
            list-style-type: none
        }

        .lst-kix_bxk20eq5dbuj-2>li:before {
            content: "\0025a0  "
        }

        ol.lst-kix_eiazrjwerwj4-2.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-2 0
        }

        .lst-kix_vcjwjha01w8n-4>li:before {
            content: "-  "
        }

        .lst-kix_vcjwjha01w8n-8>li:before {
            content: "-  "
        }

        ul.lst-kix_jkaumvtzuajs-8 {
            list-style-type: none
        }

        .lst-kix_4afi4pjkpfna-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_a5e5o2yyrzau-8>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-8
        }

        .lst-kix_vcjwjha01w8n-0>li:before {
            content: "-  "
        }

        .lst-kix_cvb2fa7g0inw-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_cvb2fa7g0inw-7 {
            list-style-type: none
        }

        .lst-kix_jkaumvtzuajs-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_cvb2fa7g0inw-8 {
            list-style-type: none
        }

        .lst-kix_4afi4pjkpfna-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_cvb2fa7g0inw-5 {
            list-style-type: none
        }

        .lst-kix_mcx9mwl64ou5-6>li {
            counter-increment: lst-ctn-kix_mcx9mwl64ou5-6
        }

        ul.lst-kix_cvb2fa7g0inw-6 {
            list-style-type: none
        }

        .lst-kix_2l2t7l68oh17-3>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_cvb2fa7g0inw-3 {
            list-style-type: none
        }

        ul.lst-kix_cvb2fa7g0inw-4 {
            list-style-type: none
        }

        .lst-kix_1ahkn1ua0ys7-4>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_cvb2fa7g0inw-1 {
            list-style-type: none
        }

        ul.lst-kix_cvb2fa7g0inw-2 {
            list-style-type: none
        }

        .lst-kix_cvb2fa7g0inw-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_wh3bw2uekp0m-3>li:before {
            content: "\0025cf  "
        }

        .lst-kix_7taahgizfshk-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_q9ohf0mp8zzu-8 {
            list-style-type: none
        }

        .lst-kix_eiazrjwerwj4-2>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-2, lower-roman) ". "
        }

        ul.lst-kix_q9ohf0mp8zzu-7 {
            list-style-type: none
        }

        .lst-kix_8mezmq2lvg5s-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_2l2t7l68oh17-7>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_q9ohf0mp8zzu-6 {
            list-style-type: none
        }

        .lst-kix_wh3bw2uekp0m-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_7taahgizfshk-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_1ahkn1ua0ys7-8>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_jkaumvtzuajs-7 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-6 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-5 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-4 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-3 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-2 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-1 {
            list-style-type: none
        }

        ul.lst-kix_jkaumvtzuajs-0 {
            list-style-type: none
        }

        ul.lst-kix_q9ohf0mp8zzu-1 {
            list-style-type: none
        }

        ul.lst-kix_q9ohf0mp8zzu-0 {
            list-style-type: none
        }

        .lst-kix_eiazrjwerwj4-6>li:before {
            content: ""counter(lst-ctn-kix_eiazrjwerwj4-6, decimal) ". "
        }

        .lst-kix_bxk20eq5dbuj-6>li:before {
            content: "\0025cf  "
        }

        .lst-kix_8mezmq2lvg5s-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_q9ohf0mp8zzu-5 {
            list-style-type: none
        }

        ul.lst-kix_q9ohf0mp8zzu-4 {
            list-style-type: none
        }

        ul.lst-kix_q9ohf0mp8zzu-3 {
            list-style-type: none
        }

        ul.lst-kix_q9ohf0mp8zzu-2 {
            list-style-type: none
        }

        .lst-kix_q9ohf0mp8zzu-5>li:before {
            content: "\0025a0  "
        }

        .lst-kix_q9ohf0mp8zzu-1>li:before {
            content: "\0025cb  "
        }

        .lst-kix_vumiwzrrem62-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_g9u3sdyi5sn-3 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-5 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-2 {
            list-style-type: none
        }

        .lst-kix_vumiwzrrem62-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_317oy7bf1j1g-4 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-5 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-3 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-4 {
            list-style-type: none
        }

        .lst-kix_qgdrxcrwev1n-7>li:before {
            content: "\0025cb  "
        }

        .lst-kix_7t2fnfiik8pc-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_317oy7bf1j1g-2 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-4.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-4 0
        }

        ul.lst-kix_g9u3sdyi5sn-7 {
            list-style-type: none
        }

        .lst-kix_6qd5emnhfh12-0>li:before {
            content: "\0025cf  "
        }

        .lst-kix_kw79sbq53lji-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_g9u3sdyi5sn-6 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-8 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-1 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-7 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-8 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-0 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-6 {
            list-style-type: none
        }

        .lst-kix_7t2fnfiik8pc-2>li:before {
            content: "\0025a0  "
        }

        .lst-kix_oat2jjdhxffc-6>li:before {
            content: "\0025cf  "
        }

        ul.lst-kix_317oy7bf1j1g-1 {
            list-style-type: none
        }

        ul.lst-kix_317oy7bf1j1g-0 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-1 {
            list-style-type: none
        }

        ul.lst-kix_g9u3sdyi5sn-0 {
            list-style-type: none
        }

        .lst-kix_a5e5o2yyrzau-1>li {
            counter-increment: lst-ctn-kix_a5e5o2yyrzau-1
        }

        .lst-kix_6qd5emnhfh12-4>li:before {
            content: "\0025cb  "
        }

        .lst-kix_6qd5emnhfh12-8>li:before {
            content: "\0025a0  "
        }

        .lst-kix_kw79sbq53lji-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_oat2jjdhxffc-3 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-2 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-5 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-4 {
            list-style-type: none
        }

        .lst-kix_oat2jjdhxffc-2>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_oat2jjdhxffc-7 {
            list-style-type: none
        }

        ul.lst-kix_oat2jjdhxffc-6 {
            list-style-type: none
        }

        li.li-bullet-0:before {
            margin-left: -18pt;
            white-space: nowrap;
            display: inline-block;
            min-width: 18pt
        }

        ul.lst-kix_oat2jjdhxffc-8 {
            list-style-type: none
        }

        ol.lst-kix_eiazrjwerwj4-5.start {
            counter-reset: lst-ctn-kix_eiazrjwerwj4-5 0
        }

        ul.lst-kix_fk7u4doo34jn-0 {
            list-style-type: none
        }

        .lst-kix_g9u3sdyi5sn-5>li:before {
            content: "\0025a0  "
        }

        ul.lst-kix_fk7u4doo34jn-5 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-6 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-7 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-8 {
            list-style-type: none
        }

        .lst-kix_g9u3sdyi5sn-1>li:before {
            content: "\0025cb  "
        }

        ul.lst-kix_fk7u4doo34jn-1 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-2 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-3 {
            list-style-type: none
        }

        ul.lst-kix_fk7u4doo34jn-4 {
            list-style-type: none
        }

        ol {
            margin: 0;
            padding: 0
        }

        table td,
        table th {
            padding: 0
        }

        .c21 {
            -webkit-text-decoration-skip: none;
            color: #000000;
            font-weight: 400;
            text-decoration: line-through;
            vertical-align: baseline;
            text-decoration-skip-ink: none;
            font-size: 11pt;
            font-family: "Arial";
            font-style: normal
        }

        .c12 {
            padding-top: 16pt;
            padding-bottom: 4pt;
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .c27 {
            background-color: #ffffff;
            padding-top: 14pt;
            padding-bottom: -6pt;
            line-height: 1.3043478260869565;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .c9 {
            color: #666666;
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 12pt;
            font-family: "Arial";
            font-style: normal
        }

        .c15 {
            color: #000000;
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 16pt;
            font-family: "Arial";
            font-style: normal
        }

        .c11 {
            color: #434343;
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 14pt;
            font-family: "Arial";
            font-style: normal
        }

        .c14 {
            padding-top: 14pt;
            padding-bottom: 4pt;
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .c8 {
            padding-top: 18pt;
            padding-bottom: 6pt;
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .c0 {
            color: #000000;
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 11pt;
            font-family: "Arial";
            font-style: normal
        }

        .c25 {
            color: #000000;
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 20pt;
            font-family: "Arial";
            font-style: normal
        }

        .c18 {
            color: #000000;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 11pt;
            font-family: "Arial";
            font-style: normal
        }

        .c3 {
            padding-top: 0pt;
            padding-bottom: 0pt;
            line-height: 1.15;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .c16 {
            padding-top: 0pt;
            padding-bottom: 0pt;
            line-height: 1.15;
            orphans: 2;
            widows: 2;
            text-align: justify
        }

        .c23 {
            font-weight: 400;
            text-decoration: none;
            vertical-align: baseline;
            font-size: 11pt;
            font-family: "Arial";
            font-style: normal
        }

        .c17 {
            padding-top: 0pt;
            padding-bottom: 0pt;
            line-height: 1.15;
            orphans: 2;
            widows: 2;
            text-align: center
        }

        .c7 {
            text-decoration-skip-ink: none;
            -webkit-text-decoration-skip: none;
            color: #1155cc;
            text-decoration: underline
        }

        .c13 {
            text-decoration-skip-ink: none;
            -webkit-text-decoration-skip: none;
            text-decoration: underline
        }

        .c26 {
            background-color: #ffffff;
            max-width: 468pt;
            padding: 72pt 72pt 72pt 72pt
        }

        .c4 {
            margin-left: 36pt;
            padding-left: 0pt
        }

        .c2 {
            color: inherit;
            text-decoration: inherit
        }

        .c1 {
            padding: 0;
            margin: 0
        }

        .c10 {
            color: #292929
        }

        .c5 {
            font-weight: 700
        }

        .c22 {
            margin-left: 36pt
        }

        .c24 {
            font-size: 24pt
        }

        .c6 {
            height: 11pt
        }

        .c19 {
            height: 12pt
        }

        .c20 {
            color: #ff0000
        }

        .title {
            padding-top: 0pt;
            color: #000000;
            font-size: 26pt;
            padding-bottom: 3pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        .subtitle {
            padding-top: 0pt;
            color: #666666;
            font-size: 15pt;
            padding-bottom: 16pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        li {
            color: #000000;
            font-size: 11pt;
            font-family: "Arial"
        }

        p {
            margin: 0;
            color: #000000;
            font-size: 11pt;
            font-family: "Arial"
        }

        h1 {
            padding-top: 20pt;
            color: #000000;
            font-size: 20pt;
            padding-bottom: 6pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        h2 {
            padding-top: 18pt;
            color: #000000;
            font-size: 16pt;
            padding-bottom: 6pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        h3 {
            padding-top: 16pt;
            color: #434343;
            font-size: 14pt;
            padding-bottom: 4pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        h4 {
            padding-top: 14pt;
            color: #666666;
            font-size: 12pt;
            padding-bottom: 4pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        h5 {
            padding-top: 12pt;
            color: #666666;
            font-size: 11pt;
            padding-bottom: 4pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            orphans: 2;
            widows: 2;
            text-align: left
        }

        h6 {
            padding-top: 12pt;
            color: #666666;
            font-size: 11pt;
            padding-bottom: 4pt;
            font-family: "Arial";
            line-height: 1.15;
            page-break-after: avoid;
            font-style: italic;
            orphans: 2;
            widows: 2;
            text-align: left
        }
    </style>
</head>

<body class="c26">
    <h1 class="c27" id="h.dv4387c2mosm"><span class="c10 c5 c24">LAION-5B: </span><span class="c5 c24">A new era of open
            large-scale multi-modal datasets</span></h1>
    <p class="c3"><span>We present a </span><span class="c0">dataset of 5,85 billion CLIP-filtered image-text pairs, 14x
            bigger than LAION-400M, previously the biggest openly accessible image-text dataset in the world.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>Authors </span><span class="c5">Christoph Schuhmann</span><span>, </span><span
            class="c5">Richard Vencu, Romain Beaumont,</span><span class="c5 c20">&nbsp;</span><span class="c5">Theo
            Coombes, Cade Gordon, Aarush Katta, Robert Kaczmarczyk, Jenia Jitsev</span></p>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 624.00px; height: 378.67px;"><img
                alt="" src="images/image7.jpg"
                style="width: 624.00px; height: 378.67px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span class="c0">Large image-text models like ALIGN, BASIC, Turing Bletchly, FLORENCE &amp; GLIDE have
            shown better and better performance compared to previous flagship models like CLIP and DALL-E. </span></p>
    <p class="c3"><span class="c0">Most of them had been trained on billions of image-text pairs and unfortunately, no
            datasets of this size had been openly available until now.</span></p>
    <p class="c3"><span class="c0">To address this problem we present LAION 5B, a large-scale dataset for research
            purposes consisting of 5,85B CLIP-filtered image-text pairs. 2,3B contain English language, 2,2B samples
            from 100+ other languages and 1B samples have texts that do not allow a certain language assignment (e.g.
            names ).</span></p>
    <p class="c3"><span>Additionally, we provide several nearest neighbor indices, an improved web interface for
            exploration &amp; subset creation as well as detection scores for watermark and NSFW. We also announce a
            full reproduction of a clip training trained on LAION-400M at </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/mlfoundations/open_clip&amp;sa=D&amp;source=editors&amp;ust=1648758687772610&amp;usg=AOvVaw27Mtu0oza9wac_owhNsZXc">open_clip</a></span><span
            class="c0">.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>Explore the dataset at the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://rom1504.github.io/clip-retrieval/&amp;sa=D&amp;source=editors&amp;ust=1648758687773016&amp;usg=AOvVaw2_2QYBH0gSvWWM7vSG4RFr">search
                demo</a></span><span>. See also the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://laion.ai/laion-5b-open-dataset&amp;sa=D&amp;source=editors&amp;ust=1648758687773167&amp;usg=AOvVaw17AXcwvMcWyx___PNF7LzM">same
                post on laion website</a></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>We thank our sponsors huggingface, doodlebot and stability for providing us with computing
            resources to produce this dataset!</span></p>
    <h2 class="c8" id="h.o2doa4ymw3ni"><span class="c15">Disclaimer on dataset purpose and content warning</span></h2>
    <p class="c16"><span>The motivation behind dataset creation is to democratize research and experimentation around
            large-scale multi-modal model training and handling of uncurated, large-scale datasets crawled from
            publically available internet. Our recommendation is therefore to use the dataset for </span><span
            class="c5">research purposes</span><span class="c0">. </span></p>
    <p class="c16 c6"><span class="c0"></span></p>
    <p class="c16"><span class="c0">Be aware that this large-scale dataset is uncurated. Keep in mind that the uncurated
            nature of the dataset means that collected links may lead to strongly discomforting and disturbing content
            for a human viewer. Therefore, please use the demo links with caution and at your own risk. It is possible
            to extract a &ldquo;safe&rdquo; subset by filtering out samples based on the safety tags (using a customized
            trained NSFW classifier that we built). While this strongly reduces the chance for encountering potentially
            harmful content when viewing, we cannot entirely exclude the possibility for harmful content being still
            present in safe mode, so that the warning holds also there.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c16"><span>We think that providing the dataset openly to broad research and other interested communities
            will allow for transparent investigation of benefits that come along with training large-scale models as
            well as pitfalls and dangers that may stay unreported or unnoticed when working with closed large datasets
            that remain restricted to a small community. Providing our dataset openly, we however </span><span
            class="c5">do not</span><span>&nbsp;recommend using it for creating ready-to-go industrial products, as the
            basic research about general properties and safety of such large-scale models, which we would like to
            encourage with this release, is still in progress.</span></p>
    <h2 class="c8" id="h.un9xxtc17gjy"><span class="c15">Introduction</span></h2>
    <p class="c3"><span>Since the release of CLIP &amp; DALL-E in January 2021, several similar large multi-modal
            language-vision models have been trained by large groups. Models like FLORENCE, Turing Bletchley, ALIGN
            &amp; BASIC demonstrated very strong transfer capab</span><span class="c10">ilit</span><span class="c10">ies
            on</span><span class="c10">&nbsp;nov</span><span class="c0">el datasets in absence of per-sample labels,
            which also steadily improved when growing training data amount, following scaling laws observed in previous
            research work.</span></p>
    <p class="c3"><span class="c0">These models require billions of image-text pairs to achieve competitive performances
            and unfortunately, no billion-scale image-text pair dataset had been openly available up until now.</span>
    </p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">To address this problem we release LAION 5B, a CLIP-filtered dataset of 5,85 billion
            high-quality image-text pairs, their CLIP ViT-L/14 embeddings, kNN-indices, a web interface for exploration
            &amp; subset-creation and NSFW- and watermark-detection scores and tools.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We describe the procedure to create the dataset and demonstrate successful training
            of DALL-E architecture. Having sufficiently large scales, the dataset opens venues for research on
            multi-modal language-vision models to a broad community.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h2 class="c8" id="h.nfkpzt5c4bmf"><span class="c15">Download the data</span></h2>
    <p class="c3"><span class="c0">We release the following packages under the LAION-5B project: </span></p>
    <ul class="c1 lst-kix_wh3bw2uekp0m-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en&amp;sa=D&amp;source=editors&amp;ust=1648758687774769&amp;usg=AOvVaw3dqpNc_7HJIqs-ly2ywswe">laion2B-en</a></span><span
                class="c0">&nbsp;2.32 billion of these contain texts in the English language</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi&amp;sa=D&amp;source=editors&amp;ust=1648758687775016&amp;usg=AOvVaw1xvDBQRX1r11l9hNzU54nS">laion2B-multi</a></span><span
                class="c0">&nbsp;2.26 billion contain texts from 100+ other languages</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang&amp;sa=D&amp;source=editors&amp;ust=1648758687775275&amp;usg=AOvVaw1oWNR75W_Vu_5OQS5NeoHk">laion1B-nolang</a></span><span
                class="c0">&nbsp;1.27 billion have texts where a particular language couldn&#39;t be clearly
                detected.</span></li>
    </ul>
    <p class="c3"><span>The data can comfortably be downloaded with </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/img2dataset&amp;sa=D&amp;source=editors&amp;ust=1648758687775524&amp;usg=AOvVaw0yFa78TVIkkfn9McHjL9eR">img2dataset</a></span><span
            class="c0">&nbsp;</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>For training usage, we recommend reading the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/laion-prepro/blob/main/laion5B/usage_guide/preparing_data_for_training.md&amp;sa=D&amp;source=editors&amp;ust=1648758687775853&amp;usg=AOvVaw1HuqZfRUTx9E5cbUgCjcHl">usage
                guide for training</a></span><span class="c0">&nbsp;</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">In particular, we release this data:</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_g9u3sdyi5sn-0 start">
        <li class="c3 c4 li-bullet-0"><span>5.85 billion pairs of image URLs and the corresponding metadata at
            </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en&amp;sa=D&amp;source=editors&amp;ust=1648758687776382&amp;usg=AOvVaw2kOTaWVs2HFcsUiiaSFoVM">laion2B-en</a></span><span>&nbsp;</span><span
                class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi&amp;sa=D&amp;source=editors&amp;ust=1648758687776534&amp;usg=AOvVaw35uCWyF5UjhGZDz0CEiNIF">laion2B-multi</a></span><span>&nbsp;
            </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang&amp;sa=D&amp;source=editors&amp;ust=1648758687776686&amp;usg=AOvVaw2U7IwHAE_0DOx2pG8H5fPH">laion1B-nolang</a></span><span>&nbsp;</span>
        </li>
        <li class="c3 c4 li-bullet-0"><span>A </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion5B-index&amp;sa=D&amp;source=editors&amp;ust=1648758687776923&amp;usg=AOvVaw2BetxhVW6X2vUPDnOVc4nS">knn
                    index</a></span><span class="c0">&nbsp;that enables quick search in the dataset</span></li>
        <li class="c3 c4 li-bullet-0"><span>Web demo of image-text search on LAION-5B </span><span class="c7"><a
                    class="c2"
                    href="https://www.google.com/url?q=https://rom1504.github.io/clip-retrieval/&amp;sa=D&amp;source=editors&amp;ust=1648758687777164&amp;usg=AOvVaw0IT6nEzBZKVE2rpQ-nAwGe">clip-retrieval</a></span>
        </li>
        <li class="c3 c4 li-bullet-0"><span>Safety tags at </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687777398&amp;usg=AOvVaw1Ne8JPM2GopP-Z-SHOMtz6">laion2B-en-safety</a></span><span>&nbsp;</span><span
                class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687777599&amp;usg=AOvVaw0-ZXddpKxHecyXdiiOecmL">laion2B-multi-safety</a></span><span>&nbsp;</span><span
                class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687777767&amp;usg=AOvVaw2C0An63p6jEXoO6ZFUVimE">laion1B-nolang-safety</a></span>
        </li>
        <li class="c3 c4 li-bullet-0"><span>Watermark tags at </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en-watermark&amp;sa=D&amp;source=editors&amp;ust=1648758687778000&amp;usg=AOvVaw2CefYTmMTa2OO323Fo8J46">laion2B-en-watermark</a></span><span>&nbsp;
            </span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi-watermark&amp;sa=D&amp;source=editors&amp;ust=1648758687778161&amp;usg=AOvVaw1GLuVCHZaXE7plzS1NmIsP">laion2B-multi-watermark</a></span><span>&nbsp;
                &nbsp;</span><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang-watermark&amp;sa=D&amp;source=editors&amp;ust=1648758687778316&amp;usg=AOvVaw36UNG3mB3h3tJGT5fu_TxU">laion1B-nolang-watermark</a></span>
        </li>
    </ul>
    <p class="c3 c22 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">The metadata files are parquet files that contain the following attributes: URL,
            TEXT, the cosine similarity score between the text and image embedding and height and width of the
            image.</span></p>
    <p class="c3"><span>Watermark and safety tags can be joined with the metadata prior to downloading by using
        </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/laion-prepro/blob/main/laion5B/safety/join.py&amp;sa=D&amp;source=editors&amp;ust=1648758687778667&amp;usg=AOvVaw3DMkTBDGXf_LTbscoM-_mu">this
                script</a></span><span class="c0">. Once that is done, they can easily be filtered upon with a
            probability threshold at your choice (we recommend 0.5 for safety and 0.8 for watermark).</span></p>
    <p class="c3"><span>You can also find the prejoined files at </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en-joined&amp;sa=D&amp;source=editors&amp;ust=1648758687778890&amp;usg=AOvVaw0SglRdxUdwqHqrRJIQZlJD">laion2B-en-joined</a></span><span>&nbsp;</span><span
            class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi-joined&amp;sa=D&amp;source=editors&amp;ust=1648758687779040&amp;usg=AOvVaw1zqMvk0PxBQgSPKTaVKSEO">laion2B-multi-joined</a></span><span>&nbsp;
        </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang-joined&amp;sa=D&amp;source=editors&amp;ust=1648758687779231&amp;usg=AOvVaw2clKqrzIJdp3zUnxAGFIDA">laion1B-nolang-joined</a></span><span>&nbsp;</span>
    </p>
    <h2 class="c8" id="h.yk6nlyomahzc"><span class="c15">License</span></h2>
    <p class="c3"><span>We distribute the metadata dataset (the parquet files) under the </span><span class="c7"><a
                class="c2"
                href="https://www.google.com/url?q=https://creativecommons.org/licenses/by/4.0/&amp;sa=D&amp;source=editors&amp;ust=1648758687779578&amp;usg=AOvVaw0FtSnKhEEA02f8ZJqkmakK">Creative
                Common CC-BY 4.0</a></span><span class="c0">&nbsp;license, which poses no particular restriction. The
            images are under their copyright.</span></p>
    <h2 class="c8" id="h.sn5ctv72o7fo"><span class="c15">Dataset Statistics</span></h2>
    <p class="c3"><span class="c0">We computed some statistics on the datasets to let people understand better:</span>
    </p>
    <h3 class="c12" id="h.kgob18dhp8cg"><span class="c11">Laion2B-en</span></h3>
    <p class="c3"><span class="c0">Total: 2.3B samples</span></p>
    <p class="c3"><span class="c0">Number with height and width bigger than</span></p>
    <ul class="c1 lst-kix_joualahdnmon-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">256 -&gt; 1324M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">512 -&gt; 488M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">1024 -&gt; &nbsp;76M</span></li>
    </ul>
    <p class="c3"><span class="c0">Number of unsafe samples with a probability threshold of 0.5: 0.029</span></p>
    <p class="c3"><span>Number of watermarked samples with a probability threshold of 0.8: 0.061</span></p>
    <h3 class="c12" id="h.g97ckz4pv5x8"><span class="c11">Laion2B-multi</span></h3>
    <p class="c3"><span class="c0">Total: 2.2B samples</span></p>
    <p class="c3"><span class="c0">Number with height and width bigger than</span></p>
    <ul class="c1 lst-kix_1ahkn1ua0ys7-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">256 -&gt; 1299M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">512 -&gt; 480M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">1024 -&gt; 57M</span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Top 10 languages:</span></p>
    <p class="c3"><span
            class="c0">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;LANGUAGE&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;count&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;proportion</span>
    </p>
    <p class="c3"><span
            class="c0">0&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ru&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 241M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.106</span></p>
    <p class="c3"><span
            class="c0">1&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;fr&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;168M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.074</span></p>
    <p class="c3"><span
            class="c0">2&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;de&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;150M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.066</span></p>
    <p class="c3"><span
            class="c0">3&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;es&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;149M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.066</span></p>
    <p class="c3"><span
            class="c0">4&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;zh&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; 143M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.063</span></p>
    <p class="c3"><span
            class="c0">5&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ja&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;131M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.057</span></p>
    <p class="c3"><span
            class="c0">6&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;it&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; 95M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.042</span></p>
    <p class="c3"><span
            class="c0">7&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;pt&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;88M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.038</span></p>
    <p class="c3"><span
            class="c0">8&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;nl&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; 66M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.029</span></p>
    <p class="c3"><span
            class="c0">9&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;pl&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp;62M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.027</span></p>
    <p class="c3"><span
            class="c0">10&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;no&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
            &nbsp; &nbsp; &nbsp; &nbsp; 49M&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0.021</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Number of unsafe samples with a probability threshold of 0.5: 0.033</span></p>
    <p class="c3"><span class="c0">Number of watermarked samples with a probability threshold of 0.8: 0.056</span></p>
    <h3 class="c12" id="h.7tg7pgyua06l"><span class="c11">Laion1B-nolang</span></h3>
    <p class="c3"><span class="c0">Total: 1.2B samples</span></p>
    <p class="c3"><span class="c0">Number with height and width bigger than</span></p>
    <ul class="c1 lst-kix_1ahkn1ua0ys7-0">
        <li class="c3 c4 li-bullet-0"><span class="c0">256 -&gt; 1324M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">512 -&gt; 488M</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">1024 -&gt; 76M</span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Number of unsafe samples with a probability threshold of 0.5: 0.03</span></p>
    <p class="c3"><span class="c0">Number of watermarked samples with a probability threshold of 0.8: 0.04</span></p>
    <h2 class="c8" id="h.2lqvourwymb7"><span class="c15">Acquisition pipeline</span></h2>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 572.00px; height: 531.00px;"><img
                alt="" src="images/image3.jpg"
                style="width: 572.00px; height: 531.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span class="c0">The acquisition pipeline follows the flowchart above and can be split into three
            major components: </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_7t2fnfiik8pc-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">Distributed processing of petabyte-scale Common Crawl dataset,
                which produces a collection of matching URLs and captions (preprocessing phase)</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">The distributed download of images based on shuffled data to pick
                a correct distribution of URLs, to avoid too heavy request loads on single websites</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">Few GPU node post-processing of the data, which is much lighter
                and can be run in a few days, producing the final dataset.</span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.ip97p46rfwvt"><span class="c11">Distributed processing of Common Crawl</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>To create image-text pairs, we parse through WAT files from Common Crawl and parse out all HTML
            IMG tags containing an alt-text attribute. At the same time, we perform a language detection on text with
            three possible outputs: English language with confidence, another language with confidence, no language
            which contains </span><span>&ldquo;no detection&rdquo; and &ldquo;detection under the confidence
            threshold&rdquo;</span><span>. The </span><span>&ldquo;no language&rdquo; set often contains short texts,
            mostly with </span><span class="c0">names of people and places.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">All extracted information by the preprocessing workers were packed and sent to the
            Postgresql node for storage using the COPY command. The Postgresql server was maintained to keep about 500M
            records at all times by means of balancing the ingress and egress of data from the database.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.gy1y76r9504w"><span class="c11">Distributed downloading of the images</span></h3>
    <p class="c3"><span class="c0">We download the raw images from the parsed URLs with asynchronous requests using Trio
            and Asks libraries in order to maximize all resources usage: vCPUs, RAM and bandwidth. We found that a
            single node in the cloud with 1-2 vCPUs, 0.5-1GB RAM and 5-10Mbps download bandwidth is inexpensive enough
            to allow downloading on a limited budget. Such a unit can process 10000 links in about 10-15 minutes. Each
            batch consisted of 10000 links taken from the Postgresql server by using the TABLESAMPLE technique, ensuring
            that the distribution among the 10000 links was following the distribution of the existing 500M records
            available on the database. We found that the distribution is still good when in the database are still above
            20M records to be processed given that we had some 300 downloading workers at any time.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">The above techniques allowed both maximizing downloading speed and minimizing IP
            reputation damages.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.mqqotorkbsu0"><span class="c11">CLIP inference at the post-processing stage</span></h3>
    <p class="c3"><span>The data pipeline continued with GPU nodes doing inference on the collected image-text pairs,
            and calculating the similarity of the embeddings for the image and the text. After the similarity score was
            established we removed the pairs under the threshold we decided to use, i.e 0.28 for the English dataset
        </span><span>( with CLIP ViT B/32 )</span><span>&nbsp;and 0.26 for the rest</span><span>&nbsp;(with
            mCLIP)</span><span class="c0">.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">As an estimation, we removed about 90% of the samples, trimming the 50+ billion of
            candidates to just below 6 billion.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.t3ojkgetnumb"><span class="c11">Filtering out unsuitable image-text pairs</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">After downloading the WAT files from Common Crawl, we apply the following filtering
            conditions:</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_317oy7bf1j1g-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">All samples with less than 5 characters alt-text length or less
                than 5 KB image size are dropped.</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">All images with the too big resolution, potentially DOS bombs,
                were dropped before attempting to process them.</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">Duplicate removal is performed with a bloom filter based on URL.
                Future runs would include more variate deduplication rules, such as URL + language for the multilanguage
                dataset.</span></li>
        <li class="c3 c4 li-bullet-0"><span>We use CLIP respectively MCLIP to compute embeddings of the image and
                alt-text. Then we compute the cosine similarity of both embeddings and drop all samples with cosine
                similarity below </span><span>0.28</span><span class="c0">&nbsp;for the English language ( with CLIP
                B/32) and 0.26 for the multilingual dataset (MCLIP). These thresholds were selected based on human
                inspection of the test results. </span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">We use the CLIP embeddings of images and texts to filter out to
                the possible extent the illegal content. </span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h2 class="c8" id="h.bvwysbb2s9w"><span class="c15">Dataset preparation pipeline</span></h2>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>After processing and filtering common crawl, </span><span class="c10">5,85B</span><span>&nbsp;of
            URL/text samples</span><span>&nbsp;remained</span><span class="c0">.</span></p>
    <p class="c3"><span class="c0">We did additional steps after that in order to prepare the dataset.</span></p>
    <p class="c3"><span>See </span><span>this</span><span>&nbsp;</span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://rom1504.medium.com/semantic-search-with-embeddings-index-anything-8fb18556443c&amp;sa=D&amp;source=editors&amp;ust=1648758687785006&amp;usg=AOvVaw0DdE-W891eCM8j-ttMjhoa">semantic
                search blogpost</a></span><span>&nbsp;and the readme of </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/clip-retrieval&amp;sa=D&amp;source=editors&amp;ust=1648758687785208&amp;usg=AOvVaw18OznpciiN47BHQfxWNTiL">clip-retrieval</a></span><span>&nbsp;for
            additional details about this process.</span><span>&nbsp;See also </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://medium.com/@rom1504/semantic-search-at-billions-scale-95f21695689a&amp;sa=D&amp;source=editors&amp;ust=1648758687785386&amp;usg=AOvVaw0QpQIBeDLbYAuD9Hv7iXyr">semantic
                search at billions scale</a></span><span>&nbsp;for more technical details of the process that was done
            for laion5B.</span><span class="c0">&nbsp;</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ol class="c1 lst-kix_eiazrjwerwj4-0 start" start="1">
        <li class="c3 c4 li-bullet-0"><span class="c0">&nbsp;Downloading the data as webdataset with distributed
                img2dataset</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">&nbsp;Computing Vit-L/14 embeddings with distributed
                clip-inference</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">&nbsp;Computing a KNN index from these embeddings using
                autofaiss</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">&nbsp;Computing additional tags (NSFW and watermark) using clip
                embeddings</span></li>
    </ol>
    <h3 class="c12" id="h.f8l0nmzuh0n"><span class="c11">Distributed img2dataset</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>We developed the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/img2dataset&amp;sa=D&amp;source=editors&amp;ust=1648758687786165&amp;usg=AOvVaw33Oebi99PQWfS4TkFLeut-">img2dataset</a></span><span
            class="c0">&nbsp;library to comfortably download from a given set of URLs, resize and store the images and
            captions in the webdataset format. This allows downloading 100 million images from our list of URLs in 20
            hours with a single node (1Gbps connection speed, 32GB of RAM, an i7 CPU with 16 cores), which allows anyone
            to obtain the whole dataset or a smaller subset.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>For </span><span>LAION-</span><span>5B we introduced a </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/img2dataset/blob/main/dataset_examples/laion5B.md&amp;sa=D&amp;source=editors&amp;ust=1648758687786547&amp;usg=AOvVaw3QQs9-dymOA7h3GKn13Gw2">distributed
                mode</a></span><span>&nbsp;for this tool, allowing to downloading the 5</span><span>,85</span><span
            class="c0">B samples in a week using 10 nodes.</span></p>
    <h3 class="c12" id="h.p2x2civuvg3w"><span class="c11">Distributed clip inference</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>From these images, the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/clip-retrieval%257D%257Bhttps://github.com/rom1504/clip-retrieval&amp;sa=D&amp;source=editors&amp;ust=1648758687787043&amp;usg=AOvVaw2UF7tn8LQW7zSf4chNwIM1">clip
                retrieval</a></span><span>&nbsp;inference tool was used to compute ViT-L/14 embeddings, allowing for a
            better analysis capacity of the data. In particular, a </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/clip-retrieval/blob/main/docs/distributed_clip_inference.md&amp;sa=D&amp;source=editors&amp;ust=1648758687787275&amp;usg=AOvVaw3jZ2h3jVOK5ywf6LHHBOED">distributed
                mode</a></span><span class="c0">&nbsp;made it possible to compute these embeddings in a week using 32
            A100: this larger clip model can only be computed at a speed of 312 sample/s per GPU, compared to 1800
            sample/s for ViT-B/32.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>The resulting embeddings are available for everyone to use </span><span>e.g.</span><span
            class="c0">&nbsp;for clustering, indexing, linear inference.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.urs2txvu7z3h"><span class="c11">Distributed indexing</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>We then used </span><span>these</span><span>&nbsp;9 TB of image embeddings to build a large
            PQ128 knn index using the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/criteo/autofaiss&amp;sa=D&amp;source=editors&amp;ust=1648758687787988&amp;usg=AOvVaw2Fup4GzGby5De0RXVAb8V4">autofaiss</a></span><span>&nbsp;tool.
            To make this run faster, a </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/criteo/autofaiss/blob/master/docs/distributed/distributed_autofaiss.md&amp;sa=D&amp;source=editors&amp;ust=1648758687788217&amp;usg=AOvVaw3_tbwgawOy4TAcaHquTYB8">distributed
                mode</a></span><span class="c0">&nbsp;is available.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.cjkq5ch882vs"><span class="c11">Integration in the search UI</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>In order to demonstrate the value of this data, we integrated this index into the </span><span
            class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://knn5.laion.ai&amp;sa=D&amp;source=editors&amp;ust=1648758687788588&amp;usg=AOvVaw1Q17Ch3r_3DhNL_JuyuWqW">knn
                search UI</a></span><span>. It is powered by the code called </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/clip-retrieval&amp;sa=D&amp;source=editors&amp;ust=1648758687788764&amp;usg=AOvVaw0EI8fCnFrkzbLlh3_PZ_2S">clip
                back</a></span><span>.</span></p>
    <p class="c3"><span class="c0">The knn index is 800GB and the metadata (URL and captions) as well, so memory mapping
            is used for both in order to use no ram, only an SSD drive of that capacity is required.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h3 class="c12" id="h.5m14yzj22p9m"><span class="c11">Watermark and safety inference</span></h3>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We wanted to give users the ability to remove unsafe examples, and watermarked
            examples. To do that we collected training and test sets. The training set was augmented with examples
            retrieved from the knn index, while the test set samples were selected to represent well the dataset
            distribution, but were all manually annotated.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>The inference is done using the </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/rom1504/embedding-reader&amp;sa=D&amp;source=editors&amp;ust=1648758687789432&amp;usg=AOvVaw2iWPsg50GxIWbO0fQdGdiQ">embedding-reader</a></span><span>&nbsp;module
            for NSFW and </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/Zasder3/LAION-5B-WatermarkDetection&amp;sa=D&amp;source=editors&amp;ust=1648758687789606&amp;usg=AOvVaw3XpoySzE4BrbfNdB6eHwVz">LAION-5B-WatermarkDetection</a></span><span
            class="c0">&nbsp;for watermarks</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">These tags were also integrated into the UI, allowing everyone to observe that the
            safety tags indeed filter out almost all the unsafe results, and giving confidence that training a
            generative model on this data will not result in unexpectedly unsafe images.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.nycf7y1gt8ak"><span class="c9">Watermark</span></h4>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 414.14px; height: 259.50px;"><img
                alt="" src="images/image4.png"
                style="width: 414.14px; height: 259.50px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span class="c0">The training dataset is 90000 samples (45222 watermarks, 44778 clear).</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Watermarked images are a big problem when training generative models like DALL-E or
            GLIDE. To tackle this problem we trained a watermark detection model and used it to calculate confidence
            scores for every image in LAION-5B.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Therefore we created a training dataset consiting of 90.000 images with 50%
            watermarked and 50% clean images. </span></p>
    <p class="c3"><span class="c0">The majority of the watermarked images have been extracted from the LAION-400M kNN
            index through the use of several text prompts like &ldquo;clip art watermark&rdquo;, &ldquo;cat
            watermark&rdquo; or &ldquo;landscape watermark&rdquo;.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">The images in the cleaned category were composed of images from the Open Images
            dataset and images that contained texts, but no watermarks, like PPT slides and memes, also retrieved from
            the kNN indices of LAION-400M.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">While we tried to curate a test set to evaluate the quality of our watermark
            detection model, we realized that it is almost impossible to draw a clear line between what actually is a
            watermark and what is not. For example pictures with small transparent texts at the bottom had been
            considered by some people as watermarked, by others not. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">In the end we decided to choose a model based on our consensual judgment. It seems to
            be &ldquo;good&rdquo; at spotting obvious watermarks like those used on popular stock image sites. </span>
    </p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">The creation of high-quality, openly accessible watermark detection test sets with
            clear and plausible definitions of what should be considered a watermark and what not, remains a challenge
            for future projects.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Nevertheless we are convinced that removing images with a high confidence score for
            containing a watermark based on our model will significantly reduce the percentage of images that would be
            considered as obvious watermarks.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>The m</span><span class="c10">odel is </span><span>available at </span><span class="c7"><a
                class="c2"
                href="https://www.google.com/url?q=https://github.com/LAION-AI/watermark-detection&amp;sa=D&amp;source=editors&amp;ust=1648758687791279&amp;usg=AOvVaw1uLytFPpKj_1huVU4pRpbJ">https://github.com/LAION-AI/watermark-detection</a></span><span>&nbsp;and
        </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/LAION-AI/LAION-5B-WatermarkDetection/releases/tag/1.0&amp;sa=D&amp;source=editors&amp;ust=1648758687791463&amp;usg=AOvVaw1qT49gJf7eTzUnhsFQ-gKN">https://github.com/LAION-AI/LAION-5B-WatermarkDetection/releases/tag/1.0</a></span><span
            class="c0">&nbsp;</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.nmr5fq1ho424"><span class="c9">Safety</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>On a balanced manually annotated safety test set </span><span>with 3000 samples</span><span
            class="c0">:</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_q9ohf0mp8zzu-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">the accuracy of the B32 NSFW classifier is: 0.960</span></li>
        <li class="c3 c4 li-bullet-0"><span>the accuracy of the ViT L 14 </span><span>NSFW</span><span
                class="c20">&nbsp;</span><span class="c0">classifier is: 0.961</span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>The model, as well as the training code, </span><span>are </span><span>available at
            &nbsp;</span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/LAION-AI/CLIP-based-NSFW-Detector&amp;sa=D&amp;source=editors&amp;ust=1648758687792325&amp;usg=AOvVaw1xoI8D7ER4QEAc_Rf-7kFi">CLIP-based-NSFW-Detector</a></span>
    </p>
    <p class="c3"><span>The tags are available at </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-en-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687792549&amp;usg=AOvVaw379XEJRJAq2m3cmEEWycg1">laion2B-en-safety</a></span><span>&nbsp;</span><span
            class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion2B-multi-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687792698&amp;usg=AOvVaw2qRNz-AFXt_w9wG-k9_DYP">laion2B-multi-safety</a></span><span>&nbsp;</span><span
            class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://huggingface.co/datasets/laion/laion1B-nolang-safety&amp;sa=D&amp;source=editors&amp;ust=1648758687792898&amp;usg=AOvVaw0U_ybZ42TLaBOe3-N6nbmP">laion1B-nolang-safety</a></span>
    </p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>Demo at </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://rom1504.github.io/clip-retrieval/&amp;sa=D&amp;source=editors&amp;ust=1648758687793498&amp;usg=AOvVaw2tQXsgv8ZH9PEf0DCUjvVJ">clip-retrieval</a></span><span
            class="c0">&nbsp;(check/uncheck safe mode)</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h2 class="c8" id="h.t5ilga5mdr57"><span class="c15">Using laion datasets</span></h2>
    <p class="c3"><span>Laion5B and </span><span>LAION-400M</span><span class="c20">&nbsp;</span><span class="c0">can be
            used to train</span></p>
    <ul class="c1 lst-kix_6qd5emnhfh12-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c0">Generative models: training image/text generative models, e.g
                autoregressive models like DALL-E or diffusion models like GLIDE</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">Models with contrastive losses: self-supervised training on
                image/text pairs using contrastive losses, e.g CLIP</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">Classification models: e.g, performing zero-shot classification
                by extracting pseudo labels from queries on the dataset </span></li>
    </ul>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We present here a few examples of models that were trained on laion datasets with
            success.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.wp24g6ilr66e"><span class="c9">CLIP</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We, LAION, are currently working together with the Cross Sectional Team Deep Learning
            (CST-DL), Scalable Learning and Multi-Purpose AI Lab (SLAMPAI) at the J&uuml;lich Supercomputing Centre
            (JSC) and the Open CLIP team in the replication of OpenAI&#39;s CLIP results.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">At the time of writing, we just finished the training of a CLIP ViT-B/32 on
            LAION-400M that matches the performance of OpenAI&#39;s original ViT-B/32 CLIP.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c17"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 504.14px; height: 233.63px;"><img
                alt="" src="images/image5.jpg"
                style="width: 504.14px; height: 233.63px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c17"><span class="c0">( The results in the right column are from our model. - huge thanks to Cade Gordon
            &amp; Ross Wightman for performing the training run )</span></p>
    <p class="c17 c6"><span class="c0"></span></p>
    <p class="c17 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">The repository with the training code and the model checkpoints can be found
            here:</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c13"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/mlfoundations/open_clip&amp;sa=D&amp;source=editors&amp;ust=1648758687795515&amp;usg=AOvVaw2g7cSf8Bzzvl2HSDctKCc5">https://github.com/mlfoundations/open_clip</a></span>
    </p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>We gratefully acknowledge the Gauss Centre for Supercomputing e.V. (www.gauss-centre.eu) for
            funding this part of work by providing computing time through the John von Neumann Institute for Computing
            (NIC) on the GCS Supercomputer JUWELS Booster at J&uuml;lich Supercomputing Centre (JSC).</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.tv6rbzfx25w0"><span class="c9">BLIP inference tuning</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c13"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/salesforce/BLIP&amp;sa=D&amp;source=editors&amp;ust=1648758687796004&amp;usg=AOvVaw3Gy2aGhz1fuP5ISfpzLVhf">BLIP</a></span><span
            class="c0">&nbsp;is a model that was trained for both image-text matching and image captioning.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">It was trained on a 115M subset of LAION-400M.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">To improve the results of the generated captions we (LAION) performed over 100
            experiments to determine the hyperparameters that maximize the BLEU-4 score compared to MS COCO
            captions.</span></p>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <p class="c3"><span>Here you can see some of our</span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=http://captions.christoph-schuhmann.de/eval_b_auto/eval.html&amp;sa=D&amp;source=editors&amp;ust=1648758687796493&amp;usg=AOvVaw1QRTjyhGCKjKBw4xctn9CK">&nbsp;results</a></span><span
            class="c0">. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 465.45px; height: 426.18px;"><img
                alt="" src="images/image9.jpg"
                style="width: 465.45px; height: 426.18px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span class="c0">eval_best_auto0185: An orange cat is looking at its reflection in the mirror.</span>
    </p>
    <p class="c3 c6"><span class="c20 c23"></span></p>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 465.45px; height: 268.36px;"><img
                alt="" src="images/image8.jpg"
                style="width: 465.45px; height: 268.36px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span class="c0">eval_best_auto0190: A green highway sign with the words Queens Bronx.</span></p>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <p class="c3"><span class="c0">We found that we can significantly improve the quality of the captions by generating
            40 (or more) candidate captions for each image and then ranking them using OpenAI&rsquo;s CLIP ViT-L/14
            &amp; CLIP-Resnet50x64.</span></p>
    <p class="c3"><span class="c0">First we ranked all candidates with ViT-L/14 and then we ranked the top-5 results
            again using Resnet50x64.</span></p>
    <p class="c3"><span class="c0">Preliminary results of human evaluations indicate that:</span></p>
    <ol class="c1 lst-kix_mcx9mwl64ou5-0 start" start="1">
        <li class="c3 c4 li-bullet-0"><span class="c0">our evaluators gave the generated captions an average quality
                rating of 3,8 &nbsp;on a scale from 0 to 5, with a standard deviation of 0,9 ( in this paricular
                hyperparameter configuration n= 600)</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c0">our evaluators gave original human captions from MS COCO an
                average quality rating of 3,9 with a standard deviation of 0,8 ( n = 2100 )</span></li>
    </ol>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">&mdash;&gt; We hypothesize that the generated captions match (&amp; sometimes even
            surpass) the average quality of the human captions of MS COCO (which are sometimes also far from perfect) in
            most cases, but sometimes ( in less than &lt;10% ) contain obvious mistakes, that humans would not make,
            because deeper kind of world knowledge &amp; &bdquo;common sense&ldquo; would be necessary in those
            cases.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.pmr8jv7ys5o3"><span class="c9">GLIDE</span></h4>
    <p class="c3"><span>Clay Mullis (alias </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/afiaka87&amp;sa=D&amp;source=editors&amp;ust=1648758687797794&amp;usg=AOvVaw1NZ2sDiKvmDC0o7GADDFz4">afiaka87</a></span><span>)
            used LAON-2B to fine-tune the OpenAi </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/openai/glide-text2im&amp;sa=D&amp;source=editors&amp;ust=1648758687797936&amp;usg=AOvVaw2zM6EvKSyfXYRLJYKufQng">glide</a></span><span
            class="c0">&nbsp;model and managed to reintroduce human generations.</span></p>
    <p class="c3"><span class="c0">Samples</span></p>
    <ul class="c1 lst-kix_fk7u4doo34jn-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://replicate.com/afiaka87/laionide-v4&amp;sa=D&amp;source=editors&amp;ust=1648758687798181&amp;usg=AOvVaw3uIIiOjEryftPTWKjx0dQz">https://replicate.com/afiaka87/laionide-v4</a></span><span
                class="c0">&nbsp;</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://wandb.ai/afiaka87/glide_compare/reports/Finetuning-GLIDE-on-Laion5B--VmlldzoxNTg3MTkz&amp;sa=D&amp;source=editors&amp;ust=1648758687798421&amp;usg=AOvVaw0ONNoepvzScME9NG7IVDBB">https://wandb.ai/afiaka87/glide_compare/reports/Finetuning-GLIDE-on-Laion5B--VmlldzoxNTg3MTkz</a></span><span
                class="c0">&nbsp;</span></li>
    </ul>
    <ul class="c1 lst-kix_vumiwzrrem62-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://wandb.ai/afiaka87/laionide-v3-glide/reports/Laionide-Version-3-Benchmark--VmlldzoxNjE0MTE3&amp;sa=D&amp;source=editors&amp;ust=1648758687798661&amp;usg=AOvVaw2PQCBFE24ZyzT2Moe-dEaQ">https://wandb.ai/afiaka87/laionide-v3-glide/reports/Laionide-Version-3-Benchmark--VmlldzoxNjE0MTE3</a></span><span
                class="c0">&nbsp;</span></li>
    </ul>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 624.00px; height: 256.00px;"><img
                alt="" src="images/image1.png"
                style="width: 624.00px; height: 256.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 624.00px; height: 256.00px;"><img
                alt="" src="images/image10.png"
                style="width: 624.00px; height: 256.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c3"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 624.00px; height: 256.00px;"><img
                alt="" src="images/image2.png"
                style="width: 624.00px; height: 256.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <h4 class="c14 c19" id="h.m2mq3qnjhg0u"><span class="c9"></span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.ujq042cvjk5e"><span class="c9">Semantic search and subset extraction</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>The </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://rom1504.github.io/clip-retrieval/&amp;sa=D&amp;source=editors&amp;ust=1648758687799317&amp;usg=AOvVaw1aGGlzFho_Mff8qWggUpbc">clip-retrieval</a></span><span
            class="c0">&nbsp;interface allows a user to search images and texts based on a query image or text using the
            CLIP embeddings of the input and our precomputed kNN indices. It demonstrates the diversity of images and
            captions that can be found in LAION-5B as well as high semantic relevance shows the distribution of image
            sizes of LAION-5B. Given the abundance of high-resolution images, one can produce subsets of images for
            training various customized models, and also choose image resolution that is suitable for the purpose of
            particular training. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <h4 class="c14" id="h.tvtd551tw56e"><span class="c9">CLOOB</span></h4>
    <p class="c3"><span class="c0">Katherine Crowson and John David Pressman recently trained a CLOOB ViT-B/16, variant
            of CLIP, for 32 epochs on LAION-400M and got preliminary results, that come close to the performance of
            OpenAI&#39;s ViT-B/32, even though this was an early run with unoptimized hyperparameters.</span></p>
    <p class="c3"><span class="c0">The checkpoints can be found here: </span></p>
    <p class="c3"><span class="c13"><a class="c2"
                href="https://www.google.com/url?q=https://github.com/crowsonkb/cloob-training&amp;sa=D&amp;source=editors&amp;ust=1648758687799887&amp;usg=AOvVaw2-e4jQqW02gMKW-A1ksNBV">https://github.com/crowsonkb/cloob-training</a></span>
    </p>
    <p class="c17"><span
            style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 300.50px; height: 248.97px;"><img
                alt="" src="images/image6.png"
                style="width: 300.50px; height: 248.97px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);"
                title=""></span></p>
    <p class="c17"><span class="c0">zero-shot accuracies on Imagenet-1K</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We are in touch with Andreas F&uuml;rst, one of the original CLOOB authors, and
            learned from him that their team is currently (at the time of writing) training a CLOOB ViT-B/32 with
            LAION-400M with optimized hyperparameters and very promising results so far (53% zero-shot accuracy on
            Imagenet after 7 epochs).</span></p>
    <h4 class="c14" id="h.f6ybvk2zzawp"><span class="c9">Papers citing LAION 400M</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>After the release of </span><span>LAION-400M</span><span>, several papers used
        </span><span>LAION-400M</span><span class="c0">&nbsp;for image generation, text to image generation, image to
            text generation and text image matching:</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_v0ratkyi1f8x-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://arxiv.org/abs/2111.14822.pdf&amp;sa=D&amp;source=editors&amp;ust=1648758687800615&amp;usg=AOvVaw3pa-wTYzIkrbc6Stk-_CV3">Vector
                    Quantized Diffusion Model for Text-to-Image Synthesis</a></span><span>&nbsp;used
            </span><span>LAION-400M</span><span class="c0">&nbsp;to train VQ diffusion text to image generation
                models</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://arxiv.org/abs/2112.10752.pdf&amp;sa=D&amp;source=editors&amp;ust=1648758687800878&amp;usg=AOvVaw3ktH3Y2ASeRe4YjWJng5rI">High-Resolution
                    Image Synthesis with Latent Diffusion Models</a></span><span>&nbsp;used a</span><span>&nbsp;subset
                of </span><span>LAION-400M</span><span>&nbsp;to t</span><span class="c0">rain latent diffusion
                models</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://arxiv.org/abs/2112.03109.pdf&amp;sa=D&amp;source=editors&amp;ust=1648758687801141&amp;usg=AOvVaw1OuwC_bwqdXvSPv1yKFDH6">General
                    Facial Representation Learning in a Visual-Linguistic
                    Manner</a></span><span>&nbsp;</span><span>LAION-400M</span><span class="c0">&nbsp;face subset to
                train a face clip</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://arxiv.org/abs/2201.12086&amp;sa=D&amp;source=editors&amp;ust=1648758687801417&amp;usg=AOvVaw3FatImL3KSLCIB683Nn71J">BLIP:
                    Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and
                    Generation</a></span><span>&nbsp;image captioning using </span><span>LAION-400M </span><span
                class="c0">subset</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c7"><a class="c2"
                    href="https://www.google.com/url?q=https://arxiv.org/pdf/2112.05253.pdf&amp;sa=D&amp;source=editors&amp;ust=1648758687801666&amp;usg=AOvVaw1K4m3tMx_Ct76-4mZzzHGJ">MAGMA
                    &ndash; Multimodal Augmentation of Generative Models through Adapter-based
                    Finetuning</a></span><span>&nbsp;was trained on image question answering using a
            </span><span>LAION-400M </span><span class="c0">subset</span></li>
    </ul>
    <p class="c3 c6"><span class="c23 c20"></span></p>
    <h4 class="c14" id="h.4k7rwp6m1kdg"><span class="c9">Conclusion</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>By releasing an updated version of an openly available dataset that contains 5 billion
            image-text pairs, </span><span class="c0">we have set new Standards for the scale of openly available
            datasets and enable researchers from all over the world to train state-of-the-art language-vision models
            like GLIDE or Turing Bletchley. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>As proof of concept, we demonstrated that a subset of our dataset can be used to train various
        </span><span>CLIP-like</span><span class="c0">&nbsp;models, producing samples of sufficient quality. This
            dataset extends the possibilities in multi-language large-scale training and research of language-vision
            models, that were previously restricted to those having access to proprietary large datasets, to the broad
            community. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.1jzn37c7jk4y"><span class="c9">What&rsquo;s next?</span></h4>
    <p class="c3"><span class="c0">This is only the beginning! Now that this huge and open dataset is released, it can
            be used to train many models, such as gigantic clip models, image/text generation models and much
            more.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We have so many projects going on that it&#39;s probably best, if you are interested,
            to join our Discord server and check out what&#39;s going on. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">We are and always will be a grassroots community that works openly and welcomes
            everyone who is kind and passionate and for machine learning.</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span>Join us in </span><span class="c7"><a class="c2"
                href="https://www.google.com/url?q=https://discord.gg/eq3cAMZtCC&amp;sa=D&amp;source=editors&amp;ust=1648758687802894&amp;usg=AOvVaw2QmqjI-0wz8K4aszxrZodW">discord</a></span><span
            class="c0">&nbsp;and help us to train models like CLIP, BLIP, GLIDE, Dall-E, SimMIM, AudioCLIP and don&#39;t
            hesitate to share your ideas for new projects with us. </span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3"><span class="c0">Become a part of our constantly growing crowd of supporters who help us to make
            machine learning dreams come true!</span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <p class="c3 c6"><span class="c0"></span></p>
    <h4 class="c14" id="h.w1bd4ld2q51x"><span class="c9">Credit Assignment</span></h4>
    <p class="c3 c6"><span class="c0"></span></p>
    <ul class="c1 lst-kix_jkaumvtzuajs-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c5">Christoph Schuhmann</span><span>: He led this project
            </span><span class="c0">and built POCs for most of its components including clip filtering,the safety model,
                the watermark model and the Blip inference tuning project.</span></li>
    </ul>
    <p class="c3 c6 c22"><span class="c0"></span></p>
    <ul class="c1 lst-kix_uk2bl0dl1nvh-0 start">
        <li class="c3 c4 li-bullet-0"><span class="c5">Richard Vencu</span><span class="c0">: System architecture and
                download script optimizations, GPU assisted filtering. Set up the AWS infrastructure.</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Romain Beaumont</span><span class="c0">: Guidance on scaling for
                the common crawl filtering pipeline. Built and ran the dataset preparation pipeline: pyspark
                deduplication job, img2dataset, clip inference, autofaiss, safety tags.</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Clayton Mullis</span><span class="c0">: DALLE-pytorch
                training/analysis, glide training, WDS filtering</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Jenia Jitsev</span><span class="c0">: scientific organization
                &amp; writing, experiments planning and design, compute resource acquisition, general supervision</span>
        </li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Robert Kaczmarczyk</span><span class="c0">: Established WDS
                architecture, performed DALL-E training runs, balancing calculation, sample (NSFW, watermark, caption
                quality) annotation and manuscript revision</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Andreas K&ouml;pf:</span><span class="c0">&nbsp;He conducted the
                hyperparameter search for the inference strategies with the BLIP image-captioning model</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Aarush Katta: </span><span class="c0">Trained the watermark
                model</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Cade Gordon</span><span class="c0">: Run distributed inference
                for the watermark tags &amp; trained the CLIP B/32 model on JUWELS Booster</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Ross Wightman:</span><span>&nbsp;Ross helped Cade with the
                debugging &amp; training of the CLIP-B/32 model and executed experiments on JUWELS Booster</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Katherine Crowson and John David Pressman: </span><span
                class="c0">Trained the CLOOB model</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Ar</span><span class="c5">an Komatsuzaki</span><span>: Led an
                image-text-pair dataset building project, which inspired this project.</span></li>
        <li class="c3 c4 li-bullet-0"><span class="c5">Bokai Yu</span><span class="c0">: Accomplished most of the work
                to make the knn index building tool autofaiss work in a distributed setting</span></li>
    </ul>
</body>

</html>