add multiprocessing pool #91

openrightsgroup · Mar 20, 2021 · 7a822da · 7a822da
1 parent c8f0455
commit 7a822da
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/importers/nominet/import-nominet.example.cfg b/importers/nominet/import-nominet.example.cfg
@@ -13,3 +13,6 @@ path=
 
 [paths]
 download=
+
+[worker]
+threads=
diff --git a/importers/nominet/import-nominet.py b/importers/nominet/import-nominet.py
@@ -8,6 +8,7 @@
 import datetime
 import subprocess
 import configparser
+import multiprocessing
 
 import psycopg2
 import paramiko
@@ -153,9 +154,13 @@ def main():
 
     unpack(cfg, filename)
 
+    pool = multiprocessing.Pool(cfg.getint('worker', 'threads'))
+
     conn = connect_db(cfg)
 
-    for name in resolve_iter(compare(cfg, filename)):
+    for name in pool.imap_unordered(resolve, compare(cfg, filename), chunksize=16):
+        if name is None:
+            continue
         logging.info("Got: %s", name)
         dbstore(conn, name)
         if args.debug: