diff --git a/importers/nominet/import-nominet.example.cfg b/importers/nominet/import-nominet.example.cfg index 990c06e..2e11222 100644 --- a/importers/nominet/import-nominet.example.cfg +++ b/importers/nominet/import-nominet.example.cfg @@ -13,3 +13,6 @@ path= [paths] download= + +[worker] +threads= diff --git a/importers/nominet/import-nominet.py b/importers/nominet/import-nominet.py index 24c10b9..d77cb8b 100644 --- a/importers/nominet/import-nominet.py +++ b/importers/nominet/import-nominet.py @@ -8,6 +8,7 @@ import datetime import subprocess import configparser +import multiprocessing import psycopg2 import paramiko @@ -153,9 +154,13 @@ def main(): unpack(cfg, filename) + pool = multiprocessing.Pool(cfg.getint('worker', 'threads')) + conn = connect_db(cfg) - for name in resolve_iter(compare(cfg, filename)): + for name in pool.imap_unordered(resolve, compare(cfg, filename), chunksize=16): + if name is None: + continue logging.info("Got: %s", name) dbstore(conn, name) if args.debug: