crawl manager: added missing HCF_CONSUMER_FRONTIER to scheduled job

scrapinghub · Feb 5, 2019 · 76d8783 · 76d8783
1 parent 55373a5
commit 76d8783
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/hcf_backend/utils/crawlmanager.py b/hcf_backend/utils/crawlmanager.py
@@ -61,7 +61,8 @@ def workflow_loop(self):
 
             for slot in available_slots:
                 frontera_settings_json = json.dumps({
-                    'HCF_CONSUMER_SLOT': slot
+                    'HCF_CONSUMER_SLOT': slot,
+                    'HCF_CONSUMER_FRONTIER': self.args.frontier,
                 })
                 logger.info(f"Will schedule spider job with frontera settings {frontera_settings_json}")
                 jobkey = self.schedule_spider(spider_args_override={'frontera_settings_json': frontera_settings_json})