Skip to content

Commit

Permalink
added crawl manager description
Browse files Browse the repository at this point in the history
  • Loading branch information
kalessin committed Feb 5, 2019
1 parent 04999b8 commit 55373a5
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions hcf_backend/utils/crawlmanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Script for easy managing of consumer spiders from multiple slots. It checks available slots and schedules
a job for each one.
"""

import re
import json
import random
Expand Down Expand Up @@ -29,6 +30,10 @@ def add_argparser_options(self):
self.argparser.add_argument('frontier', help='Frontier name')
self.argparser.add_argument('prefix', help='Slot prefix')

@property
def description(self):
return __doc__

def workflow_loop(self):
slot_re = re.compile(rf"{self.args.prefix}\d+")
available_slots = set(slot for slot in self.hcfpal.get_slots(self.args.frontier) if slot_re.match(slot))
Expand Down

0 comments on commit 55373a5

Please sign in to comment.