forked from openscanhub/fedora-infra
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add resalloc agent spawner in the hub container
Signed-off-by: Siteshwar Vashisht <[email protected]>
- Loading branch information
Showing
3 changed files
with
204 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| # Configuration for resalloc-agent-spawner.service. YAML format. | ||
| # Specify groups of agents within the `agent_groups` section that agent spawner | ||
| # should take care of. | ||
|
|
||
| agent_groups: | ||
| workers: | ||
| # These commands are executed in the background async as they may take | ||
| # quite some time to process. If `cmd_prepare` fails (exit non-zero), the | ||
| # agent immediately marked for removal. `cmd_terminate` exit status is | ||
| # just ignored (we need to remove the agent no matter what). | ||
|
|
||
| # Prepare the agent. Variable $RESALLOC_RESOURCE_DATA (base64 encoded) | ||
| # is provided in the script environment. Other variables like | ||
| # RESOURCE_NAME, RESOURCE_POOL_ID, etc. are provided as well. | ||
| cmd_prepare: /usr/bin/osh-worker-manager --create-worker $(echo "$AGENT_SPAWNER_RESOURCE_DATA" | base64 -d | head -1) &> /tmp/$$-preparation.log | ||
| # cmd_prepare: echo "$RESALLOC_RESOURCE_DATA" | base64 -d | head -1 &> /tmp/$$-preparation.log | ||
| # cmd_prepare: env &> /tmp/$$-preparation.log | ||
| # Prepare the agent for termination. Upon finishing this command, the | ||
| # resalloc resource ticket is closed and the resource deallocated. | ||
| cmd_terminate: /usr/bin/osh-worker-manager --delete-worker $(echo "$AGENT_SPAWNER_RESOURCE_DATA" | base64 -d | head -1) &> /tmp/$$-deletion.log | ||
| # cmd_terminate: /sbin/osh-resalloc --delete-worker "$(echo "$RESALLOC_RESOURCE_DATA" | base64 -d | head -1)" | ||
|
|
||
| # The following commands are executed synchronously by the agent spawner | ||
| # daemon (polling). Please keep them super fast to avoid overall system # | ||
| # halt! | ||
|
|
||
| # The `cmd_converge_to` needs to print integer number (the currently ideal | ||
| # number of agents to converge to) onto stdout. | ||
| cmd_converge_to: /usr/bin/osh-worker-manager --workers-needed | ||
|
|
||
| # Agents may decide to stop themselves. This hook is used to detect | ||
| # such a case -> if exit status 0 is returned, agent is going to be | ||
| # terminated (cmd_terminate is called against it). | ||
| cmd_check_finished: /usr/bin/osh-worker-manager --check-throwaway $(echo "$AGENT_SPAWNER_RESOURCE_DATA" | base64 -d | head -1) &> /tmp/$$-check-finished.log | ||
|
|
||
| # Some agents might be expected to run long-term (or indefinitely). This | ||
| # hook helps us to politely ask the agent whether it is OK to terminate. | ||
| # Returning exit status 1 means the agent can not be terminated. | ||
| # Returning 0 means that the agent was prepared for termination, and | ||
| # this has to be removed now. This is useful for gently downsizing | ||
| # the agent count while converging to `cmd_converge_to`. | ||
| cmd_try_release: /bin/false | ||
|
|
||
| # List of resalloc tags to use while requesting tickets | ||
| tags: | ||
| - arch_x86_64 | ||
|
|
||
| # Note that we use the 'request_survives_server_restart' resalloc client option, | ||
| # so the resalloc server must be running to avoid the overall system hang! | ||
| resalloc_server: "http://resalloc-server:49100" | ||
|
|
||
| # Where to log events. | ||
| logfile: /var/log/resalloc-agent-spawner/agent-spawner.log | ||
|
|
||
| # How to connect to redis-db. By default connects to 127.0.0.1:6379. | ||
| #redis_db: null | ||
| redis_host: redis | ||
| #redis_port": null | ||
| #redis_password": null |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| """ | ||
| resalloc agent spawner helpers | ||
| """ | ||
|
|
||
| import os | ||
| import subprocess | ||
|
|
||
| from resalloc.helpers import load_config_file | ||
|
|
||
|
|
||
| def rk_to_tid(key): | ||
| """ Redis agent key to Resalloc Ticket ID """ | ||
| return int(key.split(":")[1]) | ||
|
|
||
|
|
||
| def tid_to_rk(ticket): | ||
| """ Resalloc Ticket ID to Redis Agent key """ | ||
| return f"agent:{ticket}" | ||
|
|
||
|
|
||
| def get_config(): | ||
| """ | ||
| Load the agent-spawner YAML configuration | ||
| """ | ||
| conf_dir = os.environ.get("CONFIG_DIR", "/etc/resalloc-agent-spawner") | ||
| config_file = os.path.join(conf_dir, "config.yaml") | ||
| config = load_config_file(config_file) | ||
| config.setdefault("agent_groups", {}) | ||
| config.setdefault("resalloc_server", "http://localhost:49100") | ||
| config.setdefault("logfile", "/tmp/agent-spawner.log") | ||
| groups = config["agent_groups"] | ||
| for group_id in groups.keys(): | ||
| group = groups[group_id] | ||
| group.setdefault("cmd_converge_to", "/usr/bin/echo 1") | ||
| group.setdefault("cmd_check_finished", "/bin/false") | ||
| group.setdefault("cmd_prepare", "/bin/true") | ||
| group.setdefault("cmd_terminate", "/bin/true") | ||
| group.setdefault("cmd_try_release", "/bin/false") | ||
| group.setdefault("tags", ["please-specify-some-tags"]) | ||
|
|
||
| # This dictionary is passed to redis APIs and other parts of code which | ||
| # expects configurations to be attributes rather than key/value pairs. | ||
| # This class works around this requirement. | ||
| # Thanks to https://stackoverflow.com/a/14620633 | ||
| class AttrDict(dict): | ||
| def __init__(self, *args, **kwargs): | ||
| super(AttrDict, self).__init__(*args, **kwargs) | ||
| self.__dict__ = self | ||
|
|
||
| return AttrDict(config) | ||
|
|
||
|
|
||
| class CmdCallerMixin: | ||
| """ | ||
| Wrapper around calling command hooks. | ||
| """ | ||
|
|
||
| def _cmd(self, group_id, cmd_id): | ||
| return self.opts["agent_groups"][group_id][cmd_id] | ||
|
|
||
| def cmd_converge_to(self, group_id): | ||
| """ | ||
| Query the outside world for the ideal number of agents in given group. | ||
| """ | ||
| result = subprocess.run( | ||
| self._cmd(group_id, "cmd_converge_to"), | ||
| stdout=subprocess.PIPE, check=False, shell=True) | ||
| if result.returncode == 0: | ||
| try: | ||
| return int(result.stdout.decode("utf-8").strip()) | ||
| except ValueError: | ||
| self.log.error("Converge-to hook failure, expected int, " | ||
| "got: %s", result.stdout) | ||
| return None | ||
|
|
||
| self.log.debug("Failing to run converge-to hook") | ||
| return None | ||
|
|
||
| def cmd_try_release(self, group_id, data): | ||
| """ | ||
| Call hook that releases the resource | ||
| """ | ||
| cmd = self._cmd(group_id, "cmd_try_release") | ||
| result = subprocess.run(cmd, check=False, **self.subproces_kwargs(data)) | ||
| return not result.returncode | ||
|
|
||
| def cmd_is_finished(self, group_id, data): | ||
| """ | ||
| Call hook that releases the resource | ||
| """ | ||
| result = subprocess.run( | ||
| self._cmd(group_id, "cmd_check_finished"), | ||
| check=False, **self.subproces_kwargs(data)) | ||
| return not result.returncode | ||
|
|
||
| def cmd_take(self, group_id, data): | ||
| """ | ||
| Initialize the agent | ||
| """ | ||
| return not subprocess.run( | ||
| self._cmd(group_id, "cmd_prepare"), check=False, | ||
| **self.subproces_kwargs(data), | ||
| ).returncode | ||
|
|
||
| def cmd_terminate(self, group_id, data): | ||
| """ | ||
| Prepare the agent for removal. | ||
| """ | ||
| subprocess.run( | ||
| self._cmd(group_id, "cmd_terminate"), | ||
| check=False, | ||
| **self.subproces_kwargs(data), | ||
| ) | ||
|
|
||
| def subproces_kwargs(self, data): | ||
| """ | ||
| generate "generic" subprocess.Popen kwargs | ||
| """ | ||
| return { | ||
| "env": { | ||
| "AGENT_SPAWNER_RESOURCE_DATA": str(data), | ||
| }, | ||
| "shell": True, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters