From 2257b09f0ca8f0cd92abdb791c9078f4d9a72e4b Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Fri, 4 Apr 2014 22:42:07 -0300 Subject: [PATCH 01/14] First attempt at the composer file. --- composer.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 composer.json diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..0bcacdc --- /dev/null +++ b/composer.json @@ -0,0 +1,24 @@ +{ + "name": "jpirkey/gearmanmanager", + "homepage": "http://brian.moonspot.net/GearmanManager", + "description": "PHP daemon for managing gearman workers", + "keywords": ["gearman","net_gearman","gearman-manager"], + "type": "library", + "license": "BSD", + "authors": [ + { + "name": "Brian Moon", + "homepage": "http://brian.moonspot.net" + } + ], + "suggest": { + "jpirkey/net_gearman" : "A PHP interface for Danga's Gearman", + "ext-gearman" : "PHP wrapper to libgearman" + }, + "support": { + "issues": "https://github.com/brianlmoon/GearmanManager/issues" + }, + "require": { + "php": ">=5.2.0" + } +} \ No newline at end of file From 2c339112b91491967ada91ea2f1a6e97d0404aaa Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 00:34:04 -0300 Subject: [PATCH 02/14] Moving GearmanPearManager and GearmanPeclManager classes to their own file. --- GearmanManager.php | 66 +++++++++++++++------- pear-manager.php => GearmanPearManager.php | 17 ------ pecl-manager.php => GearmanPeclManager.php | 22 +------- 3 files changed, 46 insertions(+), 59 deletions(-) rename pear-manager.php => GearmanPearManager.php (95%) mode change 100755 => 100644 rename pecl-manager.php => GearmanPeclManager.php (95%) mode change 100755 => 100644 diff --git a/GearmanManager.php b/GearmanManager.php index 7ed0334..180007b 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -213,11 +213,17 @@ abstract class GearmanManager { */ protected $prefix = ""; + public function __construct($config = null) + { + $this->getopt($config); + $this->pid = getmypid(); + } + /** * Creates the manager and gets things going * */ - public function __construct() { + public function run($config = null) { if(!function_exists("posix_kill")){ $this->show_help("The function posix_kill was not found. Please ensure POSIX functions are installed"); @@ -227,12 +233,12 @@ public function __construct() { $this->show_help("The function pcntl_fork was not found. Please ensure Process Control functions are installed"); } - $this->pid = getmypid(); - - /** - * Parse command line options. Loads the config file as well - */ - $this->getopt(); + if ( ! empty($config)) { + /** + * Parse command line options. Loads the config file as well + */ + $this->getopt($config); + } /** * Register signal listeners @@ -353,25 +359,45 @@ public function __destruct() { * Parses the command line options * */ - protected function getopt() { + protected function getopt($config = array()) { - $opts = getopt("ac:dD:h:Hl:o:p:P:u:v::w:r:x:Z"); + if (empty($config)) { + $opts = getopt("ac:dD:h:Hl:o:p:P:u:v::w:r:x:Z"); - if(isset($opts["H"])){ - $this->show_help(); - } + if(isset($opts["H"])){ + $this->show_help(); + } + + if(isset($opts["c"]) && !file_exists($opts["c"])){ + $this->show_help("Config file $opts[c] not found."); + } - if(isset($opts["c"]) && !file_exists($opts["c"])){ - $this->show_help("Config file $opts[c] not found."); + /** + * parse the config file + */ + if(isset($opts["c"])){ + $this->parse_config($opts["c"]); + } } + else if (is_string($config)) { + if (!file_exists($config)){ + $this->show_help("Config file $config not found."); + } - /** - * parse the config file - */ - if(isset($opts["c"])){ - $this->parse_config($opts["c"]); + /** + * parse the config file + */ + $this->parse_config($config); + } + else { + /** + * set the config to the passed in array + * @todo we should probably do more checks to validate the values + */ + $this->config = $config; } + /** * command line opts always override config file */ @@ -1170,5 +1196,3 @@ protected function show_help($msg = "") { } } - -?> diff --git a/pear-manager.php b/GearmanPearManager.php old mode 100755 new mode 100644 similarity index 95% rename from pear-manager.php rename to GearmanPearManager.php index e1058a3..c42ec77 --- a/pear-manager.php +++ b/GearmanPearManager.php @@ -1,17 +1,4 @@ -#!/usr/bin/env php - * @copyright 1997-Present Brian Moon - * @package GearmanManager - * - */ - -declare(ticks = 1); - /** * Uncomment and set to your prefix. */ @@ -225,7 +212,3 @@ protected function validate_lib_workers() { } } - -$mgr = new GearmanPearManager(); - -?> diff --git a/pecl-manager.php b/GearmanPeclManager.php old mode 100755 new mode 100644 similarity index 95% rename from pecl-manager.php rename to GearmanPeclManager.php index 721749e..7977aff --- a/pecl-manager.php +++ b/GearmanPeclManager.php @@ -1,19 +1,4 @@ -#!/usr/bin/env php - * @copyright 1997-Present Brian Moon - * @package GearmanManager - * - */ - -declare(ticks = 1); - -require dirname(__FILE__)."/GearmanManager.php"; - /** * Implements the worker portions of the pecl/gearman library */ @@ -225,9 +210,4 @@ protected function validate_lib_workers() { } -} - -$mgr = new GearmanPeclManager(); - -?> - +} \ No newline at end of file From aa2b6dc6da12b386c17377890783b7e83df7c810 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 00:34:27 -0300 Subject: [PATCH 03/14] Adding the driver files back. --- pear-manager.php | 18 ++++++++++++++++++ pecl-manager.php | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100755 pear-manager.php create mode 100755 pecl-manager.php diff --git a/pear-manager.php b/pear-manager.php new file mode 100755 index 0000000..7a104f7 --- /dev/null +++ b/pear-manager.php @@ -0,0 +1,18 @@ +#!/usr/bin/env php + + * @copyright 1997-Present Brian Moon + * @package GearmanManager + * + */ + +declare(ticks = 1); + +require dirname(__FILE__) . "/GearmanManager.php"; +require dirname(__FILE__) . "/GearmanPearManager.php"; +$mgr = new GearmanPearManager(); +$mgr->run(); \ No newline at end of file diff --git a/pecl-manager.php b/pecl-manager.php new file mode 100755 index 0000000..72ff3b8 --- /dev/null +++ b/pecl-manager.php @@ -0,0 +1,19 @@ +#!/usr/bin/env php + + * @copyright 1997-Present Brian Moon + * @package GearmanManager + * + */ + +declare(ticks = 1); + +require dirname(__FILE__) . "/GearmanManager.php"; +require dirname(__FILE__) . "/GearmanPeclManager.php"; + +$mgr = new GearmanPeclManager(); +$mgr->run(); \ No newline at end of file From d6a69d03bac14b1985e771b61b5743bcf2154104 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 11:01:40 -0300 Subject: [PATCH 04/14] Using the same spacing in both driver files. --- pear-manager.php | 1 + 1 file changed, 1 insertion(+) diff --git a/pear-manager.php b/pear-manager.php index 7a104f7..94ca442 100755 --- a/pear-manager.php +++ b/pear-manager.php @@ -14,5 +14,6 @@ require dirname(__FILE__) . "/GearmanManager.php"; require dirname(__FILE__) . "/GearmanPearManager.php"; + $mgr = new GearmanPearManager(); $mgr->run(); \ No newline at end of file From 5df8bbfc751c55bcc06e67c701758d18eb2eb2da Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 11:44:05 -0300 Subject: [PATCH 05/14] Adding the 3 classes to the auto load config. --- composer.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/composer.json b/composer.json index 0bcacdc..17da454 100644 --- a/composer.json +++ b/composer.json @@ -20,5 +20,8 @@ }, "require": { "php": ">=5.2.0" + }, + "autoload": { + "classmap": [ "GearmanManager.php", "GearmanPearManager.php", "GearmanPeclManager.php" ] } } \ No newline at end of file From 4035dff95d51b45fee67b2a4ae26a386aecccace Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 21:00:33 -0300 Subject: [PATCH 06/14] Passing in the configuration to the job. --- GearmanPearManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanPearManager.php b/GearmanPearManager.php index c42ec77..f2f3134 100644 --- a/GearmanPearManager.php +++ b/GearmanPearManager.php @@ -51,7 +51,7 @@ protected function start_lib_worker($worker_list, $timeouts = array()) { $message.= "; timeout: $timeout"; } $this->log($message, GearmanManager::LOG_LEVEL_WORKER_INFO); - $worker->addAbility($w, $timeout); + $worker->addAbility($w, $timeout, $this->functions[$w]); } $worker->attachCallback(array($this, 'job_start'), Net_Gearman_Worker::JOB_START); From 5053a147721eefc4fc4f964c16435aa28ce92a3c Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 21:23:02 -0300 Subject: [PATCH 07/14] Adding name to the worker configuration. --- GearmanManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanManager.php b/GearmanManager.php index 180007b..7c13a2d 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -674,7 +674,7 @@ protected function load_workers() { } if(!isset($this->functions[$function])){ - $this->functions[$function] = array(); + $this->functions[$function] = array('name' => $function); } if(!empty($this->config['functions'][$function]['dedicated_only'])){ From fffa6226baee578ee2d707728c7e53e39311ba45 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 21:24:01 -0300 Subject: [PATCH 08/14] Being more descriptive what -p option is for. --- GearmanManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanManager.php b/GearmanManager.php index 7c13a2d..cd34efe 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -1182,7 +1182,7 @@ protected function show_help($msg = "") { echo " -h HOST[:PORT] Connect to HOST and optional PORT\n"; echo " -H Shows this help\n"; echo " -l LOG_FILE Log output to LOG_FILE or use keyword 'syslog' for syslog support\n"; - echo " -p PREFIX Optional prefix for functions/classes of PECL workers. PEAR requires a constant be defined in code.\n"; + echo " -p PREFIX Optional log line prefix for functions/classes of PECL workers. PEAR requires a constant be defined in code.\n"; echo " -P PID_FILE File to write process ID out to\n"; echo " -u USERNAME Run wokers as USERNAME\n"; echo " -v Increase verbosity level by one\n"; From 28ad6ca25be9d023c07d7b127a990c4517b2db2f Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sat, 5 Apr 2014 21:28:13 -0300 Subject: [PATCH 09/14] Adding a number of additional config options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding support for ‘verbose’ in the $config array. Added -I : Ignore idle (no_job) responses in 'Maximum job iterations per worker' setting Added -J PREFIX: Job class prefix [default is 'Net_Gearman_Job_'] (PEAR ONLY) Added -L NUMBER: Number of process loops to run --- GearmanManager.php | 91 ++++++++++++++++++++++++++++++++---------- GearmanPearManager.php | 10 ++++- 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/GearmanManager.php b/GearmanManager.php index cd34efe..4a0305f 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -286,6 +286,7 @@ protected function process_loop() { /** * Main processing loop for the parent process */ + $counter = 0; while(!$this->stop_work || count($this->children)) { $status = null; @@ -309,12 +310,18 @@ protected function process_loop() { $code = pcntl_wexitstatus($status); $this->log("Child $exited exited with error code of $code (".implode(",", $worker).")", GearmanManager::LOG_LEVEL_PROC_INFO); if(!$this->stop_work){ - $this->start_worker($worker); + $counter++; + if ( empty($this->config['process_loops']) || $counter <= $this->config['process_loops']) { + // We want to start another another in its place + $this->start_worker($worker); + } + else { + $this->stop_work = true; + } } } } - if($this->stop_work && time() - $this->stop_time > 60){ $this->log("Children have not exited, killing.", GearmanManager::LOG_LEVEL_PROC_INFO); $this->stop_children(SIGKILL); @@ -332,6 +339,12 @@ protected function process_loop() { } } + + if ( ! empty($this->config['process_loops']) && $counter >= $this->config['process_loops']) { + $this->stop_work = true; + $this->log("Counter {$counter} is greater than or equal to process loop configuration of {$this->config['process_loops']}", GearmanManager::LOG_LEVEL_DEBUG); + } + /** * php will eat up your cpu if you don't have this */ @@ -362,22 +375,7 @@ public function __destruct() { protected function getopt($config = array()) { if (empty($config)) { - $opts = getopt("ac:dD:h:Hl:o:p:P:u:v::w:r:x:Z"); - - if(isset($opts["H"])){ - $this->show_help(); - } - - if(isset($opts["c"]) && !file_exists($opts["c"])){ - $this->show_help("Config file $opts[c] not found."); - } - - /** - * parse the config file - */ - if(isset($opts["c"])){ - $this->parse_config($opts["c"]); - } + $opts = getopt("ac:dD:h:HIJl:L:o:p:P:u:v::w:r:x:Z"); } else if (is_string($config)) { if (!file_exists($config)){ @@ -397,6 +395,33 @@ protected function getopt($config = array()) { $this->config = $config; } + if(isset($opts["H"])){ + $this->show_help(); + } + + if(isset($opts["c"]) && !file_exists($opts["c"])){ + $this->show_help("Config file $opts[c] not found."); + } + + /** + * parse the config file + */ + if(isset($opts["c"])){ + $this->parse_config($opts["c"]); + } + + $this->config['job_class_prefix'] = null; + if (isset($opt["J"])) { + $this->config['job_class_prefix'] = $opt['J']; + } + + if (defined('NET_GEARMAN_JOB_CLASS_PREFIX')) { + $this->config['job_class_prefix'] = NET_GEARMAN_JOB_CLASS_PREFIX; + } + + if ( isset($config['job_class_prefix']) ){ + $this->config['job_class_prefix'] = $config['job_class_prefix']; + } /** * command line opts always override config file @@ -405,8 +430,8 @@ protected function getopt($config = array()) { $this->config['pid_file'] = $opts['P']; } - if(isset($opts["l"])){ - $this->config['log_file'] = $opts["l"]; + if(isset($opts['l'])){ + $this->config['log_file'] = $opts['l']; } if (isset($opts['a'])) { @@ -423,6 +448,12 @@ protected function getopt($config = array()) { if (isset($opts['r'])) { $this->config['max_runs_per_worker'] = (int)$opts['r']; + $this->config['ignore_idle_run_counts'] = false; + } + + if (isset($opts['R'])) { + $this->config['max_runs_per_worker'] = (int)$opts['R']; + $this->config['ignore_idle_run_counts'] = true; } if (isset($opts['D'])) { @@ -449,6 +480,14 @@ protected function getopt($config = array()) { $this->user = $this->config["user"]; } + if (isset($opts['I'])) { + $this->config['ignore_idle_in_run_counts'] = true; + } + + if (isset($opts['L'])) { + $this->config['process_loops'] = (int) $opts['L']; + } + /** * If we want to daemonize, fork here and exit */ @@ -503,6 +542,10 @@ protected function getopt($config = array()) { } } + if (isset($this->config['verbose'])) { + $this->verbose = $this->config['verbose']; + } + if($this->user) { $user = posix_getpwnam($this->user); if (!$user || !isset($user['uid'])) { @@ -707,6 +750,9 @@ protected function load_workers() { } $this->functions[$function]['path'] = $file; + if ($this->config['job_class_prefix']) { + $this->functions[$function]['class_name'] = $this->config['job_class_prefix'] . $function; + } /** * Note about priority. This exploits an undocumented feature @@ -1181,10 +1227,13 @@ protected function show_help($msg = "") { echo " -D NUMBER Start NUMBER workers that do all jobs\n"; echo " -h HOST[:PORT] Connect to HOST and optional PORT\n"; echo " -H Shows this help\n"; + echo " -I Ignore idle (no_job) responses in 'Maximum job iterations per worker' setting\n"; + echo " -J PREFIX Job class prefix [default is 'Net_Gearman_Job_'] (PEAR ONLY)\n"; echo " -l LOG_FILE Log output to LOG_FILE or use keyword 'syslog' for syslog support\n"; + echo " -L NUMBER Number of process loops to run\n"; echo " -p PREFIX Optional log line prefix for functions/classes of PECL workers. PEAR requires a constant be defined in code.\n"; echo " -P PID_FILE File to write process ID out to\n"; - echo " -u USERNAME Run wokers as USERNAME\n"; + echo " -u USERNAME Run workers as USERNAME\n"; echo " -v Increase verbosity level by one\n"; echo " -w DIR Directory where workers are located, defaults to ./workers. If you are using PECL, you can provide multiple directories separated by a comma.\n"; echo " -r NUMBER Maximum job iterations per worker\n"; diff --git a/GearmanPearManager.php b/GearmanPearManager.php index f2f3134..08015f6 100644 --- a/GearmanPearManager.php +++ b/GearmanPearManager.php @@ -84,7 +84,13 @@ public function monitor($idle, $lastJob) { $this->log("Worker's last job $time seconds ago", GearmanManager::LOG_LEVEL_CRAZY); - if(!empty($this->config["max_runs_per_worker"]) && $this->job_execution_count >= $this->config["max_runs_per_worker"]) { + if ( $idle && ! empty($this->config['ignore_idle_in_run_counts']) ) { + // Keep looking for a job + $this->log("No job returned, continuing to wait", GearmanManager::LOG_LEVEL_CRAZY); + return false; + } + + if( !empty($this->config["max_runs_per_worker"]) && $this->job_execution_count >= $this->config["max_runs_per_worker"]) { $this->log("Ran $this->job_execution_count jobs which is over the maximum({$this->config['max_runs_per_worker']}), exiting", GearmanManager::LOG_LEVEL_WORKER_INFO); $this->stop_work = true; } @@ -198,7 +204,7 @@ protected function validate_lib_workers() { * Validate functions */ foreach($this->functions as $name => $func){ - $class = NET_GEARMAN_JOB_CLASS_PREFIX.$name; + $class = $this->config['job_class_prefix'].$name; if(!class_exists($class)) { include $func['path']; } From 64aa6649bf0027fa6c11a58f70c23706ef9e90ef Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sun, 6 Apr 2014 01:05:19 -0300 Subject: [PATCH 10/14] Fixing an error in the comment. --- GearmanPearManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanPearManager.php b/GearmanPearManager.php index 08015f6..2cd486c 100644 --- a/GearmanPearManager.php +++ b/GearmanPearManager.php @@ -183,7 +183,7 @@ private function log_result($handle, $result) { /** - * Validates the PECL compatible worker files/functions + * Validates the PEAR compatible worker files/functions */ protected function validate_lib_workers() { From a34c25ab42d3dc8140ee8898a6dacd2d840b66a6 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sun, 6 Apr 2014 01:05:43 -0300 Subject: [PATCH 11/14] Adding support for passing in the daemon param via the config array. --- GearmanManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanManager.php b/GearmanManager.php index 4a0305f..6ca0d48 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -491,7 +491,7 @@ protected function getopt($config = array()) { /** * If we want to daemonize, fork here and exit */ - if(isset($opts["d"])){ + if(isset($opts["d"]) || ! empty($config['daemon'])){ $pid = pcntl_fork(); if($pid>0){ $this->isparent = false; From cb0df6d4327df957809e35af0e03fe2f650f58d3 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Sun, 6 Apr 2014 02:53:05 -0300 Subject: [PATCH 12/14] Adding a @todo for fixing the handler when multiple workers are running. --- GearmanManager.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/GearmanManager.php b/GearmanManager.php index 6ca0d48..2a35e35 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -316,6 +316,10 @@ protected function process_loop() { $this->start_worker($worker); } else { + /** + * @todo Don't want to do this because it kills all children; which I need to + * figure out how to handle that better + */ $this->stop_work = true; } } From 9ba1d23f73e4f6c5af75e688a6cc10edc50bdb3c Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Thu, 31 Jul 2014 14:05:05 -0400 Subject: [PATCH 13/14] Allow for empty class prefixes in config array. --- GearmanManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GearmanManager.php b/GearmanManager.php index 9ca0121..232f66f 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -451,7 +451,7 @@ protected function getopt($config = array()) { if (isset($opts['p'])) { $this->prefix = $opts['p']; - } elseif(!empty($this->config['prefix'])) { + } elseif(isset($this->config['prefix'])) { $this->prefix = $this->config['prefix']; } elseif(defined('NET_GEARMAN_JOB_CLASS_PREFIX')) { $this->prefix = NET_GEARMAN_JOB_CLASS_PREFIX; From ee415e50058d3639256b700c31886e15c7553f68 Mon Sep 17 00:00:00 2001 From: Jason Pirkey Date: Thu, 31 Jul 2014 14:05:43 -0400 Subject: [PATCH 14/14] Defaulting PEAR manager prefix to Net_Gearman_Job_ All the examples use Net_Gearman_Job_ as the prefix, so we should default that. --- GearmanManager.php | 2 -- GearmanPearManager.php | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/GearmanManager.php b/GearmanManager.php index 232f66f..066da02 100644 --- a/GearmanManager.php +++ b/GearmanManager.php @@ -455,8 +455,6 @@ protected function getopt($config = array()) { $this->prefix = $this->config['prefix']; } elseif(defined('NET_GEARMAN_JOB_CLASS_PREFIX')) { $this->prefix = NET_GEARMAN_JOB_CLASS_PREFIX; - } else { - $this->prefix = 'Net_Gearman_Job_'; } if(isset($opts['u'])){ diff --git a/GearmanPearManager.php b/GearmanPearManager.php index be2724e..d4b6e8c 100644 --- a/GearmanPearManager.php +++ b/GearmanPearManager.php @@ -13,6 +13,12 @@ class GearmanPearManager extends GearmanManager { private $start_time; + /** + * Job Class prefix to use. + * @var string + */ + protected $prefix = 'Net_Gearman_Job_'; + /** * Starts a worker for the PEAR library *