From 41f3b40fc145965466bcbd0ee88161c7752ea264 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sun, 19 Nov 2023 02:59:20 +0000 Subject: [PATCH 01/14] Get the plugin working as a plain transport --- README.md | 10 +++ src/Controller/AppController.php | 10 +++ src/Controller/WebhooksController.php | 25 ++++++ src/Mailer/Transport/SendGridTransport.php | 2 +- src/Plugin.php | 27 ------- src/SendGridPlugin.php | 93 ++++++++++++++++++++++ 6 files changed, 139 insertions(+), 28 deletions(-) create mode 100644 src/Controller/AppController.php create mode 100644 src/Controller/WebhooksController.php delete mode 100644 src/Plugin.php create mode 100644 src/SendGridPlugin.php diff --git a/README.md b/README.md index 6e300b2..a98e42d 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,16 @@ $email->setTo('foo@example.com.com') ->setSendAt(1649500630) ->deliver(); ``` +### Webhooks +You can receive status events from SendGrid. + + + https://app.sendgrid.com/settings/mail_settings/webhook_settings + https://docs.sendgrid.com/for-developers/tracking-events/event#security-features + +/send-grid/webhook + + ## Reporting Issues diff --git a/src/Controller/AppController.php b/src/Controller/AppController.php new file mode 100644 index 0000000..5803c41 --- /dev/null +++ b/src/Controller/AppController.php @@ -0,0 +1,10 @@ + 'https://api.sendgrid.com/v3', 'apiKey' => '', ]; diff --git a/src/Plugin.php b/src/Plugin.php deleted file mode 100644 index b79ecee..0000000 --- a/src/Plugin.php +++ /dev/null @@ -1,27 +0,0 @@ -plugin( + 'SendGrid', + ['path' => '/send-grid'], + function (RouteBuilder $builder) { + // Add custom routes here + $builder->fallbacks(); + } + ); + parent::routes($routes); + } + + /** + * Add middleware for the plugin. + * + * @param \Cake\Http\MiddlewareQueue $middlewareQueue The middleware queue to update. + * @return \Cake\Http\MiddlewareQueue + */ + public function middleware(MiddlewareQueue $middlewareQueue): MiddlewareQueue + { + // Add your middlewares here + + return $middlewareQueue; + } + + /** + * Add commands for the plugin. + * + * @param \Cake\Console\CommandCollection $commands The command collection to update. + * @return \Cake\Console\CommandCollection + */ + public function console(CommandCollection $commands): CommandCollection + { + // Add your commands here + + $commands = parent::console($commands); + + return $commands; + } + + /** + * Register application container services. + * + * @param \Cake\Core\ContainerInterface $container The Container to update. + * @return void + * @link https://book.cakephp.org/4/en/development/dependency-injection.html#dependency-injection + */ + public function services(ContainerInterface $container): void + { + // Add your services here + } +} From 492b703b0956f0dc606bb1468943a5147e312b44 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Wed, 22 Nov 2023 18:47:44 +0000 Subject: [PATCH 02/14] Default Config for Transport --- src/Mailer/Transport/SendGridTransport.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Mailer/Transport/SendGridTransport.php b/src/Mailer/Transport/SendGridTransport.php index 35c6309..a26e5c7 100644 --- a/src/Mailer/Transport/SendGridTransport.php +++ b/src/Mailer/Transport/SendGridTransport.php @@ -36,6 +36,14 @@ class SendGridTransport extends AbstractTransport protected array $_defaultConfig = [ 'apiEndpoint' => 'https://api.sendgrid.com/v3', 'apiKey' => '', + 'enableWebhooks' => false, // If you want to use webhooks to monitor delivery ane error reports + 'webhookConfig' => [ + 'domain' => 'example.com', // The domain used for Message IDs does not need to be + 'tableName' => 'email_queue', // The table name that stores email data + 'uniqueIdField' => 'id', // The field name that stores the unique message ID char(36) uid + 'statusField' => 'status', // The field name that stores the status of the email char(50) status + 'statusMessageField' => 'status_message', // The field name that stores the status message TEXT status_message + ] ]; /** @@ -109,6 +117,8 @@ public function send(Message $message): array $this->_reqParams['subject'] = $message->getSubject(); + // $this->_reqParams['custom_args'] ="{'email_msg_id': '106'}"; + $emailFormat = $message->getEmailFormat(); if (!empty($message->getBodyHtml())) { $this->_reqParams['content'][] = (object)[ @@ -253,6 +263,7 @@ protected function _processAttachments(Message $message) */ protected function _sendEmail() { + $options = [ 'type' => 'json', 'headers' => [ @@ -260,6 +271,7 @@ protected function _sendEmail() 'Authorization' => 'Bearer ' . $this->getConfig('apiKey'), ], ]; + $response = $this->Client ->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options); @@ -267,6 +279,7 @@ protected function _sendEmail() $result = []; $result['apiResponse'] = $response->getJson(); $result['responseCode'] = $response->getStatusCode(); + $result['messageId'] = $response->getHeader('X-Message-Id') ?? ''; $result['status'] = $result['responseCode'] == 202 ? 'OK' : 'ERROR'; if (Configure::read('debug')) { $result['reqParams'] = $this->_reqParams; From 5025970513a6eb3531ff9d78d0e7af551775a47a Mon Sep 17 00:00:00 2001 From: Jubbs Date: Thu, 23 Nov 2023 20:53:01 +0000 Subject: [PATCH 03/14] Webhooks --- src/Controller/WebhooksController.php | 50 +++++++++++++++---- src/Mailer/Exception/SendGridApiException.php | 3 +- src/SendGridPlugin.php | 6 ++- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index b387d42..41c4149 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -1,25 +1,53 @@ skipCheckCallback(function ($request) { + * // Skip token check for API URLs. + * Log::write('debug', json_encode($request->getParam('controller'))); + * if ($request->getParam('controller') === 'Webhooks') { + * return true; + * } + * }); * - * Add your application-wide methods in the class below, your controllers - * will inherit them. + * // Ensure routing middleware is added to the queue before CSRF protection middleware. + * $middlewareQueue->add($csrf); * - * @link https://book.cakephp.org/4/en/controllers.html#the-app-controller + * return $middlewareQueue; + * + * test curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '{"login":"my_login","password":"my_password"}' */ -class AppController extends AppController + +namespace SendGrid\Controller; + +use SendGrid\Controller\AppController; +use Cake\Log\Engine\FileLog; + + +class WebHooksController extends AppController { public function index() { - + //debug($this->request->getData()); + $this->viewBuilder()->setLayout('ajax'); + + // Log the incoming data + $this->log(json_encode($this->request->getParam('controller')), 'debug'); } -} \ No newline at end of file +} diff --git a/src/Mailer/Exception/SendGridApiException.php b/src/Mailer/Exception/SendGridApiException.php index b222ec2..9fb2eac 100644 --- a/src/Mailer/Exception/SendGridApiException.php +++ b/src/Mailer/Exception/SendGridApiException.php @@ -1,6 +1,7 @@ Date: Sun, 26 Nov 2023 21:32:03 +0000 Subject: [PATCH 04/14] Update Status --- README.md | 63 ++++++++++++++++++++++++--- src/Controller/WebhooksController.php | 60 +++++++++++++------------ 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a98e42d..39da6cd 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ This plugin provides email delivery using [SendGrid](https://sendgrid.com/). -This branch is for use with CakePHP 4.0+. For CakePHP 3, please use cake-3.x branch. +This branch is for use with CakePHP 5.0+. For CakePHP 4, please use cake-4.x branch. ## Requirements This plugin has the following requirements: -* CakePHP 4.0 or greater. +* CakePHP 5.0 or greater. * PHP 7.2 or greater. ## Installation @@ -137,15 +137,64 @@ $email->setTo('foo@example.com.com') ->setSendAt(1649500630) ->deliver(); ``` -### Webhooks -You can receive status events from SendGrid. +## Webhooks +You can receive status events from SendGrid. This allows you ensure that SendGrid was able to send the email recording bounces etc. +### Webhook Config +You will require a Table in the database to record the emails sent. You can use the lorenzo/cakephp-email-queue plugin to queue the emails and in that case you would +use the email_queue table. However you can create your own table/Model as long as it has at least three columns. They can be called anything but they must have the correct types. - https://app.sendgrid.com/settings/mail_settings/webhook_settings - https://docs.sendgrid.com/for-developers/tracking-events/event#security-features +When you send the email the deliver function will return an array with a 'messageId' element if it successfully connected to SendGrid. This needs to be recorded in the status_id field. + +* status_id VARCHAR(100) +* status VARCHAR(100) +* status_message TEXT + +You need to map this table and these fields in you app_local.php config + +```php + + 'sendgridWebhook' => [ + 'tableClass' => 'EmailQueue', // The table name that stores email data + 'uniqueIdField' => 'status_id', // The field name that stores the unique message ID VARCHAR(100) + 'statusField' => 'status', // The field name that stores the status of the email status VARCHAR(100) + 'statusMessageField' => 'status_message', // The field name that stores the status messages TEXT + ], + +``` + +You will need to login to your SendGrid Account and configure your domain and the events that you want to track + + https://app.sendgrid.com/settings/mail_settings/webhook_settings + +The return url needs to be set to +* https://YOUR DOMAIN/send-grid/webhook + +Security needs to allow this action to be posted to TODO test with Auth plugin + +The CSRF protection middleware needs to allow posts to the webhooks controller in Application.php +Remove the current CSRF protection middleware and replace it with the following. If you already have CSRF exceptions then add the Webhooks one + + ```php + $csrf = new CsrfProtectionMiddleware(); + + $csrf->skipCheckCallback(function ($request) { + // Skip token check for API URLs. + if ($request->getParam('controller') === 'Webhooks') { + return true; + } + }); + + // Ensure routing middleware is added to the queue before CSRF protection middleware. + $middlewareQueue->add($csrf); + + return $middlewareQueue; -/send-grid/webhook + ``` +TODO enable SendGrid security + https://docs.sendgrid.com/for-developers/tracking-events/event#security-features + ## Reporting Issues diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index 41c4149..ed2204f 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -8,46 +8,52 @@ //https://app.sendgrid.com/settings/mail_settings /** + * test * - * The CSRF protection middleware needs to allow posts to the webhooks controller - * in Application.php - * - * Remove the current CSRF protection middleware and replace it with the following. If you already have CSRF exceptions then add the Webhooks one - * - * $csrf = new CsrfProtectionMiddleware(); - * - * - * $csrf->skipCheckCallback(function ($request) { - * // Skip token check for API URLs. - * Log::write('debug', json_encode($request->getParam('controller'))); - * if ($request->getParam('controller') === 'Webhooks') { - * return true; - * } - * }); - * - * // Ensure routing middleware is added to the queue before CSRF protection middleware. - * $middlewareQueue->add($csrf); - * - * return $middlewareQueue; - * - * test curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '{"login":"my_login","password":"my_password"}' + curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}' */ namespace SendGrid\Controller; use SendGrid\Controller\AppController; use Cake\Log\Engine\FileLog; - +use Cake\View\JsonView; +use Cake\Core\Configure; +use Cake\I18n\DateTime; class WebHooksController extends AppController { + + public function viewClasses(): array + { + return [JsonView::class]; + } + + public function index() { - //debug($this->request->getData()); - $this->viewBuilder()->setLayout('ajax'); - // Log the incoming data - $this->log(json_encode($this->request->getParam('controller')), 'debug'); + $this->viewBuilder()->setClassName("Json"); + + $result = $this->request->getData(); + $this->set('result',$result); + + $config = Configure::read('sendgridWebhook'); + + $emailTable = $this->getTableLocator()->get($config['tableClass']); + + $email_record = $emailTable->find('all')->contain([ + $config['uniqueIdField'], + $config['statusField'], + $config['statusMessageField'], + ])->where([$config['uniqueIdField']=>$result->sg_message_id])->first(); + + $email_record->$config['statusMessageField'] .= "
".(new DateTime())->format('d/m/Y H:i:s').":".$result->event." ".$result->response??" ".$result->reason??" "; + $email_record->$config['statusField'] = $result->event; + $emailTable->save($email_record); + + $ok = "OK"; + $this->viewBuilder()->setOption('serialize', $ok); } } From a3b55eab4bb9a261be6ae382cbee6a33c68bff45 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Mon, 27 Nov 2023 09:31:22 +0000 Subject: [PATCH 05/14] getting Webhook working --- src/Controller/WebhooksController.php | 51 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index ed2204f..0bda2f6 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -16,11 +16,13 @@ namespace SendGrid\Controller; use SendGrid\Controller\AppController; -use Cake\Log\Engine\FileLog; + use Cake\View\JsonView; use Cake\Core\Configure; use Cake\I18n\DateTime; +use Cake\Log\Log; + class WebHooksController extends AppController { @@ -34,25 +36,34 @@ public function viewClasses(): array public function index() { - $this->viewBuilder()->setClassName("Json"); - - $result = $this->request->getData(); - $this->set('result',$result); - - $config = Configure::read('sendgridWebhook'); - - $emailTable = $this->getTableLocator()->get($config['tableClass']); - - $email_record = $emailTable->find('all')->contain([ - $config['uniqueIdField'], - $config['statusField'], - $config['statusMessageField'], - ])->where([$config['uniqueIdField']=>$result->sg_message_id])->first(); - - $email_record->$config['statusMessageField'] .= "
".(new DateTime())->format('d/m/Y H:i:s').":".$result->event." ".$result->response??" ".$result->reason??" "; - $email_record->$config['statusField'] = $result->event; - $emailTable->save($email_record); - + $this->viewBuilder()->setClassName("Json"); + + $result = $this->request->getData(); + $this->set('result', $result); + + $config = Configure::read('sendgridWebhook'); + + if (isset($config['debug']) && $config['debug'] == 'true') { + Log::debug(json_encode($result)); + } + + $emailTable = $this->getTableLocator()->get($config['tableClass']); + + foreach ($result as $event) { + $message_id = explode(".", $event['sg_message_id'])[0]; + $email_record = $emailTable->find('all')->select(["id", + $config['uniqueIdField'], + $config['statusField'], + $config['statusMessageField'], + ])->where([$config['uniqueIdField'] => $message_id])->first(); + if ($email_record) { + $email_record->set($config['statusMessageField'],$email_record->get($config['statusMessageField']). "
" . (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . + (isset($event['response'])? $event['response']:""). " " . + (isset($event['reason'])? $event['reason']:"")); + $email_record->set($config['statusField'],$event['event']); + $emailTable->save($email_record); + } + } $ok = "OK"; $this->viewBuilder()->setOption('serialize', $ok); } From 932fe3ad570d58cffbc0efa65abd34f2499d5c30 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Mon, 27 Nov 2023 09:32:10 +0000 Subject: [PATCH 06/14] break at end --- src/Controller/WebhooksController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index 0bda2f6..a7e3cc7 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -57,10 +57,10 @@ public function index() $config['statusMessageField'], ])->where([$config['uniqueIdField'] => $message_id])->first(); if ($email_record) { - $email_record->set($config['statusMessageField'],$email_record->get($config['statusMessageField']). "
" . (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . + $email_record->set($config['statusMessageField'],$email_record->get($config['statusMessageField']). (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . (isset($event['response'])? $event['response']:""). " " . (isset($event['reason'])? $event['reason']:"")); - $email_record->set($config['statusField'],$event['event']); + $email_record->set($config['statusField'],$event['event']."
"); $emailTable->save($email_record); } } From 3e97472c49573b00a1c5f6ba493f68b19605e749 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Fri, 1 Dec 2023 20:47:02 +0000 Subject: [PATCH 07/14] testing webhook --- src/Controller/WebhooksController.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index a7e3cc7..d31f5a5 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -10,7 +10,7 @@ /** * test * - curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}' + curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' */ namespace SendGrid\Controller; @@ -48,7 +48,7 @@ public function index() } $emailTable = $this->getTableLocator()->get($config['tableClass']); - + $count = 0; foreach ($result as $event) { $message_id = explode(".", $event['sg_message_id'])[0]; $email_record = $emailTable->find('all')->select(["id", @@ -57,14 +57,18 @@ public function index() $config['statusMessageField'], ])->where([$config['uniqueIdField'] => $message_id])->first(); if ($email_record) { + $count++; $email_record->set($config['statusMessageField'],$email_record->get($config['statusMessageField']). (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . (isset($event['response'])? $event['response']:""). " " . - (isset($event['reason'])? $event['reason']:"")); - $email_record->set($config['statusField'],$event['event']."
"); + (isset($event['reason'])? $event['reason']:"
")); + $email_record->set($config['statusField'],$event['event']); $emailTable->save($email_record); } } - $ok = "OK"; - $this->viewBuilder()->setOption('serialize', $ok); + if (isset($config['debug']) && $config['debug'] == 'true') { + Log::debug("Updated $count Email records"); + } + $this->set('OK', "OK"); + $this->viewBuilder()->setOption('serialize', "OK"); } } From e75cc6df229ea28b1ec5489ff76a758c8ba0c344 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Fri, 1 Dec 2023 20:52:38 +0000 Subject: [PATCH 08/14] TODO comments --- README.md | 11 +++++------ src/Controller/WebhooksController.php | 6 +++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 39da6cd..c00ce57 100644 --- a/README.md +++ b/README.md @@ -176,24 +176,23 @@ The CSRF protection middleware needs to allow posts to the webhooks controller i Remove the current CSRF protection middleware and replace it with the following. If you already have CSRF exceptions then add the Webhooks one ```php - $csrf = new CsrfProtectionMiddleware(); + $csrf = new CsrfProtectionMiddleware(); - $csrf->skipCheckCallback(function ($request) { + $csrf->skipCheckCallback(function ($request) { // Skip token check for API URLs. if ($request->getParam('controller') === 'Webhooks') { return true; } - }); + }); // Ensure routing middleware is added to the queue before CSRF protection middleware. - $middlewareQueue->add($csrf); + $middlewareQueue->add($csrf); return $middlewareQueue; ``` -TODO enable SendGrid security - https://docs.sendgrid.com/for-developers/tracking-events/event#security-features + diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index d31f5a5..28a38a6 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -10,7 +10,11 @@ /** * test * - curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' + * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' + * + * TODO enable SendGrid security + * https://docs.sendgrid.com/for-developers/tracking-events/event#security-features + */ namespace SendGrid\Controller; From 90900324dc6bcb178e9f40000676be267743d82d Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sat, 2 Dec 2023 05:57:16 +0000 Subject: [PATCH 09/14] Get Elliptic Curve Working --- README.md | 9 ++- composer.json | 3 +- src/Controller/WebhooksController.php | 82 +++++++++++++++++++++----- src/Util/EllipticCurve/Ecdsa.php | 26 +++++++++ src/Util/EllipticCurve/PrivateKey.php | 84 +++++++++++++++++++++++++++ src/Util/EllipticCurve/PublicKey.php | 64 ++++++++++++++++++++ src/Util/EllipticCurve/Signature.php | 29 +++++++++ 7 files changed, 280 insertions(+), 17 deletions(-) create mode 100644 src/Util/EllipticCurve/Ecdsa.php create mode 100644 src/Util/EllipticCurve/PrivateKey.php create mode 100644 src/Util/EllipticCurve/PublicKey.php create mode 100644 src/Util/EllipticCurve/Signature.php diff --git a/README.md b/README.md index c00ce57..ce3af92 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,9 @@ You need to map this table and these fields in you app_local.php config 'uniqueIdField' => 'status_id', // The field name that stores the unique message ID VARCHAR(100) 'statusField' => 'status', // The field name that stores the status of the email status VARCHAR(100) 'statusMessageField' => 'status_message', // The field name that stores the status messages TEXT + 'debug' => 'true', // write incoming requests to debug log + 'secure' => 'true', // enable SendGrid signed webhook security. You should enable this in production + 'verification_key' => '', // The verification key from SendGrid ], ``` @@ -170,7 +173,6 @@ You will need to login to your SendGrid Account and configure your domain and th The return url needs to be set to * https://YOUR DOMAIN/send-grid/webhook -Security needs to allow this action to be posted to TODO test with Auth plugin The CSRF protection middleware needs to allow posts to the webhooks controller in Application.php Remove the current CSRF protection middleware and replace it with the following. If you already have CSRF exceptions then add the Webhooks one @@ -192,9 +194,12 @@ Remove the current CSRF protection middleware and replace it with the following. ``` +If the authentication plugin (https://book.cakephp.org/authentication/3/en/index.html) is used for authentication the webhook action should work OK. If you have a different authentication method then you will need to add an exception for the webhook action. /send-grid/webhooks/index - +#### Webhook Signature Verification +SendGrid allows you to sign the webhook requests. This is a good idea in production. You will need to enable this in your SendGrid account and then set secure to true and add your verification key to your app_local.php config file. +https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features. Enable signed event webhook and follow the instructions to get the verification key. ## Reporting Issues diff --git a/composer.json b/composer.json index 856f1c5..153d5ec 100644 --- a/composer.json +++ b/composer.json @@ -30,7 +30,8 @@ }, "require": { "php": ">=7.2", - "cakephp/cakephp": "^4.0.0" + "cakephp/cakephp": "^5.0.0", + "starkbank/ecdsa": "^0.0.5" }, "require-dev": { "phpunit/phpunit": "^8.5 || ^9.3", diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index 28a38a6..820b2c6 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -6,16 +6,20 @@ //https://docs.sendgrid.com/for-developers/tracking-events/event //Config //https://app.sendgrid.com/settings/mail_settings +//Key Verification +//https://docs.sendgrid.com/for-developers/tracking-events/event#security-features /** * test - * * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' * - * TODO enable SendGrid security - * https://docs.sendgrid.com/for-developers/tracking-events/event#security-features - + * security test + * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Signature: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyV0a3iQnESb1wrXjheHcWTkNwdzrMc9LkCVKXadf3fjbWOfGZP8PRK17fg1SQHb6LAmpBkNRqkVqLl9Swpk2mg==' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' + * + * */ +//curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Timestamp: 1701474264'-H 'X-Twilio-Email-Event-Webhook-Signature: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyV0a3iQnESb1wrXjheHcWTkNwdzrMc9LkCVKXadf3fjbWOfGZP8PRK17fg1SQHb6LAmpBkNRqkVqLl9Swpk2mg==' -d '[{"email":"justin.g.harrison@gmail.com","event":"delivered","ip":"159.183.224.105","response":"250 2.0.0 OK 1701474014 bs32-20020a05620a472000b0077d85c98bb3si4555619qkb.194 - gsmtp","sg_event_id":"ZGVsaXZlcmVkLTAtMzkzNjk0MTYtZkdtQlRKdjNTbFM4TEhITU9ETTlvdy0w","sg_message_id":"fGmBTJv3SlS8LHHMODM9ow.filterdrecv-6bb8954444-svvb8-1-656A6EDB-1D.0","smtp-id":"","timestamp":1701474014,"tls":1},{"email":"justin.g.harrison@gmail.com","event":"processed","send_at":0,"sg_event_id":"cHJvY2Vzc2VkLTM5MzY5NDE2LWZHbUJUSnYzU2xTOExISE1PRE05b3ctMA","sg_message_id":"fGmBTJv3SlS8LHHMODM9ow.filterdrecv-6bb8954444-svvb8-1-656A6EDB-1D.0","smtp-id":"","timestamp":1701474011}]' + namespace SendGrid\Controller; @@ -24,13 +28,32 @@ use Cake\View\JsonView; use Cake\Core\Configure; use Cake\I18n\DateTime; - +use Cake\Event\EventInterface; use Cake\Log\Log; +use SendGrid\Util\EllipticCurve\Ecdsa; +use SendGrid\Util\EllipticCurve\PublicKey; +use SendGrid\Util\EllipticCurve\PrivateKey; +use SendGrid\Util\EllipticCurve\Signature; class WebHooksController extends AppController - { + const SIGNATURE = "X-Twilio-Email-Event-Webhook-Signature"; + const TIMESTAMP = "X-Twilio-Email-Event-Webhook-Timestamp"; + /** + * beforeFilter callback. + * + * @param \Cake\Event\EventInterface<\Cake\Controller\Controller> $event Event. + * @return \Cake\Http\Response|null|void + */ + public function beforeFilter(EventInterface $event) + { + // If the authentication plugin is loaded then open up the index action + if (isset($this->Authentication)) { + $this->Authentication->allowUnauthenticated(['index']); + } + } + public function viewClasses(): array { return [JsonView::class]; @@ -39,33 +62,63 @@ public function viewClasses(): array public function index() { - + $this->request->allowMethod(['post']); $this->viewBuilder()->setClassName("Json"); - $result = $this->request->getData(); - $this->set('result', $result); + $this->set('result', $result); $config = Configure::read('sendgridWebhook'); if (isset($config['debug']) && $config['debug'] == 'true') { Log::debug(json_encode($result)); + Log::debug(json_encode($this->request->getHeaders())); + } + + if (isset($config['secure']) && $config['secure'] == 'true') { + $this->request->getBody()->rewind(); + $payload = $this->request->getBody()->getContents(); + // Log::debug($payload); + + if (!isset($config['verification_key'])) { + if (isset($config['debug']) && $config['debug'] == 'true') { + Log::debug("Verfication Failed: Webhook Signature verification key not set in app_local.php"); + } + $this->set('error', "Invalid Signature"); + $this->viewBuilder()->setOption('serialize', "error"); + return; + } + + $publicKey = PublicKey::fromString($config['verification_key']); + + $timestampedPayload = $this->request->getHeaderLine($this::TIMESTAMP) . $payload; + $decodedSignature = Signature::fromBase64($this->request->getHeaderLine($this::SIGNATURE)); + + if (!Ecdsa::verify($timestampedPayload, $decodedSignature, $publicKey)) { + if (isset($config['debug']) && $config['debug'] == 'true') { + Log::debug("Verfication Failed: Webhook Signature does not verify against the verification key"); + } + $this->set('error', "Invalid Signature"); + $this->viewBuilder()->setOption('serialize', "error"); + return; + } } $emailTable = $this->getTableLocator()->get($config['tableClass']); $count = 0; foreach ($result as $event) { $message_id = explode(".", $event['sg_message_id'])[0]; - $email_record = $emailTable->find('all')->select(["id", + $email_record = $emailTable->find('all')->select([ + "id", $config['uniqueIdField'], $config['statusField'], $config['statusMessageField'], ])->where([$config['uniqueIdField'] => $message_id])->first(); if ($email_record) { $count++; - $email_record->set($config['statusMessageField'],$email_record->get($config['statusMessageField']). (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . - (isset($event['response'])? $event['response']:""). " " . - (isset($event['reason'])? $event['reason']:"
")); - $email_record->set($config['statusField'],$event['event']); + $email_record->set($config['statusMessageField'], $email_record->get($config['statusMessageField']) . (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " . + (isset($event['response']) ? $event['response'] : "") . " " . + (isset($event['reason']) ? $event['reason'] : "
")); + $email_record->set($config['statusField'], $event['event']); $emailTable->save($email_record); } } @@ -75,4 +128,5 @@ public function index() $this->set('OK', "OK"); $this->viewBuilder()->setOption('serialize', "OK"); } + } diff --git a/src/Util/EllipticCurve/Ecdsa.php b/src/Util/EllipticCurve/Ecdsa.php new file mode 100644 index 0000000..9168d7d --- /dev/null +++ b/src/Util/EllipticCurve/Ecdsa.php @@ -0,0 +1,26 @@ +openSslPrivateKey, OPENSSL_ALGO_SHA256); + return new Signature($signature); + } + + public static function verify ($message, $signature, $publicKey) { + $success = openssl_verify($message, $signature->toDer(), $publicKey->openSslPublicKey, OPENSSL_ALGO_SHA256); + if ($success == 1) { + return true; + } + return false; + } +} + +?> \ No newline at end of file diff --git a/src/Util/EllipticCurve/PrivateKey.php b/src/Util/EllipticCurve/PrivateKey.php new file mode 100644 index 0000000..4f39510 --- /dev/null +++ b/src/Util/EllipticCurve/PrivateKey.php @@ -0,0 +1,84 @@ + "sha256", + "private_key_bits" => 2048, + "private_key_type" => OPENSSL_KEYTYPE_EC, + "curve_name" => $curve + ); + + $response = openssl_pkey_new($config); + + openssl_pkey_export($response, $openSslPrivateKey, null, $config); + + $openSslPrivateKey = openssl_pkey_get_private($openSslPrivateKey); + } + + $this->openSslPrivateKey = $openSslPrivateKey; + } + + function publicKey() { + $openSslPublicKey = openssl_pkey_get_details($this->openSslPrivateKey)["key"]; + + return new PublicKey($openSslPublicKey); + } + + function toString () { + return base64_encode($this->toDer()); + } + + function toDer () { + $pem = $this->toPem(); + + $lines = array(); + foreach(explode("\n", $pem) as $value) { + if (substr($value, 0, 5) !== "-----") { + array_push($lines, $value); + } + } + + $pem_data = join("", $lines); + + return base64_decode($pem_data); + } + + function toPem () { + openssl_pkey_export($this->openSslPrivateKey, $out, null); + return $out; + } + + static function fromPem ($str) { + $rebuilt = array(); + foreach(explode("\n", $str) as $line) { + $line = trim($line); + if (strlen($line) > 1) { + array_push($rebuilt, $line); + } + }; + $rebuilt = join("\n", $rebuilt) . "\n"; + return new PrivateKey(null, openssl_get_privatekey($rebuilt)); + } + + static function fromDer ($str) { + $pem_data = base64_encode($str); + $pem = "-----BEGIN EC PRIVATE KEY-----\n" . substr($pem_data, 0, 64) . "\n" . substr($pem_data, 64, 64) . "\n" . substr($pem_data, 128, 64) . "\n-----END EC PRIVATE KEY-----\n"; + return new PrivateKey(null, openssl_get_privatekey($pem)); + } + + static function fromString ($str) { + return PrivateKey::fromDer(base64_decode($str)); + } + +} + +?> \ No newline at end of file diff --git a/src/Util/EllipticCurve/PublicKey.php b/src/Util/EllipticCurve/PublicKey.php new file mode 100644 index 0000000..a60a1ca --- /dev/null +++ b/src/Util/EllipticCurve/PublicKey.php @@ -0,0 +1,64 @@ +pem = $pem; + $this->openSslPublicKey = openssl_get_publickey($pem); + } + + function toString () { + return base64_encode($this->toDer()); + } + + function toDer () { + $pem = $this->toPem(); + + $lines = array(); + foreach(explode("\n", $pem) as $value) { + if (substr($value, 0, 5) !== "-----") { + array_push($lines, $value); + } + } + + $pem_data = join("", $lines); + + return base64_decode($pem_data); + } + + function toPem () { + return $this->pem; + } + + static function fromPem ($str) { + $rebuilt = array(); + foreach(explode("\n", $str) as $line) { + $line = trim($line); + if (strlen($line) > 1) { + array_push($rebuilt, $line); + } + }; + $rebuilt = join("\n", $rebuilt) . "\n"; + return new PublicKey($rebuilt); + } + + static function fromDer ($str) { + $pem_data = base64_encode($str); + $pem = "-----BEGIN PUBLIC KEY-----\n" . substr($pem_data, 0, 64) . "\n" . substr($pem_data, 64) . "\n-----END PUBLIC KEY-----\n"; + return new PublicKey($pem); + } + + static function fromString ($str) { + return PublicKey::fromDer(base64_decode($str)); + } + +} + +?> \ No newline at end of file diff --git a/src/Util/EllipticCurve/Signature.php b/src/Util/EllipticCurve/Signature.php new file mode 100644 index 0000000..0db15d5 --- /dev/null +++ b/src/Util/EllipticCurve/Signature.php @@ -0,0 +1,29 @@ +der = $der; + } + + function toDer () { + return $this->der; + } + + function toBase64 () { + return base64_encode($this->der); + } + + static function fromDer ($str) { + return new Signature($str); + } + + static function fromBase64 ($str) { + return new Signature(base64_decode($str)); + } +} + +?> \ No newline at end of file From 20d6f2a48c55d3ccdd97cd70addc886ad66cceef Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sat, 2 Dec 2023 06:00:17 +0000 Subject: [PATCH 10/14] Tidy up --- README.md | 2 +- src/Controller/WebhooksController.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ce3af92..d00e4cc 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Remove the current CSRF protection middleware and replace it with the following. If the authentication plugin (https://book.cakephp.org/authentication/3/en/index.html) is used for authentication the webhook action should work OK. If you have a different authentication method then you will need to add an exception for the webhook action. /send-grid/webhooks/index #### Webhook Signature Verification -SendGrid allows you to sign the webhook requests. This is a good idea in production. You will need to enable this in your SendGrid account and then set secure to true and add your verification key to your app_local.php config file. +SendGrid allows you to sign the webhook requests. This is a good idea in production to keep the webhook secure. You will need to enable this in your SendGrid account and then set secure to true and add your verification key to your app_local.php config file. https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features. Enable signed event webhook and follow the instructions to get the verification key. diff --git a/src/Controller/WebhooksController.php b/src/Controller/WebhooksController.php index 820b2c6..3567576 100644 --- a/src/Controller/WebhooksController.php +++ b/src/Controller/WebhooksController.php @@ -14,11 +14,10 @@ * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' * * security test - * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Signature: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyV0a3iQnESb1wrXjheHcWTkNwdzrMc9LkCVKXadf3fjbWOfGZP8PRK17fg1SQHb6LAmpBkNRqkVqLl9Swpk2mg==' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' + * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Signature: MFk..........2mg==' -d '[{"timestamp": 1700762652, "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]' * * */ -//curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Timestamp: 1701474264'-H 'X-Twilio-Email-Event-Webhook-Signature: MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyV0a3iQnESb1wrXjheHcWTkNwdzrMc9LkCVKXadf3fjbWOfGZP8PRK17fg1SQHb6LAmpBkNRqkVqLl9Swpk2mg==' -d '[{"email":"justin.g.harrison@gmail.com","event":"delivered","ip":"159.183.224.105","response":"250 2.0.0 OK 1701474014 bs32-20020a05620a472000b0077d85c98bb3si4555619qkb.194 - gsmtp","sg_event_id":"ZGVsaXZlcmVkLTAtMzkzNjk0MTYtZkdtQlRKdjNTbFM4TEhITU9ETTlvdy0w","sg_message_id":"fGmBTJv3SlS8LHHMODM9ow.filterdrecv-6bb8954444-svvb8-1-656A6EDB-1D.0","smtp-id":"","timestamp":1701474014,"tls":1},{"email":"justin.g.harrison@gmail.com","event":"processed","send_at":0,"sg_event_id":"cHJvY2Vzc2VkLTM5MzY5NDE2LWZHbUJUSnYzU2xTOExISE1PRE05b3ctMA","sg_message_id":"fGmBTJv3SlS8LHHMODM9ow.filterdrecv-6bb8954444-svvb8-1-656A6EDB-1D.0","smtp-id":"","timestamp":1701474011}]' namespace SendGrid\Controller; @@ -32,7 +31,6 @@ use Cake\Log\Log; use SendGrid\Util\EllipticCurve\Ecdsa; use SendGrid\Util\EllipticCurve\PublicKey; -use SendGrid\Util\EllipticCurve\PrivateKey; use SendGrid\Util\EllipticCurve\Signature; class WebHooksController extends AppController From eff7818e00df2115cfd8349cf58133f270cd13ee Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sat, 2 Dec 2023 20:17:04 +0000 Subject: [PATCH 11/14] Remove ESCD library --- composer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 153d5ec..cdcdd54 100644 --- a/composer.json +++ b/composer.json @@ -30,8 +30,7 @@ }, "require": { "php": ">=7.2", - "cakephp/cakephp": "^5.0.0", - "starkbank/ecdsa": "^0.0.5" + "cakephp/cakephp": "^5.0.0" }, "require-dev": { "phpunit/phpunit": "^8.5 || ^9.3", From 416c5a8bc329db3a0442638e90b9230557c7b859 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sat, 2 Dec 2023 20:32:02 +0000 Subject: [PATCH 12/14] PR#7 --- src/Mailer/Transport/SendGridTransport.php | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Mailer/Transport/SendGridTransport.php b/src/Mailer/Transport/SendGridTransport.php index a26e5c7..528dd91 100644 --- a/src/Mailer/Transport/SendGridTransport.php +++ b/src/Mailer/Transport/SendGridTransport.php @@ -1,4 +1,5 @@ 'id', // The field name that stores the unique message ID char(36) uid 'statusField' => 'status', // The field name that stores the status of the email char(50) status 'statusMessageField' => 'status_message', // The field name that stores the status message TEXT status_message - ] + ] ]; /** @@ -117,15 +118,8 @@ public function send(Message $message): array $this->_reqParams['subject'] = $message->getSubject(); - // $this->_reqParams['custom_args'] ="{'email_msg_id': '106'}"; - $emailFormat = $message->getEmailFormat(); - if (!empty($message->getBodyHtml())) { - $this->_reqParams['content'][] = (object)[ - 'type' => 'text/html', - 'value' => trim($message->getBodyHtml()), - ]; - } + if ($emailFormat == 'both' || $emailFormat == 'text') { $this->_reqParams['content'][] = (object)[ 'type' => 'text/plain', @@ -133,6 +127,13 @@ public function send(Message $message): array ]; } + if (!empty($message->getBodyHtml())) { + $this->_reqParams['content'][] = (object)[ + 'type' => 'text/html', + 'value' => trim($message->getBodyHtml()), + ]; + } + $this->_processHeaders($message); $this->_processAttachments($message); @@ -271,7 +272,7 @@ protected function _sendEmail() 'Authorization' => 'Bearer ' . $this->getConfig('apiKey'), ], ]; - + $response = $this->Client ->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options); From 6f0493d639127bccbac655102d411078014f5cc7 Mon Sep 17 00:00:00 2001 From: Jubbs Date: Sat, 2 Dec 2023 21:06:52 +0000 Subject: [PATCH 13/14] Get ReplyTo Working --- src/Mailer/Transport/SendGridTransport.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Mailer/Transport/SendGridTransport.php b/src/Mailer/Transport/SendGridTransport.php index 528dd91..dc59264 100644 --- a/src/Mailer/Transport/SendGridTransport.php +++ b/src/Mailer/Transport/SendGridTransport.php @@ -189,6 +189,16 @@ protected function _prepareEmailAddresses(Message $message) $this->_reqParams['from'] = (object)['email' => key($from)]; } + $replyTo = $message->getReplyTo(); + if (!empty($replyTo)) { + if (key($replyTo) != $replyTo[key($replyTo)]) { + $this->_reqParams['reply_to'] = (object)['email' => key($replyTo), 'name' => $replyTo[key($replyTo)]]; + + } else { + $this->_reqParams['reply_to'] = (object)['email' => key($replyTo)]; + } + } + $emails = []; foreach ($message->getTo() as $toEmail => $toName) { $emails['to'][] = [ @@ -273,9 +283,7 @@ protected function _sendEmail() ], ]; - - $response = $this->Client - ->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options); + $response = $this->Client->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options); $result = []; $result['apiResponse'] = $response->getJson(); From c9cec35421ab37875340bd71a25fbbbc34a356aa Mon Sep 17 00:00:00 2001 From: Jubbs Date: Tue, 5 Dec 2023 02:18:36 +0000 Subject: [PATCH 14/14] PHP version bump --- README.md | 2 +- composer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d00e4cc..261a753 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This branch is for use with CakePHP 5.0+. For CakePHP 4, please use cake-4.x bra This plugin has the following requirements: * CakePHP 5.0 or greater. -* PHP 7.2 or greater. +* PHP 8.1 or greater. ## Installation diff --git a/composer.json b/composer.json index cdcdd54..2e2feb9 100644 --- a/composer.json +++ b/composer.json @@ -29,7 +29,7 @@ "issues": "https://github.com/sprintcube/cakephp-sendgrid/issues" }, "require": { - "php": ">=7.2", + "php": ">=8.1", "cakephp/cakephp": "^5.0.0" }, "require-dev": {