From 063c6b108a4db3a74765df71a83cc45169b92e2c Mon Sep 17 00:00:00 2001
From: Ian
Date: Mon, 11 Nov 2019 16:27:52 +0000
Subject: [PATCH 1/5] Add configuration schema validation
Including adding to monitoring checks.
---
config/local_info.xml | 2 +
config/local_info.xsd | 132 ++++++++++++++++
config/web_portal/menu.xsd | 36 +++++
config/web_portal/menu.xslt | 48 ++++++
htdocs/web_portal/GOCDB_monitor/index.php | 75 +++++----
htdocs/web_portal/GOCDB_monitor/tests.php | 143 ++++++++++++++----
.../GOCDB_monitor/validate_local_info_xml.php | 62 ++++++++
lib/Gocdb_Services/Config.php | 15 +-
tests/miscTests/ValidateLocalInfoXML.php | 22 +++
9 files changed, 479 insertions(+), 56 deletions(-)
create mode 100755 config/local_info.xsd
create mode 100644 config/web_portal/menu.xsd
create mode 100644 config/web_portal/menu.xslt
create mode 100644 htdocs/web_portal/GOCDB_monitor/validate_local_info_xml.php
create mode 100644 tests/miscTests/ValidateLocalInfoXML.php
diff --git a/config/local_info.xml b/config/local_info.xml
index 6115d7ed8..ee3cdb787 100755
--- a/config/local_info.xml
+++ b/config/local_info.xml
@@ -1,9 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/web_portal/menu.xsd b/config/web_portal/menu.xsd
new file mode 100644
index 000000000..7a62fad05
--- /dev/null
+++ b/config/web_portal/menu.xsd
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/web_portal/menu.xslt b/config/web_portal/menu.xslt
new file mode 100644
index 000000000..fc90c9619
--- /dev/null
+++ b/config/web_portal/menu.xslt
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+
+ ** DO NOT EDIT THIS FILE DIRECTLY. **
+ ** THIS FILE IS AUTO GENERATED. **
+ ** SEE menu.xslt FOR DETAILS. **
+
+ <xs:schema version="1.0"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ >
+
+ <xs:complexType name="validMenus">
+ <xs:all>
+
+
+ </xs:all>
+ </xs:complexType>
+</xs:schema>
+
+
+
+
+
+ <xs:element name="
+
+ " type="showType" minOccurs="0"/>
+
+
+
+
+
+
diff --git a/htdocs/web_portal/GOCDB_monitor/index.php b/htdocs/web_portal/GOCDB_monitor/index.php
index 1d1637582..0cb47da1d 100644
--- a/htdocs/web_portal/GOCDB_monitor/index.php
+++ b/htdocs/web_portal/GOCDB_monitor/index.php
@@ -5,36 +5,43 @@
URLs as defined by local_info.xml
";
-
-$piUrl = Factory::getConfigService()->GetPiUrl().get_testPiMethod();
-echo "PI URL is: ".$piUrl."
";
-
-$portalUrl = Factory::getConfigService()->GetPortalURL();
-echo "Portal URl is: ".$portalUrl."
";
-
-$baseUrl = Factory::getConfigService()->getServerBaseUrl();
-echo "Server Base URL is: ".$baseUrl."
";
+$config = Factory::getConfigService();
// GOCDB5 DB connection
$res = test_db_connection();
-$test_statuses["GOCDB5 DB connection"] = $res["status"];
-$test_messages["GOCDB5 DB connection"] = $res["message"];
+$test_statuses[TEST_1] = $res["status"];
+$test_messages[TEST_1] = $res["message"];
-// GOCDBPI v5
-$res = test_url($piUrl);
-$test_statuses["GOCDBPI_v5 availability"] = $res["status"];
-$test_messages["GOCDBPI_v5 availability"] = $res["message"];
+// GOCDB5 configuration
+$res = test_config($config);
+$test_statuses[TEST_4] = $res["status"];
+$test_messages[TEST_4] = $res["message"];
-// GOCDB5 web portal
-$res = test_url($portalUrl);
-$test_statuses["GOCDB5 central portal availability"] = $res["status"];
-$test_messages["GOCDB5 central portal availability"] = $res["message"];
+// Following tests depend on the config file being valid.
+if (strcasecmp($res["status"], OK) == 0) {
+ define_test_urls($config);
+ // GOCDBPI v5
+ $res = test_url(PI_URL);
+ $test_statuses[TEST_2] = $res["status"];
+ $test_messages[TEST_2] = $res["message"];
+ // GOCDB5 web portal
+ $res = test_url(SERVER_BASE_URL);
+ $test_statuses[TEST_3] = $res["status"];
+ $test_messages[TEST_3] = $res["message"];
-// DISPLAY RESULTS
+ // DISPLAY RESULTS
+ echo "URLs as defined by local_info.xml
";
+ echo "PI URL is: " . PI_URL . "
";
+ echo "Portal URl is: " . PORTAL_URL . "
";
+ echo "Server Base URL is: " . SERVER_BASE_URL . "
";
+} else {
+ echo "Unable to extract URL information due to configuration test failure.
";
+}
?>
Service status overview
+Other tests may have dependencies on the server configuration so may
+show ERROR or UNKNOWN if the configuration is invalid.
@@ -48,10 +55,10 @@
$status) {
echo("
");
- echo("$test | ");
+ echo("$test | ");
echo($disp[$status]);
- echo("{$test_messages[$test]} | ");
- echo("{$test_doc[$test]} | ");
+ echo("{$test_messages[$test]} | ");
+ echo("{$test_doc[$test]} | ");
echo("
");
}
?>
@@ -62,12 +69,24 @@
Other tests and check pages
- - GOCDB server ganglia page - Useful to see if there are memory or CPU problems
- - Status check - non vebose check of GOCDB service status. Just returns 'OK' if all the tests in \"service status overview\" are fine, 'WARNING' or 'ERROR' otherwise. Used for automatic tests
+ - GOCDB server
+ ganglia page - Useful to see if there are memory or CPU
+ problems
+ - Status check - a less vebose check of
+ GOCDB service status. Returns the single line 'All GOCDB tests
+ are looking good' if all tests run without error and 'GOCDB
+ Web Portal is unable to connect to the GOCDB back end database'
+ otherwise. Used for automated tests.
Further documentation
diff --git a/htdocs/web_portal/GOCDB_monitor/tests.php b/htdocs/web_portal/GOCDB_monitor/tests.php
index ffb2c3aec..a78d395c9 100644
--- a/htdocs/web_portal/GOCDB_monitor/tests.php
+++ b/htdocs/web_portal/GOCDB_monitor/tests.php
@@ -1,5 +1,24 @@
setLocalInfoFileLocation(...)
+
\Factory::getConfigService()->setLocalInfoOverride($_SERVER['SERVER_NAME']);
$test_statuses = array(
- "GOCDB5 DB connection" => "unknown",
- "GOCDBPI_v5 availability" => "unknown",
- "GOCDB5 central portal availability" => "unknown"
+ TEST_1 => UKN,
+ TEST_2 => UKN,
+ TEST_3 => UKN,
+ TEST_4 => UKN
);
$test_desc = array(
- "GOCDB5 DB connection" =>
- "Connect to GOCDB5 (RAL/master instance) from this machine using EntityManager->getConnection()->connect()",
- "GOCDBPI_v5 availability" =>
- "Retrieve https://goc.egi.eu/gocdbpi/?method=get_site_list&sitename=RAL-LCG2 using PHP CURL",
- "GOCDB5 central portal availability" => "N/A",
+ TEST_1 =>
+ "Connect to GOCDB5 (RAL/master instance) from this " .
+ "machine using EntityManager->getConnection()->connect()",
+ TEST_2 =>
+ "Retrieve https://goc.egi.eu/gocdbpi/?" .
+ "method=get_site_list&sitename=RAL-LCG2 using PHP CURL",
+ TEST_3 =>
+ "N/A",
+ TEST_4 =>
+ "Server XML configuration validation."
);
$test_doc = array(
- "GOCDB5 DB connection" =>
- "documentation/recipe",
- "GOCDBPI_v5 availability" =>
- "documentation/recipe",
- "GOCDB5 central portal availability" =>
- "documentation/recipe"
+ TEST_1 =>
+ "" .
+ "documentation/recipe",
+ TEST_2 =>
+ "" .
+ "documentation/recipe",
+ TEST_3 =>
+ "" .
+ "documentation/recipe",
+ TEST_4 =>
+ "Contact GOCDB service managers." .
+ "
Other tests have dependencies on the server configuration " .
+ "
so may show errors if the configuration is invalid.
"
);
$test_messages = array(
- "GOCDB5 DB connection" => "no information",
- "GOCDBPI_v5 availability" => "no information",
- "GOCDB5 central portal availability" => "no information"
+ TEST_1 => UKNMSG,
+ TEST_2 => UKNMSG,
+ TEST_3 => UKNMSG,
+ TEST_4 => UKNMSG
);
$disp = array(
@@ -50,6 +87,47 @@
"ok" => "OK | ",
);
+// Run the tests but return nothing but a count of passes and failures
+function get_test_counts($config)
+{
+ $res[1] = test_db_connection();
+ $res[4] = test_config($config);
+
+ if ($res[4]["status"] != "error") {
+ // Only define test URLs if the config is valid
+ define_test_urls($config);
+
+ $res[2] = test_url(PI_URL);
+ $res[3] = test_url(SERVER_BASE_URL);
+ }
+
+ $counts = array("ok" => 0,
+ "warn" => 0,
+ "error" => 0
+ );
+
+ foreach ($res as $r) {
+ $counts[$r["status"]]++;
+ }
+
+ return $counts;
+}
+
+// Define url constants for testing.
+// Note: Should only be called if test_config is successful
+function define_test_urls(\org\gocdb\services\config $config)
+{
+
+ list($serverBaseURL, $webPortalURL, $piURL) = $config->getURLs();
+
+ define("PI_URL", $piURL . get_testPiMethod());
+ define("PORTAL_URL", $webPortalURL);
+ define("SERVER_BASE_URL", $serverBaseURL);
+
+ //define("SERVER_SSLCERT", "/etc/grid-security/hostcert.pem");
+ //define("SERVER_SSLKEY", "/etc/pki/tls/private/hostkey.pem");
+}
+
// Test the connection to the database using Doctrine
function test_db_connection()
{
@@ -57,11 +135,11 @@ function test_db_connection()
try {
$entityManager = Factory::getNewEntityManager();
$entityManager->getConnection()->connect();
- $retval["status"] = "ok";
- $retval["message"] = "everything is well";
+ $retval["status"] = OK;
+ $retval["message"] = OKMSG;
} catch (\Exception $e) {
$message = $e->getMessage();
- $retval["status"] = "error";
+ $retval["status"] = NOK;
$retval["message"] = "$message";
}
@@ -73,11 +151,11 @@ function test_url($url)
$retval = [];
try {
get_https2($url);
- $retval["status"] = "ok";
- $retval["message"] = "everything is well";
+ $retval["status"] = OK;
+ $retval["message"] = OKMSG;
} catch (Exception $exception) {
$message = $exception->getMessage();
- $retval["status"] = "error";
+ $retval["status"] = NOK;
$retval["message"] = "$message";
}
return $retval;
@@ -180,3 +258,16 @@ function run_tests(&$message)
return $errorCount;
}
+function test_config($config)
+{
+ $retval = [];
+ try {
+ validate_local_info_xml($config->getLocalInfoFileLocation());
+ $retval["status"] = OK;
+ $retval["message"] = OKMSG;
+ } catch (Exception $exception) {
+ $retval["status"] = NOK;
+ $retval["message"] = $exception->getMessage();
+ }
+ return $retval;
+}
diff --git a/htdocs/web_portal/GOCDB_monitor/validate_local_info_xml.php b/htdocs/web_portal/GOCDB_monitor/validate_local_info_xml.php
new file mode 100644
index 000000000..9e8b98259
--- /dev/null
+++ b/htdocs/web_portal/GOCDB_monitor/validate_local_info_xml.php
@@ -0,0 +1,62 @@
+\n";
+
+ switch ($error->level) {
+ case LIBXML_ERR_WARNING:$return .= "Warning $error->code: ";
+ break;
+ case LIBXML_ERR_ERROR:$return .= "Error $error->code: ";
+ break;
+ case LIBXML_ERR_FATAL:$return .= "Fatal Error $error->code: ";
+ break;
+ }
+ $return .= trim($error->message);
+
+ if ($error->file) {
+ $return .= " in $error->file";
+ }
+ $return .= " on line $error->line\n";
+ return $return;
+}
+/**
+ * Loop over all errors printing a message for each
+ */
+function libxml_display_errors()
+{
+ $message = "";
+
+ $errors = libxml_get_errors();
+ foreach ($errors as $error) {
+ $message .= libxml_display_error($error);
+ }
+ libxml_clear_errors();
+
+ return $message;
+}
+/**
+ * Check that the given xml matches its schema.
+ * The schema .xsd file must have the same name prefix and
+ * be in the same dir as the input .xml file
+ */
+function validate_local_info_xml ($path)
+{
+ // Enable user error handling
+ libxml_use_internal_errors(true);
+
+ $xml = new DOMDocument();
+
+ $xml->load($path);
+
+ $xsd = preg_replace( '/\.xml$/', '.xsd', $path);
+
+ if (!$xml->schemaValidate($xsd)) {
+ throw new Exception (libxml_display_errors());
+ }
+
+ return;
+}
+?>
diff --git a/lib/Gocdb_Services/Config.php b/lib/Gocdb_Services/Config.php
index 80c9de109..59576f417 100644
--- a/lib/Gocdb_Services/Config.php
+++ b/lib/Gocdb_Services/Config.php
@@ -157,7 +157,6 @@ private function readLocalInfoXML($path, $url = null)
if (!$base) {
$this->throwXmlErrors('Failed to load configuration file ' . $path);
}
-
// Search the input XML for a 'local_info' section that does NOT have a url attribute
// specified. This is the default spec.
$unqualified = $base->xpath("//local_info[not(@url)]");
@@ -401,8 +400,9 @@ public function GetPortalURL()
*/
public function isRestrictPDByRole($forceStrict = false)
{
- if ($forceStrict === true)
+ if ($forceStrict === true) {
return true;
+ }
$localInfo = $this->GetLocalInfoXML();
$value = $localInfo->restrict_personal_data;
@@ -432,7 +432,18 @@ public function getServerBaseUrl()
$url = $localInfo->server_base_url;
return strval($url);
}
+ /**
+ * Convenience function to return the 3 configuration URLs
+ */
+ public function getURLs()
+ {
+ $localInfo = $this->GetLocalInfoXML();
+ $serverBaseUrl = $localInfo->server_base_url;
+ $webPortalUrl = $localInfo->web_portal_url;
+ $piUrl = $localInfo->pi_url;
+ return array($serverBaseUrl, $webPortalUrl, $piUrl);
+ }
/**
* The write API documentation URL as recorded in local_info.xml.
* This URL is given to users of the write API in error messages
diff --git a/tests/miscTests/ValidateLocalInfoXML.php b/tests/miscTests/ValidateLocalInfoXML.php
new file mode 100644
index 000000000..86e81d96d
--- /dev/null
+++ b/tests/miscTests/ValidateLocalInfoXML.php
@@ -0,0 +1,22 @@
+load('../../config/local_info.xml');
+
+if (!$xml->schemaValidate('../../config/local_info.xsd')) {
+ print 'Errors found.
';
+ print libxml_display_errors();
+} else {
+ print 'Validated. No errors found.
';
+}
From 5aaf1437edceea060be12cbd7d0e36e3cd729deb Mon Sep 17 00:00:00 2001
From: Ian Neilson
Date: Mon, 27 Mar 2023 16:19:12 +0000
Subject: [PATCH 2/5] Include menu validation README
---
config/web_portal/README.md | 53 +++++++++++++++++++++++++++++++++++++
1 file changed, 53 insertions(+)
create mode 100644 config/web_portal/README.md
diff --git a/config/web_portal/README.md b/config/web_portal/README.md
new file mode 100644
index 000000000..a639ad5a0
--- /dev/null
+++ b/config/web_portal/README.md
@@ -0,0 +1,53 @@
+## What *config/web_portal/menu.xsd* is for.
+
+This file is the schema file for the definiton of GOCDB's left-hand column menu, used to check that only valid menu names are included in the GOCDB main configuration file config/local_info.xml. The menu.xsd file is built automatically, **do not edit it** - see the next question.
+
+GOCDB uses an XML definition (config/web_portal/menu.xml) to programatically draw the left-hand menu items on the user
+interface. Which menu items are drawn is selected via The \