Merge pull request #1451 from GlobalNOC/2.0.16-dev

2.0.16 dev

.env.example

@@ -64,7 +64,14 @@ OESS_LOCAL_ASN=55038
 #   nso+vpn-mpls: runs both vpn-mpls and nso southbounds simultaneously
 #
 OESS_NETWORK_TYPE=vpn-mpls
-
+#
+# Email used to receive administrative notifications
+#
+# OESS_ADMIN_EMAIL=root@localhost
+#
+# Email address used to receive endpoint approval requests
+#
+# OESS_APPROVAL_EMAIL=root@localhost
 
 # Authentication details for NETCONF connection to network devices. Is
 # required for vpn-mpls, evpn-vxlan, and nso+vpn-mpls

Dockerfile.dev

@@ -8,7 +8,7 @@ RUN yum makecache
 RUN yum -y install epel-release
 RUN yum -y install perl httpd mariadb-server rabbitmq-server
 RUN yum -y install perl-Carp-Always perl-Test-Deep perl-Test-Exception perl-Test-Pod perl-Test-Pod-Coverage perl-Devel-Cover perl-Net-AMQP-RabbitMQ perl-LWP-Protocol-https perl-AnyEvent-HTTP perl-GRNOC-CLI perl-Text-CSV perl-Net-IP
-RUN yum -y install perl-OESS oess-core oess-frontend yui2
+RUN yum -y install perl-OESS oess-core oess-frontend yui2 postfix
 
 COPY app/mpls/mpls_discovery.pl /usr/bin/mpls_discovery.pl
 COPY app/mpls/mpls_fwdctl.pl /usr/bin/mpls_fwdctl.pl
@@ -24,6 +24,7 @@ COPY perl-lib/OESS/t/conf/interface-speed-config.xml /etc/oess/interface-speed-c
 COPY perl-lib/OESS/t/conf/logging.conf /etc/oess/logging.conf
 COPY perl-lib/OESS/t/conf/passwd.xml /etc/oess/.passwd.xml
 COPY perl-lib/OESS/t/conf/integration.sql /etc/oess/integration.sql
+COPY perl-lib/OESS/etc/* /usr/share/oess-core/
 
 COPY perl-lib/OESS/entrypoint.dev.sh /entrypoint.sh
 RUN chmod 777 /entrypoint.sh

Makefile

@@ -1,6 +1,7 @@
-OESS_VERSION=2.0.15
+OESS_VERSION=2.0.16
 OESS_NETWORK=oess
 TEST_FILES=
+MAILHOST=localhost
 
 include .env
 
@@ -21,12 +22,6 @@ test:
 	docker run --rm -it -e OESS_TEST_FILES="$(TEST_FILES)" --volume ${PWD}/perl-lib/OESS:/perl-lib/OESS oess-test
 
 # For single container builds. Should only be used for testing.
-container:
-	docker build -f Dockerfile.dev --tag oess:${OESS_VERSION} .
-
-clean-container:
-	docker build --no-cache -f Dockerfile.dev --tag oess:${OESS_VERSION} .
-
 # To attach OESS to an existing docker network:
 # --network ${OESS_NETWORK}
 #
@@ -38,6 +33,7 @@ clean-container:
 # --cap-add=NET_ADMIN
 #
 dev:
+	docker build --no-cache -f Dockerfile.dev --tag oess:${OESS_VERSION} .
 	docker run -it \
 	--name oess-dev \
 	--rm \
@@ -48,8 +44,11 @@ dev:
 	--mount type=bind,src=${PWD}/perl-lib/OESS/lib/OESS,dst=/usr/share/perl5/vendor_perl/OESS \
 	--mount type=bind,src=${PWD}/frontend,dst=/usr/share/oess-frontend \
 	--mount type=bind,src=${PWD}/perl-lib/OESS/share,dst=/usr/share/doc/perl-OESS-${OESS_VERSION}/share \
+	--mount type=bind,src=${PWD}/perl-lib/OESS/t,dst=/usr/share/doc/perl-OESS-${OESS_VERSION}/t \
 	--mount type=bind,src=${PWD}/app,dst=/usr/bin/oess \
 	--mount type=bind,src=${PWD}/docs,dst=/docs \
+	--mount type=bind,src=${PWD}/perl-lib/OESS/etc,dst=/usr/share/oess-core \
+	--hostname ${MAILHOST} \
 	oess:${OESS_VERSION} /bin/bash
 
 documentation:

app/Makefile

@@ -1,5 +1,5 @@
 NAME= oess-core
-VERSION = 2.0.15
+VERSION = 2.0.16
 
 rpm:	dist
 	rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz

app/oess-core.spec

@@ -1,5 +1,5 @@
 Name:		oess-core
-Version:	2.0.15
+Version:	2.0.16
 Release:	1%{?dist}
 Summary:	The core OESS service providers
 
@@ -20,7 +20,7 @@ Requires: /bin/bash
 Requires: /usr/bin/perl
 Requires: perl(base), perl(constant), perl(strict), perl(warnings)
 
-Requires: perl-OESS >= 2.0.15
+Requires: perl-OESS >= 2.0.16
 
 Requires: perl(AnyEvent), perl(AnyEvent::DBus), perl(AnyEvent::RabbitMQ)
 Requires: perl(CPAN), perl(CPAN::Shell)

app/oess_pull_azure_interfaces.pl

@@ -37,48 +37,65 @@ sub main {
     die $azure_err if defined $azure_err;
 
     foreach my $ep (@$endpoints) {
+        if (!defined $ep->cloud_connection_id) {
+            warn "Endpoint with missing cloud_connection_id found: " . Dumper($ep->to_hash);
+            next;
+        }
         my $conn = $conns->{$ep->cloud_connection_id};
-        my $remote_peers = $syncer->get_peering_addresses_from_azure($conn, $ep->cloud_interconnect_id);
-
-        $ep->load_peers;
-        my $local_peers = $ep->peers;
-
-        my $i = 0;
-        while ($i < @$remote_peers) {
-            my $peer;
-            if ($i+1 > @$local_peers) {
-                # While more remote_peers than local_peers create one
-                my $peer = new OESS::Peer(
-                    db => $db,
-                    model => {
-                        local_ip   => $remote_peers->[$i]->{local_ip},
-                        peer_asn   => $remote_peers->[$i]->{remote_asn},
-                        peer_ip    => $remote_peers->[$i]->{remote_ip},
-                        status     => 'up',
-                        ip_version => $remote_peers->[$i]->{ip_version}
-                    }
-                );
-                $peer->create(vrf_ep_id => $ep->vrf_endpoint_id);
-            } else {
-                $local_peers->[$i]->local_ip($remote_peers->[$i]->{local_ip});
-                $local_peers->[$i]->peer_asn($remote_peers->[$i]->{remote_asn});
-                $local_peers->[$i]->peer_ip($remote_peers->[$i]->{remote_ip});
-                $local_peers->[$i]->ip_version($remote_peers->[$i]->{ip_version});
-                $local_peers->[$i]->update;
-            }
-
-            $i++;
+        if (defined $conn->{error} || defined $conn->{Error}) {
+            warn "Error fetching connection details from azure for " . $ep->cloud_connection_id;
+            warn Dumper($conn);
+            next;
         }
 
-        while ($i < @$local_peers) {
-            # While more local_peers than remote_peers remove one
-            $local_peers->[$i]->decom;
-            $i++;
+        # Only sync peers for l3 connections
+        if (defined $ep->vrf_endpoint_id) {
+            my $remote_peers = $syncer->get_peering_addresses_from_azure($conn, $ep->cloud_interconnect_id);
+
+            $ep->load_peers;
+            my $local_peers = $ep->peers;
+
+            my $i = 0;
+            while ($i < @$remote_peers) {
+                my $peer;
+                if ($i+1 > @$local_peers) {
+                    # While more remote_peers than local_peers create one
+                    my $peer = new OESS::Peer(
+                        db => $db,
+                        model => {
+                            local_ip   => $remote_peers->[$i]->{local_ip},
+                            peer_asn   => $remote_peers->[$i]->{remote_asn},
+                            peer_ip    => $remote_peers->[$i]->{remote_ip},
+                            status     => 'up',
+                            ip_version => $remote_peers->[$i]->{ip_version}
+                        }
+                    );
+                    $peer->create(vrf_ep_id => $ep->vrf_endpoint_id);
+                } else {
+                    $local_peers->[$i]->local_ip($remote_peers->[$i]->{local_ip});
+                    $local_peers->[$i]->peer_asn($remote_peers->[$i]->{remote_asn});
+                    $local_peers->[$i]->peer_ip($remote_peers->[$i]->{remote_ip});
+                    $local_peers->[$i]->ip_version($remote_peers->[$i]->{ip_version});
+                    $local_peers->[$i]->update;
+                }
+
+                $i++;
+            }
+
+            while ($i < @$local_peers) {
+                # While more local_peers than remote_peers remove one
+                $local_peers->[$i]->decom;
+                $i++;
+            }
         }
 
         if ($ep->bandwidth != $conn->{properties}->{bandwidthInMbps}) {
+            if (defined $ep->circuit_ep_id) {
+                warn "Setting bandwidth of $ep->{bandwidth} on l2ep $ep->{circuit_ep_id} for circuit $ep->{circuit_id}";
+            }
             $ep->bandwidth($conn->{properties}->{bandwidthInMbps});
-            $ep->update_db;
+            my $bandwidth_err = $ep->update_db;
+            warn "Error setting bandwidth: $bandwidth_err" if defined $bandwidth_err;
         }
     }
 }

app/oess_pull_gcp_interfaces.pl

@@ -24,6 +24,7 @@ sub main{
 
     $client->set_url($config->base_url() . "/services/vrf.cgi");
     my $vrfs = $client->get_vrfs(workgroup_id => $workgroup_id);
+    die $client->get_error if !defined $vrfs;
 
     my $gcp_ints = get_gcp_virtual_interface($config);
 
@@ -46,6 +47,7 @@ sub find_gcp_workgroup{
 
     $client->set_url($config->base_url() . "/services/user.cgi");
     my $res = $client->get_current();
+    die $client->get_error if !defined $res;
 
     my $workgroups = $res->{'results'}->[0]->{'workgroups'};
     foreach my $wg (@$workgroups){
@@ -63,7 +65,7 @@ sub connect_to_ws {
     my $creds = $config->get_cloud_config();
 
     my $client = GRNOC::WebService::Client->new(
-        url     => $config->base_url() . "services/vrf.cgi",
+        url     => $config->base_url() . "/services/vrf.cgi",
         uid     => $creds->{'user'},
         passwd  => $creds->{'password'},
         realm   => $creds->{'realm'},
@@ -116,8 +118,13 @@ sub compare_and_update_vrfs{
 
             warn "get_vrf_gcp_details";
             my $peering = get_vrf_gcp_details(gcp_ints => $gcp_ints, cloud_connection_id => $connection_id);
-	    next if(!defined($peering) || $peering eq '');
+            next if(!defined($peering) || $peering eq '');
+
             my $update = update_endpoint_values($endpoint->{'peers'}->[0], $peering);
+            if ($endpoint->{mtu} != $peering->{mtu}) {
+                $update = 1;
+                $endpoint->{mtu} = $peering->{mtu}
+            }
             if($update){
                 update_oess_vrf($vrf,$client);
             }else{
@@ -156,6 +163,7 @@ sub update_oess_vrf{
                                                   inner_tag => $ep->{'inner_tag'},
                                                   vrf_endpoint_id => $ep->{'vrf_endpoint_id'},
                                                   circuit_ep_id => $ep->{'circuit_ep_id'},
+                                                  cloud_gateway_type => $ep->{'mtu'},
                                                   peers => \@peerings}));
     }
 

docs/_data/api/admin-admin/get_endpoints_in_review.json

@@ -0,0 +1 @@
+{"method_deprecated":0,"output_type":"application/json","input_params":{},"name":"get_endpoints_in_review","description":"Returns a list of endpoints that require admin review.","default_input_validators":[],"expires":"-1d"}

docs/_data/api/admin-admin/review_endpoint.json

@@ -0,0 +1 @@
+{"method_deprecated":0,"output_type":"application/json","input_params":{"approve":{"allow_null":0,"name":"approve","ignore_default_input_validators":0,"min_length":null,"description":"Set to 1 for approval or 0 for denial.","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter approve only accepts integer numbers.","required":1,"multiple":0,"max_length":null},"vrf_ep_id":{"allow_null":1,"name":"vrf_ep_id","ignore_default_input_validators":0,"min_length":null,"description":"VRF ID of the Endpoint","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter vrf_ep_id only accepts integer numbers.","required":0,"multiple":0,"max_length":null},"circuit_ep_id":{"allow_null":1,"name":"circuit_ep_id","ignore_default_input_validators":0,"min_length":null,"description":"Circuit ID of the Endpoint","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter circuit_ep_id only accepts integer numbers.","required":0,"multiple":0,"max_length":null},"skip_cloud_provisioning":{"allow_null":1,"name":"skip_cloud_provisioning","default":0,"ignore_default_input_validators":0,"min_length":null,"description":"If set to 1 cloud provider configurations will not be performed.","pattern":"^(\\-?\\d+)$","input_validators":[],"validation_error_text":"Parameter skip_cloud_provisioning only accepts integer numbers.","required":0,"multiple":0,"max_length":null}},"name":"review_endpoint","description":"Approve or deny an endpoint in review.","default_input_validators":[],"expires":"-1d"}

docs/_data/api/interface/get_options.json

@@ -0,0 +1 @@
+{"method_deprecated":0,"output_type":"application/json","input_params":{},"name":"get_options","description":"get_options returns provisioning options for interfaces","default_input_validators":[],"expires":"-1d"}

frontend/Makefile

@@ -1,5 +1,5 @@
 NAME= oess-frontend
-VERSION = 2.0.15
+VERSION = 2.0.16
 
 rpm:	dist
 	rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz

frontend/oess-frontend.spec

@@ -1,5 +1,5 @@
 Name:		oess-frontend
-Version:	2.0.15
+Version:	2.0.16
 Release:	1%{?dist}
 Summary:	The OESS webservices and user interface
 
@@ -14,14 +14,14 @@ BuildRequires: python >= 2.6, python-libs >= 2.6
 BuildRequires: python-simplejson
 BuildRequires: rh-nodejs8-nodejs == 8.11.4
 
-Requires: oess-core >= 2.0.15
+Requires: oess-core >= 2.0.16
 Requires: yui
 Requires: httpd, mod_ssl
 Requires: nddi-tiles
 Requires: perl-Crypt-SSLeay
 Requires: xmlsec1, xmlsec1-openssl
 
-Requires: perl-OESS >= 2.0.15
+Requires: perl-OESS >= 2.0.16
 
 Requires: perl(strict), perl(warnings)
 Requires: perl(AnyEvent)