Merge pull request #80 from GlobalNOC/1.7.0-dev

1.7.0 dev

CHANGES.md

@@ -1,3 +1,13 @@
+## GRNOC TSDS Services 1.7.0 --
+
+### Features:
+
+* Adds in local changes made to code as 'hot fixes' that were never added to the Git repo.
+* Adds in support for OL8 packaging
+* Allows writers to move failed writers to separate queue to keep pipeline clear and allow for
+examination later.
+* Adds default 2-day expiry for measurement decomissioning
+
 ## GRNOC TSDS Services 1.6.9 -- Wed Mar 16 2022
 
 ### Features:

MANIFEST

@@ -50,6 +50,7 @@ INSTALL.md
 lib/GRNOC/TSDS.pm
 lib/GRNOC/TSDS/AggregateDocument.pm
 lib/GRNOC/TSDS/AggregatePoint.pm
+lib/GRNOC/TSDS/Aggregate/Histogram.pm
 lib/GRNOC/TSDS/Constants.pm
 lib/GRNOC/TSDS/Constraints.pm
 lib/GRNOC/TSDS/DataDocument.pm

Makefile

@@ -0,0 +1,22 @@
+VERSION = 1.7.0
+NAME = grnoc-tsds-services
+
+rpm: dist
+	cd dist; tar -czvf $(NAME)-$(VERSION).tar.gz $(NAME)-$(VERSION)
+	rpmbuild -ta  dist/$(NAME)-$(VERSION).tar.gz
+
+clean:
+	rm -rf dist/$(NAME)-$(VERSION)/
+	rm -rf dist
+
+dist: clean venv
+	rm -rf dist/$(NAME)-$(VERSION)/
+	mkdir -p dist/$(NAME)-$(VERSION)/
+	cp -rv bin conf lib systemd www init.d CHANGES.md INSTALL.md venv $(NAME).spec dist/$(NAME)-$(VERSION)/
+	cd dist; tar -czvf $(NAME)-$(VERSION).tar.gz $(NAME)-$(VERSION)/
+
+test:
+	/usr/bin/perl -I lib/ t/TEST 1
+
+venv:
+	carton install --path=venv

bin/tsds-change-required-metadata.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Data::Dumper;
 use GRNOC::Config;
 use GRNOC::CLI;

bin/tsds-decom.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use GRNOC::Log;
 
 use GRNOC::TSDS::MeasurementDecommer;

bin/tsds_add_interface_addresses.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 # One time script to add interface_address metafield
 # Adds interface_address as an array with interface_address.type, interface_address.mask, and interface_address.value
 

bin/tsds_add_meta_optical.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 # Script to add metadata fields to optical data
 
 use GRNOC::Config;

bin/tsds_add_meta_power.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 # One time script to add metafields for power
 # Adds network, node_type, node_role, type, description, and alternate_intf fields
 

bin/tsds_adjust_metafields.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 # One time fix to make field adjustments
 # Deletes fields from tsds, drops the corresponding indexes and unsets them in the db
 # Following fields will be removed by this script - pop.owner, pop.hands_and_eyes, pop.pop_id, pop.role, ...

bin/tsds_expire.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use GRNOC::TSDS::Expire;
 use JSON::XS;
 

bin/tsds_find_unsharded.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Data::Dumper;
 use Getopt::Long;
 

bin/tsds_firehose.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use JSON::XS;
 use Net::AMQP::RabbitMQ;
 use Net::RabbitMQ::Management::API;

bin/tsds_fix_measurements.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use GRNOC::Config;
 
 use MongoDB;

bin/tsds_fix_start_date.pl

@@ -32,6 +32,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Data::Dumper;
 use GRNOC::Log;
 use Getopt::Long;

bin/tsds_install.pl

@@ -13,6 +13,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use constant DEFAULT_CONFIG_FILE => '/etc/grnoc/tsds/services/config.xml';
 
 use GRNOC::TSDS::Install;

bin/tsds_meta.pl

@@ -2,9 +2,12 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Data::Dumper;
 use Getopt::Long;
-use lib '../lib/';
+
 use GRNOC::TSDS::MetaStats;
 
 ### constants ###                                                                                                               

bin/tsds_search_indexer.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Getopt::Long;
 use Data::Dumper;
 use GRNOC::TSDS::Parser; # including b/c Marpa will be a PITA otherwise

bin/tsds_setup.pl

@@ -3,10 +3,12 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Marpa::R2;
 use Getopt::Long;
 use Data::Dumper;
-use lib '../lib/';
 use GRNOC::TSDS::Setup;
 
 ### constants ###

bin/tsds_upgrade.pl

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use Marpa::R2;
 use GRNOC::TSDS::Upgrade;
 use Getopt::Long;

bin/tsds_writer

@@ -3,6 +3,9 @@
 use strict;
 use warnings;
 
+use lib '/opt/grnoc/venv/grnoc-tsds-services/lib/perl5';
+use lib './venv/lib/perl5';
+
 use GRNOC::TSDS::Writer;
 
 use Getopt::Long;

cpanfile

@@ -0,0 +1,37 @@
+requires 'Cache::Memcached::Fast', '0.23';
+requires 'Class::Accessor', '0.34';
+requires 'Clone', '0.34';
+requires 'Data::Compare', '1.29';
+requires 'DateTime', '1.04';
+requires 'DateTime::Format::Strptime', '1.54';
+requires 'Env::C', '0.13';
+requires 'File::Slurp', '9999.19';
+requires 'Hash::Merge', '0.200';
+requires 'HTML::Parser', '3.71';
+requires 'JSON', '2.59';
+requires 'JSON::XS', '3.01';
+requires 'List::Flatten::Recursive', '0.103460';
+requires 'List::MoreUtils', '0.33';
+requires 'List::Util', '1.47';
+requires 'LockFile::Simple', '0.208';
+requires 'Marpa::R2', '3.000000';
+requires 'Math::Round', '0.06';
+requires 'MongoDB', 'v1.8.1';
+requires 'Moo', '2.003000';
+requires 'Net::AMQP::RabbitMQ', '2.30000';
+requires 'Number::Format', '1.73';
+requires 'Parallel::ForkManager', '1.18';
+requires 'Proc::Daemon', '0.19';
+requires 'Redis', '1.991';
+requires 'Redis::DistLock', '0.07';
+requires 'Sort::Versions', '1.5';
+requires 'Statistics::LineFit', '0.06';
+requires 'Test::Deep', '1.126';
+requires 'Time::HiRes', '1.9725';
+requires 'Tie::IxHash', '1.22';
+requires 'Try::Tiny', '0.30';
+requires 'Type::Tiny', '1.004002';
+requires 'Types::XSD::Lite', '0.005';
+requires 'WWW::Mechanize::PhantomJS', '0.14';
+requires 'XML::Writer', '0.623';
+requires 'XSLoader';