Update dependencies and syntax

Dockerfile

@@ -96,7 +96,8 @@ RUN curl -L https://jitpack.io/com/github/KNMI/adaguc-services/1.2.4/adaguc-serv
 # Install newer numpy
     curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
     python get-pip.py && \
-    pip install numpy netcdf4 six lxml && \
+    pip install numpy netcdf4 six python3-lxml && \
+    # pip install numpy netcdf4 six lxml && \
 # Run adaguc-server functional and regression tests
     bash runtests.sh && \
 # Set same uid as vivid

Dockerfile.NetCDF-4.6.1

@@ -10,7 +10,8 @@ RUN yum update -y && yum install -y \
     epel-release deltarpm
 
 RUN yum install -y \
-    python-lxml \
+    # python-lxml \
+    python3-lxml \
     cairo \
     curl \
     gd \
@@ -78,7 +79,8 @@ RUN yum update -y && yum install -y \
     epel-release deltarpm
 
 RUN yum install -y \
-    python-lxml \
+    python3-lxml \
+    # python-lxml \
     cairo \
     curl \
     gd \

data/python/adaguc/ADAGUCFeatureFunctions.py

@@ -2,26 +2,25 @@
 
 import os
 import netCDF4
-import urllib2
+import urllib.request
 import warnings
 import numpy
-from sets import Set
 import logging
-import ADAGUCWCS
+from .ADAGUCWCS import ADAGUCWCS
 from netCDF4 import num2date 
 from datetime import date
 
 #logging.basicConfig(level=logging.DEBUG)
 
 def defaultCallback(message,percentage):
-  print "defaultCallback:: "+message+" "+str(percentage)
+  print("defaultCallback:: "+message+" "+str(percentage))
 
 def printfield(featuredata):
   for y in range(0,numpy.shape(featuredata)[0]):
     mstr = ""
     for x in range(0,numpy.shape(featuredata)[1]):
       mstr = "%s%0.2d" %(mstr, (featuredata[y][x]))
-    print mstr
+    print(mstr)
 
 def ADAGUCFeatureCombineNuts( featureNCFile,dataNCFile,bbox= "-40,20,60,85",variable = None, time= None,width=300,height=300,crs="EPSG:4326",outncrasterfile="/tmp/stat.nc",outncpointfile="/tmp/statpoints.nc", outcsvfile="/tmp/stat.csv",callback=defaultCallback, tmpFolderPath = "/tmp", homeFolderPath="/tmp"):
 

data/python/adaguc/ADAGUCWCS.py

@@ -1,5 +1,5 @@
 import urllib
-from StringIO import StringIO
+from io import StringIO
 import isodate 
 import time
 import netCDF4
@@ -10,7 +10,7 @@
 import json
 import os
 import shutil
-from mkdir_p import *
+import pathlib
 import zipfile
 from xml.sax.saxutils import escape
 from xml.dom import minidom
@@ -170,7 +170,7 @@ def iteratewcs(TIME = "",BBOX = "-180,-90,180,90",CRS = "EPSG:4326",RESX=None,RE
   CALLBACK("Starting iterateWCS",1)
   tmpdir = TMP+"/iteratewcstmp";
   shutil.rmtree(tmpdir, ignore_errors=True)
-  mkdir_p(tmpdir);
+  pathlib.Path(tmpdir).mkdir(parents=True, exist_ok=True)
 
   """ Determine which dates to do based on describe coverage call"""
   CALLBACK("Starting WCS DescribeCoverage request",1)
@@ -274,7 +274,7 @@ def makeGetCoverage(single_date, env = {}):
       raise ValueError ("Succesfully completed WCS GetCoverage, but no data found for "+url+"\n"+adaguclog+"\n");
 
     if(CALLBACK==None):
-      print str(int((float(datesdone)/numdatestodo)*90.))
+      print(str(int((float(datesdone)/numdatestodo)*90.)))
     else:
       CALLBACK(messagetime,((float(datesdone)/float(numdatestodo))*90.))
     return filetogenerate
@@ -297,7 +297,7 @@ def monitor2(line):
     try:
       data = json.loads(line)
       if(CALLBACK == None):
-        print float(data["percentage"])*(1./10)+90
+        print(float(data["percentage"])*(1./10)+90)
       else:
         CALLBACK(data["message"],float(data["percentage"])*(1./10)+90)
     except:

dataset-scanner.Dockerfile

@@ -84,7 +84,8 @@ RUN curl -L https://jitpack.io/com/github/KNMI/adaguc-services/1.2.0/adaguc-serv
 # Install newer numpy
     curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
     python get-pip.py && \
-    pip install numpy netcdf4 six lxml && \
+    pip install numpy netcdf4 six python3-lxml && \
+    # pip install numpy netcdf4 six lxml && \
 # Run adaguc-server functional and regression tests
     bash runtests.sh && \
 # Set same uid as vivid

tests/AdagucTests/AdagucTestTools.py

@@ -1,5 +1,5 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -25,9 +25,9 @@ def getLogFile(self):
 
   def printLogFile(self):
       ADAGUC_LOGFILE = os.environ['ADAGUC_LOGFILE']
-      print "\n=== START ADAGUC LOGS ==="
-      print self.getLogFile()
-      print "=== END ADAGUC LOGS ==="
+      print("\n=== START ADAGUC LOGS ===")
+      print(self.getLogFile())
+      print("=== END ADAGUC LOGS ===")
 
   def runADAGUCServer(self, url = None, env = [], path = None, args = None, isCGI = True, showLogOnError = True, showLog = False):
 
@@ -67,16 +67,16 @@ def runADAGUCServer(self, url = None, env = [], path = None, args = None, isCGI
       print("\n\n--- START ADAGUC DEBUG INFO ---")
       print("Adaguc-server has non zero exit status %d " % status)
       if isCGI == False:
-        print filetogenerate.getvalue()
+        print(filetogenerate.getvalue())
       else:
         self.printLogFile();
       if status == -9: print("Process: Killed")
       if status == -11: print("Process: Segmentation Fault ")
 
       if len(headers)!=0: 
-        print "=== START ADAGUC HTTP HEADER ==="
-        print headers
-        print "=== END ADAGUC HTTP HEADER ==="
+        print("=== START ADAGUC HTTP HEADER ===")
+        print(headers)
+        print("=== END ADAGUC HTTP HEADER ===")
       else: 
         print("Process: No HTTP Headers written")
 

tests/AdagucTests/TestADAGUCFeatureFunctions.py

@@ -1,5 +1,5 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 from adaguc.ADAGUCFeatureFunctions import ADAGUCFeatureCombineNuts
 import unittest
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestCSV.py

@@ -1,5 +1,5 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestGeoJSON.py

@@ -1,5 +1,5 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestMetadataService.py

@@ -1,5 +1,5 @@
 import os, os.path
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestOpenDAPServer.py

@@ -1,13 +1,13 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
 import subprocess
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestWMS.py

@@ -1,14 +1,14 @@
 import os, os.path
-from StringIO import StringIO
-from adaguc.CGIRunner import CGIRunner
+from io import StringIO
+# from adaguc import CGIRunner
 import unittest
 import shutil
 import subprocess
 import json
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestWMSDocumentCache.py

@@ -1,13 +1,13 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
 import subprocess
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestWMSPolylineRenderer.py

@@ -1,5 +1,5 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']
 

tests/AdagucTests/TestWMSSLD.py

@@ -1,5 +1,5 @@
 import os, os.path
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
@@ -8,7 +8,7 @@
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 # import sys
 # sys.path.insert(0, '/nobackup/users/bennekom/adaguc-webmapjs-sld-server/env/lib/python3.6/site-packages/flask/__init__.py')
 # from flask import Flask

tests/AdagucTests/TestWMSTiling.py

@@ -1,13 +1,13 @@
 import os
-from StringIO import StringIO
+from io import StringIO
 from adaguc.CGIRunner import CGIRunner
 import unittest
 import shutil
 import subprocess
 from lxml import etree
 from lxml import objectify
 import re
-from AdagucTestTools import AdagucTestTools
+from .AdagucTestTools import AdagucTestTools
 
 
 ADAGUC_PATH = os.environ['ADAGUC_PATH']

tests/requirements.txt

@@ -0,0 +1,3 @@
+netcdf4
+lxml
+urllib

tests/starttests.sh

@@ -17,7 +17,9 @@ export ADAGUC_FONT="${ADAGUC_PATH}/data/fonts/FreeSans.ttf"
 export ADAGUC_ONLINERESOURCE=""
 export ADAGUC_ENABLELOGBUFFER=FALSE
 ulimit -c unlimited
-python ${ADAGUC_PATH}/tests/functional_test.py $1
+# conda install --yes --file requirements.txt
+pip install -r requirements.txt
+python3 ${ADAGUC_PATH}/tests/functional_test.py $1
 
 
 # To run a specific test: