Py3 (#526)

* closing in, but need to go back and forth with master

* basestring thing

* flak8 passes again

* some basestring stuff

* un 2to3ed pcmdi directory tests pass again

* top dir done

* driver dir left to do

* obs seems to be the issue file, so commitng b4 converting it

* 2 more files

* seems like py3 works except it's using esmf 7.1

* test suite runs both py2 and py3

* cleanup debug statement

* travis test py3 as well expected to fail because on py3

* does not matter if py3 fails for now

* prj4 restriction

* pcoord and portrait fixed

* turn off double testing!

.travis.yml

@@ -17,11 +17,16 @@ before_install:
 
 script:
  - conda create -q -n py2 -c cdat/label/nightly -c nesii/label/dev-esmf -c conda-forge -c uvcdat -c pcmdi vcs vcsaddons cdp mesalib image-compare flake8 nose cia "proj4<5" "python<3"
+ - conda create -q -n py3 -c cdat/label/nightly -c nesii/label/dev-esmf -c conda-forge -c uvcdat -c pcmdi vcs vcsaddons cdp mesalib image-compare flake8 nose cia "proj4<5" "python>3"
  # Useful for debugging any issues with conda
  - conda info -a
  - source activate py2
  - python setup.py install
  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then python run_tests.py -v2 ; fi
+ - git clean -fd
+ - source activate py3
+ - python setup.py install
+ - if [ "$TRAVIS_OS_NAME" = "linux" ]; then python run_tests.py -v2 ; fi
  - source deactivate
 
 after_success:

recipes/pcmdi_metrics/conda_upload.sh

@@ -31,7 +31,8 @@ source deactivate
 source activate root
 python ./prep_for_build.py -l 1.1.2
 echo "starting conda build"
-conda build pcmdi_metrics -c conda-forge -c uvcdat -c pcmdi
+conda build pcmdi_metrics -c nesii/label/dev-esmf -c conda-forge -c uvcdat -c pcmdi --python=27
+conda build pcmdi_metrics -c nesii/label/dev-esmf -c conda-forge -c uvcdat -c pcmdi --python=36
 echo "starting anaconda upload"
 anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-*tar.bz2 --force
 

recipes/pcmdi_metrics/meta.yaml.in

@@ -25,6 +25,7 @@ requirements:
  - matplotlib
  - numpy
  - cia
+ - proj4 <5
 
 
 about:

run_tests.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 import glob
 import sys
 import os
@@ -12,6 +13,11 @@
 import shlex
 import cdat_info
 
+try:
+ basestring
+except Exception:
+ basestring = str
+
 root = os.getcwd()
 cpus = multiprocessing.cpu_count()
 
@@ -102,7 +108,7 @@ def findDiffFiles(log):
  file2 = log[k].split()[2]
  except:
  file2 = log[k].split()[1][:-1]+log[j].split()[0]
- print "+++++++++++++++++++++++++",file2
+ print("+++++++++++++++++++++++++",file2)
  if log[j].find("Saving image diff") > -1:
  diff = log[j].split()[-1]
  # break
@@ -113,7 +119,7 @@ def run_command(command, join_stderr=True):
  if isinstance(command, basestring):
  command = shlex.split(command)
  if args.verbosity > 0:
- print "Executing %s in %s" % (" ".join(command), os.getcwd())
+ print("Executing %s in %s" % (" ".join(command), os.getcwd()))
  if join_stderr:
  stderr = subprocess.STDOUT
  else:
@@ -126,10 +132,10 @@ def run_command(command, join_stderr=True):
  cwd=os.getcwd())
  out = []
  while P.poll() is None:
- read = P.stdout.readline().rstrip()
+ read = P.stdout.readline().rstrip().decode("utf-8")
  out.append(read)
  if args.verbosity > 1 and len(read) != 0:
- print read
+ print(read)
  return P, out
 
 
@@ -147,15 +153,15 @@ def run_nose(test_name):
 
 if args.vtk is not None:
  P, installed_vtk = run_command("conda list vtk-cdat")
- print installed_vtk
+ print(installed_vtk)
  while installed_vtk[-1] == "":
  installed_vtk.pop(-1)
  installed_vtk = installed_vtk[-1]
  installed_vtk = installed_vtk.split()
  vtk_name = installed_vtk[0]
  vtk_channel = installed_vtk[-1]
  if args.verbosity > 1:
- print "%s installed from: %s" % (vtk_name, vtk_channel)
+ print("%s installed from: %s" % (vtk_name, vtk_channel))
  run_command("conda install -f -y -c %s %s" % (args.vtk, vtk_name))
 
 sys.path.append(
@@ -170,7 +176,11 @@ def run_nose(test_name):
 
 if args.failed_only and os.path.exists(os.path.join("tests",".last_failure")):
  f = open(os.path.join("tests",".last_failure"))
- failed = set(eval(f.read().strip()))
+ read = f.read().strip()
+ try:
+ failed = set(eval(read.decode("utf-8")))
+ except Exception:
+ failed = set(eval(read))
  f.close()
  new_names = []
  for fnm in failed:
@@ -179,10 +189,10 @@ def run_nose(test_name):
  names = new_names
 
 if args.verbosity > 1:
- print("Names:", names)
+ print(("Names:", names))
 
 if len(names)==0:
- print "No tests to run"
+ print("No tests to run")
  sys.exit(0)
 
 # Make sure we have sample data
@@ -200,20 +210,20 @@ def run_nose(test_name):
 failed = []
 for d in outs:
  results.update(d)
- nm = d.keys()[0]
+ nm = list(d.keys())[0]
  if d[nm]["result"] != 0:
  failed.append(nm)
 f = open(os.path.join("tests",".last_failure"),"w")
 f.write(repr(failed))
 f.close()
 
 if args.verbosity > 0:
- print "Ran %i tests, %i failed (%.2f%% success)" %\
- (len(outs), len(failed), 100. - float(len(failed)) / len(outs) * 100.)
+ print("Ran %i tests, %i failed (%.2f%% success)" %\
+ (len(outs), len(failed), 100. - float(len(failed)) / len(outs) * 100.))
  if len(failed) > 0:
- print "Failed tests:"
+ print("Failed tests:")
  for f in failed:
- print "\t", f
+ print("\t", f)
 if args.html or args.package or args.dropbox:
  if not os.path.exists("tests_html"):
  os.makedirs("tests_html")
@@ -222,8 +232,8 @@ def run_nose(test_name):
  js = image_compare.script_data()
 
  fi = open("index.html", "w")
- print>>fi, "<!DOCTYPE html>"
- print>>fi, """<html><head><title>VCS Test Results %s</title>
+ print("<!DOCTYPE html>", file=fi)
+ print("""<html><head><title>VCS Test Results %s</title>
  <link rel="stylesheet" type="text/css" href="http:https://cdn.datatables.net/1.10.13/css/jquery.dataTables.css">
  <script type="text/javascript" src="http:https://code.jquery.com/jquery-1.12.4.js"></script>
  <script type="text/javascript" charset="utf8"
@@ -236,47 +246,47 @@ def run_nose(test_name):
  });
  } );
  </script>
- </head>""" % time.asctime()
- print>>fi, "<body><h1>VCS Test results: %s</h1>" % time.asctime()
- print>>fi, "<table id='table_id' class='display'>"
- print>>fi, "<thead><tr><th>Test</th><th>Result</th><th>Start Time</th><th>End Time</th><th>Time</th></tr></thead>"
- print>>fi, "<tfoot><tr><th>Test</th><th>Result</th><th>Start Time</th><th>End Time</th><th>Time</th></tr></tfoot>"
+ </head>""" % time.asctime(), file=fi)
+ print("<body><h1>VCS Test results: %s</h1>" % time.asctime(), file=fi)
+ print("<table id='table_id' class='display'>", file=fi)
+ print("<thead><tr><th>Test</th><th>Result</th><th>Start Time</th><th>End Time</th><th>Time</th></tr></thead>", file=fi)
+ print("<tfoot><tr><th>Test</th><th>Result</th><th>Start Time</th><th>End Time</th><th>Time</th></tr></tfoot>", file=fi)
 
  for t in sorted(results.keys()):
  result = results[t]
  nm = t.split("/")[-1][:-3]
- print>>fi, "<tr><td>%s</td>" % nm,
+ print("<tr><td>%s</td>" % nm, end=' ', file=fi)
  fe = codecs.open("%s.html" % nm, "w", encoding="utf-8")
- print>>fe, "<!DOCTYPE html>"
- print>>fe, "<html><head><title>%s</title>" % nm
+ print("<!DOCTYPE html>", file=fe)
+ print("<html><head><title>%s</title>" % nm, file=fe)
  if result["result"] == 0:
- print>>fi, "<td><a href='%s.html'>OK</a></td>" % nm,
- print>>fe, "</head><body>"
- print>>fe, "<a href='index.html'>Back To Results List</a>"
+ print("<td><a href='%s.html'>OK</a></td>" % nm, end=' ', file=fi)
+ print("</head><body>", file=fe)
+ print("<a href='index.html'>Back To Results List</a>", file=fe)
  else:
- print>>fi, "<td><a href='%s.html'>Fail</a></td>" % nm,
- print>>fe, "<script type='text/javascript'>%s</script></head><body>" % js
- print>>fe, "<a href='index.html'>Back To Results List</a>"
- print>>fe, "<h1>Failed test: %s on %s</h1>" % (nm, time.asctime())
+ print("<td><a href='%s.html'>Fail</a></td>" % nm, end=' ', file=fi)
+ print("<script type='text/javascript'>%s</script></head><body>" % js, file=fe)
+ print("<a href='index.html'>Back To Results List</a>", file=fe)
+ print("<h1>Failed test: %s on %s</h1>" % (nm, time.asctime()), file=fe)
  file1, file2, diff = findDiffFiles(result["log"])
  if file1 != "":
- print>>fe, '<div id="comparison"></div><script type="text/javascript"> ImageCompare.compare(' +\
+ print('<div id="comparison"></div><script type="text/javascript"> ImageCompare.compare(' +\
  'document.getElementById("comparison"), "%s", "%s"); </script>' % (
- abspath(file2, nm, "test"), abspath(file1, nm, "source"))
- print>>fe, "<div><a href='index.html'>Back To Results List</a></div>"
- print>>fe, "<div id='diff'><img src='%s' alt='diff file'></div>" % abspath(
- diff, nm, "diff")
- print>>fe, "<div><a href='index.html'>Back To Results List</a></div>"
- print>>fe, '<div id="output"><h1>Log</h1><pre>%s</pre></div>' % "\n".join(result[
- "log"])
- print>>fe, "<a href='index.html'>Back To Results List</a>"
- print>>fe, "</body></html>"
+ abspath(file2, nm, "test"), abspath(file1, nm, "source")), file=fe)
+ print("<div><a href='index.html'>Back To Results List</a></div>", file=fe)
+ print("<div id='diff'><img src='%s' alt='diff file'></div>" % abspath(
+ diff, nm, "diff"), file=fe)
+ print("<div><a href='index.html'>Back To Results List</a></div>", file=fe)
+ print('<div id="output"><h1>Log</h1><pre>%s</pre></div>' % "\n".join(result[
+ "log"]), file=fe)
+ print("<a href='index.html'>Back To Results List</a>", file=fe)
+ print("</body></html>", file=fe)
  fe.close()
  t = result["times"]
- print>>fi, "<td>%s</td><td>%s</td><td>%s</td></tr>" % (
- time.ctime(t["start"]), time.ctime(t["end"]), t["end"] - t["start"])
+ print("<td>%s</td><td>%s</td><td>%s</td></tr>" % (
+ time.ctime(t["start"]), time.ctime(t["end"]), t["end"] - t["start"]), file=fi)
 
- print>>fi, "</table></body></html>"
+ print("</table></body></html>", file=fi)
  fi.close()
  if args.html:
  webbrowser.open("file:https://%s/index.html" % os.getcwd())
@@ -290,7 +300,7 @@ def run_nose(test_name):
  t.add("tests_html")
  t.close()
  if args.verbosity > 0:
- print "Packaged Result Info in:", tnm
+ print("Packaged Result Info in:", tnm)
 if args.dropbox: 
  import dropbox
  dbx = dropbox.Dropbox(os.environ.get("DROPBOX_TOKEN",""))

setup.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from distutils.core import setup
 import glob
 import subprocess
@@ -19,7 +20,7 @@
  stdout=subprocess.PIPE,
  stderr=subprocess.PIPE)
 try:
- descr = p.stdout.readlines()[0].strip()
+ descr = p.stdout.readlines()[0].strip().decode("utf-8")
  Version = "-".join(descr.split("-")[:-2])
  if Version == "":
  Version = descr
@@ -36,13 +37,13 @@
  stdout=subprocess.PIPE,
  stderr=subprocess.PIPE)
 try:
- commit = p.stdout.readlines()[0].split()[1]
+ commit = p.stdout.readlines()[0].split()[1].decode("utf-8")
 except:
  commit = ""
 f = open("src/python/version.py", "w")
-print >>f, "__version__ = '%s'" % Version
-print >>f, "__git_tag_describe__ = '%s'" % descr
-print >>f, "__git_sha1__ = '%s'" % commit
+print("__version__ = '%s'" % Version, file=f)
+print("__git_tag_describe__ = '%s'" % descr, file=f)
+print("__git_sha1__ = '%s'" % commit, file=f)
 f.close()
 
 portrait_files = ["src/python/graphics/share/portraits.scr", ]
@@ -66,7 +67,7 @@
 scripts += glob.glob("src/python/diurnal/scripts/*.py")
 
 demo_files = glob.glob("demo/*/*")
-print "demo files"
+print("demo files")
 
 data_files = [
  ('share/pmp/graphics/vcs', portrait_files),
@@ -98,7 +99,7 @@
  ]
 
 if install_dev:
- print "Adding experimental packages"
+ print("Adding experimental packages")
  dev_packages = glob.glob("src/python/devel/*")
  dev_packages.remove("src/python/devel/example_dev")
  for p in dev_packages:
@@ -130,7 +131,7 @@
  author='PCMDI',
  description='model metrics tools',
  url='http:https://github.com/PCMDI/pcmdi_metrics',
- packages=packages.keys(),
+ packages=list(packages.keys()),
  package_dir=packages,
  scripts=scripts,
  data_files=data_files

src/python/__init__.py

@@ -11,6 +11,6 @@
 # add the handler to the logger
 plog.addHandler(ch)
 plog.setLevel(LOG_LEVEL)
-import io # noqa
-import pcmdi # noqa
-from version import __version__, __git_sha1__, __git_tag_describe__ # noqa
+from . import io # noqa
+from . import pcmdi # noqa
+from .version import __version__, __git_sha1__, __git_tag_describe__ # noqa

src/python/devel/example_dev/lib/__init__.py

@@ -1,2 +1,2 @@
 # init file for example dev module
-from file1 import example # noqa
+from .file1 import example # noqa

src/python/devel/example_dev/lib/file1.py

@@ -1,3 +1,3 @@
 def example(a, b):
- print "Adding", a, "to", b
+ print("Adding", a, "to", b)
  return a + b

src/python/devel/example_dev/scripts/example_dev1.py

@@ -9,4 +9,4 @@
 args = parser.parse_args(sys.argv[1:])
 s = example(args.numbers[0], args.numbers[1])
 
-print "Result:", s
+print("Result:", s)

src/python/devel/monsoon_wang/graphics/SeabarChart_mpl.py

@@ -8,7 +8,7 @@ class BarChart(object):
 
  def __init__(self, mods, data, uni, fig=None, rect=111, **kwargs):
 
- print 'IN FCN..... '
+ print('IN FCN..... ')
 
  # Canvas setup
  if fig is None:
@@ -24,7 +24,7 @@ def __init__(self, mods, data, uni, fig=None, rect=111, **kwargs):
  unit = kwargs['label']
  else:
  unit = uni # 'Unit needed here ...'
- print 'data above max '
+ print('data above max ')
  ymax = max(data)
  ymin = min(data)
 # yint = float((ymax-ymin)/3.) # REPLACE INT WITH FLOAT
@@ -52,12 +52,12 @@ def __init__(self, mods, data, uni, fig=None, rect=111, **kwargs):
  # -- if we didn't provide enough colors (for the number of simulations),
  # we will use green as a default
  tmpcolors = ['g'] * len(highlights)
- for i in xrange(len(colors)):
+ for i in range(len(colors)):
  tmpcolors[i] = colors[i]
  colors = tmpcolors
  else:
  colors = ['g'] * len(highlights)
- print 'highlights = ', highlights
+ print('highlights = ', highlights)
  for highlight in highlights:
  y_highlight = [0] * len(y)
 

src/python/devel/monsoon_wang/graphics/__init__.py

@@ -1,2 +1,2 @@
 from taylorDiagram import TaylorDiagram # noqa
-from SeabarChart_mpl import BarChart # noqa
+from .SeabarChart_mpl import BarChart # noqa

src/python/diurnal/common.py

@@ -16,7 +16,7 @@ def __init__(self, args, filename, filename_template, varname="pr"):
 
 def populateStringConstructor(template, args):
  template = genutil.StringConstructor(template)
- for k in template.keys():
+ for k in list(template.keys()):
  if hasattr(args, k):
  setattr(template, k, str(getattr(args, k)))
  return template