diff --git a/.circleci/config.yml b/.circleci/config.yml index 28612456c..cc225bf27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,13 +11,13 @@ aliases: command: | mkdir -p workspace git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat --depth=1 - python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3' + python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3.6' - &create_conda_env name: create_conda_env environment: PKGS: "vcs vcsaddons mesalib matplotlib scipy cia testsrunner 'proj4<5' 'vtk-cdat>8.1'" - CHANNELS: "-c cdat/label/nightly -c conda-forge -c pcmdi" + CHANNELS: "-c cdat/label/v81 -c conda-forge -c pcmdi" command: | export PATH=$WORKDIR/miniconda/bin:$PATH conda config --set always_yes yes --set changeps1 no diff --git a/pcmdi_metrics/graphics/portraits.py b/pcmdi_metrics/graphics/portraits.py index cf8730ec1..ff7e31c4d 100644 --- a/pcmdi_metrics/graphics/portraits.py +++ b/pcmdi_metrics/graphics/portraits.py @@ -78,6 +78,15 @@ def setlogo(self, value): logo = property(getlogo, setlogo) + def _repr_png_(self): + import tempfile + tmp = tempfile.mktemp() + ".png" + self.x.png(tmp) + f = open(tmp, "rb") + st = f.read() + f.close() + return st + def __init__(self): self.x1 = .12 self.x2 = .84 @@ -142,7 +151,7 @@ def __init__(self, files_structure=None, exclude=[], **kw): self.x = kw["x"] else: self.x = vcs.init() - self.verbose = True # output files looked for to the screen + self.verbose = False # output files looked for to the screen self.files_structure = files_structure self.exclude = exclude # First determine the list of parameters on which we can have a @@ -708,15 +717,104 @@ def decorate(self, output, ynm, xnm): x.names = repr(dic) nm = '___'.join(ynm) y.id = nm + y.original_id = output.getAxis(0,).id output.setAxis(0, y) dic = {} for i in range(len(ynm)): dic[i] = ynm[i] y.names = repr(dic) + x.original_id = output.getAxis(1,).id output.setAxis(1, x) return + def generateTemplate(self): + template = vcs.createtemplate() + # Now sets all the things for the template... + # Sets a bunch of template attributes to off + for att in [ + 'line1', 'line2', 'line3', 'line4', + 'box2', 'box3', 'box4', + 'min', 'max', 'mean', + 'xtic1', 'xtic2', + 'ytic1', 'ytic2', + 'xvalue', 'yvalue', 'zvalue', 'tvalue', + 'xunits', 'yunits', 'zunits', 'tunits', + 'source', 'title', 'dataname', + ]: + a = getattr(template, att) + setattr(a, 'priority', 0) + for att in [ + 'xname', 'yname', + ]: + a = getattr(template, att) + setattr(a, 'priority', 0) + + template.data.x1 = self.PLOT_SETTINGS.x1 + template.data.x2 = self.PLOT_SETTINGS.x2 + template.data.y1 = self.PLOT_SETTINGS.y1 + template.data.y2 = self.PLOT_SETTINGS.y2 + template.box1.x1 = self.PLOT_SETTINGS.x1 + template.box1.x2 = self.PLOT_SETTINGS.x2 + template.box1.y1 = self.PLOT_SETTINGS.y1 + template.box1.y2 = self.PLOT_SETTINGS.y2 + template.xname.y = self.PLOT_SETTINGS.y2 + .02 + template.yname.x = self.PLOT_SETTINGS.x2 + .01 + template.xlabel1.y = self.PLOT_SETTINGS.y1 + template.xlabel2.y = self.PLOT_SETTINGS.y2 + template.xlabel1.texttable = self.PLOT_SETTINGS.tictable + template.xlabel2.texttable = self.PLOT_SETTINGS.tictable + template.xlabel1.textorientation = \ + self.PLOT_SETTINGS.xticorientation + template.xlabel2.textorientation = \ + self.PLOT_SETTINGS.xticorientation + template.ylabel1.x = self.PLOT_SETTINGS.x1 + template.ylabel2.x = self.PLOT_SETTINGS.x2 + template.ylabel1.texttable = self.PLOT_SETTINGS.tictable + template.ylabel2.texttable = self.PLOT_SETTINGS.tictable + template.ylabel1.textorientation = \ + self.PLOT_SETTINGS.yticorientation + template.ylabel2.textorientation = \ + self.PLOT_SETTINGS.yticorientation + + if self.PLOT_SETTINGS.xtic1.y1 is not None: + template.xtic1.y1 = self.PLOT_SETTINGS.xtic1.y1 + template.xtic1.priority = 1 + if self.PLOT_SETTINGS.xtic1.y2 is not None: + template.xtic1.y2 = self.PLOT_SETTINGS.xtic1.y2 + template.xtic1.priority = 1 + if self.PLOT_SETTINGS.xtic2.y1 is not None: + template.xtic2.y1 = self.PLOT_SETTINGS.xtic2.y1 + template.xtic2.priority = 1 + if self.PLOT_SETTINGS.xtic2.y2 is not None: + template.xtic2.y2 = self.PLOT_SETTINGS.xtic2.y2 + template.xtic2.priority = 1 + if self.PLOT_SETTINGS.ytic1.x1 is not None: + template.ytic1.x1 = self.PLOT_SETTINGS.ytic1.x1 + template.ytic1.priority = 1 + if self.PLOT_SETTINGS.ytic1.x2 is not None: + template.ytic1.x2 = self.PLOT_SETTINGS.ytic1.x2 + template.ytic1.priority = 1 + if self.PLOT_SETTINGS.ytic2.x1 is not None: + template.ytic2.priority = 1 + template.ytic2.x1 = self.PLOT_SETTINGS.ytic2.x1 + if self.PLOT_SETTINGS.ytic2.x2 is not None: + template.ytic2.priority = 1 + template.ytic2.x2 = self.PLOT_SETTINGS.ytic2.x2 + template.legend.x1 = self.PLOT_SETTINGS.legend.x1 + template.legend.x2 = self.PLOT_SETTINGS.legend.x2 + template.legend.y1 = self.PLOT_SETTINGS.legend.y1 + template.legend.y2 = self.PLOT_SETTINGS.legend.y2 + try: + tmp = vcs.createtextorientation('crap22') + except Exception: + tmp = vcs.gettextorientation('crap22') + tmp.height = 12 + # tmp.halign = 'center' + # template.legend.texttable = tmp + template.legend.textorientation = tmp + return template + def plot(self, data=None, mesh=None, template=None, meshfill=None, x=None, bg=0, multiple=1.1): self.bg = bg @@ -732,91 +830,7 @@ def plot(self, data=None, mesh=None, template=None, # Do we use a predefined template ? if template is None: - template = vcs.createtemplate() - # Now sets all the things for the template... - # Sets a bunch of template attributes to off - for att in [ - 'line1', 'line2', 'line3', 'line4', - 'box2', 'box3', 'box4', - 'min', 'max', 'mean', - 'xtic1', 'xtic2', - 'ytic1', 'ytic2', - 'xvalue', 'yvalue', 'zvalue', 'tvalue', - 'xunits', 'yunits', 'zunits', 'tunits', - 'source', 'title', 'dataname', - ]: - a = getattr(template, att) - setattr(a, 'priority', 0) - for att in [ - 'xname', 'yname', - ]: - a = getattr(template, att) - setattr(a, 'priority', 0) - - template.data.x1 = self.PLOT_SETTINGS.x1 - template.data.x2 = self.PLOT_SETTINGS.x2 - template.data.y1 = self.PLOT_SETTINGS.y1 - template.data.y2 = self.PLOT_SETTINGS.y2 - template.box1.x1 = self.PLOT_SETTINGS.x1 - template.box1.x2 = self.PLOT_SETTINGS.x2 - template.box1.y1 = self.PLOT_SETTINGS.y1 - template.box1.y2 = self.PLOT_SETTINGS.y2 - template.xname.y = self.PLOT_SETTINGS.y2 + .02 - template.yname.x = self.PLOT_SETTINGS.x2 + .01 - template.xlabel1.y = self.PLOT_SETTINGS.y1 - template.xlabel2.y = self.PLOT_SETTINGS.y2 - template.xlabel1.texttable = self.PLOT_SETTINGS.tictable - template.xlabel2.texttable = self.PLOT_SETTINGS.tictable - template.xlabel1.textorientation = \ - self.PLOT_SETTINGS.xticorientation - template.xlabel2.textorientation = \ - self.PLOT_SETTINGS.xticorientation - template.ylabel1.x = self.PLOT_SETTINGS.x1 - template.ylabel2.x = self.PLOT_SETTINGS.x2 - template.ylabel1.texttable = self.PLOT_SETTINGS.tictable - template.ylabel2.texttable = self.PLOT_SETTINGS.tictable - template.ylabel1.textorientation = \ - self.PLOT_SETTINGS.yticorientation - template.ylabel2.textorientation = \ - self.PLOT_SETTINGS.yticorientation - - if self.PLOT_SETTINGS.xtic1.y1 is not None: - template.xtic1.y1 = self.PLOT_SETTINGS.xtic1.y1 - template.xtic1.priority = 1 - if self.PLOT_SETTINGS.xtic1.y2 is not None: - template.xtic1.y2 = self.PLOT_SETTINGS.xtic1.y2 - template.xtic1.priority = 1 - if self.PLOT_SETTINGS.xtic2.y1 is not None: - template.xtic2.y1 = self.PLOT_SETTINGS.xtic2.y1 - template.xtic2.priority = 1 - if self.PLOT_SETTINGS.xtic2.y2 is not None: - template.xtic2.y2 = self.PLOT_SETTINGS.xtic2.y2 - template.xtic2.priority = 1 - if self.PLOT_SETTINGS.ytic1.x1 is not None: - template.ytic1.x1 = self.PLOT_SETTINGS.ytic1.x1 - template.ytic1.priority = 1 - if self.PLOT_SETTINGS.ytic1.x2 is not None: - template.ytic1.x2 = self.PLOT_SETTINGS.ytic1.x2 - template.ytic1.priority = 1 - if self.PLOT_SETTINGS.ytic2.x1 is not None: - template.ytic2.priority = 1 - template.ytic2.x1 = self.PLOT_SETTINGS.ytic2.x1 - if self.PLOT_SETTINGS.ytic2.x2 is not None: - template.ytic2.priority = 1 - template.ytic2.x2 = self.PLOT_SETTINGS.ytic2.x2 - template.legend.x1 = self.PLOT_SETTINGS.legend.x1 - template.legend.x2 = self.PLOT_SETTINGS.legend.x2 - template.legend.y1 = self.PLOT_SETTINGS.legend.y1 - template.legend.y2 = self.PLOT_SETTINGS.legend.y2 - try: - tmp = vcs.createtextorientation('crap22') - except Exception: - tmp = vcs.gettextorientation('crap22') - tmp.height = 12 - # tmp.halign = 'center' - # template.legend.texttable = tmp - template.legend.textorientation = tmp - + template = self.generateTemplate() else: if isinstance(template, vcs.template.P): tid = template.name @@ -826,7 +840,7 @@ def plot(self, data=None, mesh=None, template=None, raise 'Error cannot understand what you mean by template=' + \ str(template) - template = vcs.createtemplate() + template = vcs.createtemplate(source=tid) # Do we use a predefined meshfill ? if meshfill is None: @@ -847,7 +861,7 @@ def plot(self, data=None, mesh=None, template=None, meshfill.yticlabels2 = mtics if self.PLOT_SETTINGS.colormap is None: self.set_colormap() - elif x.getcolormapname() != self.PLOT_SETTINGS.colormap: + elif self.x.getcolormapname() != self.PLOT_SETTINGS.colormap: self.x.setcolormap(self.PLOT_SETTINGS.colormap) if self.PLOT_SETTINGS.levels is None: @@ -861,7 +875,11 @@ def plot(self, data=None, mesh=None, template=None, if len(levs) > 1: meshfill.levels = levs if self.PLOT_SETTINGS.fillareacolors is None: - cols = vcs.getcolors(levs, list(range(16, 40)), split=1) + if self.PLOT_SETTINGS.colormap is None: + # Default colormap only use range 16->40 + cols = vcs.getcolors(levs, list(range(16, 40)), split=1) + else: + cols = vcs.getcolors(levs, split=1) meshfill.fillareacolors = cols else: meshfill.fillareacolors = self.PLOT_SETTINGS.fillareacolors @@ -1078,7 +1096,7 @@ def plot(self, data=None, mesh=None, template=None, if p not in self.dummies and \ p not in self.auto_dummies and \ p not in axes_param: - txt = x.createtext( + txt = self.x.createtext( None, self.PLOT_SETTINGS.parametertable.name, None, @@ -1217,737 +1235,4 @@ def draw_values(self, raveled, mesh, meshfill, template): self.x.plot(tmptxt, bg=self.bg) def set_colormap(self): - cols = ( - 100, - 100, - 100, - 0, - 0, - 0, - 83.9216, - 83.9216, - 83.9216, - 30.9804, - 30.9804, - 30.9804, - 100, - 100, - 100, - 100, - 100, - 0, - 0, - 2.7451, - 100, - 0, - 5.4902, - 100, - 0, - 7.84314, - 100, - 0, - 10.9804, - 100, - 0, - 13.7255, - 100, - 0, - 16.4706, - 100, - 0, - 20.3922, - 100, - 0, - 23.1373, - 100, - 0, - 25.4902, - 100, - 0, - 30.1961, - 100, - 0, - 0, - 47.451, - 10.5882, - 13.3333, - 54.5098, - 21.5686, - 27.0588, - 61.5686, - 32.549, - 40.7843, - 68.6274, - 43.5294, - 54.5098, - 76.0784, - 48.6275, - 60.7843, - 79.2157, - 53.7255, - 67.451, - 82.7451, - 58.8235, - 73.7255, - 86.2745, - 64.3137, - 80.3922, - 89.4118, - 69.4118, - 86.6667, - 92.9412, - 74.5098, - 93.3333, - 96.4706, - 80, - 100, - 100, - 100, - 87.0588, - 85.098, - 100, - 69.4118, - 67.8431, - 100, - 52.1569, - 50.9804, - 100, - 34.5098, - 33.7255, - 100, - 17.2549, - 16.8627, - 100, - 0, - 0, - 87.451, - 0, - 0, - 74.902, - 0, - 0, - 62.7451, - 0, - 0, - 50.1961, - 0, - 0, - 37.6471, - 0, - 0, - 25.4902, - 0, - 0, - 100, - 100, - 100, - 0, - 0, - 47.451, - 0.392157, - 0.392157, - 47.451, - 0.784314, - 0.784314, - 47.8431, - 1.17647, - 1.17647, - 48.2353, - 1.56863, - 1.56863, - 48.2353, - 1.96078, - 1.96078, - 48.6275, - 2.35294, - 2.7451, - 49.0196, - 2.7451, - 3.13725, - 49.0196, - 3.13725, - 3.52941, - 49.4118, - 3.52941, - 3.92157, - 49.8039, - 3.92157, - 4.31373, - 49.8039, - 4.31373, - 5.09804, - 50.1961, - 4.70588, - 5.4902, - 50.5882, - 5.09804, - 5.88235, - 50.5882, - 5.4902, - 6.27451, - 50.9804, - 5.88235, - 6.66667, - 51.3725, - 6.27451, - 7.45098, - 51.3725, - 6.66667, - 7.84314, - 51.7647, - 7.05882, - 8.23529, - 52.1569, - 7.45098, - 8.62745, - 52.1569, - 7.84314, - 9.01961, - 52.549, - 8.23529, - 9.80392, - 52.9412, - 8.62745, - 10.1961, - 52.9412, - 9.01961, - 10.5882, - 53.3333, - 9.41177, - 10.9804, - 53.7255, - 9.80392, - 11.3725, - 53.7255, - 10.1961, - 12.1569, - 54.1176, - 10.5882, - 12.549, - 54.5098, - 10.9804, - 12.9412, - 54.5098, - 11.3725, - 13.3333, - 54.902, - 11.7647, - 13.7255, - 55.2941, - 12.1569, - 14.5098, - 55.2941, - 12.549, - 14.902, - 55.6863, - 13.3333, - 15.2941, - 56.0784, - 13.7255, - 15.6863, - 56.4706, - 14.1176, - 16.0784, - 56.4706, - 14.5098, - 16.8627, - 56.8627, - 14.902, - 17.2549, - 57.2549, - 15.2941, - 17.6471, - 57.2549, - 15.6863, - 18.0392, - 57.6471, - 16.0784, - 18.4314, - 58.0392, - 16.4706, - 19.2157, - 58.0392, - 16.8627, - 19.6078, - 58.4314, - 17.2549, - 20, - 58.8235, - 17.6471, - 20.3922, - 58.8235, - 18.0392, - 20.7843, - 59.2157, - 18.4314, - 21.5686, - 59.6078, - 18.8235, - 21.9608, - 59.6078, - 19.2157, - 22.3529, - 60, - 19.6078, - 22.7451, - 60.3922, - 20, - 23.1373, - 60.3922, - 20.3922, - 23.9216, - 60.7843, - 20.7843, - 24.3137, - 61.1765, - 21.1765, - 24.7059, - 61.1765, - 21.5686, - 25.098, - 61.5686, - 21.9608, - 25.4902, - 61.9608, - 22.3529, - 26.2745, - 61.9608, - 22.7451, - 26.6667, - 62.3529, - 23.1373, - 27.0588, - 62.7451, - 23.5294, - 27.451, - 62.7451, - 23.9216, - 27.8431, - 63.1373, - 24.3137, - 28.6275, - 63.5294, - 24.7059, - 29.0196, - 63.5294, - 25.098, - 29.4118, - 63.9216, - 25.4902, - 29.8039, - 64.3137, - 25.8824, - 30.1961, - 64.3137, - 26.6667, - 30.9804, - 64.7059, - 27.0588, - 31.3725, - 65.098, - 27.451, - 31.7647, - 65.4902, - 27.8431, - 32.1569, - 65.4902, - 28.2353, - 32.549, - 65.8824, - 28.6275, - 32.9412, - 66.2745, - 29.0196, - 33.7255, - 66.2745, - 29.4118, - 34.1176, - 66.6667, - 29.8039, - 34.5098, - 67.0588, - 30.1961, - 34.902, - 67.0588, - 30.5882, - 35.2941, - 67.451, - 30.9804, - 36.0784, - 67.8431, - 31.3725, - 36.4706, - 67.8431, - 31.7647, - 36.8627, - 68.2353, - 32.1569, - 37.2549, - 68.6274, - 32.549, - 37.6471, - 68.6274, - 32.9412, - 38.4314, - 69.0196, - 33.3333, - 38.8235, - 69.4118, - 33.7255, - 39.2157, - 69.4118, - 34.1176, - 39.6078, - 69.8039, - 34.5098, - 40, - 70.1961, - 34.902, - 40.7843, - 70.1961, - 35.2941, - 41.1765, - 70.5882, - 35.6863, - 41.5686, - 70.9804, - 36.0784, - 41.9608, - 70.9804, - 36.4706, - 42.3529, - 71.3726, - 36.8627, - 43.1373, - 71.7647, - 37.2549, - 43.5294, - 71.7647, - 37.6471, - 43.9216, - 72.1569, - 38.0392, - 44.3137, - 72.549, - 38.4314, - 44.7059, - 72.549, - 38.8235, - 45.4902, - 72.9412, - 39.2157, - 45.8824, - 73.3333, - 40, - 46.2745, - 73.7255, - 40.3922, - 46.6667, - 73.7255, - 40.7843, - 47.0588, - 74.1176, - 41.1765, - 47.8431, - 74.5098, - 41.5686, - 48.2353, - 74.5098, - 41.9608, - 48.6275, - 74.902, - 42.3529, - 49.0196, - 75.2941, - 42.7451, - 49.4118, - 75.2941, - 43.1373, - 50.1961, - 75.6863, - 43.5294, - 50.5882, - 76.0784, - 43.9216, - 50.9804, - 76.0784, - 44.3137, - 51.3725, - 76.4706, - 44.7059, - 51.7647, - 76.8627, - 45.098, - 52.549, - 76.8627, - 45.4902, - 52.9412, - 77.2549, - 45.8824, - 53.3333, - 77.6471, - 46.2745, - 53.7255, - 77.6471, - 46.6667, - 54.1176, - 78.0392, - 47.0588, - 54.902, - 78.4314, - 47.451, - 55.2941, - 78.4314, - 47.8431, - 55.6863, - 78.8235, - 48.2353, - 56.0784, - 79.2157, - 48.6275, - 56.4706, - 79.2157, - 49.0196, - 57.2549, - 79.6078, - 49.4118, - 57.6471, - 80, - 49.8039, - 58.0392, - 80, - 50.1961, - 58.4314, - 80.3922, - 50.5882, - 58.8235, - 80.7843, - 50.9804, - 59.6078, - 80.7843, - 51.3725, - 60, - 81.1765, - 51.7647, - 60.3922, - 81.5686, - 52.1569, - 60.7843, - 81.5686, - 52.549, - 61.1765, - 81.9608, - 53.3333, - 61.9608, - 82.3529, - 53.7255, - 62.3529, - 82.7451, - 54.1176, - 62.7451, - 82.7451, - 54.5098, - 63.1373, - 83.1373, - 54.902, - 63.5294, - 83.5294, - 55.2941, - 63.9216, - 83.5294, - 55.6863, - 64.7059, - 83.9216, - 56.0784, - 65.098, - 84.3137, - 56.4706, - 65.4902, - 84.3137, - 56.8627, - 65.8824, - 84.7059, - 57.2549, - 66.2745, - 85.098, - 57.6471, - 67.0588, - 85.098, - 58.0392, - 67.451, - 85.4902, - 58.4314, - 67.8431, - 85.8824, - 58.8235, - 68.2353, - 85.8824, - 59.2157, - 68.6274, - 86.2745, - 59.6078, - 69.4118, - 86.6667, - 60, - 69.8039, - 86.6667, - 60.3922, - 70.1961, - 87.0588, - 60.7843, - 70.5882, - 87.451, - 61.1765, - 70.9804, - 87.451, - 61.5686, - 71.7647, - 87.8431, - 61.9608, - 72.1569, - 88.2353, - 62.3529, - 72.549, - 88.2353, - 62.7451, - 72.9412, - 88.6274, - 63.1373, - 73.3333, - 89.0196, - 63.5294, - 74.1176, - 89.0196, - 63.9216, - 74.5098, - 89.4118, - 64.3137, - 74.902, - 89.8039, - 64.7059, - 75.2941, - 89.8039, - 65.098, - 75.6863, - 90.1961, - 65.4902, - 76.4706, - 90.5882, - 65.8824, - 76.8627, - 90.5882, - 66.6667, - 77.2549, - 90.9804, - 67.0588, - 77.6471, - 91.3726, - 67.451, - 78.0392, - 91.7647, - 67.8431, - 78.8235, - 91.7647, - 68.2353, - 79.2157, - 92.1569, - 68.6274, - 79.6078, - 92.549, - 69.0196, - 80, - 92.549, - 69.4118, - 80.3922, - 92.9412, - 69.8039, - 81.1765, - 93.3333, - 70.1961, - 81.5686, - 93.3333, - 70.5882, - 81.9608, - 93.7255, - 70.9804, - 82.3529, - 94.1176, - 71.3726, - 82.7451, - 94.1176, - 71.7647, - 83.5294, - 94.5098, - 72.1569, - 83.9216, - 94.902, - 72.549, - 84.3137, - 94.902, - 72.9412, - 84.7059, - 95.2941, - 73.3333, - 85.098, - 95.6863, - 73.7255, - 85.8824, - 95.6863, - 74.1176, - 86.2745, - 96.0784, - 74.5098, - 86.6667, - 96.4706, - 74.902, - 87.0588, - 96.4706, - 75.2941, - 87.451, - 96.8627, - 75.6863, - 88.2353, - 97.2549, - 76.0784, - 88.6274, - 97.2549, - 76.4706, - 89.0196, - 97.6471, - 76.8627, - 89.4118, - 98.0392, - 77.2549, - 89.8039, - 98.0392, - 77.6471, - 90.5882, - 98.4314, - 78.0392, - 90.9804, - 98.8235, - 78.4314, - 91.3726, - 98.8235, - 78.8235, - 91.7647, - 99.2157, - 79.2157, - 92.1569, - 99.6078, - 80, - 92.9412, - 100) - - cols = MV2.reshape(cols, (len(cols) // 3, 3)) - - for i in range(cols.shape[0]): - co = self.x.getcolorcell(i) - if (co[0] != int(cols[i][0]) or co[1] != int( - cols[i][1]) or co[2] != int(cols[i][2])): - self.x.setcolorcell( - i, int( - cols[i][0]), int( - cols[i][1]), int( - cols[i][2])) - return + self.x.setcolormap("bl_rd_12") diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py index 6a0fbed52..3a6519073 100755 --- a/pcmdi_metrics/io/base.py +++ b/pcmdi_metrics/io/base.py @@ -32,22 +32,23 @@ # Group merged axes -def groupAxes(axes, final=[], ids=None, separator="_"): - if axes == []: - return cdms2.createAxis(final, id=separator.join(ids)) - if final == []: - final = [val for val in axes[0]] +def groupAxes(axes, ids=None, separator="_"): + if ids is None: ids = [ax.id for ax in axes] - return groupAxes(axes[1:], final, ids) - axis = axes[0] - original_length = len(final) - final = final * len(axis) - idx = 0 - for val in axis: - for i in range(original_length): - final[idx] = "{}{}{}".format(final[idx], separator, val) - idx += 1 - return groupAxes(axes[1:], final, ids) + if len(ids) != len(axes): + raise RuntimeError("You need to pass as many ids as axes") + final = [] + while len(axes) > 0: + axis = axes.pop(-1) + if final == []: + final = [str(v) for v in axis] + else: + tmp = final + final = [] + for v1 in axis: + for v2 in tmp: + final += ["{}{}{}".format(v1, separator, v2)] + return cdms2.createAxis(final, id=separator.join(ids)) # cdutil region object need a serializer @@ -188,6 +189,33 @@ def generateProvenance(): return prov +def scrap(data, axis=0): + originalOrder = data.getOrder(ids=True) + if axis not in ['x', 'y', 'z', 't'] and not isinstance(axis, int): + order = "({})...".format(axis) + else: + order = "{}...".format(axis) + new = data(order=order) + axes = new.getAxisList() # Save for later + new = MV2.array(new.asma()) # lose dims + for i in range(new.shape[0] - 1, -1, -1): + tmp = new[i] + if not isinstance(tmp, (float, numpy.float)) and tmp.mask.all(): + a = new[:i] + b = new[i + 1:] + if b.shape[0] == 0: + new = a + else: + new = MV2.concatenate((a, b)) + newAxis = [] + for v in new.getAxis(0): + newAxis.append(axes[0][int(v)]) + ax = cdms2.createAxis(newAxis, id=axes[0].id) + axes[0] = ax + new.setAxisList(axes) + return new(order=originalOrder) + + class CDMSDomainsEncoder(json.JSONEncoder): def default(self, o): components = o.components()[0].kargs @@ -523,11 +551,11 @@ def get_array_values_from_dict_recursive(self, out, ids, nms, axval, axes): try: vals = vals[k] except Exception: - vals = 1.e20 + vals = 9.99e20 try: out[tuple(ids)] = float(vals) except Exception: - out[tuple(ids)] = 1.e20 + out[tuple(ids)] = 9.99e20 def __init__(self, files=[], structure=[], ignored_keys=[], oneVariablePerFile=True): @@ -595,10 +623,31 @@ def getAxisList(self): def __call__(self, merge=[], **kargs): """ Returns the array of values""" + # First clean up kargs + if "merge" in kargs: + merge = kargs["merge"] + del(kargs["merge"]) + order = None + axes_ids = self.getAxisIds() + if "order" in kargs: + # If it's an actual axis assume that it's what user wants + # Otherwise it's an out order keyword + if "order" not in axes_ids: + order = kargs["order"] + del(kargs["order"]) ab = cdms2.getAutoBounds() cdms2.setAutoBounds("off") axes = self.getAxisList() - axes_ids = self.getAxisIds() + if merge != []: + if isinstance(merge[0], str): + merge = [merge, ] + if merge != []: + for merger in merge: + for merge_axis_id in merger: + if merge_axis_id not in axes_ids: + raise RuntimeError( + "You requested to merge axis is '{}' which is not valid. Axes: {}".format( + merge_axis_id, axes_ids)) sh = [] ids = [] used_ids = [] @@ -611,9 +660,6 @@ def __call__(self, merge=[], **kargs): # first let's see which vars are actually asked for # for now assume all keys means restriction on dims - if "merge" in kargs: - merge = kargs["merge"] - del(kargs["merge"]) if not isinstance(merge, (list, tuple)): raise RuntimeError( "merge keyword must be a list of dimensions to merge together") @@ -652,8 +698,15 @@ def __call__(self, merge=[], **kargs): # Ok at this point we need to take care of merged axes # First let's create the merged axes - new_axes = [groupAxes([self.getAxis(x) for x in merger]) - for merger in merge] + axes_to_group = [] + for merger in merge: + merged_axes = [] + for axid in merger: + for ax in axes: + if ax.id == axid: + merged_axes.append(ax) + axes_to_group.append(merged_axes) + new_axes = [groupAxes(grp_axes) for grp_axes in axes_to_group] sh2 = list(sh) for merger in merge: for merger in merge: # loop through all possible merging @@ -714,7 +767,15 @@ def __call__(self, merge=[], **kargs): sub += 1 else: outData.setAxis(index - sub, new_axes[setMergedAxis]) - outData = MV2.masked_greater(outData, 9.e19) + outData = MV2.masked_greater(outData, 9.98e20) outData.id = "pmp" + if order is not None: + myorder = "".join(["({})".format(nm) for nm in order]) + outData = outData(order=myorder) + # Merge needs cleaning for extra dims crated + if merge != []: + for i in range(outData.ndim): + outData = scrap(outData, axis=i) + outData = MV2.masked_greater(outData, 9.9e19) cdms2.setAutoBounds(ab) return outData diff --git a/pcmdi_metrics/version.py b/pcmdi_metrics/version.py index 3f1704c7a..fa2bc1ce6 100644 --- a/pcmdi_metrics/version.py +++ b/pcmdi_metrics/version.py @@ -1,3 +1,3 @@ __version__ = 'v1.2' -__git_tag_describe__ = 'v1.2-40-ge4e2258' -__git_sha1__ = 'e4e2258d063e0da11442c7dae5ec6ce688387f51' +__git_tag_describe__ = 'v1.2-57-g930f3ae' +__git_sha1__ = '930f3aec5867b96e376ff3e29bb383008432398e' diff --git a/tests/io/merge.json b/tests/io/merge.json new file mode 100644 index 000000000..a02ad79d6 --- /dev/null +++ b/tests/io/merge.json @@ -0,0 +1 @@ +{"json_structure": ["upper", "lower", "numbers"], "RESULTS": {"A": {"a": {"1": 0.0, "2": 0.1, "3": 0.2, "4": 0.3, "5": 0.4, "6": 0.5, "7": 0.6, "8": 0.7}, "b": {"1": 1.0, "2": 1.1, "3": 1.2, "4": 1.3, "5": 1.4, "6": 1.5, "7": 1.6, "8": 1.7}, "c": {"1": 2.0, "2": 2.1, "3": 2.2, "4": 2.3, "5": 2.4, "6": 2.5, "7": 2.6, "8": 2.7}, "d": {"1": 3.0, "2": 3.1, "3": 3.2, "4": 3.3, "5": 3.4, "6": 3.5, "7": 3.6, "8": 3.7}, "e": {"1": 4.0, "2": 4.1, "3": 4.2, "4": 4.3, "5": 4.4, "6": 4.5, "7": 4.6, "8": 4.7}}, "B": {"a": {"1": 10.0, "2": 10.1, "3": 10.2, "4": 10.3, "5": 10.4, "6": 10.5, "7": 10.6, "8": 10.7}, "b": {"1": 11.0, "2": 11.1, "3": 11.2, "4": 11.3, "5": 11.4, "6": 11.5, "7": 11.6, "8": 11.7}, "c": {"1": 12.0, "2": 12.1, "3": 12.2, "4": 12.3, "5": 12.4, "6": 12.5, "7": 12.6, "8": 12.7}, "d": {"1": 13.0, "2": 13.1, "3": 13.2, "4": 13.3, "5": 13.4, "6": 13.5, "7": 13.6, "8": 13.7}, "e": {"1": 14.0, "2": 14.1, "3": 14.2, "4": 14.3, "5": 14.4, "6": 14.5, "7": 14.6, "8": 14.7}}, "C": {"a": {"1": 20.0, "2": 20.1, "3": 20.2, "4": 20.3, "5": 20.4, "6": 20.5, "7": 20.6, "8": 20.7}, "b": {"1": 21.0, "2": 21.1, "3": 21.2, "4": 21.3, "5": 21.4, "6": 21.5, "7": 21.6, "8": 21.7}, "c": {"1": 22.0, "2": 22.1, "3": 22.2, "4": 22.3, "5": 22.4, "6": 22.5, "7": 22.6, "8": 22.7}, "d": {"1": 23.0, "2": 23.1, "3": 23.2, "4": 23.3, "5": 23.4, "6": 23.5, "7": 23.6, "8": 23.7}, "e": {"1": 24.0, "2": 24.1, "3": 24.2, "4": 24.3, "5": 24.4, "6": 24.5, "7": 24.6, "8": 24.7}}, "D": {"a": {"1": 30.0, "2": 30.1, "3": 30.2, "4": 30.3, "5": 30.4, "6": 30.5, "7": 30.6, "8": 30.7}, "b": {"1": 31.0, "2": 31.1, "3": 31.2, "4": 31.3, "5": 31.4, "6": 31.5, "7": 31.6, "8": 31.7}, "c": {"1": 32.0, "2": 32.1, "3": 32.2, "4": 32.3, "5": 32.4, "6": 32.5, "7": 32.6, "8": 32.7}, "d": {"1": 33.0, "2": 33.1, "3": 33.2, "4": 33.3, "5": 33.4, "6": 33.5, "7": 33.6, "8": 33.7}, "e": {"1": 34.0, "2": 34.1, "3": 34.2, "4": 34.3, "5": 34.4, "6": 34.5, "7": 34.6, "8": 34.7}}}, "json_version": 3.0, "provenance": {"platform": {"OS": "Linux", "Version": "4.15.0-43-generic", "Name": "drdoom"}, "userId": "doutriaux1", "osAccess": false, "commandLine": "create_json_for_merge.py", "date": "2019-02-28 10:53:41", "conda": {}, "packages": {}, "openGL": {"GLX": {"server": {}, "client": {}}}}} \ No newline at end of file diff --git a/tests/io/merge_incomplete_and_missing.json b/tests/io/merge_incomplete_and_missing.json new file mode 100644 index 000000000..9be999be9 --- /dev/null +++ b/tests/io/merge_incomplete_and_missing.json @@ -0,0 +1 @@ +{"json_structure": ["upper", "lower", "numbers"], "RESULTS": {"A": {"a": {"1": 0.0, "2": 0.1, "3": 0.2, "4": 0.3, "5": 0.4, "6": 0.5, "7": 0.6, "8": 0.7}, "b": {"1": 1.0, "2": 1.1, "3": 1.2, "4": 1.3, "5": 1.4, "6": 1.5, "7": 1.6, "8": 1.7}, "c": {"1": 2.0, "2": 2.1, "4": 2.3, "6": 2.5, "7": 2.6}, "d": {"1": 1e+20, "2": 1e+20, "3": 1e+20, "4": 1e+20, "5": 1e+20, "6": 1e+20, "7": 1e+20, "8": 1e+20}, "e": {"1": 4.0, "2": 4.1, "3": 4.2, "4": 4.3, "5": 4.4, "6": 4.5, "7": 4.6, "8": 4.7}}, "B": {"a": {"1": 10.0, "2": 10.1, "3": 10.2, "4": 10.3, "5": 10.4, "6": 10.5, "7": 10.6, "8": 10.7}, "b": {"1": 11.0, "2": 11.1, "3": 11.2, "4": 11.3, "5": 11.4, "6": 11.5, "7": 11.6, "8": 11.7}, "c": {"1": 12.0, "2": 12.1, "4": 12.3, "6": 12.5, "7": 12.6}, "d": {"1": 1e+20, "2": 1e+20, "3": 1e+20, "4": 1e+20, "5": 1e+20, "6": 1e+20, "7": 1e+20, "8": 1e+20}, "e": {"1": 14.0, "2": 14.1, "3": 14.2, "4": 14.3, "5": 14.4, "6": 14.5, "7": 14.6, "8": 14.7}}, "C": {"a": {"1": 20.0, "2": 20.1, "3": 20.2, "4": 20.3, "5": 20.4, "6": 20.5, "7": 20.6, "8": 20.7}, "b": {"1": 21.0, "2": 21.1, "3": 21.2, "4": 21.3, "5": 21.4, "6": 21.5, "7": 21.6, "8": 21.7}, "c": {"1": 22.0, "2": 22.1, "4": 22.3, "6": 22.5, "7": 22.6}, "d": {"1": 1e+20, "2": 1e+20, "3": 1e+20, "4": 1e+20, "5": 1e+20, "6": 1e+20, "7": 1e+20, "8": 1e+20}, "e": {"1": 24.0, "2": 24.1, "3": 24.2, "4": 24.3, "5": 24.4, "6": 24.5, "7": 24.6, "8": 24.7}}, "D": {"a": {"1": 30.0, "2": 30.1, "3": 30.2, "4": 30.3, "5": 30.4, "6": 30.5, "7": 30.6, "8": 30.7}, "b": {"1": 31.0, "2": 31.1, "3": 31.2, "4": 31.3, "5": 31.4, "6": 31.5, "7": 31.6, "8": 31.7}, "c": {"1": 32.0, "2": 32.1, "4": 32.3, "6": 32.5, "7": 32.6}, "d": {"1": 1e+20, "2": 1e+20, "3": 1e+20, "4": 1e+20, "5": 1e+20, "6": 1e+20, "7": 1e+20, "8": 1e+20}, "e": {"1": 34.0, "2": 34.1, "3": 34.2, "4": 34.3, "5": 34.4, "6": 34.5, "7": 34.6, "8": 34.7}}}, "json_version": 3.0, "provenance": {"platform": {"OS": "Linux", "Version": "4.15.0-43-generic", "Name": "drdoom"}, "userId": "doutriaux1", "osAccess": false, "commandLine": "create_json_for_merge.py", "date": "2019-03-07 16:04:11", "conda": {}, "packages": {}, "openGL": {"GLX": {"server": {}, "client": {}}}}} \ No newline at end of file diff --git a/tests/test_pmp_jsons_merge.py b/tests/test_pmp_jsons_merge.py new file mode 100644 index 000000000..6a0d61ccc --- /dev/null +++ b/tests/test_pmp_jsons_merge.py @@ -0,0 +1,153 @@ +import unittest +import pcmdi_metrics +import inspect +import os +import numpy +import json + + +class TestJSONs(unittest.TestCase): + def testMerge(self): + pth = os.path.dirname(inspect.getfile(self.__class__)) + J = pcmdi_metrics.io.base.JSONs([os.path.join( + pth, "io", "merge.json")], + oneVariablePerFile=False) + axes_ids = J.getAxisIds() + axes = J.getAxisList() + self.assertEqual(axes_ids, ["upper", "lower", "numbers"]) + data = J() + self.assertEqual(data.shape, (4, 5, 8)) + self.assertEqual(data[2, 3, 6], 23.6) + + merged = J(merge=["lower", "numbers"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "lower_numbers"]) + self.assertEqual(merged.shape, (4, 40)) + self.assertEqual(merged[2, 8], 21.) + + merged = J(merge=["numbers", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "numbers_lower"]) + self.assertEqual(merged.shape, (4, 40)) + self.assertEqual(merged[2, 8], 23.1) + + merged = J(merge=["upper", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_lower", "numbers"]) + self.assertEqual(merged.shape, (20, 8)) + self.assertEqual(merged[7, 5], 12.5) + + merged = J(merge=["lower", "upper"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["lower_upper", "numbers"]) + self.assertEqual(merged.shape, (20, 8)) + self.assertEqual(merged[7, 5], 31.5) + + merged = J(merge=["upper", "numbers"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_numbers", "lower"]) + self.assertEqual(merged.shape, (32, 5)) + self.assertEqual(merged[13, 4], 14.5) + + merged = J(merge=["numbers", "upper"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["numbers_upper", "lower"]) + self.assertEqual(merged.shape, (32, 5)) + self.assertEqual(merged[14, 4], 24.3) + + with self.assertRaises(RuntimeError) as context: + merged = J(merge=["upper", "lower", "number"]) + self.assertTrue("You requested to merge axis is 'number' which is not valid." in str( + context.exception)) + + merged = J(merge=["upper", "lower", "numbers"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_lower_numbers", ]) + self.assertEqual(merged.shape, (160,)) + self.assertEqual(merged[121], 30.1) + + merged = J(merge=["upper", "numbers", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_numbers_lower", ]) + self.assertEqual(merged.shape, (160,)) + self.assertEqual(merged[121], 31.) + + merged = J(merge=["numbers", "upper", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["numbers_upper_lower", ]) + self.assertEqual(merged.shape, (160,)) + self.assertEqual(merged[121], 1.6) + + # Merge and subset in different order + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["upper", "numbers"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_numbers", "lower"]) + self.assertEqual(merged.shape, (15, 4)) + self.assertEqual(merged[9, 2], 13.2) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["numbers", "upper"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["numbers_upper", "lower"]) + self.assertEqual(merged.shape, (15, 4)) + self.assertEqual(merged[9, 2], 23.6) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["lower", "upper"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["lower_upper", "numbers"]) + self.assertEqual(merged.shape, (12, 5)) + self.assertEqual(merged[9, 2], 22.) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["upper", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper_lower", "numbers"]) + self.assertEqual(merged.shape, (12, 5)) + self.assertEqual(merged[9, 2], 30.) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["numbers", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "numbers_lower"]) + self.assertEqual(merged.shape, (3, 20)) + self.assertEqual(merged[2, 9], 30.) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["numbers", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "numbers_lower"]) + self.assertEqual(merged.shape, (3, 20)) + self.assertEqual(merged[2, 7], 32.1) + + merged = J(upper=["C", "B", "D"], numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], merge=["lower", "numbers"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "lower_numbers"]) + self.assertEqual(merged.shape, (3, 20)) + self.assertEqual(merged[2, 7], 30.) + + merged = J(numbers=["4", "2", "1", "7", "3"], lower=[ + "e", "a", "d", "c"], upper=["C", "B", "D"], merge=["lower", "numbers"], order=["lower_numbers", "upper"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["lower_numbers", "upper"]) + self.assertEqual(merged.shape, (20, 3)) + self.assertEqual(merged[7, 2], 30.) diff --git a/tests/test_pmp_jsons_merge_incomplete_missing.py b/tests/test_pmp_jsons_merge_incomplete_missing.py new file mode 100644 index 000000000..61b2e568e --- /dev/null +++ b/tests/test_pmp_jsons_merge_incomplete_missing.py @@ -0,0 +1,25 @@ +import unittest +import pcmdi_metrics +import inspect +import os +import numpy +import json + + +class TestJSONs(unittest.TestCase): + def testMerge(self): + pth = os.path.dirname(inspect.getfile(self.__class__)) + J = pcmdi_metrics.io.base.JSONs([os.path.join( + pth, "io", "merge_incomplete_and_missing.json")], + oneVariablePerFile=False) + + merged = J(merge=["numbers", "lower"]) + axes_ids = merged.getAxisIds() + axes = merged.getAxisList() + self.assertEqual(axes_ids, ["upper", "numbers_lower"]) + self.assertEqual(merged.shape, (4, 37)) + self.assertEqual(merged.getAxis(1)[:].tolist(), + ['1_a', '1_b', '1_c', '1_d', '1_e', '2_a', '2_b', '2_c', '2_d', '2_e', '3_a', '3_b', '3_d', '3_e', '4_a', '4_b', + '4_c', '4_d', '4_e', '5_a', '5_b', '5_d', '5_e', '6_a', '6_b', '6_c', '6_d', '6_e', '7_a', '7_b', '7_c', '7_d', '7_e', '8_a', '8_b', '8_d', '8_e']) + self.assertTrue(merged.asma()[1, 3] is numpy.ma.masked) + self.assertEqual(merged[1, 4], 14)