From cb232ab38cafe98344f5661700ae867940ee3c92 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 10 Jul 2019 14:58:03 -0700 Subject: [PATCH 1/2] allow to dump in multiple files --- pcmdi_metrics/driver/outputmetrics.py | 11 ++++++++++- pcmdi_metrics/io/base.py | 11 ++++++++--- pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py | 4 +++- pcmdi_metrics/version.py | 4 ++-- tests/pcmdi/basic_test_parameters_file.py | 4 ++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/pcmdi_metrics/driver/outputmetrics.py b/pcmdi_metrics/driver/outputmetrics.py index 0650f0329..f5739b131 100644 --- a/pcmdi_metrics/driver/outputmetrics.py +++ b/pcmdi_metrics/driver/outputmetrics.py @@ -90,6 +90,14 @@ def setup_out_file(self): self.out_file.realm = self.realm self.out_file.table = self.table_realm self.out_file.case_id = self.parameter.case_id + if hasattr(self, "obs_or_model"): + self.out_file.model_version = self.obs_or_model + for key in self.out_file.keys(): + if hasattr(self.parameter, key): + setattr(self.out_file, key, getattr(self.parameter, key)) + if hasattr(self, key): + setattr(self.out_file, key, getattr(self, key)) + DataSet.apply_custom_keys(self.out_file, self.parameter.custom_keys, self.var) def add_region(self, region): @@ -291,5 +299,6 @@ def write_on_exit(self): self.out_file.write(self.metrics_dictionary, json_structure=["model", "reference", "rip", "region", "statistic", "season"], indent=4, - separators=(',', ': ')) + separators=(',', ': '), + mode="r+") self.out_file.write(self.metrics_dictionary, type='txt') diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py index 20d985d71..1c0130616 100755 --- a/pcmdi_metrics/io/base.py +++ b/pcmdi_metrics/io/base.py @@ -278,7 +278,7 @@ def __call__(self): def read(self): pass - def write(self, data, type='json', *args, **kwargs): + def write(self, data, type='json', mode="w", *args, **kwargs): self.type = type.lower() file_name = self() dir_path = os.path.split(file_name)[0] @@ -309,8 +309,13 @@ def write(self, data, type='json', *args, **kwargs): del(kwargs[k]) data["json_version"] = json_version data["json_structure"] = json_structure - f = open(file_name, 'w') - out_dict = OrderedDict({"provenance": generateProvenance()}) + + if mode == "r+" and os.path.exists(file_name): + f = open(file_name) + out_dict = json.load(f) + else: + out_dict = OrderedDict({"provenance": generateProvenance()}) + f = open(file_name, "w") out_dict.update(data) json.dump(out_dict, f, cls=CDMSDomainsEncoder, *args, **kwargs) f.close() diff --git a/pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py b/pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py index 5030efa91..cb2254ad9 100644 --- a/pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py +++ b/pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py @@ -66,7 +66,8 @@ def run_diags(self): self.output_metric.add_region(self.region) # Runs obs vs obs, obs vs model, or model vs model self.run_reference_and_test_comparison() - self.output_metric.write_on_exit() + self.output_metric.setup_out_file() + self.output_metric.write_on_exit() def load_obs_dict(self): ''' Loads obs_info_dictionary.json and appends @@ -181,6 +182,7 @@ def run_reference_and_test_comparison(self): try: tst = self.determine_obs_or_model(test_data_set_is_obs, test, self.parameter.test_data_path) + self.output_metric.obs_or_model = tst.obs_or_model # TODO Make this a custom exception. This exception is for # when a model doesn't have sftlf for a given region except RuntimeError: diff --git a/pcmdi_metrics/version.py b/pcmdi_metrics/version.py index 9ded0f33c..ef79a1511 100644 --- a/pcmdi_metrics/version.py +++ b/pcmdi_metrics/version.py @@ -1,3 +1,3 @@ __version__ = 'v1.2' -__git_tag_describe__ = 'v1.2-61-gde534f5' -__git_sha1__ = 'de534f543dce2db1a7761932ef1f5bf6e32bf324' +__git_tag_describe__ = 'v1.2-58-gec93e57' +__git_sha1__ = 'ec93e57248be0ad4b6c81d860e30714717f82c6b' diff --git a/tests/pcmdi/basic_test_parameters_file.py b/tests/pcmdi/basic_test_parameters_file.py index 4f2a5258a..ea7ffe25d 100644 --- a/tests/pcmdi/basic_test_parameters_file.py +++ b/tests/pcmdi/basic_test_parameters_file.py @@ -89,6 +89,10 @@ test_clims_interpolated_output = os.path.join( 'pcmdi_install_test_results', 'interpolated_model_clims') + +# Filename template for json files: +output_json_template = "%(variable)%(level)_%(model_version)_%(target_grid_name)_%(regrid_tool)_%(regrid_method)_metricsssss" + # FILENAME FOR INTERPOLATED CLIMATOLOGIES OUTPUT filename_output_template = "%(variable)%(level)_" + \ "%(model_version)_%(table)_historical_%(realization)_%(period)" + \ From 05d57ba95a816a8542c5b8b7de53f13a6cb7c330 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 10 Jul 2019 15:17:58 -0700 Subject: [PATCH 2/2] flake8ed --- pcmdi_metrics/driver/outputmetrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/driver/outputmetrics.py b/pcmdi_metrics/driver/outputmetrics.py index f5739b131..64a47d9d0 100644 --- a/pcmdi_metrics/driver/outputmetrics.py +++ b/pcmdi_metrics/driver/outputmetrics.py @@ -97,7 +97,7 @@ def setup_out_file(self): setattr(self.out_file, key, getattr(self.parameter, key)) if hasattr(self, key): setattr(self.out_file, key, getattr(self, key)) - + DataSet.apply_custom_keys(self.out_file, self.parameter.custom_keys, self.var) def add_region(self, region):