Commit ada7d97e authored by borja.sanchis's avatar borja.sanchis
Browse files

Merge branch '7-add-metadata-file-to-output' into 'master'

Resolve "Add metadata file to output"

Closes #7

See merge request !29
parents bdf6642c 8011c6cc
......@@ -89,9 +89,9 @@ class Source:
os.makedirs(dirname, exist_ok=True)
logger.info("Skimming data from '%s'", dirname)
with utils.cd(dirname):
source_metadata = self.metadata
model_metadata = self[model].model.metadata
metadata = {**source_metadata, **model_metadata}
metadata = {} # copy() does not recurse inside dict
utils.mergedicts(metadata, self.metadata)
utils.mergedicts(metadata, self[model].model.metadata)
_skim(self[model], delta=groupby, metadata=metadata)
......
import copy
import unittest
from o3skim import utils
dict_1 = {'a': 1, 'c': 0, 'z': {'a': 1, 'c': 0}}
dict_2 = {'b': 2, 'c': 3, 'z': {'b': 2, 'c': 3}}
class Tests_mergedict(unittest.TestCase):
def test_merge_d1d2(self):
dict_3 = copy.deepcopy(dict_1)
utils.mergedicts(dict_3, dict_2)
self.assertEqual(dict_2, {'b': 2, 'c': 3, 'z': {'b': 2, 'c': 3}})
self.assertEqual(dict_3['a'], 1)
self.assertEqual(dict_3['b'], 2)
self.assertEqual(dict_3['c'], 3)
self.assertEqual(dict_3['z'], {'a': 1, 'b': 2, 'c': 3})
def test_merge_d2d1(self):
dict_3 = copy.deepcopy(dict_2)
utils.mergedicts(dict_3, dict_1)
self.assertEqual(dict_1, {'a': 1, 'c': 0, 'z': {'a': 1, 'c': 0}})
self.assertEqual(dict_3['a'], 1)
self.assertEqual(dict_3['b'], 2)
self.assertEqual(dict_3['c'], 0)
self.assertEqual(dict_3['z'], {'a': 1, 'b': 2, 'c': 0})
......@@ -88,3 +88,20 @@ def save(file_name, metadata):
"""
with open(file_name, 'w+') as ymlfile:
yaml.dump(metadata, ymlfile, allow_unicode=True)
def mergedicts(d1, d2):
"""Merges dict d2 in dict d2 recursively. If two keys exist in
both dicts, the value in d1 is superseded by the value in d2.
:param d1: Dict to be recursively completed by d2.
:type d1: dict
:param d2: Dict to be recursively merged in d1.
:type d2: dict
"""
for key in d2:
if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict):
mergedicts(d1[key], d2[key])
else:
d1[key] = d2[key]
SourceSplit:
metadata:
meta_0: Source metadata string example
meta_1: Source metadata to be replaced by model
meta_1: Source metadata to be replaced by Model
meta_2:
meta_20: Sub-metadata from Source
meta_21: Sub-metadata to be replaced by Model
ModelTCO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
name: tco3
paths: SourceSplit/tco3_????.nc
......@@ -19,7 +23,8 @@ SourceSplit:
ModelVMRO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
vmro3_zm:
name: vmro3
paths: SourceSplit/vmro3_????.nc
......@@ -34,18 +39,22 @@ SourceSplit:
ModelALL:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
name: tco3
paths: SourceSplit/tco3_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceSplit/vmro3_????.nc
coordinates:
......@@ -53,32 +62,37 @@ SourceSplit:
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
SourceMerged:
metadata:
meta_0: Source metadata string example
meta_1: Source metadata to be replaced by model
meta_1: Source metadata to be replaced by Model
meta_2:
meta_20: Sub-metadata from Source
meta_21: Sub-metadata to be replaced by Model
ModelTCO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
name: tco3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
ModelVMRO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
......@@ -86,24 +100,25 @@ SourceMerged:
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
ModelALL:
metadata:
meta_1: Model metadata string example
meta_2: 0
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
name: tco3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
......@@ -111,6 +126,3 @@ SourceMerged:
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
......@@ -98,7 +98,9 @@ class TestSkimming_Common:
def test_metadata_commons(self, metadata_dict, variable):
assert metadata_dict["meta_0"] == "Source metadata string example"
assert metadata_dict["meta_1"] == "Model metadata string example"
assert metadata_dict["meta_2"] == 0
submeta_dict = metadata_dict["meta_2"]
assert submeta_dict["meta_20"] == "Sub-metadata from Source"
assert submeta_dict["meta_21"] == "Sub-metadata from Model"
@pytest.mark.parametrize('model_name', conftest.models_tco3, indirect=True)
def test_metadata_tco3(self, metadata_dict, variable):
......
......@@ -99,7 +99,9 @@ class TestSkimming_Common:
def test_metadata_commons(self, metadata_dict, variable):
assert metadata_dict["meta_0"] == "Source metadata string example"
assert metadata_dict["meta_1"] == "Model metadata string example"
assert metadata_dict["meta_2"] == 0
submeta_dict = metadata_dict["meta_2"]
assert submeta_dict["meta_20"] == "Sub-metadata from Source"
assert submeta_dict["meta_21"] == "Sub-metadata from Model"
@pytest.mark.parametrize('model_name', conftest.models_vmro3, indirect=True)
def test_metadata_vmro3(self, metadata_dict, variable):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment