Commit b1fe6955 authored by BorjaEst's avatar BorjaEst
Browse files

Edit functional tests to split by action

parent 0b1bf15a
import unittest
import pandas as pd
import numpy as np
import xarray as xr
from o3skim import source
model = xr.Dataset(
data_vars=dict(
tco3_zm=(["lon", "lat", "time"], np.random.rand(3, 3, 25)),
vmro3_zm=(["lon", "lat", "plev", "time"], np.random.rand(3, 3, 4, 25))
),
coords=dict(
lon=[-180, 0, 180],
lat=[-90, 0, 90],
plev=[1, 10, 100, 1000],
time=pd.date_range("2000-01-01", periods=25, freq='A')
),
attrs=dict(description="Test dataset")
)
class TestsModel(unittest.TestCase):
def assertHasAttr(self, obj, intendedAttr):
testBool = hasattr(obj, intendedAttr)
msg = 'obj lacking an attribute. obj: %s, intendedAttr: %s' % (
obj, intendedAttr)
self.assertTrue(testBool, msg=msg)
def test_dataset_has_model_accessor(self):
self.assertHasAttr(model, 'model')
import unittest
import pandas as pd
import numpy as np
import xarray as xr
from o3skim import source
Source = source.Source
name = "SourceTest"
collections = {} # Empty, only to test constructor stability
class TestsSource(unittest.TestCase):
def setUp(self):
self.source = Source(name, collections)
def test_property_name(self):
self.assertEqual(name, self.source.name)
def test_property_models(self):
expected = list(collections.keys())
result = self.source.models
self.assertEqual(expected, result)
...@@ -6,28 +6,52 @@ import o3skim ...@@ -6,28 +6,52 @@ import o3skim
import pytest import pytest
import tests.mockup as mockup_data import tests.mockup as mockup_data
import xarray import xarray
import yaml
# configurations ---------------------------------------------------- # configurations ----------------------------------------------------
year_line = range(2000, 2022) year_line = range(2000, 2022)
sources = ["SourceSplit", "SourceMerged"] # Source files
sources_example = "tests/sources_example.yaml" sources_example = "tests/sources_example.yaml"
sources_err = "tests/sources_err.yaml" sources_err = "tests/sources_err.yaml"
available_configurations = {
'correct': {sources_example},
'with_errors': {sources_err}
}
models = ["ModelTCO3", "ModelVMRO3", "ModelALL"] # Sources
models_tco3 = ['ModelTCO3', 'ModelALL'] sources = ["SourceSplit", "SourceMerged"]
models_vmro3 = ['ModelVMRO3', 'ModelALL']
# Models
tco3_models = {
'SourceSplit_ModelTCO3',
'SourceSplit_ModelALL',
'SourceMerged_ModelTCO3',
'SourceMerged_ModelALL'
}
vmro3_models = {
'SourceSplit_ModelVMRO3',
'SourceSplit_ModelALL',
'SourceMerged_ModelVMRO3',
'SourceMerged_ModelALL'
}
available_models = {
'all': tco3_models | vmro3_models,
'with_tco3': tco3_models,
'only_tco3': tco3_models - vmro3_models,
'with_vmro3': vmro3_models,
'only_vmro3': vmro3_models - tco3_models
}
# session fixtures --------------------------------------------------- # session fixtures ---------------------------------------------------
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def data_dir(tmpdir_factory): def data_dir(tmpdir_factory):
data_dir = tmpdir_factory.mktemp("data") data_dir = tmpdir_factory.mktemp("data")
for source in sources: for source in sources:
source_dir = data_dir.join(source) source_dir = data_dir.join(source)
os.mkdir(source_dir) os.mkdir(source_dir)
with o3skim.cd(source_dir): with o3skim.utils.cd(source_dir):
if source == 'SourceMerged': if source == 'SourceMerged':
mockup_data.combined(year_line) mockup_data.combined(year_line)
mockup_data.noise(name='merged_noise.nc') mockup_data.noise(name='merged_noise.nc')
...@@ -40,99 +64,153 @@ def data_dir(tmpdir_factory): ...@@ -40,99 +64,153 @@ def data_dir(tmpdir_factory):
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def config_file(request): def output_dir(tmpdir_factory, groupby, model):
output_dir = tmpdir_factory.mktemp(model + "_by_" + str(groupby))
return output_dir
@pytest.fixture(scope='session')
def metadata_dir(tmpdir_factory, model):
metadata_dir = tmpdir_factory.mktemp(model + "_metadata")
return metadata_dir
@pytest.fixture(scope='session')
def config(request):
return request.param return request.param
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def config_dict(config_file): def model(request):
return o3skim.load(config_file) return request.param
# package fixtures -------------------------------------------------- @pytest.fixture(scope='session')
def config_dict(config):
return o3skim.utils.load(config)
# module fixtures --------------------------------------------------- @pytest.fixture(scope='session')
@pytest.fixture(scope='module') def load_model(data_dir, config_dict, model):
def output_dir(tmpdir_factory): model_configuration = config_dict[model]
return tmpdir_factory.mktemp("output") with o3skim.utils.cd(data_dir):
return o3skim.loading(**model_configuration)
@pytest.fixture(scope='module') @pytest.fixture(scope='session')
def groupby(request): def dataset(load_model):
dataset, metadata = load_model
return dataset
@pytest.fixture(scope='session')
def metadata(load_model):
dataset, metadata = load_model
return metadata
@pytest.fixture(scope='session')
def actions(request):
return request.param return request.param
@pytest.fixture(scope='module') @pytest.fixture(scope='session')
def source_name(request): def processed(dataset, actions):
return o3skim.processing(dataset, actions)
@pytest.fixture(scope='session')
def groupby(request):
return request.param return request.param
@pytest.fixture(scope='module') @pytest.fixture(scope='session')
def source(config_dict, source_name, data_dir): def splitted(dataset, groupby):
with o3skim.cd(data_dir): return o3skim.group(dataset, split_by=groupby)
source = o3skim.Source(source_name, **config_dict[source_name])
return source
@pytest.fixture(scope='session')
def years(splitted):
years, splitted_ds = splitted
return years
@pytest.fixture(scope='module')
def skimmed(groupby, source, output_dir):
with o3skim.cd(output_dir):
source.skim(groupby=groupby)
yield groupby, source.name
@pytest.fixture(scope='session')
def splitted_ds(splitted):
years, splitted_ds = splitted
return splitted_ds
# class fixtures --------------------------------------------------
@pytest.fixture(scope='session')
def variables(model):
variables = []
if model in available_models['with_tco3']:
variables.append('tco3_zm')
if model in available_models['with_vmro3']:
variables.append('vmro3_zm')
return variables
# function fixtures -------------------------------------------------
@pytest.fixture()
def model_name(request):
return request.param
@pytest.fixture(scope='session')
def expected_netCDF(groupby, variables):
if groupby == None:
return ["{}.nc".format(v)
for v in variables]
elif groupby == 'year':
def format(v, y): return "{}_{}-{}.nc".format(v, y, y+1)
return [format(v, y)
for y in year_line
for v in variables]
elif groupby == 'decade':
def format(v, y): return "{}_{}-{}.nc".format(v, y, y+10)
return [format(v, y)
for y in year_line if y % 10 == 0
for v in variables]
else:
raise KeyError("bad groupby {}".format(groupby))
@pytest.fixture()
def model(source, model_name):
return source[model_name]
@pytest.fixture(scope='session')
def save_netCDF(output_dir, splitted_ds, groupby, years):
with o3skim.utils.cd(output_dir):
o3skim.saving(splitted_ds, groupby, years)
@pytest.fixture()
def year(request):
return request.param
@pytest.fixture(scope='session')
def saved_ds(output_dir, save_netCDF, expected_netCDF):
with o3skim.utils.cd(output_dir):
return xarray.open_mfdataset(expected_netCDF)
@pytest.fixture()
def variable(request):
return request.param
@pytest.fixture(scope='session')
def save_metadata(metadata_dir, metadata):
with o3skim.utils.cd(metadata_dir):
o3skim.utils.save("metadata.yaml", metadata)
@pytest.fixture()
def metadata_file(skimmed, model_name):
_, source_name = skimmed
with o3skim.cd("{}_{}".format(source_name, model_name)):
yield "metadata.yaml"
@pytest.fixture(scope='session')
def metadata_file_content(metadata_dir, save_metadata):
with o3skim.utils.cd(metadata_dir):
return o3skim.utils.load("metadata.yaml")
@pytest.fixture()
def metadata_dict(metadata_file): # package fixtures --------------------------------------------------
with open(metadata_file, "r") as ymlfile:
config = yaml.safe_load(ymlfile)
return config
# module fixtures ---------------------------------------------------
# class fixtures --------------------------------------------------
# function fixtures -------------------------------------------------
@pytest.fixture() @pytest.fixture()
def skimed_file(skimmed, model_name, variable, year): def output_cd(output_dir):
groupby, source_name = skimmed with o3skim.utils.cd(output_dir):
with o3skim.cd("{}_{}".format(source_name, model_name)): yield None
if not groupby:
yield "{}.nc".format(variable)
if groupby == 'year':
yield "{}_{}-{}.nc".format(variable, year, year + 1)
if groupby == 'decade':
yield "{}_{}-{}.nc".format(variable, year, year + 10)
@pytest.fixture() @pytest.fixture()
def skimmed_model(skimed_file): def metadata_cd(metadata_dir):
with xarray.open_dataset(skimed_file) as model: with o3skim.utils.cd(metadata_dir):
yield model yield None
SourceSplit: SourceSplit_ModelTCO3:
metadata: metadata:
meta_0: Source metadata string example meta_1: Model metadata string example
meta_1: Source metadata to be replaced by Model
meta_2: meta_2:
meta_20: Sub-metadata from Source meta_21: Sub-metadata from Model
meta_21: Sub-metadata to be replaced by Model tco3_zm:
ModelTCO3: name: tco3
paths: SourceSplit/tco3_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata: metadata:
meta_1: Model metadata string example meta_tco3_1: TCO3 metadata string example
meta_2: meta_tco3_2: 0
meta_21: Sub-metadata from Model SourceSplit_ModelVMRO3:
tco3_zm: metadata:
name: tco3 meta_1: Model metadata string example
paths: SourceSplit/tco3_????.nc meta_2:
coordinates: meta_21: Sub-metadata from Model
time: time vmro3_zm:
lat: latitude name: vmro3
lon: longitude paths: SourceSplit/vmro3_????.nc
metadata: coordinates:
meta_tco3_1: TCO3 metadata string example time: time
meta_tco3_2: 0 plev: pressure_level
ModelVMRO3: lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
SourceSplit_ModelALL:
metadata:
meta_1: Model metadata string example
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
metadata: metadata:
meta_1: Model metadata string example meta_tco3_1: TCO3 metadata string example
meta_2: meta_tco3_2: 0
meta_21: Sub-metadata from Model name: tco3
vmro3_zm: paths: SourceSplit/tco3_????.nc
name: vmro3 coordinates:
paths: SourceSplit/vmro3_????.nc time: time
coordinates: lat: latitude
time: time lon: longitude
plev: pressure_level vmro3_zm:
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
ModelALL:
metadata: metadata:
meta_1: Model metadata string example meta_vmro3_1: VMRO3 metadata string example
meta_2: meta_vmro3_2: 0
meta_21: Sub-metadata from Model name: vmro3
tco3_zm: paths: SourceSplit/vmro3_????.nc
metadata: coordinates:
meta_tco3_1: TCO3 metadata string example time: time
meta_tco3_2: 0 plev: pressure_level
name: tco3 lat: latitude
paths: SourceSplit/tco3_????.nc lon: longitude
coordinates: SourceMerged_ModelTCO3:
time: time
lat: latitude
lon: longitude
vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceSplit/vmro3_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
SourceMerged:
metadata: metadata:
meta_0: Source metadata string example meta_1: Model metadata string example
meta_1: Source metadata to be replaced by Model
meta_2: meta_2:
meta_20: Sub-metadata from Source meta_21: Sub-metadata from Model
meta_21: Sub-metadata to be replaced by Model tco3_zm:
ModelTCO3:
metadata: metadata:
meta_1: Model metadata string example meta_tco3_1: TCO3 metadata string example
meta_2: meta_tco3_2: 0
meta_21: Sub-metadata from Model name: tco3
tco3_zm: paths: SourceMerged/merged_????.nc
metadata: coordinates:
meta_tco3_1: TCO3 metadata string example time: time
meta_tco3_2: 0 lat: latitude
name: tco3 lon: longitude
paths: SourceMerged/merged_????.nc SourceMerged_ModelVMRO3:
coordinates: metadata:
time: time meta_1: Model metadata string example
lat: latitude meta_2:
lon: longitude meta_21: Sub-metadata from Model
ModelVMRO3: vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
SourceMerged_ModelALL:
metadata:
meta_1: Model metadata string example
meta_2:
meta_21: Sub-metadata from Model
tco3_zm:
metadata: metadata:
meta_1: Model metadata string example meta_tco3_1: TCO3 metadata string example
meta_2: meta_tco3_2: 0
meta_21: Sub-metadata from Model name: tco3
vmro3_zm: paths: SourceMerged/merged_????.nc
metadata: coordinates:
meta_vmro3_1: VMRO3 metadata string example time: time
meta_vmro3_2: 0 lat: latitude
name: vmro3 lon: longitude
paths: SourceMerged/merged_????.nc vmro3_zm:
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
ModelALL:
metadata: metadata:
meta_1: Model metadata string example meta_vmro3_1: VMRO3 metadata string example
meta_2: meta_vmro3_2: 0
meta_21: Sub-metadata from Model name: vmro3
tco3_zm: paths: SourceMerged/merged_????.nc
metadata: coordinates:
meta_tco3_1: TCO3 metadata string example time: time
meta_tco3_2: 0 plev: pressure_level
name: tco3 lat: latitude
paths: SourceMerged/merged_????.nc lon: longitude
coordinates:
time: time
lat: latitude
lon: longitude
vmro3_zm:
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
...@@ -6,16 +6,30 @@ configs = conftest.available_configurations ...@@ -6,16 +6,30 @@ configs = conftest.available_configurations
models = conftest.available_models models = conftest.available_models
@pytest.mark.parametrize('config', configs['correct'], indirect=True)
@pytest.mark.parametrize('model', models['all'], indirect=True)
class TestO3CommonsLoad:
def test_attrs(self, dataset):
assert dataset.attrs == dict(description="Test ozone dataset")
def test_metadata(self, metadata):
assert metadata["meta_1"] == "Model metadata string example"
assert metadata["meta_2"]["meta_21"] == "Sub-metadata from Model"
@pytest.mark.parametrize('config', configs['correct'], indirect=True) @pytest.mark.parametrize('config', configs['correct'],