Skip to content

GitLab

Commit bfd21b18 authored by BorjaEst's avatar BorjaEst
Browse files

Merge branch 'dev' into 7-add-metadata-file-to-output

parents cf003d33 d068463d
Show whitespace changes
Inline Side-by-side
README.md
View file @ bfd21b18
......@@ -115,7 +115,7 @@ On top, [tox](https://tox.readthedocs.io/en/latest/) automation is used to simpl
To run white and black box tests use:
```sh
$ tox tests o3skim/*.py
$ tox
```
## BlackBox tests - [Unittest framework](https://docs.python.org/3/library/unittest.html)
......@@ -123,7 +123,7 @@ Located inside package modules (./o3skim). This helps to test easily functions a
To run only white tests use:
```sh
$ tox o3skim/*.py
$ tox o3skim
```
## BlackBox tests - [Pytest framework](https://docs.pytest.org/en/stable/)
......
docs/dev_guide/tests.rst
View file @ bfd21b18
......@@ -29,7 +29,7 @@ To run White and Black-Box tests use:
.. code-block:: bash
$ tox tests o3skim/*.py
$ tox
...
py36: commands succeeded
...
......@@ -49,14 +49,12 @@ naming conventions. Therefore all Black-Box tests should be
located on the **tests** folder at the package root and start
with **test**. For example *test_sources.py*.
More than 500 test combinations are generated using which otherwise
might not be feasible using other python test frameworks.
.. _pytest: https://docs.pytest.org/en/stable/
.. _test_discovery: https://docs.pytest.org/en/reorganize-docs/new-docs/user/naming_conventions.html
To run only Black-Box tests simply call tox followed by the
folder with the test location:
More than 500 test combinations are generated using which otherwise
might not be feasible using other python test frameworks. To run
only Black-Box tests simply call tox followed by the folder with the
test location:
.. code-block:: bash
......@@ -78,15 +76,12 @@ makes it not suitable for Black-Box testing without a very complex
customization.
To simplify code usage and testing, the white tests should be located
on the same file than the function / class are supposed to test.
To run only White tests simply call tox followed by the module files
you would like to test. You can also use the wildcard '*' to selected
and test all python modules:
inside the package folder. To run only White tests simply call tox
followed by the package name:
.. code-block:: bash
$ tox o3skim/*.py
$ tox o3skim
...
py36: commands succeeded
...
......
example.yaml
View file @ bfd21b18
# For more information see the documentation at
# ./docs/user_guide/source-file.rst
CCMI-1:
IPSL:
SourceSplit:
metadata:
meta_0: Source metadata string example
ModelTCO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
tco3_zm:
name: toz
paths: Ccmi/mon/toz/*.nc
name: tco3
paths: SourceSplit/tco3_????.nc
coordinates:
time: time
lat: lat
lon: lon
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
ModelVMRO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
vmro3_zm:
name: vmro3
paths: Ccmi/mon/toz/*.nc
paths: SourceSplit/vmro3_????.nc
coordinates:
time: time
plev: plev
lat: lat
lon: lon
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
ModelALL:
metadata:
meta_1: Model metadata string example
meta_2: 0
tco3_zm:
name: tco3
paths: SourceSplit/tco3_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
vmro3_zm:
name: vmro3
paths: SourceSplit/vmro3_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
SourceMerged:
metadata:
meta_0: Source metadata string example
ModelTCO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
tco3_zm:
name: tco3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
ModelVMRO3:
metadata:
meta_1: Model metadata string example
meta_2: 0
vmro3_zm:
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
ModelALL:
metadata:
meta_1: Model metadata string example
meta_2: 0
tco3_zm:
name: tco3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
lat: latitude
lon: longitude
metadata:
meta_tco3_1: TCO3 metadata string example
meta_tco3_2: 0
vmro3_zm:
name: vmro3
paths: SourceMerged/merged_????.nc
coordinates:
time: time
plev: pressure_level
lat: latitude
lon: longitude
metadata:
meta_vmro3_1: VMRO3 metadata string example
meta_vmro3_2: 0
o3skim/extended_xarray.py
View file @ bfd21b18
......@@ -63,102 +63,3 @@ class ModelAccessor:
logger.debug("Skimming model")
return self._model.mean(mean_coord)
class Tests(unittest.TestCase):
tco3 = np.random.rand(3, 3, 25)
vmro3 = np.random.rand(3, 3, 4, 25)
longitude = [-180, 0, 180]
latitude = [-90, 0, 90]
pressure_level = [1, 10, 100, 1000]
time = pd.date_range("2000-01-01", periods=25, freq='A')
@staticmethod
def tco3_datarray():
return xr.DataArray(
data=Tests.tco3,
dims=["lon", "lat", "time"],
coords=dict(
lon=Tests.longitude,
lat=Tests.latitude,
time=Tests.time
),
attrs=dict(description="Test tco3 datarray")
)
@staticmethod
def vmro3_datarray():
return xr.DataArray(
data=Tests.vmro3,
dims=["lon", "lat", "plev", "time"],
coords=dict(
lon=Tests.longitude,
lat=Tests.latitude,
plev=Tests.pressure_level,
time=Tests.time
),
attrs=dict(description="Test vmro3 datarray")
)
def setUp(self):
self.ds = xr.Dataset(
data_vars=dict(
tco3_zm=self.tco3_datarray(),
vmro3_zm=self.vmro3_datarray()
),
coords=dict(
lon=Tests.longitude,
lat=Tests.latitude,
plev=Tests.pressure_level,
time=Tests.time
),
attrs=dict(description="Test dataset")
)
def test_tco3_property(self):
expected = Tests.tco3_datarray().to_dataset(name="tco3_zm")
xr.testing.assert_equal(self.ds.model.tco3, expected)
def test_vmro3_property(self):
expected = Tests.vmro3_datarray().to_dataset(name="vmro3_zm")
xr.testing.assert_equal(self.ds.model.vmro3, expected)
def test_metadata_property(self):
metadata = self.ds.model.metadata
self.assertEqual(metadata["description"], "Test dataset")
self.assertEqual(metadata["tco3_zm"]
["description"], "Test tco3 datarray")
self.assertEqual(metadata["vmro3_zm"]
["description"], "Test vmro3 datarray")
def test_groupby_year(self):
groups = self.ds.model.groupby_year()
self.assertEqual(25, len(groups))
for year, dataset in groups:
self.assertIsInstance(year, np.int64)
self.assertIsInstance(dataset, xr.Dataset)
def test_groupby_decade(self):
groups = self.ds.model.groupby_decade()
self.assertEqual(3, len(groups))
for decade, dataset in groups:
self.assertIsInstance(decade, np.int64)
self.assertIsInstance(dataset, xr.Dataset)
def test_skimming(self):
result = self.ds.model.skim()
# Test general coordinates
self.assertIn('time', result.coords)
self.assertIn('lat', result.coords)
self.assertIn('plev', result.coords)
self.assertNotIn('lon', result.coords)
# Test tco3 coordinates
self.assertIn('time', result.model.tco3.coords)
self.assertIn('lat', result.model.tco3.coords)
self.assertNotIn('plev', result.model.tco3.coords)
self.assertNotIn('lon', result.model.tco3.coords)
# Test vmro3 coordinates
self.assertIn('time', result.model.vmro3.coords)
self.assertIn('lat', result.model.vmro3.coords)
self.assertIn('plev', result.model.vmro3.coords)
self.assertNotIn('lon', result.model.vmro3.coords)
o3skim/source.py
View file @ bfd21b18
......@@ -182,54 +182,3 @@ def _skim(model, delta=None, metadata=None):
if metadata:
logger.debug("Creating metadata.yaml file")
utils.save(file_name="metadata.yaml", metadata=metadata)
class TestsSource(unittest.TestCase):
name = "SourceTest"
collections = {} # Empty, only to test constructor stability
def setUp(self):
self.source = Source(TestsSource.name, TestsSource.collections)
def test_property_name(self):
expected = TestsSource.name
result = self.source.name
self.assertEqual(expected, result)
def test_property_models(self):
expected = list(TestsSource.collections.keys())
result = self.source.models
self.assertEqual(expected, result)
class TestsModel(unittest.TestCase):
tco3 = np.random.rand(3, 3, 25)
vmro3 = np.random.rand(3, 3, 4, 25)
@ staticmethod
def model():
return xr.Dataset(
data_vars=dict(
tco3_zm=(["lon", "lat", "time"], TestsModel.tco3),
vmro3_zm=(["lon", "lat", "plev", "time"], TestsModel.vmro3)
),
coords=dict(
lon=[-180, 0, 180],
lat=[-90, 0, 90],
plev=[1, 10, 100, 1000],
time=pd.date_range("2000-01-01", periods=25, freq='A')
),
attrs=dict(description="Test dataset")
)
def assertHasAttr(self, obj, intendedAttr):
testBool = hasattr(obj, intendedAttr)
msg = 'obj lacking an attribute. obj: %s, intendedAttr: %s' % (
obj, intendedAttr)
self.assertTrue(testBool, msg=msg)
def test_dataset_has_model_accessor(self):
model = TestsModel.model()
self.assertHasAttr(model, 'model')
o3skim/standardization.py
View file @ bfd21b18
......@@ -83,107 +83,3 @@ def sort(array):
"""Sorts an array by coordinates"""
logger.debug("Sorting coordinates in dataset")
return array.sortby(list(array.coords))
class TestsTCO3(unittest.TestCase):
data_s = np.mgrid[1:3:3j, 1:3:3j, 1:3:3j][0]
data_r = np.mgrid[3:1:3j, 3:1:3j, 1:1:1j, 3:1:3j][0]
varname = "tco3"
coords = {'lon': 'long', 'lat': 'latd', 'time': 'time'}
@staticmethod
def non_standard_ds():
return xr.Dataset(
data_vars=dict(
tco3=(["long", "latd", "high", "time"], TestsTCO3.data_r)
),
coords=dict(
long=[180, 0, -180],
latd=[90, 0, -90],
high=[1],
time=pd.date_range("2000-01-03", periods=3, freq='-1d')
),
attrs=dict(description="Non standardized dataset")
)
@staticmethod
def standard_ds():
return xr.Dataset(
data_vars=dict(
tco3_zm=(["lon", "lat", "time"], TestsTCO3.data_s)
),
coords=dict(
time=pd.date_range("2000-01-01", periods=3, freq='1d'),
lat=[-90, 0, 90],
lon=[-180, 0, 180]
),
attrs=dict(description="Standardized dataset")
)
def test_standardize(self):
standardized_tco3 = standardize_tco3(
dataset=TestsTCO3.non_standard_ds(),
variable=TestsTCO3.varname,
coordinates=TestsTCO3.coords)
xr.testing.assert_equal(TestsTCO3.standard_ds(), standardized_tco3)
def test_fail_returns_empty_dataset(self):
empty_dataset = standardize_tco3(
dataset=TestsTCO3.non_standard_ds(),
variable="badVariable",
coordinates=TestsTCO3.coords)
xr.testing.assert_equal(xr.Dataset(), empty_dataset)
class TestsVMRO3(unittest.TestCase):
data_s = np.mgrid[1:3:3j, 1:3:3j, 1:4:4j, 1:3:3j][0]
data_r = np.mgrid[3:1:3j, 3:1:3j, 4:1:4j, 3:1:3j][0]
varname = "vmro3"
coords = {'lon': 'longit', 'lat': 'latitu',
'plev': 'level', 'time': 't'}
@staticmethod
def non_standard_ds():
return xr.Dataset(
data_vars=dict(
vmro3=(["longit", "latitu", "level", "t"], TestsVMRO3.data_r)
),
coords=dict(
longit=[180, 0, -180],
latitu=[90, 0, -90],
level=[1000, 100, 10, 1],
t=pd.date_range("2000-01-03", periods=3, freq='-1d')
),
attrs=dict(description="Non standardized dataset")
)
@staticmethod
def standard_ds():
return xr.Dataset(
data_vars=dict(
vmro3_zm=(["lon", "lat", "plev", "time"], TestsVMRO3.data_s)
),
coords=dict(
time=pd.date_range("2000-01-01", periods=3, freq='1d'),
plev=[1, 10, 100, 1000],
lat=[-90, 0, 90],
lon=[-180, 0, 180]
),
attrs=dict(description="Standardized dataset")
)
def test_standardize(self):
standardized_vmro3 = standardize_vmro3(
dataset=TestsVMRO3.non_standard_ds(),
variable=TestsVMRO3.varname,
coordinates=TestsVMRO3.coords)
xr.testing.assert_equal(TestsVMRO3.standard_ds(), standardized_vmro3)
def test_fail_returns_empty_dataset(self):
empty_dataset = standardize_vmro3(
dataset=TestsVMRO3.non_standard_ds(),
variable="badVariable",
coordinates=TestsVMRO3.coords)
xr.testing.assert_equal(xr.Dataset(), empty_dataset)
o3skim/test/__init__.py 0 → 100644
View file @ bfd21b18
""" """
pass
\ No newline at end of file
o3skim/test/test_extended_xarray.py 0 → 100644
View file @ bfd21b18
import unittest
import xarray as xr
import pandas as pd
import numpy as np
from o3skim import extended_xarray
tco3 = np.random.rand(3, 3, 25)
vmro3 = np.random.rand(3, 3, 4, 25)
longitude = [-180, 0, 180]
latitude = [-90, 0, 90]
pressure_level = [1, 10, 100, 1000]
time = pd.date_range("2000-01-01", periods=25, freq='A')
tco3_datarray = xr.DataArray(
data=tco3,
dims=["lon", "lat", "time"],
coords=dict(
lon=longitude,
lat=latitude,
time=time
),
attrs=dict(description="Test tco3 xarray")
)
vmro3_datarray = xr.DataArray(
data=vmro3,
dims=["lon", "lat", "plev", "time"],
coords=dict(
lon=longitude,
lat=latitude,
plev=pressure_level,
time=time
),
attrs=dict(description="Test vmro3 xarray")
)
dataset = xr.Dataset(
data_vars=dict(
tco3_zm=tco3_datarray,
vmro3_zm=vmro3_datarray
),
coords=dict(
lon=longitude,
lat=latitude,
plev=pressure_level,
time=time
),
attrs=dict(description="Test dataset")
)
class Tests(unittest.TestCase):
def test_tco3_property(self):
expected = tco3_datarray.to_dataset(name="tco3_zm")
xr.testing.assert_equal(dataset.model.tco3, expected)
def test_vmro3_property(self):
expected = vmro3_datarray.to_dataset(name="vmro3_zm")
xr.testing.assert_equal(dataset.model.vmro3, expected)
def test_metadata_property(self):
meta = dataset.model.metadata
self.assertEqual(meta["description"], "Test dataset")
self.assertEqual(meta["tco3_zm"]["description"], "Test tco3 xarray")
self.assertEqual(meta["vmro3_zm"]["description"], "Test vmro3 xarray")
def test_groupby_year(self):
groups = dataset.model.groupby_year()
self.assertEqual(25, len(groups))
for year, ds in groups:
self.assertIsInstance(year, np.int64)
self.assertIsInstance(ds, xr.Dataset)
def test_groupby_decade(self):
groups = dataset.model.groupby_decade()
self.assertEqual(3, len(groups))
for decade, ds in groups:
self.assertIsInstance(decade, np.int64)
self.assertIsInstance(ds, xr.Dataset)
def test_skimming_gen_coords(self):
result = dataset.model.skim()
self.assertIn('time', result.coords)
self.assertIn('lat', result.coords)
self.assertIn('plev', result.coords)
self.assertNotIn('lon', result.coords)
def test_skimming_tco3_coords(self):
result = dataset.model.skim()
self.assertIn('time', result.model.tco3.coords)
self.assertIn('lat', result.model.tco3.coords)
self.assertNotIn('plev', result.model.tco3.coords)
self.assertNotIn('lon', result.model.tco3.coords)
def test_skimming_vmro3_coords(self):
result = dataset.model.skim()
self.assertIn('time', result.model.vmro3.coords)
self.assertIn('lat', result.model.vmro3.coords)
self.assertIn('plev', result.model.vmro3.coords)
self.assertNotIn('lon', result.model.vmro3.coords)
o3skim/test/test_model.py 0 → 100644
View file @ bfd21b18
import unittest
import pandas as pd
import numpy as np
import xarray as xr
from o3skim import source
model = xr.Dataset(
data_vars=dict(
tco3_zm=(["lon", "lat", "time"], np.random.rand(3, 3, 25)),
vmro3_zm=(["lon", "lat", "plev", "time"], np.random.rand(3, 3, 4, 25))
),
coords=dict(
lon=[-180, 0, 180],
lat=[-90, 0, 90],
plev=[1, 10, 100, 1000],
time=pd.date_range("2000-01-01", periods=25, freq='A')
),
attrs=dict(description="Test dataset")
)
class TestsModel(unittest.TestCase):
def assertHasAttr(self, obj, intendedAttr):
testBool = hasattr(obj, intendedAttr)
msg = 'obj lacking an attribute. obj: %s, intendedAttr: %s' % (
obj, intendedAttr)
self.assertTrue(testBool, msg=msg)
def test_dataset_has_model_accessor(self):
self.assertHasAttr(model, 'model')
o3skim/test/test_source.py 0 → 100644
View file @ bfd21b18
import unittest
import pandas as pd
import numpy as np