Commit 35ad9caf authored by BorjaEst's avatar BorjaEst
Browse files

Merge branch '9-implement-output'

parents 9a34b212 1a5aa128
"""This module creates the sources objects"""
import glob
import yaml
import netCDF4
import xarray as xr
import os.path
def load(yaml_file):
......@@ -12,43 +14,68 @@ def load(yaml_file):
class Source:
"""Standarized datasets and methods from a data source"""
def __init__(self, collections):
self.model = {}
def __init__(self, sname, collections):
self._name = sname
self._models = {}
for name, variables in collections.items():
self.model[name] = Model(variables)
def get_tco3_zm(tco3_zm=None, **kwarg):
"""Gets and standarises the tco3_zm data"""
if tco3_zm:
fnames = glob.glob(tco3_zm['dir'] + "/*.nc")
with xr.open_mfdataset(fnames) as ds:
tco3_zm = ds.rename({
tco3_zm['name']: 'tco3_zm',
tco3_zm['coordinades']['time']: 'time',
tco3_zm['coordinades']['lat']: 'lat',
tco3_zm['coordinades']['lon']: 'lon'
})['tco3_zm']
return tco3_zm
def get_vrm_zm(vrm_zm=None, **kwarg):
"""Gets and standarises the vrm_zm data"""
if vrm_zm:
fnames = glob.glob(vrm_zm['dir'] + "/*.nc")
with xr.open_mfdataset(fnames) as ds:
vrm_zm = ds.rename({
vrm_zm['name']: 'tco3_zm',
vrm_zm['coordinades']['time']: 'time',
vrm_zm['coordinades']['lat']: 'lat',
vrm_zm['coordinades']['lon']: 'lon'
})['tco3_zm']
return vrm_zm
self._models[name] = Model(variables)
def skim(self):
for name, model in self._models.items():
path = self._name + "_" + name
os.makedirs(path, exist_ok=True)
model.skim(path)
class Model:
"""Standarised model with standarised variables"""
def __init__(self, variables):
self.tco3_zm = get_tco3_zm(**variables)
self.vrm_zm = get_vrm_zm(**variables)
self.get_tco3_zm(**variables)
self.get_vrm_zm(**variables)
def skim(self, path):
if hasattr(self, '_tco3_zm'):
to_netcdf(path, "tco3_zm", self._tco3_zm)
if hasattr(self, '_vrm_zm'):
to_netcdf(path, "vrm_zm", self._vrm_zm)
def get_tco3_zm(self, tco3_zm=None, **kwarg):
"""Gets and standarises the tco3_zm data"""
if tco3_zm:
fnames = glob.glob(tco3_zm['dir'] + "/*.nc")
with xr.open_mfdataset(fnames) as ds:
self._tco3_zm = ds.rename({
tco3_zm['name']: 'tco3_zm',
tco3_zm['coordinades']['time']: 'time',
tco3_zm['coordinades']['lat']: 'lat',
tco3_zm['coordinades']['lon']: 'lon'
})['tco3_zm'].to_dataset()
def get_vrm_zm(self, vrm_zm=None, **kwarg):
"""Gets and standarises the vrm_zm data"""
if vrm_zm:
fnames = glob.glob(vrm_zm['dir'] + "/*.nc")
with xr.open_mfdataset(fnames) as ds:
self._vrm_zm = ds.rename({
vrm_zm['name']: 'tco3_zm',
vrm_zm['coordinades']['time']: 'time',
vrm_zm['coordinades']['lat']: 'lat',
vrm_zm['coordinades']['lon']: 'lon'
})['tco3_zm'].to_dataset()
def create_empty_netCDF(fname):
"""Creates a new empty netCDF file"""
root_grp = netCDF4.Dataset(fname, 'w', format='NETCDF4')
root_grp.description = 'Example simulation data'
root_grp.close()
def to_netcdf(path, name, dataset):
"""Creates or appends data to named netcdf files"""
years, dsx = zip(*dataset.groupby("time.year"))
fnames = [path + "/" + name + "_%s.nc" % y for y in years]
[create_empty_netCDF(fn) for fn in fnames if not os.path.isfile(fn)]
xr.save_mfdataset(dsx, fnames, mode='a')
......@@ -5,11 +5,12 @@ import numpy as np
import netCDF4
import os.path
import datetime
from o3skim import sources
base = datetime.datetime(2000, 1, 1)
indexes = {
'time': [base + datetime.timedelta(days=i) for i in range(365)],
'time': [base + datetime.timedelta(days=10*i) for i in range(99)],
'plev': [x for x in range(1, 1000, 100)],
'lat': [x for x in range(-90, 90, 10)],
'lon': [x for x in range(-180, 180, 20)]
......@@ -36,17 +37,7 @@ def dataset(name, coordinades):
)
def create_empty_netCDF(fname):
"""Creates a new empty netCDF file"""
root_grp = netCDF4.Dataset(fname, 'w', format='NETCDF4')
root_grp.description = 'Example simulation data'
root_grp.close()
def netcdf(path, name, coordinades, **kwarg):
"""Creates or appends data to a mock netcdf file"""
ds = dataset(name, coordinades)
months, dsx = zip(*ds.groupby("time.month"))
fnames = [path + "/source_file_%s.nc" % m for m in months]
[create_empty_netCDF(fn) for fn in fnames if not os.path.isfile(fn)]
xr.save_mfdataset(dsx, fnames, mode='a')
sources.to_netcdf(path, name, ds)
......@@ -28,9 +28,9 @@ class TestO3SKIM_sources(unittest.TestCase):
def create_mock_datasets(self):
"""Creates mock data files according to the loaded configuration"""
for source, collection in self.config.items():
for model, variables in collection.items():
for v, vinfo in variables.items():
for _, collection in self.config.items():
for _, variables in collection.items():
for _, vinfo in variables.items():
path = "data/" + vinfo["dir"]
os.makedirs(path, exist_ok=True)
mockup_data.netcdf(path, **vinfo)
......@@ -60,19 +60,43 @@ class TestO3SKIM_sources(unittest.TestCase):
def test_000_SourcesFromConfig(self):
"""Creates the different sources from the configuration file"""
with utils.cd("data"):
ds = {source: sources.Source(collection) for
source, collection in self.config.items()}
ds = {name: sources.Source(name, collection) for
name, collection in self.config.items()}
# CCMI-1 tco3_zm asserts
self.assertTrue('time' in ds['CCMI-1'].model['IPSL'].tco3_zm.coords)
self.assertTrue('lon' in ds['CCMI-1'].model['IPSL'].tco3_zm.coords)
self.assertTrue('lat' in ds['CCMI-1'].model['IPSL'].tco3_zm.coords)
self.assertTrue('time' in ds['CCMI-1']._models['IPSL']._tco3_zm.coords)
self.assertTrue('lon' in ds['CCMI-1']._models['IPSL']._tco3_zm.coords)
self.assertTrue('lat' in ds['CCMI-1']._models['IPSL']._tco3_zm.coords)
# CCMI-1 vrm_zm asserts
self.assertTrue('time' in ds['CCMI-1'].model['IPSL'].vrm_zm.coords)
self.assertTrue('plev' in ds['CCMI-1'].model['IPSL'].vrm_zm.coords)
self.assertTrue('lon' in ds['CCMI-1'].model['IPSL'].vrm_zm.coords)
self.assertTrue('lat' in ds['CCMI-1'].model['IPSL'].vrm_zm.coords)
self.assertTrue('time' in ds['CCMI-1']._models['IPSL']._vrm_zm.coords)
self.assertTrue('plev' in ds['CCMI-1']._models['IPSL']._vrm_zm.coords)
self.assertTrue('lon' in ds['CCMI-1']._models['IPSL']._vrm_zm.coords)
self.assertTrue('lat' in ds['CCMI-1']._models['IPSL']._vrm_zm.coords)
# Checks the original data has not been modified
self.assert_with_backup()
def test_000_OutputFromSources(self):
"""Skims the data into the output folder"""
with utils.cd("data"):
ds = {name: sources.Source(name, collection) for
name, collection in self.config.items()}
with utils.cd("output"):
[source.skim() for source in ds.values()]
# CCMI-1 data skim asserts
self.assertTrue(os.path.isdir("output/CCMI-1_IPSL"))
self.assertTrue(os.path.exists("output/CCMI-1_IPSL/tco3_zm_2000.nc"))
self.assertTrue(os.path.exists("output/CCMI-1_IPSL/vrm_zm_2000.nc"))
# ECMWF data skim asserts
self.assertTrue(os.path.isdir("output/ECMWF_ERA-5"))
self.assertTrue(os.path.exists("output/ECMWF_ERA-5/tco3_zm_2000.nc"))
self.assertTrue(os.path.isdir("output/ECMWF_ERA-i"))
self.assertTrue(os.path.exists("output/ECMWF_ERA-i/tco3_zm_2000.nc"))
self.assertTrue(os.path.exists("output/ECMWF_ERA-i/vrm_zm_2000.nc"))
# Checks the original data has not been modified
self.assert_with_backup()
......@@ -24,7 +24,7 @@ ECMWF:
lon: longitude
lat: latitude
time: time
ERA-interim:
ERA-i:
tco3_zm:
name: toz
dir: Ecmwf/Erai
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment