Commit 4342d3a7 authored by thomas.forbriger's avatar thomas.forbriger
Browse files

libdatrwxx [MERGE] quickfix mseed: provide consistency check control

Merge branch '25-libdatrwxx-miniseed-consistency-checks'

Introduce new format modifiers for mseed to provide control over consistency
check fatality and application to the user. This is required to gain access to
some ill-formatted MiniSEED data file.
parents a6fcffa4 126a90eb
......@@ -51,26 +51,34 @@ namespace datrw {
\author Thomas Forbriger
\since March 2004
\date September 2011
\date July 2016
\version V2.1
\date 18/07/2013
\date 05/07/2016
Contents of this page:
\tableofcontents
\section sec_main_contents Contents
\subsection subsec_main_contents_page This page
- \ref sec_concept
- \ref sec_Fortran
Other pages:
- For library users:
- \ref page_formats
- \ref page_reading_howto
- \ref page_reading_operation
- \ref page_writing_howto
- \ref page_writing_operation
- For library maintainers and programmers:
- \ref page_implement_istream
- \ref page_implement_ostream
- \ref page_modifiers
- \ref page_notes
\subsection subsec_main_others Other pages
\subsubsection subsubsec_main_others_users For library users
- \subpage page_formats
- \subpage page_reading_howto
- \subpage page_reading_operation
- \subpage page_writing_howto
- \subpage page_writing_operation
\subsubsection subsubsec_main_others_programmers For library maintainers and programmers
- \subpage page_implement_istream
- \subpage page_implement_ostream
- \subpage page_modifiers
- \subpage page_utilities
- \subpage page_notes
.
----
\section sec_concept Concept of the library
......@@ -629,7 +637,7 @@ The operators are provided in namespace ::datrw.
\section sec_binary_io I/O for binary data formats
Currently the library uses stream function read and write for binary output
in modules like the \ref mseed.
in modules like the \ref group_mseed.
These functions apparently are remainders of a pre-standard implementation
of streams (prior to the invention of stream buffers).
The GNU libstdc++ documentations recommends to do binary I/O on the
......
......@@ -16,4 +16,39 @@
* ============================================================================
*/
/*! \page page_utilities Internal utilities
\tableofcontents
\note
This page is still under construction.
\section sec_util_formatmodifiers
\sa \ref formatmodifiers
\section sec_util_datatypes
\sa \ref datrw::datatypes
\sa datatypes.h
\section sec_util_properties
\sa \subpage page_properties
\sa \ref properties
\section sec_util_bytesex
\sa \ref group_bytesex
\section sec_util_error
\sa \ref group_error
\section sec_util_reservoir
\sa reservoir.h
*/
/* ----- END OF doxygen_utilities.txt ----- */
......@@ -28,8 +28,10 @@
* Copyright (c) 2011 by Thomas Forbriger (BFO Schiltach)
*
* REVISIONS and CHANGES
* - 28/07/2011 V1.0 Thomas Forbriger
* - 28/07/2011 V1.0 Thomas Forbriger (thof)
* - 21/01/2012 V1.1 provide online help output support class
* - 05/07/2016 V1.2 thof: provide output operator for line indentation
* with modifier help
*
* ============================================================================
*/
......@@ -38,7 +40,7 @@
#ifndef DATRW_FORMATMODIFIER_H_VERSION
#define DATRW_FORMATMODIFIER_H_VERSION \
"DATRW_FORMATMODIFIER_H V1.1"
"DATRW_FORMATMODIFIER_H V1.2"
#include <string>
#include <sstream>
......@@ -132,7 +134,7 @@ namespace datrw {
/*! \brief Help formatting modifier online help
* \ingroup formatmodifiers
* \sa datrw::su::osustream::help()
* \sa datrw::osustream::help()
*/
class ModifierHelp
{
......@@ -149,6 +151,18 @@ namespace datrw {
unsigned int Mwidth;
}; // class ModifierHelp
/*! \brief Output operator for ModifierHelp
* \ingroup formatmodifiers
*
* For line indentation an object of type ModifierHelp can be used like an
* output stream;
*
* \sa datrw::mseed::help()
*/
template<class C>
std::ostream& operator<<(ModifierHelp& mh, const C& c)
{ return(mh() << c); }
} // namespace formatmodifiers
/*----------------------------------------------------------------------*/
......
......@@ -32,6 +32,7 @@
* - 13/09/2011 V1.2 support format modifiers for ASCII dump
* - 08/05/2014 V1.3 Add field for timing tolerance value
* - 22/07/2014 V1.4 thof: support new format modifier: estimateNframes
* - 05/07/2016 V1.5 thof: provide consistency check control flags
*
* ============================================================================
*/
......@@ -40,7 +41,7 @@
#ifndef DATRW_MSEED_H_VERSION
#define DATRW_MSEED_H_VERSION \
"DATRW_MSEED_H V1.4"
"DATRW_MSEED_H V1.5"
#include<datrwxx/datread.h>
......@@ -52,14 +53,76 @@ namespace datrw {
extern const char* const streamID;
} // namespace mseed
/*----------------------------------------------------------------------*/
namespace mseed {
/*! Container for flags to control consistency checks
*
* \ingroup group_mseed
*/
struct ConsistencyCheckControl {
ConsistencyCheckControl(const bool& fatal, const bool& check):
docheck(fatal), fatal(check) { }
/*! \brief Activation of check.
*
* If true, the respective check shall be applied.
*/
bool docheck;
/*! \brief Consequences of check.
*
* If true, an exception is thrown, if the respective check fails.
*/
bool fatal;
}; // struct ConsistencyCheckControl
/*----------------------------------------------------------------------*/
/*! Container for flags to control consistency checks
* \ingroup group_mseed
*/
struct ConsistencyChecks {
ConsistencyChecks(const bool& fatal, const bool& check):
nframes(fatal, check),
nsamples(fatal, check),
data(fatal, check) { }
/*! \brief Check for consistent number of frames.
*
* Number of frames to be expected is provided in the header in
* \ref page_mseed_blockette1001
* This value will be compared against the number of frames actually
* used to store data.
*/
ConsistencyCheckControl nframes;
/*! \brief Check for consistent number of samples.
*
* Number of samples to be expected is provided in the header in
* \ref page_mseed_FixedSectionDataHeader
* This value will be compared against the number of samples actually
* provided in the logical record.
*/
ConsistencyCheckControl nsamples;
/*! \brief Check for consistent data values.
*
* Page 142 of "SEED Reference Manual, Standard for the Exchange of
* Earthquake Data, SEED Format Version 2.4, August, 2012":
*
* > The reverse integrating constant also provides for a quick data
* > integrity check when compared with the last computed
* > sample. A discrepancy indicates that the contents of the data are
* > garbled.
*
* The sample value of the last sample in the record is compared against
* the value of the reverse integration constant as provided in the
* first data frame of the record.
*/
ConsistencyCheckControl data;
}; // struct ConsistencyChecks
/*----------------------------------------------------------------------*/
/*! structure to hold one MiniSEEDRecord in SFF format
*
* \ingroup mseed
* \ingroup group_mseed
*/
struct Record {
::sff::WID2 wid2;
......@@ -69,7 +132,8 @@ namespace datrw {
bool valid;
void read(std::istream& is,
const bool& dumpascii=false,
const bool& estimateNframes=false);
const bool& estimateNframes=false,
const ConsistencyChecks& checks=ConsistencyChecks(true, true));
}; // struct Record
} // namespace mseed
......@@ -78,7 +142,7 @@ namespace datrw {
/*! \brief class to read mini-SEED data
*
* \ingroup mseed
* \ingroup group_mseed
*/
class imseedstream: public idatstream {
public:
......@@ -104,6 +168,8 @@ namespace datrw {
bool Mdumpascii; //!< dump ASCII data if true
double Mttolerance; //!< timing tolerance in microseconds
bool MestimateNframes; //!< estimate frame count
//! flags controlling consistency checks
datrw::mseed::ConsistencyChecks Mchecks;
}; // class imseedstream
} // namespace datrw
......
......@@ -28,7 +28,7 @@
\section page_mseed_lib_structure Library structure.
- Module:
- \ref mseed
- \ref group_mseed
- Namespaces
- datrw::mseed
- datrw::mseed::SEED
......
......@@ -35,11 +35,13 @@
* - 13/09/2011 V1.4 support format modifiers for ASCII dump
* - 08/05/2014 V1.5 add format modifier to tolerate time jitter
* - 22/07/2014 V1.6 thof: support new format modifier: estimateNframes
* - 05/07/2016 V1.7 thof: evaluate format modifiers to adjust
* consistency checks
*
* ============================================================================
*/
#define DATRW_IMSEEDSTREAM_CC_VERSION \
"DATRW_IMSEEDSTREAM_CC V1.6"
"DATRW_IMSEEDSTREAM_CC V1.7"
#include <datrwxx/util.h>
#include <datrwxx/mseed.h>
......@@ -48,7 +50,8 @@
#include <datrwxx/debug.h>
#include <vector>
#include <string>
#include<datrwxx/formatmodifier.h>
#include <datrwxx/formatmodifier.h>
#include <datrwxx/mseed_keywords.h>
namespace datrw {
......@@ -57,7 +60,7 @@ namespace datrw {
//@{
/*! \brief Format properties
* \ingroup mseed
* \ingroup group_mseed
*/
const bool mseed::isbinary=true;
const char* const mseed::streamID="mseed";
......@@ -67,6 +70,7 @@ namespace datrw {
/*! \class imseedstream
*
* \ingroup group_mseed
* \paragraph mseed_imseedstream_internals Internals of imseedtream
*
* We must always be one record in advance. Data reading will be done by
......@@ -85,21 +89,49 @@ namespace datrw {
const std::string& modifier,
const bool& debug):
Tbase(is, true, true, true), Mdebug(debug), Mmodifier(modifier),
Mdumpascii(false)
Mdumpascii(false), Mchecks(true, true)
{
datrw::Subformat subformat(Mmodifier);
Mdumpascii=subformat.isset("dumpascii");
subformat("ttolerance", "0.") >> this->Mttolerance;
MestimateNframes=subformat.isset("estimateNframes");
Mdumpascii=subformat.isset(mseed::key::dumpascii);
subformat(mseed::key::ttolerance, "0.") >> this->Mttolerance;
MestimateNframes=subformat.isset(mseed::key::estimateNframes);
// note: flags have opposite meaning compared to subformat flags
if (subformat.isset(mseed::key::nonfatal))
{
std::string checks=subformat.value(mseed::key::nonfatal);
bool all=(checks.find(mseed::key::all)!=std::string::npos);
Mchecks.nframes.fatal
=!(all||(checks.find(mseed::key::nframes)!=std::string::npos));
Mchecks.nsamples.fatal
=!(all||(checks.find(mseed::key::nsamples)!=std::string::npos));
Mchecks.data.fatal
=!(all||(checks.find(mseed::key::data)!=std::string::npos));
}
// note: flags have opposite meaning compared to subformat flags
if (subformat.isset(mseed::key::skipcheck))
{
std::string checks=subformat.value(mseed::key::skipcheck);
bool all=(checks.find(mseed::key::all)!=std::string::npos);
Mchecks.nframes.docheck
=!(all||(checks.find(mseed::key::nframes)!=std::string::npos));
Mchecks.nsamples.docheck
=!(all||(checks.find(mseed::key::nsamples)!=std::string::npos));
Mchecks.data.docheck
=!(all||(checks.find(mseed::key::data)!=std::string::npos));
}
DATRW_assert_modifiers_are_recognized(subformat,
"imseedstream::imseedstream()");
/* We must always be one record in advance. Data reading will be done by
* concatenating several MiniSEED records. A time series will be
* terminated after reading a noncontiguous MiniSEED record. We have to
* store this record after reading and thus must always be one record
* ahead.
*/
Mrecord.read(Mis, Mdumpascii, MestimateNframes);
Mrecord.read(Mis, Mdumpascii, MestimateNframes, Mchecks);
DATRW_debug(Mdebug, "Mrecord read", Mrecord.wid2.line());
} // imseedstream::imseedstream(std::istream& is, const bool& debug)
......@@ -216,13 +248,18 @@ namespace datrw {
DATRW_debug(Mdebug, "imseedstream::read", "last data index " <<
data.last());
/*
* Remember value of last sample in previous record to be compared to
* retrospect value derived from the succeeding record. A mismatch will
* indicate non-contiguous data.
*/
int xm1=data(data.last());
nsamples += Mrecord.wid2.nsamples;
nextwid2.date=wid2line.date+dt*nsamples;
DATRW_debug(Mdebug, "imseedstream::read", "read next");
Mrecord.read(Mis, Mdumpascii, MestimateNframes);
Mrecord.read(Mis, Mdumpascii, MestimateNframes, Mchecks);
DATRW_debug(Mdebug, "imseedstream::read", "compare next");
DATRW_debug(Mdebug, "imseedstream::read", "expected wid2: " <<
......@@ -279,11 +316,6 @@ namespace datrw {
return (series);
} // Tiseries imseedstream::read(const bool& skipdata)
/*----------------------------------------------------------------------*/
void imseedstream::help(std::ostream& os)
{ datrw::mseed::help(os); }
} // namespace datrw
/* ----- END OF imseedstream.cc ----- */
/*! \file imseedstream_help.cc
* \brief imseedstream help function (implementation)
*
* ----------------------------------------------------------------------------
*
* \author Thomas Forbriger
* \date 05/07/2016
*
* imseedstream help function (implementation)
*
* Copyright (c) 2016 by Thomas Forbriger (BFO Schiltach)
*
* ----
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ----
*
* REVISIONS and CHANGES
* - 05/07/2016 V1.0 Thomas Forbriger
*
* ============================================================================
*/
#define DATRW_IMSEEDSTREAM_HELP_CC_VERSION \
"DATRW_IMSEEDSTREAM_HELP_CC V1.0"
#include <datrwxx/mseed.h>
#include <datrwxx/formatmodifier.h>
#include <datrwxx/mseed_keywords.h>
namespace datrw {
using std::endl;
using namespace mseed;
void imseedstream::help(std::ostream& os)
{
os << std::endl;
os << "MiniSEED data reading functions" << std::endl;
os << "-------------------------------" << std::endl;
os << endl;
os << "This module is designed to read MiniSEED (data only SEED) files"
<< endl;
os << "created by several different data acquisition systems. It is"
<< endl;
os << "tested with the following systems:"
<< endl;
os << " comserv with Q680 (Quanterra with SeisComP, GRSN setup)"
<< endl;
os << " seedlink (new SeisComP system with EDD plugin)"
<< endl;
os << " Earth Data Logger (EDL raw files)"
<< endl;
os << " Q330HR (Quanterra data acquisition system with Baler)"
<< endl;
os << " Taurus (Nanometrics data acquisition system)"
<< endl;
os << " Q330HR (data streamed from Q330HR)"
<< endl;
os << " Cube3 (data converted by cube2mseed)"
<< endl;
os << endl;
os << "Special features:"
<< endl;
os << "Data files written by the datalog client of comserv commence"
<< endl;
os << "with a telemetry volume header which is non-standard MiniSEED."
<< endl;
os << "This module identifies and skips the telemetry volume headers."
<< endl << endl;
os << "The module swaps data bytes if necessary. The SEED standard"
<< endl;
os << "defines (page 12 of SEED manual) that all headers must have"
<< endl;
os << "big-endian byte-sex. The byte-sex of MiniSEED data blocks is"
<< endl;
os << "defined in the header. The Earth Data Logger violates this"
<< endl;
os << "definition and writes headers in little-endian order. The"
<< endl;
os << "module tries to guess the correct byte-sex in this case"
<< endl;
os << "when reading the header blockettes."
<< endl << endl;
os << "Several consistency checks (see below) are applied to the"
<< endl;
os << "data file. The library module this way refuses to read data"
<< endl;
os << "which obviously is corrupt or which header parameters are"
<< endl;
os << "not plausible or are at variance with the data actually read."
<< endl;
os << "There are data of known sources which fail these tests,"
<< endl;
os << "although the data itself is intact. Such files are considered"
<< endl;
os << "to be at variance with the definition of Data Only SEED (MiniSEED)."
<< endl;
os << "To be able to work on such data, consistency checks can be made"
<< endl;
os << "non-fatal or even be switched of by format modifiers (see below)."
<< endl;
os << endl;
{
formatmodifiers::ModifierHelp mh(os, 24);
os << "Available format modifiers:\n";
mh(key::dumpascii)
<< "dumps ASCII data blocks to stdout\n";
mh(key::ttolerance, "T")
<< "tolerate time jitter, if not larger than\n";
mh << "\"T\", where \"T\" is given in microseconds\n";
mh(key::estimateNframes)
<< "estimate number of frames rather than reading\n";
mh << "it from blockette1001; this is required for\n";
mh << "data created by \"cube2mseed\" which contains\n";
mh << "an invalid count of frames\n";
mh(key::nonfatal, "ID[,ID,ID]")
<< "make consistency check \"ID\" non-fatal (report only)\n";
mh << "one or more check \"ID\" values may be passed in a list\n";
mh(key::skipcheck, "ID[,ID,ID]")
<< "skip consistency check \"ID\" entirely (ignore)\n";
mh << "one or more check \"ID\" values may be passed in a list\n";
os << endl;
}
{
formatmodifiers::ModifierHelp mh(os, 12);
os << "Available consistency checks (IDs to be used with modifiers):\n";
mh(key::nframes)
<< "The number of frames to be expected is specified in\n";
mh << "the header in Blockette 1001. This value will be\n";
mh << "compared against the number of frames actually\n";
mh << "used to store data.\n";
mh << "This test is only applied if the number of frames\n";
mh << "actually is specified in the header. Some data require\n";
mh << "the number of frames to be guessed (if Blockette 1001)\n";
mh << "is missing.\n";
mh(key::nsamples)
<< "The number of samples to be expected is provided in\n";
mh << "the header in the fixed section data header. This\n";
mh << "value will be compared against the number of\n";
mh << "samples actually provided in the logical record.\n";
mh(key::data)
<< "The sample value of the last sample in the record\n";
mh << "is compared against the value of the reverse\n";
mh << "integration constant as provided in the first data\n";
mh << "frame of the record. Both must be identical.\n";
mh(key::all)
<< "Select all consistency checks\n";
os << "Consistency checks will be selected upon a substring match\n"
"against check IDs. This means that by passing only a substring\n"
"in the argument list to the format modifier several checks\n"
"can be selected at once,\n";
os << endl;
}
} // void imseedstream::help(std::ostream& os)
} // namespace datrw
/* ----- END OF imseedstream_help.cc ----- */
/*! \file mseed_keywords.cc
* \brief keywords used in mseed module (implementation)
*
* ----------------------------------------------------------------------------
*
* \author Thomas Forbriger
* \date 05/07/2016
*
* keywords used in mseed module (implementation)
*
* Copyright (c) 2016 by Thomas Forbriger (BFO Schiltach)
*
* ----
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ----
*
* REVISIONS and CHANGES
* - 05/07/2016 V1.0 Thomas Forbriger
*
* ============================================================================
*/
#define DATRW_MSEED_KEYWORDS_CC_VERSION \
"DATRW_MSEED_KEYWORDS_CC V1.0"
#include <datrwxx/mseed_keywords.h>
namespace datrw {
namespace mseed {
const char* const key::dumpascii="dumpascii";