Commit 52df2ba8 authored by thomas.forbriger's avatar thomas.forbriger
Browse files

[MERGE] make recent updates from branch master available

Merge branch 'master' into 24-libdatrwxx-miniseed-format-specification
parents 181c09b1 fc78e388
......@@ -3,6 +3,12 @@ this is <CHANGELOG>
Recent development in Seitosh (bug fixes, new features, etc)
------------------------------------------------------------
06.07.2016 (4342d3a79): libdatrwxx provides new format modifiers
Format modifiers skipcheck and nonfatal which can be applied to the mseed
format provide user control over consistency checks to be applied to
MiniSEED data.
27.06.2016 (7ffa0fa1d): libtfxx provides hexdump function
(0d462617e): fix libdatrwxx
......
# this is <Makefile>
# ----------------------------------------------------------------------------
# $Id$
#
# Copyright (c) 2002 by Thomas Forbriger (IMG Frankfurt)
#
......@@ -29,6 +28,8 @@
# 09/02/2010 V1.1 checked for new compiler
# 14/01/2011 V1.2 provide reinstall target
# 03/02/2014 thof: removed comment stripping
# 30/06/2016 thof: pass proper inlcude path to CXX compiler when
# creating dependency files
#
# ============================================================================
#
......@@ -129,7 +130,7 @@ install: $(LOCLIBDIR)
# The compiler is used to create dependency files, which are included below.
%.d: %.cc
$(SHELL) -ec '$(CXX) -M $(TMPPARDEF) $< \
$(SHELL) -ec '$(CXX) $(CPPFLAGS) -M $(TMPPARDEF) $< \
| sed '\''s,\($(notdir $*)\)\.o[ :]*,$(dir $@)\1.o $@ : ,g'\'' \
> $@; \
[ -s $@ ] || rm -f $@'
......
......@@ -412,7 +412,7 @@ namespace datrw {
const std::string& datatype,
const bool& nonfatal)
{
typedef typename aff::Series<T> Tinseries;
//typedef typename aff::Series<T> Tinseries;
std::string line;
if (datatype==ascii::keydouble)
{
......
......@@ -51,26 +51,34 @@ namespace datrw {
\author Thomas Forbriger
\since March 2004
\date September 2011
\date July 2016
\version V2.1
\date 18/07/2013
\date 05/07/2016
Contents of this page:
\tableofcontents
\section sec_main_contents Contents
\subsection subsec_main_contents_page This page
- \ref sec_concept
- \ref sec_Fortran
Other pages:
- For library users:
- \ref page_formats
- \ref page_reading_howto
- \ref page_reading_operation
- \ref page_writing_howto
- \ref page_writing_operation
- For library maintainers and programmers:
- \ref page_implement_istream
- \ref page_implement_ostream
- \ref page_modifiers
- \ref page_notes
\subsection subsec_main_others Other pages
\subsubsection subsubsec_main_others_users For library users
- \subpage page_formats
- \subpage page_reading_howto
- \subpage page_reading_operation
- \subpage page_writing_howto
- \subpage page_writing_operation
\subsubsection subsubsec_main_others_programmers For library maintainers and programmers
- \subpage page_implement_istream
- \subpage page_implement_ostream
- \subpage page_modifiers
- \subpage page_utilities
- \subpage page_notes
.
----
\section sec_concept Concept of the library
......@@ -629,7 +637,7 @@ The operators are provided in namespace ::datrw.
\section sec_binary_io I/O for binary data formats
Currently the library uses stream function read and write for binary output
in modules like the \ref mseed.
in modules like the \ref group_mseed.
These functions apparently are remainders of a pre-standard implementation
of streams (prior to the invention of stream buffers).
The GNU libstdc++ documentations recommends to do binary I/O on the
......
......@@ -16,4 +16,39 @@
* ============================================================================
*/
/*! \page page_utilities Internal utilities
\tableofcontents
\note
This page is still under construction.
\section sec_util_formatmodifiers
\sa \ref formatmodifiers
\section sec_util_datatypes
\sa \ref datrw::datatypes
\sa datatypes.h
\section sec_util_properties
\sa \subpage page_properties
\sa \ref properties
\section sec_util_bytesex
\sa \ref group_bytesex
\section sec_util_error
\sa \ref group_error
\section sec_util_reservoir
\sa reservoir.h
*/
/* ----- END OF doxygen_utilities.txt ----- */
......@@ -28,8 +28,10 @@
* Copyright (c) 2011 by Thomas Forbriger (BFO Schiltach)
*
* REVISIONS and CHANGES
* - 28/07/2011 V1.0 Thomas Forbriger
* - 28/07/2011 V1.0 Thomas Forbriger (thof)
* - 21/01/2012 V1.1 provide online help output support class
* - 05/07/2016 V1.2 thof: provide output operator for line indentation
* with modifier help
*
* ============================================================================
*/
......@@ -38,7 +40,7 @@
#ifndef DATRW_FORMATMODIFIER_H_VERSION
#define DATRW_FORMATMODIFIER_H_VERSION \
"DATRW_FORMATMODIFIER_H V1.1"
"DATRW_FORMATMODIFIER_H V1.2"
#include <string>
#include <sstream>
......@@ -132,7 +134,7 @@ namespace datrw {
/*! \brief Help formatting modifier online help
* \ingroup formatmodifiers
* \sa datrw::su::osustream::help()
* \sa datrw::osustream::help()
*/
class ModifierHelp
{
......@@ -149,6 +151,18 @@ namespace datrw {
unsigned int Mwidth;
}; // class ModifierHelp
/*! \brief Output operator for ModifierHelp
* \ingroup formatmodifiers
*
* For line indentation an object of type ModifierHelp can be used like an
* output stream;
*
* \sa datrw::mseed::help()
*/
template<class C>
std::ostream& operator<<(ModifierHelp& mh, const C& c)
{ return(mh() << c); }
} // namespace formatmodifiers
/*----------------------------------------------------------------------*/
......
......@@ -32,6 +32,7 @@
* - 13/09/2011 V1.2 support format modifiers for ASCII dump
* - 08/05/2014 V1.3 Add field for timing tolerance value
* - 22/07/2014 V1.4 thof: support new format modifier: estimateNframes
* - 05/07/2016 V1.5 thof: provide consistency check control flags
*
* ============================================================================
*/
......@@ -40,7 +41,7 @@
#ifndef DATRW_MSEED_H_VERSION
#define DATRW_MSEED_H_VERSION \
"DATRW_MSEED_H V1.4"
"DATRW_MSEED_H V1.5"
#include<datrwxx/datread.h>
......@@ -52,14 +53,76 @@ namespace datrw {
extern const char* const streamID;
} // namespace mseed
/*----------------------------------------------------------------------*/
namespace mseed {
/*! Container for flags to control consistency checks
*
* \ingroup group_mseed
*/
struct ConsistencyCheckControl {
ConsistencyCheckControl(const bool& fatal, const bool& check):
docheck(fatal), fatal(check) { }
/*! \brief Activation of check.
*
* If true, the respective check shall be applied.
*/
bool docheck;
/*! \brief Consequences of check.
*
* If true, an exception is thrown, if the respective check fails.
*/
bool fatal;
}; // struct ConsistencyCheckControl
/*----------------------------------------------------------------------*/
/*! Container for flags to control consistency checks
* \ingroup group_mseed
*/
struct ConsistencyChecks {
ConsistencyChecks(const bool& fatal, const bool& check):
nframes(fatal, check),
nsamples(fatal, check),
data(fatal, check) { }
/*! \brief Check for consistent number of frames.
*
* Number of frames to be expected is provided in the header in
* \ref page_mseed_blockette1001
* This value will be compared against the number of frames actually
* used to store data.
*/
ConsistencyCheckControl nframes;
/*! \brief Check for consistent number of samples.
*
* Number of samples to be expected is provided in the header in
* \ref page_mseed_FixedSectionDataHeader
* This value will be compared against the number of samples actually
* provided in the logical record.
*/
ConsistencyCheckControl nsamples;
/*! \brief Check for consistent data values.
*
* Page 142 of "SEED Reference Manual, Standard for the Exchange of
* Earthquake Data, SEED Format Version 2.4, August, 2012":
*
* > The reverse integrating constant also provides for a quick data
* > integrity check when compared with the last computed
* > sample. A discrepancy indicates that the contents of the data are
* > garbled.
*
* The sample value of the last sample in the record is compared against
* the value of the reverse integration constant as provided in the
* first data frame of the record.
*/
ConsistencyCheckControl data;
}; // struct ConsistencyChecks
/*----------------------------------------------------------------------*/
/*! structure to hold one MiniSEEDRecord in SFF format
*
* \ingroup mseed
* \ingroup group_mseed
*/
struct Record {
::sff::WID2 wid2;
......@@ -69,7 +132,8 @@ namespace datrw {
bool valid;
void read(std::istream& is,
const bool& dumpascii=false,
const bool& estimateNframes=false);
const bool& estimateNframes=false,
const ConsistencyChecks& checks=ConsistencyChecks(true, true));
}; // struct Record
} // namespace mseed
......@@ -78,7 +142,7 @@ namespace datrw {
/*! \brief class to read mini-SEED data
*
* \ingroup mseed
* \ingroup group_mseed
*/
class imseedstream: public idatstream {
public:
......@@ -104,6 +168,8 @@ namespace datrw {
bool Mdumpascii; //!< dump ASCII data if true
double Mttolerance; //!< timing tolerance in microseconds
bool MestimateNframes; //!< estimate frame count
//! flags controlling consistency checks
datrw::mseed::ConsistencyChecks Mchecks;
}; // class imseedstream
} // namespace datrw
......
......@@ -28,7 +28,7 @@
\section page_mseed_lib_structure Library structure.
- Module:
- \ref mseed
- \ref group_mseed
- Namespaces
- datrw::mseed
- datrw::mseed::SEED
......
......@@ -35,11 +35,13 @@
* - 13/09/2011 V1.4 support format modifiers for ASCII dump
* - 08/05/2014 V1.5 add format modifier to tolerate time jitter
* - 22/07/2014 V1.6 thof: support new format modifier: estimateNframes
* - 05/07/2016 V1.7 thof: evaluate format modifiers to adjust
* consistency checks
*
* ============================================================================
*/
#define DATRW_IMSEEDSTREAM_CC_VERSION \
"DATRW_IMSEEDSTREAM_CC V1.6"
"DATRW_IMSEEDSTREAM_CC V1.7"
#include <datrwxx/util.h>
#include <datrwxx/mseed.h>
......@@ -48,7 +50,8 @@
#include <datrwxx/debug.h>
#include <vector>
#include <string>
#include<datrwxx/formatmodifier.h>
#include <datrwxx/formatmodifier.h>
#include <datrwxx/mseed_keywords.h>
namespace datrw {
......@@ -57,7 +60,7 @@ namespace datrw {
//@{
/*! \brief Format properties
* \ingroup mseed
* \ingroup group_mseed
*/
const bool mseed::isbinary=true;
const char* const mseed::streamID="mseed";
......@@ -67,6 +70,7 @@ namespace datrw {
/*! \class imseedstream
*
* \ingroup group_mseed
* \paragraph mseed_imseedstream_internals Internals of imseedtream
*
* We must always be one record in advance. Data reading will be done by
......@@ -85,21 +89,49 @@ namespace datrw {
const std::string& modifier,
const bool& debug):
Tbase(is, true, true, true), Mdebug(debug), Mmodifier(modifier),
Mdumpascii(false)
Mdumpascii(false), Mchecks(true, true)
{
datrw::Subformat subformat(Mmodifier);
Mdumpascii=subformat.isset("dumpascii");
subformat("ttolerance", "0.") >> this->Mttolerance;
MestimateNframes=subformat.isset("estimateNframes");
Mdumpascii=subformat.isset(mseed::key::dumpascii);
subformat(mseed::key::ttolerance, "0.") >> this->Mttolerance;
MestimateNframes=subformat.isset(mseed::key::estimateNframes);
// note: flags have opposite meaning compared to subformat flags
if (subformat.isset(mseed::key::nonfatal))
{
std::string checks=subformat.value(mseed::key::nonfatal);
bool all=(checks.find(mseed::key::all)!=std::string::npos);
Mchecks.nframes.fatal
=!(all||(checks.find(mseed::key::nframes)!=std::string::npos));
Mchecks.nsamples.fatal
=!(all||(checks.find(mseed::key::nsamples)!=std::string::npos));
Mchecks.data.fatal
=!(all||(checks.find(mseed::key::data)!=std::string::npos));
}
// note: flags have opposite meaning compared to subformat flags
if (subformat.isset(mseed::key::skipcheck))
{
std::string checks=subformat.value(mseed::key::skipcheck);
bool all=(checks.find(mseed::key::all)!=std::string::npos);
Mchecks.nframes.docheck
=!(all||(checks.find(mseed::key::nframes)!=std::string::npos));
Mchecks.nsamples.docheck
=!(all||(checks.find(mseed::key::nsamples)!=std::string::npos));
Mchecks.data.docheck
=!(all||(checks.find(mseed::key::data)!=std::string::npos));
}
DATRW_assert_modifiers_are_recognized(subformat,
"imseedstream::imseedstream()");
/* We must always be one record in advance. Data reading will be done by
* concatenating several MiniSEED records. A time series will be
* terminated after reading a noncontiguous MiniSEED record. We have to
* store this record after reading and thus must always be one record
* ahead.
*/
Mrecord.read(Mis, Mdumpascii, MestimateNframes);
Mrecord.read(Mis, Mdumpascii, MestimateNframes, Mchecks);
DATRW_debug(Mdebug, "Mrecord read", Mrecord.wid2.line());
} // imseedstream::imseedstream(std::istream& is, const bool& debug)
......@@ -216,13 +248,18 @@ namespace datrw {
DATRW_debug(Mdebug, "imseedstream::read", "last data index " <<
data.last());
/*
* Remember value of last sample in previous record to be compared to
* retrospect value derived from the succeeding record. A mismatch will
* indicate non-contiguous data.
*/
int xm1=data(data.last());
nsamples += Mrecord.wid2.nsamples;
nextwid2.date=wid2line.date+dt*nsamples;
DATRW_debug(Mdebug, "imseedstream::read", "read next");
Mrecord.read(Mis, Mdumpascii, MestimateNframes);
Mrecord.read(Mis, Mdumpascii, MestimateNframes, Mchecks);
DATRW_debug(Mdebug, "imseedstream::read", "compare next");
DATRW_debug(Mdebug, "imseedstream::read", "expected wid2: " <<
......@@ -279,11 +316,6 @@ namespace datrw {
return (series);
} // Tiseries imseedstream::read(const bool& skipdata)
/*----------------------------------------------------------------------*/
void imseedstream::help(std::ostream& os)
{ datrw::mseed::help(os); }
} // namespace datrw
/* ----- END OF imseedstream.cc ----- */
/*! \file imseedstream_help.cc
* \brief imseedstream help function (implementation)
*
* ----------------------------------------------------------------------------
*
* \author Thomas Forbriger
* \date 05/07/2016
*
* imseedstream help function (implementation)
*
* Copyright (c) 2016 by Thomas Forbriger (BFO Schiltach)
*
* ----
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ----
*
* REVISIONS and CHANGES
* - 05/07/2016 V1.0 Thomas Forbriger
*
* ============================================================================
*/
#define DATRW_IMSEEDSTREAM_HELP_CC_VERSION \
"DATRW_IMSEEDSTREAM_HELP_CC V1.0"
#include <datrwxx/mseed.h>
#include <datrwxx/formatmodifier.h>
#include <datrwxx/mseed_keywords.h>
namespace datrw {
using std::endl;
using namespace mseed;
void imseedstream::help(std::ostream& os)
{
os << std::endl;
os << "MiniSEED data reading functions" << std::endl;
os << "-------------------------------" << std::endl;
os << endl;
os << "This module is designed to read MiniSEED (data only SEED) files"
<< endl;
os << "created by several different data acquisition systems. It is"
<< endl;
os << "tested with the following systems:"
<< endl;
os << " comserv with Q680 (Quanterra with SeisComP, GRSN setup)"
<< endl;
os << " seedlink (new SeisComP system with EDD plugin)"
<< endl;
os << " Earth Data Logger (EDL raw files)"
<< endl;
os << " Q330HR (Quanterra data acquisition system with Baler)"
<< endl;
os << " Taurus (Nanometrics data acquisition system)"
<< endl;
os << " Q330HR (data streamed from Q330HR)"
<< endl;
os << " Cube3 (data converted by cube2mseed)"
<< endl;
os << endl;
os << "Special features:"
<< endl;
os << "Data files written by the datalog client of comserv commence"
<< endl;
os << "with a telemetry volume header which is non-standard MiniSEED."
<< endl;
os << "This module identifies and skips the telemetry volume headers."
<< endl << endl;
os << "The module swaps data bytes if necessary. The SEED standard"
<< endl;
os << "defines (page 12 of SEED manual) that all headers must have"
<< endl;
os << "big-endian byte-sex. The byte-sex of MiniSEED data blocks is"
<< endl;
os << "defined in the header. The Earth Data Logger violates this"
<< endl;
os << "definition and writes headers in little-endian order. The"
<< endl;
os << "module tries to guess the correct byte-sex in this case"
<< endl;
os << "when reading the header blockettes."
<< endl << endl;
os << "Several consistency checks (see below) are applied to the"
<< endl;
os << "data file. The library module this way refuses to read data"
<< endl;
os << "which obviously is corrupt or which header parameters are"
<< endl;
os << "not plausible or are at variance with the data actually read."
<< endl;
os << "There are data of known sources which fail these tests,"
<< endl;
os << "although the data itself is intact. Such files are considered"
<< endl;
os << "to be at variance with the definition of Data Only SEED (MiniSEED)."
<< endl;
os << "To be able to work on such data, consistency checks can be made"
<< endl;
os << "non-fatal or even be switched of by format modifiers (see below)."
<< endl;
os << endl;
{
formatmodifiers::ModifierHelp mh(os, 24);
os << "Available format modifiers:\n";
mh(key::dumpascii)
<< "dumps ASCII data blocks to stdout\n";
mh(key::ttolerance, "T")
<< "tolerate time jitter, if not larger than\n";
mh << "\"T\", where \"T\" is given in microseconds\n";
mh(key::estimateNframes)
<< "estimate number of frames rather than reading\n";
mh << "it from blockette1001; this is required for\n";
mh << "data created by \"cube2mseed\" which contains\n";
mh << "an invalid count of frames\n";
mh(key::nonfatal, "ID[,ID,ID]")
<< "make consistency check \"ID\" non-fatal (report only)\n";
mh << "one or more check \"ID\" values may be passed in a list\n";
mh(key::skipcheck, "ID[,ID,ID]")
<< "skip consistency check \"ID\" entirely (ignore)\n";
mh << "one or more check \"ID\" values may be passed in a list\n";
os << endl;
}
{
formatmodifiers::ModifierHelp mh(os, 12);
os << "Available consistency checks (IDs to be used with modifiers):\n";
mh(key::nframes)
<< "The number of frames to be expected is specified in\n";
mh << "the header in Blockette 1001. This value will be\n";
mh << "compared against the number of frames actually\n";
mh << "used to store data.\n";
mh << "This test is only applied if the number of frames\n";
mh << "actually is specified in the header. Some data require\n";
mh << "the number of frames to be guessed (if Blockette 1001)\n";
mh << "is missing.\n";
mh(key::nsamples)
<< "The number of samples to be expected is provided in\n";
mh << "the header in the fixed section data header. This\n";
mh << "value will be compared against the number of\n";
mh << "samples actually provided in the logical record.\n";
mh(key::data)
<< "The sample value of the last sample in the record\n";
mh << "is compared against the value of the reverse\n";
mh << "integration constant as provided in the first data\n";
mh << "frame of the record. Both must be identical.\n";
mh(key::all)
<< "Select all consistency checks\n";
os << "Consistency checks will be selected upon a substring match\n"
"against check IDs. This means that by passing only a substring\n"
"in the argument list to the format modifier several checks\n"
"can be selected at once,\n";
os << endl;
}