From 22f6bf16579d04adf8642b8901d5aee738f80f32 Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Fri, 26 Jul 2024 15:49:56 -0500 Subject: [PATCH] Autoconversion of das2 streams to CDF files Can now write das2 input to das3 datamodel in a streaming fashion, The client API is 99.9% unchanged. Calling DasIO_model(..., 3) before running the pipeline with DasIO_readAll() is sufficent. This allows the das3_cdf program to handle das2 streams natively. --- buildfiles/Linux.mak | 6 +- das2/builder.c | 898 +------------------------------- das2/codec.c | 39 +- das2/core.h | 2 - das2/encoding.h | 2 + das2/io.c | 2 +- das2/plane.c | 3 +- das2/plane.h | 15 +- das2/serial2.c | 959 +++++++++++++++++++++++++++++++++++ das2/serial2.h | 67 +++ das2/{serial.c => serial3.c} | 2 +- das2/{serial.h => serial3.h} | 2 - das2/stream.c | 3 +- das2/variable.c | 2 +- das2/variable.h | 14 +- utilities/das3_cdf.c | 52 +- 16 files changed, 1122 insertions(+), 946 deletions(-) create mode 100644 das2/serial2.c create mode 100644 das2/serial2.h rename das2/{serial.c => serial3.c} (99%) rename das2/{serial.h => serial3.h} (99%) diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index cf8eaf4..2e19bd6 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -12,14 +12,14 @@ TARG=libdas3.0 SRCS:=das1.c array.c buffer.c builder.c cli.c codec.c credentials.c dataset.c \ datum.c descriptor.c dft.c dimension.c dsdf.c encoding.c frame.c http.c io.c \ iterator.c json.c log.c node.c oob.c operator.c packet.c plane.c processor.c \ -property.c serial.c send.c stream.c time.c tt2000.c units.c utf8.c util.c \ -value.c variable.c vector.c +property.c serial2.c serial3.c send.c stream.c time.c tt2000.c units.c utf8.c \ +util.c value.c variable.c vector.c HDRS:=defs.h time.h das1.h util.h log.h buffer.h utf8.h value.h units.h \ tt2000.h operator.h datum.h frame.h array.h encoding.h variable.h descriptor.h \ dimension.h dataset.h plane.h packet.h stream.h processor.h property.h oob.h \ io.h iterator.h builder.h dsdf.h credentials.h http.h dft.h json.h node.h cli.h \ - send.h vector.h serial.h codec.h core.h + send.h vector.h serial2.h serial3.h codec.h core.h ifeq ($(SPICE),yes) SRCS:=$(SRCS) spice.c diff --git a/das2/builder.c b/das2/builder.c index bbdf23b..01f685a 100644 --- a/das2/builder.c +++ b/das2/builder.c @@ -25,113 +25,7 @@ #include "variable.h" #include "dataset.h" #include "builder.h" - -/* Max number of dimensions in a dataset */ -#define DASBLDR_MAX_DIMS 64 -#define DASBLDR_SRC_ARY_SZ 64 - -/* ************************************************************************** */ -/* Specialized property copies only used by the das2 builder */ - - -/** Copy in dataset properties from some other descriptor - * - * This is a helper for das 2.2 streams. - * - * Any properties that don't start with a specific dimension identifier i.e. - * 'x','y','z','w' are copied into this dataset's properties dictionary. Only - * properties not present in the internal dictionary are copied in. - * - * @param pThis this dataset object - * @param pOther The descriptor containing properites to copy in - * @return The number of properties copied in - */ -int DasDs_copyInProps(DasDs* pThis, const DasDesc* pOther) -{ - const DasAry* pSource = &(pOther->properties); - size_t uProps = DasAry_lengthIn(pSource, DIM0); - - int nCopied = 0; - for(size_t u = 0; u < uProps; ++u){ - size_t uPropLen = 0; - const DasProp* pIn = (const DasProp*) DasAry_getBytesIn( - pSource, DIM1_AT(u), &uPropLen - ); - if(!DasProp_isValid(pIn)) - continue; - - const char* sName = DasProp_name(pIn); - - /* Do I want this prop? */ - if((*sName == 'x')||(*sName == 'y')||(*sName == 'z')||(*sName == '\0')) - continue; /* ... nope */ - - /* Do I have this property? ... */ - const DasProp* pOut = DasDesc_getLocal((DasDesc*)pThis, sName); - if(DasProp_isValid(pOut)) - continue; /* ... yep */ - - /* Set the property */ - if(DasDesc_setProp((DasDesc*)pThis, pIn) != DAS_OKAY){ - return nCopied; - } - ++nCopied; - } - return nCopied; -} - -/** Copy in dataset properties from some other descriptor - * - * This is a helper for das 2.2 streams as these use certian name patterns to - * indicate which dimension a property is for - * - * Any properties that start with a specific dimension identifier i.e. - * 'x','y','z','w' are copied into this dataset's properties dictionary. Only - * properties not present in the internal dictionary are copied in. - * - * @param pThis this dimension object - * @param cAxis the connonical axis to copy in. - * @param pOther The descriptor containing properites to copy in - * @return The number of properties copied in - * @memberof DasDim - */ - -int DasDim_copyInProps(DasDim* pThis, char cAxis, const DasDesc* pOther) -{ - char sNewName[32] = {'\0'}; - - const DasAry* pSrcAry = &(pOther->properties); - size_t uProps = DasAry_lengthIn(pSrcAry, DIM0); - - int nCopied = 0; - for(size_t u = 0; u < uProps; ++u){ - size_t uPropLen = 0; - const DasProp* pIn = (const DasProp*) DasAry_getBytesIn(pSrcAry, DIM1_AT(u), &uPropLen); - if(!DasProp_isValid(pIn)) - continue; - - const char* sName = DasProp_name(pIn); - - /* We only want stuff for the given axis, but we don't want to copy in - * the axis name, so make sure there's something after it. */ - if((*sName != cAxis)||( *(sName +1) == '\0')) - continue; /* ... nope */ - - /* Since we strip the x,y,z, make next char lower to preserve the look - * of the prop naming scheme */ - memset(sNewName, 0, 32); - int nLen = (int)(strlen(sName)) - 1; - nLen = nLen > 31 ? 31 : nLen; - strncpy(sNewName, sName + 1, 31); - sNewName[0] = tolower(sNewName[0]); - - DasDesc_flexSet((DasDesc*)pThis, NULL, DasProp_type(pIn), sNewName, - DasProp_value(pIn), DasProp_sep(pIn), pIn->units, DASPROP_DAS3 - ); - ++nCopied; - } - return nCopied; -} +#include "serial2.h" /* ************************************************************************** */ /* Helpers */ @@ -326,795 +220,6 @@ char* _DasDsBldr_getExistingGroup( return NULL; /* They are going to have to make something up */ } -/* ************************************************************************* */ -/* Inspect plane properties and output standardized dimension role string */ - -const char* _DasDsBldr_role(PlaneDesc* pPlane) -{ - const char* sRole = DasDesc_get((DasDesc*)pPlane, "operation"); - if(sRole == NULL) return DASVAR_CENTER; - - /* Interpret Autoplot style strings */ - if(strcmp("BIN_AVG", sRole) == 0) return DASVAR_MEAN; - if(strcmp("BIN_MAX", sRole) == 0) return DASVAR_MAX; - if(strcmp("BIN_MIN", sRole) == 0) return DASVAR_MIN; - - return DASVAR_CENTER; -} - -/* ************************************************************************* */ -/* Handle matching up planes into single dimensions, interface is really */ -/* complicated because this code was inline in another function */ - -DasDim* _DasDsBldr_getDim( - PlaneDesc* pPlane, PktDesc* pPd, DasStream* pSd, - char cAxis, - DasDs* pDs, enum dim_type dType, const char* sDimId, - DasDim** ppDims, char* pDimSrc, size_t* puDims -){ - /* if this plane has a source property then it might be grouped */ - char* p = NULL; - char sNewDimId[64] = {'\0'}; - DasDim* pDim = NULL; - - const char* sSource = DasDesc_get((DasDesc*)pPlane, "source"); - if(sSource != NULL){ - /* See if this plane source appears in an existing dim */ - for(size_t u = 0; u < *puDims; u++){ - p = pDimSrc + u*DASBLDR_SRC_ARY_SZ; - if(strncmp(sSource, p, DASBLDR_SRC_ARY_SZ) == 0){ - /* Same group, add in any plane level properties that might be - * missing in this dimension */ - if(cAxis != '\0') - DasDim_copyInProps(ppDims[u], cAxis, (DasDesc*)pPlane); - return ppDims[u]; - } - } - } - - /* Didn't find a matching dim, so add one to the dataset, record it's info - * if it has an operation source property. Also de-kludge the name by - * removing anything after the dot - */ - - if(sSource != NULL){ - p = (char*) strchr(sDimId, '.'); - if(p && (p != sDimId)){ - strncpy(sNewDimId, sDimId, 63); - p = strchr(sNewDimId, '.'); - *p = '\0'; - sDimId = sNewDimId; - } - if((pDim = DasDs_makeDim(pDs, dType, sDimId, "")) == NULL) return NULL; - - if((*puDims) + 1 >= DASBLDR_MAX_DIMS){ - das_error(DASERR_BLDR, "Too many dimensions in a single packet %d", - DASBLDR_MAX_DIMS); - return NULL; - } - - ppDims[*puDims] = pDim; - - p = pDimSrc + (*puDims)*DASBLDR_SRC_ARY_SZ; - strncpy(p, sSource, DASBLDR_SRC_ARY_SZ - 1); - *puDims = *puDims + 1; - } - else{ - if((pDim = DasDs_makeDim(pDs, dType, sDimId, ""))==NULL) return NULL; - - } - - if(cAxis != '\0'){ - DasDim_copyInProps(pDim, cAxis, (DasDesc*)pSd); - DasDim_copyInProps(pDim, cAxis, (DasDesc*)pPd); - DasDim_copyInProps(pDim, cAxis, (DasDesc*)pPlane); - } - return pDim; -} - - -/* ************************************************************************* */ -/* Add a das3 codec to a dataset that's equavalent to a das2 encoder */ -DasErrCode _DasDsBldr_addCodec( - DasDs* pDs, const char* sAryId, int nItems, DasEncoding* pEncoder -){ - - int nHash = DasEnc_hash(pEncoder); - const char* sEncType = NULL; - int nItemBytes = 0; - const char* sSemantic = "real"; - switch(nHash){ - case DAS2DT_BE_REAL_8: sEncType = "BEreal"; nItemBytes = 8; break; - case DAS2DT_LE_REAL_8: sEncType = "LEreal"; nItemBytes = 8; break; - case DAS2DT_BE_REAL_4: sEncType = "BEreal"; nItemBytes = 4; break; - case DAS2DT_LE_REAL_4: sEncType = "LEreal"; nItemBytes = 4; break; - } - if(sEncType == NULL){ - nItemBytes = pEncoder->nWidth; - sEncType = "utf8"; - if(pEncoder->nCat == DAS2DT_TIME) - sSemantic = "datetime"; - } - - return DasDs_addFixedCodec(pDs, sAryId, sSemantic, sEncType, nItemBytes, nItems); -} - - -/* ************************************************************************* */ -/* Initialize X-Y pattern - * Make one dimension for X and one each for the Y's. Could do some deeper - * inspection to see if some of the columns should be grouped into the same - * dimension. For das 2.3 streams where we can have multiple X columns - * this will be a must. */ - -void _strrep(char* pId, char c, char r) -{ - char* pTmp = pId; - while(*pTmp != '\0'){ - if(*pTmp == c) *pTmp = r; - ++pTmp; - } -} - - -DasDs* _DasDsBldr_initXY( - DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs -){ - /* If my group name is null, make up a new one appropriate to XY data */ - const char* pId = NULL; - PlaneDesc* pPlane = NULL; - DasEncoding* pEncoder = NULL; - const char* sRole = NULL; /* The reason this plane is present */ - int nY = 0; - char sBuf[64] = {'\0'}; - char sDsId[64] = {'\0'}; - if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); - if(pGroup == NULL){ - if(PktDesc_getNPlanesOfType(pPd, Y) == 1){ - pPlane = PktDesc_getPlaneByType(pPd, Y, 0); - pGroup = PlaneDesc_getName(pPlane); - } - else{ - nY = PktDesc_getNPlanesOfType(pPd, Y); - snprintf(sBuf, 63, "unknown_%dY", nY); - pGroup = sBuf; - nY = 0; - } - } - - snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); - - DasDs* pDs = new_DasDs(sDsId, pGroup, 1); - DasDim* pDim = NULL; - DasAry* pAry = NULL; - DasVar* pVar = NULL; - enum dim_type dType = DASDIM_UNK; - - /* Tracking for array groups (aka dimensions) */ - DasDim* aDims[DASBLDR_MAX_DIMS] = {NULL}; - char aDimSrc[DASBLDR_MAX_DIMS * DASBLDR_SRC_ARY_SZ] = {'\0'}; - size_t uDims = 0; - - /* Copy any properties that don't start with one of the axis prefixes */ - - /* ... actually, we output stream headers now, so this isn't needed, it - was always a bit of a hack and doesn't play well with CDF generation. - DasDs_copyInProps(pDs, (DasDesc*)pSd); - DasDs_copyInProps(pDs, (DasDesc*)pPd); - */ - - char sAryId[64] = {'\0'}; - double fill; - char cAxis = '\0'; - - for(size_t u = 0; u < pPd->uPlanes; ++u){ - pPlane = pPd->planes[u]; - pEncoder = PlaneDesc_getValEncoder(pPlane); - - pId = pPlane->sName; - if(pPlane->planeType == X){ - cAxis = 'x'; - if(pId == NULL){ - if(Units_haveCalRep(pPlane->units)) pId = "time"; - else pId = "X"; - } - - /* Fill is not allowed for Das 2.2 X planes */ - /* Also das 2.2 planes always convert ASCII times to doubles. - * Though it's probably un-needed processing, we at leas know - * the array types will always be etDouble */ - pAry = new_DasAry(pId, vtDouble, 0, NULL, RANK_1(0), pPlane->units); - } - else{ - cAxis = 'y'; - ++nY; - - /* Always copy over the name if it exists */ - if(pId == NULL) snprintf(sAryId, 63, "Y_%d", nY); - else strncpy(sAryId, pId, 63); - _strrep(sAryId, '.', '_'); /* handle amplitude.max type stuff */ - - fill = PlaneDesc_getFill(pPlane); - pAry = new_DasAry( - sAryId, vtDouble, 0, (const ubyte*)&fill, RANK_1(0), pPlane->units - ); - } - if(pAry == NULL) return NULL; - - /* add the new array to the DS so it has somewhere to put this item from - * the packets */ - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - - /* Remember how to fill this array. This will get more complicated - * when variable length packets are introduced */ - DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); - - /* On to higher level data organizational structures: Create dimensions - * and variables as needed for the new arrays */ - if(pPlane->planeType == X) dType = DASDIM_COORD; - else dType = DASDIM_DATA; - - pDim = _DasDsBldr_getDim( - pPlane, pPd, pSd, cAxis, pDs, dType, pId, aDims, aDimSrc, &uDims - ); - if(pDim == NULL) return NULL; - - pVar = new_DasVarArray(pAry, SCALAR_1(0)); - if( pVar == NULL) return NULL; - sRole = _DasDsBldr_role(pPlane); - if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; - - if(bCodecs) - if(_DasDsBldr_addCodec(pDs, sAryId, 1, pEncoder) != DAS_OKAY) - return NULL; - } - return pDs; -} - -/* ************************************************************************* */ -/* Initialize X-Y-Z pattern */ - -DasDs* _DasDsBldr_initXYZ( - DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs -){ - /* If my group name is null, make up a new one appropriate to XYZ data */ - const char* pId = NULL; - PlaneDesc* pPlane = NULL; - DasEncoding* pEncoder = NULL; - const char* sRole = NULL; /* The reason this plane is present */ - int nZ = 0; - char sBuf[64] = {'\0'}; - char sDsId[64] = {'\0'}; - if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); - if(pGroup == NULL){ - if(PktDesc_getNPlanesOfType(pPd, Z) == 1){ - pPlane = PktDesc_getPlaneByType(pPd, Z, 0); - pGroup = PlaneDesc_getName(pPlane); - } - else{ - nZ = PktDesc_getNPlanesOfType(pPd, Z); - snprintf(sBuf, 63, "unknown_%dZ", nZ); - pGroup = sBuf; - nZ = 0; - } - } - - snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); - - DasDs* pDs = new_DasDs(sDsId, pGroup, 1); - DasDim* pDim = NULL; - DasAry* pAry = NULL; - DasVar* pVar = NULL; - enum dim_type dType = DASDIM_UNK; - - /* Tracking for array groups (aka dimensions) */ - DasDim* aDims[DASBLDR_MAX_DIMS] = {NULL}; - char aDimSrc[DASBLDR_MAX_DIMS * DASBLDR_SRC_ARY_SZ] = {'\0'}; - size_t uDims = 0; - - /* Copy any properties that don't start with one of the axis prefixes */ - /* ... or don't. Messes up CDF output - DasDs_copyInProps(pDs, (DasDesc*)pSd); - DasDs_copyInProps(pDs, (DasDesc*)pPd); */ - - char sAryId[64] = {'\0'}; - double fill; - char cAxis = '\0'; - - for(size_t u = 0; u < pPd->uPlanes; ++u){ - pPlane = pPd->planes[u]; - pId = pPlane->sName; - pEncoder = PlaneDesc_getValEncoder(pPlane); - - switch(pPlane->planeType){ - case X: - cAxis = 'x'; - if(pId == NULL){ - if(Units_haveCalRep(pPlane->units)) pId = "time"; - else pId = "X"; - } - - /* Fill is not allowed for Das 2.2 X planes */ - /* Also das 2.2 planes always convert ASCII times to doubles. - * Though it's probably un-needed processing, we at least know - * the array types will always be etDouble */ - pAry = new_DasAry(pId, vtDouble, 0, NULL, RANK_1(0), pPlane->units); - break; - - case Y: - cAxis = 'y'; - if(pId == NULL)pId = "Y"; - - /* Fill is not allowed for Das 2.2 Y planes in an X,Y,Z pattern */ - pAry = new_DasAry(pId, vtDouble, 0, NULL, RANK_1(0), pPlane->units); - break; - - case Z: - cAxis = 'z'; - ++nZ; - - if(pId == NULL) snprintf(sAryId, 63, "Z_%d", nZ); - else strncpy(sAryId, pId, 63); - _strrep(sAryId, '.', '_'); /* handle amplitude.max stuff */ - - fill = PlaneDesc_getFill(pPlane); - pAry = new_DasAry( - sAryId, vtDouble, 0, (const ubyte*)&fill, RANK_1(0), pPlane->units - ); - break; - - - default: - das_error(DASERR_BLDR, "Unexpected plane type in XYZ pattern"); - return NULL; - } - - if(pAry == NULL) return NULL; - - /* add the new array to the DS so it has somewhere to put this item from - * the packets */ - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - - /* Remember how to fill this array. This will get more complicated - * when variable length packets are introduced. So this is item 'u' - * from the packet and 1 item is read for each packet */ - DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); - - /* Create dimensions and variables as needed for the new arrays */ - if(pPlane->planeType == Z) dType = DASDIM_DATA; - else dType = DASDIM_COORD; - - pDim = _DasDsBldr_getDim( - pPlane, pPd, pSd, cAxis, pDs, dType, pId, aDims, aDimSrc, &uDims - ); - if(pDim == NULL) return NULL; - - pVar = new_DasVarArray(pAry, SCALAR_1(0)); - if( pVar == NULL) return NULL; - - /* Assume these are center values unless there is a property stating - * otherwise */ - sRole = _DasDsBldr_role(pPlane); - if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; - - if(bCodecs) - if(_DasDsBldr_addCodec(pDs, sAryId, 1, pEncoder) != DAS_OKAY) - return NULL; - } - - return pDs; -} - -/* ************************************************************************* */ -/* Initialize Events Pattern */ - -DasDs* _DasDsBldr_initEvents(DasStream* pSd, PktDesc* pPd, const char* sGroupId) -{ - das_error(DASERR_BLDR, "Event stream reading has not been implemented"); - return NULL; -} - -/* ************************************************************************* */ -/* Initialize YScan Pattern */ - -bool _DasDsBldr_checkYTags(PktDesc* pPd) -{ - size_t uYScans = PktDesc_getNPlanesOfType(pPd, YScan); - if(uYScans < 2) return true; - - PlaneDesc* pFirst = PktDesc_getPlaneByType(pPd, YScan, 0); - size_t uYTags = PlaneDesc_getNItems(pFirst); - ytag_spec_t spec = PlaneDesc_getYTagSpec(pFirst); - double rInterval = -1.0, rMin = -1.0, rMax = -1.0; - if(spec == ytags_series) - PlaneDesc_getYTagSeries(pFirst, &rInterval, &rMin, &rMax); - const double* pYTags = NULL; - if(spec == ytags_list) pYTags = PlaneDesc_getYTags(pFirst); - - das_units units = PlaneDesc_getYTagUnits(pFirst); - - PlaneDesc* pNext = NULL; - size_t u,v; - double rNextInterval = -1.0, rNextMin = -1.0, rNextMax = -1.0; - const double* pNextYTags = NULL; - for(u = 1; u < uYScans; ++u){ - pNext = PktDesc_getPlaneByType(pPd, YScan, u); - if(uYTags != PlaneDesc_getNItems(pNext)) return false; - - /* The tags can be specified as none, a list, or a series */ - if(spec != PlaneDesc_getYTagSpec(pNext)) return false; - - if(units != PlaneDesc_getYTagUnits(pNext)) return false; - - switch(spec){ - case ytags_none: break; - case ytags_series: - PlaneDesc_getYTagSeries(pNext, &rNextInterval, &rNextMin, &rNextMax); - if(rInterval != rNextInterval ) return false; - if(rMin != rNextMin) return false; - if(rMax != rNextMax) return false; - break; - - case ytags_list: - pNextYTags = PlaneDesc_getYTags(pNext); - for(v = 0; vplaneType != YScan){ - das_error(DASERR_BLDR, "Program logic error"); - return NULL; - } - double* pTags = (double*)calloc(PlaneDesc_getNItems(pPlane), sizeof(double)); - size_t u, uItems = PlaneDesc_getNItems(pPlane); - const double* pListTags = NULL; - double rInterval, rMin, rMax; - switch(pPlane->ytag_spec){ - case ytags_list: - pListTags = PlaneDesc_getYTags(pPlane); - for(u = 0; u < uItems; ++u) pTags[u] = pListTags[u]; - break; - case ytags_none: - for(u = 0; u < uItems; ++u) pTags[u] = u; - break; - case ytags_series: - PlaneDesc_getYTagSeries(pPlane, &rInterval, &rMin, &rMax); - for(u = 0; u < uItems; ++u) pTags[u] = rMin + (rInterval * u); - break; - } - return pTags; -} - -bool _DasDsBldr_isWaveform(PlaneDesc* pPlane){ - - const char* sRend = DasDesc_getStr((DasDesc*)pPlane, "renderer"); - if(sRend == NULL) return false; - if(strcmp("waveform", sRend) != 0) return false; - - das_units units = PlaneDesc_getYTagUnits(pPlane); - if(! Units_canConvert(units, UNIT_SECONDS)) return false; - return true; -} - -DasDs* _DasDsBldr_initYScan( - DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs -){ - /* Make sure all the yscans have the same yTags. The assumption here is - * that a single packet only has data correlated in it's coordinates. If - * two yscans have different yTag sets then they are not correlated. */ - - if(!_DasDsBldr_checkYTags(pPd)){ - das_error(DASERR_BLDR, "YTags are not equivalent in multi-yscan packet"); - return NULL; - } - - /* If my group name is null, make up a new one appropriate to YScan data */ - PlaneDesc* pPlane = PktDesc_getPlaneByType(pPd, YScan, 0); - int nY = 0, nYScan = 0; - char sDsGroup[64] = {'\0'}; - char sDsId[64] = {'\0'}; - - if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); - if(pGroup == NULL) pGroup = PlaneDesc_getName(pPlane); - if(pGroup == NULL){ - nYScan = PktDesc_getNPlanesOfType(pPd, YScan); - snprintf(sDsGroup, 63, "default_%d_MultiZ", nYScan); - pGroup = sDsGroup; - } - snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); - - size_t uItems = PlaneDesc_getNItems(pPlane); - - DasDs* pDs = new_DasDs(sDsId, pGroup, 2); - DasDim* pDim = NULL; - DasDim* pXDim = NULL; - DasDim* pYDim = NULL; - DasVar* pVar = NULL; - DasVar* pOffset = NULL; - DasVar* pReference = NULL; - - /* Dito, see above searching for string CDF - DasDs_copyInProps(pDs, (DasDesc*)pSd); - DasDs_copyInProps(pDs, (DasDesc*)pPd); */ /*These never have props in das 2.2 */ - const char* pPlaneId = NULL; - DasEncoding* pEncoder = NULL; - const char* sRole = NULL; - das_units Yunits; - das_units Zunits; - const char* pYTagId = NULL; - DasAry* pAry = NULL; - char sAryId[64] = {'\0'}; - double fill; - - /* One thing to watch out for. The number of dimensions is not equal to - * the number of planes. If two planes have the same 'source' property - * but different roles then they go together in the same dimension. - * - * The old das2 stream model is *REALLY* showing it's age and needs to be - * updated. To much hoop jumping and heuristics are needed to shove - * complex info into such a simple structure. QStream is better, but still - * not explicit enough. We just need a break from the past. -cwp 2019-04-05 - */ - DasDim* aDims[DASBLDR_MAX_DIMS] = {NULL}; - char aDimSrc[DASBLDR_MAX_DIMS*DASBLDR_SRC_ARY_SZ] = {'\0'}; - size_t uDims = 0; - - double* pYTags = NULL; - bool bAddedYTags = false; - for(size_t u = 0; u < pPd->uPlanes; ++u){ - pPlane = pPd->planes[u]; - if(pPlane->sName != NULL) pPlaneId = pPlane->sName; - - /* Assume this is a center value unless told otherwise */ - sRole = _DasDsBldr_role(pPlane); - - pEncoder = PlaneDesc_getValEncoder(pPlane); - - /* If we're lucky the plane will tell us the purpose for it's - * values, i.e. min, center, max, err-bar */ - - switch(pPlane->planeType){ - case X: - if(pPlaneId == NULL){ - if(Units_haveCalRep(pPlane->units)) pPlaneId = "time"; - else pPlaneId = "X"; - - } - pAry = new_DasAry(pPlaneId, vtDouble, 0, NULL, RANK_1(0), pPlane->units); - if(pAry == NULL) return NULL; - DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - - pXDim = _DasDsBldr_getDim( - pPlane, pPd, pSd, 'x', pDs, DASDIM_COORD, pPlaneId, - aDims, aDimSrc, &uDims - ); - if(pXDim == NULL) return NULL; - - - /* Map index 0 to time array 0, ignore index 1 */ - pVar = new_DasVarArray(pAry, SCALAR_2(0, DASIDX_UNUSED)); - if(pVar == NULL) return NULL; - if(! DasDim_addVar(pXDim, sRole, pVar)) return NULL; - - if(bCodecs) - if(_DasDsBldr_addCodec(pDs, pPlaneId, 1, pEncoder) != DAS_OKAY) - return NULL; - break; - - case Y: - ++nY; - if(pPlaneId == NULL) - snprintf(sAryId, 63, "Y_%d", nY); - else - strncpy(sAryId, pPlaneId, 63); - _strrep(sAryId, '.', '_'); - - fill = PlaneDesc_getFill(pPlane); - pAry = new_DasAry( - sAryId, vtDouble, 0, (const ubyte*)&fill, RANK_1(0), pPlane->units - ); - if(pAry == NULL) return NULL; - DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - - /* Assume that extra Y values are more coordinates unless told - * otherwise by a setting of some sort that I don't yet know */ - pYDim = _DasDsBldr_getDim( - pPlane, pPd, pSd, 'y', pDs, DASDIM_COORD, pPlaneId, - aDims, aDimSrc, &uDims - ); - if(pYDim == NULL) return NULL; - - /* Map index 0 to this array, ignore index 1 */ - pVar = new_DasVarArray(pAry, SCALAR_2(0, DASIDX_UNUSED)); - if(pVar == NULL) return NULL; - if(! DasDim_addVar(pYDim, sRole, pVar)) return NULL; - - if(bCodecs) - if(_DasDsBldr_addCodec(pDs, pPlaneId, 1, pEncoder) != DAS_OKAY) - return NULL; - break; - - case YScan: - ++nYScan; - - /* This one is interesting, maybe have to add two arrays. - * - * Also sometimes this is a waveform and we need to set the yTags as - * an offset dimension for time. - */ - - if(!bAddedYTags){ - Yunits = PlaneDesc_getYTagUnits(pPlane); - if( Units_canConvert(Yunits, UNIT_HERTZ) ){ - pYTagId = "frequency"; - } - else{ - if(Units_canConvert(Yunits, UNIT_SECONDS)) pYTagId = "offset"; - else{ - if(Units_canConvert(Yunits, UNIT_EV)) pYTagId = "energy"; - else pYTagId = "ytags"; - } - } - pAry = new_DasAry(pYTagId, vtDouble, 0, NULL, RANK_1(uItems), Yunits); - if(pAry == NULL) return NULL; - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - pYTags = _DasDsBldr_yTagVals(pPlane); - - /* Use put instead of append since we've already allocated the space */ - DasAry_putAt(pAry, IDX0(0), (const ubyte*)pYTags, uItems); - free(pYTags); pYTags = NULL; - - /* If the data aren't waveforms we're going to need to add a new - * dimension, otherwise we'll add in the REF and OFFSET variables */ - if( _DasDsBldr_isWaveform(pPlane) ){ - /* ignore first index, map second index to array */ - pOffset = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); - DasDim_addVar(pXDim, DASVAR_OFFSET, pOffset); - - /* Convert the old CENTER variable to a Reference variable */ - pReference = DasDim_popVar(pXDim, DASVAR_CENTER); - DasDim_addVar(pXDim, DASVAR_REF, pReference); - - /* Make new center variable that is rank 2 since reference and - * offset are orthogonal */ - pVar = new_DasVarBinary("center", pReference, "+", pOffset); - if(! DasDim_addVar(pXDim, DASVAR_CENTER, pVar) ) return NULL; - } - else{ - /* If we have and then the Y's are our reference values - Convert the old CENTER variable to a Reference variable and - add the Ytags as offsets */ - if(pYDim != NULL){ - pOffset = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); - DasDim_addVar(pYDim, DASVAR_OFFSET, pOffset); - - /* Convert the old CENTER variable to a Reference variable */ - pReference = DasDim_popVar(pYDim, DASVAR_CENTER); - DasDim_addVar(pYDim, DASVAR_REF, pReference); - - /* Make new center variable that is rank 2 since reference and - * offset are orthogonal */ - pVar = new_DasVarBinary("center", pReference, "+", pOffset); - if(! DasDim_addVar(pYDim, DASVAR_CENTER, pVar) ) return NULL; - } - else{ - /* Nope no offsets in freq or time, just a new center variable */ - pDim = DasDs_makeDim(pDs, DASDIM_COORD, pYTagId, ""); - if(pDim == NULL) return NULL; - - DasDim_copyInProps(pDim, 'y', (DasDesc*)pSd); - DasDim_copyInProps(pDim, 'y', (DasDesc*)pPd); - DasDim_copyInProps(pDim, 'y', (DasDesc*)pPlane); - - /* Map index 1 to this array's index 0, ignore 0, assume */ - /* center values */ - pVar = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); - if(! DasDim_addVar(pDim, DASVAR_CENTER, pVar)) return NULL; - } - } - - bAddedYTags = true; - } - - /* Now for the actual data values array, try for a good array name */ - Zunits = PlaneDesc_getUnits(pPlane); - if(pPlaneId == NULL){ - if(Units_canConvert(Zunits, UNIT_E_SPECDENS)) - strncpy(sAryId, "e_spec_dens", 63); - else if(Units_canConvert(Zunits, UNIT_B_SPECDENS)) - strncpy(sAryId, "b_spec_dens", 63); - else - snprintf(sAryId, 63, "YScan_%d", nYScan); - } - else{ - strncpy(sAryId, pPlaneId, 63); - } - _strrep(sAryId, '.', '_'); - - - fill = PlaneDesc_getFill(pPlane); - pAry = new_DasAry( - sAryId, vtDouble, 0, (const ubyte*)&fill, RANK_2(0, uItems), Zunits - ); - if(pAry == NULL) return NULL; - if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; - DasAry_setSrc(pAry, PktDesc_getId(pPd), u, uItems); - - pDim = _DasDsBldr_getDim( - pPlane, pPd, pSd, 'z', pDs, DASDIM_DATA, pPlaneId, aDims, - aDimSrc, &uDims - ); - if(pDim == NULL) return NULL; - - /* Map index 0 to 0 and 1 to 1 */ - pVar = new_DasVarArray(pAry, SCALAR_2(0, 1)); - if(pVar == NULL) return NULL; - if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; - - if(bCodecs) - if(_DasDsBldr_addCodec(pDs, sAryId, (int)uItems, pEncoder) != DAS_OKAY) - return NULL; - - break; - - default: - das_error(DASERR_DS, "logic error"); - return NULL; - break; - } - } - return pDs; -} - -/* Generate an potentiall active dataset object from a packet descriptor - * - * This is an internal function and thus is not present in builder.h. To use it - * define the function prototype in the relavent module. - * - * @param bCodecs If true, define codecs for the generated dataset object so that - * it can be used to parse data packet payloads - */ -DasDs* dasds_from_packet(DasStream* pSd, PktDesc* pPd, const char* sGroup, bool bCodecs) -{ - /* Initialize based on the observed pattern. Das2 streams have traditionally - * followed certian layout patterns, you can't have arbitrary collections of - * and planes. */ - size_t u, uPlanes = PktDesc_getNPlanes(pPd); - PlaneDesc* pPlane = NULL; - int nXs = 0, nYs = 0, nYScans = 0, nZs = 0; - for(u = 0; u < uPlanes; ++u){ - pPlane = PktDesc_getPlane(pPd, u); - switch(PlaneDesc_getType(pPlane)){ - case X: ++nXs; break; - case Y: ++nYs; break; - case YScan: ++nYScans; break; - case Z: ++nZs; break; - case Invalid: das_error(DASERR_DS, "logic error"); return NULL; - } - } - - DasDs* pCd = NULL; - if(nYScans == 0){ - if(nZs != 0) pCd = _DasDsBldr_initXYZ(pSd, pPd, sGroup, bCodecs); - else{ - if(nXs == 2) pCd = _DasDsBldr_initEvents(pSd, pPd, sGroup); - else pCd = _DasDsBldr_initXY(pSd, pPd, sGroup, bCodecs); - } - } - else{ - pCd = _DasDsBldr_initYScan(pSd, pPd, sGroup, bCodecs); - } - return pCd; -} - /* Get the correlated dataset container that corresponds to the given packet * descriptor. This uses duck typing. If a packet descriptor has the same * number and type of planes, and each plane has the same name (and ytags if @@ -1290,7 +395,6 @@ void DasDsBldr_release(DasDsBldr* pThis){ } - DasDs** DasDsBldr_getDataSets(DasDsBldr* pThis, size_t* pLen){ if(pThis->uValidPairs == 0) return NULL; diff --git a/das2/codec.c b/das2/codec.c index 76a71e4..e054233 100644 --- a/das2/codec.c +++ b/das2/codec.c @@ -196,31 +196,40 @@ DasErrCode DasCodec_init( } else if((strcmp(sSemantic, "datetime") == 0)){ bDateTime = true; - pThis->uProc |= DASENC_PARSE; - /* If we're storing this as a datetime structure it's covered, if - we need to convert to something else the units are needed */ - if(vtAry != vtTime){ + /* For datetimes stored as text, we just run it in */ + if((vtAry != vtUByte)&&(vtAry != vtByte)){ + + pThis->uProc |= DASENC_PARSE; /* Gonna have to parse the text */ + + /* If we're storing this as a datetime structure it's covered, if + we need to convert to something else the units are needed */ + if(vtAry != vtTime){ - if( (epoch == NULL) || (! Units_haveCalRep(epoch) ) ) - goto UNSUPPORTED; + if( (epoch == NULL) || (! Units_haveCalRep(epoch) ) ) + goto UNSUPPORTED; - /* Check that the array element size is big enough for the units in - question */ - if((epoch == UNIT_TT2000)&&(vtAry != vtLong)&&(vtAry != vtDouble)) - goto UNSUPPORTED; - else - if((vtAry != vtDouble)&&(vtAry != vtFloat)) + /* Check that the array element size is big enough for the units in question + If the data are not stored as text */ + if((epoch == UNIT_TT2000)&&(vtAry != vtLong)&&(vtAry != vtDouble)) goto UNSUPPORTED; + else + if((vtAry != vtDouble)&&(vtAry != vtFloat)) + goto UNSUPPORTED; + } pThis->timeUnits = epoch; /* In addition to parsing, we have to convert */ } + else{ + pThis->timeUnits = UNIT_UTC; /* kind of a fake placeholder unit */ + } } else if(strcmp(sSemantic, "string") == 0){ - if(vtAry != vtUByte) /* Expect uByte storage for strings not */ - goto UNSUPPORTED; /* vtText as there is no external place */ - /* to put the string data */ + /* Expect uByte storage for strings not vtText as there is no external place + to put the string data */ + if((vtAry != vtUByte)&&(vtAry != vtByte)) + goto UNSUPPORTED; if(DasAry_getUsage(pThis->pAry) & D2ARY_AS_STRING){ pThis->uProc |= DASENC_NULLTERM; diff --git a/das2/core.h b/das2/core.h index 428c8f3..8a3c0d9 100644 --- a/das2/core.h +++ b/das2/core.h @@ -230,8 +230,6 @@ #include #include #include -#include /* might not need to be exposed */ -#include /* might not need to be exposed */ /* Add a utility for handling UTF-8 as an internal string format, though almost all string manipulation algorithms get by without this even when diff --git a/das2/encoding.h b/das2/encoding.h index 5a05b52..2b2b0e9 100644 --- a/das2/encoding.h +++ b/das2/encoding.h @@ -182,6 +182,8 @@ DAS_API DasEncoding* new_DasEncoding(int nCat, int nWidth, const char* sFmt); /* Das Encondings use value semantics */ #define del_DasEncoding(pEnc) free(pEnc) +#define DasEnc_isUtf8(pEnc) ((pEnc->nCat == DAS2DT_TIME)||(pEnc->nCat == DAS2DT_ASCII)) + /** @} */ /** Create a new encoding based on the encoding type string. diff --git a/das2/io.c b/das2/io.c index 4a320cc..ebbbe78 100644 --- a/das2/io.c +++ b/das2/io.c @@ -42,7 +42,7 @@ #include "util.h" /* <-- Make sure endianess macros are present */ #include "http.h" /* Get ssl helpers */ -#include "serial.h" +#include "serial3.h" #include "io.h" #ifdef _WIN32 diff --git a/das2/plane.c b/das2/plane.c index dde2f70..006dac6 100644 --- a/das2/plane.c +++ b/das2/plane.c @@ -876,7 +876,8 @@ double PlaneDesc_getFill(const PlaneDesc* pThis){ /* keyword "fill" work anywhere, but the benefits outweigh the negatives */ if(pThis->planeType == X){ - das_error(17, " planes should never have fill values\n"); + /* das_error(17, " planes should never have fill values\n"); */ + return getDas2Fill(); } /* Break const correctness, D doesn't allow this since it's worried diff --git a/das2/plane.h b/das2/plane.h index 20fa926..98a9b56 100644 --- a/das2/plane.h +++ b/das2/plane.h @@ -134,18 +134,11 @@ typedef struct plane_descriptor{ */ size_t uItems; - /* Das 2.3 note + /* Das 3.0 note: * One of the fundamental assumptions in this code, way back from when - * Jeremy started it was that all data could be converted to doubles. - * for high-time resolution data this just won't work. There is no way - * to encode time to nano-second accuracy over any appreciable time range - * in a 64 bit floating point value. - * - * Furthermore, some data types are just fine as they are, there is no - * reason to convert them. We should think about removing this restriction. - * for das 2.3. - * - * --cwp 2018-10-25 + * it started was that all data could be converted to doubles. That + * remains true for this old packet structure. For das3 streams many + * data types are supported, see codec.h and array.h for details. */ double* pData; double value; /* Convenience for planes that only store one data point */ diff --git a/das2/serial2.c b/das2/serial2.c new file mode 100644 index 0000000..74ad562 --- /dev/null +++ b/das2/serial2.c @@ -0,0 +1,959 @@ +/* Copyright (C) 2017-2024 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +#define _POSIX_C_SOURCE 200112L +#include +#include + +#include "stream.h" + +/* ************************************************************************** */ +/* Converting old style packets to datasets */ + +/* Max number of dimensions in a dataset */ +#define LEGACY_MAX_DIMS 64 +#define LEGACY_SRC_ARY_SZ 64 + +/* ************************************************************************* */ +/* Inspect plane properties and output standardized dimension role string */ + +const char* _serial_role(PlaneDesc* pPlane) +{ + const char* sRole = DasDesc_get((DasDesc*)pPlane, "operation"); + if(sRole == NULL) return DASVAR_CENTER; + + /* Interpret Autoplot style strings */ + if(strcmp("BIN_AVG", sRole) == 0) return DASVAR_MEAN; + if(strcmp("BIN_MAX", sRole) == 0) return DASVAR_MAX; + if(strcmp("BIN_MIN", sRole) == 0) return DASVAR_MIN; + + return DASVAR_CENTER; +} + +/* ************************************************************************** */ +/* Specialized property copies only used by the legacy converter */ + +/** Copy in dataset properties from some other descriptor + * + * This is a helper for das 2.2 streams. + * + * Any properties that don't start with a specific dimension identifier i.e. + * 'x','y','z','w' are copied into this dataset's properties dictionary. Only + * properties not present in the internal dictionary are copied in. + * + * @param pThis this dataset object + * @param pOther The descriptor containing properites to copy in + * @return The number of properties copied in + */ +int DasDs_copyInProps(DasDs* pThis, const DasDesc* pOther) +{ + const DasAry* pSource = &(pOther->properties); + size_t uProps = DasAry_lengthIn(pSource, DIM0); + + int nCopied = 0; + for(size_t u = 0; u < uProps; ++u){ + size_t uPropLen = 0; + const DasProp* pIn = (const DasProp*) DasAry_getBytesIn( + pSource, DIM1_AT(u), &uPropLen + ); + if(!DasProp_isValid(pIn)) + continue; + + const char* sName = DasProp_name(pIn); + + /* Do I want this prop? */ + if((*sName == 'x')||(*sName == 'y')||(*sName == 'z')||(*sName == '\0')) + continue; /* ... nope */ + + /* Do I have this property? ... */ + const DasProp* pOut = DasDesc_getLocal((DasDesc*)pThis, sName); + if(DasProp_isValid(pOut)) + continue; /* ... yep */ + + /* Set the property */ + if(DasDesc_setProp((DasDesc*)pThis, pIn) != DAS_OKAY){ + return nCopied; + } + ++nCopied; + } + return nCopied; +} + +/** Copy in dataset properties from some other descriptor + * + * This is a helper for das 2.2 streams as these use certian name patterns to + * indicate which dimension a property is for + * + * Any properties that start with a specific dimension identifier i.e. + * 'x','y','z','w' are copied into this dataset's properties dictionary. Only + * properties not present in the internal dictionary are copied in. + * + * @param pThis this dimension object + * @param cAxis the connonical axis to copy in. + * @param pOther The descriptor containing properites to copy in + * @return The number of properties copied in + * @memberof DasDim + */ + +int DasDim_copyInProps(DasDim* pThis, char cAxis, const DasDesc* pOther) +{ + char sNewName[32] = {'\0'}; + + const DasAry* pSrcAry = &(pOther->properties); + size_t uProps = DasAry_lengthIn(pSrcAry, DIM0); + + int nCopied = 0; + for(size_t u = 0; u < uProps; ++u){ + size_t uPropLen = 0; + const DasProp* pIn = (const DasProp*) DasAry_getBytesIn(pSrcAry, DIM1_AT(u), &uPropLen); + if(!DasProp_isValid(pIn)) + continue; + + const char* sName = DasProp_name(pIn); + + /* We only want stuff for the given axis, but we don't want to copy in + * the axis name, so make sure there's something after it. */ + if((*sName != cAxis)||( *(sName +1) == '\0')) + continue; /* ... nope */ + + /* Since we strip the x,y,z, make next char lower to preserve the look + * of the prop naming scheme */ + memset(sNewName, 0, 32); + int nLen = (int)(strlen(sName)) - 1; + nLen = nLen > 31 ? 31 : nLen; + strncpy(sNewName, sName + 1, 31); + sNewName[0] = tolower(sNewName[0]); + + DasDesc_flexSet((DasDesc*)pThis, NULL, DasProp_type(pIn), sNewName, + DasProp_value(pIn), DasProp_sep(pIn), pIn->units, DASPROP_DAS3 + ); + ++nCopied; + } + return nCopied; +} + +/* ************************************************************************* */ +/* Handle matching up planes into single dimensions, interface is really */ +/* complicated because this code was inline in another function */ + +DasDim* _serial_getDim( + PlaneDesc* pPlane, PktDesc* pPd, DasStream* pSd, + char cAxis, + DasDs* pDs, enum dim_type dType, const char* sDimId, + DasDim** ppDims, char* pDimSrc, size_t* puDims +){ + /* if this plane has a source property then it might be grouped */ + char* p = NULL; + char sNewDimId[64] = {'\0'}; + DasDim* pDim = NULL; + + const char* sSource = DasDesc_get((DasDesc*)pPlane, "source"); + if(sSource != NULL){ + /* See if this plane source appears in an existing dim */ + for(size_t u = 0; u < *puDims; u++){ + p = pDimSrc + u*LEGACY_SRC_ARY_SZ; + if(strncmp(sSource, p, LEGACY_SRC_ARY_SZ) == 0){ + /* Same group, add in any plane level properties that might be + * missing in this dimension */ + if(cAxis != '\0') + DasDim_copyInProps(ppDims[u], cAxis, (DasDesc*)pPlane); + return ppDims[u]; + } + } + } + + /* Didn't find a matching dim, so add one to the dataset, record it's info + * if it has an operation source property. Also de-kludge the name by + * removing anything after the dot + */ + + if(sSource != NULL){ + p = (char*) strchr(sDimId, '.'); + if(p && (p != sDimId)){ + strncpy(sNewDimId, sDimId, 63); + p = strchr(sNewDimId, '.'); + *p = '\0'; + sDimId = sNewDimId; + } + if((pDim = DasDs_makeDim(pDs, dType, sDimId, "")) == NULL) return NULL; + + if((*puDims) + 1 >= LEGACY_MAX_DIMS){ + das_error(DASERR_BLDR, "Too many dimensions in a single packet %d", + LEGACY_MAX_DIMS); + return NULL; + } + + ppDims[*puDims] = pDim; + + p = pDimSrc + (*puDims)*LEGACY_SRC_ARY_SZ; + strncpy(p, sSource, LEGACY_SRC_ARY_SZ - 1); + *puDims = *puDims + 1; + } + else{ + if((pDim = DasDs_makeDim(pDs, dType, sDimId, ""))==NULL) return NULL; + + } + + if(cAxis != '\0'){ + DasDim_copyInProps(pDim, cAxis, (DasDesc*)pSd); + DasDim_copyInProps(pDim, cAxis, (DasDesc*)pPd); + DasDim_copyInProps(pDim, cAxis, (DasDesc*)pPlane); + } + return pDim; +} + +/* ************************************************************************* */ +/* Handle making arrays suitable for receiving stream values + * + * Note: Internal arrays need to be some binary type if possible so that + * downstream programs can easily work with data. + * + * @param bRaw - If True, expect raw stream data, if False expect data + * parsed to doubles. + */ +DasAry* _serial_makeAry( + bool bRaw, const char* sAryId, DasEncoding* pEncoder, const ubyte* pFill, + int rank, size_t* shape, das_units defUnits +){ + DasAry* pAry = NULL; + das_val_type vtAry = vtDouble; /* no choice, data already read */ + das_units units = defUnits; + + /* I'm reading the stream directly and have a choice of internal encodings */ + if(bRaw){ + + switch(pEncoder->nCat){ + case DAS2DT_TIME: /* store text times as structures, and rm epoch units */ + vtAry = vtTime; + units = UNIT_UTC; + break; + case DAS2DT_ASCII: /* If over 12 chars (including whitespace) encode as double */ + vtAry = (pEncoder->nWidth > 12) ? vtDouble : vtFloat; + break; + default: /* All that's left are DAS2DT_BE_REAL & DAS2DT_LE_REAL */ + vtAry = (pEncoder->nWidth > 4) ? vtDouble : vtFloat; + break; + } + } + + pAry = new_DasAry(sAryId, vtAry, 0, pFill, rank, shape, units); + return pAry; +} + +/* ************************************************************************* */ +/* Internal storage and possible direct encoding */ + +DasErrCode _serial_addCodec( + DasDs* pDs, const char* sAryId, int nItems, DasEncoding* pEncoder +){ + + int nHash = DasEnc_hash(pEncoder); + const char* sEncType = NULL; + int nItemBytes = 0; + const char* sSemantic = "real"; + switch(nHash){ + case DAS2DT_BE_REAL_8: sEncType = "BEreal"; nItemBytes = 8; break; + case DAS2DT_LE_REAL_8: sEncType = "LEreal"; nItemBytes = 8; break; + case DAS2DT_BE_REAL_4: sEncType = "BEreal"; nItemBytes = 4; break; + case DAS2DT_LE_REAL_4: sEncType = "LEreal"; nItemBytes = 4; break; + } + if(sEncType == NULL){ + nItemBytes = pEncoder->nWidth; + sEncType = "utf8"; + if(pEncoder->nCat == DAS2DT_TIME) + sSemantic = "datetime"; + } + + return DasDs_addFixedCodec(pDs, sAryId, sSemantic, sEncType, nItemBytes, nItems); +} + + +/* ************************************************************************* */ +/* Initialize X-Y pattern + * Make one dimension for X and one each for the Y's. Could do some deeper + * inspection to see if some of the columns should be grouped into the same + * dimension. For das 2.3 streams where we can have multiple X columns + * this will be a must. */ + +void _strrep(char* pId, char c, char r) +{ + char* pTmp = pId; + while(*pTmp != '\0'){ + if(*pTmp == c) *pTmp = r; + ++pTmp; + } +} + + +DasDs* _serial_initXY( + DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs +){ + /* If my group name is null, make up a new one appropriate to XY data */ + const char* pId = NULL; + PlaneDesc* pPlane = NULL; + DasEncoding* pEncoder = NULL; + const char* sRole = NULL; /* The reason this plane is present */ + int nY = 0; + char sBuf[64] = {'\0'}; + char sDsId[64] = {'\0'}; + if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); + if(pGroup == NULL){ + if(PktDesc_getNPlanesOfType(pPd, Y) == 1){ + pPlane = PktDesc_getPlaneByType(pPd, Y, 0); + pGroup = PlaneDesc_getName(pPlane); + } + else{ + nY = PktDesc_getNPlanesOfType(pPd, Y); + snprintf(sBuf, 63, "unknown_%dY", nY); + pGroup = sBuf; + nY = 0; + } + } + + snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); + + DasDs* pDs = new_DasDs(sDsId, pGroup, 1); + DasDim* pDim = NULL; + DasAry* pAry = NULL; + DasVar* pVar = NULL; + das_units units = UNIT_DIMENSIONLESS; + enum dim_type dType = DASDIM_UNK; + + /* Tracking for array groups (aka dimensions) */ + DasDim* aDims[LEGACY_MAX_DIMS] = {NULL}; + char aDimSrc[LEGACY_MAX_DIMS * LEGACY_SRC_ARY_SZ] = {'\0'}; + size_t uDims = 0; + + /* Copy any properties that don't start with one of the axis prefixes */ + + /* ... actually, we output stream headers now, so this isn't needed, it + was always a bit of a hack and doesn't play well with CDF generation. + DasDs_copyInProps(pDs, (DasDesc*)pSd); + DasDs_copyInProps(pDs, (DasDesc*)pPd); + */ + + char sAryId[64] = {'\0'}; + double fill; + char cAxis = '\0'; + + for(size_t u = 0; u < pPd->uPlanes; ++u){ + pPlane = pPd->planes[u]; + pEncoder = PlaneDesc_getValEncoder(pPlane); + fill = PlaneDesc_getFill(pPlane); + units = pPlane->units; + + pId = pPlane->sName; + if(pPlane->planeType == X){ + cAxis = 'x'; + if(pId == NULL){ + if(Units_haveCalRep(pPlane->units)) pId = "time"; + else pId = "X"; + } + + pAry = _serial_makeAry( + bCodecs, pId, pEncoder, (const ubyte*)&fill, RANK_1(0), units + ); + } + else{ + cAxis = 'y'; + ++nY; + + /* Always copy over the name if it exists */ + if(pId == NULL) snprintf(sAryId, 63, "Y_%d", nY); + else strncpy(sAryId, pId, 63); + _strrep(sAryId, '.', '_'); /* handle amplitude.max type stuff */ + + pAry = _serial_makeAry( + bCodecs, sAryId, pEncoder, (const ubyte*)&fill, RANK_1(0), units + ); + } + if(pAry == NULL) return NULL; + + /* add the new array to the DS so it has somewhere to put this item from + * the packets */ + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + + /* Remember how to fill this array. This will get more complicated + * when variable length packets are introduced */ + DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); + + /* On to higher level data organizational structures: Create dimensions + * and variables as needed for the new arrays */ + if(pPlane->planeType == X) dType = DASDIM_COORD; + else dType = DASDIM_DATA; + + pDim = _serial_getDim( + pPlane, pPd, pSd, cAxis, pDs, dType, pId, aDims, aDimSrc, &uDims + ); + if(pDim == NULL) return NULL; + + pVar = new_DasVarArray(pAry, SCALAR_1(0)); + if( pVar == NULL) return NULL; + sRole = _serial_role(pPlane); + if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; + + if(bCodecs) + if(_serial_addCodec(pDs, sAryId, 1, pEncoder) != DAS_OKAY) + return NULL; + } + return pDs; +} + +/* ************************************************************************* */ +/* Initialize X-Y-Z pattern */ + +DasDs* _serial_initXYZ( + DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs +){ + /* If my group name is null, make up a new one appropriate to XYZ data */ + const char* pId = NULL; + PlaneDesc* pPlane = NULL; + DasEncoding* pEncoder = NULL; + const char* sRole = NULL; /* The reason this plane is present */ + int nZ = 0; + char sBuf[64] = {'\0'}; + char sDsId[64] = {'\0'}; + if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); + if(pGroup == NULL){ + if(PktDesc_getNPlanesOfType(pPd, Z) == 1){ + pPlane = PktDesc_getPlaneByType(pPd, Z, 0); + pGroup = PlaneDesc_getName(pPlane); + } + else{ + nZ = PktDesc_getNPlanesOfType(pPd, Z); + snprintf(sBuf, 63, "unknown_%dZ", nZ); + pGroup = sBuf; + nZ = 0; + } + } + + snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); + + DasDs* pDs = new_DasDs(sDsId, pGroup, 1); + DasDim* pDim = NULL; + DasAry* pAry = NULL; + DasVar* pVar = NULL; + das_units units = UNIT_DIMENSIONLESS; + + enum dim_type dType = DASDIM_UNK; + + /* Tracking for array groups (aka dimensions) */ + DasDim* aDims[LEGACY_MAX_DIMS] = {NULL}; + char aDimSrc[LEGACY_MAX_DIMS * LEGACY_SRC_ARY_SZ] = {'\0'}; + size_t uDims = 0; + + /* Copy any properties that don't start with one of the axis prefixes */ + /* ... or don't. Messes up CDF output + DasDs_copyInProps(pDs, (DasDesc*)pSd); + DasDs_copyInProps(pDs, (DasDesc*)pPd); */ + + char sAryId[64] = {'\0'}; + double fill; + char cAxis = '\0'; + + for(size_t u = 0; u < pPd->uPlanes; ++u){ + pPlane = pPd->planes[u]; + pId = pPlane->sName; + pEncoder = PlaneDesc_getValEncoder(pPlane); + fill = PlaneDesc_getFill(pPlane); + units = pPlane->units; + + switch(pPlane->planeType){ + case X: + cAxis = 'x'; + if(pId == NULL){ + if(Units_haveCalRep(pPlane->units)) pId = "time"; + else pId = "X"; + } + + /* Fill is not allowed for Das 2.2 X planes in an X,Y,Z pattern */ + pAry = _serial_makeAry( + bCodecs, pId, pEncoder, (const ubyte*)&fill, RANK_1(0), units + ); + break; + + case Y: + cAxis = 'y'; + if(pId == NULL)pId = "Y"; + + /* Fill is not allowed for Das 2.2 Y planes in an X,Y,Z pattern */ + pAry = _serial_makeAry( + bCodecs, pId, pEncoder, (const ubyte*)&fill, RANK_1(0), units + ); + break; + + case Z: + cAxis = 'z'; + ++nZ; + + if(pId == NULL) snprintf(sAryId, 63, "Z_%d", nZ); + else strncpy(sAryId, pId, 63); + _strrep(sAryId, '.', '_'); /* handle amplitude.max stuff */ + + pAry = _serial_makeAry( + bCodecs, pId, pEncoder, (const ubyte*)&fill, RANK_1(0), units + ); + break; + + + default: + das_error(DASERR_BLDR, "Unexpected plane type in XYZ pattern"); + return NULL; + } + + if(pAry == NULL) return NULL; + + /* add the new array to the DS so it has somewhere to put this item from + * the packets */ + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + + /* Remember how to fill this array. This will get more complicated + * when variable length packets are introduced. So this is item 'u' + * from the packet and 1 item is read for each packet */ + DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); + + /* Create dimensions and variables as needed for the new arrays */ + if(pPlane->planeType == Z) dType = DASDIM_DATA; + else dType = DASDIM_COORD; + + pDim = _serial_getDim( + pPlane, pPd, pSd, cAxis, pDs, dType, pId, aDims, aDimSrc, &uDims + ); + if(pDim == NULL) return NULL; + + pVar = new_DasVarArray(pAry, SCALAR_1(0)); + if( pVar == NULL) return NULL; + + /* Assume these are center values unless there is a property stating + * otherwise */ + sRole = _serial_role(pPlane); + if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; + + if(bCodecs) + if(_serial_addCodec(pDs, sAryId, 1, pEncoder) != DAS_OKAY) + return NULL; + } + + return pDs; +} + +/* ************************************************************************* */ +/* Initialize Events Pattern */ + +DasDs* _serial_initEvents(DasStream* pSd, PktDesc* pPd, const char* sGroupId) +{ + das_error(DASERR_BLDR, "Event stream reading has not been implemented"); + return NULL; +} + +/* ************************************************************************* */ +/* Initialize YScan Pattern */ + +bool _serial_checkYTags(PktDesc* pPd) +{ + size_t uYScans = PktDesc_getNPlanesOfType(pPd, YScan); + if(uYScans < 2) return true; + + PlaneDesc* pFirst = PktDesc_getPlaneByType(pPd, YScan, 0); + size_t uYTags = PlaneDesc_getNItems(pFirst); + ytag_spec_t spec = PlaneDesc_getYTagSpec(pFirst); + double rInterval = -1.0, rMin = -1.0, rMax = -1.0; + if(spec == ytags_series) + PlaneDesc_getYTagSeries(pFirst, &rInterval, &rMin, &rMax); + const double* pYTags = NULL; + if(spec == ytags_list) pYTags = PlaneDesc_getYTags(pFirst); + + das_units units = PlaneDesc_getYTagUnits(pFirst); + + PlaneDesc* pNext = NULL; + size_t u,v; + double rNextInterval = -1.0, rNextMin = -1.0, rNextMax = -1.0; + const double* pNextYTags = NULL; + for(u = 1; u < uYScans; ++u){ + pNext = PktDesc_getPlaneByType(pPd, YScan, u); + if(uYTags != PlaneDesc_getNItems(pNext)) return false; + + /* The tags can be specified as none, a list, or a series */ + if(spec != PlaneDesc_getYTagSpec(pNext)) return false; + + if(units != PlaneDesc_getYTagUnits(pNext)) return false; + + switch(spec){ + case ytags_none: break; + case ytags_series: + PlaneDesc_getYTagSeries(pNext, &rNextInterval, &rNextMin, &rNextMax); + if(rInterval != rNextInterval ) return false; + if(rMin != rNextMin) return false; + if(rMax != rNextMax) return false; + break; + + case ytags_list: + pNextYTags = PlaneDesc_getYTags(pNext); + for(v = 0; vplaneType != YScan){ + das_error(DASERR_BLDR, "Program logic error"); + return NULL; + } + double* pTags = (double*)calloc(PlaneDesc_getNItems(pPlane), sizeof(double)); + size_t u, uItems = PlaneDesc_getNItems(pPlane); + const double* pListTags = NULL; + double rInterval, rMin, rMax; + switch(pPlane->ytag_spec){ + case ytags_list: + pListTags = PlaneDesc_getYTags(pPlane); + for(u = 0; u < uItems; ++u) pTags[u] = pListTags[u]; + break; + case ytags_none: + for(u = 0; u < uItems; ++u) pTags[u] = u; + break; + case ytags_series: + PlaneDesc_getYTagSeries(pPlane, &rInterval, &rMin, &rMax); + for(u = 0; u < uItems; ++u) pTags[u] = rMin + (rInterval * u); + break; + } + return pTags; +} + +bool _serial_isWaveform(PlaneDesc* pPlane){ + + const char* sRend = DasDesc_getStr((DasDesc*)pPlane, "renderer"); + if(sRend == NULL) return false; + if(strcmp("waveform", sRend) != 0) return false; + + das_units units = PlaneDesc_getYTagUnits(pPlane); + if(! Units_canConvert(units, UNIT_SECONDS)) return false; + return true; +} + +DasDs* _serial_initYScan( + DasStream* pSd, PktDesc* pPd, const char* pGroup, bool bCodecs +){ + /* Make sure all the yscans have the same yTags. The assumption here is + * that a single packet only has data correlated in it's coordinates. If + * two yscans have different yTag sets then they are not correlated. */ + + if(!_serial_checkYTags(pPd)){ + das_error(DASERR_BLDR, "YTags are not equivalent in multi-yscan packet"); + return NULL; + } + + /* If my group name is null, make up a new one appropriate to YScan data */ + PlaneDesc* pPlane = PktDesc_getPlaneByType(pPd, YScan, 0); + int nY = 0, nYScan = 0; + char sDsGroup[64] = {'\0'}; + char sDsId[64] = {'\0'}; + + if(pGroup == NULL) pGroup = PktDesc_getGroup(pPd); + if(pGroup == NULL) pGroup = PlaneDesc_getName(pPlane); + if(pGroup == NULL){ + nYScan = PktDesc_getNPlanesOfType(pPd, YScan); + snprintf(sDsGroup, 63, "default_%d_MultiZ", nYScan); + pGroup = sDsGroup; + } + snprintf(sDsId, 63, "%s_%02d", pGroup, PktDesc_getId(pPd)); + + size_t uItems = PlaneDesc_getNItems(pPlane); + + DasDs* pDs = new_DasDs(sDsId, pGroup, 2); + DasDim* pDim = NULL; + DasDim* pXDim = NULL; + DasDim* pYDim = NULL; + DasVar* pVar = NULL; + DasVar* pOffset = NULL; + DasVar* pReference = NULL; + + /* Dito, see above searching for string CDF + DasDs_copyInProps(pDs, (DasDesc*)pSd); + DasDs_copyInProps(pDs, (DasDesc*)pPd); */ /*These never have props in das 2.2 */ + const char* pPlaneId = NULL; + DasEncoding* pEncoder = NULL; + const char* sRole = NULL; + das_units Yunits = UNIT_DIMENSIONLESS; + das_units Zunits = UNIT_DIMENSIONLESS; + const char* pYTagId = NULL; + DasAry* pAry = NULL; + char sAryId[64] = {'\0'}; + double fill; + + /* One thing to watch out for. The number of dimensions is not equal to + * the number of planes. If two planes have the same 'source' property + * but different roles then they go together in the same dimension. + * + * The old das2 stream model is *REALLY* showing it's age and needs to be + * updated. To much hoop jumping and heuristics are needed to shove + * complex info into such a simple structure. QStream is better, but still + * not explicit enough. We just need a break from the past. -cwp 2019-04-05 + */ + DasDim* aDims[LEGACY_MAX_DIMS] = {NULL}; + char aDimSrc[LEGACY_MAX_DIMS*LEGACY_SRC_ARY_SZ] = {'\0'}; + size_t uDims = 0; + + double* pYTags = NULL; + bool bAddedYTags = false; + for(size_t u = 0; u < pPd->uPlanes; ++u){ + pPlane = pPd->planes[u]; + if(pPlane->sName != NULL) pPlaneId = pPlane->sName; + + /* Assume this is a center value unless told otherwise */ + sRole = _serial_role(pPlane); + pEncoder = PlaneDesc_getValEncoder(pPlane); + fill = PlaneDesc_getFill(pPlane); + + /* If we're lucky the plane will tell us the purpose for it's + * values, i.e. min, center, max, err-bar */ + + switch(pPlane->planeType){ + case X: + if(pPlaneId == NULL){ + if(Units_haveCalRep(pPlane->units)) pPlaneId = "time"; + else pPlaneId = "X"; + } + + pAry = _serial_makeAry( + bCodecs, pPlaneId, pEncoder, (const ubyte*)&fill, RANK_1(0), pPlane->units + ); + + if(pAry == NULL) return NULL; + DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + + pXDim = _serial_getDim( + pPlane, pPd, pSd, 'x', pDs, DASDIM_COORD, pPlaneId, + aDims, aDimSrc, &uDims + ); + if(pXDim == NULL) return NULL; + + /* Map index 0 to time array 0, ignore index 1 */ + pVar = new_DasVarArray(pAry, SCALAR_2(0, DASIDX_UNUSED)); + if(pVar == NULL) return NULL; + if(! DasDim_addVar(pXDim, sRole, pVar)) return NULL; + + if(bCodecs) + if(_serial_addCodec(pDs, pPlaneId, 1, pEncoder) != DAS_OKAY) + return NULL; + break; + + case Y: + ++nY; + if(pPlaneId == NULL) + snprintf(sAryId, 63, "Y_%d", nY); + else + strncpy(sAryId, pPlaneId, 63); + _strrep(sAryId, '.', '_'); + + pAry = _serial_makeAry( + bCodecs, sAryId, pEncoder, (const ubyte*)&fill, RANK_1(0), pPlane->units + ); + if(pAry == NULL) return NULL; + + DasAry_setSrc(pAry, PktDesc_getId(pPd), u, 1); + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + + /* Assume that extra Y values are more coordinates unless told + * otherwise by a setting of some sort that I don't yet know */ + pYDim = _serial_getDim( + pPlane, pPd, pSd, 'y', pDs, DASDIM_COORD, pPlaneId, + aDims, aDimSrc, &uDims + ); + if(pYDim == NULL) return NULL; + + /* Map index 0 to this array, ignore index 1 */ + pVar = new_DasVarArray(pAry, SCALAR_2(0, DASIDX_UNUSED)); + + if(pVar == NULL) return NULL; + if(! DasDim_addVar(pYDim, sRole, pVar)) return NULL; + + if(bCodecs) + if(_serial_addCodec(pDs, pPlaneId, 1, pEncoder) != DAS_OKAY) + return NULL; + break; + + case YScan: + ++nYScan; + + /* This one is interesting, maybe have to add two arrays. + * + * Also sometimes this is a waveform and we need to set the yTags as + * an offset dimension for time. + */ + + if(!bAddedYTags){ + Yunits = PlaneDesc_getYTagUnits(pPlane); + if( Units_canConvert(Yunits, UNIT_HERTZ) ){ + pYTagId = "frequency"; + } + else{ + if(Units_canConvert(Yunits, UNIT_SECONDS)) pYTagId = "offset"; + else{ + if(Units_canConvert(Yunits, UNIT_EV)) pYTagId = "energy"; + else pYTagId = "ytags"; + } + } + pAry = new_DasAry(pYTagId, vtDouble, 0, NULL, RANK_1(uItems), Yunits); + if(pAry == NULL) return NULL; + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + pYTags = _serial_yTagVals(pPlane); + + /* Use put instead of append since we've already allocated the space */ + DasAry_putAt(pAry, IDX0(0), (const ubyte*)pYTags, uItems); + free(pYTags); pYTags = NULL; + + /* If the data aren't waveforms we're going to need to add a new + * dimension, otherwise we'll add in the REF and OFFSET variables */ + if( _serial_isWaveform(pPlane) ){ + /* ignore first index, map second index to array */ + pOffset = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); + DasDim_addVar(pXDim, DASVAR_OFFSET, pOffset); + + /* Convert the old CENTER variable to a Reference variable */ + pReference = DasDim_popVar(pXDim, DASVAR_CENTER); + DasDim_addVar(pXDim, DASVAR_REF, pReference); + + /* Make new center variable that is rank 2 since reference and + * offset are orthogonal */ + pVar = new_DasVarBinary("center", pReference, "+", pOffset); + if(! DasDim_addVar(pXDim, DASVAR_CENTER, pVar) ) return NULL; + } + else{ + /* If we have and then the Y's are our reference values + Convert the old CENTER variable to a Reference variable and + add the Ytags as offsets */ + if(pYDim != NULL){ + pOffset = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); + DasDim_addVar(pYDim, DASVAR_OFFSET, pOffset); + + /* Convert the old CENTER variable to a Reference variable */ + pReference = DasDim_popVar(pYDim, DASVAR_CENTER); + DasDim_addVar(pYDim, DASVAR_REF, pReference); + + /* Make new center variable that is rank 2 since reference and + * offset are orthogonal */ + pVar = new_DasVarBinary("center", pReference, "+", pOffset); + if(! DasDim_addVar(pYDim, DASVAR_CENTER, pVar) ) return NULL; + } + else{ + /* Nope no offsets in freq or time, just a new center variable */ + pDim = DasDs_makeDim(pDs, DASDIM_COORD, pYTagId, ""); + if(pDim == NULL) return NULL; + + DasDim_copyInProps(pDim, 'y', (DasDesc*)pSd); + DasDim_copyInProps(pDim, 'y', (DasDesc*)pPd); + DasDim_copyInProps(pDim, 'y', (DasDesc*)pPlane); + + /* Map index 1 to this array's index 0, ignore 0, assume */ + /* center values */ + pVar = new_DasVarArray(pAry, SCALAR_2(DASIDX_UNUSED, 0)); + if(! DasDim_addVar(pDim, DASVAR_CENTER, pVar)) return NULL; + } + } + + bAddedYTags = true; + } + + /* Now for the actual data values array, try for a good array name */ + Zunits = PlaneDesc_getUnits(pPlane); + if(pPlaneId == NULL){ + if(Units_canConvert(Zunits, UNIT_E_SPECDENS)) + strncpy(sAryId, "e_spec_dens", 63); + else if(Units_canConvert(Zunits, UNIT_B_SPECDENS)) + strncpy(sAryId, "b_spec_dens", 63); + else + snprintf(sAryId, 63, "YScan_%d", nYScan); + } + else{ + strncpy(sAryId, pPlaneId, 63); + } + _strrep(sAryId, '.', '_'); + + pAry = _serial_makeAry( + bCodecs, sAryId, pEncoder, (const ubyte*)&fill, RANK_2(0, uItems), Zunits + ); + if(pAry == NULL) return NULL; + + if(DasDs_addAry(pDs, pAry) != DAS_OKAY) return NULL; + DasAry_setSrc(pAry, PktDesc_getId(pPd), u, uItems); + + pDim = _serial_getDim( + pPlane, pPd, pSd, 'z', pDs, DASDIM_DATA, pPlaneId, aDims, + aDimSrc, &uDims + ); + if(pDim == NULL) return NULL; + + /* Map index 0 to 0 and 1 to 1 */ + pVar = new_DasVarArray(pAry, SCALAR_2(0, 1)); + if(pVar == NULL) return NULL; + if(! DasDim_addVar(pDim, sRole, pVar)) return NULL; + + if(bCodecs) + if(_serial_addCodec(pDs, sAryId, (int)uItems, pEncoder) != DAS_OKAY) + return NULL; + + break; + + default: + das_error(DASERR_DS, "logic error"); + return NULL; + break; + } + } + return pDs; +} + +/* bCodecs - If true, define codecs for the generated dataset object so that + * it can be used to parse data packet payloads. + */ +DasDs* dasds_from_packet(DasStream* pSd, PktDesc* pPd, const char* sGroup, bool bCodecs) +{ + /* Initialize based on the observed pattern. Das2 streams have traditionally + * followed certian layout patterns, you can't have arbitrary collections of + * and planes. */ + size_t u, uPlanes = PktDesc_getNPlanes(pPd); + PlaneDesc* pPlane = NULL; + int nXs = 0, nYs = 0, nYScans = 0, nZs = 0; + for(u = 0; u < uPlanes; ++u){ + pPlane = PktDesc_getPlane(pPd, u); + switch(PlaneDesc_getType(pPlane)){ + case X: ++nXs; break; + case Y: ++nYs; break; + case YScan: ++nYScans; break; + case Z: ++nZs; break; + case Invalid: das_error(DASERR_DS, "logic error"); return NULL; + } + } + + DasDs* pCd = NULL; + if(nYScans == 0){ + if(nZs != 0) pCd = _serial_initXYZ(pSd, pPd, sGroup, bCodecs); + else{ + if(nXs == 2) pCd = _serial_initEvents(pSd, pPd, sGroup); + else pCd = _serial_initXY(pSd, pPd, sGroup, bCodecs); + } + } + else{ + pCd = _serial_initYScan(pSd, pPd, sGroup, bCodecs); + } + return pCd; +} diff --git a/das2/serial2.h b/das2/serial2.h new file mode 100644 index 0000000..26fb042 --- /dev/null +++ b/das2/serial2.h @@ -0,0 +1,67 @@ +/* Copyright (C) 2024 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file serial.h Creating, reading and writing datasets to and from buffers */ + +#ifndef _das_serial2_h_ +#define _das_serial2_h_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Generate an potentially active dataset object from a packet descriptor. + * + * @note There are two ways to use this function: + * 1. To create a dataset structure suitable for manual data insertion of + * das2 packet data (no codecs) + * + * 2. To create a dataset structure that parses it's own data direct from + * a stream without going through das2 parsing (has codecs) + * + * Since DasPlane interprets all data at doubles, setting bCodecs to + * false will generate attached arrays that expect doubles. Setting bCodecs + * to true will generate attached arrays suitable for handling stream data + * "as-is". + * + * @param pSd - The stream descriptor, this is needed to detect for "waveform" layout + * + * @param pPd - The packet descriptor to inspect, it is not changed. + * + * @param sGroup - The group string name to assign to this dataset, may be NULL in + * which case pPd is inspected for a group name. + * + * @param bCodecs If true, define codecs for the generated dataset object so that + * it can be used to parse data packet payloads. See the note above. + * + * @returns A new DasDs object allocated on the heap if successful, NULL otherwise + * and das_error is called internally. Note that the dataset is not + * attached to the stream descriptor. It is the callers responsibility to + * attach it if desired. + */ +DAS_API DasDs* dasds_from_packet( + DasStream* pSd, PktDesc* pPd, const char* sGroup, bool bCodecs +); + + +#ifdef __cplusplus +} +#endif + +#endif /* _serial */ diff --git a/das2/serial.c b/das2/serial3.c similarity index 99% rename from das2/serial.c rename to das2/serial3.c index dbf457b..b835701 100644 --- a/das2/serial.c +++ b/das2/serial3.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2018 Chris Piker +/* Copyright (C) 2017-2024 Chris Piker * * This file is part of das2C, the Core Das2 C Library. * diff --git a/das2/serial.h b/das2/serial3.h similarity index 99% rename from das2/serial.h rename to das2/serial3.h index df12b7b..27fcfa9 100644 --- a/das2/serial.h +++ b/das2/serial3.h @@ -60,8 +60,6 @@ DAS_API DasDs* dasds_from_xmlheader(DasBuf* pBuf, StreamDesc* pParent, int nPktI */ DAS_API DasErrCode dasds_decode_data(DasDs* pDs, DasBuf* pBuf); - - #ifdef __cplusplus } #endif diff --git a/das2/stream.c b/das2/stream.c index c147513..ce72a07 100644 --- a/das2/stream.c +++ b/das2/stream.c @@ -37,7 +37,8 @@ #include -#include "serial.h" +#include "serial2.h" +#include "serial3.h" #include "stream.h" /* ************************************************************************** */ diff --git a/das2/variable.c b/das2/variable.c index 8a8fc53..c20e9c5 100644 --- a/das2/variable.c +++ b/das2/variable.c @@ -1544,7 +1544,7 @@ DasErrCode init_DasVarArray( /* Make sure that the last index < the first internal for scalar types, and that last index == first internal for rank 1 types */ - if(vtAry == vtUByte){ + if((vtAry == vtUByte)||(vtAry == vtByte)){ if((pAry->uFlags & D2ARY_AS_STRING) == D2ARY_AS_STRING){ if(nIntRank != 1) return das_error(DASERR_VAR, "Dense text needs an internal rank of 1"); diff --git a/das2/variable.h b/das2/variable.h index 2ce05fd..e2ce372 100644 --- a/das2/variable.h +++ b/das2/variable.h @@ -50,7 +50,10 @@ enum var_type { * * The vector macros do the same but they also assume check the array to * make sure that there is one extra index right after all the ones - * that are mentioned. + * that are mentioned. + * + * The string macors are the same as the vector macros, just included here + * for code readility */ #define SCALAR_0 0, (NULL), 0 #define SCALAR_1(I) 1, (int8_t[DASIDX_MAX]){I,_D,_D,_D,_D,_D,_D,_D}, 0 @@ -71,6 +74,15 @@ enum var_type { #define VEC_6(I,J,K,L,M,N) 6, (int8_t[DASIDX_MAX]){I,J,K,L,M,N,_D,_D}, 1 #define VEC_7(I,J,K,L,M,N,O) 7, (int8_t[DASIDX_MAX]){I,J,K,L,M,N,O,_D}, 1 +#define STRING_0 0, (int8_t[DASIDX_MAX]){_D,_D,_D,_D,_D,_D,_D,_D}, 1 +#define STRING_1(I) 1, (int8_t[DASIDX_MAX]){I,_D,_D,_D,_D,_D,_D,_D}, 1 +#define STRING_2(I,J) 2, (int8_t[DASIDX_MAX]){I,J,_D,_D,_D,_D,_D,_D}, 1 +#define STRING_3(I,J,K) 3, (int8_t[DASIDX_MAX]){I,J,K,_D,_D,_D,_D,_D}, 1 +#define STRING_4(I,J,K,L) 4, (int8_t[DASIDX_MAX]){I,J,K,L,_D,_D,_D,_D}, 1 +#define STRING_5(I,J,K,L,M) 5, (int8_t[DASIDX_MAX]){I,J,K,L,M,_D,_D,_D}, 1 +#define STRING_6(I,J,K,L,M,N) 6, (int8_t[DASIDX_MAX]){I,J,K,L,M,N,_D,_D}, 1 +#define STRING_7(I,J,K,L,M,N,O) 7, (int8_t[DASIDX_MAX]){I,J,K,L,M,N,O,_D}, 1 + /* Internal function for merging variable, and dimension shapes. Different * rules apply for arrays to variable shape merges. diff --git a/utilities/das3_cdf.c b/utilities/das3_cdf.c index ea634af..8a85b7f 100644 --- a/utilities/das3_cdf.c +++ b/utilities/das3_cdf.c @@ -1318,8 +1318,8 @@ long DasVar_cdfType(const DasVar* pVar) CDF_UINT1, /* vtByteSeq = 15 */ }; - /* If the units of the variable are any time units, return a type of tt2k */ - if((DasVar_units(pVar) == UNIT_TT2000)&&(DasVar_valType(pVar) == vtLong)) + /* All calendar times will be converted to TT2000 by the CDF writer */ + if(Units_haveCalRep(DasVar_units(pVar))) return CDF_TIME_TT2000; /* For other variables, we want the underlying type */ @@ -1482,9 +1482,9 @@ DasErrCode makeCdfVar( /* Make a name for this variable, since everything is flattened */ DasVar_cdfUniqName(pCtx->nCdfId, pDim, pVar, sNameBuf, DAS_MAX_ID_BUFSZ - 1); - das_val_type vt = DasVar_valType(pVar); + /* If this var is to be interpreted as a text value, we'll need strlen */ long nCharLen = 1L; - if((vt == vtText)||(vt == vtByteSeq)){ + if(DasVar_cdfType(pVar) == CDF_UCHAR){ ptrdiff_t aIntr[DASIDX_MAX] = {0}; DasVar_intrShape(pVar, aIntr); nCharLen = aIntr[0]; @@ -1571,7 +1571,7 @@ DasErrCode makeCdfVar( int nAryRank = DasAry_shape(pAry, aAryShape); size_t uLen = 0; - vt = DasAry_valType(pAry); + das_val_type vt = DasAry_valType(pAry); const ubyte* pVals = DasAry_getIn(pAry, vt, DIM0, &uLen); /* Put index information into data types needed for function call */ @@ -1919,7 +1919,7 @@ DasErrCode onDataSet(StreamDesc* pSd, int iPktId, DasDs* pDs, void* pUser) int64_t* g_pTimeValBuf = NULL; size_t g_uTimeBufLen = 0; -const ubyte* _structToTT2k(const ubyte* pData, size_t uTimes) +const ubyte* _structToTT2k(const ubyte* pData, size_t uTimes) { if(g_uTimeBufLen < uTimes){ if(g_pTimeValBuf != NULL){ @@ -1935,6 +1935,33 @@ const ubyte* _structToTT2k(const ubyte* pData, size_t uTimes) return (const ubyte*)g_pTimeValBuf; } +const ubyte* _valueToTT2k( + const ubyte* pData, size_t uTimes, das_val_type vt, das_units units +){ + if(g_uTimeBufLen < uTimes){ + if(g_pTimeValBuf != NULL){ + free(g_pTimeValBuf); + } + g_pTimeValBuf = (int64_t*)calloc(uTimes, sizeof(int64_t)); + } + + /* Just handle doubles for now, that's the most common time type */ + const double* pDblSrc = NULL; + switch(vt){ + case vtDouble: + /* TODO: Check endianness here! */ + pDblSrc = (const double*)pData; + for(size_t u = 0; u < uTimes; ++u){ + g_pTimeValBuf[u] = das_us2K_to_tt2K(Units_convertTo(UNIT_US2000, pDblSrc[u], units)); + } + return (const ubyte*)g_pTimeValBuf; + + default: + das_error(DASERR_NOTIMP, "Add conversion for epoch based from type %s", das_vt_toStr(vt)); + return NULL; + } +} + DasErrCode _writeRecVaryAry(CDFid nCdfId, DasVar* pVar, DasAry* pAry) { CDFstatus iStatus; /* Used by the CDF_MAD macro */ @@ -1960,12 +1987,17 @@ DasErrCode _writeRecVaryAry(CDFid nCdfId, DasVar* pVar, DasAry* pAry) if(pData == NULL) return PERR; - /* Hook in data conversion. If we see vtTime, that's a structure, and - it needs to be re-written to TT2K */ + /* Hook in time conversion conversion. If we see vtTime, that's a structure, and + it needs to be re-written to TT2K, if we see */ - if(DasAry_valType(pAry) == vtTime){ + if(DasAry_valType(pAry) == vtTime) pData = _structToTT2k(pData, uElements); - } + else if( + Units_haveCalRep(DasAry_units(pAry)) && + (DasAry_valType(pAry) != vtLong) && + (DasAry_units(pAry) != UNIT_TT2000) + ) + pData = _valueToTT2k(pData, uElements, DasAry_valType(pAry), DasAry_units(pAry)); int nRank = DasAry_shape(pAry, aShape);