diff --git a/ScientificDataSet/Providers/NetCDF/NetCDFDataSet.cs b/ScientificDataSet/Providers/NetCDF/NetCDFDataSet.cs
index 9ce5eb2..a6ae6d1 100644
--- a/ScientificDataSet/Providers/NetCDF/NetCDFDataSet.cs
+++ b/ScientificDataSet/Providers/NetCDF/NetCDFDataSet.cs
@@ -24,7 +24,7 @@ namespace Microsoft.Research.Science.Data.NetCDF4
///
///
/// The provider supports variables of any non-negative rank.
- ///
+ ///
///
/// The provider is associated with the provider name "nc" and extensions ".nc".
///
@@ -44,7 +44,7 @@ namespace Microsoft.Research.Science.Data.NetCDF4
/// -
/// openMode=createNew|create|open|openOrCreate|readOnly
///
- /// The flag "openMode" specifies how the data set should open a file,
+ /// The flag "openMode" specifies how the data set should open a file,
/// data base or whatever resource it uses to store the data.
/// Possible values for the flag are:
///
@@ -106,13 +106,13 @@ namespace Microsoft.Research.Science.Data.NetCDF4
/// include
/// Allows including variables as references from another , defined as a URI,
/// into this .
- /// Example: msds:memory?include=msds%3Acsv%3Ffile%example.csv%23lat%2Clon
+ /// Example: msds:memory?include=msds%3Acsv%3Ffile%example.csv%23lat%2Clon
/// (escaped version of "msds:memory?include=escape[msds:csv?file=example.csv#lat,lon]")
/// includes variables lat and lon from msds:csv?file=example.csv. If variables names are not specified,
/// all variables are included.
///
///
- ///
+ ///
///
/// The provider supports URIs containing a path
/// and appended through '?' parameters: c:\data\air0.nc?openMode=open&enableRollback=false.
@@ -132,17 +132,17 @@ namespace Microsoft.Research.Science.Data.NetCDF4
/// To enable it, implements the interface.
///
///
- /// Names. If a variable name is not complied with NetCDF naming rules,
+ /// Names. If a variable name is not complied with NetCDF naming rules,
/// its simplified version is used at the NetCDF layer, but through the it is still visible
/// unmodified.
///
///
- /// support.
- /// Since the unmanaged NetCDF4 doesn't support a date/time type, it is internally stored
- /// as a double variable with special attribute
+ /// support.
+ /// Since the unmanaged NetCDF4 doesn't support a date/time type, it is internally stored
+ /// as a double variable with special attribute
/// Units = "100-nanosecond intervals that have elapsed since 12:00:00 midnight, January 1, 0001".
/// NetCDF variables with this attribute are loaded as variables.
- /// In the current release, any other units descriptions are not considered as a date/time and
+ /// In the current release, any other units descriptions are not considered as a date/time and
/// hence the variable's final type of data is .
///
///
@@ -201,8 +201,8 @@ public class NetCDFDataSet : DataSet, IChunkSizesAdjustable
/// DataSet URI (see remarks for ).
///
///
- /// If the file specified by exists, the NetCDF will be
- /// initialized with that file. Otherwise, the new file will be created and
+ /// If the file specified by exists, the NetCDF will be
+ /// initialized with that file. Otherwise, the new file will be created and
/// the resulting NetCDFDataSet will be empty.
///
///
@@ -227,15 +227,19 @@ public NetCDFDataSet(string uri)
Debug.WriteLineIf(TraceNetCDFDataSet.TraceInfo, "Deflate mode: " + deflate);
- ResourceOpenMode openMode = this.uri.GetOpenModeOrDefault(ResourceOpenMode.OpenOrCreate);
- InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
+ if (((NetCDFUri)this.uri).Url.Length > 0) {
+ InitializeFromUrl(((NetCDFUri)this.uri).Url);
+ } else {
+ ResourceOpenMode openMode = this.uri.GetOpenModeOrDefault(ResourceOpenMode.OpenOrCreate);
+ InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
+ }
}
///
/// Initializes new instance of the NetCDFDataSet class.
///
/// DataSet URI (see remarks for ).
- /// The open mode (see
+ /// The open mode (see
/// ).
///
///
@@ -257,7 +261,205 @@ public NetCDFDataSet(string uri, ResourceOpenMode openMode)
deflate = ((NetCDFUri)this.uri).Deflate;
trimTrailingZero = ((NetCDFUri)this.uri).TrimTrailingZero;
- InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
+ if (((NetCDFUri)this.uri).Url.Length > 0) {
+ InitializeFromUrl(((NetCDFUri)this.uri).Url);
+ } else {
+ InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
+ }
+ }
+
+ private void LoadMetadata(string fileName)
+ {
+ int res;
+ //********************************************************
+ // Loading schema
+ int ndims, nvars, ngatts, unlimdimid;
+ res = NetCDF.nc_inq(ncid, out ndims, out nvars, out ngatts, out unlimdimid);
+ HandleResult(res);
+
+ Debug.WriteLineIf(NetCDFDataSet.TraceNetCDFDataSet.TraceInfo, String.Format("NetCDF dataset {0} opened: {1} dimensions, {2} variables, {3} global attributes.",
+ fileName, ndims, nvars, ngatts));
+
+ /********************************************************
+ // Reading Global Attributes
+ int nattrs;
+ res = NetCDF.nc_inq_varnatts(ncid, NetCDF.NC_GLOBAL, out nattrs);
+ AttributeTypeMap atm = new AttributeTypeMap(ncid, NetCDF.NC_GLOBAL);
+ NetCDFDataSet.HandleResult(res);
+ MetadataDictionary globalMetadata = Metadata;
+ for (int i = 0; i < nattrs; i++)
+ {
+ // Name
+ StringBuilder attName = new StringBuilder(512);
+ res = NetCDF.nc_inq_attname(ncid, NetCDF.NC_GLOBAL, i, attName);
+ NetCDFDataSet.HandleResult(res);
+ string aname = attName.ToString();
+
+ // Skip out internal attribute
+ if (aname == AttributeTypeMap.AttributeName)
+ continue;
+
+ // Type
+ object value = ReadNetCdfAttribute(NetCDF.NC_GLOBAL, aname, atm);
+ globalMetadata[aname] = value;
+ } /**/
+
+ //********************************************************
+ // Loading variables
+ int[] varIds = new int[nvars];
+ res = NetCDF.nc_inq_varids(ncid, out nvars, varIds);
+ HandleResult(res);
+
+ for (int i = 0; i < nvars; i++)
+ {
+ Variable var = ReadNetCdfVariable(varIds[i]);
+ AddVariableToCollection(var);
+ }
+
+ Variable globalMetaVar = new NetCDFGlobalMetadataVariable(this);
+ AddVariableToCollection(globalMetaVar);
+
+ //********************************************************
+ // Loading coordinate systems
+ // CS stored in two places: as global attributes and as variable's attributes
+ // for backward compatibility. We're going to use the global attributes,
+ // as it works even when there is no any variable containing the coordinate system.
+ for (int i = 0; ; i++)
+ {
+ string attName = "coordinates" + (i + 1).ToString();
+
+ // Inquire information about cs attribute
+ NcType type;
+ IntPtr len;
+ res = NetCDF.nc_inq_att(ncid, NcConst.NC_GLOBAL, attName, out type, out len);
+ if (res == (int)ResultCode.NC_ENOTATT) // that's all: no more cs
+ break;
+ if (res != (int)ResultCode.NC_NOERR) // an error has occurred
+ HandleResult(res);
+
+ if (type != NcType.NC_CHAR)
+ throw new Exception("Coordinate system defining attribute is not a string");
+
+ // Getting the value of the attribute
+ string attValue = NcGetAttText(ncid, NcConst.NC_GLOBAL, attName, (int)len, out res);
+ HandleResult(res);
+
+ string[] items = attValue.Split(' ');
+ string csName = items[0]; // csname axis1 axis2 ...
+
+ // Creating the coordinate system instance
+ Variable[] axes = new Variable[items.Length - 1];
+ bool csFound = true;
+ for (int j = 1; j < items.Length; j++) // axes names
+ {
+ int aid;
+ Variable a;
+ if (int.TryParse(items[j], out aid))
+ {
+ csFound = Variables.TryGetByID(aid, out a);
+ if (!csFound) break;
+ a = Variables.GetByID(aid);
+ }
+ else
+ {
+ csFound = ContainsRecent(Variables, items[j]);
+ if (!csFound) break;
+ a = Variables[items[j], SchemaVersion.Recent];
+ }
+ axes[j - 1] = a;
+ }
+ if (!csFound) break;
+ CoordinateSystem cs = CreateCoordinateSystem(csName, axes);
+ }
+
+ // Loading coordinate systems for variable's metadata (backward compatibility)
+ foreach (Variable var in Variables)
+ {
+ if (var.Metadata.ContainsKey("coordinates", SchemaVersion.Recent) &&
+ !var.Metadata.ContainsKey("coordinatesName", SchemaVersion.Recent))
+ {
+ string[] axes = var.Metadata["coordinates", SchemaVersion.Recent].ToString().Split(' ');
+ List axesList = new List();
+ bool axisNotFound = false;
+ foreach (var axis in axes)
+ {
+ try
+ {
+ Variable a = Variables[axis, SchemaVersion.Recent];
+ axesList.Add(a);
+ }
+ catch
+ {
+ Trace.WriteLineIf(TraceNetCDFDataSet.TraceError, "Coordinate named " + axis + " is not found in nc file");
+ axisNotFound = true;
+ break;
+ }
+ }
+ if (!axisNotFound)
+ {
+ // This is a standart NC file without SDS coordinate systems metadata!
+ //string csName = "_cs_" + var.Name + "_" + (Guid.NewGuid()).ToString();
+ //CreateCoordinateSystem(csName, axesList.ToArray());
+ break;
+ }
+ }
+ }
+
+ // Adding coordinate systems to variables
+ foreach (Variable var in Variables)
+ {
+ // Each variable has an entry named coordinates#Name, # - integer index
+ var.Metadata.ForEach(
+ delegate (KeyValuePair entry)
+ {
+ if (!entry.Key.StartsWith("coordinates") || !entry.Key.EndsWith("Name"))
+ return;
+
+ try
+ {
+ CoordinateSystem cs = CoordinateSystems[(string)entry.Value, SchemaVersion.Recent];
+ var.AddCoordinateSystem(cs);
+ }
+ catch (Exception ex)
+ {
+ Trace.WriteLineIf(TraceNetCDFDataSet.TraceError, "Coordinate system with name " + entry.Value + " not found");
+ }
+ },
+ SchemaVersion.Proposed);
+ }
+ } // eo if exists
+
+ private void InitializeFromUrl(string url)
+ {
+ try
+ {
+ initializing = true;
+ int res;
+
+ if (NetCDFDataSet.TraceNetCDFDataSet.TraceInfo)
+ {
+ IntPtr cacheSize, nelems;
+ float preemption;
+ if (NetCDF.nc_get_chunk_cache(out cacheSize, out nelems, out preemption) == (int)ResultCode.NC_NOERR)
+ {
+ string s = String.Format("NetCDF chunk cache size: {0}, nelems: {1}, preemption: {2}", cacheSize, nelems, preemption);
+ Trace.WriteLineIf(NetCDFDataSet.TraceNetCDFDataSet.TraceInfo, s);
+ }
+ }
+
+ res = NetCDF.nc_open_chunked(url, CreateMode.NC_NOWRITE /*| NetCDF.CreateMode.NC_SHARE*/, out ncid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
+ HandleResult(res);
+
+ LoadMetadata(url);
+ Commit();
+
+ SetCompleteReadOnly();
+ initializing = false;
+ }
+ finally
+ {
+ IsAutocommitEnabled = false;
+ }
}
private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
@@ -270,6 +472,7 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
int res;
bool exists = File.Exists(fileName);
+
if (openMode == ResourceOpenMode.Create && exists)
{
File.Delete(fileName);
@@ -293,7 +496,7 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
if (openMode == ResourceOpenMode.CreateNew)
throw new IOException("The open mode is createNew but the file already exists");
- if (((File.GetAttributes(fileName) & FileAttributes.ReadOnly) != 0) && openMode != ResourceOpenMode.ReadOnly)
+ if ((File.GetAttributes(fileName) & FileAttributes.ReadOnly) != 0 && openMode != ResourceOpenMode.ReadOnly)
{
openMode = ResourceOpenMode.ReadOnly;
Trace.WriteLineIf(TraceNetCDFDataSet.TraceWarning, "NetCDFDataSet: Opening file in read only mode");
@@ -319,165 +522,7 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
HandleResult(res);
- if (exists)
- {
- //********************************************************
- // Loading schema
- int ndims, nvars, ngatts, unlimdimid;
- res = NetCDF.nc_inq(ncid, out ndims, out nvars, out ngatts, out unlimdimid);
- HandleResult(res);
-
- Debug.WriteLineIf(NetCDFDataSet.TraceNetCDFDataSet.TraceInfo, String.Format("NetCDF dataset {0} opened: {1} dimensions, {2} variables, {3} global attributes.",
- fileName, ndims, nvars, ngatts));
-
- /********************************************************
- // Reading Global Attributes
- int nattrs;
- res = NetCDF.nc_inq_varnatts(ncid, NetCDF.NC_GLOBAL, out nattrs);
- AttributeTypeMap atm = new AttributeTypeMap(ncid, NetCDF.NC_GLOBAL);
- NetCDFDataSet.HandleResult(res);
- MetadataDictionary globalMetadata = Metadata;
- for (int i = 0; i < nattrs; i++)
- {
- // Name
- StringBuilder attName = new StringBuilder(512);
- res = NetCDF.nc_inq_attname(ncid, NetCDF.NC_GLOBAL, i, attName);
- NetCDFDataSet.HandleResult(res);
- string aname = attName.ToString();
-
- // Skip out internal attribute
- if (aname == AttributeTypeMap.AttributeName)
- continue;
-
- // Type
- object value = ReadNetCdfAttribute(NetCDF.NC_GLOBAL, aname, atm);
- globalMetadata[aname] = value;
- } /**/
-
- //********************************************************
- // Loading variables
- int[] varIds = new int[nvars];
- res = NetCDF.nc_inq_varids(ncid, out nvars, varIds);
- HandleResult(res);
-
- for (int i = 0; i < nvars; i++)
- {
- Variable var = ReadNetCdfVariable(varIds[i]);
- AddVariableToCollection(var);
- }
-
- Variable globalMetaVar = new NetCDFGlobalMetadataVariable(this);
- AddVariableToCollection(globalMetaVar);
-
- //********************************************************
- // Loading coordinate systems
- // CS stored in two places: as global attributes and as variable's attributes
- // for backward compatibility. We're going to use the global attributes,
- // as it works even when there is no any variable containing the coordinate system.
- for (int i = 0; ; i++)
- {
- string attName = "coordinates" + (i + 1).ToString();
-
- // Inquire information about cs attribute
- NcType type;
- IntPtr len;
- res = NetCDF.nc_inq_att(ncid, NcConst.NC_GLOBAL, attName, out type, out len);
- if (res == (int)ResultCode.NC_ENOTATT) // that's all: no more cs
- break;
- if (res != (int)ResultCode.NC_NOERR) // an error has occurred
- HandleResult(res);
-
- if (type != NcType.NC_CHAR)
- throw new Exception("Coordinate system defining attribute is not a string");
-
- // Getting the value of the attribute
- string attValue = NcGetAttText(ncid, NcConst.NC_GLOBAL, attName, (int)len, out res);
- HandleResult(res);
-
- string[] items = attValue.Split(' ');
- string csName = items[0]; // csname axis1 axis2 ...
-
- // Creating the coordinate system instance
- Variable[] axes = new Variable[items.Length - 1];
- bool csFound = true;
- for (int j = 1; j < items.Length; j++) // axes names
- {
- int aid;
- Variable a;
- if (int.TryParse(items[j], out aid))
- {
- csFound = Variables.TryGetByID(aid, out a);
- if (!csFound) break;
- a = Variables.GetByID(aid);
- }
- else
- {
- csFound = ContainsRecent(Variables, items[j]);
- if (!csFound) break;
- a = Variables[items[j], SchemaVersion.Recent];
- }
- axes[j - 1] = a;
- }
- if (!csFound) break;
- CoordinateSystem cs = CreateCoordinateSystem(csName, axes);
- }
-
- // Loading coordinate systems for variable's metadata (backward compatibility)
- foreach (Variable var in Variables)
- {
- if (var.Metadata.ContainsKey("coordinates", SchemaVersion.Recent) &&
- !var.Metadata.ContainsKey("coordinatesName", SchemaVersion.Recent))
- {
- string[] axes = var.Metadata["coordinates", SchemaVersion.Recent].ToString().Split(' ');
- List axesList = new List();
- bool axisNotFound = false;
- foreach (var axis in axes)
- {
- try
- {
- Variable a = Variables[axis, SchemaVersion.Recent];
- axesList.Add(a);
- }
- catch
- {
- Trace.WriteLineIf(TraceNetCDFDataSet.TraceError, "Coordinate named " + axis + " is not found in nc file");
- axisNotFound = true;
- break;
- }
- }
- if (!axisNotFound)
- {
- // This is a standart NC file without SDS coordinate systems metadata!
- //string csName = "_cs_" + var.Name + "_" + (Guid.NewGuid()).ToString();
- //CreateCoordinateSystem(csName, axesList.ToArray());
- break;
- }
- }
- }
-
- // Adding coordinate systems to variables
- foreach (Variable var in Variables)
- {
- // Each variable has an entry named coordinates#Name, # - integer index
- var.Metadata.ForEach(
- delegate (KeyValuePair entry)
- {
- if (!entry.Key.StartsWith("coordinates") || !entry.Key.EndsWith("Name"))
- return;
-
- try
- {
- CoordinateSystem cs = CoordinateSystems[(string)entry.Value, SchemaVersion.Recent];
- var.AddCoordinateSystem(cs);
- }
- catch (Exception ex)
- {
- Trace.WriteLineIf(TraceNetCDFDataSet.TraceError, "Coordinate system with name " + entry.Value + " not found");
- }
- },
- SchemaVersion.Proposed);
- }
- } // eo if exists
+ if (exists) LoadMetadata(fileName);
Commit();
@@ -505,7 +550,7 @@ private static bool ContainsRecent(ReadOnlyVariableCollection vars, string var)
}
///
- ///
+ ///
///
///
///
@@ -522,7 +567,7 @@ internal static NetCDFDataSet Open(string fileName, bool enableRollback)
}
///
- ///
+ ///
///
///
///
@@ -567,7 +612,7 @@ internal bool RollbackEnabled
#region Overriden Methods
///
- ///
+ ///
///
///
///
@@ -581,7 +626,7 @@ protected override Variable CreateVariable(string varName, s
}
///
- ///
+ ///
///
protected override void OnTransactionOpened()
{
@@ -614,7 +659,7 @@ private void UpdateDimIds()
}
///
- ///
+ ///
///
///
///
@@ -626,7 +671,7 @@ protected override bool OnPrecommitting(DataSet.Changes changes)
return true;
}
///
- ///
+ ///
///
///
protected override void OnPrecommit(DataSet.Changes changes)
@@ -657,7 +702,7 @@ protected override void OnPrecommit(DataSet.Changes changes)
}
}
///
- ///
+ ///
///
protected override void OnCommit()
{
@@ -673,7 +718,7 @@ protected override void OnCommit()
tempFileName = null;
}
///
- ///
+ ///
///
protected override void OnRollback()
{
@@ -719,7 +764,7 @@ protected override void OnRollback()
#region Utils
///
- ///
+ ///
///
protected override void Dispose(bool disposing)
{