Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NetCDF4 group usage is added #5

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

NetCDF4 group usage is added #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0fc9600
NetCDF4 group usage is added
Jan 28, 2019
bb25ddd
Unit tests for group usage checking are added
Jan 28, 2019
146aa01
Closing file in case of error while group reading is added. Unit test…
Jan 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions NetCDFInterop/NetCDF.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,8 @@ public static int nc_create_chunked(string path, CreateMode mode, out int ncidp,
public static int nc_enddef(int ncid) { lock (namebuf) { return NetCDFDynamic.f_nc_enddef.Invoke(ncid); } }
public static int nc_redef(int ncid) { lock (namebuf) { return NetCDFDynamic.f_nc_redef.Invoke(ncid); } }
public static int nc_inq(int ncid, out int ndims, out int nvars, out int ngatts, out int unlimdimid) { lock (namebuf) { return NetCDFDynamic.f_nc_inq.Invoke(ncid, out ndims, out nvars, out ngatts, out unlimdimid); } }
public static int nc_def_grp(int ncid, string name, out int groupId) { return NetCDFDynamic.f_nc_def_grp.Invoke(ncid, name, out groupId); }
public static int nc_inq_grp_ncid(int ncid, string name, out int groupId) { return NetCDFDynamic.f_nc_inq_grp_ncid.Invoke(ncid, name, out groupId); }
public static int nc_def_var(int ncid, string name, NcType xtype, int[] dimids, out int varidp) { lock (namebuf) { return NetCDFDynamic.f_nc_def_var.Invoke(ncid, name, xtype, dimids.Length, dimids, out varidp); } }
public static int nc_def_dim(int ncid, string name, IntPtr len, out int dimidp) { lock (namebuf) { return NetCDFDynamic.f_nc_def_dim.Invoke(ncid, name, len, out dimidp); } }
public static int nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_level) { lock (namebuf) { return NetCDFDynamic.f_nc_def_var_deflate.Invoke(ncid, varid, shuffle, deflate, deflate_level); } }
Expand Down Expand Up @@ -587,6 +589,8 @@ static NetCDFDynamic()
f_nc_def_dim = native.GetFunction<nc_def_dim>();
f_nc_def_var_deflate = native.GetFunction<nc_def_var_deflate>();
f_nc_def_var_chunking = native.GetFunction<nc_def_var_chunking>();
f_nc_def_grp = native.GetFunction<nc_def_grp>();
f_nc_inq_grp_ncid = native.GetFunction<nc_inq_grp_ncid>();
f_nc_inq_var = native.GetFunction<nc_inq_var>();
f_nc_inq_varids = native.GetFunction<nc_inq_varids>();
f_nc_inq_vartype = native.GetFunction<nc_inq_vartype>();
Expand Down Expand Up @@ -691,6 +695,8 @@ static NetCDFDynamic()
public static nc_def_dim f_nc_def_dim { get; private set; }
public static nc_def_var_deflate f_nc_def_var_deflate { get; private set; }
public static nc_def_var_chunking f_nc_def_var_chunking { get; private set; }
public static nc_def_grp f_nc_def_grp { get; private set; }
public static nc_inq_grp_ncid f_nc_inq_grp_ncid { get; private set; }
public static nc_inq_var f_nc_inq_var { get; private set; }
public static nc_inq_varids f_nc_inq_varids { get; private set; }
public static nc_inq_vartype f_nc_inq_vartype { get; private set; }
Expand Down Expand Up @@ -794,6 +800,8 @@ static NetCDFDynamic()
public delegate int nc_def_dim(int ncid, string name, IntPtr len, out int dimidp);
public delegate int nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_level);
public delegate int nc_def_var_chunking(int ncid, int varid, int contiguous, IntPtr[] chunksizes);
public delegate int nc_def_grp(int ncid, string name, out int new_ncid);
public delegate int nc_inq_grp_ncid(int ncid, string name, out int groupId);
public delegate int nc_inq_var(int ncid, int varid, StringBuilder name, out NcType type, out int ndims, int[] dimids, out int natts);
public delegate int nc_inq_varids(int ncid, out int nvars, int[] varids);
public delegate int nc_inq_vartype(int ncid, int varid, out NcType xtypep);
Expand Down
161 changes: 161 additions & 0 deletions SDSLiteTests/NetCDFTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,5 +169,166 @@ public void Empty_AddAttribute_UtfString()
Empty_AddAttribute("Русский");
}

private void CheckGroupCreating(string groupName, string parentGroupName = null)
{
Tuple<Type, string, Array>[] variableInfo =
{
new Tuple<Type, string, Array>(typeof(float), "testVar1", new float[] { 42.0f, 73.0f, 1.0f, 0.0f }),
new Tuple<Type, string, Array>(typeof(int), "testVar2", new int[] { 4, 8, 15, 16 }),
new Tuple<Type, string, Array>(typeof(string), "testVar3", new string[] { "str", "rts", "23", "---"})
};

string dimensionName = "testDim1";
string tmpFileName = System.IO.Path.GetTempFileName() + ".nc";

// create a test dataset group and create variables and dimensions, that this group must contain
using (var ds = DataSet.Open(tmpFileName + "?openMode=create&groupName=" + groupName))
{
foreach(var info in variableInfo)
{
ds.AddVariable(info.Item1, info.Item2, info.Item3, dimensionName);
}

ds.Commit();
}

// check that group contains variables and dimensions
using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=" + groupName))
{
foreach(var info in variableInfo)
{
Assert.True(ds.Variables.Contains(info.Item2));
var vals = ds.Variables[info.Item2].GetData();
for(int i = 0; i < vals.Length; i++)
{
Assert.Equals(info.Item3.GetValue(i), vals.GetValue(i));
}
}

Assert.True(ds.Dimensions.Contains(dimensionName));
}

// if the given group is not the root group
if (!string.IsNullOrEmpty(groupName) && !groupName.Equals("/"))
{
// check that the root group doesn't contain variables and dimensions from the non-root group
using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly"))
{
foreach (var info in variableInfo)
{
Assert.False(ds.Variables.Contains(info.Item2));
}

Assert.False(ds.Dimensions.Contains(dimensionName));
}
}

// if the parent group is given
if(!string.IsNullOrEmpty(parentGroupName))
{
// check that parent group doesn't contain variables and dimensions from the child group
using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=" + parentGroupName))
{
foreach(var info in variableInfo)
{
Assert.False(ds.Variables.Contains(info.Item2));
}

Assert.False(ds.Dimensions.Contains(dimensionName));
}
}

System.IO.File.Delete(tmpFileName);
}

[Test]
public void RootGroupTest1()
{
CheckGroupCreating("");
}
[Test]
public void RootGroupTest2()
{
CheckGroupCreating("/");
}
[Test]
public void SimpleGroupTest()
{
CheckGroupCreating("testGroup");
}
[Test]
public void CheckParentGroupTest()
{
CheckGroupCreating("testGroup", "/");
}
[Test]
public void NestedGroupTest1()
{
CheckGroupCreating("parent/child", "parent");
}
[Test]
public void NestedGroupTest2()
{
CheckGroupCreating("parent/child/subchild", "parent/child");
}
[Test]
public void GroupNotFoundTest()
{
string tmpFileName = System.IO.Path.GetTempFileName() + ".nc";
string rightTestGroup = "testGrp";
string wrongTestGroup = "testGrp/childGrp";

using (var ds = DataSet.Open(tmpFileName + "?openMode=create&groupName=" + rightTestGroup))
{
}

try
{
Assert.Throws<DataSetCreateException>(() =>
{
using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=" + wrongTestGroup))
{
}
});
}
finally
{
System.IO.File.Delete(tmpFileName);
}
}
[Test]
public void DifferentWaysToDefineGroupTest()
{
string tmpFileName = System.IO.Path.GetTempFileName() + ".nc";
string groupName = "group/subgroup";
string testVarName = "var";
string dimensionName = "dimension";

using (var ds = DataSet.Open(tmpFileName + "?openMode=create&groupName=" + groupName))
{
ds.AddVariable(typeof(int), testVarName, new int[] { 42 }, dimensionName);
ds.Commit();
}

using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=/" + groupName))
{
Assert.True(ds.Variables.Contains(testVarName));
Assert.True(ds.Dimensions.Contains(dimensionName));
}

using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=" + groupName + "/"))
{
Assert.True(ds.Variables.Contains(testVarName));
Assert.True(ds.Dimensions.Contains(dimensionName));
}

using (var ds = DataSet.Open(tmpFileName + "?openMode=readOnly&groupName=/" + groupName + "/"))
{
Assert.True(ds.Variables.Contains(testVarName));
Assert.True(ds.Dimensions.Contains(dimensionName));
}

System.IO.File.Delete(tmpFileName);
}
}
}
6 changes: 6 additions & 0 deletions ScientificDataSet/Core/Factory/Attributes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,12 @@ public DataSetProviderUriTypeAttribute(Type uriType)
[AttributeUsage(AttributeTargets.Property, AllowMultiple=false)]
public sealed class FileNamePropertyAttribute : Attribute { }

/// <summary>
/// Indicates that the target property contains group name.
/// </summary>
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class GroupNamePropertyAttribute : Attribute { }

/// <summary>
/// Indicates that the target property contains directory name.
/// </summary>
Expand Down
68 changes: 56 additions & 12 deletions ScientificDataSet/Providers/NetCDF/NetCDFDataSet.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ namespace Microsoft.Research.Science.Data.NetCDF4
/// See <see cref="DataSet.Clone(String)"/>.
/// </item>
/// <item>
/// <term>groupName=name_of_group</term>
/// <description>
/// Specifies a group for reading/writing. The name can include subgroups.
/// </description>
/// </item>
/// <item>
/// <term>deflate=off|store|fastest|fast|normal|good|best</term>
/// <description>
/// Defines the data compression level. Affects only new variables,
Expand Down Expand Up @@ -175,7 +181,8 @@ public class NetCDFDataSet : DataSet, IChunkSizesAdjustable

internal new const int GlobalMetadataVariableID = DataSet.GlobalMetadataVariableID;

private int ncid = -1;
private int ncid = -1; // ncid of an open group
private int fileid = -1; // ncid of an open file
private bool initializing = true; // means that we're inializing schema from the file
private bool rollbackEnabled = false;
private bool trimTrailingZero;
Expand Down Expand Up @@ -229,7 +236,7 @@ public NetCDFDataSet(string uri)
Debug.WriteLineIf(TraceNetCDFDataSet.TraceInfo, "Deflate mode: " + deflate);

ResourceOpenMode openMode = this.uri.GetOpenModeOrDefault(ResourceOpenMode.OpenOrCreate);
InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
InitializeFromFile(((NetCDFUri)this.uri).FileName, ((NetCDFUri)this.uri).GroupName, openMode);
}

/// <summary>
Expand Down Expand Up @@ -258,10 +265,10 @@ public NetCDFDataSet(string uri, ResourceOpenMode openMode)
deflate = ((NetCDFUri)this.uri).Deflate;
trimTrailingZero = ((NetCDFUri)this.uri).TrimTrailingZero;

InitializeFromFile(((NetCDFUri)this.uri).FileName, openMode);
InitializeFromFile(((NetCDFUri)this.uri).FileName, ((NetCDFUri)this.uri).GroupName, openMode);
}

private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
private void InitializeFromFile(string fileName, string groupPath, ResourceOpenMode openMode)
{
bool autoCommit = IsAutocommitEnabled;
IsAutocommitEnabled = false;
Expand Down Expand Up @@ -301,9 +308,9 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
}

if (openMode == ResourceOpenMode.ReadOnly)
res = NetCDF.nc_open_chunked(fileName, CreateMode.NC_NOWRITE /*| NetCDF.CreateMode.NC_SHARE*/, out ncid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
res = NetCDF.nc_open_chunked(fileName, CreateMode.NC_NOWRITE /*| NetCDF.CreateMode.NC_SHARE*/, out fileid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
else
res = NetCDF.nc_open_chunked(fileName, CreateMode.NC_WRITE /*| NetCDF.CreateMode.NC_SHARE*/, out ncid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
res = NetCDF.nc_open_chunked(fileName, CreateMode.NC_WRITE /*| NetCDF.CreateMode.NC_SHARE*/, out fileid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
}
else
{
Expand All @@ -312,14 +319,46 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)

if (!Path.IsPathRooted(fileName))
fileName = Path.Combine(Environment.CurrentDirectory, fileName);
res = NetCDF.nc_create_chunked(fileName, CreateMode.NC_NETCDF4 | CreateMode.NC_CLOBBER /*| NetCDF.CreateMode.NC_SHARE*/, out ncid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);

Variable globalMetaVar = new NetCDFGlobalMetadataVariable(this);
AddVariableToCollection(globalMetaVar);
res = NetCDF.nc_create_chunked(fileName, CreateMode.NC_NETCDF4 | CreateMode.NC_CLOBBER /*| NetCDF.CreateMode.NC_SHARE*/, out fileid, new IntPtr(defaultCacheSize), new IntPtr(defaultCacheNElems), defaultCachePreemption);
}

HandleResult(res);

// Opening or creating a group
ncid = fileid;
if (!string.IsNullOrEmpty(groupPath) && !groupPath.Trim().Equals("/"))
{
var pathParts = groupPath.Trim('/').Split('/');
int partNumber = 0;
int hlpNcid;

while (partNumber < pathParts.Length && NetCDF.nc_inq_grp_ncid(ncid, pathParts[partNumber], out hlpNcid) == (int)ResultCode.NC_NOERR)
{
ncid = hlpNcid;
partNumber++;
}

if (partNumber < pathParts.Length)
{
if (openMode == ResourceOpenMode.ReadOnly)
{
NetCDF.nc_close(fileid);
throw new ArgumentException($"Group \"{groupPath}\" is not found!");
}
else
{
exists = false;
}

for (; partNumber < pathParts.Length; partNumber++)
{
res = NetCDF.nc_def_grp(ncid, pathParts[partNumber], out hlpNcid);
HandleResult(res);
ncid = hlpNcid;
}
}
}

if (exists)
{
//********************************************************
Expand Down Expand Up @@ -479,6 +518,11 @@ private void InitializeFromFile(string fileName, ResourceOpenMode openMode)
SchemaVersion.Proposed);
}
} // eo if exists
else
{
Variable globalMetaVar = new NetCDFGlobalMetadataVariable(this);
AddVariableToCollection(globalMetaVar);
}

Commit();

Expand Down Expand Up @@ -724,11 +768,11 @@ protected override void OnRollback()
/// </summary>
protected override void Dispose(bool disposing)
{
if (ncid >= 0)
if (fileid >= 0)
{
try
{
int res = NetCDF.nc_close(ncid);
int res = NetCDF.nc_close(fileid);
if (disposing)
HandleResult(res);
if (tempFileName != null)
Expand Down
18 changes: 18 additions & 0 deletions ScientificDataSet/Providers/NetCDF/NetCDFUri.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,24 @@ public string FileName
set { this["file"] = value; }
}

/// <summary>
/// Specifies the group to use.
/// </summary>
[GroupNameProperty]
[Description("Specifies the group to use.")]
public string GroupName
{
get
{
if(GetParameterOccurences("groupName") == 0)
{
return "";
}

return this["groupName"];
}
}

/// <summary>
/// Indicates whether the rollback is enabled or not.
/// </summary>
Expand Down