From 54721d8e1b00ea30510c26ed152cb71adb1fbf1e Mon Sep 17 00:00:00 2001 From: mukunku Date: Tue, 8 Dec 2020 13:25:30 -0500 Subject: [PATCH] fix: fill weight overflow when there are too many columns (#22) --- src/ParquetFileViewer/Constants.cs | 13 + .../{ => Controls}/DelayedOnChangedTextBox.cs | 144 +++---- .../CustomScriptBasedSchemaAdapter.cs | 6 +- .../Helpers/ExtensionMethods.cs | 24 ++ .../{ => Helpers}/UtilityMethods.cs | 353 +++++++++--------- src/ParquetFileViewer/MainForm.Designer.cs | 114 +++--- src/ParquetFileViewer/MainForm.cs | 20 +- .../ParquetFileViewer.csproj | 8 +- 8 files changed, 358 insertions(+), 324 deletions(-) create mode 100644 src/ParquetFileViewer/Constants.cs rename src/ParquetFileViewer/{ => Controls}/DelayedOnChangedTextBox.cs (95%) rename src/ParquetFileViewer/{ => Helpers}/CustomScriptBasedSchemaAdapter.cs (97%) create mode 100644 src/ParquetFileViewer/Helpers/ExtensionMethods.cs rename src/ParquetFileViewer/{ => Helpers}/UtilityMethods.cs (87%) diff --git a/src/ParquetFileViewer/Constants.cs b/src/ParquetFileViewer/Constants.cs new file mode 100644 index 0000000..9053d30 --- /dev/null +++ b/src/ParquetFileViewer/Constants.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ParquetFileViewer +{ + public static class Constants + { + public const string FILL_WEIGHT_EXCEPTION_MESSAGE = "FillWeight"; + } +} diff --git a/src/ParquetFileViewer/DelayedOnChangedTextBox.cs b/src/ParquetFileViewer/Controls/DelayedOnChangedTextBox.cs similarity index 95% rename from src/ParquetFileViewer/DelayedOnChangedTextBox.cs rename to src/ParquetFileViewer/Controls/DelayedOnChangedTextBox.cs index d4b0ddc..814a1b8 100644 --- a/src/ParquetFileViewer/DelayedOnChangedTextBox.cs +++ b/src/ParquetFileViewer/Controls/DelayedOnChangedTextBox.cs @@ -1,72 +1,72 @@ -using System; -using System.Windows.Forms; - -namespace ParquetFileViewer -{ - public class DelayedOnChangedTextBox : TextBox - { - private Timer m_delayedTextChangedTimer; - - public event EventHandler DelayedTextChanged; - - public DelayedOnChangedTextBox() - : base() - { - this.DelayedTextChangedTimeout = 1 * 1000; - } - - public DelayedOnChangedTextBox(int secondsDelay) - : base() - { - this.DelayedTextChangedTimeout = secondsDelay * 1000; - } - - protected override void Dispose(bool disposing) - { - if (m_delayedTextChangedTimer != null) - { - m_delayedTextChangedTimer.Stop(); - if (disposing) - m_delayedTextChangedTimer.Dispose(); - } - - base.Dispose(disposing); - } - - public int DelayedTextChangedTimeout { get; set; } - - protected virtual void OnDelayedTextChanged(EventArgs e) - { - this.DelayedTextChanged?.Invoke(this, e); - } - - protected override void OnTextChanged(EventArgs e) - { - this.InitializeDelayedTextChangedEvent(); - base.OnTextChanged(e); - } - - private void InitializeDelayedTextChangedEvent() - { - if (m_delayedTextChangedTimer != null) - m_delayedTextChangedTimer.Stop(); - - if (m_delayedTextChangedTimer == null || m_delayedTextChangedTimer.Interval != this.DelayedTextChangedTimeout) - { - m_delayedTextChangedTimer = new Timer(); - m_delayedTextChangedTimer.Tick += new EventHandler(HandleDelayedTextChangedTimerTick); - m_delayedTextChangedTimer.Interval = this.DelayedTextChangedTimeout; - } - - m_delayedTextChangedTimer.Start(); - } - - private void HandleDelayedTextChangedTimerTick(object sender, EventArgs e) - { - Timer timer = sender as Timer; - timer.Stop(); - - this.OnDelayedTextChanged(EventArgs.Empty); - } - } -} +using System; +using System.Windows.Forms; + +namespace ParquetFileViewer.Controls +{ + public class DelayedOnChangedTextBox : TextBox + { + private Timer m_delayedTextChangedTimer; + + public event EventHandler DelayedTextChanged; + + public DelayedOnChangedTextBox() + : base() + { + this.DelayedTextChangedTimeout = 1 * 1000; + } + + public DelayedOnChangedTextBox(int secondsDelay) + : base() + { + this.DelayedTextChangedTimeout = secondsDelay * 1000; + } + + protected override void Dispose(bool disposing) + { + if (m_delayedTextChangedTimer != null) + { + m_delayedTextChangedTimer.Stop(); + if (disposing) + m_delayedTextChangedTimer.Dispose(); + } + + base.Dispose(disposing); + } + + public int DelayedTextChangedTimeout { get; set; } + + protected virtual void OnDelayedTextChanged(EventArgs e) + { + this.DelayedTextChanged?.Invoke(this, e); + } + + protected override void OnTextChanged(EventArgs e) + { + this.InitializeDelayedTextChangedEvent(); + base.OnTextChanged(e); + } + + private void InitializeDelayedTextChangedEvent() + { + if (m_delayedTextChangedTimer != null) + m_delayedTextChangedTimer.Stop(); + + if (m_delayedTextChangedTimer == null || m_delayedTextChangedTimer.Interval != this.DelayedTextChangedTimeout) + { + m_delayedTextChangedTimer = new Timer(); + m_delayedTextChangedTimer.Tick += new EventHandler(HandleDelayedTextChangedTimerTick); + m_delayedTextChangedTimer.Interval = this.DelayedTextChangedTimeout; + } + + m_delayedTextChangedTimer.Start(); + } + + private void HandleDelayedTextChangedTimerTick(object sender, EventArgs e) + { + Timer timer = sender as Timer; + timer.Stop(); + + this.OnDelayedTextChanged(EventArgs.Empty); + } + } +} diff --git a/src/ParquetFileViewer/CustomScriptBasedSchemaAdapter.cs b/src/ParquetFileViewer/Helpers/CustomScriptBasedSchemaAdapter.cs similarity index 97% rename from src/ParquetFileViewer/CustomScriptBasedSchemaAdapter.cs rename to src/ParquetFileViewer/Helpers/CustomScriptBasedSchemaAdapter.cs index 4926c9c..e62cef4 100644 --- a/src/ParquetFileViewer/CustomScriptBasedSchemaAdapter.cs +++ b/src/ParquetFileViewer/Helpers/CustomScriptBasedSchemaAdapter.cs @@ -6,7 +6,7 @@ using System.Text; using System.Threading.Tasks; -namespace ParquetFileViewer +namespace ParquetFileViewer.Helpers { public class CustomScriptBasedSchemaAdapter { @@ -135,11 +135,11 @@ private string MakeList(DataColumnCollection columns) { if (!flag) { - stringBuilder.Append(", "); + stringBuilder.Append(" , "); } string str = this.MakeSafe(column.ColumnName); string typeFor = this.GetTypeFor(column); - stringBuilder.AppendFormat("{0} {1}", str, typeFor); + stringBuilder.Append($"{Environment.NewLine} {str} {typeFor}"); flag = false; } return stringBuilder.ToString(); diff --git a/src/ParquetFileViewer/Helpers/ExtensionMethods.cs b/src/ParquetFileViewer/Helpers/ExtensionMethods.cs new file mode 100644 index 0000000..2a5a2b3 --- /dev/null +++ b/src/ParquetFileViewer/Helpers/ExtensionMethods.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using System.Data; +using System.Windows.Forms; + +namespace ParquetFileViewer.Helpers +{ + public static class ExtensionMethods + { + /// + /// Returns a list of all column names within a given datatable + /// + /// The datatable to retrieve the column names from + /// + public static IList GetColumnNames(this DataTable datatable) + { + List columns = new List(datatable.Columns.Count); + foreach (DataColumn column in datatable.Columns) + { + columns.Add(column.ColumnName); + } + return columns; + } + } +} diff --git a/src/ParquetFileViewer/UtilityMethods.cs b/src/ParquetFileViewer/Helpers/UtilityMethods.cs similarity index 87% rename from src/ParquetFileViewer/UtilityMethods.cs rename to src/ParquetFileViewer/Helpers/UtilityMethods.cs index ca46748..ad268e3 100644 --- a/src/ParquetFileViewer/UtilityMethods.cs +++ b/src/ParquetFileViewer/Helpers/UtilityMethods.cs @@ -1,185 +1,168 @@ -using Parquet; -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; - -namespace ParquetFileViewer -{ - public static class UtilityMethods - { - public static DataTable ParquetReaderToDataTable(ParquetReader parquetReader, out int totalRecordCount, List selectedFields, int offset, int recordCount) - { - //Get list of data fields and construct the DataTable - DataTable dataTable = new DataTable(); - List fields = new List(); - var dataFields = parquetReader.Schema.GetDataFields(); - foreach (string selectedField in selectedFields) - { - var dataField = dataFields.FirstOrDefault(f => f.Name.Equals(selectedField, StringComparison.InvariantCultureIgnoreCase)); - if (dataField != null) - { - fields.Add(dataField); - DataColumn newColumn = new DataColumn(dataField.Name, ParquetNetTypeToCSharpType(dataField.DataType)) - { - // Should not set this, or line 89 in ProcessRowGroup() will throw an exception with any required field (because assigning later than adding) - //AllowDBNull = dataField.HasNulls - }; - dataTable.Columns.Add(newColumn); - } - else - throw new Exception(string.Format("Field '{0}' does not exist", selectedField)); - } - - //Read column by column to generate each row in the datatable - totalRecordCount = 0; - for (int i = 0; i < parquetReader.RowGroupCount; i++) - { - int rowsLeftToRead = recordCount; - using (ParquetRowGroupReader groupReader = parquetReader.OpenRowGroupReader(i)) - { - if (groupReader.RowCount > int.MaxValue) - throw new ArgumentOutOfRangeException(string.Format("Cannot handle row group sizes greater than {0}", groupReader.RowCount)); - - int rowsPassedUntilThisRowGroup = totalRecordCount; - totalRecordCount += (int)groupReader.RowCount; - - if (offset >= totalRecordCount) - continue; - - if (rowsLeftToRead > 0) - { - int numberOfRecordsToReadFromThisRowGroup = Math.Min(Math.Min(totalRecordCount - offset, recordCount), (int)groupReader.RowCount); - rowsLeftToRead -= numberOfRecordsToReadFromThisRowGroup; - - int recordsToSkipInThisRowGroup = Math.Max(offset - rowsPassedUntilThisRowGroup, 0); - - ProcessRowGroup(dataTable, groupReader, fields, recordsToSkipInThisRowGroup, numberOfRecordsToReadFromThisRowGroup); - } - } - } - - return dataTable; - } - - private static void ProcessRowGroup(DataTable dataTable, ParquetRowGroupReader groupReader, List fields, int skipRecords, int readRecords) - { - int rowBeginIndex = dataTable.Rows.Count; - bool isFirstColumn = true; - - foreach (var field in fields) - { - int rowIndex = rowBeginIndex; - - int skippedRecords = 0; - foreach (var value in groupReader.ReadColumn(field).Data) - { - if (skipRecords > skippedRecords) - { - skippedRecords++; - continue; - } - - if (rowIndex >= readRecords) - break; - - if (isFirstColumn) - { - var newRow = dataTable.NewRow(); - dataTable.Rows.Add(newRow); - } - - if (value == null) - dataTable.Rows[rowIndex][field.Name] = DBNull.Value; - else if (field.DataType == Parquet.Data.DataType.DateTimeOffset) - dataTable.Rows[rowIndex][field.Name] = ((DateTimeOffset)value).DateTime; //converts to local time! - else - dataTable.Rows[rowIndex][field.Name] = value; - - rowIndex++; - } - - isFirstColumn = false; - } - } - - - public static Type ParquetNetTypeToCSharpType(Parquet.Data.DataType type) - { - Type columnType = null; - switch (type) - { - case Parquet.Data.DataType.Boolean: - columnType = typeof(bool); - break; - case Parquet.Data.DataType.Byte: - columnType = typeof(sbyte); - break; - case Parquet.Data.DataType.ByteArray: - columnType = typeof(sbyte[]); - break; - case Parquet.Data.DataType.DateTimeOffset: - //Let's treat DateTimeOffsets as DateTime - columnType = typeof(DateTime); - break; - case Parquet.Data.DataType.Decimal: - columnType = typeof(decimal); - break; - case Parquet.Data.DataType.Double: - columnType = typeof(double); - break; - case Parquet.Data.DataType.Float: - columnType = typeof(float); - break; - case Parquet.Data.DataType.Short: - case Parquet.Data.DataType.Int16: - case Parquet.Data.DataType.Int32: - case Parquet.Data.DataType.UnsignedInt16: - columnType = typeof(int); - break; - case Parquet.Data.DataType.Int64: - columnType = typeof(long); - break; - case Parquet.Data.DataType.UnsignedByte: - columnType = typeof(byte); - break; - case Parquet.Data.DataType.String: - default: - columnType = typeof(string); - break; - } - - return columnType; - } - - public static List GetDataTableColumns(DataTable datatable) - { - List columns = new List(datatable != null ? datatable.Columns.Count : 0); - if (datatable != null) - { - foreach (DataColumn column in datatable.Columns) - { - columns.Add(column.ColumnName); - } - } - return columns; - } - - public static string CleanCSVValue(string value, bool alwaysEncloseInQuotes = false) - { - if (!string.IsNullOrWhiteSpace(value)) - { - //In RFC 4180 we escape quotes with double quotes - string formattedValue = value.Replace("\"", "\"\""); - - //Enclose value with quotes if it contains commas,line feeds or other quotes - if (formattedValue.Contains(",") || formattedValue.Contains("\r") || formattedValue.Contains("\n") || formattedValue.Contains("\"\"") || alwaysEncloseInQuotes) - formattedValue = string.Concat("\"", formattedValue, "\""); - - return formattedValue; - } - else - return string.Empty; - } - } -} +using Parquet; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; + +namespace ParquetFileViewer.Helpers +{ + public static class UtilityMethods + { + public static DataTable ParquetReaderToDataTable(ParquetReader parquetReader, out int totalRecordCount, List selectedFields, int offset, int recordCount) + { + //Get list of data fields and construct the DataTable + DataTable dataTable = new DataTable(); + List fields = new List(); + var dataFields = parquetReader.Schema.GetDataFields(); + foreach (string selectedField in selectedFields) + { + var dataField = dataFields.FirstOrDefault(f => f.Name.Equals(selectedField, StringComparison.InvariantCultureIgnoreCase)); + if (dataField != null) + { + fields.Add(dataField); + DataColumn newColumn = new DataColumn(dataField.Name, ParquetNetTypeToCSharpType(dataField.DataType)); + dataTable.Columns.Add(newColumn); + } + else + throw new Exception(string.Format("Field '{0}' does not exist", selectedField)); + } + + //Read column by column to generate each row in the datatable + totalRecordCount = 0; + for (int i = 0; i < parquetReader.RowGroupCount; i++) + { + int rowsLeftToRead = recordCount; + using (ParquetRowGroupReader groupReader = parquetReader.OpenRowGroupReader(i)) + { + if (groupReader.RowCount > int.MaxValue) + throw new ArgumentOutOfRangeException(string.Format("Cannot handle row group sizes greater than {0}", groupReader.RowCount)); + + int rowsPassedUntilThisRowGroup = totalRecordCount; + totalRecordCount += (int)groupReader.RowCount; + + if (offset >= totalRecordCount) + continue; + + if (rowsLeftToRead > 0) + { + int numberOfRecordsToReadFromThisRowGroup = Math.Min(Math.Min(totalRecordCount - offset, recordCount), (int)groupReader.RowCount); + rowsLeftToRead -= numberOfRecordsToReadFromThisRowGroup; + + int recordsToSkipInThisRowGroup = Math.Max(offset - rowsPassedUntilThisRowGroup, 0); + + ProcessRowGroup(dataTable, groupReader, fields, recordsToSkipInThisRowGroup, numberOfRecordsToReadFromThisRowGroup); + } + } + } + + return dataTable; + } + + private static void ProcessRowGroup(DataTable dataTable, ParquetRowGroupReader groupReader, List fields, int skipRecords, int readRecords) + { + int rowBeginIndex = dataTable.Rows.Count; + bool isFirstColumn = true; + + foreach (var field in fields) + { + int rowIndex = rowBeginIndex; + + int skippedRecords = 0; + foreach (var value in groupReader.ReadColumn(field).Data) + { + if (skipRecords > skippedRecords) + { + skippedRecords++; + continue; + } + + if (rowIndex >= readRecords) + break; + + if (isFirstColumn) + { + var newRow = dataTable.NewRow(); + dataTable.Rows.Add(newRow); + } + + if (value == null) + dataTable.Rows[rowIndex][field.Name] = DBNull.Value; + else if (field.DataType == Parquet.Data.DataType.DateTimeOffset) + dataTable.Rows[rowIndex][field.Name] = ((DateTimeOffset)value).DateTime; //converts to local time! + else + dataTable.Rows[rowIndex][field.Name] = value; + + rowIndex++; + } + + isFirstColumn = false; + } + } + + + public static Type ParquetNetTypeToCSharpType(Parquet.Data.DataType type) + { + Type columnType = null; + switch (type) + { + case Parquet.Data.DataType.Boolean: + columnType = typeof(bool); + break; + case Parquet.Data.DataType.Byte: + columnType = typeof(sbyte); + break; + case Parquet.Data.DataType.ByteArray: + columnType = typeof(sbyte[]); + break; + case Parquet.Data.DataType.DateTimeOffset: + //Let's treat DateTimeOffsets as DateTime + columnType = typeof(DateTime); + break; + case Parquet.Data.DataType.Decimal: + columnType = typeof(decimal); + break; + case Parquet.Data.DataType.Double: + columnType = typeof(double); + break; + case Parquet.Data.DataType.Float: + columnType = typeof(float); + break; + case Parquet.Data.DataType.Short: + case Parquet.Data.DataType.Int16: + case Parquet.Data.DataType.Int32: + case Parquet.Data.DataType.UnsignedInt16: + columnType = typeof(int); + break; + case Parquet.Data.DataType.Int64: + columnType = typeof(long); + break; + case Parquet.Data.DataType.UnsignedByte: + columnType = typeof(byte); + break; + case Parquet.Data.DataType.String: + default: + columnType = typeof(string); + break; + } + + return columnType; + } + + public static string CleanCSVValue(string value, bool alwaysEncloseInQuotes = false) + { + if (!string.IsNullOrWhiteSpace(value)) + { + //In RFC 4180 we escape quotes with double quotes + string formattedValue = value.Replace("\"", "\"\""); + + //Enclose value with quotes if it contains commas,line feeds or other quotes + if (formattedValue.Contains(",") || formattedValue.Contains("\r") || formattedValue.Contains("\n") || formattedValue.Contains("\"\"") || alwaysEncloseInQuotes) + formattedValue = string.Concat("\"", formattedValue, "\""); + + return formattedValue; + } + else + return string.Empty; + } + } +} diff --git a/src/ParquetFileViewer/MainForm.Designer.cs b/src/ParquetFileViewer/MainForm.Designer.cs index fb5f5fe..419c86d 100644 --- a/src/ParquetFileViewer/MainForm.Designer.cs +++ b/src/ParquetFileViewer/MainForm.Designer.cs @@ -1,4 +1,5 @@ -namespace ParquetFileViewer + +namespace ParquetFileViewer { partial class MainForm { @@ -31,12 +32,9 @@ private void InitializeComponent() System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(MainForm)); this.mainTableLayoutPanel = new System.Windows.Forms.TableLayoutPanel(); this.recordsToLabel = new System.Windows.Forms.Label(); - this.recordCountTextBox = new ParquetFileViewer.DelayedOnChangedTextBox(); this.showRecordsFromLabel = new System.Windows.Forms.Label(); - this.offsetTextBox = new ParquetFileViewer.DelayedOnChangedTextBox(); this.runQueryButton = new System.Windows.Forms.Button(); this.searchFilterLabel = new System.Windows.Forms.LinkLabel(); - this.searchFilterTextBox = new ParquetFileViewer.DelayedOnChangedTextBox(); this.clearFilterButton = new System.Windows.Forms.Button(); this.mainGridView = new System.Windows.Forms.DataGridView(); this.openParquetFileDialog = new System.Windows.Forms.OpenFileDialog(); @@ -54,6 +52,8 @@ private void InitializeComponent() this.changeFieldsMenuStripButton = new System.Windows.Forms.ToolStripMenuItem(); this.getSQLCreateTableScriptToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.changeDateFormatToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.defaultToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.iSO8601ToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.helpToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.userGuideToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.aboutToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); @@ -70,8 +70,9 @@ private void InitializeComponent() this.mainStatusStrip = new System.Windows.Forms.StatusStrip(); this.exportFileDialog = new System.Windows.Forms.SaveFileDialog(); this.ExportFileBackgroundWorker = new System.ComponentModel.BackgroundWorker(); - this.defaultToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); - this.iSO8601ToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.recordCountTextBox = new ParquetFileViewer.Controls.DelayedOnChangedTextBox(); + this.offsetTextBox = new ParquetFileViewer.Controls.DelayedOnChangedTextBox(); + this.searchFilterTextBox = new ParquetFileViewer.Controls.DelayedOnChangedTextBox(); this.mainTableLayoutPanel.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)(this.mainGridView)).BeginInit(); this.mainMenuStrip.SuspendLayout(); @@ -122,18 +123,6 @@ private void InitializeComponent() this.recordsToLabel.Text = "Record Count:"; this.recordsToLabel.TextAlign = System.Drawing.ContentAlignment.MiddleRight; // - // recordCountTextBox - // - this.recordCountTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); - this.recordCountTextBox.DelayedTextChangedTimeout = 1000; - this.recordCountTextBox.Location = new System.Drawing.Point(752, 5); - this.recordCountTextBox.Name = "recordCountTextBox"; - this.recordCountTextBox.Size = new System.Drawing.Size(54, 20); - this.recordCountTextBox.TabIndex = 5; - this.recordCountTextBox.TextAlign = System.Windows.Forms.HorizontalAlignment.Right; - this.recordCountTextBox.DelayedTextChanged += new System.EventHandler(this.recordsToTextBox_TextChanged); - this.recordCountTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.recordsToTextBox_KeyPress); - // // showRecordsFromLabel // this.showRecordsFromLabel.Anchor = System.Windows.Forms.AnchorStyles.Right; @@ -145,18 +134,6 @@ private void InitializeComponent() this.showRecordsFromLabel.Text = "Record Offset:"; this.showRecordsFromLabel.TextAlign = System.Drawing.ContentAlignment.MiddleRight; // - // offsetTextBox - // - this.offsetTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); - this.offsetTextBox.DelayedTextChangedTimeout = 1000; - this.offsetTextBox.Location = new System.Drawing.Point(642, 5); - this.offsetTextBox.Name = "offsetTextBox"; - this.offsetTextBox.Size = new System.Drawing.Size(54, 20); - this.offsetTextBox.TabIndex = 4; - this.offsetTextBox.TextAlign = System.Windows.Forms.HorizontalAlignment.Right; - this.offsetTextBox.DelayedTextChanged += new System.EventHandler(this.offsetTextBox_TextChanged); - this.offsetTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.offsetTextBox_KeyPress); - // // runQueryButton // this.runQueryButton.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) @@ -192,18 +169,6 @@ private void InitializeComponent() this.searchFilterLabel.TextAlign = System.Drawing.ContentAlignment.MiddleRight; this.searchFilterLabel.LinkClicked += new System.Windows.Forms.LinkLabelLinkClickedEventHandler(this.searchFilterLabel_Click); // - // searchFilterTextBox - // - this.searchFilterTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); - this.mainTableLayoutPanel.SetColumnSpan(this.searchFilterTextBox, 2); - this.searchFilterTextBox.DelayedTextChangedTimeout = 1000; - this.searchFilterTextBox.Location = new System.Drawing.Point(85, 5); - this.searchFilterTextBox.Name = "searchFilterTextBox"; - this.searchFilterTextBox.Size = new System.Drawing.Size(321, 20); - this.searchFilterTextBox.TabIndex = 1; - this.searchFilterTextBox.Text = "WHERE "; - this.searchFilterTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.searchFilterTextBox_KeyPress); - // // clearFilterButton // this.clearFilterButton.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) @@ -228,7 +193,7 @@ private void InitializeComponent() this.mainGridView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) | System.Windows.Forms.AnchorStyles.Left) | System.Windows.Forms.AnchorStyles.Right))); - this.mainGridView.ColumnHeadersHeightSizeMode = System.Windows.Forms.DataGridViewColumnHeadersHeightSizeMode.AutoSize; + this.mainGridView.ColumnHeadersHeightSizeMode = System.Windows.Forms.DataGridViewColumnHeadersHeightSizeMode.DisableResizing; this.mainTableLayoutPanel.SetColumnSpan(this.mainGridView, 10); this.mainGridView.Location = new System.Drawing.Point(3, 33); this.mainGridView.Name = "mainGridView"; @@ -237,6 +202,7 @@ private void InitializeComponent() this.mainGridView.SelectionMode = System.Windows.Forms.DataGridViewSelectionMode.CellSelect; this.mainGridView.Size = new System.Drawing.Size(803, 305); this.mainGridView.TabIndex = 6; + this.mainGridView.ColumnAdded += new System.Windows.Forms.DataGridViewColumnEventHandler(this.MainGridView_ColumnAdded); this.mainGridView.DataBindingComplete += new System.Windows.Forms.DataGridViewBindingCompleteEventHandler(this.mainGridView_DataBindingComplete); this.mainGridView.MouseClick += new System.Windows.Forms.MouseEventHandler(this.mainGridView_MouseClick); // @@ -369,6 +335,20 @@ private void InitializeComponent() this.changeDateFormatToolStripMenuItem.Size = new System.Drawing.Size(217, 22); this.changeDateFormatToolStripMenuItem.Text = "Date Format"; // + // defaultToolStripMenuItem + // + this.defaultToolStripMenuItem.Name = "defaultToolStripMenuItem"; + this.defaultToolStripMenuItem.Size = new System.Drawing.Size(119, 22); + this.defaultToolStripMenuItem.Text = "Default"; + this.defaultToolStripMenuItem.Click += new System.EventHandler(this.DefaultToolStripMenuItem_Click); + // + // iSO8601ToolStripMenuItem + // + this.iSO8601ToolStripMenuItem.Name = "iSO8601ToolStripMenuItem"; + this.iSO8601ToolStripMenuItem.Size = new System.Drawing.Size(119, 22); + this.iSO8601ToolStripMenuItem.Text = "ISO 8601"; + this.iSO8601ToolStripMenuItem.Click += new System.EventHandler(this.ISO8601ToolStripMenuItem_Click); + // // helpToolStripMenuItem // this.helpToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { @@ -485,19 +465,41 @@ private void InitializeComponent() this.ExportFileBackgroundWorker.DoWork += new System.ComponentModel.DoWorkEventHandler(this.ExportFileBackgroundWorker_DoWork); this.ExportFileBackgroundWorker.RunWorkerCompleted += new System.ComponentModel.RunWorkerCompletedEventHandler(this.ExportFileBackgroundWorker_RunWorkerCompleted); // - // defaultToolStripMenuItem + // recordCountTextBox // - this.defaultToolStripMenuItem.Name = "defaultToolStripMenuItem"; - this.defaultToolStripMenuItem.Size = new System.Drawing.Size(180, 22); - this.defaultToolStripMenuItem.Text = "Default"; - this.defaultToolStripMenuItem.Click += new System.EventHandler(this.DefaultToolStripMenuItem_Click); + this.recordCountTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); + this.recordCountTextBox.DelayedTextChangedTimeout = 1000; + this.recordCountTextBox.Location = new System.Drawing.Point(752, 5); + this.recordCountTextBox.Name = "recordCountTextBox"; + this.recordCountTextBox.Size = new System.Drawing.Size(54, 20); + this.recordCountTextBox.TabIndex = 5; + this.recordCountTextBox.TextAlign = System.Windows.Forms.HorizontalAlignment.Right; + this.recordCountTextBox.DelayedTextChanged += new System.EventHandler(this.recordsToTextBox_TextChanged); + this.recordCountTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.recordsToTextBox_KeyPress); // - // iSO8601ToolStripMenuItem + // offsetTextBox // - this.iSO8601ToolStripMenuItem.Name = "iSO8601ToolStripMenuItem"; - this.iSO8601ToolStripMenuItem.Size = new System.Drawing.Size(180, 22); - this.iSO8601ToolStripMenuItem.Text = "ISO 8601"; - this.iSO8601ToolStripMenuItem.Click += new System.EventHandler(this.ISO8601ToolStripMenuItem_Click); + this.offsetTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); + this.offsetTextBox.DelayedTextChangedTimeout = 1000; + this.offsetTextBox.Location = new System.Drawing.Point(642, 5); + this.offsetTextBox.Name = "offsetTextBox"; + this.offsetTextBox.Size = new System.Drawing.Size(54, 20); + this.offsetTextBox.TabIndex = 4; + this.offsetTextBox.TextAlign = System.Windows.Forms.HorizontalAlignment.Right; + this.offsetTextBox.DelayedTextChanged += new System.EventHandler(this.offsetTextBox_TextChanged); + this.offsetTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.offsetTextBox_KeyPress); + // + // searchFilterTextBox + // + this.searchFilterTextBox.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right))); + this.mainTableLayoutPanel.SetColumnSpan(this.searchFilterTextBox, 2); + this.searchFilterTextBox.DelayedTextChangedTimeout = 1000; + this.searchFilterTextBox.Location = new System.Drawing.Point(85, 5); + this.searchFilterTextBox.Name = "searchFilterTextBox"; + this.searchFilterTextBox.Size = new System.Drawing.Size(321, 20); + this.searchFilterTextBox.TabIndex = 1; + this.searchFilterTextBox.Text = "WHERE "; + this.searchFilterTextBox.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.searchFilterTextBox_KeyPress); // // MainForm // @@ -534,9 +536,9 @@ private void InitializeComponent() private System.Windows.Forms.TableLayoutPanel mainTableLayoutPanel; private System.Windows.Forms.OpenFileDialog openParquetFileDialog; private System.Windows.Forms.Label recordsToLabel; - private DelayedOnChangedTextBox recordCountTextBox; + private ParquetFileViewer.Controls.DelayedOnChangedTextBox recordCountTextBox; private System.Windows.Forms.Label showRecordsFromLabel; - private DelayedOnChangedTextBox offsetTextBox; + private ParquetFileViewer.Controls.DelayedOnChangedTextBox offsetTextBox; private System.Windows.Forms.DataGridView mainGridView; private System.Windows.Forms.MenuStrip mainMenuStrip; private System.Windows.Forms.ToolStripMenuItem fileToolStripMenuItem; @@ -552,7 +554,7 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripMenuItem helpToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem aboutToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem cSVToolStripMenuItem; - private DelayedOnChangedTextBox searchFilterTextBox; + private ParquetFileViewer.Controls.DelayedOnChangedTextBox searchFilterTextBox; private System.ComponentModel.BackgroundWorker FileSchemaBackgroundWorker; private System.ComponentModel.BackgroundWorker ReadDataBackgroundWorker; private System.Windows.Forms.Button runQueryButton; diff --git a/src/ParquetFileViewer/MainForm.cs b/src/ParquetFileViewer/MainForm.cs index 5c94e86..ab28cb0 100644 --- a/src/ParquetFileViewer/MainForm.cs +++ b/src/ParquetFileViewer/MainForm.cs @@ -1,4 +1,5 @@ using Parquet; +using ParquetFileViewer.Helpers; using System; using System.Collections.Generic; using System.ComponentModel; @@ -8,7 +9,6 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; @@ -22,7 +22,7 @@ public partial class MainForm : Form private const int loadingPanelWidth = 200; private const int loadingPanelHeight = 200; private const string QueryUselessPartRegex = "^WHERE "; - private const string ISO8601DateTimeFormat = "yyyy-MM-ddTHH:mm:ssZ"; + private const string ISO8601DateTimeFormat = "yyyy-MM-ddTHH:mm:ss.fffZ"; private const string DefaultTableName = "MY_TABLE"; private readonly string DefaultFormTitle; @@ -143,7 +143,7 @@ public MainForm() this.OpenFilePath = null; //Set DGV to be double buffered for smoother loading and scrolling - if (!System.Windows.Forms.SystemInformation.TerminalServerSession) + if (!SystemInformation.TerminalServerSession) { Type dgvType = this.mainGridView.GetType(); System.Reflection.PropertyInfo pi = dgvType.GetProperty("DoubleBuffered", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); @@ -481,7 +481,7 @@ private void FileSchemaBackgroundWorker_DoWork(object sender, DoWorkEventArgs e) e.Result = schema; else { - //Parquet.NET doesn't have any async methods or readers that allow sequential records reading so we need to use the ThreadPool to support cancellation. + //Parquet.NET doesn't have any async methods or readers that allow sequential reading so we need to use the ThreadPool to support cancellation. var task = Task.Run(() => { //Unfortunately there's no way to quickly get the metadata from a parquet file without reading an actual data row @@ -530,7 +530,7 @@ private void FileSchemaBackgroundWorker_RunWorkerCompleted(object sender, RunWor } else { - var fieldSelectionForm = new FieldsToLoadForm(fields, UtilityMethods.GetDataTableColumns(this.MainDataSource)); + var fieldSelectionForm = new FieldsToLoadForm(fields, this.MainDataSource?.GetColumnNames() ?? new string[0]); if (fieldSelectionForm.ShowDialog(this) == DialogResult.OK) { if (fieldSelectionForm.NewSelectedFields != null && fieldSelectionForm.NewSelectedFields.Count > 0) @@ -830,5 +830,15 @@ private void ISO8601ToolStripMenuItem_Click(object sender, EventArgs e) this.ShowError(ex); } } + + private void MainGridView_ColumnAdded(object sender, DataGridViewColumnEventArgs e) + { + if (e.Column is DataGridViewColumn column) + { + //This will help avoid overflowing the sum(fillweight) of the grid's columns when there are too many of them. + //The value of this field is not important as we do not use the FILL mode for column sizing. + column.FillWeight = 0.01f; + } + } } } diff --git a/src/ParquetFileViewer/ParquetFileViewer.csproj b/src/ParquetFileViewer/ParquetFileViewer.csproj index 33e9936..f24420b 100644 --- a/src/ParquetFileViewer/ParquetFileViewer.csproj +++ b/src/ParquetFileViewer/ParquetFileViewer.csproj @@ -81,8 +81,9 @@ AboutBox.cs - - + + + Component @@ -91,6 +92,7 @@ FieldSelectionDialog.cs + Form @@ -99,7 +101,7 @@ - + AboutBox.cs