Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
ferventdesert committed May 12, 2018
1 parent c59b2c9 commit b8ee268
Show file tree
Hide file tree
Showing 15 changed files with 374 additions and 214 deletions.
5 changes: 2 additions & 3 deletions Hawk.Core/Utils/Converters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ public object Convert(object value, Type targetType, object parameter, CultureIn
if (s == null)
return unknown;
var p = s.MyType;
string[] usualStrings = "从爬虫转换 生成区间数 合并多列 写入数据表 从文本生成 空对象过滤器 删除该列 列名修改器 XPath筛选器".Split(' ');
if (usualStrings.Contains(s.Name))
return ".常用";
if (s.Description.Contains(s.Name))
return "常用";
if(p==null)
return unknown;
foreach (var item in map)
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL.Controls/DataProcessManagerUI.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@
<TextBlock TextWrapping="Wrap" Margin="0,0,-0.001,0" FontSize="{DynamicResource UltraFont}" d:LayoutOverrides="Width, GridBox"><Run Language="zh-cn" Text="快速链接"/></TextBlock>
</StackPanel>
<ListBox Grid.Column="2" Margin="30,40,30,20" Grid.Row="2" ItemTemplate="{DynamicResource DataTemplateHelp}" ItemsSource="{Binding MainFrmUI.CommandCollection[1].ChildActions}" Grid.RowSpan="2"/>
<StackPanel Grid.Column="1" d:LayoutOverrides="Width" HorizontalAlignment="Left" VerticalAlignment="Bottom" Margin="30,20,0,20">
<StackPanel VerticalAlignment="Bottom" Margin="270,0,0,30" HorizontalAlignment="Left" Width="235.283" Grid.ColumnSpan="2">
<TextBlock TextWrapping="Wrap" FontSize="16" Foreground="#FFA1A1A1" HorizontalAlignment="Left"><Run Text="数据处理革命"/></TextBlock>
<TextBlock TextWrapping="Wrap" HorizontalAlignment="Left" FontSize="16" Foreground="#FFA1A1A1" Margin="0,8,0,0"><Run Text="Smart ETL Crawler Engine"/></TextBlock>
</StackPanel>
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL.Controls/Properties/Settings.Designer.cs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

469 changes: 296 additions & 173 deletions Hawk.ETL.Controls/SmartCrawlerUI.xaml

Large diffs are not rendered by default.

46 changes: 22 additions & 24 deletions Hawk.ETL.Controls/SmartETLUI.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -168,21 +168,21 @@
</ItemsPanelTemplate>

<DataTemplate x:Key="DataTemplateList">
<StackPanel Orientation="Horizontal" Margin="0,4" >
<Rectangle Width="20" Height="20" Margin="0,0,20,0" >
<Rectangle.Fill>
<VisualBrush Stretch="Fill"
Visual="{Binding Converter={StaticResource ModuleMetroConverter}, Mode=OneWay}" />
</Rectangle.Fill>
</Rectangle>
<StackPanel HorizontalAlignment="Stretch" Height="Auto" Width="Auto" Orientation="Vertical">
<Label Content="{Binding Name}" Margin="0" Height="Auto" VerticalAlignment="Stretch" FontSize="{DynamicResource LargeFont}"
d:LayoutOverrides="GridBoX" HorizontalAlignment="Left" Width="Auto" />
<Label Content="{Binding Description}" Margin="0" Height="Auto" VerticalAlignment="Stretch"
HorizontalAlignment="Left" Width="Auto" Opacity="0.8" />

</StackPanel>
</StackPanel>
<Grid Height="Auto" Margin="0,4" Width="Auto">
<Rectangle Width="20" Height="20" HorizontalAlignment="Left" VerticalAlignment="Center" >
<Rectangle.Fill>
<VisualBrush Stretch="Fill"
Visual="{Binding Converter={StaticResource ModuleMetroConverter}, Mode=OneWay}" />
</Rectangle.Fill>
</Rectangle>
<StackPanel Height="Auto" Width="Auto" Orientation="Vertical" d:LayoutOverrides="Width, Height" Margin="30,0,0,0">
<Label Content="{Binding Name}" ToolTip="{Binding Description}" Margin="0" Height="Auto" VerticalAlignment="Stretch" FontSize="{DynamicResource LargeFont}"
d:LayoutOverrides="GridBoX" HorizontalAlignment="Left" Width="Auto" />
<TextBlock Text="{Binding Description}" Margin="0" Height="Auto" VerticalAlignment="Stretch"
HorizontalAlignment="Left" Width="Auto" Opacity="0.6" TextWrapping="WrapWithOverflow" />

</StackPanel>
</Grid>
<DataTemplate.Triggers>
<EventTrigger RoutedEvent="Mouse.MouseEnter" />
<EventTrigger RoutedEvent="Mouse.MouseLeave" />
Expand Down Expand Up @@ -354,7 +354,7 @@
<StackPanel HorizontalAlignment="Left" Orientation="Horizontal"
d:LayoutOverrides="Height" VerticalAlignment="Center">
<TextBlock TextWrapping="Wrap" HorizontalAlignment="Left" VerticalAlignment="Center" Margin="5,0,0,0"><Run Language="zh-cn" Text="任务名称"/></TextBlock>
<TextBox TextWrapping="Wrap" Text="{Binding Name}" System_Windows_Controls_WpfPropertyGrid:TextBoxExtender.CommitOnEnter="True" VerticalAlignment="Center" Margin="8,0,30,0" FontSize="16" MinWidth="100"/>
<TextBox TextWrapping="Wrap" Text="{Binding Name}" System_Windows_Controls_WpfPropertyGrid:TextBoxExtender.CommitOnEnter="True" VerticalAlignment="Center" Margin="8,0,25,0" FontSize="16" MinWidth="100"/>
<TextBlock TextWrapping="Wrap" Text="模块"
VerticalAlignment="Center" />
<TextBox Width="50" TextWrapping="Wrap" Text="{Binding ETLMount}" VerticalAlignment="Center"
Expand Down Expand Up @@ -433,9 +433,11 @@
<TextBlock TextWrapping="Wrap" Margin="0"
VerticalAlignment="Center" ><Run Language="zh-cn" Text="采样量"/></TextBlock>
<TextBox TextWrapping="Wrap" Text="{Binding SampleMount}" ToolTip="设置获取数据表的前n行" VerticalAlignment="Center"
FontSize="16" MinWidth="30" />
<CheckBox Content="自动刷新" IsChecked="{Binding IsAutoRefresh}" VerticalAlignment="Center" Margin="15,0,0,0" />
<CheckBox Content="调试详情" IsChecked="{Binding DisplayDetail}" VerticalAlignment="Center" Margin="15,0,0,0" />
FontSize="16" MinWidth="30" Margin="0,0,30,0" />
<StackPanel>
<CheckBox Content="自动刷新" IsChecked="{Binding IsAutoRefresh}" VerticalAlignment="Center" />
<CheckBox Content="调试详情" IsChecked="{Binding DisplayDetail}" VerticalAlignment="Center" />
</StackPanel>
</StackPanel>

</Grid>
Expand Down Expand Up @@ -477,7 +479,7 @@
</Grid>
</ScrollViewer>
</Grid>
<StackPanel Height="17.24" Margin="20,12.835,29.75,0" Orientation="Horizontal" VerticalAlignment="Top" d:LayoutOverrides="Width">
<StackPanel Margin="20,12.835,29.75,0" Orientation="Horizontal" VerticalAlignment="Top" d:LayoutOverrides="Width">
<Rectangle Width="15" Height="15" Margin="0,0,10,0" >
<Rectangle.Fill>
<VisualBrush Stretch="Fill"
Expand All @@ -502,9 +504,7 @@
<GroupStyle ContainerStyle="{StaticResource GroupContainerStyle}">
<GroupStyle.HeaderTemplate>
<DataTemplate>

<TextBlock Text="{Binding Name}" />

</DataTemplate>
</GroupStyle.HeaderTemplate>
</GroupStyle>
Expand All @@ -513,7 +513,6 @@
<GridSplitter HorizontalAlignment="Stretch" Height="3" VerticalAlignment="Bottom" Margin="0,0,0,-1.25" d:LayoutOverrides="VerticalAlignment, GridBox" />
<ListView x:FieldModifier="public" x:Name="CurrentETLToolList" SelectedItem="{Binding CurrentTool, Mode=OneWay}" Grid.Row="1" Margin="0,8,0,0" SelectionMode="Single" DataContext="{Binding}"
ItemsSource="{Binding CurrentETLTools}" >

<ListView.View>
<GridView>
<GridViewColumn Header="启用">
Expand All @@ -526,7 +525,6 @@
<GridViewColumn Header="位置" DisplayMemberBinding="{Binding ETLIndex}" />
<GridViewColumn Header="类型" DisplayMemberBinding="{Binding Attribute, Converter={StaticResource GroupConverter}}" />
<GridViewColumn Header="名称" DisplayMemberBinding="{Binding TypeName}" />

</GridView>

</ListView.View>
Expand Down
11 changes: 9 additions & 2 deletions Hawk.ETL/Crawlers/XPathAnalyzer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1239,8 +1239,14 @@ public static IEnumerable<CrawTarget> SearchPropertiesSmartOne(this HtmlNode doc
if (subPath2 == subPath1)
continue;

yield return GetCrawTargetOne(childNodes, crawlItem, existItem);
yield return GetCrawTargetOne(childNodes, existItem, crawlItem);
var item= GetCrawTargetOne(childNodes, crawlItem, existItem);
item.RootNode = doc2;
item.WorkMode = ScriptWorkMode.One;
yield return item;
item= GetCrawTargetOne(childNodes, existItem, crawlItem);
item.RootNode = doc2;
item.WorkMode = ScriptWorkMode.One;
yield return item;
}
}
}
Expand All @@ -1257,6 +1263,7 @@ private static CrawTarget GetCrawTargetOne(List<HtmlNode> childNodes, CrawlItem
var crawlItems = names.Zip(xpaths,
(a, b) => new CrawlItem {Name = a, XPath = b, CrawlType = valueItem.CrawlType}).ToList();
var target = getCrawTarget(crawlItems, "");

return target;
}

Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Executor/TableEX.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace Hawk.ETL.Plugins.Executor
{
[XFrmWork("写入数据表", "将数据保存至软件的数据管理器中,之后可方便进行其他处理,拖入到任意一列皆可", "column_three")]
[XFrmWork("写入数据表", "将数据保存至软件的数据管理器中,之后可方便进行其他处理,拖入到任意一列皆可,常用", "column_three")]
public class TableEX : DataExecutorBase
{
private readonly IDataManager dataManager;
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Filters/NullFT.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace Hawk.ETL.Plugins.Filters
{
[XFrmWork("空对象过滤器", "检查文本是否为空白符或null")]
[XFrmWork("空对象过滤器", "检查文本是否为空白符或null,常用")]
public class NullFT : ToolBase, IColumnDataFilter
{
#region Constructors and Destructors
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Generators/TableGE.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace Hawk.ETL.Plugins.Generators
{
[XFrmWork("从数据表生成","从数据管理中已有的数据表中生成" )]
[XFrmWork("从数据表生成","从数据管理中已有的数据表中生成,常用" )]
public class TableGE : GeneratorBase
{
private readonly IDataManager dataManager;
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Generators/TextGE.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace Hawk.ETL.Plugins.Generators
{
[XFrmWork("从文本生成","每行一条数据" )]
[XFrmWork("从文本生成","每行一条数据,常用" )]
public class TextGE : GeneratorBase
{
List<string> argsList=new List<string>();
Expand Down
4 changes: 2 additions & 2 deletions Hawk.ETL/Plugins/Transformers/CollumTransformer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace Hawk.ETL.Plugins.Transformers
{
[XFrmWork("列名修改器", "对列名进行修改","page_edit")]
[XFrmWork("列名修改器", "对列名进行修改,常用","page_edit")]
public class RenameTF : TransformerBase
{
public override bool Init(IEnumerable<IFreeDocument> docus)
Expand Down Expand Up @@ -44,7 +44,7 @@ public override object TransformData(IFreeDocument document)
}
}

[XFrmWork("删除该列","不用解释了吧","delete")]
[XFrmWork("删除该列","不用解释了吧,常用","delete")]
public class DeleteTF : RenameTF
{
public override object TransformData(IFreeDocument document)
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Transformers/CrawlerTF.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

namespace Hawk.ETL.Plugins.Transformers
{
[XFrmWork("从爬虫转换", "使用网页采集器获取网页数据,拖入的列需要为超链接","carema")]
[XFrmWork("从爬虫转换", "使用网页采集器获取网页数据,拖入的列需要为超链,常用","carema")]
public class CrawlerTF : ResponseTF
{
private BfsGE generator;
Expand Down
2 changes: 1 addition & 1 deletion Hawk.ETL/Plugins/Transformers/MergeTF.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace Hawk.ETL.Plugins.Transformers
{
[XFrmWork("合并多列", "将多个列组合成同一列,形如'http:\\{0}:{1},{2}...'输入列的序号为0,之后的1,2分别代表【其他项】的第0和第1个值")]
[XFrmWork("合并多列", "将多个列组合成同一列,形如'http:\\{0}:{1},{2}...'输入列的序号为0,之后的1,2分别代表【其他项】的第0和第1个值,常用")]
public class MergeTF : TransformerBase
{

Expand Down
7 changes: 6 additions & 1 deletion Hawk.ETL/Process/SmartCrawler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ public class SmartCrawler : AbstractProcessMethod, IView
private IEnumerator<string> currentXPaths;
public bool enableRefresh = true;
private bool hasInit;
public HtmlDocument HtmlDoc = new HtmlDocument();

public HtmlDocument HtmlDoc { get; set; }
private TextBox htmlTextBox;
private bool isBusy;

Expand All @@ -70,6 +71,7 @@ public SmartCrawler()
CrawlItems = new ObservableCollection<CrawlItem>();
helper = new HttpHelper();
URL = "";
HtmlDoc=new HtmlDocument();
SelectText = "";
IsMultiData = ScriptWorkMode.List;
IsAttribute = true;
Expand Down Expand Up @@ -916,7 +918,10 @@ private async void VisitUrlAsync()
{
dynamic invoke = control.View;
if (IsSuperMode == false)
{
invoke.UpdateHtml(URLHTML);
OnPropertyChanged("HtmlDoc");
}
else
{
invoke.UpdateHtml("超级模式下内置浏览器不展示内容,请查看左侧的文本内容");
Expand Down
30 changes: 29 additions & 1 deletion Hawk.ETL/Process/SmartETLTool.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Collections.Specialized;
Expand Down Expand Up @@ -51,6 +52,8 @@ public SmartETLTool()

ETLToolsView.GroupDescriptions.Add(new PropertyGroupDescription("Self", new GroupConverter()));
ETLToolsView.SortDescriptions.Add(new SortDescription("GroupName",ListSortDirection.Ascending));
ETLToolsView.CustomSort = new NameComparer();

}
}

Expand All @@ -65,7 +68,7 @@ public SmartETLTool()

private ListView currentToolList;
private ScrollViewer scrollViewer;
private string searchText;
private string searchText = "常用";

#endregion

Expand Down Expand Up @@ -1010,6 +1013,31 @@ public enum GroupType
Edit
}

public class NameComparer : IComparer
{

public int Compare(object x, object y)
{
var x1 = x as XFrmWorkAttribute;
var y1 = y as XFrmWorkAttribute;
var key = "常用";
if (x1.Description.Contains(key))
{
if (y1.Description.Contains(key))
return x1.Name.CompareTo(y1.Name);
return -1;

}
if (y1.Description.Contains(key))
{
return 1;

}
return x1.Name.CompareTo(y1.Name);


}
}
public class SmartGroup : PropertyChangeNotifier
{
private GroupType _groupType;
Expand Down

0 comments on commit b8ee268

Please sign in to comment.