Commit a2088003 by mahaisong

feat:调用ES,返回数据、解析,封装成指定的json格式,放入json文件

parent 7fc356d7
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<configSections>
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
<sectionGroup name="common">
<section name="logging" type="Common.Logging.ConfigurationSectionHandler, Common.Logging" />
</sectionGroup>
</configSections>
<connectionStrings>
<add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
</connectionStrings>
<common>
<logging>
<factoryAdapter type="Common.Logging.Log4Net.Log4NetLoggerFactoryAdapter, Common.Logging.Log4Net208">
<!--FILE
FILE-WATCH log4net节点在其他独立文件内部。
INLINE log4net节点在配置app.config内部。
EXTERNAL-->
<arg key="configType" value="FILE-WATCH" />
<arg key="configFile" value="~/Config/log4net.config" />
</factoryAdapter>
</logging>
</common>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8" ?>
<log4net>
<appender name="errorAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="ERROR" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\err.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="infoAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\info.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="debugAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="DEBUG" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\debug.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="perfAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\perf.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date %logger - %message%newline" />
</layout>
</appender>
<root>
<level value="ALL" />
<appender-ref ref="errorAppender" />
<appender-ref ref="infoAppender" />
<appender-ref ref="debugAppender" />
</root>
<logger name="Performance" additivity="false">
<level value="ALL" />
<appender-ref ref="perfAppender" />
</logger>
</log4net>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8" ?>
<log4net>
<appender name="errorAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="ERROR" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<File value="Logs/err.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="infoAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs/info.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="debugAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="DEBUG" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs/debug.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="perfAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs/perf.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date %logger - %message%newline" />
</layout>
</appender>
<root>
<level value="ALL" />
<appender-ref ref="errorAppender" />
<appender-ref ref="infoAppender" />
<appender-ref ref="debugAppender" />
</root>
<logger name="Performance" additivity="false">
<level value="ALL" />
<appender-ref ref="perfAppender" />
</logger>
</log4net>
\ No newline at end of file
......@@ -32,7 +32,40 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Common.Logging, Version=3.4.1.0, Culture=neutral, PublicKeyToken=af08829b84f0328e, processorArchitecture=MSIL">
<HintPath>..\packages\Common.Logging.3.4.1\lib\net40\Common.Logging.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Common.Logging.Core, Version=3.4.1.0, Culture=neutral, PublicKeyToken=af08829b84f0328e, processorArchitecture=MSIL">
<HintPath>..\packages\Common.Logging.Core.3.4.1\lib\net40\Common.Logging.Core.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Common.Logging.Log4Net208, Version=3.4.1.0, Culture=neutral, PublicKeyToken=af08829b84f0328e, processorArchitecture=MSIL">
<HintPath>..\packages\Common.Logging.Log4Net208.3.4.1\lib\net40\Common.Logging.Log4Net208.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Elasticsearch.Net, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\Elasticsearch.Net.2.5.8\lib\net45\Elasticsearch.Net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="log4net, Version=2.0.8.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
<HintPath>..\packages\log4net.2.0.8\lib\net45-full\log4net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MongoDB.Bson, Version=2.0.0.828, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Bson.2.0.0\lib\net45\MongoDB.Bson.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Nest, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\NEST.2.5.8\lib\net45\Nest.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
......@@ -45,15 +78,21 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ESClientConfigureManager.cs" />
<Compile Include="ESItemAccess.cs" />
<Compile Include="Form1.cs">
<SubType>Form</SubType>
</Compile>
<Compile Include="Form1.Designer.cs">
<DependentUpon>Form1.cs</DependentUpon>
</Compile>
<Compile Include="Item.cs" />
<Compile Include="ListModel.cs" />
<Compile Include="LogService.cs" />
<Compile Include="PerformanceUtility.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ResultModel\ResultItem.cs" />
<Compile Include="V1.cs" />
<EmbeddedResource Include="Form1.resx">
<DependentUpon>Form1.cs</DependentUpon>
......@@ -67,6 +106,13 @@
<AutoGen>True</AutoGen>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<None Include="Config\log4net.config">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="OutPut\模板.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="packages.config" />
<None Include="Properties\Settings.settings">
<Generator>SettingsSingleFileGenerator</Generator>
<LastGenOutput>Settings.Designer.cs</LastGenOutput>
......@@ -80,6 +126,7 @@
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
......
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ProjectView>ShowAllFiles</ProjectView>
</PropertyGroup>
</Project>
\ No newline at end of file
using Elasticsearch.Net;
using Nest;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace CsvCount_ES
{
public sealed class ESClientConfigureManager
{
public static List<Uri> Uris { get; private set; }
public static String DefaultIndex { get; private set; }
/// <summary>
/// ES连接设置
/// </summary>
public static ConnectionSettings ConnectSetting { get; private set; }
private const String ConnectionStringIndex = "ESDatabase";
private const String HostIndex = "host";
private const String PortIndex = "port";
private const String DefaultIndexIndex = "defaultIndex";
static ESClientConfigureManager()
{
String connectionString = ConfigurationManager.ConnectionStrings[ConnectionStringIndex].ConnectionString;
String[] namedValues = connectionString.Split(';');
Dictionary<String, String> nvPair = new Dictionary<String, String>();
foreach (String nv in namedValues)
{
String[] pair = nv.Split('=');
nvPair[pair[0]] = pair[1];
}
String _url = GetValueByName(nvPair, HostIndex, "tank.palaspom.com|mech.palaspom.com");
String _port = GetValueByName(nvPair, PortIndex, "19235");
//支持多Uri模式,要求多个链接使用 | 分隔,在分隔后数量相同的情况下按照顺序一一对应,如果其中一方仅有一个则使用它对应对方所有部分
string[] urls = _url.Split('|');
string[] ports = _port.Split('|');
if (urls == null || ports == null || urls.Length < 1 || ports.Length < 1)
throw new Exception("Unable get the elasticsearch config pool setting.");
Uris = new List<Uri>();
if (urls.Length == ports.Length)
{
for (int i = 0; i < urls.Length; i++)
{
String formattedUrl = Regex.IsMatch(urls[i], @"://") ? urls[i] : String.Format("http://{0}", urls[i]);
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[i])));
}
}
else if (urls.Length > 1 && ports.Length == 1)
{
for (int i = 0; i < urls.Length; i++)
{
String formattedUrl = Regex.IsMatch(urls[i], @"://") ? urls[i] : String.Format("http://{0}", urls[i]);
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[0])));
}
}
else if (urls.Length == 1 && ports.Length > 1)
{
String formattedUrl = Regex.IsMatch(urls[0], @"://") ? urls[0] : String.Format("http://{0}", urls[0]);
for (int i = 0; i < ports.Length; i++)
{
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[i])));
}
}
else
{
throw new Exception("The elasticsearch config pool setting is error.");
}
var pool = new StaticConnectionPool(Uris);
String index = GetValueByName(nvPair, DefaultIndexIndex, "palas");
DefaultIndex = index;
ConnectSetting = new ConnectionSettings(pool).BasicAuthentication("carey", "jfyhdcm").DefaultIndex(index).DisableDirectStreaming();
}
private static String GetValueByName(Dictionary<String, String> nvPair, String name, String defaultValue)
{
String value = null;
if (nvPair.TryGetValue(name, out value))
{
return value;
}
return defaultValue;
}
}
}
using Nest;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CsvCount_ES
{
/// <summary>
/// 直接获取ESData的特殊类
/// </summary>
public class ESItemAccess : ESAccess
{
public const string NullValue = "空空";
/// <summary>
/// 根据itemID搜索Item
/// </summary>
/// <param name="id">itemID</param>
/// <returns>返回实体</returns>
public static Item SearchByItemID(string id)
{
SearchDescriptor<Item> builder = new SearchDescriptor<Item>()
.Size(1)
.Query(q => q
.Bool(b => b
.Must(m => m
.QueryString(qs => qs.DefaultField(f => f.ItemID).Query(id))
)
)
);
try
{
return SearchTop(builder);
}
catch (Exception e)
{
LogService.WriteError(string.Format("ES SearchByItemID Fail:{0} ItemID:{1}", e.ToString(), id));
return null;
}
}
/// <summary>
/// 根据itemID搜索Item
/// </summary>
/// <param name="id">itemID</param>
/// <returns>返回实体</returns>
public static Item[] SearchByItemIDs(List<string> itemIds)
{
SearchDescriptor<Item> builder = new SearchDescriptor<Item>()
.Size(1000)
.Query(q => q
.Bool(b => b
.Must(m => m
.QueryString(qs => qs.DefaultField(f => f.ItemID).Query("\"" + string.Join("\" \"", itemIds) + "\""))
)
)
);
try
{
return Search(builder);
}
catch (Exception e)
{
LogService.WriteError(string.Format("ES SearchByItemIDs Fail:{0}", e.ToString()));
return null;
}
}
}
public class ESAccess
{
/// <summary>
/// 从ES中搜索数据,仅返回需要的数据本体
/// </summary>
/// <param name="searchQuery">查询条件</param>
/// <typeparam name="T">需要查询的类</typeparam>
/// <returns>数据结果</returns>
public static T[] Search<T>(SearchDescriptor<T> searchQuery) where T : class
{
ISearchResponse<T> result = SearchDetail(searchQuery);
if (result == null || result.Documents == null)
return null;
else
return result.Documents.ToArray();
}
/// <summary>
/// 从ES中搜索数据,仅返回第一条数据本体
/// </summary>
/// <typeparam name="T">查询条件</typeparam>
/// <param name="searchQuery">需要查询的类</param>
/// <returns>数据结果</returns>
public static T SearchTop<T>(SearchDescriptor<T> searchQuery) where T : class
{
var response = Search(searchQuery);
if (response == null)
return null;
else
return response.FirstOrDefault();
}
/// <summary>
/// 从ES中搜索数据,返回详细信息,包括数据本体,查询时间,命中总数等
/// </summary>
/// <param name="searchQuery">查询条件</param>
/// <typeparam name="T">需要查询的类</typeparam>
/// <returns>统计数据结果集</returns>
public static ISearchResponse<T> SearchDetail<T>(SearchDescriptor<T> searchQuery) where T : class
{
ElasticClient client = new ElasticClient(ESClientConfigureManager.ConnectSetting);
ISearchResponse<T> response = null;
try
{
response = PerformanceUtility.OperationTimedMonitor<ISearchResponse<T>>(
PerformanceUtility.DefaultTriggerTimeSpan,
PerformanceUtility.DefaultIntervalTimeSpan,
Serialize(searchQuery),
//searchQuery.ExplainJson(),
() => { return client.Search<T>(searchQuery); });
var isItem = typeof(T) == typeof(Item);
if (isItem)
{
foreach (var document in response.Documents)
{
var item = document as Item;
ReverseNullValueForQuery(item);
}
}
}
catch (Exception e)
{
LogService.WriteError("ESAccess SearchDetail Fail", e);
}
return response;
}
public static void ReverseNullValueForQuery(Item item)
{
if (item.DuplicationID == ESItemAccess.NullValue)
{
item.DuplicationID = null;
}
}
public static string Serialize<T>(T obj) where T : class
{
try
{
ElasticClient client = new ElasticClient(ESClientConfigureManager.ConnectSetting);
using (MemoryStream stream = new MemoryStream())
{
client.Serializer.Serialize(obj, stream);
byte[] b = stream.ToArray();
return Encoding.UTF8.GetString(b, 0, b.Length);
}
}
catch
{
return "";
}
}
public static T Deserialize<T>(string json) where T : class
{
ElasticClient client = new ElasticClient(ESClientConfigureManager.ConnectSetting);
return client.Serializer.Deserialize<T>(new MemoryStream(Encoding.UTF8.GetBytes(json)));
}
}
}
......@@ -39,6 +39,8 @@
this.textBox_path = new System.Windows.Forms.TextBox();
this.label_wait = new System.Windows.Forms.Label();
this.label4 = new System.Windows.Forms.Label();
this.label5 = new System.Windows.Forms.Label();
this.textBox_block = new System.Windows.Forms.TextBox();
this.SuspendLayout();
//
// openFileDialog1
......@@ -47,7 +49,7 @@
//
// button1
//
this.button1.Location = new System.Drawing.Point(432, 40);
this.button1.Location = new System.Drawing.Point(503, 34);
this.button1.Name = "button1";
this.button1.Size = new System.Drawing.Size(75, 23);
this.button1.TabIndex = 0;
......@@ -57,15 +59,15 @@
//
// textBox1
//
this.textBox1.Location = new System.Drawing.Point(76, 40);
this.textBox1.Location = new System.Drawing.Point(113, 37);
this.textBox1.Name = "textBox1";
this.textBox1.Size = new System.Drawing.Size(350, 20);
this.textBox1.Size = new System.Drawing.Size(352, 20);
this.textBox1.TabIndex = 1;
//
// label1
//
this.label1.AutoSize = true;
this.label1.Location = new System.Drawing.Point(3, 40);
this.label1.Location = new System.Drawing.Point(3, 37);
this.label1.Name = "label1";
this.label1.Size = new System.Drawing.Size(67, 13);
this.label1.TabIndex = 2;
......@@ -74,18 +76,18 @@
// button2
//
this.button2.Enabled = false;
this.button2.Location = new System.Drawing.Point(76, 80);
this.button2.Location = new System.Drawing.Point(503, 77);
this.button2.Name = "button2";
this.button2.Size = new System.Drawing.Size(99, 27);
this.button2.TabIndex = 3;
this.button2.Text = "2:开始执行分析";
this.button2.UseVisualStyleBackColor = true;
this.button2.Click += new System.EventHandler(this.button2_Click);
this.button2.Click += new System.EventHandler(this.button2_ES_Click);
//
// label2
//
this.label2.AutoSize = true;
this.label2.Location = new System.Drawing.Point(3, 153);
this.label2.Location = new System.Drawing.Point(6, 121);
this.label2.Name = "label2";
this.label2.Size = new System.Drawing.Size(58, 13);
this.label2.TabIndex = 5;
......@@ -95,7 +97,7 @@
//
this.listBox1.Enabled = false;
this.listBox1.FormattingEnabled = true;
this.listBox1.Location = new System.Drawing.Point(76, 122);
this.listBox1.Location = new System.Drawing.Point(113, 121);
this.listBox1.Name = "listBox1";
this.listBox1.Size = new System.Drawing.Size(431, 160);
this.listBox1.TabIndex = 8;
......@@ -113,7 +115,7 @@
// textBox_path
//
this.textBox_path.Enabled = false;
this.textBox_path.Location = new System.Drawing.Point(115, 331);
this.textBox_path.Location = new System.Drawing.Point(113, 331);
this.textBox_path.Name = "textBox_path";
this.textBox_path.Size = new System.Drawing.Size(517, 20);
this.textBox_path.TabIndex = 10;
......@@ -121,7 +123,7 @@
// label_wait
//
this.label_wait.AutoSize = true;
this.label_wait.Location = new System.Drawing.Point(220, 87);
this.label_wait.Location = new System.Drawing.Point(232, 77);
this.label_wait.Name = "label_wait";
this.label_wait.Size = new System.Drawing.Size(0, 13);
this.label_wait.TabIndex = 11;
......@@ -131,15 +133,33 @@
this.label4.AutoSize = true;
this.label4.Location = new System.Drawing.Point(6, 371);
this.label4.Name = "label4";
this.label4.Size = new System.Drawing.Size(239, 13);
this.label4.Size = new System.Drawing.Size(381, 13);
this.label4.TabIndex = 12;
this.label4.Text = "参考:1.800W条记录不查ES,20秒统计次数";
this.label4.Text = "参考:1.800W条记录不查ES,7秒统计次数(400M,分块大小10240B)";
//
// label5
//
this.label5.AutoSize = true;
this.label5.Location = new System.Drawing.Point(3, 77);
this.label5.Name = "label5";
this.label5.Size = new System.Drawing.Size(104, 13);
this.label5.TabIndex = 13;
this.label5.Text = "分块并行读取(B):";
//
// textBox_block
//
this.textBox_block.Location = new System.Drawing.Point(113, 77);
this.textBox_block.Name = "textBox_block";
this.textBox_block.Size = new System.Drawing.Size(119, 20);
this.textBox_block.TabIndex = 14;
//
// Form1
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(644, 408);
this.Controls.Add(this.textBox_block);
this.Controls.Add(this.label5);
this.Controls.Add(this.label4);
this.Controls.Add(this.label_wait);
this.Controls.Add(this.textBox_path);
......@@ -152,6 +172,7 @@
this.Controls.Add(this.button1);
this.Name = "Form1";
this.Text = "Form1";
this.Load += new System.EventHandler(this.Form1_Load);
this.ResumeLayout(false);
this.PerformLayout();
......@@ -170,6 +191,8 @@
private System.Windows.Forms.TextBox textBox_path;
private System.Windows.Forms.Label label_wait;
private System.Windows.Forms.Label label4;
private System.Windows.Forms.Label label5;
private System.Windows.Forms.TextBox textBox_block;
}
}
using MongoDB.Bson.Serialization.Attributes;
using Nest;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CsvCount_ES
{
/// <summary>
/// 全局公用的单Item数据包
/// </summary>
[ElasticsearchType(Name = "items")]
public class Item
{
public Item()
{
}
/// <summary>
/// 文章ID
/// </summary>
public string ItemID { get; set; }
/// <summary>
/// 文章Url
/// </summary>
public string Url { get; set; }
/// <summary>
/// 标题
/// </summary>
public string CleanTitle { get; set; }
/// <summary>
/// 正文
/// </summary>
public string CleanText { get; set; }
/// <summary>
/// 页面显示发布时间
/// </summary>
[BsonDateTimeOptions(Kind = DateTimeKind.Local)]
public DateTime PubDate { get; set; }
/// <summary>
/// 媒体名称(分词索引)
/// </summary>
public string MediaName { get; set; }
/// <summary>
/// 相似转载ID(早先入库的相似文章ID,第一篇该字段为空,其他均为第一篇的ID)
/// </summary>
public string DuplicationID { get; set; }
///<summary>
///分析字段,每个客户一条记录(支持不同的分析模型)
///</summary>
[Nested(IncludeInParent = true)]//(Path = "analyzeData")]
public ItemAnalyzeData[] AnalyzeData { get; set; }
}
/// <summary>
/// 语义分析结果(每Issue可不同)
/// </summary>
[Serializable]
public class ItemAnalyzeData
{
/// <summary>
/// 议题编号
/// </summary>
public string IssueID { get; set; }
/// <summary>
/// 品类/市场编号
/// </summary>
public string[] MarketIDs { set; get; }
/// <summary>
/// 主题/板块编号
/// </summary>
public string[] BlockIDs { set; get; }
/// <summary>
/// 股票编号
/// </summary>
public string[] StockIDs { set; get; }
/// <summary>
/// IG Tree
/// </summary>
public string IG { set; get; }
public ItemAnalyzeData()
{
}
}
}
using Common.Logging;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CsvCount_ES
{
public class LogService
{
private static ILog _logdebug = LogManager.GetLogger("logdebug");
private static ILog _loginfo = LogManager.GetLogger("loginfo");
private static ILog _logwarn = LogManager.GetLogger("logwarn");
private static ILog _logerror = LogManager.GetLogger("logerror");
private static ILog _logfatal = LogManager.GetLogger("logfatal");
public static void WriteDebug(string msg)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logdebug.Debug(msg);
}
public static void WriteDebug(string msg, Exception e)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logdebug.Debug(msg, e);
}
public static void WriteInfo(string msg)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_loginfo.Info(msg);
}
public static void WriteInfo(string msg, Exception e)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_loginfo.Info(msg, e);
}
public static void WriteWarn(string msg)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logwarn.Warn(msg);
}
public static void WriteWarn(string msg, Exception e)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logwarn.Warn(msg, e);
}
public static void WriteError(string msg)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logerror.Error(msg);
}
public static void WriteError(string msg, Exception e)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logerror.Error(msg, e);
}
public static void WriteFatal(string msg)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logfatal.Fatal(msg);
}
public static void WriteFatal(string msg, Exception e)
{
StackFrame sf = new StackFrame(1);
msg = string.Format("{0}.{1}:{2}", sf.GetMethod().ReflectedType.FullName, sf.GetMethod().Name, msg);
_logfatal.Fatal(msg, e);
}
}
}
//要求通过newsid=itemid(ES中的)一一对应。
//要求通过newsid=itemid(ES中的)一一对应。
//要求取得以下字段,并返回统一包装的JSON结构。(AnalyzeData只有1个,里面的数据是common+reader(两者其中有值,值也是一样的,任取其一。可能存在相互补充的情况,合并为1个。))
[
{
"ItemID": "文章ID",
"ClickCount": "点击次数",
"Url": "文章Url",
"CleanTitle": "标题",
"CleanText": "正文",
"PubDate": "页面显示发布时间",
"MediaName": "21世纪经济报道",
"DuplicationID": "相似转载ID(早先入库的相似文章ID,第一篇该字段为空,其他均为第一篇的ID)",
"AnalyzeData语义分析结果字段": {
"MarketIDs": " 品类/市场common编号reader编号:有值任取其一",
"BlockIDs": "主题/板块编号common编号,reader编号:有值任取其一",
"StockIDs": "股票编号common编号,reader编号:有值任取其一",
"IG": ""
}
}
]
\ No newline at end of file
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CsvCount_ES
{
public class PerformanceUtility
{
public static readonly TimeSpan DefaultTriggerTimeSpan = TimeSpan.FromMinutes(5);
public static readonly TimeSpan DefaultIntervalTimeSpan = TimeSpan.FromMinutes(2);
public static T OperationTimedMonitor<T>(TimeSpan trigger, TimeSpan interval, String message, Func<T> operation)
{
Stopwatch watch = new Stopwatch();
watch.Start();
Task<T> task = Task.Run<T>(operation);
Task waitForTrigger = Task.Delay(trigger);
Task.WaitAny(task, waitForTrigger);
if (task.IsCompleted)
{
return task.Result;
}
else
{
String guid = Guid.NewGuid().ToString();
LogService.WriteWarn(String.Format("Operation {0} timeout triggered, Message {1}, Time elasped {2}", guid, message, watch.ElapsedMilliseconds));
Task waitInterval = Task.Delay(interval);
Task.WaitAny(task, waitInterval);
while (!task.IsCompleted)
{
LogService.WriteWarn(String.Format("Operation {0} interval triggered, Time elasped {1}", guid, watch.ElapsedMilliseconds));
waitInterval = Task.Delay(interval);
Task.WaitAny(task, waitInterval);
}
return task.Result;
}
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CsvCount_ES.ResultModel
{
[Serializable()]
public class ResultItem
{
public ResultItem()
{
}
/// <summary>
/// 文章ID
/// </summary>
public string ItemID { get; set; }
/// <summary>
/// 点击的次数
/// </summary>
public int ClickCount { get; set; }
/// <summary>
/// 文章Url
/// </summary>
public string Url { get; set; }
/// <summary>
/// 标题
/// </summary>
public string CleanTitle { get; set; }
/// <summary>
/// 正文
/// </summary>
public string CleanText { get; set; }
/// <summary>
/// 页面显示发布时间(格式化的)
/// </summary>
public string PubDate { get; set; }
/// <summary>
/// 媒体名称(分词索引)
/// </summary>
public string MediaName { get; set; }
/// <summary>
/// 相似转载ID(早先入库的相似文章ID,第一篇该字段为空,其他均为第一篇的ID)
/// </summary>
public string DuplicationID { get; set; }
///<summary>
///合并模型
///</summary>
public ResultItemAnalyzeData AnalyzeData { get; set; }
}
/// <summary>
/// 语义分析结果(每Issue可不同)
/// </summary>
[Serializable]
public class ResultItemAnalyzeData
{
/// <summary>
/// 品类/市场编号
/// </summary>
public string[] MarketIDs { set; get; }
/// <summary>
/// 主题/板块编号
/// </summary>
public string[] BlockIDs { set; get; }
/// <summary>
/// 股票编号
/// </summary>
public string[] StockIDs { set; get; }
/// <summary>
/// IG Tree
/// </summary>
public string IG { set; get; }
public ResultItemAnalyzeData()
{
}
}
}
\ No newline at end of file
......@@ -35,6 +35,19 @@ namespace CsvCount_ES
//if (!String.IsNullOrWhiteSpace(textBox1.Text))
//{
//Int64 length = 10240;//每次块大小 512000byte 500KB 这样1个G转2010次算完
// //10兆字节(mb)=10485760字节(b)
// //UTF-8编码中,一个英文字符等于一个字节,一个中文(含繁体)等于三个字节。 本次文件中主要是英文和数字,没有中文。所以要多少字就配多大块。
// //50千字节(kb)=51200字节(b)
// //200千字节(kb)=204800字节(b)
//if (String.IsNullOrWhiteSpace(textBox_block.Text) || !Int64.TryParse(textBox_block.Text, out length))
//{
// MessageBox.Show("请正确输入分块并行读取(B)");
// return;
//}
// #region 执行内存映射方案
// label_wait.Text = "正在执行分析,请耐心等待........!";
......@@ -74,7 +87,7 @@ namespace CsvCount_ES
// //int offset = 0;
// //int length = 512000;//每次块大小 512000byte 500KB 这样1个G转2010次算完
// ////UTF-8编码中,一个英文字符等于一个字节,一个中文(含繁体)等于三个字节。 本次文件中主要是英文和数字,没有中文。所以要多少字就配多大块。
// //string fp2 = @filepath;
// //using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(fp2, FileMode.Open, "mapname"))
......@@ -416,7 +429,7 @@ namespace CsvCount_ES
////listBox1.Enabled = false;
//textBox1.Enabled = true;
//button1.Enabled = true;
//textBox_block.Enabled = true;
//int total = 0;
//foreach (var item in totalCountDic)
......
This source diff could not be displayed because it is too large. You can view the blob instead.
<?xml version="1.0" encoding="utf-8" ?>
<log4net>
<appender name="errorAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="ERROR" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\err.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="infoAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\info.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="debugAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="DEBUG" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\debug.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level %logger - %message%newline" />
</layout>
</appender>
<appender name="perfAppender" type="log4net.Appender.RollingFileAppender">
<filter type="log4net.Filter.LevelMatchFilter">
<levelToMatch value="INFO" />
</filter>
<filter type="log4net.Filter.DenyAllFilter" />
<file value="Logs\perf.log" />
<encoding value="utf-8"/>
<preserveLogFileNameExtension value="true" />
<appendToFile value="true" />
<rollingStyle value="Date" />
<datePattern value="yyyyMMdd" />
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date %logger - %message%newline" />
</layout>
</appender>
<root>
<level value="ALL" />
<appender-ref ref="errorAppender" />
<appender-ref ref="infoAppender" />
<appender-ref ref="debugAppender" />
</root>
<logger name="Performance" additivity="false">
<level value="ALL" />
<appender-ref ref="perfAppender" />
</logger>
</log4net>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<configSections>
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
<sectionGroup name="common">
<section name="logging" type="Common.Logging.ConfigurationSectionHandler, Common.Logging" />
</sectionGroup>
</configSections>
<connectionStrings>
<add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
</connectionStrings>
<common>
<logging>
<factoryAdapter type="Common.Logging.Log4Net.Log4NetLoggerFactoryAdapter, Common.Logging.Log4Net208">
<!--FILE
FILE-WATCH log4net节点在其他独立文件内部。
INLINE log4net节点在配置app.config内部。
EXTERNAL-->
<arg key="configType" value="FILE-WATCH" />
<arg key="configFile" value="~/Config/log4net.config" />
</factoryAdapter>
</logging>
</common>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<configSections>
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
<sectionGroup name="common">
<section name="logging" type="Common.Logging.ConfigurationSectionHandler, Common.Logging" />
</sectionGroup>
</configSections>
<connectionStrings>
<add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
</connectionStrings>
<common>
<logging>
<factoryAdapter type="Common.Logging.Log4Net.Log4NetLoggerFactoryAdapter, Common.Logging.Log4Net208">
<!--FILE
FILE-WATCH log4net节点在其他独立文件内部。
INLINE log4net节点在配置app.config内部。
EXTERNAL-->
<arg key="configType" value="FILE-WATCH" />
<arg key="configFile" value="~/Config/log4net.config" />
</factoryAdapter>
</logging>
</common>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
</configuration>
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
//要求通过newsid=itemid(ES中的)一一对应。
//要求通过newsid=itemid(ES中的)一一对应。
//要求取得以下字段,并返回统一包装的JSON结构。(AnalyzeData只有1个,里面的数据是common+reader(两者其中有值,值也是一样的,任取其一。可能存在相互补充的情况,合并为1个。))
[
{
"ItemID": "文章ID",
"ClickCount": "点击次数",
"Url": "文章Url",
"CleanTitle": "标题",
"CleanText": "正文",
"PubDate": "页面显示发布时间",
"MediaName": "21世纪经济报道",
"DuplicationID": "相似转载ID(早先入库的相似文章ID,第一篇该字段为空,其他均为第一篇的ID)",
"AnalyzeData语义分析结果字段": {
"MarketIDs": " 品类/市场common编号reader编号:有值任取其一",
"BlockIDs": "主题/板块编号common编号,reader编号:有值任取其一",
"StockIDs": "股票编号common编号,reader编号:有值任取其一",
"IG": ""
}
}
]
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -15,3 +15,32 @@ C:\Users\admin\Desktop\CsvCount_ES\CsvCount_ES\bin\Debug\CsvCount_ES.pdb
C:\Users\admin\Desktop\CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.Form1.resources
C:\Users\admin\Desktop\CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.Properties.Resources.resources
C:\Users\admin\Desktop\CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.csproj.GenerateResource.Cache
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Config\log4net.config
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\OutPut\模板.json
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\CsvCount_ES.exe.config
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\CsvCount_ES.exe
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\CsvCount_ES.pdb
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Core.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Log4Net208.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Elasticsearch.Net.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\log4net.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\MongoDB.Bson.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Nest.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Newtonsoft.Json.dll
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.pdb
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Core.pdb
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Core.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Log4Net208.pdb
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Common.Logging.Log4Net208.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Elasticsearch.Net.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\log4net.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\MongoDB.Bson.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Nest.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\bin\Debug\Newtonsoft.Json.xml
D:\smallproject\6.CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.Form1.resources
D:\smallproject\6.CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.Properties.Resources.resources
D:\smallproject\6.CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.csproj.GenerateResource.Cache
D:\smallproject\6.CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.exe
D:\smallproject\6.CsvCount_ES\CsvCount_ES\obj\Debug\CsvCount_ES.pdb
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Common.Logging" version="3.4.1" targetFramework="net45" />
<package id="Common.Logging.Core" version="3.4.1" targetFramework="net45" />
<package id="Common.Logging.Log4Net208" version="3.4.1" targetFramework="net45" />
<package id="Elasticsearch.Net" version="2.5.8" targetFramework="net45" />
<package id="log4net" version="2.0.8" targetFramework="net45" />
<package id="Microsoft.CSharp" version="4.0.1" targetFramework="net45" />
<package id="MongoDB.Bson" version="2.0.0" targetFramework="net45" />
<package id="NEST" version="2.5.8" targetFramework="net45" />
<package id="Newtonsoft.Json" version="11.0.1" targetFramework="net45" />
</packages>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment