Commit 21ebca38 by mahaisong

feat:开发周期性程序

parent f96a5eac


Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.25420.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinderESFromTaikorES", "MinderESFromTaikorES\MinderESFromTaikorES.csproj", "{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinderESFromBeigeOracle", "MinderESFromBeigeOracle\MinderESFromBeigeOracle.csproj", "{016A1AFD-0237-4251-B12C-793963C535CB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinderESCommon", "MinderESCommon\MinderESCommon.csproj", "{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Debug|x64.ActiveCfg = Debug|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Debug|x64.Build.0 = Debug|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Release|Any CPU.Build.0 = Release|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Release|x64.ActiveCfg = Release|Any CPU
{594EEA7B-46C2-41A4-B7B8-57298B15B5D7}.Release|x64.Build.0 = Release|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Debug|x64.ActiveCfg = Debug|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Debug|x64.Build.0 = Debug|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Release|Any CPU.Build.0 = Release|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Release|x64.ActiveCfg = Release|Any CPU
{016A1AFD-0237-4251-B12C-793963C535CB}.Release|x64.Build.0 = Release|Any CPU
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Debug|x64.ActiveCfg = Debug|x64
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Debug|x64.Build.0 = Debug|x64
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Release|Any CPU.Build.0 = Release|Any CPU
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Release|x64.ActiveCfg = Release|x64
{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
using Elasticsearch.Net;
using Elasticsearch.Net;
using Nest;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Text.RegularExpressions;
namespace MinderESCommon
{
public sealed class MinderESClientConfigureManager
{
public static List<Uri> Uris { get; private set; }
public static String DefaultIndex { get; private set; }
/// <summary>
/// ES连接设置
/// </summary>
public static ConnectionSettings ConnectSetting { get; private set; }
private const String ConnectionStringIndex = "MinderESDatabase";
private const String HostIndex = "host";
private const String PortIndex = "port";
private const String DefaultIndexIndex = "defaultIndex";
static MinderESClientConfigureManager()
{
String connectionString = ConfigurationManager.ConnectionStrings[ConnectionStringIndex].ConnectionString;
String[] namedValues = connectionString.Split(';');
Dictionary<String, String> nvPair = new Dictionary<String, String>();
foreach (String nv in namedValues)
{
String[] pair = nv.Split('=');
nvPair[pair[0]] = pair[1];
}
String _url = GetValueByName(nvPair, HostIndex, "minder");
String _port = GetValueByName(nvPair, PortIndex, "9200");
//支持多Uri模式,要求多个链接使用 | 分隔,在分隔后数量相同的情况下按照顺序一一对应,如果其中一方仅有一个则使用它对应对方所有部分
string[] urls = _url.Split('|');
string[] ports = _port.Split('|');
if (urls == null || ports == null || urls.Length < 1 || ports.Length < 1)
throw new Exception("Unable get the elasticsearch config pool setting.");
Uris = new List<Uri>();
if (urls.Length == ports.Length)
{
for (int i = 0; i < urls.Length; i++)
{
String formattedUrl = Regex.IsMatch(urls[i], @"://") ? urls[i] : String.Format("http://{0}", urls[i]);
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[i])));
}
}
else if (urls.Length > 1 && ports.Length == 1)
{
for (int i = 0; i < urls.Length; i++)
{
String formattedUrl = Regex.IsMatch(urls[i], @"://") ? urls[i] : String.Format("http://{0}", urls[i]);
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[0])));
}
}
else if (urls.Length == 1 && ports.Length > 1)
{
String formattedUrl = Regex.IsMatch(urls[0], @"://") ? urls[0] : String.Format("http://{0}", urls[0]);
for (int i = 0; i < ports.Length; i++)
{
Uris.Add(new Uri(String.Format("{0}:{1}", formattedUrl, ports[i])));
}
}
else
{
throw new Exception("The elasticsearch config pool setting is error.");
}
var pool = new StaticConnectionPool(Uris);
String index = GetValueByName(nvPair, DefaultIndexIndex, "palas_test");
DefaultIndex = index;
//ConnectSetting = new ConnectionSettings(pool).BasicAuthentication("carey", "jfyhdcm").DefaultIndex(index).DisableDirectStreaming();
ConnectSetting = new ConnectionSettings(pool).DefaultIndex(index).DisableDirectStreaming();
}
private static String GetValueByName(Dictionary<String, String> nvPair, String name, String defaultValue)
{
String value = null;
if (nvPair.TryGetValue(name, out value))
{
return value;
}
return defaultValue;
}
}
}


using Palas.Common.Data;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace MinderESCommon
{
public static class ErrorItemCache
{
/// <summary>
/// 插入出错的itemid
/// </summary>
/// <remarks>线程安全</remarks>
public static ConcurrentDictionary<string, Item> ErrorItemMap = new ConcurrentDictionary<string, Item>();
}
}
using System;
using System;
using System.Security.Cryptography;
using System.Text;
namespace MinderESCommon
{
public class MD5Helper
{
// Hash an input string and return the hash as
// a 32 character hexadecimal string.
public static string getMd5Hash(string input, bool ignore = true)
{
if (String.IsNullOrEmpty(input))
{
return null;
}
//处理百度贴吧回帖的问题以及宝宝树的回帖问题
if (ignore && ((input.Contains("tieba") || input.Contains("bbs.pcbaby.com.cn")) && input.Contains("?")))
input = input.Substring(0, input.LastIndexOf("?"));
var benchStr = input.Trim();
// Create a new instance of the MD5CryptoServiceProvider object.
MD5 md5Hasher = MD5.Create();
// Convert the input string to a byte array and compute the hash.
byte[] data = md5Hasher.ComputeHash(Encoding.UTF8.GetBytes(benchStr));
// Create a new Stringbuilder to collect the bytes
// and create a string.
StringBuilder sBuilder = new StringBuilder();
// Loop through each byte of the hashed data
// and format each one as a hexadecimal string.
for (int i = 0; i < data.Length; i++)
{
sBuilder.Append(data[i].ToString("x2"));
}
// Return the hexadecimal string.
return sBuilder.ToString();
}
// Verify a hash against a string.
public static bool verifyMd5Hash(string input, string hash)
{
// Hash the input.
string hashOfInput = getMd5Hash(input);
// Create a StringComparer an comare the hashes.
StringComparer comparer = StringComparer.OrdinalIgnoreCase;
if (0 == comparer.Compare(hashOfInput, hash))
{
return true;
}
else
{
return false;
}
}
}
}
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{B911AAD3-B7B0-4A0A-BAE6-3EAC5DF762C7}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>MinderESCommon</RootNamespace>
<AssemblyName>MinderESCommon</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="Elasticsearch.Net, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\Elasticsearch.Net.2.5.8\lib\net45\Elasticsearch.Net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="HooLab.Log">
<HintPath>..\..\..\..\worksapce\Palas\include\HooLab.Log.dll</HintPath>
</Reference>
<Reference Include="MongoDB.Bson, Version=2.5.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Bson.2.5.0\lib\net45\MongoDB.Bson.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Nest, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\NEST.2.5.8\lib\net45\Nest.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Palas.Common">
<HintPath>..\..\..\..\worksapce\Palas\src\Palas.Common\bin\Debug\Palas.Common.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ErrorItemCache.cs" />
<Compile Include="ESAccess.cs" />
<Compile Include="ESClientConfigureManager.cs" />
<Compile Include="MD5Helper.cs" />
<Compile Include="PerformanceUtility.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ProjectView>ShowAllFiles</ProjectView>
</PropertyGroup>
</Project>
\ No newline at end of file
using System;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace MinderESCommon
{
public class PerformanceUtility
{
public static readonly TimeSpan DefaultTriggerTimeSpan = TimeSpan.FromMinutes(5);
public static readonly TimeSpan DefaultIntervalTimeSpan = TimeSpan.FromMinutes(2);
public static T OperationTimedMonitor<T>(TimeSpan trigger, TimeSpan interval, String message, Func<T> operation)
{
Stopwatch watch = new Stopwatch();
watch.Start();
Task<T> task = Task.Run<T>(operation);
Task waitForTrigger = Task.Delay(trigger);
Task.WaitAny(task, waitForTrigger);
if (task.IsCompleted)
{
return task.Result;
}
else
{
String guid = Guid.NewGuid().ToString();
//LogService.WriteWarn(String.Format("Operation {0} timeout triggered, Message {1}, Time elasped {2}", guid, message, watch.ElapsedMilliseconds));
Task waitInterval = Task.Delay(interval);
Task.WaitAny(task, waitInterval);
while (!task.IsCompleted)
{
//LogService.WriteWarn(String.Format("Operation {0} interval triggered, Time elasped {1}", guid, watch.ElapsedMilliseconds));
waitInterval = Task.Delay(interval);
Task.WaitAny(task, waitInterval);
}
return task.Result;
}
}
}
}
using System.Reflection;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// 有关程序集的一般信息由以下
// 控制。更改这些特性值可修改
// 与程序集关联的信息。
[assembly: AssemblyTitle("MinderESCommon")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("MinderESCommon")]
[assembly: AssemblyCopyright("Copyright © 2018")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
//将 ComVisible 设置为 false 将使此程序集中的类型
//对 COM 组件不可见。 如果需要从 COM 访问此程序集中的类型,
//请将此类型的 ComVisible 特性设置为 true。
[assembly: ComVisible(false)]
// 如果此项目向 COM 公开,则下列 GUID 用于类型库的 ID
[assembly: Guid("b911aad3-b7b0-4a0a-bae6-3eac5df762c7")]
// 程序集的版本信息由下列四个值组成:
//
// 主版本
// 次版本
// 生成号
// 修订号
//
//可以指定所有这些值,也可以使用“生成号”和“修订号”的默认值,
// 方法是按如下所示使用“*”: :
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment