Skip to content

S
smallproject
Commit 10ef080b by mahaisong
Options

fix:加入mysql，标记上一次执行成功的日期。（解决由停电引起的数据丢失）

parent 4284c680
Showing with 102 additions and 11 deletions
10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/App.config
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -5,6 +5,8 @@
<sectionGroup name="common">
<section name="logging" type="Common.Logging.ConfigurationSectionHandler, Common.Logging" />
</sectionGroup>
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<common>
<logging>
......@@ -35,15 +37,31 @@
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="MySql.Data" publicKeyToken="c5687fc88969c44d" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.9.12.0" newVersion="6.9.12.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
<connectionStrings>
<add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />
<add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
<add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
</connectionStrings>
</configuration>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
<parameters>
<parameter value="v13.0" />
</parameters>
</defaultConnectionFactory>
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="MySql.Data.MySqlClient" type="MySql.Data.MySqlClient.MySqlProviderServices, MySql.Data.Entity.EF6, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d"></provider></providers>
</entityFramework>
<system.data>
<DbProviderFactories>
<remove invariant="MySql.Data.MySqlClient" />
<add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
</DbProviderFactories>
</system.data></configuration>
\ No newline at end of file
10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/QuartzJobs/TaikorESJob.cs
using HTCommon.Data;
using HTCommon.Data;
......@@ -2,10 +2,12 @@
using HTCommon.DataAccess;
using MinderESCommon;
using Nest;
using PalasEntityModel;
using Quartz;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data.Entity.Migrations;
using System.IO;
using System.Linq;
using System.Text;
......@@ -17,6 +19,7 @@ namespace TaikorES_MinderES.QuartzJobs
public class TaikorESJob : IJob
{
private static string sync_name = "TaikorNews_Taikor";
static Int32 Query_Count = 500;
public void Execute(IJobExecutionContext context)
{
......@@ -62,11 +65,42 @@ namespace TaikorES_MinderES.QuartzJobs
return;
}
//得到上次最后1个itemid
DateTime? QueryTime;
//首先判断上次抓取的编号
SyncOffset _syncState = null;
//没有，则创建新的
using (PalasEntityContext palasEntityContext = new PalasEntityContext())
{
_syncState = palasEntityContext.SyncOffset.Where(f => f.source_name == sync_name).FirstOrDefault();
}
//得到上次最后1个itemid
if (_syncState != null && null != _syncState.last_pubdate && !string.IsNullOrWhiteSpace(_syncState.last_pubdate.ToString()))
{
QueryTime = _syncState.last_pubdate;
}
else
{
//查询范围：昨天的0点，到昨天的23点59分59秒。
DateTime TodayQueryDate = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 0:0:0.000").AddDays(-1);
//当为空时,创建一下
_syncState = new SyncOffset();
_syncState.last_itemid = "0";
_syncState.source_name = "sync_name";
_syncState.create_date = DateTime.Now;
QueryTime = TodayQueryDate;
_syncState.last_pubdate = TodayQueryDate;
}
DateTime TodayEndTime = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 23:59:59.999").AddDays(-1);
bool essuccess = false;
foreach (var item in MatchCrawlID)
{
......@@ -78,7 +112,7 @@ namespace TaikorES_MinderES.QuartzJobs
int tryCount = 0;
string todayitemID = "";//按照同一fetchTime时间、按照todayitemID号排序，则查询时也要注意大于上次的按照todayitemID号排序号。
string crawlID = item.Value.ToString();
DateTime beginTime = TodayQueryDate;
DateTime beginTime = (DateTime)QueryTime;
//goto
TagToday:
......@@ -147,6 +181,9 @@ namespace TaikorES_MinderES.QuartzJobs
string msg = string.Format("Success sync TaikorES news at {0}, this time index {1} items.", DateTime.Now, TaiKorNewsList.Count);
LogService.WriteInfo(msg);
//Console.WriteLine(msg);
essuccess = true;
}
if (null != TaiKorNewsList && TaiKorNewsList.Count == Query_Count)
{
......@@ -173,9 +210,21 @@ namespace TaikorES_MinderES.QuartzJobs
}
if (essuccess)
{
_syncState.modify_date = DateTime.Now;
_syncState.last_pubdate = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 0:0:0.000");
using (PalasEntityContext htItemContext = new PalasEntityContext())
{
htItemContext.SyncOffset.AddOrUpdate(_syncState);
htItemContext.SaveChanges();
}
}
//失败--重试3次
LogService.WriteInfo("完成：TaikorES_MinderES:"+ totalcount+"条;" + TodayQueryDate.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
Console.WriteLine("完成：TaikorES_MinderES:" + totalcount+"条;"+ TodayQueryDate.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
LogService.WriteInfo("完成：TaikorES_MinderES:" + totalcount + "条;" + QueryTime.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
Console.WriteLine("完成：TaikorES_MinderES:" + totalcount + "条;" + QueryTime.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
}
catch (Exception ex)
......
10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/TaikorES_MinderES.csproj
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -50,10 +50,26 @@
<HintPath>..\packages\Elasticsearch.Net.2.5.8\lib\net45\Elasticsearch.Net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="log4net, Version=2.0.8.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
<HintPath>..\packages\log4net.2.0.8\lib\net45-full\log4net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MySql.Data, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
<HintPath>..\packages\MySql.Data.6.9.12\lib\net45\MySql.Data.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MySql.Data.Entity.EF6, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
<HintPath>..\packages\MySql.Data.Entity.6.9.12\lib\net45\MySql.Data.Entity.EF6.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Nest, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\NEST.2.5.8\lib\net45\Nest.dll</HintPath>
<Private>True</Private>
......@@ -62,11 +78,15 @@
<HintPath>..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="PalasEntityModel">
<HintPath>..\include\PalasEntityModel.dll</HintPath>
</Reference>
<Reference Include="Quartz, Version=2.6.0.0, Culture=neutral, PublicKeyToken=f6b8c98a402cc8a4, processorArchitecture=MSIL">
<HintPath>..\packages\Quartz.2.6.0\lib\net40\Quartz.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
......
10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/packages.config
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -4,8 +4,11 @@
<package id="Common.Logging.Core" version="3.4.1" targetFramework="net452" />
<package id="Common.Logging.Log4Net208" version="3.4.1" targetFramework="net452" />
<package id="Elasticsearch.Net" version="2.5.8" targetFramework="net452" />
<package id="EntityFramework" version="6.0.0" targetFramework="net452" />
<package id="log4net" version="2.0.8" targetFramework="net452" />
<package id="Microsoft.CSharp" version="4.0.1" targetFramework="net452" />
<package id="MySql.Data" version="6.9.12" targetFramework="net452" />
<package id="MySql.Data.Entity" version="6.9.12" targetFramework="net452" />
<package id="NEST" version="2.5.8" targetFramework="net452" />
<package id="Newtonsoft.Json" version="11.0.1" targetFramework="net452" />
<package id="Quartz" version="2.6.0" targetFramework="net452" />
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment