fix:加入mysql，标记上一次执行成功的日期。（解决由停电引起的数据丢失）

10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/App.config

-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
@@ -5,7 +5,9 @@
     <sectionGroup name="common">
       <section name="logging" type="Common.Logging.ConfigurationSectionHandler, Common.Logging" />
     </sectionGroup>
-    </configSections>
+    <!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
+    <section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
+  </configSections>
   <common>
     <logging>
       <factoryAdapter type="Common.Logging.Log4Net.Log4NetLoggerFactoryAdapter, Common.Logging.Log4Net208">
@@ -18,9 +20,9 @@
       </factoryAdapter>
     </logging>
   </common>
-    <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
-    </startup>
+  <startup>
+    <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
+  </startup>
   <runtime>
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
@@ -35,15 +37,31 @@
         <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
         <bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="MySql.Data" publicKeyToken="c5687fc88969c44d" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.9.12.0" newVersion="6.9.12.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <connectionStrings>
     <add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
-
-
     <add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />
     <add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
     <add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
- </connectionStrings>
-  
-</configuration>
+  </connectionStrings>
+  <entityFramework>
+    <defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
+      <parameters>
+        <parameter value="v13.0" />
+      </parameters>
+    </defaultConnectionFactory>
+    <providers>
+      <provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
+    <provider invariantName="MySql.Data.MySqlClient" type="MySql.Data.MySqlClient.MySqlProviderServices, MySql.Data.Entity.EF6, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d"></provider></providers>
+  </entityFramework>
+<system.data>
+    <DbProviderFactories>
+      <remove invariant="MySql.Data.MySqlClient" />
+      <add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
+    </DbProviderFactories>
+  </system.data></configuration>
\ No newline at end of file

10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/QuartzJobs/TaikorESJob.cs

-using HTCommon.Data;
+using HTCommon.Data;
@@ -2,10 +2,12 @@
 using HTCommon.DataAccess;
 using MinderESCommon;
 using Nest;
+using PalasEntityModel;
 using Quartz;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Data.Entity.Migrations;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -17,6 +19,7 @@ namespace TaikorES_MinderES.QuartzJobs
 
     public class TaikorESJob : IJob
     {
+        private static string sync_name = "TaikorNews_Taikor";
         static Int32 Query_Count = 500;
         public void Execute(IJobExecutionContext context)
         {
@@ -62,11 +65,42 @@ namespace TaikorES_MinderES.QuartzJobs
                     return;
                 }
 
-                //查询范围：昨天的0点，到昨天的23点59分59秒。 
-                DateTime TodayQueryDate = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 0:0:0.000").AddDays(-1);
+                //得到上次最后1个itemid
+
+                DateTime? QueryTime;
+                //首先判断上次抓取的编号
+                SyncOffset _syncState = null;
+                //没有，则创建新的
+                using (PalasEntityContext palasEntityContext = new PalasEntityContext())
+                {
+                    _syncState = palasEntityContext.SyncOffset.Where(f => f.source_name == sync_name).FirstOrDefault();
+                }
+
+
+                //得到上次最后1个itemid 
+                if (_syncState != null && null != _syncState.last_pubdate && !string.IsNullOrWhiteSpace(_syncState.last_pubdate.ToString()))
+                {
+                    QueryTime = _syncState.last_pubdate;
+                }
+                else
+                {
+
+                    //查询范围：昨天的0点，到昨天的23点59分59秒。 
+                    DateTime TodayQueryDate = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 0:0:0.000").AddDays(-1);
+
+                    //当为空时,创建一下
+                    _syncState = new SyncOffset();
+                    _syncState.last_itemid = "0";
+                    _syncState.source_name = "sync_name";
+                    
+                    _syncState.create_date = DateTime.Now;
+                    QueryTime = TodayQueryDate;
+                    _syncState.last_pubdate = TodayQueryDate;
+                }
 
                 DateTime TodayEndTime = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 23:59:59.999").AddDays(-1);
 
+                bool essuccess = false;
 
                 foreach (var item in MatchCrawlID)
                 {
@@ -78,7 +112,7 @@ namespace TaikorES_MinderES.QuartzJobs
                         int tryCount = 0;
                         string todayitemID = "";//按照同一fetchTime时间、按照todayitemID号排序，则查询时也要注意大于上次的按照todayitemID号排序号。
                         string crawlID = item.Value.ToString();
-                        DateTime beginTime = TodayQueryDate;
+                        DateTime beginTime = (DateTime)QueryTime;
                         //goto
                         TagToday:
 
@@ -147,6 +181,9 @@ namespace TaikorES_MinderES.QuartzJobs
                             string msg = string.Format("Success sync TaikorES news at {0}, this time index {1} items.", DateTime.Now, TaiKorNewsList.Count);
                             LogService.WriteInfo(msg);
                             //Console.WriteLine(msg);
+
+                            essuccess = true;
+
                         }
                         if (null != TaiKorNewsList && TaiKorNewsList.Count == Query_Count)
                         {
@@ -170,12 +207,24 @@ namespace TaikorES_MinderES.QuartzJobs
                         Console.WriteLine("列表中第" + item.Key.ToString() + "个爬虫:" + item.Value.ToString() + "，执行时错误！" + ex.ToString() + "数据日期为：" + TodayEndTime.ToString("yyyy-MM-dd"));
 
                     }
-
+                
                 }
 
+                if (essuccess)
+                {
+                    _syncState.modify_date = DateTime.Now;
+                    _syncState.last_pubdate = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd") + " 0:0:0.000");
+
+                    using (PalasEntityContext htItemContext = new PalasEntityContext())
+                    {
+                        htItemContext.SyncOffset.AddOrUpdate(_syncState);
+                        htItemContext.SaveChanges();
+                    }
+                }
+                 
                 //失败--重试3次
-                LogService.WriteInfo("完成：TaikorES_MinderES:"+ totalcount+"条;" + TodayQueryDate.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
-                Console.WriteLine("完成：TaikorES_MinderES:" + totalcount+"条;"+ TodayQueryDate.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
+                LogService.WriteInfo("完成：TaikorES_MinderES:" + totalcount + "条;" + QueryTime.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
+                Console.WriteLine("完成：TaikorES_MinderES:" + totalcount + "条;" + QueryTime.ToString() + " - " + TodayEndTime.ToString() + "4W爬虫抓取的所有数据均以导入到MinderES中，请检查！");
 
             }
             catch (Exception ex)

10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/TaikorES_MinderES.csproj

-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
@@ -50,10 +50,26 @@
       <HintPath>..\packages\Elasticsearch.Net.2.5.8\lib\net45\Elasticsearch.Net.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
+      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
+      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="log4net, Version=2.0.8.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
       <HintPath>..\packages\log4net.2.0.8\lib\net45-full\log4net.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="MySql.Data, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
+      <HintPath>..\packages\MySql.Data.6.9.12\lib\net45\MySql.Data.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="MySql.Data.Entity.EF6, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
+      <HintPath>..\packages\MySql.Data.Entity.6.9.12\lib\net45\MySql.Data.Entity.EF6.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="Nest, Version=2.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
       <HintPath>..\packages\NEST.2.5.8\lib\net45\Nest.dll</HintPath>
       <Private>True</Private>
@@ -62,11 +78,15 @@
       <HintPath>..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="PalasEntityModel">
+      <HintPath>..\include\PalasEntityModel.dll</HintPath>
+    </Reference>
     <Reference Include="Quartz, Version=2.6.0.0, Culture=neutral, PublicKeyToken=f6b8c98a402cc8a4, processorArchitecture=MSIL">
       <HintPath>..\packages\Quartz.2.6.0\lib\net40\Quartz.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.ComponentModel.DataAnnotations" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />

10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/packages.config

-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
@@ -4,8 +4,11 @@
   <package id="Common.Logging.Core" version="3.4.1" targetFramework="net452" />
   <package id="Common.Logging.Log4Net208" version="3.4.1" targetFramework="net452" />
   <package id="Elasticsearch.Net" version="2.5.8" targetFramework="net452" />
+  <package id="EntityFramework" version="6.0.0" targetFramework="net452" />
   <package id="log4net" version="2.0.8" targetFramework="net452" />
   <package id="Microsoft.CSharp" version="4.0.1" targetFramework="net452" />
+  <package id="MySql.Data" version="6.9.12" targetFramework="net452" />
+  <package id="MySql.Data.Entity" version="6.9.12" targetFramework="net452" />
   <package id="NEST" version="2.5.8" targetFramework="net452" />
   <package id="Newtonsoft.Json" version="11.0.1" targetFramework="net452" />
   <package id="Quartz" version="2.6.0" targetFramework="net452" />