Skip to content

S
smallproject
Commit e90b4408 by mahaisong
Options

fix：修复没有分支错误

parent 67416fd4
Showing with 255 additions and 2 deletions
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/App.config
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
...@@ -48,7 +48,13 @@ ...@@ -48,7 +48,13 @@
</runtime> </runtime>
<connectionStrings> <connectionStrings>
<add name="HJBeigeContext" providerName="Oracle.ManagedDataAccess.Client" connectionString="User Id=bigdata_admin;Password=bigdata_admin;Data Source=BEIGE_ORCL" /> <add name="HJBeigeContext" providerName="Oracle.ManagedDataAccess.Client" connectionString="User Id=bigdata_admin;Password=bigdata_admin;Data Source=BEIGE_ORCL" />
<<<<<<< master
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" /> <add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />
=======
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palasitems_v1;requesttimeout=30000" />
<!--<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />-->
>>>>>>> local
<add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" /> <add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
<add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" /> <add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
</connectionStrings> </connectionStrings>
......
10.Paul需求-JSON贝格ES导入本地ES/MinderES/TaikorES_MinderES/App.config
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
...@@ -45,7 +45,13 @@ ...@@ -45,7 +45,13 @@
</runtime> </runtime>
<connectionStrings> <connectionStrings>
<add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" /> <add name="ESDatabase" connectionString="host=mech.palaspom.com|tank.palaspom.com;port=19235;defaultIndex=palas" />
<<<<<<< master
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" /> <add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />
=======
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palasitems_v1;requesttimeout=30000" />
<!--<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />-->
>>>>>>> local
<add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" /> <add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
<add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" /> <add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
</connectionStrings> </connectionStrings>
......
10.Paul需求-JSON贝格ES导入本地ES/旧ES导入新ES/Program(keyword及from用法).cs 0 → 100644
using HJBeigeModel;
using HJBeigeModel;
using HTCommon.Data;
using HTCommon.DataAccess;
using HTCommon.Helper;
using MinderESCommon;
using Nest;
using Oracle.ManagedDataAccess.Client;
using PalasEntityModel;
using Quartz;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Entity.Migrations;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace BeigeOracle_MinderES
{
class Program
{
private static IScheduler scheduler;
private static string wstr = "";
static void Main(string[] args)
{
try
{
Console.Title = "BeigeOracle数据定时插入Minder机器ES";//设置窗口标题
Console.WindowWidth = 120;//一屏幕宽120个字，多了换行
Console.WindowHeight = 34;//一屏幕最多34行，多了就滚动
Console.BufferWidth = 120; //一行内容120个字，多了换行。 如果BufferWidth比WindowWidth大，则代表可以水平向后滚动
Console.BufferHeight = 9999; //最大高度
Console.BackgroundColor = ConsoleColor.DarkYellow; //设置背景色 黄
//开启quartz
wstr = DateTimeTool.ConvertDataTimeToString(DateTime.Now) + "： 1.程序启动";
Console.WriteLine(wstr);
LogService.WriteInfo(wstr);
try
{
Int32 Query_Count = 1000;
string crawlID = "Oracle_Beige_NEWS";
int totalcount = 0;
//得到上次最后1个itemid
long lastId = 0;
int tryCount = 0;
int fromcount = 0;
bool iscatch = false;
//goto
TagToday:
if (lastId > 0)
{
if (!iscatch)
{
fromcount += 1000;
}
iscatch = false;
}
GC.Collect();
IList<Item> BeigeNewsList = null;
try
{
//当前爬虫、指定的数据，且排除上一次的最后一条。
var search = new SearchDescriptor<Item>();
search.Query(q => q.Bool(b => b.Must(m => m.Term(t => t.Field(tf => tf.CrawlID.Suffix("keyword")).Value(crawlID)),
m2 => m2.Range(r => r.Field(f => f.ClientItemID.Suffix("keyword")).GreaterThanOrEquals(0)))))
.From(fromcount) //跳过的数据个数
.Size(Query_Count) //返回数据个数
.Sort(st => st.Ascending(asc => asc.ClientItemID.Suffix("keyword")));//排序
BeigeNewsList = ESAccess.Search_SearchDetail<Item>(search).ToList<Item>();
Console.WriteLine("查询：数量+"+ BeigeNewsList .Count+ " -" + BeigeNewsList.LastOrDefault().ClientItemID.ToString() + "--！");
}
catch (Exception ex)
{
if (tryCount < 3)
{
tryCount = tryCount + 1;
iscatch = true;
goto TagToday;
}
//失败--重试3次
LogService.WriteError(DateTime.Now.ToString() + "日期查询数据库时重试三次依然错误，无法连接或异常，请检查！" + ex.ToString());
Console.WriteLine(DateTime.Now.ToString() + "日期查询数据库时重试三次依然错误，无法连接或异常，请检查！" + ex.ToString());
GC.Collect();
iscatch = true;
goto TagToday;
}
if (BeigeNewsList != null && BeigeNewsList.Count > 0)
{
try
{
ElasticClient client = new ElasticClient(MinderESClientConfigureManager.ConnectSetting);
var Descriptor = new BulkDescriptor();
foreach (Item Product in BeigeNewsList)
{
Descriptor.Index<Item>(op => op.Document(Product));
}
client.Bulk(Descriptor);
}
catch (Exception ex)
{
Console.WriteLine("ES批量插入报错" + ex.ToString());
}
//foreach (Item newItem in BeigeNewsList)
//{
// try
// {
// MinderESAccess.Index<Item>(newItem, 0);
// }
// catch (Exception ex)
// {
// Console.WriteLine("Import to ES error! " + ex.ToString());
// }
//}
}
if (null != BeigeNewsList && BeigeNewsList.Count == Query_Count)
{
//认为当天内还有其他记录，需继续执行。
tryCount = 0;
lastId = long.Parse(BeigeNewsList.LastOrDefault().ClientItemID.ToString());
Console.WriteLine("进度： - lastId:" + lastId + ",数据均以导入到MinderES中，请检查！");
goto TagToday;
}
LogService.WriteInfo("完成：BeigeOracle_MinderES:" + totalcount + "条;" + DateTime.Now.ToString() + " -lastId: " + lastId + ",数据均以导入到MinderES中，请检查！");
Console.WriteLine("完成：BeigeOracle_MinderES:" + totalcount + "条;" + DateTime.Now.ToString() + " - lastId:" + lastId + ",数据均以导入到MinderES中，请检查！");
}
catch (Exception ex)
{
LogService.WriteError(ex.ToString());
}
wstr = DateTimeTool.ConvertDataTimeToString(DateTime.Now) + "： 2.定时任务启动--------若要关闭本程序，请输入exit！";
Console.WriteLine(wstr);
LogService.WriteInfo(wstr);
Console.WriteLine(DateTimeTool.ConvertDataTimeToString(DateTime.Now) + "： 2.1定时任务执行日志，请观察debug.log文件");
}
catch (Exception ex)
{
Console.BackgroundColor = ConsoleColor.Red;
Console.ForegroundColor = ConsoleColor.Cyan;
wstr = DateTimeTool.ConvertDataTimeToString(DateTime.Now) + "： 启动失败,定时任务自动停止！！！错误如下：" + ex.ToString();
Console.WriteLine(wstr);
LogService.WriteInfo(wstr);
try
{
//quartz 停止
scheduler.Shutdown();
}
catch { }
Console.BackgroundColor = ConsoleColor.DarkYellow; //设置默认背景色
Console.ForegroundColor = ConsoleColor.Gray; //设置默认背景色
}
finally
{
while (true)
{
string key = Console.ReadLine();
if (key.Trim().ToUpper().Equals("EXIT"))
{
//quartz 停止
scheduler.Shutdown();
break;
}
}
Console.WriteLine("3.程序关闭--手工关闭！");
LogService.WriteInfo("3.程序关闭--手工关闭！");
}
}
//显示出console中支持的背景色及前景色
//ShowColor(); //人工挑选1个需要的文本、背景颜色
//Console.ResetColor(); //将控制台的前景色和背景色设为默认值
//Console.BackgroundColor = ConsoleColor.Blue; //设置背景色 蓝
//Console.ForegroundColor = ConsoleColor.White; //设置前景色，即字体颜色 白
static void ShowColor()
{
Type type = typeof(ConsoleColor);
Console.ForegroundColor = ConsoleColor.White;
foreach (string name in Enum.GetNames(type))
{
Console.BackgroundColor = (ConsoleColor)Enum.Parse(type, name);
Console.WriteLine(name);
}
Console.BackgroundColor = ConsoleColor.Black;
foreach (string name in Enum.GetNames(type))
{
Console.ForegroundColor = (ConsoleColor)Enum.Parse(type, name);
Console.WriteLine(name);
}
foreach (string bc in Enum.GetNames(type))
{
Console.BackgroundColor = (ConsoleColor)Enum.Parse(type, bc);
foreach (string fc in Enum.GetNames(type))
{
Console.ForegroundColor = (ConsoleColor)Enum.Parse(type, fc);
Console.WriteLine("bc=" + bc + ",fc=" + fc);
}
Console.WriteLine();
}
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment