Skip to content

S
smallproject
Commit 77da7bdf by mahaisong
Options

fix:Oracle的组件老是出问题，直接写了转MODEL方式，用ado方式处理。

parent 745f30b6
Showing with 210 additions and 68 deletions
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/App.config
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -8,6 +8,8 @@
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
<section name="oracle.manageddataaccess.client" type="OracleInternal.Common.ODPMSectionHandler, Oracle.ManagedDataAccess, Version=4.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342" />
</configSections>
<common>
<logging>
......@@ -42,11 +44,6 @@
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-11.0.0.0" newVersion="11.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<publisherPolicy apply="no" />
<assemblyIdentity name="Oracle.ManagedDataAccess" publicKeyToken="89b483f429c47342" culture="neutral" />
<bindingRedirect oldVersion="4.121.0.0 - 4.65535.65535.65535" newVersion="4.122.1.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
<connectionStrings>
......@@ -54,8 +51,7 @@
<add name="MinderESDatabase" connectionString="host=minder;port=9200;defaultIndex=palas_test;requesttimeout=30000" />
<add name="PalasV5Context" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
<add name="PalasEntityContext" connectionString="server=minder;user id=palas;password=lapas;persistsecurityinfo=True;database=Palas_V5;Character Set=utf8" providerName="MySql.Data.MySqlClient" />
<add name="OracleDbContext" providerName="Oracle.ManagedDataAccess.Client" connectionString="User Id=oracle_user;Password=oracle_user_password;Data Source=oracle" />
</connectionStrings>
</connectionStrings>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
<parameters>
......@@ -65,22 +61,28 @@
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="MySql.Data.MySqlClient" type="MySql.Data.MySqlClient.MySqlProviderServices, MySql.Data.Entity.EF6, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d"></provider>
<provider invariantName="Oracle.ManagedDataAccess.Client" type="Oracle.ManagedDataAccess.EntityFramework.EFOracleProviderServices, Oracle.ManagedDataAccess.EntityFramework, Version=6.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342" />
<provider invariantName="Oracle.ManagedDataAccess.Client" type="Oracle.ManagedDataAccess.EntityFramework.EFOracleProviderServices, Oracle.ManagedDataAccess.EntityFramework, Version=6.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342" />
</providers>
</entityFramework>
<system.data>
<DbProviderFactories>
<remove invariant="MySql.Data.MySqlClient" />
<add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.12.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
<remove invariant="Oracle.ManagedDataAccess.Client" />
<add name="ODP.NET, Managed Driver" invariant="Oracle.ManagedDataAccess.Client" description="Oracle Data Provider for .NET, Managed Driver" type="Oracle.ManagedDataAccess.Client.OracleClientFactory, Oracle.ManagedDataAccess, Version=4.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342" />
<remove invariant="Oracle.ManagedDataAccess.Client"/>
<add name="ODP.NET, Managed Driver" invariant="Oracle.ManagedDataAccess.Client" description="Oracle Data Provider for .NET, Managed Driver"
type="Oracle.ManagedDataAccess.Client.OracleClientFactory, Oracle.ManagedDataAccess, Version=4.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342"/>
</DbProviderFactories>
</system.data>
<oracle.manageddataaccess.client>
<version number="*">
<dataSources>
<dataSource alias="SampleDataSource" descriptor="(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=localhost)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=ORCL))) " />
</dataSources>
</version>
</oracle.manageddataaccess.client>
<!--<oracle.manageddataaccess.client>
<version number="*">
<dataSources>
<dataSource alias="BEIGE_ORCL" descriptor="(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=140.207.41.22)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=ORCL))) " />
</dataSources>
</version>
</oracle.manageddataaccess.client>-->
</configuration>
\ No newline at end of file
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/BeigeOracle_MinderES.csproj
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -58,6 +58,10 @@
<HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="HJBeigeModel, Version=1.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\ImportLocalES\HJBeigeModel\bin\x64\Debug\HJBeigeModel.dll</HintPath>
</Reference>
<Reference Include="log4net, Version=2.0.8.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
<HintPath>..\packages\log4net.2.0.8\lib\net45-full\log4net.dll</HintPath>
<Private>True</Private>
......@@ -79,12 +83,12 @@
<Private>True</Private>
</Reference>
<Reference Include="Oracle.ManagedDataAccess, Version=4.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342, processorArchitecture=MSIL">
<HintPath>..\packages\Oracle.ManagedDataAccess.12.2.1100\lib\net40\Oracle.ManagedDataAccess.dll</HintPath>
<Private>True</Private>
<SpecificVersion>False</SpecificVersion>
<HintPath>..\include\Oracle.ManagedDataAccess.dll</HintPath>
</Reference>
<Reference Include="Oracle.ManagedDataAccess.EntityFramework, Version=6.122.1.0, Culture=neutral, PublicKeyToken=89b483f429c47342, processorArchitecture=MSIL">
<HintPath>..\packages\Oracle.ManagedDataAccess.EntityFramework.12.2.1100\lib\net45\Oracle.ManagedDataAccess.EntityFramework.dll</HintPath>
<Private>True</Private>
<SpecificVersion>False</SpecificVersion>
<HintPath>..\include\Oracle.ManagedDataAccess.EntityFramework.dll</HintPath>
</Reference>
<Reference Include="PalasEntityModel">
<HintPath>..\include\PalasEntityModel.dll</HintPath>
......@@ -109,6 +113,7 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="QuartzJobs\BeigeESJob.cs" />
<Compile Include="QuartzJobs\TestJob.cs" />
<Compile Include="ReaderToListModel.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config">
......@@ -138,10 +143,6 @@
</ItemGroup>
<ItemGroup />
<ItemGroup>
<ProjectReference Include="..\HJBeigeModel\HJBeigeModel.csproj">
<Project>{0AFE2959-EB8E-4D60-B1B1-AB7616EA319D}</Project>
<Name>HJBeigeModel</Name>
</ProjectReference>
<ProjectReference Include="..\HTCommon\HTCommon.csproj">
<Project>{E497051E-61E1-4247-93D3-7929A8C8B7F6}</Project>
<Name>HTCommon</Name>
......
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/Program.cs
using MinderESCommon;
using MinderESCommon;
......@@ -72,7 +72,8 @@ namespace BeigeOracle_MinderES
}
Console.WriteLine("3.程序关闭！");
Console.WriteLine("3.程序关闭--手工关闭！");
LogService.WriteInfo("3.程序关闭--手工关闭！");
}
}
......
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/QuartzJobs/BeigeESJob.cs
using HJBeigeModel;
using HJBeigeModel;
......@@ -12,6 +12,7 @@ using System.Data;
using System.Data.Entity.Migrations;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
......@@ -20,9 +21,11 @@ namespace BeigeOracle_MinderES.QuartzJobs
public class BeigeESJob : IJob
{
private static string sync_name = "HuaJinNews_Beige";
private static Int32 Query_Count = 500;
private static Int32 Query_Count = 100;
public void Execute(IJobExecutionContext context)
{
try
{
int totalcount = 0;
......@@ -44,7 +47,7 @@ namespace BeigeOracle_MinderES.QuartzJobs
long.TryParse(_syncState.last_itemid, out lastId);
}
else
{
{
//当为空时,创建一下
_syncState = new SyncOffset();
_syncState.last_itemid = "0";
......@@ -54,42 +57,61 @@ namespace BeigeOracle_MinderES.QuartzJobs
}
int tryCount = 0;
string connString = "Data Source=BEIGE_ORCL;User Id=bigdata_admin;Password=bigdata_admin";
OracleConnection conn = new OracleConnection();
conn.ConnectionString = connString;
//goto
TagToday:
List<BeigeNews> BeigeNewsList = null;
IList<BeigeNews> BeigeNewsList = null;
try
{
{
if(conn.State!=ConnectionState.Open)
conn.Open();
OracleCommand cmd = conn.CreateCommand();
//（接着上次的继续计算）从 Oracle 中抽取 TXT_NWS_BAS 的数据。
//TXT_NWS_BAS 的ID主键是自增INT型，所以可以顺序抽取
//TXT_NWS_BAS.ID 是TXT_NWS_BAS_TXT.ORIG_ID ， 但是也可能没有ORIG_ID。
string sqlstr = "SELECT tleft.ID AS ID, tleft.MED_NAME AS MED_NAME, tleft.PUB_DT AS PUB_DT, tleft.AUT AS AUT, tleft.TIT AS TIT , tleft.LNK_ADDR AS LNK_ADDR, tright.CONT_FMT AS CONT_FMT, tright.CONT AS CONT FROM ( SELECT * FROM ( SELECT t1.ID AS ID, t1.MED_NAME AS MED_NAME, t1.PUB_DT AS PUB_DT, t1.AUT AS AUT, t1.TIT AS TIT , t1.LNK_ADDR AS LNK_ADDR FROM BIGDATA_ADMIN.TXT_NWS_BAS t1 WHERE t1.ID > "+ lastId + " ORDER BY t1.ID ASC ) WHERE rownum <= "+ Query_Count + " ) tleft LEFT JOIN BIGDATA_ADMIN.TXT_NWS_BAS_TXT tright ON tleft.ID = tright.ORIG_ID";
cmd.CommandText = sqlstr;
//linq速度提升。
using (HJBeigeContext hjBeigeContext = new HJBeigeContext())
{
var data = from o in hjBeigeContext.TXT_NWS_BAS
join d in hjBeigeContext.TXT_NWS_BAS_TXT
on o.ID equals d.ORIG_ID into dc
from dci in dc.DefaultIfEmpty()
where o.ID > lastId
orderby o.PUB_DT, o.ID
select new BeigeNews
{
ID = o.ID,
MED_NAME = o.MED_NAME,
PUB_TIME = o.PUB_DT,
AUT = o.AUT,
TIT = o.TIT,
LNK_ADDR = o.LNK_ADDR,
CONT_FMT = dci.CONT_FMT,
CONT = dci.CONT
};
BeigeNewsList = data.ToList().Take(Query_Count).ToList();
cmd.CommandType = CommandType.Text;
}
//也会ONS未配置，但是会得到值。
OracleDataReader odr = cmd.ExecuteReader();
BeigeNewsList = ReaderToListModel.ReaderToList<BeigeNews>(odr);
conn.Close();
////linq速度提升。
//using (HJBeigeContext hjBeigeContext = new HJBeigeContext())
//{
// //优化linq
// var data = from o in hjBeigeContext.TXT_NWS_BAS
// join d in hjBeigeContext.TXT_NWS_BAS_TXT
// on o.ID equals d.ORIG_ID into dc
// from dci in dc.DefaultIfEmpty()
// where o.ID > lastId
// orderby o.ID
// select new BeigeNews
// {
// ID = o.ID,
// MED_NAME = o.MED_NAME,
// PUB_TIME = o.PUB_DT,
// AUT = o.AUT,
// TIT = o.TIT,
// LNK_ADDR = o.LNK_ADDR,
// CONT_FMT = dci.CONT_FMT,
// CONT = dci.CONT
// };
// BeigeNewsList = data.ToList().Take(Query_Count).ToList();
//}
}
catch (Exception ex)
{
......@@ -194,12 +216,12 @@ namespace BeigeOracle_MinderES.QuartzJobs
//认为当天内还有其他记录，需继续执行。
tryCount = 0;
lastId = long.Parse(BeigeNewsList.LastOrDefault().ID.ToString());
goto TagToday;
}
if (null == BeigeNewsList || 0 == BeigeNewsList.Count)
{
{
if (_syncState != null)
{
using (PalasEntityContext htItemContext = new PalasEntityContext())
......@@ -214,9 +236,9 @@ namespace BeigeOracle_MinderES.QuartzJobs
}
LogService.WriteInfo("完成：BeigeOracle_MinderES:"+ totalcount+"条;" + DateTime.Now.ToString() + " -lastId: " + lastId + ",数据均以导入到MinderES中，请检查！");
LogService.WriteInfo("完成：BeigeOracle_MinderES:" + totalcount + "条;" + DateTime.Now.ToString() + " -lastId: " + lastId + ",数据均以导入到MinderES中，请检查！");
Console.WriteLine("完成：BeigeOracle_MinderES:" + totalcount + "条;" + DateTime.Now.ToString() + " - lastId:" + lastId + ",数据均以导入到MinderES中，请检查！");
}
catch (Exception ex)
......@@ -227,6 +249,7 @@ namespace BeigeOracle_MinderES.QuartzJobs
}
}
}
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/ReaderToListModel.cs 0 → 100644
using System;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace BeigeOracle_MinderES
{
public static class ReaderToListModel
{
/// <summary>
/// DataReader转泛型
/// </summary>
/// <typeparam name="T">传入的实体类</typeparam>
/// <param name="objReader">DataReader对象</param>
/// <returns></returns>
public static IList<T> ReaderToList<T>(this IDataReader objReader)
{
using (objReader)
{
List<T> list = new List<T>();
//获取传入的数据类型
Type modelType = typeof(T);
//遍历DataReader对象
while (objReader.Read())
{
//使用与指定参数匹配最高的构造函数，来创建指定类型的实例
T model = Activator.CreateInstance<T>();
for (int i = 0; i < objReader.FieldCount; i++)
{
//判断字段值是否为空或不存在的值
if (!IsNullOrDBNull(objReader[i]))
{
//匹配字段名
PropertyInfo pi = modelType.GetProperty(objReader.GetName(i), BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);
if (pi != null)
{
//绑定实体对象中同名的字段
pi.SetValue(model, CheckType(objReader[i], pi.PropertyType), null);
}
}
}
list.Add(model);
}
return list;
}
}
/// <summary>
/// 对可空类型进行判断转换(*要不然会报错)
/// </summary>
/// <param name="value">DataReader字段的值</param>
/// <param name="conversionType">该字段的类型</param>
/// <returns></returns>
private static object CheckType(object value, Type conversionType)
{
if (conversionType.IsGenericType && conversionType.GetGenericTypeDefinition().Equals(typeof(Nullable<>)))
{
if (value == null)
return null;
System.ComponentModel.NullableConverter nullableConverter = new System.ComponentModel.NullableConverter(conversionType);
conversionType = nullableConverter.UnderlyingType;
}
return Convert.ChangeType(value, conversionType);
}
/// <summary>
/// 判断指定对象是否是有效值
/// </summary>
/// <param name="obj"></param>
/// <returns></returns>
private static bool IsNullOrDBNull(object obj)
{
return (obj == null || (obj is DBNull)) ? true : false;
}
/// <summary>
/// DataReader转模型
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="objReader"></param>
/// <returns></returns>
public static T ReaderToModel<T>(this IDataReader objReader)
{
using (objReader)
{
if (objReader.Read())
{
Type modelType = typeof(T);
int count = objReader.FieldCount;
T model = Activator.CreateInstance<T>();
for (int i = 0; i < count; i++)
{
if (!IsNullOrDBNull(objReader[i]))
{
PropertyInfo pi = modelType.GetProperty(objReader.GetName(i), BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);
if (pi != null)
{
pi.SetValue(model, CheckType(objReader[i], pi.PropertyType), null);
}
}
}
return model;
}
}
return default(T);
}
}
}
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/packages.config
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
......@@ -11,7 +11,5 @@
<package id="MySql.Data.Entity" version="6.9.12" targetFramework="net452" />
<package id="NEST" version="2.5.8" targetFramework="net452" />
<package id="Newtonsoft.Json" version="11.0.1" targetFramework="net452" />
<package id="Oracle.ManagedDataAccess" version="12.2.1100" targetFramework="net452" />
<package id="Oracle.ManagedDataAccess.EntityFramework" version="12.2.1100" targetFramework="net452" />
<package id="Quartz" version="2.6.0" targetFramework="net452" />
</packages>
\ No newline at end of file
10.Paul需求-JSON贝格ES导入本地ES/MinderES/BeigeOracle_MinderES/quartz_jobs.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml version="1.0" encoding="UTF-8"?>
......@@ -28,7 +28,7 @@
<job-name>Cycle_BeigeESJob</job-name>
<job-group>Test</job-group>
<start-time>2015-01-22T00:00:00+08:00</start-time>
<cron-expression>0/1 * * * * ? </cron-expression>
<cron-expression>0 0/5 * * * ? </cron-expression>
</cron>
</trigger>
<!--end-每5秒钟执行一次-BeigeESJob-->
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment