0
나는 80 개의 String 컬럼을 가진 파일을 가지고 있는데, 약 2 개의 Lacs 레코드가있다. 미래에 사용할 기술을 평가하고 있었는데 2000 Lacs 레코드 파일을 받게 될 것입니다. 내가 2 일스프링 배치 성능이 bufferedReader 및 JDBC 배치 삽입물보다 적은 이유는 무엇입니까?
을 평가 한오라클 11g 데이터베이스
접근 한 - 스프링 배치 그것은 2 118초했다
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:batch="http://www.springframework.org/schema/batch" xmlns:task="http://www.springframework.org/schema/task" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-2.2.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.2.xsd"> <batch:job id="trialDataLoadJob"> <batch:step id="step1"> <batch:tasklet transaction-manager="transactionManager"> <batch:chunk reader="trialInputFileReader" writer="trialJdbcWriter" commit-interval="1000"/> </batch:tasklet> </batch:step> </batch:job> <bean id="trialInputFileReader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step"> <property name="resource" value="file:#{jobParameters[inputFile]}" /> <property name="linesToSkip" value="1" /> <property name="lineMapper"> <bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper"> <property name="lineTokenizer" ref="trialLineTokenizer" /> <property name="fieldSetMapper" ref="trialFieldSetMapper" /> </bean> </property> </bean> <bean id="trialLineTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer"> <property name="delimiter" value="," /> <property name="names" value="RUN_DATE,ACODE,GMI_ACCOUNT,GMI_OFFICE,GMI_FIRM,ACCOUNT_NAME,CCY,LE_ID,MARKET,GMF,MIC_CODE,GMI_PRD,PRD,PRD_DESCRIPTION,BBG_BACK_OFFICE_TICKER,BBG_FRONT_OFFICE_TICKER,BBG_YELLOW_KEY,ROOT_RIC,RIC,TRADE_TYPE,PROMPT_DATE,EXPIRY_DECLARATION,LAST_TRADED_DATE,STRIKE_PRICE,OPTION_TYPE,DELIVERY_TYPE,TRADE_PRICE,DECIMAL_TRADE_PRICE,TRIAL_TRADEPRICE,CONTRACTS,NOTIONAL_FOR_FUTURES,CLEARING_BROKER,EXEC_BROKER,TRADE_DATE,TRANSACTION_TYPE,UNDERLYING,UNDERLYING_TYPE,GMI_MULTIPLIER,UTI,USI,ISIN,AII,CFI,SERIAL,DEALER_REFERENCE,TRADE_EXECUTION_ID,CLEARING_TIMESTAMP,EXECUTION_TIMESTAMP,CCP_LE_ID,SWAP_TYPE,EFFECTIVE_DATE, COUPON_RATE,DAY_COUNT_BASIS,ROLL_FREQUENCY,RESET_FREQUENCY,ACTIVITY_TYPE,EMPTY,PRD_ID_PREFIX_1,PRD_ID_PREFIX_2,TRIAL_ENTITY_NAME,TRIAL_ENTITY_LEI,TRIAL_REGION" /> <property name="strict" value="false"></property> </bean> <bean id="demo" class="com.mkyong.Demo"></bean> <bean id= "trialFieldSetMapper" class="org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper"> <property name="prototypeBeanName" value="demo" /> </bean> <bean id="trialJdbcWriter" class="org.springframework.batch.item.database.JdbcBatchItemWriter"> <property name="dataSource" ref="dataSource" /> <property name="sql"> <value> <![CDATA[ Insert into TB_TRANS(RUN_DATE,ACODE,GMI_ACCOUNT,GMI_OFFICE,GMI_FIRM,ACCOUNT_NAME,CCY,LE_ID,MARKET,GMF,MIC_CODE,GMI_PRD,PRD,PRD_DESCRIPTION,BBG_BACK_OFFICE_TICKER,BBG_FRONT_OFFICE_TICKER,BBG_YELLOW_KEY,ROOT_RIC,RIC,TRADE_TYPE,PROMPT_DATE,EXPIRY_DECLARATION,LAST_TRADED_DATE,STRIKE_PRICE,OPTION_TYPE,DELIVERY_TYPE,TRADE_PRICE,DECIMAL_TRADE_PRICE,TRIAL_TRADEPRICE,CONTRACTS,NOTIONAL_FOR_FUTURES,CLEARING_BROKER,EXEC_BROKER,TRADE_DATE,TRANSACTION_TYPE,UNDERLYING,UNDERLYING_TYPE,GMI_MULTIPLIER,UTI,USI,ISIN,AII,CFI,SERIAL,DEALER_REFERENCE,TRADE_EXECUTION_ID,CLEARING_TIMESTAMP,EXECUTION_TIMESTAMP,CCP_LE_ID,SWAP_TYPE,EFFECTIVE_DATE, COUPON_RATE,DAY_COUNT_BASIS,ROLL_FREQUENCY,RESET_FREQUENCY,ACTIVITY_TYPE,EMPTY,PRD_ID_PREFIX_1,PRD_ID_PREFIX_2,TRIAL_ENTITY_NAME,TRIAL_ENTITY_LEI,TRIAL_REGION) values (:RUN_DATE,:ACODE,:GMI_ACCOUNT,:GMI_OFFICE,:GMI_FIRM,:ACCOUNT_NAME,:CCY,:LE_ID,:MARKET,:GMF,:MIC_CODE,:GMI_PRD,:PRD,:PRD_DESCRIPTION,:BBG_BACK_OFFICE_TICKER,:BBG_FRONT_OFFICE_TICKER,:BBG_YELLOW_KEY,:ROOT_RIC,:RIC,:TRADE_TYPE,:PROMPT_DATE,:EXPIRY_DECLARATION,:LAST_TRADED_DATE,:STRIKE_PRICE,:OPTION_TYPE,:DELIVERY_TYPE,:TRADE_PRICE,:DECIMAL_TRADE_PRICE,:TRIAL_TRADEPRICE,:CONTRACTS,:NOTIONAL_FOR_FUTURES,:CLEARING_BROKER,:EXEC_BROKER,:TRADE_DATE,:TRANSACTION_TYPE,:UNDERLYING,:UNDERLYING_TYPE,:GMI_MULTIPLIER,:UTI,:USI,:ISIN,:AII,:CFI,:SERIAL,:DEALER_REFERENCE,:TRADE_EXECUTION_ID,:CLEARING_TIMESTAMP,:EXECUTION_TIMESTAMP,:CCP_LE_ID,:SWAP_TYPE,:EFFECTIVE_DATE,:COUPON_RATE,:DAY_COUNT_BASIS,:ROLL_FREQUENCY,:RESET_FREQUENCY,:ACTIVITY_TYPE,:EMPTY,:PRD_ID_PREFIX_1,:PRD_ID_PREFIX_2,:TRIAL_ENTITY_NAME,:TRIAL_ENTITY_LEI,:TRIAL_REGION) ]]> </value> </property> <property name="itemSqlParameterSourceProvider"> <bean class="org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider" /> </property> </bean> </beans> 구성은, 00,000 기록.
접근법 2 - BufferedReader로 & JDBC 배치는
Connection connection = dbManager.getConnection();
File file = new File(filePath);
BufferedReader br = null;
boolean error = false;
ArrayList<String[]> records = new ArrayList<String[]>(batchSize * 2);
try {
connection.setAutoCommit(false);
br = new BufferedReader(new FileReader(file));
String line;
while ((line = br.readLine()) != null) {
// System.out.println(line);
String[] values = line.split(",",-1);
records.add(values);
if (records.size() == batchSize) {
insertToDB(records, connection);
records.clear();
}
}
if (records.size() > 0) {
insertToDB(records, connection);
}
} catch (FileNotFoundException e) {
error = true;
e.printStackTrace();
} catch (IOException e) {
error = true;
e.printStackTrace();
} catch (SQLException e) {
error = true;
e.printStackTrace();
DBManager.rollback(connection);
} finally {
if (br != null)
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
if (!error) {
DBManager.commitConnection(connection);
}
if (DBManager.isConnectionClosed(connection))
DBManager.closeConnection(connection);
}
은 1,00,000 레코드를 3 초 걸렸다.
스프링 배치 버전 3이 느린 이유는 무엇입니까 ?? 나는 그 반대의 인상을 받았다.
: - 때문에 프레임 워크의 자신의 성격 - 프레임 워크는 다음 코드의 직선 조각 더 복잡하기 때문에 오버 헤드를 발생시킨다 (코드는 '아무튼 재시작 가능성에 신경 쓰지 마라. 프레임 워크는 보통 최적의 성능을 얻기 위해 위상을 미세 조정해야합니다. –