엄청난 양의 데이터를 청크로 옮기기

나는 엄청난 양의 데이터를 조건에 따라 쿼리해야하며 일부 보관 테이블로 이동해야하는 스크립트 작업을하고 있습니다. 6 개의 아카이브 테이블에 대해 INSERT 작업을 수행하기 위해 일치하는 레코드를 스캔하여 선택하기 위해 5 천만 개 이상의 레코드가 있습니다. PGA_ AGGREGATE _LIMIT 내가 그것을로드를 피할 수있는 방법으로 내 스크립트를 향상시키고 자 증가에 비해엄청난 양의 데이터를 청크로 옮기기

Error report: 
ORA-04036: PGA memory used by the instance exceeds PGA_ AGGREGATE _LIMIT

다른 기록의 수백만 실행할 때 아래 스크립트는 약 50 만 기록을 위해 잘 작동하지만 다음 예외가 발생합니다 모든 레코드를 메모리에 저장하지만 스크립트를 실행하고 테이블에 값을 청크로 삽입합니다. 현재이 작업을 어떻게 수행해야하는지에 대해서는 잘 모릅니다. 다른 사람이 스크립트를 일괄 적으로 실행하도록하여 메모리 부족 문제를 피할 수 있는지 제안 할 수 있습니까?

아래 스크립트는 내 테이블에 있습니다 (4 개 테이블에 값 삽입).

CREATE OR REPLACE TYPE R1_ID_TYPE IS TABLE OF NUMBER; 
/
CREATE OR REPLACE TYPE R5_ID_TYPE IS TABLE OF NUMBER; 
/
DECLARE 
    R01_IDS R1_ID_TYPE; 
    R05_IDS R5_ID_TYPE; 

BEGIN 
    --add the R05_IDs which are older than five years from R5_TABLE and R6_TABLE to R5_ID_TYPE nested table 
    SELECT R5.R05_ID AS R05_ID 
    BULK COLLECT INTO R05_IDS 
    FROM R6_TABLE R6 , R5_TABLE R5 
      WHERE R5.R05_ID = R6.R06_R05_ID_FK 
      AND R5.R05_DATE_TIME_CAPTURED <= TRUNC(SYSDATE) - 1825 
      AND R5.R05_STATUS = 'D' 
      AND R6.R06_STATUS = 'D'; 

    -- Inserts all the deregistered records which are older than five years from R5_TABLE and R6_TABLE tables to the relevant archive tables 
    INSERT ALL 
    INTO R5_TABLE_archived(
     R05_ID, 
     R05_R01_ID_FK, 
     R05_NUMBER, 
     R05_NUMBER_TYPE, 
     R05_STATUS, 
     R05_GSM_SUBSCRIBER_TYPE 
     R05_DATE_TIME_CAPTURED) 
     values (
     R5_R05_ID, 
     R5_R05_R01_ID_FK, 
     R5_NUMBER, 
     R5_NUMBER_TYPE, 
     R5_R05_STATUS, 
     R5_R05_GSM_SUBSCRIBER_TYPE, 
     R5_R05_DATE_TIME_CAPTURED) 
    INTO R6_TABLE_archived(
     R06_ID, 
     R06_R05_ID_FK, 
     R06_R08_ID_FK, 
     R06_STATUS, 
     R06_REFERENCE_NUMBER, 
     R06_DATE_TIME_CAPTURED, 
     R06_DATE_EXPIRED) 
     values (
     R6_R06_ID, 
     R6_R06_R05_ID_FK, 
     R6_R06_R08_ID_FK, 
     R6_R06_STATUS, 
     R6_R06_REFERENCE_NUMBER, 
     R6_R06_DATE_TIME_CAPTURED, 
     R6_R06_DATE_EXPIRED) 
    SELECT R5_R05_ID, 
     R5_R05_R01_ID_FK, 
     R5_NUMBER, 
     R5_NUMBER_TYPE, 
     R5_R05_STATUS, 
     R5_R05_GSM_SUBSCRIBER_TYPE, 
     R5_R05_DATE_TIME_CAPTURED, 
     R6_R06_ID, 
     R6_R06_R05_ID_FK, 
     R6_R06_R08_ID_FK, 
     R6_R06_CHANGE_SOURCE, 
     R6_R06_REFERENCE_NUMBER, 
     R6_R06_DATE_TIME_CAPTURED, 
     R6_R06_DATE_EXPIRED 
    FROM 
    (
    SELECT R5.R05_ID R5_R05_ID, 
     R5.R05_R01_ID_FK R5_R05_R01_ID_FK, 
     R5.R05_NUMBER R5_NUMBER, 
     R5.R05_NUMBER_TYPE R5_NUMBER_TYPE, 
     R5.R05_STATUS R5_R05_STATUS, 
     R5.R05_GSM_SUBSCRIBER_TYPE R5_R05_GSM_SUBSCRIBER_TYPE, 
     R5.R05_DATE_TIME_CAPTURED R5_R05_DATE_TIME_CAPTURED, 
     R6.R06_ID R6_R06_ID, 
     R6.R06_R05_ID_FK R6_R06_R05_ID_FK, 
     R6.R06_R08_ID_FK R6_R06_R08_ID_FK, 
     R6.R06_STATUS R6_R06_STATUS, 
     R6.R06_REFERENCE_NUMBER R6_R06_REFERENCE_NUMBER, 
     R6.R06_DATE_TIME_CAPTURED R6_R06_DATE_TIME_CAPTURED, 
     R6.R06_DATE_EXPIRED R6_R06_DATE_EXPIRED 
     FROM R6_TABLE R6 , R5_TABLE R5 
     WHERE R5.R05_ID = R6.R06_R05_ID_FK 
     AND R5.R05_DATE_TIME_CAPTURED <= TRUNC(SYSDATE) - 1825 
     AND R5.R05_STATUS = 'D' 
     AND R6.R06_STATUS = 'D');  

    --selects all the R01 IDs which matches with the above criteria and copy values to respective archive tables 
    SELECT UNIQUE R1.R01_ID AS R01_ID 
    BULK COLLECT INTO R01_IDS      
    FROM R1_TABLE R1, R5_TABLE R5 
    WHERE R5.R05_ID IN (Select column_value from table(R05_IDS)) 
    AND R1.R01_ID NOT IN (
         SELECT R01.R01_ID 
         FROM R1_TABLE R01, 
           R5_TABLE R05 
         WHERE R05.R05_STATUS != 'D' 
           AND R01.R01_ID = R05.R05_R01_ID_FK) 
    AND R1.R01_ID = R5.R05_R01_ID_FK; 

    --insert R1_TABLE tables values which matches with the above criteria into the R1_TABLE_ARCHIVED table 
    INSERT ALL 
    INTO R1_TABLE_ARCHIVED(R01_ID,R01_ID_TYPE,R01_IDENTITY_NUMBER,R01_PASSPORT_COUNTRY,R01_DATE_TIME_CAPTURED) 
    VALUES (RA1_R01_ID,RA1_R01_ID_TYPE,RA1_R01_IDENTITY_NUMBER,RA1_R01_PASSPORT_COUNTRY,RA1_R01_DATE_TIME_CAPTURED) 
    SELECT RA1_R01_ID,RA1_R01_ID_TYPE,RA1_R01_IDENTITY_NUMBER,RA1_R01_PASSPORT_COUNTRY,RA1_R01_DATE_TIME_CAPTURED 
    FROM (
     SELECT 
      r1.R01_ID RA1_R01_ID, 
      r1.R01_ID_TYPE RA1_R01_ID_TYPE, 
      r1.R01_IDENTITY_NUMBER RA1_R01_IDENTITY_NUMBER, 
      r1.R01_PASSPORT_COUNTRY RA1_R01_PASSPORT_COUNTRY, 
      r1.R01_DATE_TIME_CAPTURED RA1_R01_DATE_TIME_CAPTURED 
      FROM 
      R1_TABLE r1 
      WHERE 
      r1.R01_ID IN (Select column_value from table(R01_IDS)) 
    ); 

    --insert R2_TABLE tables values which matches with the above criteria into the R2_TABLE_ARCHIVED table 
    INSERT ALL 
    INTO R2_TABLE_ARCHIVED(R02_ID,R02_R01_ID_FK,R02_fname,R02_SURNAME,R02_CONTACT_NUMBER,R02_DATE_TIME_CAPTURED) 
    VALUES(RA2_R02_ID,RA2_R02_R01_ID_FK,RA2_R02_fname,RA2_R02_SURNAME,RA2_R02_CONTACT_NUMBER,RA2_R02_DATE_TIME_CAPTURED) 
    SELECT RA2_R02_ID,RA2_R02_R01_ID_FK,RA2_R02_fname,RA2_R02_SURNAME,RA2_R02_CONTACT_NUMBER,RA2_R02_DATE_TIME_CAPTURED 
    FROM (
      SELECT 
      r2.R02_ID RA2_R02_ID, 
      r2.R02_R01_ID_FK RA2_R02_R01_ID_FK, 
      r2.R02_fname RA2_R02_fname, 
      r2.R02_SURNAME RA2_R02_SURNAME, 
      r2.R02_CONTACT_NUMBER RA2_R02_CONTACT_NUMBER, 
      r2.R02_DATE_TIME_CAPTURED RA2_R02_DATE_TIME_CAPTURED 
      FROM 
      R2_TABLE r2 
      WHERE 
      r2.R02_R01_ID_FK IN (Select column_value from table(R01_IDS)));  


    --All the delete queries to remove the above copied values from the parent tables respectively 
    DELETE FROM R1_TABLE WHERE R01_ID IN (Select column_value from table(R01_IDS)); 
    DELETE FROM R2_TABLE WHERE R02_R01_ID_FK IN (Select column_value from table(R01_IDS)); 
    DELETE FROM R5_TABLE WHERE R05_R01_ID_FK IN (Select column_value from table(R05_IDS)); 
    DELETE FROM R6_TABLE WHERE R06_R05_ID_FK IN (R05_IDS);  


COMMIT; 
END; 
/
COMMIT;

출처

2017-12-14 sher17

콜렉션 (다른 PL/SQL 구조)은 세션 메모리에 저장된다. (전역 메모리에 저장된 쿼리 된 데이터와 달리). 세션 메모리는 사용자별로 할당되기 때문에 RAM이 여전히 비싼 리소스이기 때문에 한계가 있습니다. 세션이 모든 메모리가 PGA (세션에 사용할 수있는 메모리 풀을) 할당 최대 hoovered 때문에

그래서, 당신은 ... ...

ORA-04036: PGA memory used by the instance exceeds PGA_ AGGREGATE _LIMIT

을이 오류를 얻고있다.

수백만 개의 행으로 채우기를 채우는 중입니다. 그 행은 여전히 좁지 만 여전히 켜지지는 않습니다. 다행스럽게도 PL/SQL에는 LIMIT 절이 있습니다.

LIMIT을 사용하면 결과 집합을 청크로 채우고 처리하여 다음 청크를 얻을 수 있습니다. 변경 사항이 많지 않습니다.

DECLARE 
    R01_IDS R1_ID_TYPE; 
    R05_IDS R5_ID_TYPE; 
    cursor r5_cur is 
     SELECT R5.R05_ID 
     BULK COLLECT INTO R05_IDS 
     FROM R6_TABLE R6 , R5_TABLE R5 
      WHERE R5.R05_ID = R6.R06_R05_ID_FK 
      AND R5.R05_DATE_TIME_CAPTURED <= TRUNC(SYSDATE) - 1825 
      AND R5.R05_STATUS = 'D' 
      AND R6.R06_STATUS = 'D';  
BEGIN 
    -- this is new 
    open r5_cur; 
    loop 
     fetch r5_cur 
     BULK COLLECT INTO R05_IDS limit 100000; 
     exit when R05_IDS.count() = 0; 

     -- this is all your code 

     -- Inserts all the deregistered records which are older than five years from R5_TABLE and R6_TABLE tables to the relevant archive tables 
     INSERT ALL 
     INTO R5_TABLE_archived(
      R05_ID, 
      R05_R01_ID_FK, 
      R05_NUMBER, 
      R05_NUMBER_TYPE, 
      R05_STATUS, 
      R05_GSM_SUBSCRIBER_TYPE 
      R05_DATE_TIME_CAPTURED) 
      values (
      R5_R05_ID, 
      R5_R05_R01_ID_FK, 
      R5_NUMBER, 
      R5_NUMBER_TYPE, 
      R5_R05_STATUS, 
      R5_R05_GSM_SUBSCRIBER_TYPE, 
      R5_R05_DATE_TIME_CAPTURED) 
     INTO R6_TABLE_archived(
      R06_ID, 
      R06_R05_ID_FK, 
      R06_R08_ID_FK, 
      R06_STATUS, 
      R06_REFERENCE_NUMBER, 
      R06_DATE_TIME_CAPTURED, 
      R06_DATE_EXPIRED) 
      values (
      R6_R06_ID, 
      R6_R06_R05_ID_FK, 
      R6_R06_R08_ID_FK, 
      R6_R06_STATUS, 
      R6_R06_REFERENCE_NUMBER, 
      R6_R06_DATE_TIME_CAPTURED, 
      R6_R06_DATE_EXPIRED) 
     SELECT R5_R05_ID, 
      R5_R05_R01_ID_FK, 
      R5_NUMBER, 
      R5_NUMBER_TYPE, 
      R5_R05_STATUS, 
      R5_R05_GSM_SUBSCRIBER_TYPE, 
      R5_R05_DATE_TIME_CAPTURED, 
      R6_R06_ID, 
      R6_R06_R05_ID_FK, 
      R6_R06_R08_ID_FK, 
      R6_R06_CHANGE_SOURCE, 
      R6_R06_REFERENCE_NUMBER, 
      R6_R06_DATE_TIME_CAPTURED, 
      R6_R06_DATE_EXPIRED 
     FROM 
     (
     SELECT R5.R05_ID R5_R05_ID, 
      R5.R05_R01_ID_FK R5_R05_R01_ID_FK, 
      R5.R05_NUMBER R5_NUMBER, 
      R5.R05_NUMBER_TYPE R5_NUMBER_TYPE, 
      R5.R05_STATUS R5_R05_STATUS, 
      R5.R05_GSM_SUBSCRIBER_TYPE R5_R05_GSM_SUBSCRIBER_TYPE, 
      R5.R05_DATE_TIME_CAPTURED R5_R05_DATE_TIME_CAPTURED, 
      R6.R06_ID R6_R06_ID, 
      R6.R06_R05_ID_FK R6_R06_R05_ID_FK, 
      R6.R06_R08_ID_FK R6_R06_R08_ID_FK, 
      R6.R06_STATUS R6_R06_STATUS, 
      R6.R06_REFERENCE_NUMBER R6_R06_REFERENCE_NUMBER, 
      R6.R06_DATE_TIME_CAPTURED R6_R06_DATE_TIME_CAPTURED, 
      R6.R06_DATE_EXPIRED R6_R06_DATE_EXPIRED 
      FROM R6_TABLE R6 , R5_TABLE R5 
      WHERE R5.R05_ID = R6.R06_R05_ID_FK 
      AND R5.R05_DATE_TIME_CAPTURED <= TRUNC(SYSDATE) - 1825 
      AND R5.R05_STATUS = 'D' 
      AND R6.R06_STATUS = 'D');  

     --selects all the R01 IDs which matches with the above criteria and copy values to respective archive tables 
     SELECT UNIQUE R1.R01_ID AS R01_ID 
     BULK COLLECT INTO R01_IDS      
     FROM R1_TABLE R1, R5_TABLE R5 
     WHERE R5.R05_ID IN (Select column_value from table(R05_IDS)) 
     AND R1.R01_ID NOT IN (
          SELECT R01.R01_ID 
          FROM R1_TABLE R01, 
            R5_TABLE R05 
          WHERE R05.R05_STATUS != 'D' 
            AND R01.R01_ID = R05.R05_R01_ID_FK) 
     AND R1.R01_ID = R5.R05_R01_ID_FK; 

     --insert R1_TABLE tables values which matches with the above criteria into the R1_TABLE_ARCHIVED table 
     INSERT ALL 
     INTO R1_TABLE_ARCHIVED(R01_ID,R01_ID_TYPE,R01_IDENTITY_NUMBER,R01_PASSPORT_COUNTRY,R01_DATE_TIME_CAPTURED) 
     VALUES (RA1_R01_ID,RA1_R01_ID_TYPE,RA1_R01_IDENTITY_NUMBER,RA1_R01_PASSPORT_COUNTRY,RA1_R01_DATE_TIME_CAPTURED) 
     SELECT RA1_R01_ID,RA1_R01_ID_TYPE,RA1_R01_IDENTITY_NUMBER,RA1_R01_PASSPORT_COUNTRY,RA1_R01_DATE_TIME_CAPTURED 
     FROM (
      SELECT 
       r1.R01_ID RA1_R01_ID, 
       r1.R01_ID_TYPE RA1_R01_ID_TYPE, 
       r1.R01_IDENTITY_NUMBER RA1_R01_IDENTITY_NUMBER, 
       r1.R01_PASSPORT_COUNTRY RA1_R01_PASSPORT_COUNTRY, 
       r1.R01_DATE_TIME_CAPTURED RA1_R01_DATE_TIME_CAPTURED 
       FROM 
       R1_TABLE r1 
       WHERE 
       r1.R01_ID IN (Select column_value from table(R01_IDS)) 
     ); 

     --insert R2_TABLE tables values which matches with the above criteria into the R2_TABLE_ARCHIVED table 
     INSERT ALL 
     INTO R2_TABLE_ARCHIVED(R02_ID,R02_R01_ID_FK,R02_fname,R02_SURNAME,R02_CONTACT_NUMBER,R02_DATE_TIME_CAPTURED) 
     VALUES(RA2_R02_ID,RA2_R02_R01_ID_FK,RA2_R02_fname,RA2_R02_SURNAME,RA2_R02_CONTACT_NUMBER,RA2_R02_DATE_TIME_CAPTURED) 
     SELECT RA2_R02_ID,RA2_R02_R01_ID_FK,RA2_R02_fname,RA2_R02_SURNAME,RA2_R02_CONTACT_NUMBER,RA2_R02_DATE_TIME_CAPTURED 
     FROM (
       SELECT 
       r2.R02_ID RA2_R02_ID, 
       r2.R02_R01_ID_FK RA2_R02_R01_ID_FK, 
       r2.R02_fname RA2_R02_fname, 
       r2.R02_SURNAME RA2_R02_SURNAME, 
       r2.R02_CONTACT_NUMBER RA2_R02_CONTACT_NUMBER, 
       r2.R02_DATE_TIME_CAPTURED RA2_R02_DATE_TIME_CAPTURED 
       FROM 
       R2_TABLE r2 
       WHERE 
       r2.R02_R01_ID_FK IN (Select column_value from table(R01_IDS)));  


     --All the delete queries to remove the above copied values from the parent tables respectively 
     DELETE FROM R1_TABLE WHERE R01_ID IN (Select column_value from table(R01_IDS)); 
     DELETE FROM R2_TABLE WHERE R02_R01_ID_FK IN (Select column_value from table(R01_IDS)); 
     DELETE FROM R5_TABLE WHERE R05_R01_ID_FK IN (Select column_value from table(R05_IDS)); 
     DELETE FROM R6_TABLE WHERE R06_R05_ID_FK IN (R05_IDS);  


    end loop;   
    close r5_cur; 
    COMMIT; 
END; 
/

END LOOP로 스크롤하여 커서를 닫는 것을 잊지 마십시오!

출처

2017-12-14 08:19:13 APC

잘 설명해 주셔서 감사합니다. 이 코드는 데이터베이스에서 한 시간 이상 실행되어 ORA-01653 오류로 반환되었습니다 : USERS 01653 테이블 스페이스에서 8192로 R5_TABLE_archived 테이블을 확장 할 수 없습니다. "테이블 스페이스 % 1! % s. % s을 (를) 확장 할 수 없습니다" s " * 원인 : 표시된 테이블 공간에 테이블 세그먼트 에 필요한 블록 수의 범위를 할당하는 데 실패했습니다. * 조치 : ALTER TABLESPACE ADD DATAFILE 문을 사용하여 하나 이상의 파일을 표시된 테이블 공간에 추가하십시오. – sher17

여기서 발생하는 테이블 공간 문제에 대한 해결 방법이 있습니까 – sher17

분명히하기 위해 이동해야 할 요구 사항을 충족하는 R5 테이블에 31931421 개의 레코드가 있습니다. – sher17

테이블 공간은 물리적 스토리지에 대한 매핑입니다. 다른 데이터 파일을 추가하도록 DBA에 요청하는 것 이외에는 해결 방법이별로 없습니다.

출처

2017-12-14 11:29:49 APC

엄청난 양의 데이터를 청크로 옮기기

답변

관련 문제