'메모리 부족!' 매우 관여하는 Perl 스크립트의 오류

Question

나는 우분투 질문에 얼마 전에 this question을 물었고 여기에 약간의 코드를 제공하도록 지시 받았다. 아마 최적화 문제 일 것이다. 전체 스크립트를 포함했습니다.

일반적인 목표는 약 7000 개의 HTML 파일을 디렉토리에서 분석하고 특정 정보를 구문 분석하여 텍스트 파일로 한 줄로 내보내는 것입니다.

#!/usr/bin/perl 

use Switch; 
use strict; 

use HTML::Query 'Query'; 

my $dir = '/home/mark/Documents/Perl/garchivesfiles/completeresults'; 

opendir my $dh, $dir or die "Can't open $dir: $!"; 
my @files = map {"$dir/$_"} grep { $_ !~ /^\./ } readdir $dh; 
closedir $dh; 

my $total; 

my %xlateNum2Text = qw (0 January 
         1 Febuary 
         2 March 
         3 April 
         4 May 
         5 June 
         6 July 
         7 August 
         8 September 
         9 October 
         10 November 
         11 December     
         ); 


my $inc = 0; 
foreach my $file (@files) { 
    open FILE, $file; 
    my $html = do { local $/; <FILE> }; 
    my $q = Query(text => $html); 

    my @homescore = $q->query("span.homeScore"); 
    my @awayscore = $q->query("span.awayScore"); 
    my $singlehomescore = $homescore[0]->as_text(); 
    my $singleawayscore = $homescore[0]->as_text(); 

    my @hometeam = $q->query("table.teaminfo td.home span"); 
    my @awayteam = $q->query("table.teaminfo td.away span"); 
    my $singlehometeam = rightTeamName($hometeam[0]->as_text()); 
    my $singleawayteam = rightTeamName($awayteam[0]->as_text()); 


    my @homegoalstotal; 
    my @awaygoalstotal; 
    my @datearray; 
    my @fixtureinfo; 

    my @newhomegoals; 
    my @newawaygoals; 

    my @allinfogoals; 

    if($singlehomescore ne "0" || $singleawayscore ne "0") { 
     @homegoalstotal = $q->query("div.home ul li"); 
     @awaygoalstotal = $q->query("div.away ul li"); 
     my $i = 0; 

     @datearray = $q->query("p.fixtureinfo span"); 
     my $finaldate = $datearray[0]->as_text(); 
     my @datecomponents = split(" ", $finaldate); 
     my $mysqlyyyy = $datecomponents[3]; 
     my $mysqlmm = monthConvert($datecomponents[2]); 
     my $mysqldd = $datecomponents[1]; 

     my $mysqldate; 

     if(length($mysqlmm) == 1) { 
      $mysqlmm = "0".$mysqlmm; 
     } 

     if(length($mysqldd) == 1) { 
      $mysqldd = "0".$mysqldd; 
     } 

     $mysqldate = $mysqlyyyy."-".$mysqlmm."-".$mysqldd; 


     @fixtureinfo = $q->query("p.fixtureinfo"); 
     my $fixtureinfoinit = $fixtureinfo[0]->as_text(); 
     my @fixtureinfobrokenup = split(/ \| /, $fixtureinfoinit); 
     my $fixtureinfostring = $fixtureinfobrokenup[1]; 

     foreach my $goal (@homegoalstotal) { 
      my $tempmodifier = $goal->as_text(); 
      $tempmodifier =~ s/\)//g; 
      my @tempcomponents = split(' \(', $tempmodifier); 
      my $substitutetemp; 
      my @extratimesplit; 
      my $compositetime; 

      if(index($tempcomponents[1], ",") != -1) { 
       my @goaltimes = split('\,', $tempcomponents[1]); 
       foreach my $individmultgoal (@goaltimes) { 
        $individmultgoal =~ s/Pen//g; 
        $individmultgoal =~ s/ //g; 
        if(index($individmultgoal, "OG") == -1) { 
         if(index($individmultgoal, "+") != -1) { 
          @extratimesplit = split('\+', $individmultgoal); 
          $compositetime = $extratimesplit[0]; 
          push (@{$allinfogoals[$i]}, ($tempcomponents[0], $compositetime, "for:".$singlehometeam, $singleawayteam, $datecomponents[1], ,$datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, $extratimesplit[1])); 
          $i++; 
         } else { 
          push (@{$allinfogoals[$i]}, ($tempcomponents[0], $individmultgoal, "for:".$singlehometeam, $singleawayteam, $datecomponents[1], ,$datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, 0)); 
          $i++; 
         } 
        } 
       } 
      } else { 
       $substitutetemp = $tempcomponents[1]; 
       $substitutetemp =~ s/Pen//g; 
       $substitutetemp =~ s/ //g; 
       if(index($substitutetemp, "OG") == -1) { 
        if(index($substitutetemp, "+") != -1) { 
         @extratimesplit = split('\+', $substitutetemp); 
         $compositetime = $extratimesplit[0]; 
         push (@{$allinfogoals[$i]}, ($tempcomponents[0], $compositetime, "for:".$singlehometeam, $singleawayteam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, $extratimesplit[1])); 
         $i++; 
        } else { 
         push (@{$allinfogoals[$i]}, ($tempcomponents[0], $substitutetemp, "for:".$singlehometeam, $singleawayteam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, 0)); 
         $i++; 
        } 
       } 
      } 
     } 

     foreach my $goal (@awaygoalstotal) { 
      my $tempmodifier2 = $goal->as_text(); 
      $tempmodifier2 =~ s/\)//g; 
      my @tempcomponents2 = split(' \(', $tempmodifier2); 
      my $substitutetemp2; 
      my @extratimesplit2; 
      my $compositetime2; 

      if(index($tempcomponents2[1], ",") != -1) { 
       my @goaltimes2 = split('\,', $tempcomponents2[1]); 
       foreach my $individmultgoal2 (@goaltimes2) { 
        $individmultgoal2 =~ s/Pen//g; 
        $individmultgoal2 =~ s/ //g; 
        if(index($individmultgoal2, "OG") == -1) { 
         if(index($individmultgoal2, "+") != -1) { 
          @extratimesplit2 = split('\+', $individmultgoal2); 
          $compositetime2 = $extratimesplit2[0]; 
          push (@{$allinfogoals[$i]}, ($tempcomponents2[0], $compositetime2, "for:".$singleawayteam, $singlehometeam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, $extratimesplit2[1])); 
          $i++; 
         } else { 
          push (@{$allinfogoals[$i]}, ($tempcomponents2[0], $individmultgoal2, "for:".$singleawayteam, $singlehometeam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, 0)); 
          $i++; 
         } 
        } 
       } 
      } else { 
       $substitutetemp2 = $tempcomponents2[1]; 
       $substitutetemp2 =~ s/Pen//g; 
       $substitutetemp2 =~ s/ //g; 
       if(index($substitutetemp2, "OG") == -1) { 
        if(index($substitutetemp2, "+") != -1) { 
         @extratimesplit2 = split('\+', $substitutetemp2); 
         $compositetime2 = $extratimesplit2[0]; 
         push(@{$allinfogoals[$i]}, ($tempcomponents2[0], $compositetime2, "for:".$singleawayteam, $singlehometeam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, $extratimesplit2[1])); 
         $i++; 
        } else { 
         push(@{$allinfogoals[$i]}, ($tempcomponents2[0], $substitutetemp2, "for:".$singleawayteam, $singlehometeam, $datecomponents[1], $datecomponents[2], $datecomponents[3], $fixtureinfostring, "Barclays Premier League", monthConvert($datecomponents[2]), $mysqldate, 0)); 
         $i++; 
        } 
       } 
      } 
     } 


     @allinfogoals = sort { $a->[1] <=> $b->[1] || $a->[12] <=> $b->[12] } @allinfogoals; 

     open(GOALCSV, '>>goalcsv.txt'); 

     my $homegoalcount = 0; 
     my $awaygoalcount = 0; 

     foreach my $row(@allinfogoals){ 
      foreach my $val(@$row){ 
       if($val eq "for:".$singlehometeam) { 
        $homegoalcount++; 
        print GOALCSV "$val,".$homegoalcount.",".$awaygoalcount.",true,"; 
        print "$val,".$homegoalcount.",".$awaygoalcount.",true,"; 
       } elsif($val eq "for:".$singleawayteam) { 
        $awaygoalcount++; 
        print GOALCSV "$val,".$awaygoalcount.",".$homegoalcount.",false,"; 
        print "$val,".$awaygoalcount.",".$homegoalcount.",false,"; 
       } else { 
        print GOALCSV "$val,"; 
        print "$val,"; 
       } 
      } 
      print GOALCSV "\n"; 
      print "\n"; 
     } 
    } 

} 

sub rightTeamName{ 
    my $teamname = $_[0]; 

    switch($teamname) { 
     case "Nott'm Forest" { return "Nottingham Forest" } 
     case "QPR" { return "Queens Park Rangers" } 
     case "Southampton" { return "Southampton FC" } 
     case "Norwich" { return "Norwich City" } 
     case "Tottenham" { return "Tottenham Hotspur" } 
     case "Leeds" { return "Leeds United" } 
     case "Middlesbrough" { return "Middlesbrough FC" } 
     case "Chelsea" { return "Chelsea FC" } 
     case "Arsenal" { return "Arsenal FC" } 
     case "Oldham" { return "Oldham Athletic" } 
     case "Ipswich" { return "Ipswich Town" } 
     case "Man Utd" { return "Manchester United" } 
     case "Man City" { return "Manchester City" } 
     case "Sheffield Wed" { return "Sheffield Wednesday" } 
     case "Man City" { return "Manchester City" } 
     case "Blackburn" { return "Blackburn Rovers" } 
     case "Wimbledon" { return "AFC Wimbledon" } 
     case "Liverpool" { return "Liverpool FC" } 
     case "Coventry" { return "Coventry City" } 
     else  { return $teamname } 

    } 
} 

sub monthConvert{ 
     switch($_[0]) { 
      case "January" { return 1 } 
      case "February" { return 2 } 
      case "March" { return 3 } 
      case "April" { return 4 } 
      case "May" { return 5 } 
      case "June" { return 6 } 
      case "July" { return 7 } 
      case "August" { return 8 } 
      case "September" { return 9 } 
      case "October" { return 10 } 
      case "November" { return 11} 
      case "December" { return 12 } 
     } 
} 

Answer 1

HTML :: Query는 HTML :: Element 및 HTML :: TreeBuilder를 사용하여 문서의 노드를 모델링합니다. 노드가 복잡한 방법으로 연결되어 Perl 가비지 수집기가 노드를 정리할 수 없습니다. 따라서 약한 참조를 사용할 수있는 HTML :: Element 버전을 가지고 있다고 가정하십시오.

• 이것들은 가비지 콜렉션을 막지 않습니다. use HTML::TreeBuilder 5 -weak 트릭을해야합니다.

• query 방법의 결과로 delete으로 전화하십시오.

자세한 내용은 설명서 (예 : HTML::Element)를 참조하십시오.

다음은 코드 복제를 줄이기 위해 스크립트를 정리 한 것입니다 (원본 코드에 & 붙여 넣기의 명확한 징후가 있음). 여전히 아름답지는 않지만 일부 WTF는 남아 있지만 유지 보수성이 개선되어야합니다. 특히, @allinfogoals의 12 번째 열이 (정렬에서) 무엇인지 또는 CSV가이 이상한 방식으로 방출 된 이유를 알지 못합니다 (우리는 이미 for: 열 (→ 2)의 색인을 알고 있으므로 우리는 모든 열과 예상 값이 일치하지 않음).

일부 누락 된 if-elses 이해를위한 힌트 : 문자열에 특정 하위 문자열이 포함되어 있지 않으면 문자열을 해당 하위 문자열로 분할하는 반환 값은 원래 문자열과 동일합니다. 코드로 :

여기

use Test::More; 
my ($string, $substring) = ("foo+bar", "-"); # try it yourself! 
my ($split) = split /\Q$substring\E/, $string; 
if (-1 == index $string, $substring) { 
    is $split, $string; 
} else { 
    isnt $split, $string; 
} 
done_testing; 

는 청소 버전 :

#!/usr/bin/perl 

use strict; use warnings; 

use HTML::TreeBuilder 5 -weak; 
use HTML::Query; 

my $dir = '/home/mark/Documents/Perl/garchivesfiles/completeresults'; 

opendir my $dh, $dir or die "Can't open $dir: $!"; 

while (my $filename = readdir $dh) { 
    next if $filename =~ /^\./; 
    my $q = HTML::Query->new(file => "$dir/$filename"); 

    my $homescore = $q->query("span.homeScore")->first->as_text; 
    my $awayscore = $q->query("span.awayScore")->first->as_text; 

    my $hometeam = correctTeamName($q->query("table.teaminfo td.home span")->first->as_text); 
    my $awayteam = correctTeamName($q->query("table.teaminfo td.away span")->first->as_text); 

    my @allinfogoals; 

    if($homescore ne "0" || $awayscore ne "0") { 

     my ($fixtureinfo_span) = $q->query("p.fixtureinfo span"); 
     my (undef, $day, $month, $year) = split ' ', $fixtureinfo_span->as_text; 
     my $mysqldate = sprintf '%04d-%02d-%02d', $year, monthConvert($month), $day; 

     my ($fixtureinfo) = $q->query('p.fixtureinfo'); 
     my (undef, $fixtureinfostring) = split/\| /, $fixtureinfo->as_text; 

     for my $goal_list (
      [$hometeam, $awayteam, [$q->query("div.home ul li")->as_text]], 
      [$awayteam, $hometeam, [$q->query("div.away ul li")->as_text]] 
     ) { 
      my ($thisteam, $otherteam, $goalstotal) = @$goal_list; 
      for my $goal (@$goalstotal) { 
       $goal =~ s/\)//g; 
       my ($tempcomponent_1, $tempcomponent) = split/\(/, $goal; 

       for my $individmultgoal (split/,/, $tempcomponent) { 
        next if -1 != index $individmultgoal, 'OG'; 
        $individmultgoal =~ s/Pen//g; 
        $individmultgoal =~ s/ //g; 
        my @timesplit = 
         (index($individmultgoal, "+") != -1) 
         ? (split /\+/, $individmultgoal) 
         : ($individmultgoal, 0); 
        push @allinfogoals, [ 
         $tempcomponent_1, 
         $timesplit[0], 
         "for:$thisteam", 
         $otherteam, 
         $day, 
         $month, 
         $year, 
         $fixtureinfostring, 
         "Barclays Premier League", 
         monthConvert($month), 
         $mysqldate, 
         $timesplit[1], 
        ]; 
       } 
      } 
     } 

     @allinfogoals = sort { $a->[1] <=> $b->[1] || $a->[12] <=> $b->[12] } @allinfogoals; 

     open my $GOALCSV, '>>', 'goalcsv.txt' or die "Can't open goalcsv.txt: $!"; 

     my $print_both = sub { 
      print {$GOALCSV} @_; 
      print   @_; 
     }; 

     my $homegoalcount = 0; 
     my $awaygoalcount = 0; 

     for my $row (@allinfogoals){ 
      for my $val(@$row){ 
       if($val eq "for:$hometeam") { 
        $homegoalcount++; 
        $print_both->("$val,$homegoalcount,$awaygoalcount,true,"); 
       } elsif($val eq "for:$awayteam") { 
        $awaygoalcount++; 
        $print_both->("$val,$awaygoalcount,$homegoalcount,false,"); 
       } else { 
        $print_both->("$val,"); 
       } 
      } 
      $print_both->("\n"); 
     } 
    } 
} 

closedir $dh; 

sub correctTeamName{ 
    my %teamnames = (
     "Nott'm Forest" => "Nottingham Forest", 
     "QPR"   => "Queens Park Rangers", 
     "Southampton" => "Southampton FC", 
     "Norwich"  => "Norwich City", 
     "Tottenham"  => "Tottenham Hotspur", 
     "Leeds"   => "Leeds United", 
     "Middlesbrough" => "Middlesbrough FC", 
     "Chelsea"  => "Chelsea FC", 
     "Arsenal"  => "Arsenal FC", 
     "Oldham"  => "Oldham Athletic", 
     "Ipswich"  => "Ipswich Town", 
     "Man Utd"  => "Manchester United", 
     "Man City"  => "Manchester City", 
     "Sheffield Wed" => "Sheffield Wednesday", 
     "Man City"  => "Manchester City", 
     "Blackburn"  => "Blackburn Rovers", 
     "Wimbledon"  => "AFC Wimbledon", 
     "Liverpool"  => "Liverpool FC", 
     "Coventry"  => "Coventry City", 
    ); 
    return exists $teamnames{$_[1]} ? $teamnames{$_[1]} : $_[0]; 
} 

sub monthConvert{ 
    my $i = 1; 
    my %months = map { $_ => $i++ } qw/ 
     January February March 
     April May   June 
     July August  September 
     October November December 
    /; 
    exists $months{$_[0]} or die "Unknown month name $_[0]"; 
    return $months{$_[0]}; 
} 

참고 : 예제 파일이 제공되지 않았다 같은 코드, 테스트되지 않은 것입니다. 적어도 컴파일됩니다.

Answer 2

대부분 하나 이상의 파일이 매우 많습니다.

파일 이름을 출력합니다. 매번 코드 중 하나에 코드 분할이 표시됩니다.