#!/usr/bin/perl

# fold_reports1.pl $HOME/tmp/missing.txt $HOME/tmp/packets_per_exposure.txt $translate_log $HOME/tmp/exp_missing.txt $HOME/tmp/reformat_summary.txt $HOME/tmp/seq_cont.txt $rescue_translation_log $data_loss_log

use Time::Local;
use Date::Calc;
use Env qw(EGSE_SW_LIB);

do "$EGSE_SW_LIB/date_calc.pl"     || die "Can't do date_calc.pl: $!\n";
do "$EGSE_SW_LIB/pipeline_defs.pl" || die "Can't do pipeline_defs.pl: $!\n";

###die "Wrong number of arguments" if $#ARGV != 6;

$missing = shift;	# $HOME/tmp/missing.txt
$counts  = shift;	# $HOME/tmp/packets_per_exposure.txt
$exposures = shift;	# $translate_log ($HOME/data/staging/logs/md_translation/md_translation_yyyymmdd.txt)
$exp_missing = shift;	# $HOME/tmp/exp_missing.txt
$output  = shift;	# $HOME/tmp/reformat_summary.txt
$cont_check = shift;	# $HOME/tmp/seq_cont.txt

# New
$rescued = shift;	# $pipeline_log/md_translation_rescue_yyyymmdd.txt
%details = ();
# /New

# New mcrw 20140506 Get data for blank pixels
$blank_pixels = shift;  # $HOME/data/staging/logs/data_loss/yyyymmdd_data_loss.txt
# /New

open(FR_LOG, "> $HOME/tmp/fold_reports1_log.txt") or die "Can't open fold reports log: $!\n";

print FR_LOG "missing     = $missing\n";
print FR_LOG "counts      = $counts\n";
print FR_LOG "exposures   = $exposures\n";
print FR_LOG "exp_missing = $exp_missing\n";
print FR_LOG "output      = $output\n";
print FR_LOG "cont_check  = $cont_check\n";
print FR_LOG "rescued     = $rescued\n";

# $HOME/tmp/missing.txt
###open(MISS, "< $missing") or die "Can't open missing file: $!";
my $number_of_items = 0;
if(open(MISS, "< $missing")) {
    while(<MISS>) {
	next if /^Filename/;
	chomp;
	# $fname is ccsds packet file name
	($fname, $rec, $miss, $gaps, $date) = split;
	$details{$fname} = [$rec,               #  0 ccsds packets received
			    $miss,              #  1 ccsds packets missing
			    $gaps,              #  2 number of gaps between ccsds packets
			    "Failed",           #  3 
			    'ppe',              #  4 
			    'nexps',            #  5 
			    'tlid',             #  6 
			    'req',              #  7 
			    'done',             #  8 
			    'rid',              #  9 
			    'mid',              # 10 
			    'fx',               # 11 
			    'fy',               # 12 
			    'expperpos',        # 13 
			    'vol',              # 14 
			    'acr',              # 15 
			    'tot_exps_expect',  # 16 
			    'md_hdrs_expect',   # 17 
			    '',                 # 18 fits filename
			    'hdrs',             # 19 
			    'start',            # 20 
			    'end',              # 21 
			    'diff',             # 22 
			    'rec',              # 23 
			    0.00,               # 24 percent blank pixels
			    0.00,		# 25 compression factor
			    'N/A',		# 26 compression type
	    ];
	++$number_of_items;
    }
    close MISS;
    print FR_LOG "\n\n Read $number_of_items from $missing\n";
}
else {
    print FR_LOG "Can't open missing files file: $!\n";
#    die "Can't open missing file: $!";
}

# $HOME/tmp/packets_per_exposure.txt
###open(CNT, "< $counts") or die "Can't open counts file: $!";
$number_of_items = 0;
if(open(CNT, "< $counts")) {
    while(<CNT>) {
	next if /Filename/;
	chomp;
	($fname, $ppe, $nexps, $tlid, $req, $done, $rid, $mid, $fx, $fy, $expperpos, $vol, $acr) = split;
	$tot_exps_expect = $nexps;
	$md_hdrs_expect  = $tot_exps_expect * $ppe;
#	push @{$details{$fname}}, ($ppe, $nexps, $tlid, $req, $done, $rid, $mid, $fx, $fy, $expperpos, $vol, $acr, $tot_exps_expect, $md_hdrs_expect);
=pod deprecated mcrw 20200211
	@{$details{$fname}}->[4] = $ppe;
	@{$details{$fname}}->[5] = $nexps;
	@{$details{$fname}}->[6] = $tlid;
	@{$details{$fname}}->[7] = $req;
	@{$details{$fname}}->[8] = $done;
	@{$details{$fname}}->[9] = $rid;
	@{$details{$fname}}->[10] = $mid;
	@{$details{$fname}}->[11] = $fx;
	@{$details{$fname}}->[12] = $fy;
	@{$details{$fname}}->[13] = $expperpos;
	@{$details{$fname}}->[14] = $vol;
	@{$details{$fname}}->[15] = $acr;
	@{$details{$fname}}->[16] = $tot_exps_expect;
	@{$details{$fname}}->[17] = $md_hdrs_expect;
=cut
	$details{$fname}[4] = $ppe;
	$details{$fname}[5] = $nexps;
	$details{$fname}[6] = $tlid;
	$details{$fname}[7] = $req;
	$details{$fname}[8] = $done;
	$details{$fname}[9] = $rid;
	$details{$fname}[10] = $mid;
	$details{$fname}[11] = $fx;
	$details{$fname}[12] = $fy;
	$details{$fname}[13] = $expperpos;
	$details{$fname}[14] = $vol;
	$details{$fname}[15] = $acr;
	$details{$fname}[16] = $tot_exps_expect;
	$details{$fname}[17] = $md_hdrs_expect;

	++$number_of_items;
    }
    close CNT;
    print FR_LOG "Read $number_of_items from $counts\n";
}
else {
    print FR_LOG "Can't open counts file: $!\n";
#    die "Can't open counts file: $!";
}

# $translate_log ($HOME/data/staging/logs/md_translation/md_translation_yyyymmdd.txt)
###open(EXP, "< $exposures") or die "Can't open missing file: $!";
$number_of_items = 0;
if(open(EXP, "< $exposures")) {
    while(<EXP>) {
	chomp;
	($fname, $fffname) = split;
#	push @{$details{$fname}}, ($fffname);
=pod deprecated mcrw 20200211
	@{$details{$fname}}->[18] = $fffname;
	@{$details{$fname}}->[3] = "Ok";
=cut
	$details{$fname}[18] = $fffname;
	$details{$fname}[3] = "Ok";

	++$number_of_items;
    }
    close EXP;
    print FR_LOG "Read $number_of_items from $exposures\n";
}
else {
    print FR_LOG "Can't open exposures log: $!\n";
#    die "Can't open translate log: $!";
}

# $HOME/tmp/exp_missing.txt
###open(EXP, "< $exp_missing") or die "Can't open exposures missing file: $!";
$number_of_items = 0;
if(open(EXP, "< $exp_missing")) {
    while(<EXP>) {
	next if /Filename/;
	chomp;
	($fname, $hdrs) = split;
#	push @{$details{$fname}}, ($hdrs);
=pod deprecatd mcrw 20200211
	@{$details{$fname}}->[19] = $hdrs;
=cut
	$details{$fname}[19] = $hdrs;

	++$number_of_items;
    }
    close EXP;
    print FR_LOG "Read $number_of_items from $exp_missing\n";
}
else {
    print FR_LOG "Can't open exposures missing log: $!\n";
#    die "Can't open exposures missing file: $!";
}


# $HOME/tmp/seq_cont.txt
###open(EXP, "< $cont_check") or die "Can't open sequence continuity check file ($cont_check): $!";
$number_of_items = 0;
if(open(EXP, "< $cont_check")) {
    while(<EXP>) {
	chomp;
	next if /Filename/;
	($fname, $start, $end, $diff, $rec) = split;
#	push @{$details{$fname}}, ($start, $end, $diff, $rec);
=pod deprecated mcrw 20200211
	@{$details{$fname}}->[20] = $start;
	@{$details{$fname}}->[21] = $end;
	@{$details{$fname}}->[22] = $diff;
	@{$details{$fname}}->[23] = $rec;
=cut
	$details{$fname}[20] = $start;
	$details{$fname}[21] = $end;
	$details{$fname}[22] = $diff;
	$details{$fname}[23] = $rec;

	++$number_of_items;
    }
    close EXP;
    print FR_LOG "Read $number_of_items from $cont_check\n";
}
else {
    print FR_LOG "Can't open sequence continuity check log: $!\n";
#    die "Can't open sequence continuity check file ($cont_check): $!";
}

###open(RES, "< $rescued") or die "Can't open rescued file ($rescued): $!";
$number_of_items = 0;
if(open(RES, "< $rescued")) {
    while(<RES>) {
	chomp;
	next if /Filename/;
	($fname, $fffname) = split;
#	if(defined(@{$details{$fname}})) {
=pod deprecated mcrw 20200211
	@{$details{$fname}}->[18] = $fffname;
	@{$details{$fname}}->[3] = "Rescued";
=cut
	$details{$fname}[18] = $fffname;
	$details{$fname}[3] = "Rescued";

	++$number_of_items;
#    }
    }
    close RES;
    print FR_LOG "Read $number_of_items from $rescued\n";
}
else {
    print FR_LOG "Can't open rescued log ($rescued): $!\n";
#    die "Can't open rescued file ($rescued): $!";
}

# mcrw 20140512
goto skip_blank;
$number_of_items = 0;
if(open(BLNK, "< $blank_pixels")) {
    while(<BLNK>) {
	chomp;
	next if /Filename/;
	my @data = split;
	
	foreach $key (keys %details) {
	    if($details{$key} eq $data[0]) {
		
=pod deprecated mcrw 20200211
    		@{$details{$key}}->[24] = $data[5];
=cut
		$details{$key}[24] = $data[5];

		++$number_of_items;
		last;
	    }
	}
    }
    close BLNK;
    print FR_LOG "Read $number_of_items from $blank_pixels\n";
}
else {
    print FR_LOG "Can't open blank pixel file ($blank_pixels): $!\n";
}
skip_blank:

# mcrw 20141022 - add compression factor data
goto skip_factors;
my $comp_data = "$HOME/data/staging/logs/decompression/merge_decomp_record.txt";
my $entered = 0;
$number_of_items = 0;
if(open(FACTORS, "< $comp_data")) {
    while(<FACTORS>) {
	chomp;
	++$number_of_items;
	my ($fname,$in,$out,$percent,$factor,$comp,$type,$table) = split;
=pod deprecated mcrw 20200211
	next if(!defined(@{$details{$fname}}));
	if(@{$details{$fname}}->[18] ne '') {
	    @{$details{$fname}}->[25] = $factor;
	    @{$details{$fname}}->[26] = $type;
=cut
	next if(!defined($details{$fname}));
	if($details{$fname}[18] ne '') {
	    $details{$fname}[25] = $factor;
	    $details{$fname}[26] = $type;

	    ++$entered;
	}
    }
    close FACTORS;
    print FR_LOG "Read $number_of_items from $comp_data, used $number\n";
}
else {
    print FR_LOG "Can't open compression file ($comp_data): $!\n";
}
skip_factors:

# $HOME/tmp/reformat_summary.txt
####open(OUTPUT, "> $output") or die "Can't open output file: $!";
if(open(OUTPUT, "> $output")) {
    print FR_LG " Writing to $output\n";
    print OUTPUT "Filename                    Tl id    Rid   Req  Done                         Acronym     ExpExp ExpGot  Eloss% Ccsds Rec    Missing     Gaps  Status                      Fits File\n";

    $previous_fname = "";
    $egot_total = 0;
    foreach $fname (sort keys %details) {
=pod deprecated mcrw 20200211
	$tlid   = @{$details{$fname}}->[6];     # Timeline id
	$rid    = @{$details{$fname}}->[9];     # Raster id
	$req    = @{$details{$fname}}->[7];     # Rasters required
	$done   = @{$details{$fname}}->[8];     # Rasters performed
	$status = @{$details{$fname}}->[3];     # Reformat status
	$acr    = @{$details{$fname}}->[15];    # Acronym
	$rec    = @{$details{$fname}}->[0];     # Ccsds received 
	$miss   = @{$details{$fname}}->[1];     # Ccsds missing
# mcrw 20190827
#	$miss  += @{$details{$previous_fname}}->[22] if $previous_fname ne "";;
	$gaps   = @{$details{$fname}}->[2];     # Ccsds gaps
# mcrw 20190827
#	++$gaps if ((@{$details{$fname}}->[22] != 0) && ($previous_fname ne ""));
	$ff     = @{$details{$fname}}->[18];    # Fits filename
	$tote   = @{$details{$fname}}->[16];    # Total exposures
	$mde    = @{$details{$fname}}->[17];    # Mission data expected
	$ppe    = @{$details{$fname}}->[4];     # Packets per exposure
	$mds    = @{$details{$fname}}->[19];    # 

#	$blnk   = @{$details{$fname}}->[24];    # Blank pixels number
#	$fact   = @{$details{$fname}}->[25];    # Compression factor
#	$ctype  = @{$details{$fname}}->[26] || "Unknown";    # Compressions type
=cut
	$tlid   = $details{$fname}[6];     # Timeline id
	$rid    = $details{$fname}[9];     # Raster id
	$req    = $details{$fname}[7];     # Rasters required
	$done   = $details{$fname}[8];     # Rasters performed
	$status = $details{$fname}[3];     # Reformat status
	$acr    = $details{$fname}[15];    # Acronym
	$rec    = $details{$fname}[0];     # Ccsds received 
	$miss   = $details{$fname}[1];     # Ccsds missing
# mcrw 20190827
#	$miss  += @{$details{$previous_fname}}->[22] if $previous_fname ne "";;
	$gaps   = $details{$fname}[2];     # Ccsds gaps
# mcrw 20190827
#	++$gaps if ((@{$details{$fname}}->[22] != 0) && ($previous_fname ne ""));
	$ff     = $details{$fname}[18];    # Fits filename
	$tote   = $details{$fname}[16];    # Total exposures
	$mde    = $details{$fname}[17];    # Mission data expected
	$ppe    = $details{$fname}[4];     # Packets per exposure
	$mds    = $details{$fname}[19];    # 

#	$blnk   = @{$details{$fname}}->[24];    # Blank pixels number
#	$fact   = @{$details{$fname}}->[25];    # Compression factor
#	$ctype  = @{$details{$fname}}->[26] || "Unknown";    # Compressions type

	$egot = ($ppe != 0) ? $mds / $ppe : 0;
	
#    $epc  = ($tote != 0) ? ($egot / $tote) * 100.0 : 0.0;
	$epc  = ($mde != 0) ? 100.0 - (($mds / $mde)) * 100.0 : 100.0;
	$egot_total += $egot;
=pod deprecated mcrw 20200211	
	$start = @{$details{$fname}}->[20];
	$diff = @{$details{$fname}}->[22];
=cut
	$start = $details{$fname}[20];
	$diff = $details{$fname}[22];
	
	$previous_fname = $fname;
	write OUTPUT;
	
    }
    close OUTPUT;
}
else {
    print FR_LOG "Can't open $output : $!\n";
#    die "Can't open output file: $!";
}
close FR_LOG;

format OUTPUT = 
@<<<<<<<<<<<<<<<<<<<<<<<<< @##### @##### @#### @#### @>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>     @##### @##### @###.##   @######    @######  @######  @<<<<<<<  @<<<<<<<<<<<<<<<<<<<<<<<<<<
$fname, $tlid, $rid, $req, $done, $acr, $tote, $egot, $epc, $rec, $miss, $gaps, $status, $ff
.