#!/usr/local/bin/perl

#
# rescue_reformat1.pl 200803 (for example)
#

use Time::Local;
use Date::Calc;
use Env qw(EGSE_SW_LIB);

# Instead of die-ing would be better to report the error in a file somewhere
# and then exit. This applies to all instances of die of course. Todo.

do "$EGSE_SW_LIB/date_calc.pl"     || die "Can't do date_calc.pl: $!\n";
do "$EGSE_SW_LIB/pipeline_defs.pl" || die "Can't do pipeline_defs.pl: $!\n";
do "$EGSE_SW_LIB/pipeline.pl"      || die "Can't do pipeline.pl: $!\n";

$rescue_fits = "$HOME/work/localdata/pipeline/rescue_fits";

open(MLOG, ">> $pipeline_log/pipeline_log_rescue.txt") || die "Can't open $pipeline_log/$pipeline_log_rescue.txt for writing: $!";
print MLOG scalar(localtime), " (JST) rescue_reformat started ";

# The start time will either be passed in (interactive mode)
# or will be generated from cron using todays date minus 7 days

$state = "interactive"; # Default to cron job
#($log, $perform, $title) = init_log_and_perform(*LOG, "interactive");

@today = today();	# @today[0..6] = seconds,minutes,hours,day,month,year,day_of_week (0 = sunday)
$syear  = sprintf "%04u", $today[5];	# Get starting year
$smonth = sprintf "%02u", $today[4];	# Get starting month
$sday   = sprintf "%02u", $today[3];	# Get starting day

$year = $syear;
$month = $smonth;
$day = $sday;

$sdate = "$syear$smonth$sday";

###$sdate = $ARGV[0];
# Check to see if -m is set. If it is then there will be 2 args, $#ARGV will == 1
if($#ARGV == 1) { my $junk = shift; $merge_rescue_flag = 1; }
$sdate = shift;

$pipeline_mission_rescue_log = "$pipeline_log/pipeline_mission_rescue_log_rescue_$sdate";
open(LOG, ">> $pipeline_mission_rescue_log") || die "Can't open $pipeline_mission_rescue_log for writing: $!";
#    ($pipeline_log, $pipeline_perform, $pipeline_stage_title) = init_pipeline(*LOG, "interactive");
##    &$pipeline_stage_title("Started (manually)");
#    &$pipeline_log("rescue_reformat.pl $sdate");
print LOG "Started ", scalar(localtime), " (JST)\n";
print MLOG "manually for $sdate\n";

$date_string = $sdate;


############################################### 10 - MISSION DATA REFORMAT

#&$pipeline_stage_title("Starting mission data reformat");
stage_title(*LOG, $state, "Starting mission data reformat");
#print STDOUT scalar(localtime), " (JST) - Starting mission data reformat\n" if $state eq "interactive";
#print LOG "\n", scalar(localtime), " (JST) - Starting mission data reformat\n";
#print LOG "\t(IDL) fits_script, '$merge_dir/decompressed/', '$temp_fits/'\n";

#$date_string  = "$year$month$day";
$translate_log_name = "md_translation_rescue" . "_" . "$date_string" . ".txt";
$translate_log = "$pipeline_log/" . $translate_log_name;

$output = <<EOF;
.comp $local_idl/eis_read_orbit_events_reformatter

;.comp $local_idl/merge/eis_hdr__define
;.comp $local_idl/merge/eis_data__readfits

.comp $local_idl/merge/eis_plan__define

.comp $local_idl/merge/eis_cal__define
.comp $local_idl/merge/eis_data__define
.comp $local_idl/merge/eis_data__readccsds
.comp $local_idl/merge/eis_fits_calib
.comp $local_idl/merge/eis_fits_coord
.comp $local_idl/merge/eis_fits_dataid
;.comp $local_idl/merge/hw_hdr__define
;.comp $local_idl/merge/hw_data__define

;.comp $local_idl/merge/eis_spcd__define
.comp $local_idl/merge/eis_mkfits

!quiet=0
eis_fits_script,'$merge_dir/decompressed/', '$rescue_fits/',logfile='$translate_log', /rescued

EOF

# ;eis_fits_script, ... , /rescued, /noplan
# ;eis_fits_script, ... , /nospcd

open(MD_FITS, "> $temp_idl/md_fits.pro");
print MD_FITS $output;
close MD_FITS;

log_msg(*LOG, $state, "IDL file:\n$output");

@post_decompression = `ls $merge_dir/decompressed/ | grep eis_md`;
$number_post_decompression = @post_decompression;

$total_reformat_count = 0;
$more_files = 1;
#$repeats = 0;
while($more_files) {
#    $reformat_log = "$pipeline_log/md_reformat_rescue" . "_$date_string" . "_$repeats" . ".txt";
    $reformat_log = "$pipeline_log/md_reformat_rescue" . "_$date_string" . ".txt";
#    perform(*LOG, $state, "/nasA_Solar1/home/sbukeis/ssw_idl < $temp_idl/md_fits.pro >> $reformat_log 2>&1");
    perform(*LOG, $state, "$HOME/ssw_idl < $temp_idl/md_fits.pro >> $reformat_log 2>&1");

    # Get a count of number of mission data files here
    @reformat_count = `ls $rescue_fits/ | grep eis_l0`;
    $reformat_count = @reformat_count;
    $total_reformat_count += $reformat_count;
    last if $total_reformat_count == $number_post_decompression;
    
    # Reformatter failed on a file.
    # Remove decompressed files which have been reformatted.
    # Move existing fits to temp_fits directory
    # Move failed source file back to incomplete directory.
    # Start reformat again
    if($reformat_count > 0) {
	while($reformat_count--) {
#foreach $count (0 .. ($reformat_count - 1)) {
	    $file = shift @post_decompression;
	    chomp $file;
	    perform(*LOG, $state, "/bin/rm $merge_dir/decompressed/$file");
	}
    }
    $file = shift @post_decompression;
    chomp $file;
    perform(*LOG, $state, "/bin/mv $merge_dir/rescue/$file $merge_dir/incomplete/");
    perform(*LOG, $state, "/bin/rm $merge_dir/decompressed/$file");
    perform(*LOG, $state, "/bin/mv $rescue_fits/* $temp_fits");
###    perform(*LOG, $state, "/bin/mv $rescue_fits/* $temp_temp_fits");
    perform(*LOG, $state, "echo >> $reformat_log");
    perform(*LOG, $state, "echo $file failed. Restarting. >> $reformat_log");
    perform(*LOG, $state, "echo >> $reformat_log");
    ++$total_reformat_count;
#    ++$repeats;
    $more_files = ($total_reformat_count < $number_post_decompression);
}

perform(*LOG, $state, "/bin/mv $rescue_fits/* $temp_fits");
###perform(*LOG, $state, "/bin/mv $rescue_fits/* $temp_temp_fits");
perform(*LOG, $state, "/bin/cp $reformat_log $darts_staging/logs/md_reformat/rescue/");
perform(*LOG, $state, "/bin/cp $translate_log $darts_staging/logs/md_translation/rescue/");

# Get a count of number of mission data files here
@post_reformat = `ls $temp_fits/eis_l0*`;
###@post_reformat = `ls $temp_temp_fits/eis_l0*`;
#print STDOUT "Number of reformatted files: ", scalar(@post_reformat), "\n" if $state eq "interactive";
#print LOG  "\tNumber of reformatted files: ", scalar(@post_reformat), "\n";
log_msg(*LOG, $state, "Number of reformatted files: " . scalar(@post_reformat));;

goto skip1 if $merge_rescue_flag;

############################################### 5 - UPDATE MD TRANSLATION DBASE

#$current_time = scalar(localtime);
#print STDOUT $current_time, " (JST) - Update md translation database\n" if $state eq "interactive";
#print LOG "\n", $current_time, " (JST) - Update md translation database\n";
stage_title(*LOG, $state, "Update md translation database");

# eis_mkfits writes all the ccsds packet filenames and the corresponding fits filenames processed
# to a text file. Open the text file and transfer the information to a perl database.

if(dbmopen(%trans, "$pipeline_log/rescue_mission_name_dbase", 0644)) {
#    print STDOUT "rescue_mission_name_dbase opened\n" if $state eq "interactive";
#    print LOG "rescue_mission_name_dbase opened\n";
    log_msg(*LOG, $state, "rescue_mission_name_dbase opened");
    if(open(REF_LOG, "< $darts_staging/logs/md_translation/rescue/$translate_log_name")) {
#	print STDOUT "Opened $translate_log ok\n" if $state eq "interactive";
#	print LOG "Opened $translate_log ok\n";
	log_msg(*LOG, $state, "Opened $translate_log_name ok");
	while(<REF_LOG>) {
	    @files = split;
	    $ccsds = $files[0];
	    $fits = $files[1];
#	    ($ccsds, $fits) = split;
	    $ccsds =~ s/ //g;
	    $fits =~ s/ //g;
	    chomp $fits;
#	    print STDOUT "UPDATE: (ccsds,fits) = ($ccsds,$fits)\n";
#	    print LOG "Updating mission_name_database: $ccsds => $fits\n";
	    log_msg(*LOG, $state, "Updating mission_name_database: $ccsds => $fits");
	    $trans{$ccsds} = $fits;
	}
	close(REF_LOG);
    }
    else {
#	print STDOUT "Can't open $reformat_log: $!\n" if $state eq "interactive";
#	print LOG "Can't open $reformat_log: $!";
	log_msg(*LOG, $state, "Can't open $reformat_log: $!");
    }
#    print STDOUT "Closing rescue_mission_name_dbase\n" if $state eq "interactive";
#    print LOG  "\tClosing rescue_mission_name_dbase\n";
    log_msg(*LOG, $atate,  "\tClosing rescue_mission_name_dbase");
    if(dbmclose(%trans)) {
#	print STDOUT "Closed rescue_mission_name_dbase ok\n" if $state eq "interactive";
#	print LOG  "\tClosed rescue_mission_name_dbase ok\n";
	log_msg(*LOG, $state,  "Closed rescue_mission_name_dbase ok");
    }
    else {
#	print STDOUT "Closing rescue_mission_name_dbase failed: $!\n" if $state eq "interactive";
#	print LOG  "\tClosing rescue_mission_name_dbase failed: $!\n";
	log_msg(*LOG, $state, "Closing rescue_mission_name_dbase failed: $!");
    }
}
else {
#    print STDOUT "Cannot open rescue_mission_name_dbase\n" if $state eq "interactive";
#    print LOG "Cannot open rescue_mission_name_dbase\n";
    log_msg(*LOG, $state, "Cannot open rescue_mission_name_dbase");
}

# Copy the database to the staging area even if it was not updated
#print STDOUT "/bin/cp $pipeline_log/rescue_mission_name_dbase $darts_staging/logs/\n" if $state eq "interactive";
#print LOG  "\t/bin/cp $pipeline_log/rescue_mission_name_dbase $darts_staging/logs/\n";
#system("/bin/cp $pipeline_log/rescue_mission_name_dbase $darts_staging/logs/");
perform(*LOG, $state, "/bin/cp $pipeline_log/rescue_mission_name_dbase $darts_staging/logs/");


############################################### 5A - GENERATE FITS HEADER DUMP

#&$pipeline_stage_title("Generating fits header dump");
stage_title(*LOG, $state, "Generating fits header dump");
#print STDOUT "Generating fits header dump\n" if $state eq "interactive";
#print LOG "\n", scalar(localtime), " (JST) - Generating fits header dump\n";

#&$pipeline_perform("$HOME/bin/fits_study_metric.pl $temp_fits/* > $pipeline_log/fits_dump_$year$month$day.txt");
perform(*LOG, $state, "$HOME/bin/fits_study_metric.pl $temp_fits/* > $pipeline_log/fits_dump_rescue_$sdate.txt");
#print LOG "\t$HOME/bin/fits_study_metric.pl $temp_fits/* > $pipeline_log/fits_dump_rescue_$year$month$day.txt\n";
#system("$HOME/bin/fits_study_metric.pl $temp_fits/* > $pipeline_log/fits_dump_rescue_$year$month$day.txt");

#&pipeline_perform("/bin/cp $pipeline_log/fits_dump_$year$month$day.txt $HOME/data/staging/packet_summary");
perform(*LOG, $state, "/bin/cp $pipeline_log/fits_dump_rescue_$sdate.txt $HOME/data/staging/packet_summary");
#print LOG "\t/bin/mv $pipeline_log/fits_dump_rescue_$year$month$day.txt $HOME/data/staging/packet_summary\n";
#system("/bin/mv $pipeline_log/fits_dump_rescue_$year$month$day.txt $HOME/data/staging/packet_summary");


############################################### 6 - COMPRESS

# Now gzip everything in the temporary fits directory
# before moving to the DARTS directories

stage_title(*LOG, $state, "Gzip temporary fits files");
#$current_time = scalar(localtime);
#print STDOUT $current_time, " (JST) - Gzip temporary fits files\n" if $state eq "interactive";
#print LOG "\n", $current_time, " (JST) - Gzip temporary fits files\n";
#print LOG "\tgzip -f $temp_fits/*\n";
#system("gzip -f $temp_fits/*");
perform(*LOG, $state, "gzip -f $temp_fits/*");


############################################### 7 - COPY ENGINEERING FILES TO CALIBRATION AREA

goto skip;

$current_time = scalar(localtime);
print STDOUT $current_time, " (JST) - Copy engineering files to calibration area\n" if $state eq "interactive";
print LOG "\n", $current_time, " (JST) - Copy engineering files to calibration area\n";

# If the engineering.txt file exists then there is engineering data to be dealt with.
# Use the ccsds packet filename from the engineering.txt file to look up in the translation
# log to get the fits filename and use the engineering study type information to copy the
# fits file to the calibration directory.

$eng_dbase = 0;
if(-e "$merge_dir/engineering.txt") {
    print STDOUT "Engineering summary exists\n" if $state eq "interactive";
    print LOG "Engineering summary exists\n";
    $eng_dbase = 1 if(dbmopen(%eng_dbase, "$pipeline_log/engineering_name_dbase", 0644));
    if(dbmopen(%trans, "$pipeline_log/mission_name_dbase", undef)) {
	if(open(ENG_LOG, "< $merge_dir/engineering.txt")) {
	    $line = <ENG_LOG>;				# Read title
	    while($line = <ENG_LOG>) {			# Read ccsds packet filename
		last if $line =~ /^$/;			# Finish on a blank line
		chop $line;				# Remove \n
		($path, $type) = split(/:/, $line);	# Get filename and type of engineering study
		print LOG "(path,type) = ($path, $type)\n";
		@path = split(/\//, $path);		# Split up filename path
		$ccsds_file = $path[-1];
#		$fits_file = $trans{$path[-1]};		# Get fits filename from ccsds filename
		$fits_file = $trans{$ccsds_file};	# Get fits filename from ccsds filename
		if($fits_file) {
		    $eng_dbase{$fits_file} = $type if $eng_dbase;	# Update engineering database
		    $destination = eng_path($type);		# Get where to copy the fits file and copy it
		    $source = $temp_fits . "/"  . $fits_file . ".gz";
#		    system("/bin/cp $temp_fits/$fits_file.gz $destination") if($destination);
		    if($destination ne "") {
			$dest1 = "$darts_mission" . $destination;
			$dest2 = "$soda_darts_mission" . $destination;
			print STDOUT "Copying $source to $dest1\n" if $state eq "interactive";
			print LOG  "\tCopying $source to $dest1 and $dest2\n";
			system("/bin/cp $source $dest1");
			system("/bin/cp $source $dest2");
		    }
		}
		else {
		    print STDOUT "No translation found for $ccsds_file\n" if $state eq "interactive";
		    print LOG "No translation found for $ccsds_file\n";
		}
	    }
	    close(ENG_LOG);
	}
	dbmclose(%trans);
    }
    dbmclose(%eng_dbase) if($eng_dabse);
    system("/bin/mv $merge_dir/engineering.txt $HOME/data/mission/$year/$month/engineering.$sdate.txt");
}
else {
    print LOG "No engineering summary file\n";
}
# Copy the engineering database to the staging area even if it was not updated
#print STDOUT "/bin/cp $pipeline_log/engineering_name_dbase $darts_staging/logs/\n" if $state eq "interactive";
#print LOG  "\t/bin/cp $pipeline_log/engineering_name_dbase $darts_staging/logs/\n";
#system("/bin/cp $pipeline_log/engineering_name_dbase $darts_staging/logs/");

 skip:

############################################### 8 - COPY MISSION FITS TO DARTS
    goto skip1;

stage_title(*LOG, $state, "Moving mission data fits files to DARTS");

#
# Filenames:
#	eis_l0_yyyymmdd_hhmmss.fits
#
@files = `ls $temp_fits`;
foreach $file (@files) {
    next if !($file =~ /l0/);
    chomp $file;
    # Match destination directory with filename
    $darts_year  = substr $file,  7, 4;
    $darts_month = substr $file, 11, 2;
    $darts_day   = substr $file, 13, 2;
    $soda_md_dir = "$soda_darts_mission" . "/" . "$darts_year" . "/" . "$darts_month" . "/" . "$darts_day";
    perform(*LOG, $state, "/bin/cp $temp_fits/$file $soda_md_dir");
#    print "/bin/mv $temp_fits/$file $soda_md_dir\n";
#    system("/bin/mv $temp_fits/$file $soda_md_dir");
}

# Generate the local study descriptions file
#perform(*LOG, $state, "fits_study_metric.csh $temp_fits >  $HOME/tmp/studies.txt");
#perform(*LOG, $state, "sed -e '/SSW/,/Type/d' $HOME/tmp/studies.txt > $HOME/tmp/rescue_studies.html");
#perform(*LOG, $state, "/bin/rm $HOME/tmp/studies.txt");

skip1:
############################################### 15 - CLEAN UP

#&$pipeline_stage_title("Cleaning up");
stage_title(*LOG, $state, "Cleaning up");
#print STDOUT scalar(localtime), " (JST) - Cleaning up\n" if $state eq "interactive";
#print LOG "\n", scalar(localtime), " (JST) - Cleaning up\n";

#&pipeline_perform("/bin/rm $temp_idl/*pro");
#print LOG "\t/bin/rm $temp_idl/*pro\n";
#system("/bin/rm $temp_idl/*pro");
perform(*LOG, $state, "/bin/rm $temp_idl/*pro");

#print LOG "\t/bin/rm $temp_fits/*\n";
#system("/bin/rm $temp_fits/*");

pipeline_exit("rescue", 0) unless $merge_rescue_flag;