#!/usr/local/bin/perl

=pod

=head1 NAME

daily_merge_status1.pl

=head1 SYNOPSIS

daily_merge_status1.pl [options] [start_date end_date] [attempt]

=over 1

=item C<daily_merge_status1.pl -c>

Started by the cron job in order to process new status data.

=item C<daily_merge_status1.pl -c 20180601 20180602 1>

Started by the cron job when attempting to process pending data.
The last number is the current attempt.
Cron will try 7 times (7 subsequent days) to process the data for this day.
After 7 days this date is removed from the pending list ($HOME/track/status_pending.txt)

=item C<daily_merge_status1.pl 20180601 20180602>

Usage when started interactively at the command line in order to process the data for a particular day.

=back

=head1 FILES

=over

=item $HOME/track/status_pending.txt

=item $HOME/track/latest_status_reformat.txt

=back

=head1 AUTHOR

Matt Whillock (L<m.whillock@ucl.ac.uk>)

=cut


use Getopt::Long;
use Time::Local;
use Date::Calc;
use Env qw(EGSE_SW_LIB);

###use Hinode::EIS::EisPipeline::pipeline_defs;

$date_calc_fail = $pipeline_defs_fail = $pipeline_fail = 0;

#do "$EGSE_SW_LIB/date_calc.pl"     || die "Can't do date_calc.pl: $!\n";
#do "$EGSE_SW_LIB/pipeline_defs.pl" || die "Can't do pipeline_defs.pl: $!\n";
#do "$EGSE_SW_LIB/pipeline.pl"      || die "Can't do pipeline.pl: $!\n";
do "$EGSE_SW_LIB/date_calc.pl"     or $date_calc_fail = 1;
do "$EGSE_SW_LIB/pipeline_defs.pl" or $pipeline_defs_fail = 1;
do "$EGSE_SW_LIB/pipeline.pl"      or $pipeline_fail = 1;

#print FOO "Done do...\n";
#print FOO "Library reads: $date_calc_fail | $pipeline_defs_fail | $pipeline_fail\n";
#print FOO "Trying to open $pipeline_log/pipeline_log.txt\n";

open(MLOG, ">> $pipeline_log/pipeline_log.txt") || die "Can't open $pipeline_log/pipeline_log.txt for writing: $!";
print MLOG scalar(localtime), " (JST) daily_merge_status1 started ";

#print FOO "Opened $pipeline_log/pipeline_log.txt\n";
#close(FOO);

if ($date_calc_fail || $pipeline_defs_fail || $pipeline_fail) {
    print MLOG "Library read failed ($date_calc_fail, $pipeline_defs_fail, $pipeline_fail)\n";
    pipeline_exit("status", 99);
}

# The start time will either be passed in (interactive mode)
# or will be generated from cron using todays date minus 7 days

$state = "cron"; # Default to cron job

$ant='usc34';	# Not needed for merge as sdtp will substitute merge ant
$band=3;	# Merge telemetry
#$mode='merge';	# Merge telemetry
$mode='status_merge';	# Merge telemetry

# Set default values for start date, end date and start time to empty strings. If in
# interactive state these are not set then bail out.

$sdate = "";
$edate = "";
$stime = "";


# Moved this up from the cron branch in case there are so many catching-up reformats the date goes past
# midnight before the cron branch starts. This would lead to a gap in the data.
@today = today();	# @today[0..6] = seconds,minutes,hours,day,month,year,day_of_week (0 = sunday)

# If there is anything on the command line then this is interactive mode

###
# Check for -c: this option indicates the program has been run by cron...
###

# If ARGV is -1 then no command line parameters have been supplied.
# ARGV is always one less than the number of command line parameters.
# USAGE: daily_merge_status1.pl [options] [start_date end_date] [attempt]
# where:
#	-c started by cron

my $run_message = "Started (manually) ";
my $CHECK_ONLY = 0;
my $NO_UPDATE_DARTS = 0;

my $argv_count = $#ARGV;
my $options = "";
if($#ARGV != -1) {
    $state = "interactive";

    while($#ARGV + 1) {
	if ($#ARGV == 1) {
	    if(@ARGV) { $sdate = shift; }
	    if(@ARGV) { $edate = shift; }
	    last;
	}
	my $arg = shift;
	if ($arg eq "-p") {
	    $state = "cron pending";
	    $run_message = "Started (cron pending) ";
	}
	if ($arg eq "CHECK_ONLY") {
	    $CHECK_ONLY = 1;
	    $options .= " CHECK_ONLY";
	}
	if ($arg eq "NO_UPDATE_DARTS") {
	    $NO_UPDATE_DARTS = 1;
	    $options .= " NO_UPDATE_DARTS";
	}
    }

    $pipeline_status_log = "$pipeline_log/pipeline_status_log_$sdate.txt";

    pipeline_die("status", 10, "No start date") if $sdate eq "";
    pipeline_die("status", 20, "No end date")   if $edate eq "";
    open(LOG, ">> $pipeline_status_log") || die "Can't open $pipeline_status_log for writing: $!";
    open(LOG, ">> $pipeline_status_log") || pipeline_die("status", 30, "Can't open $pipeline_status_log for writing: $!");
#    ($pipeline_log, $pipeline_perform, $pipeline_stage_title) = init_pipeline(*LOG, "interactive");
#    &$pipeline_stage_title("Started (manually)");
#    &$pipeline_log("daily_merge_status1 $sdate $edate");
###    print LOG "Started (manually) " . scalar(localtime) . " (JST)\n";
    print LOG $run_message . scalar(localtime) . " (JST)\n";
###    ($log, $perform, $title) = init_log_and_perform(*LOG, "interactive");
    ($log, $perform, $title) = init_log_and_perform(*LOG, $state);

    if ($state eq "interactive") {
	print MLOG "manually: daily_merge_status1.pl $options $sdate $edate\n";
    }
    else {
	print MLOG "cron pending: daily_merge_status1.pl $sdate $edate\n";
    }
}
else {
    # Started by cron. Calculate start date, end date and start time
######    @today = today();	# @today[0..6] = seconds,minutes,hours,day,month,year,day_of_week (0 = sunday)

#    @start_day = other_date(@today,-5);		# Go back 5 days
#    @next_day  = other_date(@today,-4);		# Go back 4 days
    @start_day = other_date(@today,-4);		# Go back 4 days
    @next_day  = other_date(@today,-3);		# Go back 3 days

    $syear  = sprintf "%04u", $start_day[5];	# Get starting year
    $smonth = sprintf "%02u", $start_day[4];	# Get starting month
    $sday   = sprintf "%02u", $start_day[3];	# Get starting day
    $eyear  = sprintf "%04u", $next_day[5];	# Get ending year
    $emonth = sprintf "%02u", $next_day[4];	# Get ending month
    $eday   = sprintf "%02u", $next_day[3];	# Get ending day
    
    $sdate = "$syear$smonth$sday";
    $edate = "$eyear$emonth$eday";
    
    $pipeline_status_log = "$pipeline_log/pipeline_status_log_$sdate.txt";

#    open(LOG, ">> $pipeline_status_log") || die "Can't open $pipeline_status_log for writing: $!";
    open(LOG, ">> $pipeline_status_log") || pipeline_die("status", 40, "Can't open $pipeline_status_log for writing: $!");
#    ($pipeline_log, $pipeline_perform, $pipeline_stage_title) = init_pipeline(*LOG, "cron");
#    &$pipeline_stage_title("Started (by cron)");
    print LOG "Started ", scalar(localtime), " (JST)\n";
    ($log, $perform, $title) = init_log_and_perform(*LOG, "cron");
#    ($pipeline_log, $pipeline_perform, $pipeline_stage_title) = init_pipeline(*LOG, "cron");
    print MLOG "by cron: daily_merge_status1.pl $sdate $edate\n";
}

# Split the start date into constituent parts
$year  = substr $sdate, 0, 4;
$month = substr $sdate, 4, 2;
$day   = substr $sdate, 6, 2;

# And bung them back again
$ymd = "$year$month$day";

goto check_only if $CHECK_ONLY;

# Start times
@stimes=('0000','0130','0300','0430','0600','0730','0900','1030','1200','1330','1500','1630','1800','1930','2100');

# End times
@etimes=('0130','0300','0430','0600','0730','0900','1030','1200','1330','1500','1630','1800','1930','2100','2230');

############################################### 0 - CLEAR OLD DATA


#&$pipeline_stage_title("Removing old data in $merge_dir");
print STDOUT scalar(localtime), " (JST) - Removing old data in $merge_dir\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Removing old data in $merge_dir\n";

print LOG "\t/bin/rm -f $merge_dir/eis_*\n";
system("cd $merge_dir && /bin/rm -f eis_md_2*");
system("cd $merge_dir && /bin/rm -f eis_sts*");
system("cd $merge_dir && /bin/rm -f eis_dmp*");
#&$pipeline_log("/bin/rm -f $merge_dir/eis_*)";
#&$pipeline_perform("cd $merge_dir && /bin/rm -f eis_md_2*");
#&$pipeline_perform("cd $merge_dir && /bin/rm -f eis_sts*");
#&$pipeline_perform("cd $merge_dir && /bin/rm -f eis_dmp*");

# Used in the missing packets section
my $timing_dir = "$HOME/work/localdata/log/merge";
print LOG "\t/bin/rm -f $timing_dir/2* $timing_dir/tmp/* $darts_timing_directory/*\n";
system("/bin/rm $timing_dir/2* $timing_dir/tmp/* $darts_timing_directory/*");
#&$pipeline_perform("/bin/rm $timing_dir/2* $timing_dir/tmp/* $darts_timing_directory/*");


############################################### 1 - FETCH DATA


#&$pipeline_stage_title("Fetching data");
print STDOUT scalar(localtime), " (JST) - Fetching data\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Fetching data\n";

# Index into end times array
$index = 0;

# Whizz through the start times array and fetch status telemetry
foreach $start (@stimes) {
    print STDOUT "$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$sdate stime=$start etime=$etimes[$index]\n" if $state eq "interactive";
    print LOG  "\t$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$sdate stime=$start etime=$etimes[$index]\n";
    system("$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$sdate stime=$start etime=$etimes[$index]");
#    &$pipeline_perform("$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$sdate stime=$start etime=$etimes[$index]");
    ++$index;
}

# Last one uses the edate (next day)
#system("$HOME/bin/sdtp daily $ant band=$band sdate=$sdate edate=$edate stime=2230 etime=0000");
print STDOUT "$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$sdate stime=2230 etime=0000\n" if $state eq "interactive";
print LOG  "\t$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$edate stime=2230 etime=0000\n";
system("$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$edate stime=2230 etime=0000");
#&$pipeline_perform("$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$edate stime=2230 etime=0000");

#system("$HOME/bin/sdtp $mode $ant band=$band sdate=$sdate edate=$edate stime=$stime etime=$etime");

#goto missing_data;


############################################### 2 - DATA AVAILABLE CHECK


#&$pipeline_stage_title("Data available check");
@files = `ls $merge_dir/eis_sts*`;
if(! @files) {
    print STDOUT scalar(localtime), " (JST) - No ccsds status files\n" if $state eq "interactive";
    print LOG "\n", scalar(localtime), " (JST) - No ccsds status files\n";
#    &$pipeline_log("No ccsds status files");

    # Does the current date already exist in the controller file?
    # Not really necessary as the kick program uses a hash...
    $exists = `grep '$sdate $edate' ~/track/status_pending.txt`;
#    $exists = `grep '$sdate $edate' ~/tmp/status_pending.txt` ne "";

    # Write out start/end date to controlling file if not
    `echo -p $sdate $edate >> ~/track/status_pending.txt` if (($exists eq "") and ($CHECK_ONLY == 0));
    system("/bin/cp ~/track/status_pending.txt $HOME/data/staging/logs/");
    print MLOG "\tNo ccsds packets available\n";

    if (($exists eq "") and ($CHECK_ONLY == 0)) {
	`echo -p $sdate $edate >> ~/track/status_pending.txt`;
	system("/bin/cp ~/track/status_pending.txt $HOME/data/staging/logs/");
	print MLOG "\tNo ccsds packets available - saving dates in status pending file\n";
    }
    else {
	print MLOG "\tNo ccsds packets available\n";
    }
    pipeline_exit("status", 1);
}

# Do only if started by cron
system("unset noclobber");
system("echo $sdate > $HOME/track/latest_status_reformat.txt");
system("/bin/cp ~/track/latest_status_reformat.txt $HOME/data/staging/logs/");
system("set noclobber");
#&pipeline_perform(*LOG, $state, "unset noclobber");
#&pipeline_perform(*LOG, $state, "echo $sdate > $HOME/tmp/latest_status_reformat.txt");
#&pipeline_perform(*LOG, $state, "set noclobber");

# Need a more sophisticated ccsds sequence count check
#system("$HOME/bin/pipeline_ccsds_check sts $merge_dir > $merge_dir/ccsds_hdr_check.txt");
#system("/bin/mv $merge_dir/ccsds_hdr_check.txt $HOME/data/status/$year/$month/ccsds_hdrs.$year$month$day.txt");

############################################### 3 - STATUS REFORMAT


print STDOUT scalar(localtime), " (JST) - Starting status reformat\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Starting status reformat\n";
#&$pipeline_stage_title("Starting reformat");

# Does the current date already exist in the controller file?
#$exists = `grep '$sdate $edate' ~/tmp/status_pending.txt`;

# If it does then remove current date from the controller file
#if($exists ne "") {
#    `cat ~/tmp/status_pending.txt | grep -v '$sdate $edate' >> ~/tmp/$$.txt`;
#    `mv ~/tmp/$$.txt ~/tmp/status_pending.txt`;
#}

print LOG "\t(IDL) auto_mk_fits, '$merge_dir', '$temp_fits/'\n";
#&$pipeline_log("(IDL) auto_mk_fits, '$merge_dir', '$temp_fits/'");

# Temporary .comps

#$output = <<EOF;
#.comp $local_idl/eis_ccsds_interface__define
#.comp $local_idl/eis_make_status_fits
#.comp $local_idl/eis_ti2utc
#.comp $local_idl/auto_mk_fits
#auto_mk_fits, '$merge_dir', '$temp_fits/'
#EOF

$output = <<EOF;
!quiet=1
.comp $local_idl/eis_status_parameter__define
.comp $local_idl/eis_make_status_fits
;.comp $local_idl/eis_ti2utc
;.comp $local_idl/auto_mk_fits
auto_mk_fits, '$merge_dir', '$temp_fits/'
EOF

open(STAT_FITS, "> $temp_idl/status_fits.pro");
print STAT_FITS $output;
close STAT_FITS;

log_msg(*LOG, $state, "IDL file:\n$output");
#&$pipeline_log("IDL file:\n$output");

# Start the status reformatter
$log_file = "sts_reformat_$year$month$day.txt";
#system("/san_darts/solar/ssw/gen/setup/ssw_idl < $temp_idl/status_fits.pro > $pipeline_log/$log_file");
system("$HOME/ssw_idl < $temp_idl/status_fits.pro > $pipeline_log/$log_file");
#&$pipeline_perform("$HOME/ssw_idl < $temp_idl/status_fits.pro > $pipeline_log/$log_file");

# Move report to log directory
#print LOG "\t/bin/mv $pipeline_log/$log_file $staging/logs/sts_reformat/\n";
#system("/bin/mv $pipeline_log/$log_file $staging/logs/sts_reformat/");
##&$pipeline_perform("/bin/mv $pipeline_log/$log_file $staging/logs/sts_reformat/");

# mcrw 20190130
# Commented out previous block which moved report to log directory
# Remove log file as not actually useful
print LOG "\t/bin/rm $pipeline_log/$log_file";
system("/bin/rm $pipeline_log/$log_file");
#&$pipeline_perform("/bin/rm $pipeline_log/$log_file");


@files = `ls $temp_fits/*fits`;
if(! @files) {
    print STDOUT scalar(localtime), " (JST) - No status fits files\n" if $state eq "interactive";
    print LOG "\n", scalar(localtime), " (JST) - No status fits files\n";
    print MLOG "\tNo fits files produced\n";
    pipeline_exit("status", 2);
}

############################################### 3A - Missing data

missing_data:

# Need also to split the ccsds sts archives into types (sts1, 2, 3, etc).

#goto skip_missing_data_section;

#&$pipeline_stage_title("Missing data section");
print STDOUT scalar(localtime), " (JST) - Missing data section\n" if $state eq "interactive";
print LOG "\n\n", scalar(localtime), " (JST) - Missing data section\n";

my %intervals = (STS1 => '2',
		 STS2 => '10',
		 STS3 => '10',
		 HK1  => '2',
		 HK2  => '2',
		 HK3  => '1',
		 SOT  => '2',
		 MDP  => '2');

my %fits_files = (STS1 => 'eis_sts1',
		  STS2 => 'eis_sts2',
		  STS3 => 'eis_sts3',
		  HK1  => 'eis_hk1',
		  HK2  => 'eis_hk2',
		  HK3  => 'eis_aocs1',
		  SOT  => 'eis_sot',
		  MDP  => 'eis_eismdp');

my $tmp_dir = $timing_dir . "/tmp";

# Get a list of the files in date order
###my @timing_files = `ls $timing_dir/$ymd*`;
my @timing_files = `ls $timing_dir/*log`;
print LOG "\tls $timing_dir/*log\n";
goto bad_section unless @timing_files;

goto improved_method;

for($i = 0; $i < 16; ++$i) {
    $offset[$i] = sprintf("%uL", 5400 * $i);
}

my @fits_files = `ls $temp_fits`;

my $missing_packets_idl_file = $temp_idl . "/missing_packets.pro";

$foo = open(IDL, "> $missing_packets_idl_file");
goto bad_section unless $foo == 1;
#print IDL ".comp $local_idl/eis_status_sts2_conversions\n";
print IDL ".comp $local_idl/eis_missing_status_packets\n";
foreach $type (qw(STS1 STS2 STS3 HK1 HK2 HK3 SOT MDP)) {
    $count = 0;
    foreach $file (@timing_files) {
	chomp $file;
	my $new_file = $tmp_dir . "/" . substr($file, -17, 14) . "$type";
	my $time = substr($file, -9, 5);
##	print "grep $type $file | awk '{print \$11}' > $new_file\n";
##	system("grep $type $file | awk '{print \$11}' > $new_file");
	`grep $type $file | awk '{print \$11}' > $new_file`;
	
	## Get the number of packets expected in file
	my $packet_count = `wc -l $new_file`;

	# Now get a handle on the fits file to use
	my $yyyymmddhhmm = $ymd . $time;
	$fits_file = `ls $temp_fits/$fits_files{$type}_$yyyymmddhhmm*.fits`;
	chomp $fits_file;
	print IDL "eis_missing_status_packets, '$type', '$new_file', $offset[$count], $intervals{$type}, '$fits_file', '$HOME/tmp/status_reformat/'\n";
#	print IDL "eis_mak_status_fits_$type, 'ccsds_file', '$new_file'(timing_file), $intervals{$type}, num_packets=$packet_count '$temp_fits/pad'\n";
	++$count;
    }
}
close IDL;
#log_msg(*LOG, $state, "IDL file:\n$output");
#&$pipeline_log("IDL file:\n$output");

system("/bin/cp $missing_packets_idl_file $staging");

 improved_method:
    print LOG "\tCopying files from $timing_dir to $darts_timing_directory\n";
foreach $file (@timing_files) {
    chomp $file;
    print LOG "\t/bin/cp $file $darts_timing_directory\n";
    system("/bin/cp $file $darts_timing_directory");
}

goto skip_missing_data_section;

# Will rename each timing file with the date/time of each corresponding ccsds file
# Then start (awk) program to transform entries for each type of status file:
# Format:
# Seconds start offset
# Seconds time stamp for each packet type
#
# Reformatter does each status type in turn or all status types for each orbit period in turn?

###system("$HOME/ssw_idl < $temp_idl/missing_packets.pro");
 bad_section:
    print LOG "\tNo files found in $timing_dir\n";

#    `/bin/rm $tmp_dir/$ymd* $timing_dir/*`;
    ;
 skip_missing_data_section:
    ;
############################################### 4 - PS PLOTS

goto check_only if $CHECK_ONLY;

#&$pipeline_stage_title("Starting daily plots");
print STDOUT scalar(localtime), " (JST) - Starting daily plot\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Starting daily plot\n";

#&$pipeline_log("(IDL) eis_daily_plots, '$tep_fits/','$temp_plots/'");
print LOG "\t(IDL) eis_daily_plots, '$temp_fits/','$temp_plots/'\n";

# Now do the daily plots

# Temporary .comps
###foreach $i (`ls $local_idl/eis_convert*.pro`) {
###    $foo .= ".comp $i\n";
###}
###$output = <<EOF;
###$foo
###.comp $local_idl/eis_get_daily_data
###.comp $local_idl/eis_daily_plots
###.comp $local_idl/eis_status_sts1_conversions
###.comp $local_idl/eis_status_sts2_conversions
###.comp $local_idl/eis_status_sts3_conversions
###.comp $local_idl/eis_hk1_temperatures
###eis_daily_plots,'$temp_fits/','$temp_plots/'
###EOF

# Temporary .comps
$output = <<EOF;
!quiet=1
;.comp $local_idl/eis_status_sts2_conversions
;;.comp $local_idl/eis_convert_mhc_temperature
;.comp $local_idl/eis_status_sts3_conversions
;.comp $local_idl/eis_status_mdp_conversions
;.comp $local_idl/eis_daily_plots
eis_daily_plots,'$temp_fits/','$temp_plots/'
EOF

open(PLOT, "> $temp_idl/daily_plot.pro");
print PLOT $output;
close PLOT;

log_msg(*LOG, $state, "IDL file:\n$output");
#&$pipeline_log("IDL file:\n$output");

# Now start IDL and do the plots
#system("/san_darts/solar/ssw/gen/setup/ssw_idl < $temp_idl/daily_plot.pro");
system("$HOME/ssw_idl < $temp_idl/daily_plot.pro");
#&$pipeline_perform("$HOME/ssw_idl < $temp_idl/daily_plot.pro");


############################################### 5 - PS -> PDF


# Now create pdfs out of the ps files

#&$pipeline_stage_title("Convert ps to pdf and compress");
print STDOUT scalar(localtime), " (JST) - Converting ps to pdf and compressing\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Converting ps to pdf and compressing\n";

print LOG "\t$HOME/bin/daily_pdfs $temp_plots $temp_plots\n";
system("$HOME/bin/daily_pdfs $temp_plots $temp_plots");
#&$pipeline_perform("$HOME/bin/daily_pdfs $temp_plots $temp_plots");

print LOG "\tgzip -f $temp_plots/*pdf\n";
system("gzip -f $temp_plots/*pdf");
#&$pipeline_perform("gzip -f $temp_plots/*pdf");


############################################### 6 - MOVE PDFS TO DARTS


#&$pipeline_stage_title("Moving pdfs to DARTS");
print STDOUT scalar(localtime), " (JST) - Moving pdfs to DARTS\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Moving pdfs to DARTS\n";

goto skip_plot_move if $NO_UPDATE_DARTS;

print LOG "\t/bin/mkdir $darts_status/$year/$month/$day/plots\n";
system("/bin/mkdir $darts_status/$year/$month/$day/plots");
#&$pipeline_perform("/bin/mkdir $darts_status/$year/$month/$day/plots");

print LOG "\t/bin/mv $temp_plots/*gz $darts_status/$year/$month/$day/plots\n";
system("/bin/mv $temp_plots/*gz $darts_status/$year/$month/$day/plots");
#&$pipeline_perform("/bin/mv $temp_plots/*gz $darts_status/$year/$month/$day/plots");

skip_plot_move:

############################################### 7 - COMPRESS TEMP FITS DIRECTORY FILES

# Now gzip everything in the temporary fits directory
# before moving to the DARTS directories
#&$pipeline_stage_title("Gzip temporary fits files");
print STDOUT scalar(localtime), " (JST) - Gzip temporary fits files\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Gzip temporary fits files\n";

print LOG "\tgzip -f $temp_fits/*\n";
system("gzip -f $temp_fits/*");
#&$pipeline_perform("gzip -f $temp_fits/*");


############################################### 8 - MOVE STATUS FITS FILES TO DARTS

# Move the status fits files before mission data reformatting as the
# mission data reformatter looks for housekeeping date in the DARTS
# system. It also needs the status fits files to be un-gzipped. They
# will be gzipped up later in DARTS.

goto skip_data_move if $NO_UPDATE_DARTS;
#&$pipeline_stage_title("Moving status fits files to DARTS");
print STDOUT scalar(localtime), " (JST) - Moving status fits files to DARTS\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Moving status fits files to DARTS\n";

#
# Filenames:
#	eis_aocs1_yyyymmdd_hhmmss.fits
#	eis_hk1_yyyymmdd_hhmmss.fits
#	eis_hk2_yyyymmdd_hhmmss.fits
#	eis_sot_yyyymmdd_hhmmss.fits
#	eis_sts1_yyyymmdd_hhmmss.fits
#	eis_sts2_yyyymmdd_hhmmss.fits
#	eis_sts3_yyyymmdd_hhmmss.fits
#

#@files = `ls $temp_fits/*fits`;
@files = `ls $temp_fits`;
foreach $file (@files) {
    next if $file =~ /^eis_l0/;
    next unless $file =~ /^eis_/;
    chomp $file;
    # Match destination directory with filename
    $index  = 11 if $file =~ /eismdp/;
    $index  = 10 if $file =~ /aocs/;
    $index  =  9 if $file =~ /sts/;
    $index  =  8 if $file =~ /hk/;
    $index  =  8 if $file =~ /sot/;
    $darts_year  = substr $file, $index, 4;
    $darts_month = substr $file, $index + 4, 2;
    $darts_day   = substr $file, $index + 6, 2;

#    print LOG "\t/bin/mv $temp_fits/$file $darts_status/$darts_year/$darts_month/$darts_day/ccsds\n";
#    system("/bin/mv $temp_fits/$file $darts_status/$darts_year/$darts_month/$darts_day/ccsds");
    print LOG "\t/bin/cp $temp_fits/$file $darts_status/$darts_year/$darts_month/$darts_day\n";
    system("/bin/cp $temp_fits/$file $darts_status/$darts_year/$darts_month/$darts_day/");
#    &$pipeline_perform("/bin/cp $temp_fits/$file $darts_status/$darts_year/$darts_month/$darts_day/");
}

skip_data_move:

############################################### 9 - GRID FTP

#print STDOUT scalar(localtime), " (JST) - Starting gridFTP session\n" if $state eq "interactive";
#print LOG "\n", scalar(localtime), " (JST) - Starting gridFTP session\n";

###@files = `ls $temp_fits`;
#@files = `ls $darts_status/$darts_year/$darts_month/$darts_day/`;
#foreach $file (@files) {
#    chomp $file;
#    print LOG "\tgridFTP $file\n";
#}


############################################### 10 - COMPRESS STATUS FITS FILES IN DARTS

#print STDOUT scalar(localtime), " (JST) - Compress status fits files in DARTS\n" if $state eq "interactive";
#print LOG "\n", scalar(localtime), " (JST) - Compress status fits files in DARTS\n";
#
## The fits files in the temp_fits directory have been compressed but not the ones in DARTS.
## Get a listing of the temp_fits directory and substitute the .gz with nothing
#
#@files = `ls $temp_fits`;
#foreach $file (@files) {
#    next if $file =~ /eis_l0/;
#    chomp $file;
#    # Match destination directory with filename
#    $index  = 10 if $file =~ /aocs/;
#    $index  =  9 if $file =~ /sts/;
#    $index  =  8 if $file =~ /hk/;
#    $darts_year  = substr $file, $index, 4;
#    $darts_month = substr $file, $index + 4, 2;
#    $darts_day   = substr $file, $index + 6, 2;
#
#    $file =~ s/\.gz//;
#
#    print LOG "\tgzip -f $darts_status/$darts_year/$darts_month/$darts_day/$file\n";
#    system("gzip -f $darts_status/$darts_year/$darts_month/$darts_day/$file");
#}


############################################### 11 - DO DAILY CHECK

  check_only:

#&$pipeline_stage_title("Starting daily check");
print STDOUT scalar(localtime), " (JST) - Starting daily check\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Starting daily check\n";

$report_name = ($CHECK_ONLY) ? "$year" . "$month" . "$day" . "_check.txt" : "$darts_year" . "$darts_month" . "$darts_day" . "_check.txt";
print LOG "\t(IDL) eis_daily_check, '$temp_fits/','$temp_plots/$report_name'\n";
#&$pipeline_log("(IDL) eis_daily_check, '$temp_fits/','$temp_plots/$report_name'");

#.comp $local_idl/eis_get_daily_data
###.comp $local_idl/eis_hk1_temperatures

my $fits_dir = ($CHECK_ONLY) ? "$darts_status/$year/$month/$day/" : "$temp_fits/";

# Temporary .comps
$output = <<EOF;
!quiet=1
;.comp $local_idl/eis_count_check	; for eis_status_sts3_check.pro
;.comp $local_idl/eis_num2str
;.comp $local_idl/eis_min_max_str
;.comp $local_idl/eis_min_max_str1
;.comp $local_idl/eis_param_string
; Temporary!
;;;.comp $local_idl/eis_convert_mhc_temperature
;.comp $local_idl/eis_status_sts1_check
;.comp $local_idl/eis_status_sts2_check
;.comp $local_idl/eis_status_sts3_check
;.comp $local_idl/eis_status_hk1_check
;.comp $local_idl/eis_daily_check
;eis_daily_check,'$temp_fits/','$temp_plots/$report_name'
eis_daily_check,'$fits_dir','$temp_plots/$report_name'
EOF

#$output = <<EOF;
#eis_daily_check,'$temp_fits/','$temp_plots/$report_name'
#EOF

open(PLOT, "> $temp_idl/daily_check.pro");
print PLOT $output;
close PLOT;

log_msg(*LOG, $state, "IDL file:\n$output");
#&$pipeline_log("IDL file:\n$output");

# Now start IDL and do the check
$sts_check_log = "sts_check_$year$month$day.txt";
#system("/san_darts/solar/ssw/gen/setup/ssw_idl < $temp_idl/daily_check.pro > $pipeline_log/$sts_check_log");
system("$HOME/ssw_idl < $temp_idl/daily_check.pro > $pipeline_log/$sts_check_log");
#&$pipeline_perform("$HOME/ssw_idl < $temp_idl/daily_check.pro > $pipeline_log/$sts_check_log");

# Move report to log directory
#print LOG "\t/bin/mv $pipeline_log/$sts_check_log $staging/logs/sts_check/\n";
#system("/bin/mv $pipeline_log/$sts_check_log $staging/logs/sts_check/");
##&$pipeline_perform("/bin/mv $pipeline_log/$sts_check_log $staging/logs/sts_check/");

# mcrw 20190130
# Remove log - not actually useful
# Commented out previous block to move log to directory
print LOG "\t/bin/rm $pipeline_log/$sts_check_log\n";
system("/bin/rm $pipeline_log/$sts_check_log");
#&$pipeline_perform("/bin/rm $pipeline_log/$sts_check_log");


############################################### 12 - MOVE REPORT TO DARTS

goto skip_move_reports if $NO_UPDATE_DARTS;

#&$pipeline_stage_title("Moving daily check report to DARTS");
print STDOUT scalar(localtime), " (JST) - Moving daily check report to DARTS\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Moving daily check report to DARTS\n";

#print LOG "\t/bin/mv $temp_plots/*txt $darts_status/$year/$month/$day/plots\n";
#system("/bin/mv $temp_plots/*txt $darts_status/$year/$month/$day/plots");
#&$pipeline_perform("/bin/mv $temp_plots/*txt $darts_status/$year/$month/$day/plots");

print STDOUT "cat $temp_plots/$report_name | awk -f '$HOME/src/add_range.awk' > $darts_status/$year/$month/$day/plots/$report_name\n";
print LOG "cat $temp_plots/$report_name | awk -f '$HOME/src/add_range.awk' > $darts_status/$year/$month/$day/plots/$report_name\n";
system("cat $temp_plots/$report_name | awk -f '$HOME/src/add_range.awk' > $darts_status/$year/$month/$day/plots/$report_name");

print STDOUT "\tRemoving check log $temp_plots/$report_name";
print LOG "\tRemoving check log $temp_plots/$report_name\n";
system("/bin/rm $temp_plots/$report_name");

pipeline_exit("status", 999) if $CHECK_ONLY;

skip_move_reports:

############################################### 13 - UPDATE TRENDS AND SS MOVE LOG

#&$pipeline_stage_title("Starting monthly trends");
print STDOUT scalar(localtime), " (JST) - Starting monthly trends\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Starting monthly trends\n";
print LOG "\t(IDL) eis_monthly_trends, '$darts_status/$darts_year/$darts_month', '$darts_year$darts_month'\n";
#&$pipeline_log("(IDL) eis_monthly_trends, '$darts_status/$darts_year/$darts_month', '$darts_year$darts_month'");

$ccd_temp_trend_file     = "eis_ccd_temp_trend_"     . "$darts_year$darts_month" . ".txt";
$str_temp_trend_file     = "eis_str_temp_trend_"     . "$darts_year$darts_month" . ".txt";
$mbus_pwr_trend_file     = "eis_mbus_pwr_trend_"     . "$darts_year$darts_month" . ".txt";
$icu_psu_pwr_trend_file  = "eis_icu_psu_pwr_trend_"  . "$darts_year$darts_month" . ".txt";
$mhc_gp0_temp_trend_file = "eis_mhc_gp0_temp_trend_" . "$darts_year$darts_month" . ".txt";
$mhc_gp2_temp_trend_file = "eis_mhc_gp2_temp_trend_" . "$darts_year$darts_month" . ".txt";
$cam_pwr_trend_file      = "eis_cam_pwr_trend_"      . "$darts_year$darts_month" . ".txt";
$mhc_psu_pwr_trend_file  = "eis_mhc_psu_pwr_trend_"  . "$darts_year$darts_month" . ".txt";
$cam_temp_trend_file     = "eis_cam_temp_trend_"     . "$darts_year$darts_month" . ".txt";

$plot_title = date_to_natural_lang($darts_year, $darts_month);

$ss_move_log_file = "ss_moves_" . "$darts_year$darts_month" . ".txt";

#$trends = "trends";
$trends = "trends1";

$output = <<EOF;
!quiet=1
;.comp $local_idl/$trends/eis_no_data_string
;.comp $local_idl/$trends/eis_ccd_temp_trend
;.comp $local_idl/$trends/eis_mbus_pwr_trend
;.comp $local_idl/$trends/eis_mhc_gp0_temp_trend
;.comp $local_idl/$trends/eis_mhc_gp2_temp_trend
;.comp $local_idl/$trends/eis_str_temp_trend
;.comp $local_idl/$trends/eis_icu_psu_pwr_trend
;.comp $local_idl/$trends/eis_cam_pwr_trend
;.comp $local_idl/$trends/eis_mhc_psu_pwr_trend
;.comp $local_idl/$trends/eis_cam_temp_trend
;.comp $local_idl/$trends/eis_monthly_trends
;.comp $local_idl/$trends/eis_plot_trend_data
;.comp $local_idl/$trends/eis_plot_ccd_temp_trend
;.comp $local_idl/$trends/eis_plot_mbus_pwr_trend
; Temporary!
;;;.comp $local_idl/eis_convert_mhc_temperature
;.comp $local_idl/$trends/eis_plot_mhc_gp0_temp_trend
;.comp $local_idl/$trends/eis_plot_mhc_gp2_temp_trend
;.comp $local_idl/$trends/eis_plot_str_temp_trend
;.comp $local_idl/$trends/eis_plot_cam_pwr_trend
;.comp $local_idl/$trends/eis_plot_icu_psu_pwr_trend
;.comp $local_idl/$trends/eis_plot_mhc_psu_pwr_trend
;.comp $local_idl/$trends/eis_plot_cam_temp_trend
;.comp $local_idl/$trends/eis_plot_trends
;.comp $local_idl/eis_count_check
;.comp $local_idl/eis_mhc_mech_moves
eis_monthly_trends, '$darts_status/$darts_year/$darts_month', '$darts_year$darts_month', '$sdate', '$edate'
eis_plot_trends, '$darts_year$darts_month', '$plot_title', '$HOME/work/localdata/log/status/trends/'
eis_mhc_mech_moves, '$darts_status/$darts_year/$darts_month', log='$HOME/work/localdata/log/status/ss/$ss_move_log_file'
EOF

open(MONTH_TREND, "> $temp_idl/monthly_trends.pro");
print MONTH_TREND $output;
close MONTH_TREND;

log_msg(*LOG, $state, "IDL file:\n$output");
#&$pipeline_log("IDL file:\n$output");

#system("/san_darts/solar/ssw/gen/setup/ssw_idl < $temp_idl/monthly_trends.pro");
system("$HOME/ssw_idl < $temp_idl/monthly_trends.pro");
#&$pipeline_perform("$HOME/ssw_idl < $temp_idl/monthly_trends.pro");

print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$str_temp_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$mbus_pwr_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$icu_psu_pwr_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp0_temp_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp2_temp_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$cam_pwr_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$mhc_psu_pwr_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/$cam_temp_trend_file $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/trends/*.ps $darts_status/$year/$month/trends/\n";
print LOG "\t/bin/mv $HOME/work/localdata/log/status/ss/$ss_move_log_file $darts_status/$year/$month/trends/\n";
###print LOG "\t/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq >  $darts_status/qcm1_log.txt\n";
###print LOG "\t/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq >  $darts_status/qcm2_log.txt\n";

goto skip_move_trends if $NO_UPDATE_DARTS;

#print STDOUT "/bin/cat $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file | awk -f '$HOME/src/add_ccd_range.awk' > $darts_status/$year/$month/trends/$ccd_temp_trend_file\n";
#print LOG "/bin/cat $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file | awk -f '$HOME/src/add_ccd_range.awk' > $darts_status/$year/$month/trends/$ccd_temp_trend_file\n";
#system("/bin/cat $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file | awk -f '$HOME/src/add_ccd_range.awk' > $darts_status/$year/$month/trends/$ccd_temp_trend_file");

system("/bin/mv $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$str_temp_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$mbus_pwr_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$icu_psu_pwr_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp0_temp_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp2_temp_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$cam_pwr_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_psu_pwr_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/$cam_temp_trend_file $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/trends/*.ps $darts_status/$year/$month/trends/");
system("/bin/mv $HOME/work/localdata/log/status/ss/$ss_move_log_file $darts_status/$year/$month/trends/");

#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$str_temp_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$mbus_pwr_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$icu_psu_pwr_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp0_temp_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_gp2_temp_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$cam_pwr_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$mhc_psu_pwr_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/$cam_temp_trend_file $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/trends/*.ps $darts_status/$year/$month/trends/");
#&$pipeline_perform("/bin/mv $HOME/work/localdata/log/status/ss/$ss_move_log_file $darts_status/$year/$month/trends/");

#system("/bin/cp $HOME/work/localdata/log/status/trends/$ccd_temp_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$str_temp_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$mbus_pwr_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$icu_psu_pwr_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$mhc_gp0_temp_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$mhc_gp2_temp_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$cam_pwr_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$mhc_psu_pwr_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/$cam_temp_trend_file $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/trends/*.ps $darts_status/$year/$month/trends/");
#system("/bin/cp $HOME/work/localdata/log/status/ss/$ss_move_log_file $darts_status/$year/$month/trends/");


# Copy qcm logs to DARTS
system("/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq > $darts_status/qcm1_log.txt");
system("/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq > $darts_status/qcm2_log.txt");
#&$pipeline_perform("/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq > $darts_status/qcm1_log.txt");
#&$pipeline_perform("/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq > $darts_status/qcm2_log.txt");

skip_move_trends:

############################################### 14 - CCSDS MOVE TO DARTS

# TEMPORARY!

# Move the ccsds status and memory dump archives to the DARTS directory
#&$pipeline_stage_title("Copying status ccsds files to DARTS (TEMPORARY)");
print STDOUT scalar(localtime), " (JST) - Copying status ccsds files to DARTS (TEMPORARY)\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Copying status ccsds files to DARTS (TEMPORARY)\n";

print LOG "\t/bin/mkdir $darts_status/$year/$month/$day/ccsds\n";
system("/bin/mkdir $darts_status/$year/$month/$day/ccsds");
#&$pipeline_perform("/bin/mkdir $darts_status/$year/$month/$day/ccsds");

print LOG "\t/bin/mv $merge_dir/eis_sts* $darts_status/$year/$month/$day/ccsds\n";
system("/bin/mv $merge_dir/eis_sts* $darts_status/$year/$month/$day/ccsds");
#&$pipeline_perform("/bin/mv $merge_dir/eis_sts* $darts_status/$year/$month/$day/ccsds");

print LOG "\t/bin/mv $merge_dir/eis_dmp* $darts_status/$year/$month/$day/ccsds\n";
system("/bin/mv $merge_dir/eis_dmp* $darts_status/$year/$month/$day/ccsds");
#&$pipeline_perform("/bin/mv $merge_dir/eis_dmp* $darts_status/$year/$month/$day/ccsds");

$seq_counts_outfile = "$year$month$day" . "_sequence_counts.txt";
print LOG "\t(cd $darts_status/$year/$month/$day/ccsds/ ; status_sequence_counts.pl eis_sts* > $seq_counts_outfile)";
system("(cd $darts_status/$year/$month/$day/ccsds/ ; status_sequence_counts.pl eis_sts* > $seq_counts_outfile)");

print LOG "\tgzip -f $darts_status/$year/$month/$day/ccsds/eis_sts*\n";
system("gzip -f $darts_status/$year/$month/$day/ccsds/eis_sts*");
#print LOG "\tgzip -f $merge_dir/eis_sts*\n";
#system("gzip -f $merge_dir/eis_sts*");
#&$pipeline_perform("gzip -f $merge_dir/eis_sts*");

print LOG "\tgzip -f $darts_status/$year/$month/$day/ccsds/eis_dmp*\n";
system("gzip -f $darts_status/$year/$month/$day/ccsds/eis_dmp*");
#&$pipeline_perform("gzip -f $merge_dir/eis_dmp*");


############################################### 15 - CLEAN UP

#&$pipeline_stage_title("Cleaning up");
print STDOUT scalar(localtime), " (JST) - Cleaning up\n" if $state eq "interactive";
print LOG "\n", scalar(localtime), " (JST) - Cleaning up\n";

###print LOG "\t/bin/rm $temp_idl/*pro\n";
###system("/bin/rm $temp_idl/*pro");
#&$pipeline_perform("/bin/rm $temp_idl/*pro");

print LOG "\t/bin/rm $temp_fits/*\n";
system("/bin/rm $temp_fits/*");

system("cd $temp_fits && /bin/rm *fits*");

#&$pipeline_perform("cd $temp_fits && /bin/rm *");

print LOG "\t/bin/rm $temp_plots/*\n";
system("/bin/rm $temp_plots/*");
#&$pipeline_perform("/bin/rm $temp_plots/*");

print LOG "\t/bin/rm $merge_dir/eis_md*\n";
system("/bin/rm $merge_dir/eis_md*");
#&$pipeline_perform("/bin/rm $merge_dir/eis_md*");

#print LOG "\t/bin/cp $pipeline_log/pipeline_status_log_$sdate $darts_staging/logs/\n";
#system("/bin/cp $pipeline_log/pipeline_status_log_$sdate $darts_staging/logs/");

# mcrw 20200727
print LOG "\t/bin/rm $HOME/work/localdata/log/dr/*";
system("/bin/rm $HOME/work/localdata/log/dr/*");

goto dont_do;
$merge_log = "$HOME/tmp/status_merge_record.txt";
$dated_merge_log = "$darts_staging/status/status_merge_record_$year$month$day.txt";

print LOG "\t/bin/mv $merge_log $dated_merge_log";
system("/bin/mv $merge_log $dated_merge_log");

print LOG "\tgzip -f $dated_merge_log";
system("\tgzip -f $dated_merge_log");
dont_do:

############################################### 16 - REMOVE QUICKLOOK STATUS DATA

#&$pipeline_stage_title("Removing quicklook status data for $year/$month/$day");
$current_time = scalar(localtime);
print STDOUT $current_time, " (JST) - Removing quicklook status data for $year/$month/$day\n" if $state eq "interactive";
print LOG "\n\n", $current_time, " (JST) - Removing quicklook status data for $year/$month/$day\n";

# Remove directory in DARTS if it exists
if(-e "$darts_status_ql/$year/$month/$day") {
    print LOG "\t/bin/rm -rf $darts_status_ql/$year/$month/$day\n";
    system("/bin/rm -rf $darts_status_ql/$year/$month/$day");
#    &$pipeline_perform("/bin/rm -rf $darts_status_ql/$year/$month/$day");
}
else {
    print LOG "\tDirectory $darts_mission_ql/$year/$month/$day does not exist\n";
#    &$pipeline_log("Directory $darts_mission_ql/$year/$month/$day does not exist");
}

# Remove status directory in sdtp if it exists
if(-e "$ql_status/$year$month$day") {
    print LOG "\t/bin/rm -rf $ql_status/$year$month$day\n";
    system("/bin/rm -rf $ql_status/$year$month$day");
#    &$pipeline_perform("/bin/rm -rf $ql_status/$year$month$day");
}
else {
    print LOG "\tDirectory $ql_status/$year$month$day does not exist\n";
#    &$pipeline_log("Directory $ql_status/$year$month$day does not exist");
}

# Remove status fits directory in sdtp if it exists
if(-e "$ql_status_fits/$year$month$day") {
    print LOG "\t/bin/rm -rf $ql_status_fits/$year$month$day\n";
    system("/bin/rm -rf $ql_status_fits/$year$month$day");
#    &$pipeline_perform("/bin/rm -rf $ql_status_fits/$year$month$day");
}
else {
    print LOG "\tDirectory $ql_status_fits/$year$month$day does not exist\n";
#    &$pipeline_log("Directory $ql_status_fits/$year$month$day does not exist");
}

# Remove status fits directory in sdtp if it exists
if(-e "$ql_status_plots/$year$month$day") {
    print LOG "\t/bin/rm -rf $ql_status_plots/$year$month$day\n";
    system("/bin/rm -rf $ql_status_plots/$year$month$day");
#    &$pipeline_perform("/bin/rm -rf $ql_status_plots/$year$month$day");
}
else {
    print LOG "\tDirectory $ql_status_plots/$year$month$day does not exist\n";
#    &$pipeline_log("Directory $ql_status_plots/$year$month$day does not exist");
}


############################################### 17 - UPDATE QCM LOGS

#&$pipeline_stage_title("Updating QCM logs");
$current_time = scalar(localtime);
print STDOUT $current_time, " (JST) - Updating QCM logs\n" if $state eq "interactive";
print LOG "\n", $current_time, " (JST) - Updating QCM logs\n";

print LOG "\t/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq >  $HOME/data/status/qcm1_log.txt\n";
print LOG "\t/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq >  $HOME/data/status/qcm2_log.txt\n";
system("/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq >  $HOME/data/status/qcm1_log.txt");
system("/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq >  $HOME/data/status/qcm2_log.txt");
#&$pipeline_perform("/bin/sort $HOME/work/localdata/log/status/qcm/qcm1_log.txt | uniq >  $HOME/data/status/qcm1_log.txt");
#&$pipeline_perform("/bin/sort $HOME/work/localdata/log/status/qcm/qcm2_log.txt | uniq >  $HOME/data/status/qcm2_log.txt");

system("/bin/cp ~/track/status_pending.txt $HOME/data/staging/logs/");

############################################### 18 - EXIT

#&$pipeline_stage_title("Exiting");
print MLOG "\tCompleted\n";
pipeline_exit("status", 0);