package Finalize;
use strict;
use English;
use Carp;
use vars
  qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
use BlackBoard;

# Declare identity, version, author, date, etc.
$name    = __PACKAGE__;
$VERSION = '2.19';
$version = $VERSION;
$author  = 'Richard West,Dean Hinshaw,Duncan John Fyfe,Ian Stewart,Duncan Law-Green,Ed Chapin';
$date    = '2013-06-11';

#
# ChangeLog
# =========
#
# Version 2.19 - 2013-06-11 (EC)
# ------------
#
# + Merge changes from SOC version.
# + This version includes for the first time the changes made to this module 
#   in Slew pipeline (2.06->2.18).
#
# version 2.18 - 2010-08-17 DLG
# ------------
#
# + SLEW ONLY: Try forcing DPSS skip via blackboard flags
#
# version 2.17 - 2010-08-10 DLG
# ------------
# 
# + SLEW ONLY: Check for ignored('SlewProcess')
# 
# version 2.16 - 2010-06-17 DLG
# ------------
#
# + SLEW ONLY: Additional addattibute call to add standard headers to PINDEX file
#
# version 2.15 - 2010-06-15 DLG
# ------------
# 
# + SLEW ONLY: Added block to attempt to hack OBSERVER keyword in product output
#
# version 2.14 - 2010-06-02 DLG
# ------------
# 
# + SLEW ONLY: Now uses 'ppsslewsumm' instead of 'ppssumm'
#
# version 2.13 - 2010-05-25 DLG
# ------------
#
# + SLEW ONLY: Modified OBJECT keyword to 'Slew' in StampKeywords section
#
# version 2.12 - 2010-05-17 DLG
# ------------
# 
# + SLEW ONLY: Absorb StampKeywords into slew Finalize pro tem
#
# version 2.11 - 2010-05-13 DLG
# ------------
# 
# + SLEW ONLY: Modify flow control to follow StampKeywords module call
#
# version 2.10 - 2010-02-10 DLG
# ------------
#
# + Use latest version of eslewchain (1.4) which generates agreed PCMS-compliant filenames
#
# version 2.09 - 2010-01-25 DLG
# ------------
#
# + Convert slew band numbers to PCMS standard
#
# version 2.08 - 2010-01-13 DLG
# ------------
#
# + Iterate over slew bands, rename step images
#
# version 2.07 - 2009-12-16 DLG
# ------------
#
# + Started slew products cleanup and generation of PCMS-compliant names
#
# version 2.06 - 2009-06-23 DLG
# ------------
#
# + Added start dependence on SlewProcess
#
# version 2.05 - 2006-12-05 DJF
# ------------
#
# + Need to call newFile to create the XCORRE product file name and db entry if it has not been 
#   returned by ACDS.  If we don't use new file there will be no DB entry and it won't be packed 
#   into the .XMM file.
#
# version 2.04 - 2006-03-23 DJF
# ------------
#
# + Replace dependence on StampEndKeywords.  That module has been removed
#
# version 2.03 - 2006-03-23 DJF
# ------------
#
# + Compression now tests for previously compressed files remivng the need for a specialised prodPack.
#   
# version 2.02 - 2005-12-15 DJF
# ------------
#
# Separated product compression and product packing in module resources.
# Finalize amended to take adavantage of ths wrt ppssumm HTM pages 
#
# version 2.01 - 2005-12-02 DJF
# ------------
#
# + Changed dependence from StampKeywords to StampEndKeywords
#
# version 2.00 - 2005-11-21 IMS
# ------------
#
# + Add explicit perl module headers.  Previously these were supplied
#   at compile time.  This will make debugging and extending the modules
#   through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic.  Now $Version = version = ''
#
# Version 1.22 - 2004-02-21 (DJF)
# ------------
#
# + SAS 6.0 ppssumm needs a filename of the catpage 
#   parameter even if the file does not exist. 
#  
#
# Version 1.21 - 2003-07-17 (DJF)
# ------------
#
# + Adapted ppssumm command line to deal with absent
#   catalogue index file.
#
# Version 1.20 - 2003-06-20 (DJF)
# ------------
#
# + Changed to take account of ppssumm 3.0+ which now 
#   generates all summary pages in one go.
#
# Version 1.19 - 2002-02-26 (DH)
# ------------
#
# + added additional output to logs to indicate progress
#
# Version 1.18 - 2002-02-26 (DH)
# ------------
#
# + Bug fixes for 1.17.
#
# Version 1.17 - 2002-02-20 (DJF)
# ------------
#
# + Change newFile to findFile for index files to be created.
#   These files are now created for obssumm and populated here.
#
# Version 1.16 - 2001-11-04 (DH)
# ------------
#
# + Remove dependence on SendDPSS in evaluateRules.
#
# Version 1.15 - 2001-11-03 (DH)
# ------------
#
# + Remove sequenceComplete call from the end of the module.
#
# Version 1.14 - 2001-03-21 (DH)
# ------------
#
# + Replace getFile call for SSC logo to findFile call.
#
# Version 1.13 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
#   tracking purposes.
#
# Version 1.12 - 2001-03-13 (DH)
# ------------
#
# + Add prodPack(), which creates the XFTS file for sending
#   to the SOC, to the end of the processing.
#
# Version 1.11 - 2001-03-08 (DH)
# ------------
#
# + Move obs summary file production to VerifyODF module.
#
# Version 1.10 - 2001-02-19 (DH)
# ------------
#
# + Put in the new obssumm parameter useinfile.
#
# Version 1.09 - 2001-01-17 (DH)
# ------------
#
# + Change name of used logo from 'SSC LOGO 1'
#   to 'SSC LOGO 2' to avoid conflicts with the ACDS.
#
# Version 1.08 - 2000-12-19 (DH)
# ------------
#
# + Fix bug with path to logo files.
#
#
# Version 1.07 - 2000-12-14 (DH)
# ------------
#
# + First production version.
#


# Declare list of instruments this module is interested in
@instList=qw(all);

# Number of streams
sub numberOfStreams {
    return 1;
}

# Rules method
sub evaluateRules {
    
    if ( $ENV{PCMS_ISSLEW} )
    {
        start()
        if allComplete( module  => 'SlewProcess' )
	}
    else 
    {
        start()
        if allComplete( module  => 'ReceiveACDS' )
                    and allComplete( module => 'RGSProducts' )
                    and allComplete( module => 'RGSMakeFluxed' )
                    and allComplete( module => 'OMSourceCombine' )
                    and allComplete( module => 'OMMosaic' )
                    and allComplete( module => 'OMFastAnalyse' )
                    and allComplete( module => 'EPICSourceProducts' )
    }
}

# Action method
sub performAction {
	info("Module version number: $version");

my ( @name, @type, @value, @comment, @withcomment );

if ( $ENV{PCMS_ISSLEW} )
{
    # Set up parameters for slew product tidy ----------------
    # Modified for eslewchain-1.4, PCMS-"compliant" filenames

	my $rootdir = $ModuleResources::Seqdb->{'directories'}{'root'};
	info("Sequence root directory: $rootdir");

    # move files to intermediate subdir
    # Does the new eslewchain still create these?

    #	moveFile(source => 'histo.fits',
    #		 destination => 'intermediate/histo.fits');

    #	moveFile(source => 'image.fits',
    #		 destination => 'intermediate/image.fits');

    #	moveFile(source => 'spectrum.fits',
    #		 destination => 'intermediate/spectrum.fits');

    #	moveFile(source => "temp_sp_*",
    #		 destination => 'intermediate/');

	moveFile(source => "*Badpixels.ds",
		 destination => 'intermediate/') unless ignored( module => 'SlewProcess', stream => 1 );

    # Identify output products and assign PCMS-compatible filenames

    # Attitude file:P*OBX000ATTTSR0000.FITS

	my $obsid = $ENV{'SAS_RAND_SEED'};

	my ($attfile) = glob("$rootdir/P*OBX000ATTTSR0000.ds");
	my $attdest = newFile(class => 'product'
			      ,instrument => 'ob'
			      ,content => 'SLEW ATTITUDE FILE'
			      ,format => 'FITS'
			      );

	moveFile(source => "$attfile"
		 ,destination => "$attdest") unless ignored( module => 'SlewProcess', stream => 1 );
	
	my $sourcename = sprintf("P%010s",$obsid);
	
	doCommand(
		  'addattribute', set => $attdest
		  , attributename => 'OBJECT'
		  , attributetype => 'string'
		  , stringvalue => "$sourcename"
#		  , attributecomment => '"Object name"'
		  ) unless ignored(module => 'SlewProcess', stream => 1 );

    # Single raw events file: P*PNS003SLEVLI0000.FITS

	my ($rawfile) = glob("$rootdir/*ImagingEvts.ds");
#	my ($rawfile) = glob("$rootdir/P*PNS003PIEVLI0SLW.ds");
	my $rawdest = newFile(class=>'product'
			      ,instrument => 'epn'
			      ,exp_id => 'S003'
			      ,content => 'SLEW SINGLE RAW EVENT LIST'
			      ,format => 'FITS'
			      );

	moveFile(source => "$rawfile"
		 ,destination => "$rawdest") unless ignored( module => 'SlewProcess', stream => 1 );
	
	my $sourcename = sprintf("P%010s",$obsid);
	
	doCommand(
		  'addattribute', set => $rawdest
		  , attributename => 'OBJECT'
		  , attributetype => 'string'
		  , stringvalue => "$sourcename"
#		  , attributecomment => '"Object name"'
		  ) unless ignored( module => 'SlewProcess', stream => 1 );

    # filtered event files per step: P*PNS003PIEVLI0*.FITS

	my $filtfile;
	my $filtdest;
	my $filtstep = 0;
	my @filtlist = glob("$rootdir/P*PNS003PIEVLI0*.ds");
	foreach $filtfile (@filtlist) {
	    $filtdest = newFile(class => 'product'
				,instrument => 'epn'
				,exp_id => 'S003'
				,content => 'SLEW STEP FILTERED EVENT LIST'
				,format => 'FITS'
				,src_num => "$filtstep" # this needs to be in decimal, not hex
				);
	    
	    moveFile(source => "$filtfile"
		     ,destination => "$filtdest");

	    my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
	    
	    doCommand(
		      'addattribute', set => $filtdest
		      , attributename => 'OBJECT'
		      , attributetype => 'string'
		      , stringvalue => "$sourcename"
#		      , attributecomment => '"Object name"'
		      );

	    $filtstep++;
	}

    # Slew step band exposure map: P*PNS003EXPMAPbiii.FITS
    # Loop over slew bands

	my $slewband;
	my $stepfile;
	my $stepdest;

	foreach $slewband ( '6', '7', '8' ) {

	    $filtstep = 0;
	    my @steplist = glob("$rootdir/P*PNS003EXPMAP${slewband}*.ds");
	    foreach $stepfile (@steplist) {

		my $stepdest = newFile(class => 'product'
				       , instrument => 'epn'
				       , exp_id => 'S003'
				       , content => 'SLEW STEP EXPOSURE MAP'
				       , format => 'FITS'
				       , src_num => "$filtstep"
				       , band => "$slewband"
				       );

		moveFile(source => "$stepfile"
			 ,destination=> "$stepdest");

		my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);

		doCommand(
			  'addattribute', set => $stepdest
			  , attributename => 'OBJECT'
			  , attributetype => 'string'
			  , stringvalue => "$sourcename"
#			  , attributecomment => '"Object name"'
			  );

		$filtstep++;

	    }

	}

    # Slew step band images: P*PNS003IMAGE_biii.FITS
    # Loop over slew bands

	foreach $slewband ( "6", "7", "8" ) {

	    $filtstep = 0;
	    my @steplist = glob("$rootdir/P*PNS003IMAGE_${slewband}*.ds");
	    foreach $stepfile (@steplist) {

		my $stepdest = newFile(class => 'product'
				       , instrument => 'epn'
				       , exp_id => 'S003'
				       , content => 'SLEW STEP IMAGE'
				       , format => 'FITS'
				       , src_num => "$filtstep"
				       , band => "$slewband"
				       );

		moveFile(source => "$stepfile"
			 ,destination=> "$stepdest");

		# Set OBJECT header to source step name (Nora's request)

		my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);

		doCommand(
			  'addattribute', set => $stepdest
			  , attributename => 'OBJECT'
			  , attributetype => 'string'
			  , stringvalue => "$sourcename"
#			  , attributecomment => '"Object name"'
			  );

		$filtstep++;

	    }

	}

    # Unfiltered step images: P*PNS003UNFDAT8*FITS

	$filtstep = 0;
	my $unffile;
	my $unfdest;
	my @unflist = glob("$rootdir/P*PNS003UNFDAT8*.ds");
	foreach $unffile (@unflist) {
	    $unfdest = newFile(class => 'product'
			       ,instrument => 'epn'
			       ,exp_id => 'S003'
			       ,content => 'UNFILTERED SLEW STEP IMAGE'
			       ,format => 'FITS'
			       ,src_num => "$filtstep" # this needs to be in decimal, not hex
			       ,band => '8'
			       );

	    moveFile(source => "$unffile"
		     ,destination => "$unfdest");
	    
	    # Set OBJECT header to source step name (Nora's request)
	    
	    my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
	    
	    doCommand(
		      'addattribute', set => $unfdest
		      , attributename => 'OBJECT'
		      , attributetype => 'string'
		      , stringvalue => "$sourcename"
#		      , attributecomment => '"Object name"'
		      );
	    
	    
	    $filtstep++;
	}

    # Tidy: Drop all ds files from root directory

	my @unlinklist = glob("$rootdir/*.ds");
	unlink @unlinklist unless ignored( module => 'SlewProcess', stream => 1 );

    # --------------------------------------------------------
    # +++ StampKeywords +++
    # Import portion of StampKeywords module to run here for slew

    # Add basic headers to all FITS product files

	info("Starting StampKeywords subroutine...");

	# Fetch list of product files
	my @list = findFile(
			    class => 'product'
			    , 'format' => 'FITS'
			    );
	
	# We really really really want to avoid running addattribute on ACDS products.
	# This can happen if we restart a post ACDS sequence at a pre ACDs stage and apply --acds_magic=1
	@list = grep { $_ !~ /CAX000/} @list;
	
	return success() unless @list;
	
	# Fetch observation properties
	my $obs    = getObservationProperties();
	my $odsver = getOdsVer();
	my $odfver = getOdfVer();
	
	# Use any old file to get global properties (like seq id)
	my $info = fileInfo( class => 'product', name => $list[0] );
	
	# Set values of keywords
	# Reset OBJECT keyword to 'Slew' for slew sequences

#	push @name, 'OBJECT';
#	push @type, 'string';
#	push @value, 'Slew';
#	push @comment, 'Object name';
#	push @withcomment, 'y';
	push @name, 'SEQ_ID';
	push @type, 'string';
	push @value, $info->{sequence};
	push @comment, 'Pipeline sequence';
	push @withcomment, 'y';
	push @name, 'REVOLUT';
	push @type, 'string';
	push @value, $obs->{orbit};
	push @comment, 'Satellite Revolution Number';
	push @withcomment, 'y';
	push @name, 'PROCDATE';
	push @type, 'string';
	push @value, $obs->{procdate};
	push @comment, 'Processing date';
	push @withcomment, 'y';
	push @name, 'PROCREV';
	push @type, 'string';
	push @value, $obs->{procrevision};
	push @comment, 'Processing revision';
	push @withcomment, 'y';
	push @name, 'PPSVERS';
	push @type, 'string';
	push @value, $obs->{ppsversion};
	push @comment, 'PPS configuration';
	push @withcomment, 'y';
	push @name, 'SASVERS';
	push @type, 'string';
	push @value, $obs->{sasversion};
	push @comment, 'SAS version';
	push @withcomment, 'y';
	push @name, 'ODSVER';
	push @type, 'string';
	push @value, "$odsver";
	push @comment, 'ODS version';
	push @withcomment, 'y';
	push @name, 'ORIGIN';
	push @type, 'string';
	push @value, 'Leicester/SSC';
	push @comment, 'Origin of FITS file';
	push @withcomment, 'y';

	# Added block to hack OBSERVER keyword name in slews - DLG

	push @name, 'OBSERVER';
	push @type, 'string';
	push @value, 'XMM-Newton';
	push @comment, 'Observer';
	push @withcomment, 'y';
	
	push @name, 'ODFVER';
	push @type, 'string';
	push @value, sprintf('%03d',$odfver);
	push @comment, 'ODF_VERSION';
	push @withcomment, 'y';

	# CONTENT keyword must be last in the list, as we set its value in
	# the loop
	push @name, 'CONTENT';
	push @type, 'string';
	push @comment, 'Contents of file';
	push @withcomment, 'y';
	
	# Loop over each file
	foreach my $file (@list)
	{
	    next unless fileExists( file => $file );
	    my $l_info = fileInfo( class => 'product', name => $file );
	    
	    # Set the other keywords
	    doCommand(
		      'addattribute', set => $file
		      , attributename => [@name]
		      , attributetype => [@type]
		      , stringvalue => [ @value, uc( $l_info->{content} ) ]
		      , attributecomment => [@comment]
		      )
		or return exception();
	}

	info("StampKeywords subroutine complete!");

    # --------------------------------------------------------
}

	# Fetch name of product index file
	my $prodIndex=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS product index'
	);

	# Compress all product .FIT to .FTZ
	&compressProducts()
		or return exception()
	;

	info("Product index file: $prodIndex");
	# Create index file
	makePPSIndex(file => $prodIndex) 
		or return exception()
	;

    if ( $ENV{PCMS_ISSLEW} )
    {
	    # Set other keywords for product index file
	    my $l_info = fileInfo( class => 'product', name => $prodIndex );
	    doCommand(
    		  'addattribute', set => $prodIndex
	    	  , attributename => [@name]
		      , attributetype => [@type]
		      , stringvalue => [ @value, uc( $l_info->{content} ) ]
		      , attributecomment => [@comment]
		      )
	        or return exception();
    }

	&compress(file => $prodIndex);
		
	# The product index might be compressed. 	
	# Re-find the filename
	$prodIndex=findFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS product index'
	);

	info("Found product index file: $prodIndex");
	# Get logo
	my $logoFrom = &logoFile();
	my $logoTo=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'SSC LOGO 2'
		,format => 'PNG'
	);

	copyFile(source => $logoFrom
		,destination => $logoTo
	);

	# Remove directories from file name
	my $obssummLogo = $logoTo;
	$obssummLogo =~ s/^.*\///;

	copyFile(source => $logoTo
		,destination => $obssummLogo
	);

	# Summary file names
	
	my $obsSummary=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS OBSERVATION SUMMARY'
		,format => 'HTML'
	);

	my $ppsSummary=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS RUN SUMMARY'
		,format => 'HTML'
	);


	my $epicSummary=newFile(class => 'product'
		,instrument => 'epic'
		,content => 'EPIC observation summary'
		,format => 'HTML'
	);

	my $rgsSummary=newFile(class => 'product'
		,instrument => 'rgs'
		,content => 'RGS observation summary'
		,format => 'HTML'
	);

	my $omSummary=newFile(class => 'product'
		,instrument => 'om'
		,content => 'OM observation summary'
		,format => 'HTML'
	);

	# ppssumm command line parameters
	my @ppssumm = (
		pindex => $prodIndex
		,obspage => $obsSummary
		,ppspage => $ppsSummary
		,epxpage => $epicSummary
		,omxpage => $omSummary
		,rgxpage => $rgsSummary
		,logofile => $obssummLogo
	);

	# The catalogue page may not exist
	my @catSummary=findFile(class => 'product'
		,instrument => 'cat'
		,content => 'MAIN CROSS CORRELATION PAGE'
		,format => 'HTML'
	);
	my $catSummary;
	if ( @catSummary )
	{	
		$catSummary = $catSummary[0];
	} 
	else
	{
		# Create a DB entry for this file.  It MUST be allocated to the SendACDS module
		# otherwise restarting Finalize will delete this file.  
		# This is a work around made necessary by an early cat2 bug.
		# This does mean this file cannot be recreated by running the Finalize module.
		$catSummary = newFile( class => 'product'
			, instrument => 'cat'
			, content => 'MAIN CROSS CORRELATION PAGE'
			, format => 'HTML'
			, module => 'SendACDS'
		);
	}
	push ( @ppssumm , ( catpage => $catSummary) );
			  
	# Fetch name of PPS log file. Ditto the comment above.
	my $logFile=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS script log'
		,format => 'ASCII'
	);

	# Create summary files
	info("Creating Summary Files");

    if ( $ENV{PCMS_ISSLEW} )
    {
        doCommand('ppsslewsumm'
    		,@ppssumm
	    ) or return exception();

        # Hack fix for product index file path in PPS summary file
	    Exec::system("sed -i 's/PINDEX0000.FIT/PINDEX0000.FTZ/g' $ppsSummary");
    }
    else
    {
	    doCommand('ppssumm'
		    ,@ppssumm
	    ) or return exception();
    }

	# Fetch the name of index html file (INDEX.HTM)
	my $htmlIndex=newFile(class => 'product'
		,instrument => 'ob'
		,content => 'PPS OBSERVATION SUMMARY'
		,name => 'INDEX.HTM'
		,format => 'HTML'
	);

	# Its just a copy of the obssumm page

	copyFile(source => $obsSummary, destination => $htmlIndex);    


	# Create log file
	createLogScript(file => $logFile)
		or return exception()
	;

	ModuleResources::Createprodset();

	# Pack the products up into an XFTS package
	prodPack() 
		or return exception()
	;
#	prodPacknoCompression() 
#		or return exception()
#	;
    if ( $ENV{PCMS_ISSLEW} )
	{
        # ------------------------------------------------
        # set blackboard flags forcing DPSS skip

	    my %senddpss = ( module => 'SendDPSS', stream => 1, instrument => 'all');
	    BlackBoard::raiseFlag( %senddpss, value => 'complete');

	    my %dpssdts = ( module => 'DPSSDTS', stream => 1, instrument => 'all');
	    BlackBoard::raiseFlag( %dpssdts, value => 'complete');

	    my %receivedpss = ( module => 'ReceiveDPSS', stream => 1, instrument => 'all');
	    BlackBoard::raiseFlag( %receivedpss, value => 'complete');

        # ------------------------------------------------
    }
	#
	return success();
}

1;