package Finalize;
use strict;
use English;
use Carp;
use vars
qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
use BlackBoard;
# Declare identity, version, author, date, etc.
$name = __PACKAGE__;
$VERSION = '2.22';
$version = $VERSION;
$author = 'Richard West,Dean Hinshaw,Duncan John Fyfe,Ian Stewart,Duncan Law-Green,Ed Chapin,Pedro Rodriguez, Jose V Perea';
$date = '2017-01-30';
#
# ChangeLog
# =========
#
# Version 2.22 - 2017-01-30 (JVP)
# ------------
#
# + Temporal fix for an issue in the production of the EPIC Observation Summary HTM table
# when a Merged LightCurves PDF plot appears.
#
# Version 2.21 - 2016-07-07 (PR)
# ------------
#
# + Move slew products tidy to SlewProcess module
#
# Version 2.20 - 2014-03-11 (EC)
# ------------
#
# + Check for pnTimingProducts before starting
#
# Version 2.19 - 2013-06-11 (EC)
# ------------
#
# + Merge changes from SOC version.
# + This version includes for the first time the changes made to this module
# in Slew pipeline (2.06->2.18).
#
# version 2.18 - 2010-08-17 DLG
# ------------
#
# + SLEW ONLY: Try forcing DPSS skip via blackboard flags
#
# version 2.17 - 2010-08-10 DLG
# ------------
#
# + SLEW ONLY: Check for ignored('SlewProcess')
#
# version 2.16 - 2010-06-17 DLG
# ------------
#
# + SLEW ONLY: Additional addattibute call to add standard headers to PINDEX file
#
# version 2.15 - 2010-06-15 DLG
# ------------
#
# + SLEW ONLY: Added block to attempt to hack OBSERVER keyword in product output
#
# version 2.14 - 2010-06-02 DLG
# ------------
#
# + SLEW ONLY: Now uses 'ppsslewsumm' instead of 'ppssumm'
#
# version 2.13 - 2010-05-25 DLG
# ------------
#
# + SLEW ONLY: Modified OBJECT keyword to 'Slew' in StampKeywords section
#
# version 2.12 - 2010-05-17 DLG
# ------------
#
# + SLEW ONLY: Absorb StampKeywords into slew Finalize pro tem
#
# version 2.11 - 2010-05-13 DLG
# ------------
#
# + SLEW ONLY: Modify flow control to follow StampKeywords module call
#
# version 2.10 - 2010-02-10 DLG
# ------------
#
# + Use latest version of eslewchain (1.4) which generates agreed PCMS-compliant filenames
#
# version 2.09 - 2010-01-25 DLG
# ------------
#
# + Convert slew band numbers to PCMS standard
#
# version 2.08 - 2010-01-13 DLG
# ------------
#
# + Iterate over slew bands, rename step images
#
# version 2.07 - 2009-12-16 DLG
# ------------
#
# + Started slew products cleanup and generation of PCMS-compliant names
#
# version 2.06 - 2009-06-23 DLG
# ------------
#
# + Added start dependence on SlewProcess
#
# version 2.05 - 2006-12-05 DJF
# ------------
#
# + Need to call newFile to create the XCORRE product file name and db entry if it has not been
# returned by ACDS. If we don't use new file there will be no DB entry and it won't be packed
# into the .XMM file.
#
# version 2.04 - 2006-03-23 DJF
# ------------
#
# + Replace dependence on StampEndKeywords. That module has been removed
#
# version 2.03 - 2006-03-23 DJF
# ------------
#
# + Compression now tests for previously compressed files remivng the need for a specialised prodPack.
#
# version 2.02 - 2005-12-15 DJF
# ------------
#
# Separated product compression and product packing in module resources.
# Finalize amended to take adavantage of ths wrt ppssumm HTM pages
#
# version 2.01 - 2005-12-02 DJF
# ------------
#
# + Changed dependence from StampKeywords to StampEndKeywords
#
# version 2.00 - 2005-11-21 IMS
# ------------
#
# + Add explicit perl module headers. Previously these were supplied
# at compile time. This will make debugging and extending the modules
# through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic. Now $Version = version = ''
#
# Version 1.22 - 2004-02-21 (DJF)
# ------------
#
# + SAS 6.0 ppssumm needs a filename of the catpage
# parameter even if the file does not exist.
#
#
# Version 1.21 - 2003-07-17 (DJF)
# ------------
#
# + Adapted ppssumm command line to deal with absent
# catalogue index file.
#
# Version 1.20 - 2003-06-20 (DJF)
# ------------
#
# + Changed to take account of ppssumm 3.0+ which now
# generates all summary pages in one go.
#
# Version 1.19 - 2002-02-26 (DH)
# ------------
#
# + added additional output to logs to indicate progress
#
# Version 1.18 - 2002-02-26 (DH)
# ------------
#
# + Bug fixes for 1.17.
#
# Version 1.17 - 2002-02-20 (DJF)
# ------------
#
# + Change newFile to findFile for index files to be created.
# These files are now created for obssumm and populated here.
#
# Version 1.16 - 2001-11-04 (DH)
# ------------
#
# + Remove dependence on SendDPSS in evaluateRules.
#
# Version 1.15 - 2001-11-03 (DH)
# ------------
#
# + Remove sequenceComplete call from the end of the module.
#
# Version 1.14 - 2001-03-21 (DH)
# ------------
#
# + Replace getFile call for SSC logo to findFile call.
#
# Version 1.13 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
# tracking purposes.
#
# Version 1.12 - 2001-03-13 (DH)
# ------------
#
# + Add prodPack(), which creates the XFTS file for sending
# to the SOC, to the end of the processing.
#
# Version 1.11 - 2001-03-08 (DH)
# ------------
#
# + Move obs summary file production to VerifyODF module.
#
# Version 1.10 - 2001-02-19 (DH)
# ------------
#
# + Put in the new obssumm parameter useinfile.
#
# Version 1.09 - 2001-01-17 (DH)
# ------------
#
# + Change name of used logo from 'SSC LOGO 1'
# to 'SSC LOGO 2' to avoid conflicts with the ACDS.
#
# Version 1.08 - 2000-12-19 (DH)
# ------------
#
# + Fix bug with path to logo files.
#
#
# Version 1.07 - 2000-12-14 (DH)
# ------------
#
# + First production version.
#
# Declare list of instruments this module is interested in
@instList=qw(all);
# Number of streams
sub numberOfStreams {
return 1;
}
# Rules method
sub evaluateRules {
if ( $ENV{PCMS_ISSLEW} )
{
start()
if allComplete( module => 'SlewProcess' )
}
else
{
start()
if allComplete( module => 'ReceiveACDS' )
and allComplete( module => 'RGSProducts' )
and allComplete( module => 'RGSMakeFluxed' )
and allComplete( module => 'OMSourceCombine' )
and allComplete( module => 'OMMosaic' )
and allComplete( module => 'OMFastAnalyse' )
and allComplete( module => 'EPICSourceProducts' )
and allComplete( module => 'pnTimingProducts' )
and allComplete( module => 'LCMerge' )
}
}
# Action method
sub performAction {
info("Module version number: $version");
# Fetch name of product index file
my $prodIndex=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS product index'
);
# Compress all product .FIT to .FTZ
&compressProducts()
or return exception()
;
info("Product index file: $prodIndex");
# Create index file
makePPSIndex(file => $prodIndex)
or return exception()
;
if ( $ENV{PCMS_ISSLEW} )
{
info("Starting StampKeywords subroutine for the PPS product index ...");
my ( @name, @type, @value, @comment, @withcomment );
my @list = findFile(
class => 'product'
, 'format' => 'FITS'
);
# We really really really want to avoid running addattribute on ACDS products.
# This can happen if we restart a post ACDS sequence at a pre ACDs stage and apply --acds_magic=1
@list = grep { $_ !~ /CAX000/} @list;
return success() unless @list;
# Fetch observation properties
my $obs = getObservationProperties();
my $odsver = getOdsVer();
my $odfver = getOdfVer();
# Use any old file to get global properties (like seq id)
my $info = fileInfo( class => 'product', name => $list[0] );
push @name, 'SEQ_ID';
push @type, 'string';
push @value, $info->{sequence};
push @comment, 'Pipeline sequence';
push @withcomment, 'y';
push @name, 'REVOLUT';
push @type, 'string';
push @value, $obs->{orbit};
push @comment, 'Satellite Revolution Number';
push @withcomment, 'y';
push @name, 'PROCDATE';
push @type, 'string';
push @value, $obs->{procdate};
push @comment, 'Processing date';
push @withcomment, 'y';
push @name, 'PROCREV';
push @type, 'string';
push @value, $obs->{procrevision};
push @comment, 'Processing revision';
push @withcomment, 'y';
push @name, 'PPSVERS';
push @type, 'string';
push @value, $obs->{ppsversion};
push @comment, 'PPS configuration';
push @withcomment, 'y';
push @name, 'SASVERS';
push @type, 'string';
push @value, $obs->{sasversion};
push @comment, 'SAS version';
push @withcomment, 'y';
push @name, 'ODSVER';
push @type, 'string';
push @value, "$odsver";
push @comment, 'ODS version';
push @withcomment, 'y';
push @name, 'ORIGIN';
push @type, 'string';
push @value, 'Leicester/SSC';
push @comment, 'Origin of FITS file';
push @withcomment, 'y';
# Added block to hack OBSERVER keyword name in slews - DLG
push @name, 'OBSERVER';
push @type, 'string';
push @value, 'XMM-Newton';
push @comment, 'Observer';
push @withcomment, 'y';
push @name, 'ODFVER';
push @type, 'string';
push @value, sprintf('%03d',$odfver);
push @comment, 'ODF_VERSION';
push @withcomment, 'y';
# CONTENT keyword must be last in the list, as we set its value in
# the loop
push @name, 'CONTENT';
push @type, 'string';
push @comment, 'Contents of file';
push @withcomment, 'y';
# Set other keywords for product index file
my $l_info = fileInfo( class => 'product', name => $prodIndex );
doCommand(
'addattribute', set => $prodIndex
, attributename => [@name]
, attributetype => [@type]
, stringvalue => [ @value, uc( $l_info->{content} ) ]
, attributecomment => [@comment]
)
or return exception();
info("StampKeywords subroutine complete for the PPS product index");
}
&compress(file => $prodIndex);
# The product index might be compressed.
# Re-find the filename
$prodIndex=findFile(class => 'product'
,instrument => 'ob'
,content => 'PPS product index'
);
info("Found product index file: $prodIndex");
# Get logo
my $logoFrom = &logoFile();
my $logoTo=newFile(class => 'product'
,instrument => 'ob'
,content => 'SSC LOGO 2'
,format => 'PNG'
);
copyFile(source => $logoFrom
,destination => $logoTo
);
# Remove directories from file name
my $obssummLogo = $logoTo;
$obssummLogo =~ s/^.*\///;
copyFile(source => $logoTo
,destination => $obssummLogo
);
# Summary file names
my $obsSummary=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS OBSERVATION SUMMARY'
,format => 'HTML'
);
my $ppsSummary=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS RUN SUMMARY'
,format => 'HTML'
);
my $epicSummary=newFile(class => 'product'
,instrument => 'epic'
,content => 'EPIC observation summary'
,format => 'HTML'
);
my $epicSummaryCorrected=newFile(class => 'intermediate'
,instrument => 'epic'
,content => 'EPIC observation summary corrected'
,format => 'HTML'
);
my $rgsSummary=newFile(class => 'product'
,instrument => 'rgs'
,content => 'RGS observation summary'
,format => 'HTML'
);
my $omSummary=newFile(class => 'product'
,instrument => 'om'
,content => 'OM observation summary'
,format => 'HTML'
);
# ppssumm command line parameters
my @ppssumm = (
pindex => $prodIndex
,obspage => $obsSummary
,ppspage => $ppsSummary
,epxpage => $epicSummary
,omxpage => $omSummary
,rgxpage => $rgsSummary
,logofile => $obssummLogo
);
# The catalogue page may not exist
my @catSummary=findFile(class => 'product'
,instrument => 'cat'
,content => 'MAIN CROSS CORRELATION PAGE'
,format => 'HTML'
);
my $catSummary;
if ( @catSummary )
{
$catSummary = $catSummary[0];
}
else
{
# Create a DB entry for this file. It MUST be allocated to the SendACDS module
# otherwise restarting Finalize will delete this file.
# This is a work around made necessary by an early cat2 bug.
# This does mean this file cannot be recreated by running the Finalize module.
$catSummary = newFile( class => 'product'
, instrument => 'cat'
, content => 'MAIN CROSS CORRELATION PAGE'
, format => 'HTML'
, module => 'SendACDS'
);
}
push ( @ppssumm , ( catpage => $catSummary) );
# Fetch name of PPS log file. Ditto the comment above.
my $logFile=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS script log'
,format => 'ASCII'
);
# Create summary files
info("Creating Summary Files");
if ( $ENV{PCMS_ISSLEW} )
{
doCommand('ppsslewsumm'
,@ppssumm
) or return exception();
# Hack fix for product index file path in PPS summary file
Exec::system("sed -i 's/PINDEX0000.FIT/PINDEX0000.FTZ/g' $ppsSummary");
}
else
{
doCommand('ppssumm'
,@ppssumm
) or return exception();
#=======================>
# Temporal fix for an issue in the production of the EPIC Observation Summary HTM table
# when a Merged LightCurves PDF plot appears.
if ( fileExists( file => $epicSummary )){
my $trimPPS = trimPPSsum(file => $epicSummary, outfile => $epicSummaryCorrected);
}
copyFile(source => $epicSummaryCorrected, destination => $epicSummary);
#=======================>
}
# Fetch the name of index html file (INDEX.HTM)
my $htmlIndex=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS OBSERVATION SUMMARY'
,name => 'INDEX.HTM'
,format => 'HTML'
);
# Its just a copy of the obssumm page
copyFile(source => $obsSummary, destination => $htmlIndex);
# Create log file
createLogScript(file => $logFile)
or return exception()
;
ModuleResources::Createprodset();
# Pack the products up into an XFTS package
prodPack()
or return exception()
;
# prodPacknoCompression()
# or return exception()
# ;
if ( $ENV{PCMS_ISSLEW} )
{
# ------------------------------------------------
# set blackboard flags forcing DPSS skip
my %senddpss = ( module => 'SendDPSS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %senddpss, value => 'complete');
my %dpssdts = ( module => 'DPSSDTS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %dpssdts, value => 'complete');
my %receivedpss = ( module => 'ReceiveDPSS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %receivedpss, value => 'complete');
# ------------------------------------------------
}
#
return success();
}
1;