package Finalize;
use strict;
use English;
use Carp;
use vars
qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
use BlackBoard;
# Declare identity, version, author, date, etc.
$name = __PACKAGE__;
$VERSION = '2.20';
$version = $VERSION;
$author = 'Richard West,Dean Hinshaw,Duncan John Fyfe,Ian Stewart,Duncan Law-Green,Ed Chapin';
$date = '2014-03-11';
#
# ChangeLog
# =========
#
# Version 2.20 - 2014-03-11 (EC)
# ------------
#
# + Check for pnTimingProducts before starting
#
# Version 2.19 - 2013-06-11 (EC)
# ------------
#
# + Merge changes from SOC version.
# + This version includes for the first time the changes made to this module
# in Slew pipeline (2.06->2.18).
#
# version 2.18 - 2010-08-17 DLG
# ------------
#
# + SLEW ONLY: Try forcing DPSS skip via blackboard flags
#
# version 2.17 - 2010-08-10 DLG
# ------------
#
# + SLEW ONLY: Check for ignored('SlewProcess')
#
# version 2.16 - 2010-06-17 DLG
# ------------
#
# + SLEW ONLY: Additional addattibute call to add standard headers to PINDEX file
#
# version 2.15 - 2010-06-15 DLG
# ------------
#
# + SLEW ONLY: Added block to attempt to hack OBSERVER keyword in product output
#
# version 2.14 - 2010-06-02 DLG
# ------------
#
# + SLEW ONLY: Now uses 'ppsslewsumm' instead of 'ppssumm'
#
# version 2.13 - 2010-05-25 DLG
# ------------
#
# + SLEW ONLY: Modified OBJECT keyword to 'Slew' in StampKeywords section
#
# version 2.12 - 2010-05-17 DLG
# ------------
#
# + SLEW ONLY: Absorb StampKeywords into slew Finalize pro tem
#
# version 2.11 - 2010-05-13 DLG
# ------------
#
# + SLEW ONLY: Modify flow control to follow StampKeywords module call
#
# version 2.10 - 2010-02-10 DLG
# ------------
#
# + Use latest version of eslewchain (1.4) which generates agreed PCMS-compliant filenames
#
# version 2.09 - 2010-01-25 DLG
# ------------
#
# + Convert slew band numbers to PCMS standard
#
# version 2.08 - 2010-01-13 DLG
# ------------
#
# + Iterate over slew bands, rename step images
#
# version 2.07 - 2009-12-16 DLG
# ------------
#
# + Started slew products cleanup and generation of PCMS-compliant names
#
# version 2.06 - 2009-06-23 DLG
# ------------
#
# + Added start dependence on SlewProcess
#
# version 2.05 - 2006-12-05 DJF
# ------------
#
# + Need to call newFile to create the XCORRE product file name and db entry if it has not been
# returned by ACDS. If we don't use new file there will be no DB entry and it won't be packed
# into the .XMM file.
#
# version 2.04 - 2006-03-23 DJF
# ------------
#
# + Replace dependence on StampEndKeywords. That module has been removed
#
# version 2.03 - 2006-03-23 DJF
# ------------
#
# + Compression now tests for previously compressed files remivng the need for a specialised prodPack.
#
# version 2.02 - 2005-12-15 DJF
# ------------
#
# Separated product compression and product packing in module resources.
# Finalize amended to take adavantage of ths wrt ppssumm HTM pages
#
# version 2.01 - 2005-12-02 DJF
# ------------
#
# + Changed dependence from StampKeywords to StampEndKeywords
#
# version 2.00 - 2005-11-21 IMS
# ------------
#
# + Add explicit perl module headers. Previously these were supplied
# at compile time. This will make debugging and extending the modules
# through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic. Now $Version = version = ''
#
# Version 1.22 - 2004-02-21 (DJF)
# ------------
#
# + SAS 6.0 ppssumm needs a filename of the catpage
# parameter even if the file does not exist.
#
#
# Version 1.21 - 2003-07-17 (DJF)
# ------------
#
# + Adapted ppssumm command line to deal with absent
# catalogue index file.
#
# Version 1.20 - 2003-06-20 (DJF)
# ------------
#
# + Changed to take account of ppssumm 3.0+ which now
# generates all summary pages in one go.
#
# Version 1.19 - 2002-02-26 (DH)
# ------------
#
# + added additional output to logs to indicate progress
#
# Version 1.18 - 2002-02-26 (DH)
# ------------
#
# + Bug fixes for 1.17.
#
# Version 1.17 - 2002-02-20 (DJF)
# ------------
#
# + Change newFile to findFile for index files to be created.
# These files are now created for obssumm and populated here.
#
# Version 1.16 - 2001-11-04 (DH)
# ------------
#
# + Remove dependence on SendDPSS in evaluateRules.
#
# Version 1.15 - 2001-11-03 (DH)
# ------------
#
# + Remove sequenceComplete call from the end of the module.
#
# Version 1.14 - 2001-03-21 (DH)
# ------------
#
# + Replace getFile call for SSC logo to findFile call.
#
# Version 1.13 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
# tracking purposes.
#
# Version 1.12 - 2001-03-13 (DH)
# ------------
#
# + Add prodPack(), which creates the XFTS file for sending
# to the SOC, to the end of the processing.
#
# Version 1.11 - 2001-03-08 (DH)
# ------------
#
# + Move obs summary file production to VerifyODF module.
#
# Version 1.10 - 2001-02-19 (DH)
# ------------
#
# + Put in the new obssumm parameter useinfile.
#
# Version 1.09 - 2001-01-17 (DH)
# ------------
#
# + Change name of used logo from 'SSC LOGO 1'
# to 'SSC LOGO 2' to avoid conflicts with the ACDS.
#
# Version 1.08 - 2000-12-19 (DH)
# ------------
#
# + Fix bug with path to logo files.
#
#
# Version 1.07 - 2000-12-14 (DH)
# ------------
#
# + First production version.
#
# Declare list of instruments this module is interested in
@instList=qw(all);
# Number of streams
sub numberOfStreams {
return 1;
}
# Rules method
sub evaluateRules {
if ( $ENV{PCMS_ISSLEW} )
{
start()
if allComplete( module => 'SlewProcess' )
}
else
{
start()
if allComplete( module => 'ReceiveACDS' )
and allComplete( module => 'RGSProducts' )
and allComplete( module => 'RGSMakeFluxed' )
and allComplete( module => 'OMSourceCombine' )
and allComplete( module => 'OMMosaic' )
and allComplete( module => 'OMFastAnalyse' )
and allComplete( module => 'EPICSourceProducts' )
and allComplete( module => 'pnTimingProducts' )
}
}
# Action method
sub performAction {
info("Module version number: $version");
my ( @name, @type, @value, @comment, @withcomment );
if ( $ENV{PCMS_ISSLEW} )
{
# Set up parameters for slew product tidy ----------------
# Modified for eslewchain-1.4, PCMS-"compliant" filenames
my $rootdir = $ModuleResources::Seqdb->{'directories'}{'root'};
info("Sequence root directory: $rootdir");
# move files to intermediate subdir
# Does the new eslewchain still create these?
# moveFile(source => 'histo.fits',
# destination => 'intermediate/histo.fits');
# moveFile(source => 'image.fits',
# destination => 'intermediate/image.fits');
# moveFile(source => 'spectrum.fits',
# destination => 'intermediate/spectrum.fits');
# moveFile(source => "temp_sp_*",
# destination => 'intermediate/');
moveFile(source => "*Badpixels.ds",
destination => 'intermediate/') unless ignored( module => 'SlewProcess', stream => 1 );
# Identify output products and assign PCMS-compatible filenames
# Attitude file:P*OBX000ATTTSR0000.FITS
my $obsid = $ENV{'SAS_RAND_SEED'};
my ($attfile) = glob("$rootdir/P*OBX000ATTTSR0000.ds");
my $attdest = newFile(class => 'product'
,instrument => 'ob'
,content => 'SLEW ATTITUDE FILE'
,format => 'FITS'
);
moveFile(source => "$attfile"
,destination => "$attdest") unless ignored( module => 'SlewProcess', stream => 1 );
my $sourcename = sprintf("P%010s",$obsid);
doCommand(
'addattribute', set => $attdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
) unless ignored(module => 'SlewProcess', stream => 1 );
# Single raw events file: P*PNS003SLEVLI0000.FITS
my ($rawfile) = glob("$rootdir/*ImagingEvts.ds");
# my ($rawfile) = glob("$rootdir/P*PNS003PIEVLI0SLW.ds");
my $rawdest = newFile(class=>'product'
,instrument => 'epn'
,exp_id => 'S003'
,content => 'SLEW SINGLE RAW EVENT LIST'
,format => 'FITS'
);
moveFile(source => "$rawfile"
,destination => "$rawdest") unless ignored( module => 'SlewProcess', stream => 1 );
my $sourcename = sprintf("P%010s",$obsid);
doCommand(
'addattribute', set => $rawdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
) unless ignored( module => 'SlewProcess', stream => 1 );
# filtered event files per step: P*PNS003PIEVLI0*.FITS
my $filtfile;
my $filtdest;
my $filtstep = 0;
my @filtlist = glob("$rootdir/P*PNS003PIEVLI0*.ds");
foreach $filtfile (@filtlist) {
$filtdest = newFile(class => 'product'
,instrument => 'epn'
,exp_id => 'S003'
,content => 'SLEW STEP FILTERED EVENT LIST'
,format => 'FITS'
,src_num => "$filtstep" # this needs to be in decimal, not hex
);
moveFile(source => "$filtfile"
,destination => "$filtdest");
my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
doCommand(
'addattribute', set => $filtdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
);
$filtstep++;
}
# Slew step band exposure map: P*PNS003EXPMAPbiii.FITS
# Loop over slew bands
my $slewband;
my $stepfile;
my $stepdest;
foreach $slewband ( '6', '7', '8' ) {
$filtstep = 0;
my @steplist = glob("$rootdir/P*PNS003EXPMAP${slewband}*.ds");
foreach $stepfile (@steplist) {
my $stepdest = newFile(class => 'product'
, instrument => 'epn'
, exp_id => 'S003'
, content => 'SLEW STEP EXPOSURE MAP'
, format => 'FITS'
, src_num => "$filtstep"
, band => "$slewband"
);
moveFile(source => "$stepfile"
,destination=> "$stepdest");
my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
doCommand(
'addattribute', set => $stepdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
);
$filtstep++;
}
}
# Slew step band images: P*PNS003IMAGE_biii.FITS
# Loop over slew bands
foreach $slewband ( "6", "7", "8" ) {
$filtstep = 0;
my @steplist = glob("$rootdir/P*PNS003IMAGE_${slewband}*.ds");
foreach $stepfile (@steplist) {
my $stepdest = newFile(class => 'product'
, instrument => 'epn'
, exp_id => 'S003'
, content => 'SLEW STEP IMAGE'
, format => 'FITS'
, src_num => "$filtstep"
, band => "$slewband"
);
moveFile(source => "$stepfile"
,destination=> "$stepdest");
# Set OBJECT header to source step name (Nora's request)
my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
doCommand(
'addattribute', set => $stepdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
);
$filtstep++;
}
}
# Unfiltered step images: P*PNS003UNFDAT8*FITS
$filtstep = 0;
my $unffile;
my $unfdest;
my @unflist = glob("$rootdir/P*PNS003UNFDAT8*.ds");
foreach $unffile (@unflist) {
$unfdest = newFile(class => 'product'
,instrument => 'epn'
,exp_id => 'S003'
,content => 'UNFILTERED SLEW STEP IMAGE'
,format => 'FITS'
,src_num => "$filtstep" # this needs to be in decimal, not hex
,band => '8'
);
moveFile(source => "$unffile"
,destination => "$unfdest");
# Set OBJECT header to source step name (Nora's request)
my $sourcename = sprintf("P%010s_%03x",$obsid,$filtstep);
doCommand(
'addattribute', set => $unfdest
, attributename => 'OBJECT'
, attributetype => 'string'
, stringvalue => "$sourcename"
# , attributecomment => '"Object name"'
);
$filtstep++;
}
# Tidy: Drop all ds files from root directory
my @unlinklist = glob("$rootdir/*.ds");
unlink @unlinklist unless ignored( module => 'SlewProcess', stream => 1 );
# --------------------------------------------------------
# +++ StampKeywords +++
# Import portion of StampKeywords module to run here for slew
# Add basic headers to all FITS product files
info("Starting StampKeywords subroutine...");
# Fetch list of product files
my @list = findFile(
class => 'product'
, 'format' => 'FITS'
);
# We really really really want to avoid running addattribute on ACDS products.
# This can happen if we restart a post ACDS sequence at a pre ACDs stage and apply --acds_magic=1
@list = grep { $_ !~ /CAX000/} @list;
return success() unless @list;
# Fetch observation properties
my $obs = getObservationProperties();
my $odsver = getOdsVer();
my $odfver = getOdfVer();
# Use any old file to get global properties (like seq id)
my $info = fileInfo( class => 'product', name => $list[0] );
# Set values of keywords
# Reset OBJECT keyword to 'Slew' for slew sequences
# push @name, 'OBJECT';
# push @type, 'string';
# push @value, 'Slew';
# push @comment, 'Object name';
# push @withcomment, 'y';
push @name, 'SEQ_ID';
push @type, 'string';
push @value, $info->{sequence};
push @comment, 'Pipeline sequence';
push @withcomment, 'y';
push @name, 'REVOLUT';
push @type, 'string';
push @value, $obs->{orbit};
push @comment, 'Satellite Revolution Number';
push @withcomment, 'y';
push @name, 'PROCDATE';
push @type, 'string';
push @value, $obs->{procdate};
push @comment, 'Processing date';
push @withcomment, 'y';
push @name, 'PROCREV';
push @type, 'string';
push @value, $obs->{procrevision};
push @comment, 'Processing revision';
push @withcomment, 'y';
push @name, 'PPSVERS';
push @type, 'string';
push @value, $obs->{ppsversion};
push @comment, 'PPS configuration';
push @withcomment, 'y';
push @name, 'SASVERS';
push @type, 'string';
push @value, $obs->{sasversion};
push @comment, 'SAS version';
push @withcomment, 'y';
push @name, 'ODSVER';
push @type, 'string';
push @value, "$odsver";
push @comment, 'ODS version';
push @withcomment, 'y';
push @name, 'ORIGIN';
push @type, 'string';
push @value, 'Leicester/SSC';
push @comment, 'Origin of FITS file';
push @withcomment, 'y';
# Added block to hack OBSERVER keyword name in slews - DLG
push @name, 'OBSERVER';
push @type, 'string';
push @value, 'XMM-Newton';
push @comment, 'Observer';
push @withcomment, 'y';
push @name, 'ODFVER';
push @type, 'string';
push @value, sprintf('%03d',$odfver);
push @comment, 'ODF_VERSION';
push @withcomment, 'y';
# CONTENT keyword must be last in the list, as we set its value in
# the loop
push @name, 'CONTENT';
push @type, 'string';
push @comment, 'Contents of file';
push @withcomment, 'y';
# Loop over each file
foreach my $file (@list)
{
next unless fileExists( file => $file );
my $l_info = fileInfo( class => 'product', name => $file );
# Set the other keywords
doCommand(
'addattribute', set => $file
, attributename => [@name]
, attributetype => [@type]
, stringvalue => [ @value, uc( $l_info->{content} ) ]
, attributecomment => [@comment]
)
or return exception();
}
info("StampKeywords subroutine complete!");
# --------------------------------------------------------
}
# Fetch name of product index file
my $prodIndex=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS product index'
);
# Compress all product .FIT to .FTZ
&compressProducts()
or return exception()
;
info("Product index file: $prodIndex");
# Create index file
makePPSIndex(file => $prodIndex)
or return exception()
;
if ( $ENV{PCMS_ISSLEW} )
{
# Set other keywords for product index file
my $l_info = fileInfo( class => 'product', name => $prodIndex );
doCommand(
'addattribute', set => $prodIndex
, attributename => [@name]
, attributetype => [@type]
, stringvalue => [ @value, uc( $l_info->{content} ) ]
, attributecomment => [@comment]
)
or return exception();
}
&compress(file => $prodIndex);
# The product index might be compressed.
# Re-find the filename
$prodIndex=findFile(class => 'product'
,instrument => 'ob'
,content => 'PPS product index'
);
info("Found product index file: $prodIndex");
# Get logo
my $logoFrom = &logoFile();
my $logoTo=newFile(class => 'product'
,instrument => 'ob'
,content => 'SSC LOGO 2'
,format => 'PNG'
);
copyFile(source => $logoFrom
,destination => $logoTo
);
# Remove directories from file name
my $obssummLogo = $logoTo;
$obssummLogo =~ s/^.*\///;
copyFile(source => $logoTo
,destination => $obssummLogo
);
# Summary file names
my $obsSummary=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS OBSERVATION SUMMARY'
,format => 'HTML'
);
my $ppsSummary=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS RUN SUMMARY'
,format => 'HTML'
);
my $epicSummary=newFile(class => 'product'
,instrument => 'epic'
,content => 'EPIC observation summary'
,format => 'HTML'
);
my $rgsSummary=newFile(class => 'product'
,instrument => 'rgs'
,content => 'RGS observation summary'
,format => 'HTML'
);
my $omSummary=newFile(class => 'product'
,instrument => 'om'
,content => 'OM observation summary'
,format => 'HTML'
);
# ppssumm command line parameters
my @ppssumm = (
pindex => $prodIndex
,obspage => $obsSummary
,ppspage => $ppsSummary
,epxpage => $epicSummary
,omxpage => $omSummary
,rgxpage => $rgsSummary
,logofile => $obssummLogo
);
# The catalogue page may not exist
my @catSummary=findFile(class => 'product'
,instrument => 'cat'
,content => 'MAIN CROSS CORRELATION PAGE'
,format => 'HTML'
);
my $catSummary;
if ( @catSummary )
{
$catSummary = $catSummary[0];
}
else
{
# Create a DB entry for this file. It MUST be allocated to the SendACDS module
# otherwise restarting Finalize will delete this file.
# This is a work around made necessary by an early cat2 bug.
# This does mean this file cannot be recreated by running the Finalize module.
$catSummary = newFile( class => 'product'
, instrument => 'cat'
, content => 'MAIN CROSS CORRELATION PAGE'
, format => 'HTML'
, module => 'SendACDS'
);
}
push ( @ppssumm , ( catpage => $catSummary) );
# Fetch name of PPS log file. Ditto the comment above.
my $logFile=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS script log'
,format => 'ASCII'
);
# Create summary files
info("Creating Summary Files");
if ( $ENV{PCMS_ISSLEW} )
{
doCommand('ppsslewsumm'
,@ppssumm
) or return exception();
# Hack fix for product index file path in PPS summary file
Exec::system("sed -i 's/PINDEX0000.FIT/PINDEX0000.FTZ/g' $ppsSummary");
}
else
{
doCommand('ppssumm'
,@ppssumm
) or return exception();
}
# Fetch the name of index html file (INDEX.HTM)
my $htmlIndex=newFile(class => 'product'
,instrument => 'ob'
,content => 'PPS OBSERVATION SUMMARY'
,name => 'INDEX.HTM'
,format => 'HTML'
);
# Its just a copy of the obssumm page
copyFile(source => $obsSummary, destination => $htmlIndex);
# Create log file
createLogScript(file => $logFile)
or return exception()
;
ModuleResources::Createprodset();
# Pack the products up into an XFTS package
prodPack()
or return exception()
;
# prodPacknoCompression()
# or return exception()
# ;
if ( $ENV{PCMS_ISSLEW} )
{
# ------------------------------------------------
# set blackboard flags forcing DPSS skip
my %senddpss = ( module => 'SendDPSS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %senddpss, value => 'complete');
my %dpssdts = ( module => 'DPSSDTS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %dpssdts, value => 'complete');
my %receivedpss = ( module => 'ReceiveDPSS', stream => 1, instrument => 'all');
BlackBoard::raiseFlag( %receivedpss, value => 'complete');
# ------------------------------------------------
}
#
return success();
}
1;