package OMImageAnalyse;
use strict;
use English;
use Carp;
use Regexp::Common qw(boolean);
use vars
qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
use Astro::FITS::CFITSIO qw(TRUE FALSE);
use ModuleUtil;
#### DEBUG
use Data::Dumper;
#### DEBUG
# Declare identity, version, author, date, etc.
$name = __PACKAGE__;
$VERSION = '3.23';
$version = $VERSION;
$author = 'Duncan Law-Green,Dean Hinshaw,Duncan John Fyfe,Ian Stewart,Eduardo Ojero,Ed Chapin,Jose Vicente Perea';
$date = '2014-04-10';
#
# ChangeLog
# =========
#
# Version 3.23 - 2014-04-10 (JVP)
# ------------
#
# + The values in the QUALITY maps from omqualitymap are wrong when created in 64bits machines.
# Therefore, the calls to omqualitymap introduced after PPP SCR 7143 have been dropped.
#
# Version 3.22 - 2013-10-07 (EC)
# ------------
#
# + Add omqualitymap calls (Mantis 0007143)
#
# Version 3.21 - 2013-06-11 (EC)
# ------------
#
# + Merge changes from SOC version.
#
# version 3.20 - 2012-05-03 DLG
# ------------
#
# + Eliminate use of GIF bitmaps
#
# version 3.19 - 2012-03-22 DLG
# ------------
#
# + omatt input changed from reference catalogue intermediate to product
#
# version 3.18.1 - 2011-08-09 EOP
# --------------
#
# + Set implot to produce directly PNG files instead of GIF.
#
# version 3.18 - 2010-09-02 DLG
# ------------
#
# + Fixed name of USNO extract catalogue to 'Reference catalogue'
#
# version 3.17 - 2008-10-06 DLG
# ------------
#
# + Added debugging code
# + Change minsignificance to 3 in line with the omichain and the value
# used to construct the omcatalogue
#
# version 3.16 - 2006-09-20 DJF
# ------------
#
# + Record use of exposures for source specific product creation in the database.
#
# version 3.15 - 2006-09-05 DJF
# ------------
#
# + changed 'OM OSW SOURCE LIST' to 'OM OSW GRISM SOURCE LIST' for GRISM processing.
# + Added new product 'OM OSW REGION FILE' generated by omdetect for non-grism images.
#
# version 3.14 - 2006-08-24 DJF
# ------------
#
# + Disabled OM GRISM-ALIGNED IMAGE PNG file creation. implot does not
# support the required CTYPE1|2 yet.
#
# version 3.13 - 2006-08-12 DJF
# ------------
#
# + Added code to make intermediate file OM GRISM-ALIGNED IMAGE PNG file for evaluation.
#
# version 3.12 - 2006-07-13 DJF
# ------------
#
# + Use OmGWIN keyword. Only continue omgrism processing if omgprep sets OMGWIN to true.
#
# version 3.11 - 2006-02-20 DJF
# ------------
#
# + Change omdetect:minsignificane=2.0 to 2.5 by request from Simon Rosen.
# This should cut down on the number of spurious sources.
#
# version 3.10 - 2006-02-07 DJF
# ------------
#
# + omcomb should now set a keywork IMGGAP to indicate any gabs in an ENG2 combined image.
# It does not make sense to process the combined image where IMGGAP = 2|3 and the filter is GRISM.
# In this case we process the individual components as USER mode images.
# Otherwise the combined image is usefull and we process it normally.
#
# version 3.09 - 2006-02-06 DJF
# ------------
#
# + Now collecting keywords from ENG4 files. This was preventing identification of
# ENG4+GRISM
#
# version 3.08 - 2006-01-19 DJF
# ------------
#
# + It had been assumed that if the filter was GRISM then the data was imaging
# mode. This ios not true. 0448_0153750701 has OM Fast mode data (E4I)
# taken with the grism filter. This module has been amended to skip such
# data.
# + Perform isImageEmpty test before calling implot.
#
# version 3.07 - 2005-11-18 IMS
# ------------
#
# + USNO extract now done by module USNOExtract. 'start' dependency (srcMerge no longer necessary) and findFile of usno altered accordingly.
#
# version 3.06 - 2005-11-17 IMS
# ------------
#
# + Tests now also for the existence of the omgrism output dataset before proceeding.
#
# version 3.05 - 2005-10-24 RGW
# ------------
#
# + don't try to create a OM grism spectrum plot if the FITS spectrum file doesn't contain a spectrum
# + backed out the hack preventning ENG4 data being analysed
#
# version 3.04 - 2005-09-14 RGW
# ------------
#
# + updated trigger conditions (now depends on SrcMerge)
# + changed minsignificance=2.0 on omdetect
# + omatt now uses the USNO catalogue for rectification
#
# version 3.03 - 2005-09-13 IMS
# ------------
#
# + Fixed bug in which keywords from the ODF images were not read when in PFENG2 mode (immediate effect was that grism images weren't being processed as such).
#
# version 3.02 - 2005-09-07 RGW
# ------------
#
# + hack: PFENG2 images are not processed unless there are 4 of them
#
# version 3.01 - 2005-07-28 IMS
# ------------
#
# + Added grism handling. This necessitates a very ugly query of FILTER keyword in order to cope with the clunky omprep interface.
# + Final sky image accounting file is not written in grism case (prevents OMMosaic working on grism images).
#
# version 3.00 - 2005-05-09 DJF
# ------------
#
# + Add explicit perl module headers. Previously these were supplied
# at compile time. This will make debugging and extending the modules
# through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic. Now $Version = version = ''
##
# Version 2.31 - 25-Feb-2004 (DJF)
# ------------
#
# + Explicitly set omprep modeset=0
#
# Version 2.30 - 10-Dec-2003 (DJF)
# ------------
#
# + Adapted doCommand to use anonymous lists for list parameters.
#
# Version 2.29 - 13-Dec-2002 (DJF)
# ------------
#
# + implot parameter changes. Merged plotfile/Device
#
# Version 2.28 - 11-Dec-2002 (DJF)
# ------------
#
# + Remove redundant omdetect parameters (mod8set, flatset)
# + Added parameters nsigma , minsignificance & detectextended
# to omdetect using default values 2.0 , 3.0 anmd 'Y' respectively.
# + Remove redundant ommag parmater wdxset
# + Change implot parameter withsrclistset to withsrclisttab (SAS 5.4)
#
# Version 2.27 - 30-Oct-2002 (DJF)
# ------------
#
# + OM Source rectification removed to a separate module
#
# Version 2.24 - 2.26 - 23-Oct-2002 (DJF)
# ------------
#
# + Position recitification isn't working. Attempt to fix by
# removing preselection from USNO.
#
# Version 2.23 - 30-May-2002 (DJF)
# ------------
#
# + Added position rectification against usno as an intermediate product
# for testing.
#
# Version 2.22 - 23-Jan-2002 (DH)
# ------------
#
# + Remove 'use strict' from file.
#
# Version 2.21 - 23-Jan-2002 (DFJ)
# ------------
#
# + Added exposure time (duration) and exp_id to
# sky image details temporary file for use by OMMosaic Module
#
# Version 2.20 - 2001-12-04 (DFJ)
# ------------
#
# + Added tolerance to mode identification to deal with Rudi-5 strip
# clipping
#
# Version 2.19 - 2001-11-29 (DFJ)
# ------------
#
# + Fixed bug which prevented pthe processing of RUDI5_HR images
#
# Version 2.18 - 2001-11-28 (DFJ)
# ------------
#
# + Added creation of PNG for each sky image
#
# Version 2.17 - 2001-11-28 (DFJ)
# ------------
#
# +Fix information written to ASCII files
#
# Version 2.16 - 2001-11-27 (DFJ)
# ------------
#
# +Fix use of writeASCIIFile
#
# Version 2.15 - 2001-11-27 (DFJ)
# ------------
#
# + Remove debugging code
# + Fix osw/filter selection on image output mode
#
# Version 2.14 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed error in writing of ASCII image details files
#
# Version 2.13 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed mode selection fo products details
#
# Version 2.12 - 2001-11-27 (DFJ)
# ------------
#
# + Added debugging code to help track problem with image creation
#
# Version 2.11 - 2001-11-27 (DFJ)
# ------------
#
# + Remove debugging code.
# + Retrieve FILTER keyword from primary header of raw image file
#
# Version 2.10 - 2001-11-27 (DFJ)
# ------------
#
# + Simplified logic of bad header ENG2/USER mode differentiation logic
# in attemt to eliminate a bug
#
# Version 2.09 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed keyword usage in image mode identification
#
# Version 2.08 - 2001-11-27 (DFJ)3# ------------
#
# + Added some debugging code to help track down a problem with image mode indentification
#
# Version 2.07 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in logic of bad header ENG2/USER mode differentiation logic
#
# Version 2.06 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.05 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.04 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.03 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.02 - 2001-11-26 (DFJ)
# ------------
#
# + Added OM File name filter abbreviations
#
# Version 2.01 - 2001-11-26 (DFJ)
# ------------
#
# + Changed recording of modes in ASCII list files. ENG2 products
# are now all recorded with as same mode ($ENG2)
# + Added new product content strings
#
# Version 2.00 - 2001-11-26 (DFJ)
# ------------
#
# + Major re-writing. Now incorporates Image mode determination
# and writing of product, mode and filter information to a log file
# for use by OMMosaic and OMSourceCombine
# + Change in product naming convention to include filter in place of the osw
# for Engineering 2 and 4 products.
#
# Version 1.22 - 2001-11-16 (DFJ)
# ------------
#
# + Extract filer and window size information from the images.
# + Add new product classes based on image information and classification
#
# Version 1.21 - 2001-11-05 (DFJ)
# ------------
#
# + Fix selection of osw. Was failing to recognize 0 as a valid osw.
#
# Version 1.20 - 2001-11-02 (DFJ)
# ------------
#
# + Fix for EN2. Was looking in odf rather than intermediate for combined image.
#
# Version 1.19 - 2001-11-02 (DFJ)
# ------------
#
# + Changed dummy osw value to 9 to mame it more obvious.
# + osw = 9 when no valid osw is found in the odf.
#
# Version 1.18 - 2001-11-02 (DFJ)
# ------------
#
# + Fix setting of osw. Was adding characters OSW to value then using it.
#
# Version 1.17 - 2001-10-31 (DFJ)
# ------------
#
# + Change determination of osw to account for different sources (osw or osw_id)
#
# Version 1.16 - 2001-10-30 (DFJ)
# ------------
#
# + Amended use of osw info.
#
# Version 1.15 - 2001-10-19 (DFJ)
# ------------
#
# + Added default osw id of zero for engineering mode 2
#
# Version 1.14 - 2001-10-18 (DH)
# ------------
#
# + Fix minor bugs in evaluateRules().
# + Update changed parameter names in omflatfield and ommodmap.
#
# Version 1.13 - 2001-10-11 (DJF)
# ------------
#
# + Remove exposure processing which is commom to both image and fast chains.
# This has been moved to modules OMExpAnalyse
#
# Version 1.12 - 2001-04-23 (DH)
# ------------
#
# + Change the OM OSW image from an intermediate
# file to a product.
#
# Version 1.11 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
# tracking purposes.
#
# Version 1.10 - 2001-03-08 (DH)
# ------------
#
# + Change om flat field from being a product file to
# being an intermediate file.
#
# Version 1.09 - 2000-11-29 (DH)
# ------------
#
# + First production version.
#
# Declare list of instruments this module is interested in
@instList = qw(om);
sub Instlist
{
return qw(om);
}
# Number of streams
sub numberOfStreams
{
return numberOfExposures();
}
sub evaluateRules
{
# If upstream module has been ignored, so if this one
return ignore()
if ignored(
module => 'OMgetFlat'
, instrument => thisInstrument
, stream => 1
);
return ignore()
if ignored(
module => 'OMExpAnalyse'
, instrument => thisInstrument
, stream => thisStream
);
# Start conditions
start()
if complete(
module => 'OMExpAnalyse'
, instrument => thisInstrument
, stream => thisStream
)
# and complete (
# module => 'SrcMerge'
# , instrument => 'all'
# , stream => 1
# );
and complete (
module => 'USNOExtract'
, instrument => 'all'
, stream => 1
);
}
use vars qw( %Filters );
my %Filters = (
'BLOCKED' => '9'
, 'V' => 'V'
, 'U' => 'U'
, 'B' => 'B'
, 'WHITE' => 'W'
, 'GRISM2' => 'G'
, 'GRISM1' => 'H'
, 'UVW1' => 'L'
, 'UVW2' => 'S'
, 'UVM2' => 'M'
);
sub getfiltershortname
{
my ($c,$f) = (@_);
$f = uc($f);
return exists( $Filters{$f} )
? $Filters{$f}
: exception( Exception->('BADOMFILTER','Unrecognized filter :',$f) );
;
}
sub performAction
{
info("Module version number: $version");
# Find exposure ID
my $exp_id = exposureID(
instrument => thisInstrument
,stream => thisStream
);
# FUNNY is not a real mode. It is used when determining image formats which as it suggests may be funny
# (eg. cases where the FITS header are known to be wrong)
# PFENG2 = Partial Frame Engineering 2, FFENG2 = Full frame Engineering 2
# PFENG2 and FFENG2 are different as input but produce the same output as more ENG2
# This hash describes the output products
my %products = (
USER => {
IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
, SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
, RUDI5_LR => {
IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
, SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
, RUDI5_HR => {
IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
, SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
, FFENG2 => {
IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
, SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
, PFENG2 => {
IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
, SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
, ENG4 => {
IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
, SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
, SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
}
);
# This hash describes the (eventual) input images.
my %inputimage = (
RUDI5_LR => {
CLASS => 'odf'
, CONTENT => 'Imaging mode image'
, LIST => []
}
, RUDI5_HR => {
CLASS => 'odf'
, CONTENT => 'Imaging mode image'
, LIST => []
}
, PFENG2 => {
CLASS => 'intermediate'
, CONTENT => 'Engineering Mode 2 Combined Image'
, LIST => []
}
, FFENG2 => {
CLASS => 'odf'
, CONTENT => 'Engineering mode image'
, LIST => []
}
, ENG4 => {
CLASS => 'odf'
, CONTENT => 'Engineering mode image'
, LIST => []
}
, USER => {
CLASS => 'odf'
, CONTENT => 'Imaging mode image'
, LIST => []
}
);
my $count = 0;
info("Looking for Engineering Mode Files...");
my @englist = findFile(
class => 'odf'
, instrument => thisInstrument
, exp_id => $exp_id
, content => 'Engineering mode image'
);
$count = scalar(@englist) || "None";
info("Found $count Engineering mode Windows.");
# Collect relevant details from file headers
my %keywords;
###
### Look for engineering 4
### Extract the necessary keywords.
###
$inputimage{ENG4}{LIST} = [ grep ( /E4I/, @englist ) ];
foreach my $e4i ( @{$inputimage{ENG4}{LIST}} )
{
$keywords{$e4i} = { &get_keywords($e4i,'OME4I1') };
}
$count = scalar(@{$inputimage{ENG4}{LIST}}) || 'No';
info("Found $count Full Frame Engineering 4 Windows.");
###
### Look for full-frame Engineering mode 2
### Extract the necessary keywords.
###
# NB. At the time of writing E2I files are not part of the ODF but this is a future proposal.
# If they appear E2I should be a precombined low res full sky image
# replacing omcomb on partial frame (IMI) images below.
# Keep separate FFENG2 and PFENG2 lists so we know what came from where.
push @{$inputimage{FFENG2}{LIST}} , ( grep ( /E2I/, @englist ));
foreach my $e2i ( @{ $inputimage{FFENG2}{LIST} } )
{
$keywords{$e2i} = { &get_keywords($e2i,'OME2I1') };
}
$count = scalar(@{$inputimage{FFENG2}{LIST}}) || 'no';
info("Found $count Full Frame Engineering 2 Windows.");
###
### Image Modes
###
info("Looking for Image Mode Files...");
my @imiList = findFile(
class => 'odf'
, instrument => thisInstrument
, exp_id => $exp_id
, content => 'Imaging mode image'
);
$count = scalar(@imiList) || "no";
info("Found $count Image mode images.");
# Determine the mode of each image file
### Extract the necessary keywords.
foreach my $imi (@imiList)
{
$keywords{$imi} = { &get_keywords($imi,'OMIMI1') };
&image_format( $imi , $keywords{$imi} );
my $mode = $keywords{$imi}{MODE};
info("Identified mode: $mode");
push( @{ $inputimage{$mode}{LIST} }, $imi );
}
###
### Pre-process Partial-Frame Engineering mode 2
###
### If we have engineering 2 pieces we need to combine them.
### If they fail to combine we assume they are USER defined images.
my $combine = 0;
my $imggap;
if ( @{ $inputimage{PFENG2}{LIST} } > 2 )
{
my @pfeng2list = @{ $inputimage{PFENG2}{LIST} };
$inputimage{PFENG2}{LIST} = [];
info("Might have Partial-Frame Engineering Mode 2 images.");
info("Trying to generate a combined image.");
# omcomb should now add a keyword IMGGAP to the final image.
# If it is 2 or 3 and filter == GRISM then we should not use the merged image
# but process the components as USER mode images instead.
my $en2comb = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, content => 'Engineering Mode 2 Combined Image'
);
$combine = doCommand(
'omcomb', imagesets => [ @pfeng2list ]
, outset => $en2comb
);
# If combining succeded...
if ($combine)
{
info("Successfully created a combined image.");
$keywords{$en2comb} = { &get_keywords($en2comb , 'OMIMI1') };
# IMGGAP tells us of any missing Partial-Frame windows.
$imggap = readFITSKeyword(
class => 'odf'
, file => $en2comb
, extension => 'OMIMI1'
, keyword => 'IMGGAP'
);
if ($imggap)
{
# If GRISM image is missing a central window process as USER mode.
if (( $keywords{$en2comb}{FILTER} eq '200' || $keywords{$en2comb}{FILTER} eq '1000' )
&& ($imggap eq '2' || $imggap eq '3')
){
info( "Inner segment of combined image missing with GRISM filter." );
info( "Will process as individual USER mode images." );
$combine = 0;
push @{ $inputimage{USER}{LIST} } , @pfeng2list;
}
else
{
info( "Outer segment of combined image missing with GRISM filter." );
info( "Will continue to process as an ENG2 image." );
}
}
else
{
info( "Processing as Engineering mode 2" );
push ( @{ $inputimage{PFENG2}{LIST} } , $en2comb );
}
}
}
else
{
info(
"Failed to make a suitable combined image. Will process components as USER mode."
);
push( @{ $inputimage{USER}{LIST} } , @{ $inputimage{PFENG2}{LIST} });
$inputimage{PFENG2}{LIST} = [];
}
###
###
###
#
# Find necessary files
# %p hold parameters necessary for processing images.
my %p;
$p{windowFile} = findFile(
class => 'odf'
, instrument => thisInstrument
, exp_id => $exp_id
, content => 'Priority window data'
, required => 'true'
);
# Find periodic housekeeping file
$p{phkFile} = findFile(
class => 'odf'
, instrument => thisInstrument
, content => 'Periodic housekeeping'
, required => 'true'
);
# Find non-periodic housekeeping file
$p{nphkFile} = findFile(
class => 'odf'
, instrument => thisInstrument
, content => 'Non-periodic housekeeping'
, required => 'true'
);
# Find omprep tracking history
$p{prepThFile} = findFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, content => 'omprep tracking history'
);
# Flat created by omflatgen
$p{flatGenFile} = findFile(
class => 'intermediate'
, instrument => thisInstrument
, content => 'OM Flatfield'
);
#Arrays for Image and Source details
my @skyimagedetails = ();
my @sourcedetails = ();
# Loop over all modes
foreach my $mode ( 'USER', 'RUDI5_LR', 'RUDI5_HR', 'PFENG2' , 'FFENG2' , 'ENG4' )
{
info("Processing $mode Images.");
my @imList = @{ $inputimage{$mode}{LIST} };
$count = scalar(@imList);
info("$count images to process.");
next unless ($count);
# Loop over all images of a given mode
foreach my $imFile (@imList)
{
# Default to osw of 9 to make it obvious where a value was missing.
Msg->mark( __PACKAGE__ , INPUTIMAGE => \%inputimage, MODE => $mode );
my $imInfo = fileInfo(
class => $inputimage{$mode}{CLASS}
, name => $imFile
);
my $osw = $imInfo->{'osw'};
$osw = $imInfo->{'osw_id'}
unless ( defined($osw) )
;
$osw = 9
unless ( defined($osw) )
; # The value 9 should make it obvious when an osw value is missing.
info("Processing $exp_id:$osw:$imFile");
$p{rawImFile} = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'Raw Image'
);
# Have to process grism separately. Whether grism or not is decided on the value of the FILTER kwd:
my $result;
if ($keywords{$imFile}{FILTER} eq '200' || $keywords{$imFile}{FILTER} eq '1000')
{
$result = &process_grism_image(exp_id => $exp_id
,osw => $osw
,imFile => $imFile
,mode => $mode
,keywords => $keywords{$imFile}
,skyimageoutput => $products{$mode}{SKYIMG}
,srcoutput => $products{$mode}{SRC}
,imgoutput => $products{$mode}{IMG}
,%p
);
}
else
{
$result = &process_image(exp_id => $exp_id
,osw => $osw
,imFile => $imFile
,mode => $mode
,keywords => $keywords{$imFile}
,skyimageoutput => $products{$mode}{SKYIMG}
,srcoutput => $products{$mode}{SRC}
,imgoutput => $products{$mode}{IMG}
,%p
);
}
if ($result)
{
#### DEBUG
# $result->{srcdetails} is returned as an AoA. WHY??
# flatten srcdetails AoA
my @tmpsrcdetails = $result->{srcdetails};
my $di = 0;
my @flatsrcdetails = ();
for ($di = 0; $di <= $#tmpsrcdetails; $di++) {
push (@flatsrcdetails, $tmpsrcdetails[$di][0]);
}
# flatten skyimgdetails AoA
my @tmpskyimgdetails = $result->{skyimgdetails};
my $di = 0;
my @flatskyimgdetails = ();
for ($di = 0; $di <= $#tmpskyimgdetails; $di++) {
push (@flatskyimgdetails, $tmpskyimgdetails[$di][0]);
}
# push @skyimagedetails , $result->{skyimgdetails};
# push @sourcedetails , $result->{srcdetails};
push @skyimagedetails, @flatskyimgdetails;
push @sourcedetails, @flatsrcdetails;
#### DEBUG
}
}
}
# Everything was OK
info("All images processed");
info("Writing accounting information");
#### DEBUG
my $debugsourcedetails = Dumper(\@sourcedetails);
info("DEBUG: sourcedetails array: $debugsourcedetails");
my $debugskyimagedetails = Dumper(\@skyimagedetails);
info("DEBUG: skyimagedetails array: $debugskyimagedetails");
#### DEBUG
#Write SkyImage and Source information for use by later modules
if (@sourcedetails)
{
my $sourcedetailsfile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, format => "ASCII"
, content => "OM SRC IMAGE DETAILS"
);
writeASCIIFile(
name => $sourcedetailsfile
, text => \@sourcedetails
);
}
# Note that sky image details are not stored for grism images. This has the effect of preventing mosaics of grism images being made by OMMosaic. IMS 2005-08-02.
if (@skyimagedetails)
{
my $skyimagedetailsfile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, format => "ASCII"
, content => "OM SKY IMAGE DETAILS"
);
writeASCIIFile(
name => $skyimagedetailsfile
, text => \@skyimagedetails
);
}
return success();
}
sub process_image
{
my ( %h ) = (@_);
my ($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $flatGenFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords );
($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $flatGenFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords )
= @h{qw( exp_id osw imFile rawImFile nphkFile phkFile windowFile prepThFile flatGenFile skyimageoutput srcoutput imgoutput mode keywords)}
;
my $rtn = { srcdetails => [] , skyimgdetails => []};
# Run omprep on the image
# Find window data file
doCommand(
'omprep', set => $imFile
, nphset => $nphkFile
, pehset => $phkFile
, wdxset => $windowFile
, outset => $rawImFile
, modeset => 0
)
or return exception();
# Get the filter value now omprep has converted it from a code to a recognizable value
# Also get value from EXPOSURE keyword as duration.
info("Retrieving FILTER value");
my $filter = readFITSKeyword(
class => 'intermediate'
, file => "$rawImFile"
, extension => 'PRIMARY'
, keyword => 'FILTER'
);
# Filter to single character translation
$filter = __PACKAGE__->getfiltershortname($filter);
info("FILTER = $filter");
info("Retrieving EXPOSURE value");
my $duration = readFITSKeyword(
class => 'intermediate'
, file => "$rawImFile"
, extension => 'PRIMARY'
, keyword => 'EXPOSURE'
);
info("EXPOSURE = $duration");
if ( ( $mode eq 'RUDI5_LR' )
&& !( ( $keywords->{WINDOWDX} == 976 )
&& ( $keywords->{WINDOWDY} == 960 )
)){
$duration *= -1;
}
# Will the output use the osw or filter character ?
$osw = ( $mode =~ /ENG[24]/ ) ? $filter : $osw;
# omcosflag
doCommand(
'omcosflag', set => $rawImFile
, thxset => $prepThFile
)
or return exception();
# omflatfield
my $flatOut = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OSW Flat Field'
);
my $flattedIm = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OSW Flat Fielded Image'
);
doCommand(
'omflatfield', set => $rawImFile
, thxset => $prepThFile
, inorbitflatset => $flatGenFile
, tsflatset => $flatOut
, outset => $flattedIm
)
or return exception();
# ommodmap
# IMAGE
my $finalImage = newFile(
class => $imgoutput->{CLASS}
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => $imgoutput->{CONTENT}
);
my $mod8Im = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OSW MODULO 8 IMAGE'
);
doCommand(
'ommodmap', set => $flattedIm
, mod8product => 'Y'
, mod8set => $mod8Im
, outset => $finalImage
, nsig => 6
, nbox => 16
)
or return exception();
# omqualitymap
#doCommand('omqualitymap', set => $finalImage
# , srclistset => ''
# , outset => $finalImage
# , mode => 'setqualityimage'
# )
# or return exception();
# omdetect
# SOURCE LIST
my $sourceListFile = newFile(
class => $srcoutput->{CLASS}
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => $srcoutput->{CONTENT}
);
my $RegionFile=newFile(class => 'product'
,instrument => thisInstrument
,exp_id => $exp_id
,osw_id => $osw
,content => 'OM OSW REGION FILE'
,format => 'ASCII'
);
doCommand(
'omdetect', set => $finalImage
, nsigma => 2.0
, minsignificance => 3
, detectextended => 'Y'
, regionfile => $RegionFile
, outset => $sourceListFile
)
or return exception();
# omqualitymap again to forward flags to source lists
#doCommand('omqualitymap', set => $finalImage
# , srclistset => $sourceListFile
# , outset => $finalImage
# , mode => 'setqualityimage'
# )
# or return exception();
push( @{$rtn->{srcdetails}}
, "$sourceListFile\|$mode\|$filter\n"
);
# Write a dummy value of OBJECT to trick omsrclistcomb later in the pipeline
# ommag
doCommand( 'ommag', set => $sourceListFile )
or return exception();
# Use omatt to make the sky image.
# SKY IMAGE
my $usnoList = findFile(
class => 'product'
, instrument => 'all'
, content => 'Reference catalogue'
, format => 'FITS'
);
info("No USNO extract to work from. Unable to continue.") and ignore() unless defined ($usnoList);
my $skyImage = newFile(
class => $skyimageoutput->{CLASS}
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => $skyimageoutput->{CONTENT}
);
doCommand(
'omatt', set => $finalImage
, sourcelistset => $sourceListFile
, ppsoswset => $skyImage
, usecat => 'Y'
, catfile => $usnoList
) or return exception();
push( @{$rtn->{skyimgdetails}}
, "$skyImage\|$mode\|$filter\|$duration\|$exp_id\n"
);
# Make sure the image isn't empty first
my $imgStat = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, format => "ASCII"
, content => "OSW $osw sky image statistics"
);
unless ( &ModuleUtil::isImageEmpty( $skyImage , $imgStat ) )
{
# CREATE PNG FROM SKY IMAGE
my $gifFile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => $skyimageoutput->{CONTENT}
, format => 'GIF'
);
my $pngFile = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => $skyimageoutput->{CONTENT}
, format => 'PNG'
);
doCommand(
'implot', set => $skyImage
, device => "$pngFile/PNG"
, withsrclisttab => 'no'
) or return exception();
# # Convert to PNG format
# GIFtoPNG(
# source => $gifFile
# , destination => $pngFile
# ) or return exception();
} # End if isImageEMpty
return $rtn;
}
sub process_grism_image
{
my ( %h ) = (@_);
my ($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords );
($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords )
= @h{qw( exp_id osw imFile rawImFile nphkFile phkFile windowFile prepThFile skyimageoutput srcoutput imgoutput mode keywords)}
;
my $rtn = { srcdetails => [] , skyimgdetails => []};
doCommand(
'omprep', set => $imFile
, nphset => $nphkFile
, pehset => $phkFile
, wdxset => $windowFile
, outset => $rawImFile
, modeset => 4
)
or return exception();
# Get the filter value now omprep has converted it from a code to a recognizable value
# Also get value from EXPOSURE keyword as duration.
info("Retrieving FILTER value");
my $filter = readFITSKeyword(
class => 'intermediate'
, file => $rawImFile
, extension => 'PRIMARY'
, keyword => 'FILTER'
);
# Filter to single character translation
$filter = __PACKAGE__->getfiltershortname($filter);
info("FILTER = $filter");
$filter = uc($filter);
info("FILTER = $filter");
info("Retrieving EXPOSURE value");
my $duration = readFITSKeyword(
class => 'intermediate'
, file => $rawImFile
, extension => 'PRIMARY'
, keyword => 'EXPOSURE'
);
info("EXPOSURE = $duration");
my $mod8Im = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OSW Modulo 8 Image'
);
my $mod8CorrectedIm = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OSW Mod-8-corrected Image'
);
# Following parameter values are those in omgchain-1.0.12, except --flatset and --outflatset are left at defaults.
doCommand(
'ommodmap'
, set => $rawImFile
, mod8product => 'yes'
, mod8set => $mod8Im
, outset => $mod8CorrectedIm
, nsig => 3
, nbox => 16
)
or return exception();
my $finalImage = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM-ALIGNED IMAGE'
);
doCommand(
'omgprep'
, set => $mod8CorrectedIm
, outset => $finalImage
)
or return exception();
# Make it easy to disable during testing / discussions
# Disabled until implot supports CTYPE?=PIXELS
if (0)
{
my $imgStat = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, format => "ASCII"
, content => "Aligned image statistics"
);
if ( !ModuleUtil::isImageEmpty( $finalImage , $imgStat ) )
{
ppd( "000023" );
# CREATE PNG FROM GRISM IMAGE
my $gifFile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM-ALIGNED IMAGE'
, format => 'GIF'
);
my $pngFile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM-ALIGNED IMAGE'
, format => 'PNG'
);
doCommand(
'implot', set => $finalImage
, device => "$pngFile/PNG"
, withsrclisttab => 'no'
) or warn('implot failed to create an OM GRISM-ALIGNED IMAGE PNG');
# Disable excepton during testing of potential product.
#) or return exception();
# # Convert to PNG format
# GIFtoPNG(
# source => $gifFile
# , destination => $pngFile
# ) or return exception();
}
else
{
ppd( "000024" );
}
}
# Read OMGWIN keyword.
my $omgwin = readFITSKeyword(
file => $finalImage
, extension => "PRIMARY"
, keyword => "OMGWIN"
);
if ($omgwin =~ /$RE{boolean}{true}/s)
{
ppd( "000021" );
my $omdetectRegionFile = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM DS9 REGIONS'
, format => "ASCII"
);
my $sourceListFile = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM OSW GRISM SOURCE LIST'
);
# Following parameter values are those in omgchain-1.0.12. Note that they are different to the non-grism omdetect parameters (as of OMImageAnalyse-3.01).
doCommand(
'omdetect'
, set => $finalImage
, nsigma => 2.0
, regionfile => $omdetectRegionFile
, outset => $sourceListFile
)
or return exception();
push( @{$rtn->{sourcedetails}}
, "$sourceListFile\|$mode\|$filter\n" );
my $omgrismRegionFile = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM DS9 SPECTRUM REGIONS'
, format => "ASCII"
);
my $listOfGrismSpectra = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM SPECTRA LIST'
);
my $omgrismSpectrum = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
, content => 'OM GRISM SOURCE SPECTRUM'
);
# omgchain-1.0.12 default parameter values.
doCommand(
'omgrism'
, set => $finalImage
, sourcelistset => $sourceListFile
, outset => $omgrismSpectrum
, bkgoffsetleft => 6.0
, bkgwidthleft => -6.0
, bkgoffsetright => 6.0
, bkgwidthright => -6.0
, spectrumhalfwidth => -6.0
, spectrumsmoothlength => 0
, extractionmode => 0
, extractfieldspectra => 'no'
, regionfile => $omdetectRegionFile
, spectraregionfile => $omgrismRegionFile
, outspectralistset => $listOfGrismSpectra
)
or return exception();
### How does omgrismplot know to make a plot just for the target? I don't think I want to know... :( ...but I better find out some time.
if(fileExists(
file => $omgrismSpectrum
) && hasFITSExtension(
file => $omgrismSpectrum
,extension => 'RATE001')
)
{
setExposureProperty( instrument => thisInstrument , exp_id => $exp_id , name => 'ssp' , value => 0+TRUE );
my $tempPlotFile = newFile(
class => 'intermediate'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
### , src_num => $source
, format => 'PS'
, content => "OM grism source spectrum"
);
# omgchain-1.0.12 default parameter values.
doCommand(
'omgrismplot'
, set => $omgrismSpectrum
, scalebkgplot => 'no'
, binsize => 1
, plotflux => 2
, plotfile => $tempPlotFile
)
or return exception();
my $grismSpectrumPlot = newFile(
class => 'product'
, instrument => thisInstrument
, exp_id => $exp_id
, osw_id => $osw
### , src_num => $source
, 'format' => 'PDF'
, content => "OM GRISM SOURCE SPECTRUM"
);
PStoPDF(
source => $tempPlotFile
, destination => $grismSpectrumPlot
)
or return exception();
}
}
else
{
ppd( "000022" );
}
return $rtn;
}
###
### OM OSW IMAGE IDENTIFICATION
###
### RUDI-5 WINDOWS:
### Exp. WINDOWDX WINDOWDY BIN
### --- --------- --------- ---
### 1 224 224 1x1
### 1 976 960 2x2
### 2 224 224 1x1
### 2 480 1792 2x2
### 3 224 224 1x1
### 3 1792 480 2x2
### 4 224 224 1x1
### 4 480 1792 2x2
### 5 224 224 1x1
### 5 1792 480 2x2
###
### ENGINEERING 2 FULL FRAME IMAGE
### Exp. WINDOWDX WINDOWDY BIN
### - 2048 2048 1X1
###
### ENGINEERING 2 PARTIAL FRAME IMAGE
### With Correct Headers
### WINDOWX WINDOWY NAXIS1 NAXIS2 WINDOWDX WINDOWDY BINBPE
### n x 512 0 256 1024 512 2048 F
###
### ENGINEERING 2 PARTIAL FRAME IMAGE
### With Incorrect Headers
### WINDOWX WINDOWY NAXIS1 NAXIS2 WINDOWDX WINDOWDY BINBPE
### n x 256 0 256 1024 256 1024 T
###
### ENGINEERING 4
### Always E4I.FIT
###
### USER DEFINED
### Not one of the above
###
### NB. Some of the above values are obtained from the imFile
### and others from the omprep generated raw image. Which we get
### from where depends on the image format.
###
use vars qw(%Windowdefinition $Tolerance $Window1 $Window2 $Window1min $Window1max $Window2min $Window2max );
%Windowdefinition = (
#These keys are {WINDOWDXxWINDOWDY} and {BINAX1xBINAX2}
# FUNNY is used for images which could be ENG2 partial frame with bad headers or USER
# mode. Further tests are conducted below.
'976x960' => { '2x2' => 'RUDI5_LR' }
, '480x1792' => { '2x2' => 'RUDI5_LR' }
, '1792x480' => { '2x2' => 'RUDI5_LR' }
, '224x224' => { '1x1' => 'RUDI5_HR' }
, '2048x2048' => { '1x1' => 'FFENG2' } # Eng-2 Full frame image
, '512x2048' => { '1x1' => 'PFENG2' } # Eng-2 Partial frame image
, '256x1024' => { '1x1' => 'FUNNY' } # Needs more analysis to determine.
);
$Tolerance = 50;
($Window1 , $Window2) = (1792 , 480);
($Window1min , $Window1max) = ( $Window1 - $Tolerance , $Window1 + $Tolerance);
($Window2min , $Window2max) = ( $Window2 - $Tolerance , $Window2 + $Tolerance);
sub image_format
{
my ( $image, $kw ) = @_;
info("Analysing $image for image identification");
my $windowsize = $kw->{WINDOWDX}.'x'.$kw->{WINDOWDY};
my $naxissize = $kw->{NAXIS1}.'x'.$kw->{NAXIS2};
my $windowbin = $kw->{BINAX1}.'x'.$kw->{BINAX2};
my $mode = (exists $Windowdefinition{$windowsize}{$windowbin}) ? $Windowdefinition{$windowsize}{$windowbin} : 'USER';
# Need some tolerance due to clipping of RUDI5_LR windows
# For RUDI-5 WINDOWDX/Y +/- $tolerance of above values and EXP_ID of 4nn
if ( ( $mode eq 'USER' ) && ( $kw->{EXP_ID} =~ /4\d\d$/ ) )
{
my $tolerance = 50;
my ( $val1, $val2 ) = ( 1792, 480 );
# second dimension is the lesser of the two.
my ( $dim2, $dim1 ) = sort {$a <=> $b} ($kw->{WINDOWDX} , $kw->{WINDOWDY});
$mode = 'RUDI5_LR'
if ( ( ($dim1 == $Window1) && ($dim2 == $Window2) )
|| ( ( $dim1 > $Window1min ) && ( $dim1 < $Window1max )
&& ( $dim2 > $Window2min ) && ( $dim2 < $Window2max ) )
);
}
# Further refinement of mode (ie. handle bad headers etc)
if ( $mode eq 'FUNNY' )
{
info("Peculiar case: Could be ENG2 or USER mode. Testing...");
# This if accounts for wrongly headered ENG2 partial frame data in the odf
if ( ( $windowsize eq $naxissize ) && ( $kw->{BINBPE} eq 'F' ) )
{
info( "WINDOWDX/Y = NAXIS1/2 and BINBPE = F indicates an PFENG2 component image");
$mode = 'PFENG2';
}
else
{
$mode = 'USER';
}
}
$kw->{MODE} = $mode;
}
sub get_keywords
{
my ($file,$extn) = (shift,shift);
my %keywords = (
EXP_ID => ''
, WINDOWDX => 0
, WINDOWDY => 0
, BINAX1 => 0
, BINAX2 => 0
, BINBPE => 0
, FILTER => 0
, NAXIS1 => 0
, NAXIS2 => 0
);
# Extract keywords with which to identify image
foreach my $k ( keys(%keywords) )
{
$keywords{$k} = readFITSKeyword(
class => 'odf'
, file => $file
, extension => $extn
, keyword => "$k"
);
}
return %keywords;
}
1;