package OMSourceCombine;
use strict;
use English;
use Carp;
use vars
qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
#### DEBUG
use Data::Dumper;
#### DEBUG
# Declare identity, version, author, date, etc.
$name = __PACKAGE__;
$VERSION = '3.9';
$version = $VERSION;
$author = 'Richard West,Duncan John Fyfe,Ian Stewart,Duncan Law-Green,Jose Vicente Perea, Pedro Rodriguez';
$date = '2019-07-01';
#
# ChangeLog
# =========
#
# Version 3.9 - 2019-07-01 (JVP)
# -----------
#
# + Check if files in ascii file $listmosaicsourcelists exist before continuing
#
# Version 3.8 - 2019-04-24 (JVP)
# -----------
#
# + OM Final Combined Source List and Combined Region only if OM Mosaic source list is not empty
#
# Version 3.7 - 2016-03-29 (JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
# Final Combined Source List ('ommergelists') and Combined Region as final products:
# 'OM OBSERVATION SOURCE LIST' now includes source detection from stacked images (mosaic)
# 'OM OBSERVATION SOURCE REGION' new ds9 region product containing all detected sources
#
# Version 3.6 - 2016-03-23 (PR, JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
# Final Combined Source List ('ommergelists') and Combined Region already in place (as intermediate products).
#
# Version 3.5 - 2016-03-22 (PR, JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
# Final Combined Source List ('ommergelists') pending.
#
# Version 3.4 - 2012-03-22 (DLG)
# -----------
#
# + change omsrclistcomb input from intermediate reference catalogue to product
#
# Version 3.3 - 2008-09-29 (DLG)
# -----------
#
# Add support for {Partial|Full} Frame Engineering Mode 2: PFENG2, FFENG2
#
# Version 3.2 - 2008-09-19 (DLG)
# -----------
#
# Modifications to track/fix missing OM image products in PCMS-8.1
#
# Version 3.1 - 2006-01-18 (DJF)
# ------------
#
# + Was looking for filter value /grism/i rather than /G|H/i to indicate grism data.
#
# Version 3.0 - 2005-11-18 (IMS)
# ------------
#
# + Add explicit perl module headers. Previously these were supplied
# at compile time. This will make debugging and extending the modules
# through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic. Now $Version = version = ''
#
# + No longer generates the USNO extract. This job is taken over by module USNOExtract. Added 'start' dependency on this module.
#
# Version 2.12 - 2005-09-28 (RGW)
# ------------
#
# + exclude grism exposure source lists from merging operation
#
# Version 2.11 - 2005-09-26 (RGW)
# ------------
#
# + added fall-back on to an OSW sky image if no mosaic is available
# (image is currently chosen randomly, should maybe be the "largest"?)
#
# Version 2.10 - 2005-09-15 (RGW)
# ------------
#
# + removed dependency on SrcMerge, this module now creates its own
# USNO digest
#
# Version 2.09 - 2005-09-14 (RGW)
# ------------
#
# + updated trigger conditions to depend on SrcMerge
# + omsrclistcomb now uses the USNO digest as input
#
# Version 2.08 - 2004-03-27 (DFJ)
# ------------
#
# + omsrclistcomb was being called with a 'joined list' rather than a list reference
#
# Version 2.07 - 2003-12-10 (DFJ)
# ------------
#
# + Adapted doCommand to use anonymous lists for list parameters.
#
# Version 2.06 - 2001-11-29 (DFJ)
# ------------
#
# + fixed evaluate rules was refering to intrument.
# + Removed dependance on OMFastAnalyse
#
# Version 2.05 - 2001-11-29 (DFJ)
# ------------
#
# + Corrected grouping of images. The end conditions meant the
# last image got missed.
#
# Version 2.04 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.03 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.02 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.01 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.00 - 2001-11-28 (DJF)
# ------------
#
# + Major rewrite to accomodate new OM product names
#
# Version 1.10 - 2001-10-30 (DJF)
# ------------
#
# + Further amended evaluateRules to account for OMFastAnalyse and OMImageAnalyse running in parallel.
#
# Version 1.09 - 2001-10-25 (DJF)
# ------------
#
# + Amened evaluateRules to account for new OMFastAnalyse module
#
# Version 1.08 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
# tracking purposes.
#
# Version 1.07 - 2001-03-08 (DH)
# ------------
#
# + Make combined source list a product file rather than
# an intermediate file.
#
# Version 1.06 - 2001-01-10 (DH)
# ------------
#
# + Add OMgetFlat to start rule.
#
# Version 1.05 - 2001-01-10 (DH)
# ------------
#
# + Add ignore rule.
#
# Version 1.04 - 2000-11-30 (DH)
# ------------
#
# + First production version.
#
# Declare list of instruments this module is interested in
@instList=qw(om);
# Number of streams
sub numberOfStreams{ return 1; }
sub evaluateRules {
# Start conditions
ignore() if ignored(module => 'OMgetFlat'
,stream => 1);
ignore() if allIgnored(module=>'OMImageAnalyse'
,instrument => thisInstrument
);
ignore() if ignored(module => 'OMMosaic');
##### Now that we don't have to make a usno extract, do we need OMImageAnalyse and OMMosaic?
start()
if allComplete(
module => 'OMImageAnalyse'
, instrument => thisInstrument
)
and complete(
module => 'OMMosaic'
)
and complete(
module => 'USNOExtract'
, instrument => 'all'
, stream => 1
);
}
sub performAction {
info("Module version number: $version");
# OM modes are: USER , RUDI5_LR , RUDI5_HR , ENG2 , PFENG2, FFENG2, ENG4 and FAST
# Modified to include PFENG2, FFENG2 - DLG
# We do not Combine FAST images
# Modes written by OMImageAnalyse
my ($USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4) = ('USER','RUDI5_LR','ENG2','PFENG2','FFENG2','ENG4');
my @CombineModesMOS = ('USER','RUDI5_LR','ENG2','PFENG2','FFENG2','ENG4');
# Output Modes used by OMSourceCombine (we don't generate High Resoultion RUDI5 mosaics
# assume PFENG2, FFENG2 use same combine modes as ENG2
my $CombineModes = [$USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4];
info("Looking For Source Lists to Combine...");
#Determine what Sky Images were written
my @SrcLstDetailsFiles = findFile(class =>'intermediate'
,instrument => thisInstrument
,format => "ASCII"
,content => "OM SRC IMAGE DETAILS"
);
unless (scalar(@SrcLstDetailsFiles)) {
info("No Source Lists Images to Combine.");
return success();
}
my @SrcLstDetails = ();
foreach my $file (@SrcLstDetailsFiles) {
#### DEBUG
info("DEBUG: file: $file");
#### DEBUG
my @details = readASCIIFile(name=>$file);
next unless (scalar(@details));
# Fetch exposure ID
my $info=fileInfo(class => 'intermediate',
name => $file);
my $expid=$$info{'exp_id'};
my $filter=getExposureProperty(instrument => 'om',
exp_id => $expid,
name => 'filter');
#my $mode=getExposureProperty(instrument => 'om',
# exp_id => $expid,
# name => 'mode'
#);
if ($filter =~ /grism/i)
{
info("Ignoring grism source list for exposure=$expid");
next;
}
foreach my $line (@details)
{
# Shouldn't really need this second test but as with all things OM belt and braces and superglue oft saves the day.
$line =~ /([GH])$/;
if ($1)
{
info("Ignoring grism source list for exposure=$expid");
next;
}
my @tmp = split("\\|",$line);
#### DEBUG
info("DEBUG: line $line");
my $debugtmp = Dumper(\@tmp);
info("DEBUG: split to tmp: $debugtmp");
#### DEBUG
push (@SrcLstDetails,\@tmp)
if (scalar(@tmp));
}
}
#### DEBUG
my $debugsrclstdetails = Dumper(\@SrcLstDetails);
info("DEBUG: SrcLstDetails: $debugsrclstdetails");
#### DEBUG
my $count = scalar(@SrcLstDetails);
unless ($count) {
info("No source lists to combine.");
return success();
}
info("Found $count Lists");
info ("Grouping Lists by Mode then Filter");
my $usnoList = findFile(
class => 'product'
, instrument => 'all'
, content => 'Reference catalogue'
, format => 'FITS'
, required => 'true'
);
# Sort details on: OM Mode, Filter
# 0 = Filename , 1 = Src Mode , 2 = Filter
my ($filename_p,$mode_p,$filter_p) = (0,1,2);
my @SortedSrcLstDetails = sort { ${$a}[$mode_p] cmp ${$b}[$mode_p]
|| ${$a}[$filter_p] cmp ${$b}[$filter_p]
|| ${$a}[$filename_p] cmp ${$b}[$filename_p]
} @SrcLstDetails;
#### DEBUG
my $debugsortedsrclstdetails = Dumper(\@SortedSrcLstDetails);
info("DEBUG: SortedSrcLstDetails: $debugsortedsrclstdetails");
#### DEBUG
# separate into image groups on mode then filter
my @SrcGroups = ();
my $Src = shift (@SortedSrcLstDetails);
while ( defined($Src) ) {
my $oldmode = $$Src[$mode_p];
info("Mode: $oldmode");
my @mode_tmp=();
#### DEBUG
info("DEBUG: Src: $Src");
info("DEBUG: mode_p: $mode_p");
# my @tempsrc = $$Src;
# my $debugtempsrc = Dumper(\@tempsrc);
info("DEBUG: SSrc[modep]: $$Src[$mode_p]");
info("DEBUG: SSrc[filterp]: $$Src[$filter_p]");
#### DEBUG
while ( (defined($Src)) && ($$Src[$mode_p] eq $oldmode) ) {
my $oldfilter = $$Src[$filter_p];
info("Filter: $oldfilter");
my @filter_tmp = ();
while ( (defined($Src)) && ($$Src[$filter_p] eq $oldfilter) && ($$Src[$mode_p] eq $oldmode) ) {
info("Image File : $$Src[$filename_p]");
push(@filter_tmp,$$Src[$filename_p]);
$Src = shift (@SortedSrcLstDetails);
}
push (@mode_tmp,[$oldfilter,\@filter_tmp]);
}
push(@SrcGroups,[$oldmode,\@mode_tmp]);
}
&per_mode($CombineModes, $usnoList, \@SrcGroups);
###
#### Now try Mosaic detections
###
#
my $listmosaicsourcelists = findFile(
class=> 'intermediate'
, instrument => thisInstrument
, format => "ASCII"
, content => "MOSAIC SOURCE LISTS"
);
unless ($listmosaicsourcelists){
info("No MOSAIC lists to combine");
return success();
};
info ("reading file $listmosaicsourcelists");
my @mosaiclists = readASCIIFile(name=>$listmosaicsourcelists);
#unless (scalar(@mosaiclists)){ ...}
# Check if the files in ascii file $listmosaicsourcelists exist
my @mosaiclistsExists = ();
foreach my $i (@mosaiclists){
push (@mosaiclistsExists, $i) unless ( ! fileExists( file => $i ) );
}
unless (scalar(@mosaiclistsExists)){
info("No MOSAIC lists to combine. None of the files in listmosaicsourcelists exist.");
return success();
};
my $content = "OM OBSERVATION SOURCE LIST MOSAIC";
&combine($content, $usnoList ,\@mosaiclists);
my $SrcListMosaic = findFile(
class=> "product"
, instrument => thisInstrument
, content => "OM OBSERVATION SOURCE LIST MOSAIC"
);
my $SrcListWindow = findFile(
class=> "product"
, instrument => thisInstrument
, content => "OM OBSERVATION SOURCE LIST"
);
# Final Combined Source List and Combined Region
my $newSrcList = newFile(
class => 'intermediate'
,instrument => thisInstrument
,content => "final merged list"
);
if ( numberFITSRows(file=>$SrcListMosaic, extension=>'SRCLIST') > 0 ) {
doCommand('ommergelists'
, srclist1 => $SrcListWindow
, srclist2 => $SrcListMosaic
, outset => $newSrcList
, tolerance => 2
, maxrawrate => 5
) or return exception();
# Overwrite $SrcListWindow 'OM OBSERVATION SOURCE LIST' with the Combined Source List
info ("DEBUG Copy $newSrcList to $SrcListWindow");
copyFile(source => "$newSrcList", destination => $SrcListWindow); # Overwrite original
my $newSrcRegion = newFile(
class => 'product'
,instrument => thisInstrument
,content => "OM OBSERVATION SOURCE REGION"
,format => "ASCII"
);
# Create $newSrcRegion 'OM OBSERVATION SOURCE REGION' : Combined Region
doCommand('omregion'
, set => $newSrcList
, srcnumber => 1
, srcradius => 3
, nfwhm => 3
, bkginner => 1.2
, bkgouter => 1.8
, srcfile => $newSrcRegion
) or return exception();
} else {
info("OM Mosaic source list $SrcListMosaic is empty. No final Combined Source List and Combined Region");
}
return success();
}
sub per_mode {
my ($CombineModes, $usnoList, $Groups) = @_;
my ($USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4) = @$CombineModes;
my $content = "OM OBSERVATION SOURCE LIST";
info("Creating Combined Lists...");
###
### Current Rules ###
### Combine all OSW source lists except RUDI5_HR and OMFastAnalyse
### This code knows nothing of OMFastAnalyse source lists so we don't need to worry about it.
### We need to discard all RUDI5_HR mode lines and combine the others into a single list
###
my @tobecombined = ();
foreach my $group (@$Groups) {
my ($mode,$filters) = @$group;
info("MODE: $mode");
if ($mode eq $USER) {
push(@tobecombined,@{&per_filter($filters)});
}
elsif ($mode eq $RUDI5_LR) {
push(@tobecombined,@{&per_filter($filters)});
}
elsif ($mode eq $ENG2) {
push(@tobecombined,@{&per_filter($filters)});
}
elsif ($mode eq $PFENG2) {
push(@tobecombined,@{&per_filter($filters)});
}
elsif ($mode eq $FFENG2) {
push(@tobecombined,@{&per_filter($filters)});
}
elsif ($mode eq $ENG4) {
push(@tobecombined,@{&per_filter($filters)});
}
else {
info("There are no rules for generating a combined source list for this mode ($mode).");
info("No product will be generated.");
}
info("Done: $mode");
}
&combine($content, $usnoList, \@tobecombined);
}
sub per_filter {
my ($Filters) = @_;
my @list = ();
foreach my $f (@$Filters) {
my ($filter,$lists) = @$f;
info("FILTER: $filter");
push(@list,@$lists);
}
return \@list;
}
sub combine {
my ($content, $usnoList, $SrcLists) = @_;
my $newSrcList=newFile(class=>'product'
,instrument => thisInstrument
,content => "$content"
);
info("CREATING: $newSrcList");
doCommand('omsrclistcomb'
,sourcelistsets => $SrcLists
,outset => $newSrcList
, usecat => 'Y'
, catfile => $usnoList
) or return exception();
}
1;