package OMSourceCombine;
use strict;
use English;
use Carp;
use vars
  qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
#### DEBUG
use Data::Dumper;
#### DEBUG

# Declare identity, version, author, date, etc.
$name    = __PACKAGE__;
$VERSION = '3.9';
$version = $VERSION;
$author  = 'Richard West,Duncan John Fyfe,Ian Stewart,Duncan Law-Green,Jose Vicente Perea, Pedro Rodriguez';
$date    = '2019-07-01';

#
# ChangeLog
# =========
#
# Version 3.9 - 2019-07-01 (JVP)
# -----------
#
# + Check if files in ascii file $listmosaicsourcelists exist before continuing
#
# Version 3.8 - 2019-04-24 (JVP)
# -----------
#
# + OM Final Combined Source List and Combined Region only if OM Mosaic source list is not empty
#
# Version 3.7 - 2016-03-29 (JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
#   Final Combined Source List ('ommergelists') and Combined Region as final products:
#	'OM OBSERVATION SOURCE LIST'	now includes source detection from stacked images (mosaic)
#	'OM OBSERVATION SOURCE REGION'	new ds9 region product containing all detected sources 
#
# Version 3.6 - 2016-03-23 (PR, JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
#   Final Combined Source List ('ommergelists') and Combined Region already in place (as intermediate products).
#
# Version 3.5 - 2016-03-22 (PR, JVP)
# -----------
#
# + Mosaic detections (detections from stacked images)
#   Final Combined Source List ('ommergelists') pending.
#
# Version 3.4 - 2012-03-22 (DLG)
# -----------
#
# + change omsrclistcomb input from intermediate reference catalogue to product
#
# Version 3.3 - 2008-09-29 (DLG)
# -----------
# 
# Add support for {Partial|Full} Frame Engineering Mode 2: PFENG2, FFENG2
#
# Version 3.2 - 2008-09-19 (DLG)
# -----------
# 
# Modifications to track/fix missing OM image products in PCMS-8.1
#
# Version 3.1 - 2006-01-18 (DJF)
# ------------
#
# + Was looking for filter value /grism/i rather than /G|H/i to indicate grism data.  
#
# Version 3.0 - 2005-11-18 (IMS)
# ------------
#
# + Add explicit perl module headers.  Previously these were supplied
#   at compile time.  This will make debugging and extending the modules
#   through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic.  Now $Version = version = ''
#
# + No longer generates the USNO extract. This job is taken over by module USNOExtract. Added 'start' dependency on this module.
#
# Version 2.12 - 2005-09-28 (RGW)
# ------------
#
# + exclude grism exposure source lists from merging operation
#
# Version 2.11 - 2005-09-26 (RGW)
# ------------
#
# + added fall-back on to an OSW sky image if no mosaic is available
#   (image is currently chosen randomly, should maybe be the "largest"?)
#
# Version 2.10 - 2005-09-15 (RGW)
# ------------
#
# + removed dependency on SrcMerge, this module now creates its own
#   USNO digest
#
# Version 2.09 - 2005-09-14 (RGW)
# ------------
#
# + updated trigger conditions to depend on SrcMerge
# + omsrclistcomb now uses the USNO digest as input
#
# Version 2.08 - 2004-03-27 (DFJ)
# ------------
#
# + omsrclistcomb was being called with a 'joined list' rather than a list reference
#
# Version 2.07 - 2003-12-10 (DFJ)
# ------------
# 
# + Adapted doCommand to use anonymous lists for list parameters.
#
# Version 2.06 - 2001-11-29 (DFJ)
# ------------
# 
# + fixed evaluate rules was refering to intrument.
# + Removed dependance on OMFastAnalyse
# 
# Version 2.05 - 2001-11-29 (DFJ)
# ------------
# 
# + Corrected grouping of images. The end conditions meant the 
#   last image got missed.
# 
# Version 2.04 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.03 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.02 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.01 - 2001-11-28 (DJF)
# ------------
#
# + Fix call to omsrclistcomb
#
# Version 2.00 - 2001-11-28 (DJF)
# ------------
#
# + Major rewrite to accomodate new OM product names
#
# Version 1.10 - 2001-10-30 (DJF)
# ------------
#
# + Further amended evaluateRules to account for OMFastAnalyse and OMImageAnalyse running in parallel.
#
# Version 1.09 - 2001-10-25 (DJF)
# ------------
#
# + Amened evaluateRules to account for new OMFastAnalyse module
#
# Version 1.08 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
#   tracking purposes.
#
# Version 1.07 - 2001-03-08 (DH)
# ------------
#
# + Make combined source list a product file rather than
#   an intermediate file.
#
# Version 1.06 - 2001-01-10 (DH)
# ------------
#
# + Add OMgetFlat to start rule.
#
# Version 1.05 - 2001-01-10 (DH)
# ------------
#
# + Add ignore rule.
#
# Version 1.04 - 2000-11-30 (DH)
# ------------
#
# + First production version.
#

# Declare list of instruments this module is interested in
@instList=qw(om);

# Number of streams
sub numberOfStreams{ return 1; }

sub evaluateRules {

	# Start conditions
	ignore() if ignored(module => 'OMgetFlat'
		,stream => 1);

	ignore() if allIgnored(module=>'OMImageAnalyse'
		,instrument => thisInstrument
	);

	ignore() if ignored(module => 'OMMosaic');

##### Now that we don't have to make a usno extract, do we need OMImageAnalyse and OMMosaic?
    start()
      if allComplete(
        module => 'OMImageAnalyse'
        , instrument => thisInstrument
      )
      and complete(
        module => 'OMMosaic'
      )
      and complete(
        module => 'USNOExtract'
        , instrument => 'all'
        , stream => 1
     );

}

sub performAction {
	info("Module version number: $version");

	# OM modes are: USER , RUDI5_LR , RUDI5_HR , ENG2 , PFENG2, FFENG2, ENG4 and FAST
	# Modified to include PFENG2, FFENG2 - DLG

	# We do not Combine FAST images

	# Modes written by OMImageAnalyse
	my ($USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4) = ('USER','RUDI5_LR','ENG2','PFENG2','FFENG2','ENG4');
	my @CombineModesMOS = ('USER','RUDI5_LR','ENG2','PFENG2','FFENG2','ENG4');

	# Output Modes used by OMSourceCombine (we don't generate High Resoultion RUDI5 mosaics
	# assume PFENG2, FFENG2 use same combine modes as ENG2
	my $CombineModes = [$USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4];


	info("Looking For Source Lists to Combine...");
	#Determine what Sky Images were written
	my @SrcLstDetailsFiles = findFile(class =>'intermediate'
		,instrument => thisInstrument
		,format => "ASCII"
		,content => "OM SRC IMAGE DETAILS"
	);
	
	unless (scalar(@SrcLstDetailsFiles)) {
		info("No Source Lists Images to Combine.");	
		return success();
	}
	
	my @SrcLstDetails = ();
	foreach my $file (@SrcLstDetailsFiles) {
#### DEBUG
	    info("DEBUG: file: $file");
#### DEBUG
		my @details = readASCIIFile(name=>$file);
		next unless (scalar(@details));

		# Fetch exposure ID
		my $info=fileInfo(class => 'intermediate',
				  name => $file);
		my $expid=$$info{'exp_id'};
		my $filter=getExposureProperty(instrument => 'om',
					       exp_id => $expid,
					       name => 'filter');
		#my $mode=getExposureProperty(instrument => 'om',
		#				 exp_id => $expid,
		#				 name => 'mode'
		#);

		if ($filter =~ /grism/i)
		{
			info("Ignoring grism source list for exposure=$expid");
			next;
		}
		
		foreach my $line (@details) 
		{
			# Shouldn't really need this second test but as with all things OM belt and braces and superglue oft saves the day.
			$line =~ /([GH])$/;
			if ($1)
			{
				info("Ignoring grism source list for exposure=$expid");
				next;
			}
			
			my @tmp = split("\\|",$line);
#### DEBUG
			info("DEBUG: line $line");
			my $debugtmp = Dumper(\@tmp);
			info("DEBUG: split to tmp: $debugtmp");

#### DEBUG		
			push (@SrcLstDetails,\@tmp)
			    if (scalar(@tmp));
		}
	}

#### DEBUG
	my $debugsrclstdetails = Dumper(\@SrcLstDetails);
	info("DEBUG: SrcLstDetails: $debugsrclstdetails");
#### DEBUG

	my $count = scalar(@SrcLstDetails);
	unless ($count) {
		info("No source lists to combine.");	
		return success();
	}
	info("Found $count Lists");
	info ("Grouping Lists by Mode then Filter");

    my $usnoList = findFile(
			    class => 'product'
			    , instrument => 'all'
			    , content => 'Reference catalogue'
			    , format => 'FITS'
			    , required => 'true'
			    );

	# Sort details on: OM Mode, Filter	
	# 0 = Filename , 1 = Src Mode , 2 = Filter 

	my ($filename_p,$mode_p,$filter_p) = (0,1,2);
	my @SortedSrcLstDetails = sort { ${$a}[$mode_p] cmp ${$b}[$mode_p] 
		|| ${$a}[$filter_p] cmp ${$b}[$filter_p] 
		|| ${$a}[$filename_p] cmp ${$b}[$filename_p] 
	} @SrcLstDetails;

#### DEBUG
	my $debugsortedsrclstdetails = Dumper(\@SortedSrcLstDetails);
	info("DEBUG: SortedSrcLstDetails: $debugsortedsrclstdetails");
#### DEBUG

	# separate into image groups on mode then filter

	my @SrcGroups = ();
	my $Src = shift (@SortedSrcLstDetails);
	while ( defined($Src) ) {
		my $oldmode = $$Src[$mode_p];
		info("Mode: $oldmode");
		my @mode_tmp=();

#### DEBUG
		info("DEBUG: Src: $Src");
		info("DEBUG: mode_p: $mode_p");
#		my @tempsrc = $$Src; 
#		my $debugtempsrc = Dumper(\@tempsrc);
		info("DEBUG: SSrc[modep]: $$Src[$mode_p]");
		info("DEBUG: SSrc[filterp]: $$Src[$filter_p]");
#### DEBUG
		
		while ( (defined($Src)) && ($$Src[$mode_p] eq $oldmode) ) {

			my $oldfilter = $$Src[$filter_p];
			info("Filter: $oldfilter");
			my @filter_tmp = ();

			while ( (defined($Src)) && ($$Src[$filter_p] eq $oldfilter) && ($$Src[$mode_p] eq $oldmode) ) {
			
				info("Image File : $$Src[$filename_p]");
				push(@filter_tmp,$$Src[$filename_p]);
				$Src = shift (@SortedSrcLstDetails);

			}
			push (@mode_tmp,[$oldfilter,\@filter_tmp]);
		}
		push(@SrcGroups,[$oldmode,\@mode_tmp]);
	}

	&per_mode($CombineModes, $usnoList, \@SrcGroups);


###
####  Now try Mosaic detections
###
#

        my $listmosaicsourcelists = findFile(
         class=> 'intermediate'
	 , instrument => thisInstrument
	 , format => "ASCII"
         , content => "MOSAIC SOURCE LISTS"
	);
	unless ($listmosaicsourcelists){
	    info("No MOSAIC lists to combine");
	    return success();
	};
        info ("reading file $listmosaicsourcelists");
	my @mosaiclists = readASCIIFile(name=>$listmosaicsourcelists);
	#unless (scalar(@mosaiclists)){ ...}

	# Check if the files in ascii file $listmosaicsourcelists exist
	my @mosaiclistsExists = ();
	foreach my $i (@mosaiclists){
	 push (@mosaiclistsExists, $i) unless ( ! fileExists( file => $i ) );
	}
        unless (scalar(@mosaiclistsExists)){
	    info("No MOSAIC lists to combine. None of the files in listmosaicsourcelists exist.");
            return success();
        };

	my $content = "OM OBSERVATION SOURCE LIST MOSAIC";


        &combine($content, $usnoList ,\@mosaiclists);

	my $SrcListMosaic = findFile(
	     class=> "product"
	     , instrument => thisInstrument
             , content => "OM OBSERVATION SOURCE LIST MOSAIC"
        );
	my $SrcListWindow = findFile(
	     class=> "product"
	     , instrument => thisInstrument
             , content => "OM OBSERVATION SOURCE LIST"
        );


        # Final Combined Source List and Combined Region
        my $newSrcList = newFile(
         class => 'intermediate'
        ,instrument => thisInstrument
        ,content => "final merged list"
       );

       if ( numberFITSRows(file=>$SrcListMosaic, extension=>'SRCLIST') > 0 ) {
	  doCommand('ommergelists'
		  , srclist1 => $SrcListWindow
		  , srclist2 => $SrcListMosaic
		  , outset => $newSrcList
		  , tolerance => 2
		   , maxrawrate => 5
	  ) or return exception();


	   # Overwrite $SrcListWindow 'OM OBSERVATION SOURCE LIST' with the Combined Source List
	   info ("DEBUG Copy $newSrcList to $SrcListWindow");
	   copyFile(source => "$newSrcList", destination => $SrcListWindow);	# Overwrite original


	   my $newSrcRegion = newFile(
	    class => 'product'
	   ,instrument => thisInstrument
	   ,content => "OM OBSERVATION SOURCE REGION"
	   ,format => "ASCII"
           );


           # Create $newSrcRegion 'OM OBSERVATION SOURCE REGION' : Combined Region
	   doCommand('omregion'
		  , set =>  $newSrcList
		  , srcnumber  => 1
		  , srcradius => 3
		  , nfwhm => 3
		   , bkginner => 1.2
		   , bkgouter => 1.8
		   , srcfile => $newSrcRegion
	  ) or return exception();
       } else {
          info("OM Mosaic source list $SrcListMosaic is empty. No final Combined Source List and Combined Region");
       }

	return success();
}

sub per_mode {
	my ($CombineModes, $usnoList, $Groups) = @_;
	my ($USER,$RUDI5_LR,$ENG2,$PFENG2,$FFENG2,$ENG4) = @$CombineModes;
	my $content = "OM OBSERVATION SOURCE LIST";

	info("Creating Combined Lists...");	

	###
	### Current Rules	###
	### Combine all OSW source lists except RUDI5_HR and OMFastAnalyse
	###	This code knows nothing of OMFastAnalyse source lists so we don't need to worry about it.
	### We need to discard all RUDI5_HR mode lines and combine the others into a single list
	###

	my @tobecombined = ();
	foreach my $group (@$Groups) {
		my ($mode,$filters) = @$group;
		info("MODE: $mode");
		if ($mode eq $USER) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		elsif ($mode eq $RUDI5_LR) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		elsif ($mode eq $ENG2) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		elsif ($mode eq $PFENG2) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		elsif ($mode eq $FFENG2) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		elsif ($mode eq $ENG4) {
			push(@tobecombined,@{&per_filter($filters)});
		}
		else {
			info("There are no rules for generating a combined source list for this mode ($mode).");
			info("No product will be generated.");
		}
		info("Done: $mode");
	
	}

	&combine($content, $usnoList, \@tobecombined);
}

sub per_filter {
	my ($Filters) = @_;

	my @list = ();
	foreach my $f (@$Filters) {
		my ($filter,$lists) = @$f;
		info("FILTER: $filter");
		push(@list,@$lists);	
	}
	return \@list;
}

sub combine {
	my ($content, $usnoList, $SrcLists) = @_;
	
	my $newSrcList=newFile(class=>'product'
		,instrument => thisInstrument
		,content => "$content"
	);

	info("CREATING: $newSrcList");

	doCommand('omsrclistcomb'
		,sourcelistsets => $SrcLists
		,outset => $newSrcList
		, usecat => 'Y'
		, catfile => $usnoList
	) or return exception();

}
1;