package OMImageAnalyse;
use strict;
use English;
use Carp;
use Regexp::Common qw(boolean);
use vars
  qw(@ISA $VERSION $name $author $date $version @instList $numberOfStreams);
@ISA = qw(Module);
use ModuleResources;
use Astro::FITS::CFITSIO qw(TRUE FALSE);
use ModuleUtil;
#### DEBUG
use Data::Dumper;
#### DEBUG

# Declare identity, version, author, date, etc.
$name    = __PACKAGE__;
$VERSION = '3.24';
$version = $VERSION;
$author  = 'Duncan Law-Green,Dean Hinshaw,Duncan John Fyfe,Ian Stewart,Eduardo Ojero,Ed Chapin,Pedro Rodriguez,Jose Vicente Perea';
$date    = '2016-02-18';


#
# ChangeLog
# =========
#
# Version 3.24 - 2016-02-18 (PR, JVP)
# ------------
#
# + All window exposures except central high-res from RUDI-5 are considered as USER mode.
#   Resume 'omqualitymap' calls after correction for 64bits (PPR#7282). Also needed for stacked images
#
# Version 3.23 - 2014-04-10 (JVP)
# ------------
#
# + The values in the QUALITY maps from omqualitymap are wrong when created in 64bits machines.
#   Therefore, the calls to omqualitymap introduced after PPP SCR 7143 have been dropped.
#
# Version 3.22 - 2013-10-07 (EC)
# ------------
#
# + Add omqualitymap calls (Mantis 0007143)
#
# Version 3.21 - 2013-06-11 (EC)
# ------------
#
# + Merge changes from SOC version.
#
# version 3.20 - 2012-05-03 DLG
# ------------
#
# + Eliminate use of GIF bitmaps
#
# version 3.19 - 2012-03-22 DLG
# ------------
#
# + omatt input changed from reference catalogue intermediate to product
#
# version 3.18.1 - 2011-08-09 EOP
# --------------
# 
# + Set implot to produce directly PNG files instead of GIF.
#
# version 3.18 - 2010-09-02 DLG
# ------------
# 
# + Fixed name of USNO extract catalogue to 'Reference catalogue'
#
# version 3.17 - 2008-10-06 DLG
# ------------
# 
# + Added debugging code
# + Change minsignificance to 3 in line with the omichain and the value
#   used to construct the omcatalogue
#
# version 3.16 - 2006-09-20 DJF
# ------------
#
# + Record use of exposures for source specific product creation in the database.
#
# version 3.15 - 2006-09-05 DJF
# ------------
#
# + changed 'OM OSW SOURCE LIST' to 'OM OSW GRISM SOURCE LIST' for GRISM processing.
# + Added new product 'OM OSW REGION FILE' generated by omdetect for non-grism images.
#
# version 3.14 - 2006-08-24 DJF
# ------------
#
# + Disabled OM GRISM-ALIGNED IMAGE PNG file creation.  implot does not
#   support the required CTYPE1|2 yet.
#
# version 3.13 - 2006-08-12 DJF
# ------------
#
# + Added code to make intermediate file OM GRISM-ALIGNED IMAGE PNG file for evaluation.
#
# version 3.12 - 2006-07-13 DJF
# ------------
#
# + Use OmGWIN keyword.  Only continue omgrism processing if omgprep sets OMGWIN to true.
#
# version 3.11 - 2006-02-20 DJF
# ------------
#
# + Change omdetect:minsignificane=2.0 to 2.5 by request from Simon Rosen.
#   This should cut down on the number of spurious sources.
#
# version 3.10 - 2006-02-07 DJF
# ------------
#
# + omcomb should now set a keywork IMGGAP to indicate any gabs in an ENG2 combined image.
#   It does not make sense to process the combined image where IMGGAP = 2|3 and the filter is GRISM.
#   In this case we process the individual components as USER mode images.
#   Otherwise the combined image is usefull and we process it normally.
#
# version 3.09 - 2006-02-06 DJF
# ------------
#
# + Now collecting keywords from ENG4 files.  This was preventing identification of 
#   ENG4+GRISM
#
# version 3.08 - 2006-01-19 DJF
# ------------
#
# + It had been assumed that if the filter was GRISM then the data was imaging
#   mode.  This ios not true.  0448_0153750701 has OM Fast mode data (E4I)
#   taken with the grism filter.  This module has been amended to skip such 
#   data.
# + Perform isImageEmpty test before calling implot.
#
# version 3.07 - 2005-11-18 IMS
# ------------
#
# + USNO extract now done by module USNOExtract. 'start' dependency (srcMerge no longer necessary) and findFile of usno altered accordingly.
#
# version 3.06 - 2005-11-17 IMS
# ------------
#
# + Tests now also for the existence of the omgrism output dataset before proceeding.
#
# version 3.05 - 2005-10-24 RGW
# ------------
#
# + don't try to create a OM grism spectrum plot if the FITS spectrum file doesn't contain a spectrum
# + backed out the hack preventning ENG4 data being analysed
#
# version 3.04 - 2005-09-14 RGW
# ------------
#
# + updated trigger conditions (now depends on SrcMerge)
# + changed minsignificance=2.0 on omdetect
# + omatt now uses the USNO catalogue for rectification
#
# version 3.03 - 2005-09-13 IMS
# ------------
#
# + Fixed bug in which keywords from the ODF images were not read when in PFENG2 mode (immediate effect was that grism images weren't being processed as such).
#
# version 3.02 - 2005-09-07 RGW
# ------------
#
# + hack: PFENG2 images are not processed unless there are 4 of them
#
# version 3.01 - 2005-07-28 IMS
# ------------
#
# + Added grism handling. This necessitates a very ugly query of FILTER keyword in order to cope with the clunky omprep interface.
# + Final sky image accounting file is not written in grism case (prevents OMMosaic working on grism images).
#
# version 3.00 - 2005-05-09 DJF
# ------------
#
# + Add explicit perl module headers.  Previously these were supplied
#   at compile time.  This will make debugging and extending the modules
#   through additional perl libraries easier.
#
# + Code layout made more uniform with perltidy
# + Standerdized date format (ccyy-mm-dd)
# + Standerdized author list to use comma separator
# + Make use of perl $VERSION magic.  Now $Version = version = ''
##
# Version 2.31 -  25-Feb-2004 (DJF)
# ------------
#
# + Explicitly set omprep modeset=0
#
# Version 2.30 -  10-Dec-2003 (DJF)
# ------------
#
# + Adapted doCommand to use anonymous lists for list parameters.
#
# Version 2.29 -  13-Dec-2002 (DJF)
# ------------
#
# + implot parameter changes.  Merged plotfile/Device
#
# Version 2.28 -  11-Dec-2002 (DJF)
# ------------
#
# + Remove redundant omdetect parameters (mod8set, flatset)
# + Added parameters nsigma , minsignificance & detectextended
#   to omdetect using default values 2.0 , 3.0 anmd 'Y' respectively.
# + Remove redundant ommag parmater wdxset
# + Change implot parameter withsrclistset to withsrclisttab (SAS 5.4)
#
# Version 2.27 -  30-Oct-2002 (DJF)
# ------------
#
# + OM Source rectification removed to a separate module
#
# Version 2.24 - 2.26 - 23-Oct-2002 (DJF)
# ------------
#
# + Position recitification isn't working.  Attempt to fix by
# removing preselection from USNO.
#
# Version 2.23 - 30-May-2002 (DJF)
# ------------
#
# + Added position rectification against usno as an intermediate product
#   for testing.
#
# Version 2.22 - 23-Jan-2002 (DH)
# ------------
#
# + Remove 'use strict' from file.
#
# Version 2.21 - 23-Jan-2002 (DFJ)
# ------------
#
# + Added exposure time (duration) and exp_id to
#   sky image details temporary file for use by OMMosaic Module
#
# Version 2.20 - 2001-12-04 (DFJ)
# ------------
#
# + Added tolerance to mode identification to deal with Rudi-5 strip
#   clipping
#
# Version 2.19 - 2001-11-29 (DFJ)
# ------------
#
# + Fixed bug which prevented pthe processing of RUDI5_HR images
#
# Version 2.18 - 2001-11-28 (DFJ)
# ------------
#
# + Added creation of PNG for each sky image
#
# Version 2.17 - 2001-11-28 (DFJ)
# ------------
#
# +Fix information written to ASCII files
#
# Version 2.16 - 2001-11-27 (DFJ)
# ------------
#
# +Fix use of writeASCIIFile
#
# Version 2.15 - 2001-11-27 (DFJ)
# ------------
#
# + Remove debugging code
# + Fix osw/filter selection on image output mode
#
# Version 2.14 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed error in writing of ASCII image details files
#
# Version 2.13 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed mode selection fo products details
#
# Version 2.12 - 2001-11-27 (DFJ)
# ------------
#
# + Added debugging code to help track problem with image creation
#
# Version 2.11 - 2001-11-27 (DFJ)
# ------------
#
# + Remove debugging code.
# + Retrieve FILTER keyword from primary header of raw image file
#
# Version 2.10 - 2001-11-27 (DFJ)
# ------------
#
# + Simplified logic of bad header ENG2/USER mode differentiation logic
#   in attemt to eliminate a bug
#
# Version 2.09 - 2001-11-27 (DFJ)
# ------------
#
# + Fixed keyword usage in image mode identification
#
# Version 2.08 - 2001-11-27 (DFJ)3# ------------
#
# +  Added some debugging code to help track down a problem with image mode indentification
#
# Version 2.07 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in logic of bad header ENG2/USER mode differentiation logic
#
# Version 2.06 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.05 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.04 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.03 - 2001-11-27 (DFJ)
# ------------
#
# + Corrected bug in use of readFITSKeyword
#
# Version 2.02 - 2001-11-26 (DFJ)
# ------------
#
# + Added OM File name filter abbreviations
#
# Version 2.01 - 2001-11-26 (DFJ)
# ------------
#
# + Changed recording of modes in ASCII list files.  ENG2 products
#  are now all recorded with as same mode ($ENG2)
# + Added new product content strings
#
# Version 2.00 - 2001-11-26 (DFJ)
# ------------
#
# + Major re-writing.  Now incorporates Image mode determination
#   and writing of product, mode and filter information to a log file
#   for use by OMMosaic and OMSourceCombine
# + Change in product naming convention to include filter in place of the osw
#   for Engineering 2 and 4 products.
#
# Version 1.22 - 2001-11-16 (DFJ)
# ------------
#
# + Extract filer and window size information from the images.
# + Add new product classes based on image information and classification
#
# Version 1.21 - 2001-11-05 (DFJ)
# ------------
#
# + Fix selection of osw. Was failing to recognize 0 as a valid osw.
#
# Version 1.20 - 2001-11-02 (DFJ)
# ------------
#
# + Fix for EN2. Was looking in odf rather than intermediate for combined image.
#
# Version 1.19 - 2001-11-02 (DFJ)
# ------------
#
# + Changed dummy osw value to 9 to mame it more obvious.
# + osw = 9 when no valid osw is found in the odf.
#
# Version 1.18 - 2001-11-02 (DFJ)
# ------------
#
# + Fix setting of osw. Was adding characters OSW to value then using it.
#
# Version 1.17 - 2001-10-31 (DFJ)
# ------------
#
# + Change determination of osw to account for different sources (osw or osw_id)
#
# Version 1.16 - 2001-10-30 (DFJ)
# ------------
#
# + Amended use of osw info.
#
# Version 1.15 - 2001-10-19 (DFJ)
# ------------
#
# + Added default osw id of zero for engineering mode 2
#
# Version 1.14 - 2001-10-18 (DH)
# ------------
#
# + Fix minor bugs in evaluateRules().
# + Update changed parameter names in omflatfield and ommodmap.
#
# Version 1.13 - 2001-10-11 (DJF)
# ------------
#
# + Remove exposure processing which is commom to both image and fast chains.
#	This has been moved to modules OMExpAnalyse
#
# Version 1.12 - 2001-04-23 (DH)
# ------------
#
# + Change the OM OSW image from an intermediate
#	file to a product.
#
# Version 1.11 - 2001-03-16 (DH)
# ------------
#
# + Print out version number in performAction() for
#	tracking purposes.
#
# Version 1.10 - 2001-03-08 (DH)
# ------------
#
# + Change om flat field from being a product file to
#	being an intermediate file.
#
# Version 1.09 - 2000-11-29 (DH)
# ------------
#
# + First production version.
#
# Declare list of instruments this module is interested in
@instList = qw(om);
sub Instlist
{
	return qw(om);
}

# Number of streams
sub numberOfStreams
{
    return numberOfExposures();
}

sub evaluateRules
{

    # If upstream module has been ignored, so if this one
    return ignore()
      if ignored(
        module => 'OMgetFlat'
        , instrument => thisInstrument
        , stream => 1
      );
    return ignore()
      if ignored(
        module => 'OMExpAnalyse'
        , instrument => thisInstrument
        , stream => thisStream
      );

    # Start conditions
    start()
      if complete(
        module => 'OMExpAnalyse'
        , instrument => thisInstrument
        , stream => thisStream
      )
#      and complete (
#	module => 'SrcMerge'
#        , instrument => 'all'
#	, stream => 1
#      );
      and complete (
	module => 'USNOExtract'
        , instrument => 'all'
	, stream => 1
      );
}

use vars qw( %Filters );
my %Filters = (
	'BLOCKED' => '9'
	, 'V' => 'V'
	, 'U' => 'U'
	, 'B' => 'B'
	, 'WHITE' => 'W'
	, 'GRISM2' => 'G'
	, 'GRISM1' => 'H'
	, 'UVW1' => 'L'
	, 'UVW2' => 'S'
	, 'UVM2' => 'M'
);
sub getfiltershortname
{
	my ($c,$f) = (@_);
	$f = uc($f);
	return exists(  $Filters{$f} ) 
		? $Filters{$f}
		: exception( Exception->('BADOMFILTER','Unrecognized filter :',$f) );
	;
}

sub performAction
{
    info("Module version number: $version");

    # Find exposure ID
    my $exp_id = exposureID(
    	instrument => thisInstrument
        ,stream => thisStream
    );

# FUNNY is not a real mode.  It is used when determining image formats which as it suggests may be funny
# (eg. cases where the FITS header are known to be wrong)
# PFENG2 = Partial Frame Engineering 2, FFENG2 = Full frame Engineering 2
# PFENG2 and FFENG2 are different as input but produce the same output as more ENG2

    # This hash describes the  output products
    my %products = (
        USER => {
            IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
            , SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
          }
        , RUDI5_LR => {
            IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
            , SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
          }
        , RUDI5_HR => {
            IMG => { CLASS => 'product', CONTENT => 'OM OSW IMAGE' }
            , SKYIMG => { CLASS => 'product', CONTENT => 'OM OSW SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
          }
		, FFENG2 => {
            IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
            , SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
          }
		, PFENG2 => {
            IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
            , SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
          }
        , ENG4 => {
            IMG => { CLASS => 'product', CONTENT => 'OM FULL-FRAME IMAGE' }
            , SKYIMG => { CLASS => 'product' , CONTENT => 'OM FULL-FRAME SKY IMAGE' }
            , SRC => { CLASS => 'product', CONTENT => 'OM OSW SOURCE LIST' }
        }
    );

    # This hash describes the (eventual) input images.
    my %inputimage = (
        RUDI5_LR => {
            CLASS => 'odf'
            , CONTENT => 'Imaging mode image'
            , LIST => []
          }
        , RUDI5_HR => {
            CLASS => 'odf'
            , CONTENT => 'Imaging mode image'
            , LIST => []
          }
        , PFENG2 => {
            CLASS => 'intermediate'
            , CONTENT => 'Engineering Mode 2 Combined Image'
            , LIST => []
          }
        , FFENG2 => {
            CLASS => 'odf'
            , CONTENT => 'Engineering mode image'
            , LIST => []
          }
        , ENG4 => {
            CLASS => 'odf'
            , CONTENT => 'Engineering mode image'
            , LIST => []
          }
        , USER => {
            CLASS => 'odf'
            , CONTENT => 'Imaging mode image'
            , LIST => []
        }
    );
    my $count = 0;
    info("Looking for Engineering Mode Files...");
    my @englist = findFile(
        class => 'odf'
        , instrument => thisInstrument
        , exp_id => $exp_id
        , content => 'Engineering mode image'
    );
    $count = scalar(@englist) || "None";
    info("Found $count Engineering mode Windows.");

	# Collect relevant details from file headers
    my %keywords;


	###
	### Look for engineering 4
	### Extract the necessary keywords.
	###

	$inputimage{ENG4}{LIST} = [ grep ( /E4I/, @englist ) ];
	foreach my $e4i ( @{$inputimage{ENG4}{LIST}} )
	{
		$keywords{$e4i} = { &get_keywords($e4i,'OME4I1') };
	}

	$count = scalar(@{$inputimage{ENG4}{LIST}}) || 'No';
	info("Found $count Full Frame Engineering 4 Windows.");

	###
	### Look for full-frame Engineering mode 2
	### Extract the necessary keywords.
	###
	# NB. At the time of writing E2I files are not part of the ODF but this is a future proposal.
	# If they appear E2I should be a precombined low res full sky image
	# replacing omcomb on partial frame (IMI) images below.
	# Keep separate FFENG2 and PFENG2 lists so we know what came from where.

	push @{$inputimage{FFENG2}{LIST}} , ( grep ( /E2I/, @englist ));
	foreach my $e2i ( @{ $inputimage{FFENG2}{LIST} } )
	{
		$keywords{$e2i} = { &get_keywords($e2i,'OME2I1') };
	}
	$count = scalar(@{$inputimage{FFENG2}{LIST}}) || 'no';
	info("Found $count Full Frame Engineering 2 Windows.");

	###
    ### Image Modes
	###
    info("Looking for Image Mode Files...");
    my @imiList = findFile(
        class => 'odf'
        , instrument => thisInstrument
        , exp_id => $exp_id
	, content => 'Imaging mode image'
    );
    $count = scalar(@imiList) || "no";
    info("Found $count Image mode images.");

    # Determine the mode of each image file
	### Extract the necessary keywords.
    foreach my $imi (@imiList)
    {
        $keywords{$imi} = { &get_keywords($imi,'OMIMI1') };
        &image_format( $imi , $keywords{$imi} );
		my $mode = $keywords{$imi}{MODE};
        info("Identified mode: $mode");
        push( @{ $inputimage{$mode}{LIST} }, $imi );
    }

	### 
	### Pre-process Partial-Frame Engineering mode 2
	### 
	### If we have engineering 2 pieces we need to combine them.  
	### If they fail to combine we assume they are USER defined images.

	my $combine = 0;
	my $imggap;
    if ( @{ $inputimage{PFENG2}{LIST} } > 2 )
    {
		my @pfeng2list = @{ $inputimage{PFENG2}{LIST} };
		$inputimage{PFENG2}{LIST} = [];
		info("Might have Partial-Frame Engineering Mode 2 images.");
		info("Trying to generate a combined image.");

	# omcomb should now add a keyword IMGGAP to the final image. 
	# If it is 2 or 3 and filter == GRISM then we should not use the merged image
    # but process the components as USER mode images instead.

        my $en2comb = newFile(
            class => 'intermediate'
            , instrument => thisInstrument
            , exp_id => $exp_id
            , content => 'Engineering Mode 2 Combined Image'
        );
        $combine = doCommand(
            'omcomb', imagesets => [ @pfeng2list ]
            , outset => $en2comb
        );

		# If combining succeded...
        if ($combine)
		{
			info("Successfully created a combined image.");
			
			$keywords{$en2comb} = { &get_keywords($en2comb , 'OMIMI1') };

			# IMGGAP tells us of any missing Partial-Frame windows.
			$imggap = readFITSKeyword(
				class => 'odf'
				, file => $en2comb
				, extension => 'OMIMI1'
				, keyword => 'IMGGAP'
			);

			if ($imggap)
			{
				# If GRISM image is missing a central window process as USER mode.
				if (( $keywords{$en2comb}{FILTER}  eq '200' || $keywords{$en2comb}{FILTER} eq '1000' )
					&& ($imggap eq '2' || $imggap eq '3')
				){
					info( "Inner segment of combined image missing with GRISM filter." );
					info( "Will process as individual USER mode images." );
					$combine = 0;
					push  @{ $inputimage{USER}{LIST} } ,  @pfeng2list;
				}	
				else
				{
					info( "Outer segment of combined image missing with GRISM filter." );
					info( "Will continue to process as an ENG2 image." );
				}
			}
			else 
			{
				info( "Processing as Engineering mode 2" );
				push ( @{ $inputimage{PFENG2}{LIST} } , $en2comb );
			}
        }
	}
	else 
	{
		info(
			"Failed to make a suitable combined image. Will process components as USER mode."
		);
		push( @{ $inputimage{USER}{LIST} } , @{ $inputimage{PFENG2}{LIST} });
		$inputimage{PFENG2}{LIST} = [];
	}
    ###
    ###
    ###
    #
    # Find necessary files
	# %p hold parameters necessary for processing images.
	my %p;
    $p{windowFile} = findFile(
        class => 'odf'
        , instrument => thisInstrument
        , exp_id => $exp_id
        , content => 'Priority window data'
        , required => 'true'
    );

    # Find periodic housekeeping file
    $p{phkFile} = findFile(
        class => 'odf'
        , instrument => thisInstrument
        , content => 'Periodic housekeeping'
        , required => 'true'
    );

    # Find non-periodic housekeeping file
    $p{nphkFile} = findFile(
        class => 'odf'
        , instrument => thisInstrument
        , content => 'Non-periodic housekeeping'
        , required => 'true'
    );

    # Find omprep tracking history
    $p{prepThFile} = findFile(
        class => 'intermediate'
        , instrument => thisInstrument
        , exp_id => $exp_id
        , content => 'omprep tracking history'
    );

    # Flat created by omflatgen
    $p{flatGenFile} = findFile(
        class => 'intermediate'
        , instrument => thisInstrument
        , content => 'OM Flatfield'
    );

    #Arrays for Image and Source details
    my @skyimagedetails = ();
    my @sourcedetails    = ();

    # Loop over all modes
    foreach my $mode ( 'USER', 'RUDI5_LR', 'RUDI5_HR', 'PFENG2' , 'FFENG2' , 'ENG4' )
    {
        info("Processing $mode Images.");
        my @imList = @{ $inputimage{$mode}{LIST} };
        $count = scalar(@imList);
        info("$count images to process.");
        next unless ($count);

        # Loop over all images of a given mode
        foreach my $imFile (@imList)
        {
			# Default to osw of 9 to make it obvious where a value was missing.
			Msg->mark( __PACKAGE__ , INPUTIMAGE => \%inputimage, MODE => $mode );
            my $imInfo = fileInfo(
                class => $inputimage{$mode}{CLASS}
                , name => $imFile
            );
            my $osw = $imInfo->{'osw'};
            $osw = $imInfo->{'osw_id'} 
				unless ( defined($osw) )
			;
            $osw = 9
            	unless ( defined($osw) )
            ; # The value 9 should make it obvious when an osw value is missing.
            info("Processing $exp_id:$osw:$imFile");

			$p{rawImFile} = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
				, content => 'Raw Image'
			);

# Have to process grism separately. Whether grism or not is decided on the value of the FILTER kwd:
			
			my $result;
            if ($keywords{$imFile}{FILTER} eq '200' || $keywords{$imFile}{FILTER} eq '1000')
            {
                $result = &process_grism_image(exp_id => $exp_id
					,osw =>  $osw
					,imFile => $imFile
					,mode => $mode
					,keywords => $keywords{$imFile}
					,skyimageoutput => $products{$mode}{SKYIMG}
					,srcoutput => $products{$mode}{SRC}
					,imgoutput => $products{$mode}{IMG}
					,%p
				);
            }
			else
			{
                 $result = &process_image(exp_id => $exp_id
					,osw =>  $osw
					,imFile => $imFile
					,mode => $mode
					,keywords => $keywords{$imFile}
					,skyimageoutput => $products{$mode}{SKYIMG}
					,srcoutput => $products{$mode}{SRC}
					,imgoutput => $products{$mode}{IMG}
					,%p
				);
			}
			if ($result)
			{
#### DEBUG
# $result->{srcdetails} is returned as an AoA. WHY??
# flatten srcdetails AoA
			    my @tmpsrcdetails = $result->{srcdetails};
			    my $di = 0; 
			    my @flatsrcdetails = ();
			    for ($di = 0; $di <= $#tmpsrcdetails; $di++) {
				push (@flatsrcdetails, $tmpsrcdetails[$di][0]);
			    }

# flatten skyimgdetails AoA

			    my @tmpskyimgdetails = $result->{skyimgdetails};
			    my $di = 0;
			    my @flatskyimgdetails = ();
			    for ($di = 0; $di <= $#tmpskyimgdetails; $di++) {
				push (@flatskyimgdetails, $tmpskyimgdetails[$di][0]);
			    }

#				push @skyimagedetails , $result->{skyimgdetails};
#				push @sourcedetails , $result->{srcdetails};

			    push @skyimagedetails, @flatskyimgdetails;
			    push @sourcedetails, @flatsrcdetails;

#### DEBUG
			}
		}
	}
# Everything was OK
    info("All images processed");
    info("Writing accounting information");

#### DEBUG
    my $debugsourcedetails = Dumper(\@sourcedetails);
    info("DEBUG: sourcedetails array: $debugsourcedetails");
    my $debugskyimagedetails = Dumper(\@skyimagedetails);
    info("DEBUG: skyimagedetails array: $debugskyimagedetails");
#### DEBUG


    #Write SkyImage and Source information for use by later modules
	if (@sourcedetails)
	{
		my $sourcedetailsfile = newFile(
			class => 'intermediate'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, format => "ASCII"
			, content => "OM SRC IMAGE DETAILS"
		);
		writeASCIIFile(
			name => $sourcedetailsfile
			, text => \@sourcedetails
		);
	}

# Note that sky image details are not stored for grism images. This has the effect of preventing mosaics of grism images being made by OMMosaic. IMS 2005-08-02.

	if (@skyimagedetails)
	{
		my $skyimagedetailsfile = newFile(
			class => 'intermediate'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, format => "ASCII"
			, content => "OM SKY IMAGE DETAILS"
		);
		writeASCIIFile(
			name => $skyimagedetailsfile
			, text => \@skyimagedetails
		);
	}
    return success();
}

sub process_image
{
	my ( %h ) = (@_);
	my ($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $flatGenFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords );
	($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $flatGenFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords )
		= @h{qw( exp_id osw imFile rawImFile nphkFile phkFile windowFile prepThFile flatGenFile skyimageoutput srcoutput imgoutput mode keywords)}
	;
	my $rtn = { srcdetails => [] , skyimgdetails => []};
	# Run omprep on the image
	# Find window data file
	doCommand(
		'omprep', set => $imFile
		, nphset => $nphkFile
		, pehset => $phkFile
		, wdxset => $windowFile
		, outset => $rawImFile
		, modeset => 0
	  )
	  or return exception();

	# Get the filter value now omprep has converted it from a code to a recognizable value
	# Also get value from EXPOSURE keyword as duration.
	info("Retrieving FILTER value");
	my $filter = readFITSKeyword(
		class => 'intermediate'
		, file => "$rawImFile"
		, extension => 'PRIMARY'
		, keyword => 'FILTER'
	);

	# Filter to single character translation
	$filter = __PACKAGE__->getfiltershortname($filter);
	info("FILTER = $filter");
	info("Retrieving EXPOSURE value");
	my $duration = readFITSKeyword(
		class => 'intermediate'
		, file => "$rawImFile"
		, extension => 'PRIMARY'
		, keyword => 'EXPOSURE'
	);
	info("EXPOSURE = $duration");

	if ( ( $mode eq 'RUDI5_LR' )
		&& !( ( $keywords->{WINDOWDX} == 976 )
			&& ( $keywords->{WINDOWDY} == 960 )
	)){
		$duration *= -1;
	}

	# Will the output use the osw or filter character ?
	$osw = ( $mode =~ /ENG[24]/ ) ? $filter : $osw;

	# omcosflag
	doCommand(
		'omcosflag', set => $rawImFile
		, thxset => $prepThFile
	  )
	  or return exception();

	# omflatfield
	my $flatOut = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OSW Flat Field'
	);
	my $flattedIm = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OSW Flat Fielded Image'
	);
	doCommand(
		'omflatfield', set => $rawImFile
		, thxset => $prepThFile
		, inorbitflatset => $flatGenFile
		, tsflatset => $flatOut
		, outset => $flattedIm
	  )
	  or return exception();

	# ommodmap
# IMAGE
	my $finalImage = newFile(
		class => $imgoutput->{CLASS}
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => $imgoutput->{CONTENT}
	);
	my $mod8Im = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OSW MODULO 8 IMAGE'
	);
	doCommand(
		'ommodmap', set => $flattedIm
		, mod8product => 'Y'
		, mod8set => $mod8Im
		, outset => $finalImage
		, nsig => 6
		, nbox => 16
	  )
	  or return exception();

        # omqualitymap
        doCommand('omqualitymap', set => $finalImage
                  , srclistset => ''
                  , outset => $finalImage
                  , mode => 'setqualityimage'
                  )
            or return exception();

	# omdetect
# SOURCE LIST
	my $sourceListFile = newFile(
		class => $srcoutput->{CLASS}
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => $srcoutput->{CONTENT}
	);

	my $RegionFile=newFile(class => 'product'
		,instrument => thisInstrument
		,exp_id => $exp_id
		,osw_id => $osw
		,content => 'OM OSW REGION FILE'
		,format => 'ASCII'
	);

	doCommand(
		'omdetect', set => $finalImage
		, nsigma => 2.0
		, minsignificance => 3
		, detectextended => 'Y'
		, regionfile => $RegionFile
		, outset => $sourceListFile
	  )
	  or return exception();

        # omqualitymap again to forward flags to source lists
        
        doCommand('omqualitymap', set => $finalImage
                  , srclistset => $sourceListFile
                  , outset => $finalImage
                  , mode => 'setqualityimage'
                  )
            or return exception();

	push( @{$rtn->{srcdetails}}
		, "$sourceListFile\|$mode\|$filter\n" 
	);

# Write a dummy value of OBJECT to trick omsrclistcomb later in the pipeline

	# ommag
	doCommand( 'ommag', set => $sourceListFile )
	  or return exception();

	# Use omatt to make the sky image.
# SKY IMAGE
	my $usnoList = findFile(
				class => 'product'
				, instrument => 'all'
				, content => 'Reference catalogue'
				, format => 'FITS'
				);
	info("No USNO extract to work from.  Unable to continue.") and ignore() unless defined ($usnoList);
	my $skyImage = newFile(
		class => $skyimageoutput->{CLASS}
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => $skyimageoutput->{CONTENT}
	);
	doCommand(
		'omatt', set => $finalImage
		, sourcelistset => $sourceListFile
		, ppsoswset => $skyImage
		, usecat => 'Y'
		, catfile => $usnoList
	) or return exception();
	push( @{$rtn->{skyimgdetails}}
		, "$skyImage\|$mode\|$filter\|$duration\|$exp_id\n"
	);

	# Make sure the image isn't empty first
	my $imgStat = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, format => "ASCII"
		, content => "OSW $osw sky image statistics"
	);
	unless ( &ModuleUtil::isImageEmpty( $skyImage , $imgStat ) )
	{

	# CREATE PNG FROM SKY IMAGE
		my $gifFile = newFile(
			class => 'intermediate'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => $skyimageoutput->{CONTENT}
			, format => 'GIF'
		);
		my $pngFile = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => $skyimageoutput->{CONTENT}
			, format => 'PNG'
		);
		doCommand(
			'implot', set => $skyImage
			, device => "$pngFile/PNG"
			, withsrclisttab => 'no'
		) or return exception();

#		# Convert to PNG format
#		GIFtoPNG(
#			source => $gifFile
#			, destination => $pngFile
#		) or return exception();
	} # End if isImageEMpty
	return $rtn;
}

sub process_grism_image
{
	my ( %h ) = (@_);
	my ($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords );
	($exp_id , $osw , $imFile , $rawImFile , $nphkFile , $phkFile , $windowFile , $prepThFile , $skyimageoutput , $srcoutput , $imgoutput , $mode , $keywords )
		= @h{qw( exp_id osw imFile rawImFile nphkFile phkFile windowFile prepThFile skyimageoutput srcoutput imgoutput mode keywords)}
	;
	my $rtn = { srcdetails => [] , skyimgdetails => []};

	doCommand(
		'omprep', set => $imFile
		, nphset => $nphkFile
		, pehset => $phkFile
		, wdxset => $windowFile
		, outset => $rawImFile
		, modeset => 4
	  )
	  or return exception();

# Get the filter value now omprep has converted it from a code to a recognizable value
# Also get value from EXPOSURE keyword as duration.
	info("Retrieving FILTER value");
	my $filter = readFITSKeyword(
		class => 'intermediate'
		, file => $rawImFile
		, extension => 'PRIMARY'
		, keyword => 'FILTER'
	);

	# Filter to single character translation
	$filter = __PACKAGE__->getfiltershortname($filter);
	info("FILTER = $filter");
	$filter = uc($filter);
	info("FILTER = $filter");
	info("Retrieving EXPOSURE value");
	my $duration = readFITSKeyword(
		class => 'intermediate'
		, file => $rawImFile
		, extension => 'PRIMARY'
		, keyword => 'EXPOSURE'
	);
	info("EXPOSURE = $duration");

	my $mod8Im = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OSW Modulo 8 Image'
	);
	my $mod8CorrectedIm = newFile(
		class => 'intermediate'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OSW Mod-8-corrected Image'
	);
# Following parameter values are those in omgchain-1.0.12, except --flatset and --outflatset are left at defaults.
	doCommand(
		'ommodmap'
		, set => $rawImFile
		, mod8product => 'yes'
		, mod8set => $mod8Im
		, outset => $mod8CorrectedIm
		, nsig => 3
		, nbox => 16
	  )
	  or return exception();

	my $finalImage = newFile(
		class => 'product'
		, instrument => thisInstrument
		, exp_id => $exp_id
		, osw_id => $osw
		, content => 'OM GRISM-ALIGNED IMAGE'
	);

	doCommand(
		'omgprep'
		, set => $mod8CorrectedIm
		, outset => $finalImage
	)
	or return exception();


	# Make it easy to disable during testing / discussions
	# Disabled until implot supports CTYPE?=PIXELS
	if (0)
	{
		my $imgStat = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
				, format => "ASCII"
				, content => "Aligned image statistics"
		);

		if ( !ModuleUtil::isImageEmpty( $finalImage , $imgStat ) )
		{
			ppd( "000023" );

			# CREATE PNG FROM GRISM IMAGE
			my $gifFile = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
				, content => 'OM GRISM-ALIGNED IMAGE'
				, format => 'GIF'
			);
			my $pngFile = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
				, content => 'OM GRISM-ALIGNED IMAGE'
				, format => 'PNG'
			);
			doCommand(
				'implot', set => $finalImage
				, device => "$pngFile/PNG"
				, withsrclisttab => 'no'
			) or warn('implot failed to create an OM GRISM-ALIGNED IMAGE PNG'); 
			# Disable excepton during testing of potential product.
			#) or return exception();

#			# Convert to PNG format
#			GIFtoPNG(
#				source => $gifFile
#				, destination => $pngFile
#			) or return exception();
		}
		else
		{
			ppd( "000024" );
		}
	}

	# Read OMGWIN keyword.
	my $omgwin = readFITSKeyword(
		file => $finalImage
		, extension => "PRIMARY"
		, keyword => "OMGWIN"
	);

	if ($omgwin =~ /$RE{boolean}{true}/s)
	{
		ppd( "000021" );
		my $omdetectRegionFile = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => 'OM GRISM DS9 REGIONS'
			, format => "ASCII"
		);
		my $sourceListFile = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => 'OM OSW GRISM SOURCE LIST'
		);
# Following parameter values are those in omgchain-1.0.12. Note that they are different to the non-grism omdetect parameters (as of OMImageAnalyse-3.01).
		doCommand(
			'omdetect'
			, set => $finalImage
			, nsigma => 2.0
			, regionfile => $omdetectRegionFile
			, outset => $sourceListFile
		  )
		  or return exception();
		push( @{$rtn->{sourcedetails}}
			, "$sourceListFile\|$mode\|$filter\n" );

		my $omgrismRegionFile = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => 'OM GRISM DS9 SPECTRUM REGIONS'
			, format => "ASCII"
		);
		my $listOfGrismSpectra = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => 'OM GRISM SPECTRA LIST'
		);
		my $omgrismSpectrum = newFile(
			class => 'product'
			, instrument => thisInstrument
			, exp_id => $exp_id
			, osw_id => $osw
			, content => 'OM GRISM SOURCE SPECTRUM'
		);
# omgchain-1.0.12 default parameter values.
		doCommand(
			'omgrism'
			, set => $finalImage
			, sourcelistset => $sourceListFile
			, outset => $omgrismSpectrum
			, bkgoffsetleft => 6.0
			, bkgwidthleft => -6.0
			, bkgoffsetright => 6.0
			, bkgwidthright => -6.0
			, spectrumhalfwidth => -6.0
			, spectrumsmoothlength => 0
			, extractionmode => 0
			, extractfieldspectra => 'no'
			, regionfile => $omdetectRegionFile
			, spectraregionfile => $omgrismRegionFile
			, outspectralistset => $listOfGrismSpectra
		  )
		  or return exception();


### How does omgrismplot know to make a plot just for the target? I don't think I want to know... :(  ...but I better find out some time.
		if(fileExists(
		  file => $omgrismSpectrum
		) && hasFITSExtension(
		  file => $omgrismSpectrum
		  ,extension => 'RATE001')
		)
		{
			setExposureProperty( instrument => thisInstrument , exp_id => $exp_id , name => 'ssp' , value => 0+TRUE );
			my $tempPlotFile = newFile(
				class => 'intermediate'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
###                        , src_num => $source
			, format => 'PS'
			, content => "OM grism source spectrum"
			);
# omgchain-1.0.12 default parameter values.
			doCommand(
				'omgrismplot'
				, set => $omgrismSpectrum
				, scalebkgplot => 'no'
				, binsize => 1
				, plotflux => 2
				, plotfile => $tempPlotFile
			  )
			  or return exception();
			my $grismSpectrumPlot = newFile(
				class => 'product'
				, instrument => thisInstrument
				, exp_id => $exp_id
				, osw_id => $osw
###                        , src_num => $source
				, 'format' => 'PDF'
				, content => "OM GRISM SOURCE SPECTRUM"
			  );
			PStoPDF(
				source => $tempPlotFile
				, destination => $grismSpectrumPlot
			  )
			  or return exception();
		}
	}
	else
	{	
		ppd( "000022" );
	}
	return $rtn;
}
###
### OM OSW IMAGE IDENTIFICATION
###
### RUDI-5 WINDOWS:
###	Exp.	WINDOWDX	WINDOWDY	BIN
###	---		---------	---------	---
###	1		224			224			1x1
###	1		976			960			2x2
###	2		224			224			1x1
###	2		480			1792		2x2
###	3		224			224			1x1
###	3		1792		480			2x2
###	4		224			224			1x1
###	4		480			1792		2x2
###	5		224			224			1x1
###	5		1792		480			2x2
###
### ENGINEERING 2 FULL FRAME IMAGE
###	Exp.	WINDOWDX	WINDOWDY	BIN
### -		2048		2048		1X1
###
### ENGINEERING 2 PARTIAL FRAME IMAGE
### With Correct Headers
### WINDOWX		WINDOWY	NAXIS1	NAXIS2	WINDOWDX	WINDOWDY	BINBPE
### n x 512		0		256		1024	512			2048		F
###
### ENGINEERING 2 PARTIAL FRAME IMAGE
### With Incorrect Headers
### WINDOWX		WINDOWY	NAXIS1	NAXIS2	WINDOWDX	WINDOWDY	BINBPE
### n x 256		0		256		1024	256			1024		T
###
### ENGINEERING 4
### Always E4I.FIT
###
### USER DEFINED
### Not one of the above
###
### NB. Some of the above values are obtained from the imFile
### and others from the omprep generated raw image. Which we get
### from where depends on the image format.
###

use vars qw(%Windowdefinition $Tolerance $Window1 $Window2 $Window1min $Window1max $Window2min $Window2max );
%Windowdefinition = (
	#These keys are {WINDOWDXxWINDOWDY} and {BINAX1xBINAX2}
	# FUNNY is used for images which could be ENG2 partial frame with bad headers or USER
	# mode.  Further tests are conducted below.
#	'976x960' => { '2x2' => 'RUDI5_LR' }
#	, '480x1792' => { '2x2' => 'RUDI5_LR' }
#	, '1792x480' => { '2x2' => 'RUDI5_LR' }
	'976x960' => { '2x2' => 'USER' }
	, '480x1792' => { '2x2' => 'USER' }
	, '1792x480' => { '2x2' => 'USER' }
	, '224x224' => { '1x1' => 'RUDI5_HR' }
	, '2048x2048' => { '1x1' => 'FFENG2' }   # Eng-2 Full frame image
	, '512x2048' => { '1x1' => 'PFENG2' } # Eng-2 Partial frame image
	, '256x1024' => { '1x1' => 'FUNNY' } # Needs more analysis to determine.
);
$Tolerance = 50;
($Window1 , $Window2) = (1792 , 480);
($Window1min , $Window1max) = ( $Window1 - $Tolerance , $Window1 + $Tolerance);
($Window2min , $Window2max) = ( $Window2 - $Tolerance , $Window2 + $Tolerance);

sub image_format
{
    my ( $image, $kw ) = @_;
    info("Analysing $image for image identification");

    my $windowsize   = $kw->{WINDOWDX}.'x'.$kw->{WINDOWDY};
    my $naxissize = $kw->{NAXIS1}.'x'.$kw->{NAXIS2};
    my $windowbin    = $kw->{BINAX1}.'x'.$kw->{BINAX2};

    my $mode  =  (exists $Windowdefinition{$windowsize}{$windowbin}) ? $Windowdefinition{$windowsize}{$windowbin} : 'USER';

    # Need some tolerance due to clipping of RUDI5_LR windows
    # For RUDI-5 WINDOWDX/Y +/- $tolerance  of above values and EXP_ID of 4nn
    if ( ( $mode eq 'USER' ) && ( $kw->{EXP_ID} =~ /4\d\d$/ ) )
    {
        my $tolerance = 50;
        my ( $val1, $val2 ) = ( 1792, 480 );
		# second dimension is the lesser of the two.
	my ( $dim2, $dim1 ) = sort {$a <=> $b} ($kw->{WINDOWDX} , $kw->{WINDOWDY});
#        $mode = 'RUDI5_LR'
        $mode = 'USER'
          if ( ( ($dim1 == $Window1) && ($dim2 == $Window2) )
			|| 	( ( $dim1 > $Window1min ) && ( $dim1 < $Window1max )  
				&& ( $dim2 > $Window2min ) && ( $dim2 < $Window2max ) )
		);
    }

    # Further refinement of mode (ie. handle bad headers etc)
    if ( $mode eq 'FUNNY' )
    {
        info("Peculiar case: Could be ENG2 or USER mode.  Testing...");
	# This if accounts for wrongly headered ENG2 partial frame data in the odf
        if (   ( $windowsize eq $naxissize ) && ( $kw->{BINBPE} eq 'F' ) )
        {
            info( "WINDOWDX/Y = NAXIS1/2 and BINBPE = F indicates an PFENG2 component image");

            $mode = 'PFENG2';
        }
        else
        {
            $mode = 'USER';
        }
    }
	$kw->{MODE} = $mode;
}

sub get_keywords
{
	my ($file,$extn) = (shift,shift);
    my %keywords = (
        EXP_ID => ''
        , WINDOWDX => 0
        , WINDOWDY => 0
        , BINAX1 => 0
        , BINAX2 => 0
        , BINBPE => 0
        , FILTER => 0
        , NAXIS1 => 0
        , NAXIS2 => 0
    );

    # Extract keywords with which to identify image
    foreach my $k ( keys(%keywords) )
    {
        $keywords{$k} = readFITSKeyword(
            class => 'odf'
            , file => $file
            , extension => $extn
            , keyword => "$k"
        );
    }
	return %keywords;
}

1;