[gnuastro-commits] master 5b6b502c 2/6: dither-simulate: analysis done within a Makefile

[Mohammad Akhlaghi]

branch: master
commit 5b6b502c1006d46f2a798194dd15d822a0da93d6
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    dither-simulate: analysis done within a Makefile
    
    Until now, the newly added 'bin/script/dither-simulate.in' would simply
    create the dithering pattern in series, within the shell script itself.
    
    With this commit, following the experience that was learnt in the Zeropoint
    script, a separate Makefile has been written for this script to call. This
    significantly improves the speed.
---
 bin/script/dither-simulate.in | 152 ++++++++++++++++++++++++++++++------------
 bin/script/dither-simulate.mk |  60 +++++++++++++++++
 bin/script/zeropoint.in       |   3 +-
 bin/script/zeropoint.mk       |   7 --
 4 files changed, 169 insertions(+), 53 deletions(-)

diff --git a/bin/script/dither-simulate.in b/bin/script/dither-simulate.in
index a18afec0..92788650 100644
--- a/bin/script/dither-simulate.in
+++ b/bin/script/dither-simulate.in
@@ -26,18 +26,20 @@ export LC_NUMERIC=C
 
 # Default values.
 img=""
+hdu=1
 cat=""
 width=""
 imghdu=1
 center=""
 keeptmp=0
-output=""
 racol="RA"
 deccol="DEC"
 widthinpix=0
+numthreads=0
 version=@VERSION@
 scriptname=@SCRIPT_NAME@
-output=dither-check.fits
+output=dither-simulate.fits
+installdir=@PREFIX@/share/gnuastro
 
 
 
@@ -66,6 +68,7 @@ $scriptname options:
   -C, --center=FLT,FLT    Center of output stack (in RA,Dec).
   -w, --width=FLT,FLT     Width of output stack (in WCS).
       --widthinpix        Interpret '--width' values in pixels.
+      --mksrc=STR         Makefile (for developers when debugging).
 
  Output:
   -o, --output=STR        Name of finally stacked image.
@@ -74,6 +77,7 @@ $scriptname options:
 
  Operating mode:
   -h, --help              Print this help.
+  -N, --numthreads=INT    Number of threads; maximum if not given.
       --cite              BibTeX citation for this program.
   -q, --quiet             Don't print any extra information in stdout.
   -V, --version           Print program version.
@@ -90,7 +94,7 @@ EOF
 print_version() {
      cat <<EOF
 $scriptname (GNU Astronomy Utilities) $version
-Copyright (C) 2020-2023 Free Software Foundation, Inc.
+Copyright (C) 2023-2023 Free Software Foundation, Inc.
 License GPLv3+: GNU General public license version 3 or later.
 This is free software: you are free to change and redistribute it.
 There is NO WARRANTY, to the extent permitted by law.
@@ -167,9 +171,9 @@ EOF
 
 
 
+
 # Read the command-line options
-# Separate command-line arguments from options and put the option values
-# into the respective variables.
+# -----------------------------
 #
 # OPTIONS WITH A VALUE:
 #
@@ -220,9 +224,12 @@ do
     -w|--width)      width="$2";                                  check_v "$1" 
"$width";  shift;shift;;
     -w=*|--width=*)  width="${1#*=}";                             check_v "$1" 
"$width";  shift;;
     -w*)             width=$(echo "$1" | sed -e's/-w//');         check_v "$1" 
"$width";  shift;;
+    --mksrc)         mksrc="$2";                                  check_v "$1" 
"$mksrc";  shift;shift;;
+    --mksrc=*)       mksrc="${1#*=}";                             check_v "$1" 
"$mksrc";  shift;;
     --widthinpix)    widthinpix=1;                                             
           shift;;
     --widthinpix=*)  on_off_option_error --quiet -q;;
 
+
    # Output parameters
    -k|--keeptmp)           keeptmp=1; shift;;
    -k*|--keeptmp=*)        on_off_option_error --keeptmp -k;;
@@ -242,6 +249,9 @@ do
     -V*|--version=*)        on_off_option_error --version -V;;
     --cite)                 print_citation; exit 0;;
     --cite=*)               on_off_option_error --cite;;
+    -N|--numthreads)        numthreads="$2";                          check_v 
"$1" "$numthreads";  shift;shift;;
+    -N=*|--numthreads=*)    numthreads="${1#*=}";                     check_v 
"$1" "$numthreads";  shift;;
+    -N*)                    numthreads=$(echo "$1" | sed -e's/-N//'); check_v 
"$1" "$numthreads";  shift;;
 
     # Unrecognized option:
     -*) echo "$scriptname: unknown option '$1'"; exit 1;;
@@ -261,7 +271,11 @@ done
 
 
 
-# Necessary options.
+# Sanity checks
+# -------------
+#
+# The following options are necessary (the script will not work without
+# them).
 if [ x"$cat" = x ]; then
     echo "$scriptname: no argument (position catalog) given"; exit 1
 fi
@@ -278,12 +292,26 @@ fi
 
 
 
-# Set the temporary directory name and delete its contents if there is
-# any.
+
+# Temporary directory
+# -------------------
+#
+# The directory to keep temporary files.
 if [ x"$tmpdir" = x ]; then
-    tmpdir=dither-visualize-tmp
+    namecnosuffix=${cat%.*}
+    namecnodir=$(echo ${namecnosuffix##*/})
+    nameinosuffix=${img%.*}
+    nameinodir=$(echo ${nameinosuffix##*/})
+    namewidth=$(echo $width | sed 's|,|-|')
+    namecenter=$(echo $center | sed 's|,|-|')
+    tmpdir=dither-tmp-$namecnodir-$nameinodir-$namecenter-$namewidth
 fi
-if [ -d $tmpdir ]; then rm -f $tmpdir/*
+if [ -d $tmpdir ]; then
+    #################################
+    ######## FOR DEVELOPMENT ########
+    rm -f $tmpdir/*
+    #echo NOT CLEANING TMPDIR
+    #################################
 else                    mkdir $tmpdir
 fi
 
@@ -291,47 +319,83 @@ fi
 
 
 
-# Simulate the image for each dither pattern.
-counter=1
-elist=$tmpdir/exp-list.txt
-const=$tmpdir/constant.fits
-icopy=$tmpdir/input-copy.fits
-rm -f $elist
-if [ $widthinpix = 0 ]; then wip=""; else wip="--widthinpix"; fi
-asttable $cat -c$racol,$deccol \
-    | while read r d; do
+# Configuration file
+# ------------------
+#
+# All the settings given by the user and data are passed to Make through
+# this configuration file (and the variables within it).
+counter=1;
+config=$tmpdir/dither-simulate.conf
+ndither=$(astfits $cat --hdu=$hdu --keyvalue=NAXIS2 --quiet)
+echo "img = $img" > $config
+echo "width = $width" >> $config
+echo "center = $center" >> $config
+echo "output = $output" >> $config
+echo "imghdu = $imghdu" >> $config
+echo "dithers = $(seq 24 | tr '\n' ' ')" >> $config
+if [ $widthinpix = 1 ]; then
+    echo "widthinpix = --widthinpix" >> $config
+else
+    echo "widthinpix = " >> $config
+fi
+asttable $cat -c$racol,$deccol | while read r d; do
+
+    # Add the RA and Dec of the center of each pointing.
+    echo "$counter-ra  = $r" >> $config
+    echo "$counter-dec = $d" >> $config
+
+    # Increment the counter.
+    counter=$((counter+1))
+done
+
 
-        # Copy the input image HDU into the temporary directory for
-        # modification.
-        astfits $img --copy=$imghdu --output=$icopy
-       astfits $icopy --update=CRVAL1,$r --update=CRVAL2,$d
 
-       # Set all the pixel values to 1.
-       astarithmetic $icopy 1 uint8 constant --output=$const
-       rm $icopy
 
-       # Warp the image to the final pixel grid.
-       warped=$tmpdir/warped-$counter.fits
-       astwarp $const --center=$center --width=$width $wip \
-               --output=$warped
-       rm $const
 
-       # In this scenario, we do not care about the values, so set
-       # all the non-blank pixels to 1.
-       exp=$tmpdir/exp-$counter.fits
-       astarithmetic $warped isnotblank -o$exp
-       rm $warped
+# If the user hasn't set the number threads, find it
+# --------------------------------------------------
+#
+# Note that on different operating systems, the command to read the number
+# of threads differs: for example in GNU/Linux based systems, it is
+# 'nproc', but in macOS is is a special case of 'sysctl'.
+if [ x"$numthreads" = x0 ]; then
+    if type nproc > /dev/null 2> /dev/null; then
+        numthreads=$(nproc --all);
+    else
+        numthreads=$(sysctl -a | awk '/^hw\.ncpu/{print $2}')
+        if [ x"$numthreads" = x ]; then numthreads=1; fi
+    fi
+fi
+
+
+
+
+
+# Call the Makefile
+# -----------------
+#
+# Here, Makefile is invoked.
+if [ x$mksrc = x ]; then
+    mksrc=$installdir/dither-simulate.mk
+fi
+make -f $mksrc tmpdir=$tmpdir --jobs=$numthreads
+
+
+
+
+
+# Remove temporary directory
+# --------------------------
+#
+# If user does not specify to keep the build file with the option of
+# --keeptmp', then the directory will be removed.
+if [ x"$keeptmp" = x ]; then
+   rm -r $tmpdir
+fi
+
 
-       # Add this image to the list of images to add.
-       printf "%s " $exp >> $elist
 
-       # Increment the counter.
-        counter=$((counter+1))
-done
 
-# Stack all the exposure maps.
-astarithmetic $(cat $elist) $(wc $elist -w | awk '{print $1}') \
-             sum -g1 --output=$output
 
 # Delete the temporary directory if necessary
 if [ "$keeptmp" = 0 ]; then
diff --git a/bin/script/dither-simulate.mk b/bin/script/dither-simulate.mk
new file mode 100644
index 00000000..f7fcaa8e
--- /dev/null
+++ b/bin/script/dither-simulate.mk
@@ -0,0 +1,60 @@
+# Makefile to do the number-crunching of the 'dither-simulate.in' script.
+#
+# Copyright (C) 2023-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Include the parameters from the user.
+include $(tmpdir)/dither-simulate.conf
+
+# Final target.
+all: $(output)
+
+# Second expansion.
+.SECONDEXPANSION:
+
+# Stop the recipe's shell if the command fails
+.SHELLFLAGS = -ec
+
+
+
+
+# Build each separate exposure.
+exposures=$(foreach d,$(dithers),$(tmpdir)/exp-$(d).fits)
+$(exposures): $(tmpdir)/exp-%.fits: $(img) | $(tmpdir)
+
+#      Copy the input into a temporary one and edit its keywords to adjust
+#      to this pointings position.
+       copy=$(subst .fits,-copy.fits,$@); \
+       warped=$(subst .fits,-warped.fits,$@); \
+       const=$(subst .fits,-constant.fits,$@); \
+       astfits $(img) --copy=$(imghdu) --output=$$copy; \
+       astfits --update=CRVAL1,$($*-ra) $$copy \
+               --update=CRVAL2,$($*-dec); \
+       astarithmetic $$copy 1 uint8 constant --output=$$const; \
+       rm $$copy; \
+       astwarp $$const --center=$(center) --width=$(width) \
+               $(widthinpix) --output=$$warped; \
+       rm $$const; \
+       astarithmetic $$warped isnotblank -o$@; \
+       rm $$warped
+
+
+
+
+
+# Build the stack
+$(output): $(exposures)
+       astarithmetic $(exposures) $(words $(exposures)) \
+                     sum -g1 --output=$@
diff --git a/bin/script/zeropoint.in b/bin/script/zeropoint.in
index cc470c43..2a7f9ef5 100644
--- a/bin/script/zeropoint.in
+++ b/bin/script/zeropoint.in
@@ -329,7 +329,7 @@ do
         # Operating mode options.
         -N|--numthreads)        numthreads="$2";                               
check_v "$1" "$numthreads";  shift;shift;;
         -N=*|--numthreads=*)    numthreads="${1#*=}";                          
check_v "$1" "$numthreads";  shift;;
-        -N*)                    numthreads=$(echo "$1" | sed -e's/-j//');      
check_v "$1" "$numthreads";  shift;;
+        -N*)                    numthreads=$(echo "$1" | sed -e's/-N//');      
check_v "$1" "$numthreads";  shift;;
         -q|--quiet)             quiet=" -q"; shift;;
         -q*|--quiet=*)          on_off_option_error --quiet -q;;
         -?|--help)              print_help; exit 0;;
@@ -634,7 +634,6 @@ fi
 
 # Call the Makefile
 # -----------------
-
 #
 # Here, Makefile is invoked.
 if [ x$mksrc = x ]; then
diff --git a/bin/script/zeropoint.mk b/bin/script/zeropoint.mk
index 20c14685..575e0557 100644
--- a/bin/script/zeropoint.mk
+++ b/bin/script/zeropoint.mk
@@ -53,13 +53,6 @@ include $(tmpdir)/zeropoint.conf
 
 
 
-# Build the main directory for saving the outputs.
-$(tmpdir):; mkdir $@
-
-
-
-
-
 # Catalog of stars
 # ----------------
 #