[gnuastro-commits] master 6ef71c2: Query: when no --output specified, name includes DATASUM of data

[Mohammad Akhlaghi]

branch: master
commit 6ef71c2fd39f3f1bea850e93936ae70647185d3d
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    Query: when no --output specified, name includes DATASUM of data
    
    Unlike all the other Gnuastro programs, that have an input file, Query
    doesn't have any. So its output filename was simply 'query.fits'. But given
    that it also contacts an outside server (which may not be available later),
    having such a generic name (which can easily be over-written by a new call)
    could cause the loss of data and major frustration.
    
    Also, until now, the raw downloaded file was given to the user. However, we
    recently discovered that Gaia provides compressed FITS files, so a '.fits'
    name could be confusing (even though technically it has no problem within
    Gnuastro).
    
    To fix both problems, with this commit, the raw data file is actually
    written into a temporary named file (that is unique and will certainly not
    exist, thanks to the C library's 'mkstemp' function). That file is then
    read into memory and written into an output table. In the process, when no
    output name is provided, the 'DATASUM' of the downloaded data (part of the
    FITS standard) is calculated and written into the name of the output file.
    The extra processing steps will take negligible processing power compared
    to contacting a remote server, so they don't add any extra burden.
    
    The fact that we read the table and write it out, allows the produced
    tables from any database to have a similar/unified format for the
    users. For example, when the output is a FITS table in the zero-th/first
    extension, there will be keywords containing the values of all the given
    options to Query (so they can "reproduce" the result later: "reproduce"
    because something may change on the server, but that is beyond our
    control).
    
    This was suggested by Francois Ochsenbein.
---
 NEWS                             |  2 +
 bin/fits/fits.c                  | 17 +-------
 bin/query/args.h                 |  3 +-
 bin/query/gaia.c                 |  5 ++-
 bin/query/main.h                 |  5 +++
 bin/query/query.c                | 56 +++++++++++++++++++++++---
 bin/query/ui.c                   | 85 +++++++++++++---------------------------
 doc/gnuastro.texi                | 24 ++++++++++--
 lib/checkset.c                   | 82 ++++++++++++++++++++++++++++++++++++++
 lib/fits.c                       | 43 ++++++++++++++++++++
 lib/gnuastro-internal/checkset.h |  6 +++
 lib/gnuastro/fits.h              |  6 +++
 12 files changed, 249 insertions(+), 85 deletions(-)

diff --git a/NEWS b/NEWS
index fd21a72..1bd79f5 100644
--- a/NEWS
+++ b/NEWS
@@ -104,6 +104,8 @@ See the end of the file for license conditions.
    - GAL_ARITHMETIC_OP_MAKENEW: new 'makenew' operator.
    - gal_blank_flag_remove: Remove all flagged elements in a dataset.
    - gal_blank_remove_rows: remove all rows that have at least one blank.
+   - gal_fits_hdu_datasum: calculate DATASUM of given HDU in given FITS file.
+   - gal_fits_hdu_datasum_ptr: calculate DATASUM of opened FITS file pointer.
    - gal_wcs_coverage: Return the sky coverage of given image HDU.
    - gal_wcs_dimension_name: return the name of the requested WCS dim.
 
diff --git a/bin/fits/fits.c b/bin/fits/fits.c
index 2a6e386..e91adf9 100644
--- a/bin/fits/fits.c
+++ b/bin/fits/fits.c
@@ -281,22 +281,7 @@ fits_hdu_number(struct fitsparams *p)
 static void
 fits_datasum(struct fitsparams *p)
 {
-  int status=0;
-  fitsfile *fptr;
-  unsigned long datasum, hdusum;
-
-  /* Read the desired extension (necessary for reading the rest). */
-  fptr=gal_fits_hdu_open(p->filename, p->cp.hdu, READONLY);
-
-  /* Calculate the checksum and datasum of the opened HDU. */
-  fits_get_chksum(fptr, &datasum, &hdusum, &status);
-  gal_fits_io_error(status, "estimating datasum");
-
-  /* Close the file. */
-  fits_close_file(fptr, &status);
-
-  /* Print the datasum */
-  printf("%ld\n", datasum);
+  printf("%ld\n", gal_fits_hdu_datasum(p->filename, p->cp.hdu));
 }
 
 
diff --git a/bin/query/args.h b/bin/query/args.h
index b18a02e..15023f7 100644
--- a/bin/query/args.h
+++ b/bin/query/args.h
@@ -39,12 +39,11 @@ struct argp_option program_options[] =
       0,
       "Name of database (e.g., 'esa').",
       GAL_OPTIONS_GROUP_INPUT,
-      &p->database,
+      &p->databasestr,
       GAL_TYPE_STRING,
       GAL_OPTIONS_RANGE_ANY,
       GAL_OPTIONS_NOT_MANDATORY,
       GAL_OPTIONS_NOT_SET,
-      ui_parse_database
     },
     {
       "query",
diff --git a/bin/query/gaia.c b/bin/query/gaia.c
index d07407a..5142dce 100644
--- a/bin/query/gaia.c
+++ b/bin/query/gaia.c
@@ -174,7 +174,7 @@ gaia_query(struct queryparams *p)
   /* Build the calling command. */
   if( asprintf(&command, "curl -o%s --form LANG=ADQL --form FORMAT=fits "
                "--form REQUEST=doQuery --form QUERY=\"%s\" "
-               "https://gea.esac.esa.int/tap-server/tap/sync";, p->cp.output,
+               "https://gea.esac.esa.int/tap-server/tap/sync";, p->downloadname,
                querystr)<0 )
     error(EXIT_FAILURE, 0, "%s: asprintf allocation ('command')", __func__);
 
@@ -189,4 +189,7 @@ gaia_query(struct queryparams *p)
           p->cp.quiet==0 ? "printed above " : "",
           p->cp.quiet==0 ? "" : " (the command can be printed "
           "if you don't use the option '--quiet', or '-q')");
+
+  /* Clean up. */
+  free(command);
 }
diff --git a/bin/query/main.h b/bin/query/main.h
index f716ea3..941d3e9 100644
--- a/bin/query/main.h
+++ b/bin/query/main.h
@@ -54,6 +54,11 @@ struct queryparams
   char                  *query;  /* Raw query string.                  */
   gal_list_str_t      *columns;  /* Columns to extract from database.  */
 
+  /* Internal variables. */
+  char            *databasestr;  /* Name of input database.            */
+  char           *downloadname;  /* Temporary output name.             */
+  char          *processedname;  /* Temporary output name.             */
+
   /* Output: */
   time_t               rawtime;  /* Starting time of the program.      */
 };
diff --git a/bin/query/query.c b/bin/query/query.c
index c4ecc10..f04d2a9 100644
--- a/bin/query/query.c
+++ b/bin/query/query.c
@@ -46,25 +46,71 @@ query_check_download(struct queryparams *p)
   int status=0;
   char *logname;
   fitsfile *fptr;
+  gal_data_t *table;
+  unsigned long datasum;
 
   /* Open the FITS file and if the status value is still zero, it means
      everything worked properly. */
-  fits_open_file(&fptr, p->cp.output, READONLY, &status);
-  if(status==0) fits_close_file(fptr, &status);
+  fits_open_file(&fptr, p->downloadname, READONLY, &status);
+  if(status==0)
+    {
+      /* Close the FITS file pointer. */
+      fits_close_file(fptr, &status);
+
+      /* Read the table and write it into a clean output (in case the
+         downloaded table is compressed in any special FITS way). */
+      table=gal_table_read(p->downloadname, "1", NULL, NULL,
+                           GAL_TABLE_SEARCH_NAME, 1, p->cp.minmapsize,
+                           p->cp.quietmmap, NULL);
+      gal_table_write(table, NULL, NULL, p->cp.tableformat,
+                      p->cp.output ? p->cp.output : p->processedname,
+                      "QUERY", 0);
+
+      /* Delete the raw downloaded file. */
+      remove(p->downloadname);
+      free(p->downloadname);
+
+      /* If no output name was specified, calculate the 'datasum' of the
+         table and put that after the file name. */
+      if(p->cp.output==NULL)
+        {
+          /* Calculate the extension's datasum. */
+          datasum=gal_fits_hdu_datasum(p->processedname, "1");
+
+          /* Allocate the output name. */
+          if( asprintf(&p->cp.output, "%s-%lu.fits", p->databasestr, 
datasum)<0 )
+            error(EXIT_FAILURE, 0, "%s: asprintf allocation", __func__);
+
+          /* Make sure the desired output name doesn't exist. */
+          gal_checkset_writable_remove(p->cp.output, p->cp.keep,
+                                       p->cp.dontdelete);
+
+          /* Rename the processed name to the desired output. */
+          rename(p->processedname, p->cp.output);
+          free(p->processedname);
+        }
+    }
   else
     {
       /* Add a '.log' suffix to the output filename. */
-      len=strlen(p->cp.output);
+      len=strlen(p->downloadname);
       logname=gal_pointer_allocate(GAL_TYPE_STRING, len+10, 1,
                                    __func__, "logname");
-      sprintf(logname, "%s.log", p->cp.output);
+      sprintf(logname, "%s.log", p->downloadname);
 
       /* Rename the output file to the logname file and let the user
          know. */
-      rename(p->cp.output, logname);
+      rename(p->downloadname, logname);
       error(EXIT_FAILURE, 0, "the requested dataset could not be retrieved! "
             "For more, please see '%s'", logname);
     }
+
+  /* Add the query keywords to the first extension (if the output was a
+     FITS file). */
+  if( gal_fits_name_is_fits (p->cp.output) )
+    gal_fits_key_write_config(&p->cp.okeys, "Query settings",
+                              "QUERY-CONFIG", p->cp.output, "0");
+
 }
 
 
diff --git a/bin/query/ui.c b/bin/query/ui.c
index 7bce4de..ead6968 100644
--- a/bin/query/ui.c
+++ b/bin/query/ui.c
@@ -124,7 +124,6 @@ ui_initialize_options(struct queryparams *p,
         case GAL_OPTIONS_KEY_IGNORECASE:
         case GAL_OPTIONS_KEY_NUMTHREADS:
         case GAL_OPTIONS_KEY_MINMAPSIZE:
-        case GAL_OPTIONS_KEY_TABLEFORMAT:
         case GAL_OPTIONS_KEY_STDINTIMEOUT:
         case GAL_OPTIONS_KEY_KEEPINPUTDIR:
           cp->coptions[i].flags=OPTION_HIDDEN;
@@ -189,52 +188,6 @@ parse_opt(int key, char *arg, struct argp_state *state)
 
 
 
-/* Parse the database. */
-void *
-ui_parse_database(struct argp_option *option, char *arg,
-                  char *filename, size_t lineno, void *junk)
-{
-  char *str;
-  int database;
-
-  /* We want to print the stored values. */
-  if(lineno==-1)
-    {
-      /* Based on the value, write the database. */
-      database=*(int *)(option->value);
-      switch(database)
-        {
-        case QUERY_DATABASE_GAIA:
-          gal_checkset_allocate_copy("gaia", &str);
-          break;
-        default:
-          error(EXIT_FAILURE, 0, "%s: a bug! Please contact us at %s to "
-                "address the problem. The code %d isn't recognized for "
-                "'p->database'", __func__, PACKAGE_BUGREPORT, database);
-        }
-      return str;
-    }
-  else
-    {
-      /* Convert the given string into a code. */
-      if( !strcmp(arg, "gaia") ) database=QUERY_DATABASE_GAIA;
-      else
-        error(EXIT_FAILURE, 0, "'%s' is not a recognized database.\n\n"
-              "For the full list of recognized databases, please see the "
-              "documentation (with the command 'info astquery')", arg);
-
-      /* Write the database code into the option. */
-      *(int *)(option->value)=database;
-
-      /* Our job is done, return NULL. */
-      return NULL;
-    }
-}
-
-
-
-
-
 char *
 ui_strlist_to_str(gal_list_str_t *input)
 {
@@ -300,12 +253,20 @@ ui_read_check_only_options(struct queryparams *p)
   gal_data_t *tmp;
 
   /* See if database has been specified. */
-  if(p->database==0)
+  if(p->databasestr==NULL)
     error(EXIT_FAILURE, 0, "no input dataset.\n\n"
           "Please use the '--database' ('-d') option to specify your "
           "desired database, see manual ('info gnuastro astquery' "
           "command) for the current databases");
 
+  /* Convert the given string into a code. */
+  if( !strcmp(p->databasestr, "gaia") ) p->database=QUERY_DATABASE_GAIA;
+  else
+    error(EXIT_FAILURE, 0, "'%s' is not a recognized database.\n\n"
+          "For the full list of recognized databases, please see the "
+          "documentation (with the command 'info astquery')", p->databasestr);
+
+
   /* Make sure that '--query' and '--center' are not called together. */
   if(p->query && (p->center || p->overlapwith) )
     error(EXIT_FAILURE, 0, "the '--query' option cannot be called together "
@@ -367,16 +328,20 @@ ui_read_check_only_options(struct queryparams *p)
                 "be negative");
     }
 
-  /* If an output name isn't given, set one. */
-  if(p->cp.output)
-    {
-      /* Make sure its a FITS table. */
-      if( gal_fits_name_is_fits(p->cp.output)==0 )
-        error(EXIT_FAILURE, 0, "'%s' is not a FITS file. Currently only "
-              "FITS file outputs are supported", p->cp.output);
-    }
-  else
-    p->cp.output=gal_checkset_automatic_output(&p->cp, "./query.fits", 
".fits");
+  /* Make sure that the output name is in a writable location and that it
+     doesn't exist. If it exists, and the user hasn't called
+     '--dontdelete', then delete the existing file. */
+  gal_checkset_writable_remove(p->cp.output, p->cp.keep,
+                               p->cp.dontdelete);
+
+  /* Set the name for the downloaded and processed files. These are due to
+     an internal low-level processing that will be done on the raw
+     downloaded file. */
+  p->processedname=gal_checkset_make_unique_suffix(p->cp.output
+                                                   ? p->cp.output
+                                                   : "query.fits",
+                                                   ".fits");
+  p->downloadname=gal_checkset_make_unique_suffix(p->processedname, NULL);
 }
 
 
@@ -484,6 +449,10 @@ ui_read_check_inputs_setup(int argc, char *argv[], struct 
queryparams *p)
   gal_options_print_state(&p->cp);
 
 
+  /* Prepare all the options as FITS keywords to write in output later. */
+  gal_options_as_fits_keywords(&p->cp);
+
+
   /* Check that the options and arguments fit well with each other. Note
      that arguments don't go in a configuration file. So this test should
      be done after (possibly) printing the option values. */
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index 27ba32e..4a33e0b 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -8511,7 +8511,7 @@ Calculate and print the given HDU's "datasum" to stdout.
 The given HDU is specified with the @option{--hdu} (or @option{-h}) option.
 This number is calculated by parsing all the bytes of the given HDU's data 
records (excluding keywords).
 This option ignores any possibly existing @code{DATASUM} keyword in the HDU.
-For more on the datasum feature of the FITS standard, see @ref{Keyword 
manipulation} (under the @code{checksum} component of @option{--write}).
+For more on @code{DATASUM} in the FITS standard, see @ref{Keyword 
manipulation} (under the @code{checksum} component of @option{--write}).
 
 You can use this option to confirm that the data in two different HDUs 
(possibly with different keywords) is identical.
 Its advantage over @option{--write=datasum} (which writes the @code{DATASUM} 
keyword into the given HDU) is that it doesn't require write permissions.
@@ -10184,13 +10184,19 @@ $ astquery --database=gaia --query="XXXX YYYY" 
--output=my-gaia.fits
 
 Query doesn't take any input argument, because the main goal is to retreive 
data from external sources.
 The main input to Query is the @option{--database} option which specifies 
which database should be contacted for submitting the query.
-The name of the downloaded output file can optionally be set with 
@option{--output}.
-If @option{--output} is not set, an automatically generated name will be used.
+The name of the downloaded output file can optionally be set with 
@option{--output} (see below for when @option{--output} is not specified).
+Like all Gnuastro programs, if the output is a FITS file, the zero-th/first 
HDU of the output will contain all the command-line options given to Query.
 
 There are two methods to query the database: 1) with @option{--query} you can 
directly give a raw query statement that is recognized by the database, 2) with 
the @option{--center} and @option{--radius}, the low-level query will 
constructed automatically for the particular database.
 The former is very low level and will require some knowledge of the database's 
query language, but of course, it is much more powerful.
 The latter is much more limited in terms of capabilities (the only constraint 
is the location of the objects compared to the given center), but doesn't 
require any knowledge of the database's query language.
 
+@cindex @code{DATASUM}: FITS keyword
+If @option{--output} is not set, the output name will be in the format of 
@file{STRING-NUMBER.fits}, where @file{STRING} is the name of the database, and 
@file{NUMBER} is the @code{DATASUM} of the downloaded table, which will be 
unique for a given table's data (For more on @code{DATASUM} in the FITS 
standard, see @ref{Keyword manipulation}, under the @code{checksum} component 
of @option{--write}).
+With this feature, a second run of @command{astquery} that isn't called with 
@option{--output}, and downloads the same final table data from the same 
database will have the same output name.
+However, a query (that is again not called with @option{--output}) which 
results in a different downloaded table (even differing by a single number) 
will have a different output name to avoid overriding a previously downloaded 
dataset.
+Generally, when calling Query more than once, it is recommended to set an 
output name for each call based on your project's context.
+
 @cartouche
 @noindent
 @strong{Under development, request for feedback:} Query is a new member of the 
Gnuastro family of programs.
@@ -22805,6 +22811,18 @@ extension will be returned.
 Return the number of HDUs/extensions in @file{filename}.
 @end deftypefun
 
+@deftypefun {unsigned long} gal_fits_hdu_datasum (char @code{*filename}, char 
@code{*hdu})
+@cindex @code{DATASUM}: FITS keyword
+Return the @code{DATASUM} of the given HDU in the given FITS file.
+For more on @code{DATASUM} in the FITS standard, see @ref{Keyword 
manipulation} (under the @code{checksum} component of @option{--write}).
+@end deftypefun
+
+@deftypefun {unsigned long} gal_fits_hdu_datasum_ptr (char @code{*fptr})
+@cindex @code{DATASUM}: FITS keyword
+Return the @code{DATASUM} of the already opened HDU in @code{fptr}.
+For more on @code{DATASUM} in the FITS standard, see @ref{Keyword 
manipulation} (under the @code{checksum} component of @option{--write}).
+@end deftypefun
+
 @deftypefun int gal_fits_hdu_format (char @code{*filename}, char @code{*hdu})
 Return the format of the HDU as one of CFITSIO's recognized macros:
 @code{IMAGE_HDU}, @code{ASCII_TBL}, or @code{BINARY_TBL}.
diff --git a/lib/checkset.c b/lib/checkset.c
index a5069f6..4891e6c 100644
--- a/lib/checkset.c
+++ b/lib/checkset.c
@@ -292,6 +292,88 @@ gal_checkset_not_dir_part(char *filename)
 
 
 
+/* Make an allocated copy of the input string, then remove the suffix from
+   that string. */
+char *
+gal_checkset_suffix_separate(char *name, char **outsuffix)
+{
+  char *c, *out=NULL, *suffix=NULL;
+
+  /* Make a copy of the input. */
+  gal_checkset_allocate_copy(name, &out);
+
+  /* Parse the string from the end and stop when we hit a '.'. */
+  c=out+strlen(out)-1;
+  while(c!=out)
+    {
+      /* As soon as we hit the first '.' take a copy of the string after it
+         and put it in 'suffix'. */
+      if(*c=='.')
+        {
+          gal_checkset_allocate_copy(c, &suffix);
+          *c='\0';
+          break;
+        }
+      --c;
+    }
+
+  /* Put the 'suffix' in the output pointer and return the string with no
+     suffix. */
+  *outsuffix=suffix;
+  return out;
+}
+
+
+
+
+
+/* Given a reference filename, add a format of AAAAA-XXXXXX.CCCC where
+   'AAAAA' is the base name of the 'reference' argument, 'XXXXX' is a
+   random/unique sequence of characters, and 'YYYYY' is the string given to
+   'suffix'. If 'suffix' is NULL, the suffix of 'reference' will be used.*/
+char *
+gal_checkset_make_unique_suffix(char *reference, char *suffix)
+{
+  int tmpnamefile;
+  char *nosuff, *tmpname;
+  char *out=NULL, *insuffix;
+
+  /* Remove the suffix. */
+  nosuff=gal_checkset_suffix_separate(reference, &insuffix);
+
+  /* First generate the input to 'mkstemp' (the 'XXXXXX's will be replaced
+     with a unique set of strings with same number of characters). */
+  if( asprintf(&tmpname, "%s-XXXXXX", nosuff)<0 )
+    error(EXIT_FAILURE, 0, "%s: asprintf allocation", __func__);
+
+  /* Generate the unique name with 'mkstemp', but it will actually open the
+     file (to make sure that the name is not used), so we need to close it
+     afterwards. */
+  tmpnamefile=mkstemp(tmpname);
+  errno=0;
+  if( close(tmpnamefile) != 0 )
+    error(EXIT_FAILURE, errno, "couldn't close temporary file");
+
+  /* Delete the temporarily created file. */
+  remove(tmpname);
+
+  /* Add the suffix. */
+  out = ( suffix
+          ? gal_checkset_malloc_cat(tmpname, suffix)
+          : ( insuffix
+              ? gal_checkset_malloc_cat(tmpname, insuffix)
+              : tmpname ) );
+
+  /* Clean up and return the output. */
+  if(tmpname!=out) free(tmpname);
+  if(insuffix) free(insuffix);
+  free(nosuff);
+  return out;
+}
+
+
+
+
 /* Check if a file exists and report if it doesn't. */
 void
 gal_checkset_check_file(char *filename)
diff --git a/lib/fits.c b/lib/fits.c
index 36eaa28..cccfd32 100644
--- a/lib/fits.c
+++ b/lib/fits.c
@@ -649,6 +649,49 @@ gal_fits_hdu_num(char *filename)
 
 
 
+/* Calculate the datasum of the given HDU in the given file. */
+unsigned long
+gal_fits_hdu_datasum(char *filename, char *hdu)
+{
+  int status=0;
+  fitsfile *fptr;
+  unsigned long datasum;
+
+  /* Read the desired extension (necessary for reading the rest). */
+  fptr=gal_fits_hdu_open(filename, hdu, READONLY);
+
+  /* Calculate the datasum. */
+  datasum=gal_fits_hdu_datasum_ptr(fptr);
+
+  /* Close the file and return. */
+  fits_close_file(fptr, &status);
+  gal_fits_io_error(status, "closing file");
+  return datasum;
+}
+
+
+
+
+
+/* Calculate the FITS standard datasum for the opened FITS pointer. */
+unsigned long
+gal_fits_hdu_datasum_ptr(fitsfile *fptr)
+{
+  int status=0;
+  unsigned long datasum, hdusum;
+
+  /* Calculate the checksum and datasum of the opened HDU. */
+  fits_get_chksum(fptr, &datasum, &hdusum, &status);
+  gal_fits_io_error(status, "estimating datasum");
+
+  /* Return the datasum. */
+  return datasum;
+}
+
+
+
+
+
 /* Given the filename and HDU, this function will return the CFITSIO code
    for the type of data it contains (table, or image). The CFITSIO codes
    are:
diff --git a/lib/gnuastro-internal/checkset.h b/lib/gnuastro-internal/checkset.h
index 0960815..9ad3f2e 100644
--- a/lib/gnuastro-internal/checkset.h
+++ b/lib/gnuastro-internal/checkset.h
@@ -98,6 +98,12 @@ gal_checkset_dir_part(char *filename);
 char *
 gal_checkset_not_dir_part(char *filename);
 
+char *
+gal_checkset_suffix_separate(char *name, char **suffix);
+
+char *
+gal_checkset_make_unique_suffix(char *reference, char *suffix);
+
 void
 gal_checkset_check_file(char *filename);
 
diff --git a/lib/gnuastro/fits.h b/lib/gnuastro/fits.h
index 5794296..9032db9 100644
--- a/lib/gnuastro/fits.h
+++ b/lib/gnuastro/fits.h
@@ -154,6 +154,12 @@ gal_fits_open_to_write(char *filename);
 size_t
 gal_fits_hdu_num(char *filename);
 
+unsigned long
+gal_fits_hdu_datasum(char *filename, char *hdu);
+
+unsigned long
+gal_fits_hdu_datasum_ptr(fitsfile *fptr);
+
 int
 gal_fits_hdu_format(char *filename, char *hdu);