[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gnuastro-commits] master 99ab4e9: Library and book: memory management b
|
From: |
Mohammad Akhlaghi |
|
Subject: |
[gnuastro-commits] master 99ab4e9: Library and book: memory management by checking available RAM |
|
Date: |
Tue, 24 Nov 2020 18:57:13 -0500 (EST) |
branch: master
commit 99ab4e98458fe1069e5a1717dc36697d0546d12b
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Library and book: memory management by checking available RAM
Until recently, the main way to manage the memory was to set
'--minmapsize', but for that the user needed to know/worry-about the size
of the arrays. So a feature was added to read the available space on RAM
before every allocation. However, in Linux, there is no system call for
this, so we have to read from a file and this made some programs (which
just need to allocate very small arrays) very slow.
With this commit, things have been cleaned up into modular functions (some
of them new) to help in the management. Now, the available memory is only
checked when we array to allocate is larger than 10MB. Also the "Memory
management" section of the book has been updated to be more clear.
---
NEWS | 7 +
bin/arithmetic/arithmetic.c | 3 +-
bin/convolve/convolve.c | 23 ++-
bin/crop/crop.c | 10 +-
bin/mkcatalog/mkcatalog.c | 3 +-
bin/mkprof/mkprof.c | 11 +-
bin/noisechisel/detection.c | 6 +-
bin/noisechisel/sky.c | 3 +-
bin/noisechisel/threshold.c | 9 +-
bin/segment/clumps.c | 6 +-
bin/segment/segment.c | 6 +-
bin/statistics/sky.c | 3 +-
bin/warp/warp.c | 11 +-
doc/gnuastro.texi | 383 ++++++++++++++++++++-------------------
lib/arithmetic.c | 4 +-
lib/checkset.c | 190 +++++++++++++++++++
lib/convolve.c | 3 +-
lib/data.c | 199 +-------------------
lib/fits.c | 3 +-
lib/gnuastro-internal/checkset.h | 7 +
lib/gnuastro/pointer.h | 9 +-
lib/gnuastro/threads.h | 6 +-
lib/interpolate.c | 3 +-
lib/pointer.c | 75 +++++++-
lib/threads.c | 31 +++-
lib/tile.c | 3 +-
tests/lib/multithread.c | 8 +-
27 files changed, 598 insertions(+), 427 deletions(-)
diff --git a/NEWS b/NEWS
index 8623e68..6666289 100644
--- a/NEWS
+++ b/NEWS
@@ -106,6 +106,9 @@ See the end of the file for license conditions.
- gal_blank_remove_rows: remove all rows that have at least one blank.
- gal_fits_hdu_datasum: calculate DATASUM of given HDU in given FITS file.
- gal_fits_hdu_datasum_ptr: calculate DATASUM of opened FITS file pointer.
+ - gal_pointer_allocate_ram_or_mmap: allocate space either in RAM or as a
+ memory-mapped file.
+ - gal_pointer_mmap_free: "free" (actually delete) the memory-mapped file.
- gal_wcs_coverage: Return the sky coverage of given image HDU.
- gal_wcs_dimension_name: return the name of the requested WCS dim.
@@ -136,6 +139,10 @@ See the end of the file for license conditions.
delimiters ('_:_:_').
Library:
+ - gal_pointer_mmap_allocate: new name for 'gal_pointer_allocate_mmap'.
+ - gal_threads_dist_in_threads: now accounts for billions of threads,
+ thus includes memory management options.
+ - gal_threads_spin_off: now accounts for memory management.
- gal_units_degree_to_ra: new 'usecolon' argument to optionally format
output string with colons as delimiters ('_:_:_'). When this option is
zero, the string will be in the '_h_m_s' format.
diff --git a/bin/arithmetic/arithmetic.c b/bin/arithmetic/arithmetic.c
index 1277153..e69066a 100644
--- a/bin/arithmetic/arithmetic.c
+++ b/bin/arithmetic/arithmetic.c
@@ -512,7 +512,8 @@ wrapper_for_filter(struct arithmeticparams *p, char *token,
int operator)
/* Spin off threads for each pixel. */
gal_threads_spin_off(arithmetic_filter, &afp, afp.input->size,
- p->cp.numthreads);
+ p->cp.numthreads, p->cp.minmapsize,
+ p->cp.quietmmap);
}
diff --git a/bin/convolve/convolve.c b/bin/convolve/convolve.c
index 2804ff1..d2b71d7 100644
--- a/bin/convolve/convolve.c
+++ b/bin/convolve/convolve.c
@@ -520,6 +520,7 @@ twodimensionfft(struct convolveparams *p, struct
fftonthreadparams *fp,
{
int err;
pthread_t t; /* All thread ids saved in this, not used. */
+ char *mmapname=NULL;
pthread_attr_t attr;
pthread_barrier_t b;
size_t i, nb, *indexs, thrdcols;
@@ -539,7 +540,10 @@ twodimensionfft(struct convolveparams *p, struct
fftonthreadparams *fp,
/* ==================== */
/* 1D FFT on each row. */
/* ==================== */
- gal_threads_dist_in_threads(multiple*p->ps0, nt, &indexs, &thrdcols);
+ mmapname=gal_threads_dist_in_threads(multiple*p->ps0, nt,
+ p->input->minmapsize,
+ p->cp.quietmmap,
+ &indexs, &thrdcols);
if(nt==1)
{
fp[0].stride=1;
@@ -577,15 +581,21 @@ twodimensionfft(struct convolveparams *p, struct
fftonthreadparams *fp,
pthread_attr_destroy(&attr);
pthread_barrier_destroy(&b);
}
- free(indexs);
+
+ /* Clean up. */
+ if(mmapname) gal_pointer_mmap_free(&mmapname, p->cp.quietmmap);
+ else free(indexs);
/* ====================== */
/* 1D FFT on each column. */
/* ====================== */
- /* No comments, exact duplicate, except the p->ps1s! */
- gal_threads_dist_in_threads(multiple*p->ps1, nt, &indexs, &thrdcols);
+ /* No comments, exact duplicate of above, except the p->ps1s! */
+ mmapname=gal_threads_dist_in_threads(multiple*p->ps1, nt,
+ p->input->minmapsize,
+ p->cp.quietmmap,
+ &indexs, &thrdcols);
if(nt==1)
{
fp[0].stride=p->ps1;
@@ -614,7 +624,10 @@ twodimensionfft(struct convolveparams *p, struct
fftonthreadparams *fp,
pthread_attr_destroy(&attr);
pthread_barrier_destroy(&b);
}
- free(indexs);
+
+ /* Clean up, note that 'indexs' may be memory-mapped. */
+ if(mmapname) gal_pointer_mmap_free(&mmapname, p->cp.quietmmap);
+ else free(indexs);
}
diff --git a/bin/crop/crop.c b/bin/crop/crop.c
index 7e9444c..902cfbc 100644
--- a/bin/crop/crop.c
+++ b/bin/crop/crop.c
@@ -31,6 +31,7 @@ along with Gnuastro. If not, see
<http://www.gnu.org/licenses/>.
#include <gnuastro/fits.h>
#include <gnuastro/threads.h>
+#include <gnuastro/pointer.h>
#include <gnuastro-internal/timing.h>
#include <gnuastro-internal/checkset.h>
@@ -404,6 +405,7 @@ crop(struct cropparams *p)
int err=0;
char *tmp;
pthread_t t; /* We don't use the thread id, so all are saved here. */
+ char *mmapname;
pthread_attr_t attr;
pthread_barrier_t b;
struct onecropparams *crp;
@@ -428,8 +430,9 @@ crop(struct cropparams *p)
/* Distribute the indexs into the threads (for clarity, this is needed
even if we only have one object). */
- gal_threads_dist_in_threads(p->catname ? p->numout : 1, nt,
- &indexs, &thrdcols);
+ mmapname=gal_threads_dist_in_threads(p->catname ? p->numout : 1, nt,
+ p->cp.minmapsize, p->cp.quietmmap,
+ &indexs, &thrdcols);
/* Run the job, if there is only one thread, don't go through the
@@ -487,7 +490,8 @@ crop(struct cropparams *p)
}
/* Print the final verbose info, save log, and clean up: */
+ if(mmapname) gal_pointer_mmap_free(&mmapname, p->cp.quietmmap);
+ else free(indexs);
crop_verbose_final(p);
- free(indexs);
free(crp);
}
diff --git a/bin/mkcatalog/mkcatalog.c b/bin/mkcatalog/mkcatalog.c
index 8d42dc7..22b628c 100644
--- a/bin/mkcatalog/mkcatalog.c
+++ b/bin/mkcatalog/mkcatalog.c
@@ -796,7 +796,8 @@ mkcatalog(struct mkcatalogparams *p)
/* Do the processing on each thread. */
gal_threads_spin_off(mkcatalog_single_object, p, p->numobjects,
- p->cp.numthreads);
+ p->cp.numthreads, p->cp.minmapsize,
+ p->cp.quietmmap);
/* Post-thread processing, for example to convert image coordinates to RA
and Dec. */
diff --git a/bin/mkprof/mkprof.c b/bin/mkprof/mkprof.c
index 12289c9..e2a1e86 100644
--- a/bin/mkprof/mkprof.c
+++ b/bin/mkprof/mkprof.c
@@ -747,12 +747,12 @@ mkprof_write(struct mkprofparams *p)
void
mkprof(struct mkprofparams *p)
{
- char *tmp;
pthread_t t; /* Thread id not used, all are saved here. */
pthread_attr_t attr;
pthread_barrier_t b;
size_t numforprint=50;
struct mkonthread *mkp;
+ char *tmp, *mmapname=NULL;
gal_list_str_t *comments=NULL;
int err, origquiet=p->cp.quiet;
size_t i, fi, *indexs, thrdcols;
@@ -772,7 +772,9 @@ mkprof(struct mkprofparams *p)
/* Distribute the different profiles for different threads. Note
that one thread is left out for writing, while nt-1 are left
for building. */
- gal_threads_dist_in_threads(p->num, nt, &indexs, &thrdcols);
+ mmapname=gal_threads_dist_in_threads(p->num, nt,
+ p->cp.minmapsize, p->cp.quietmmap,
+ &indexs, &thrdcols);
/* 'onaxes' are size of the merged output image without over-sampling or
@@ -885,7 +887,8 @@ mkprof(struct mkprofparams *p)
printf(" -- Output: %s\n", p->mergedimgname);
/* Clean up. */
- free(mkp);
- free(indexs);
+ if(mmapname) gal_pointer_mmap_free(&mmapname, p->cp.quietmmap);
+ else free(indexs);
if(onaxes) free(onaxes);
+ free(mkp);
}
diff --git a/bin/noisechisel/detection.c b/bin/noisechisel/detection.c
index a15061d..631d642 100644
--- a/bin/noisechisel/detection.c
+++ b/bin/noisechisel/detection.c
@@ -373,7 +373,8 @@ detection_pseudo_find(struct noisechiselparams *p,
gal_data_t *workbin,
/* Do the respective step. */
gal_threads_spin_off(detection_fill_holes_open, &fho_prm,
- p->ltl.tottiles, p->cp.numthreads);
+ p->ltl.tottiles, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
/* Reset the blank values (if they were changed). */
if( p->blankasforeground==0 && gal_blank_present(p->input,0) )
@@ -424,7 +425,8 @@ detection_pseudo_find(struct noisechiselparams *p,
gal_data_t *workbin,
}
else
gal_threads_spin_off(detection_fill_holes_open, &fho_prm,
- p->ltl.tottiles, p->cp.numthreads);
+ p->ltl.tottiles, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
/* Clean up. */
free(fho_prm.copyspace);
diff --git a/bin/noisechisel/sky.c b/bin/noisechisel/sky.c
index ff336b2..f200969 100644
--- a/bin/noisechisel/sky.c
+++ b/bin/noisechisel/sky.c
@@ -207,7 +207,8 @@ sky_and_std(struct noisechiselparams *p, char *checkname)
/* Find the Sky and its STD on proper tiles. */
gal_threads_spin_off(sky_mean_std_undetected, p, tl->tottiles,
- cp->numthreads);
+ cp->numthreads, p->cp.minmapsize,
+ p->cp.quietmmap);
if(checkname)
{
p->sky->name="SKY";
diff --git a/bin/noisechisel/threshold.c b/bin/noisechisel/threshold.c
index f318d47..56159a4 100644
--- a/bin/noisechisel/threshold.c
+++ b/bin/noisechisel/threshold.c
@@ -162,8 +162,9 @@ threshold_apply(struct noisechiselparams *p, float *value1,
float *value2, int kind)
{
struct threshold_apply_p taprm={value1, value2, kind, p};
- gal_threads_spin_off(threshold_apply_on_thread, &taprm, p->cp.tl.tottiles,
- p->cp.numthreads);
+ gal_threads_spin_off(threshold_apply_on_thread, &taprm,
+ p->cp.tl.tottiles, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
}
@@ -631,7 +632,9 @@ threshold_quantile_find_apply(struct noisechiselparams *p)
elements, it is only necessary to check one (with the 'updateflag'
value set to 1), then update the next. */
qprm.p=p;
- gal_threads_spin_off(qthresh_on_tile, &qprm, tl->tottiles, cp->numthreads);
+ gal_threads_spin_off(qthresh_on_tile, &qprm, tl->tottiles,
+ cp->numthreads, cp->minmapsize,
+ cp->quietmmap);
free(qprm.usage);
if( gal_blank_present(qprm.erode_th, 1) )
{
diff --git a/bin/segment/clumps.c b/bin/segment/clumps.c
index d860696..a62f4f1 100644
--- a/bin/segment/clumps.c
+++ b/bin/segment/clumps.c
@@ -626,7 +626,8 @@ clumps_true_find_sn_thresh(struct segmentparams *p)
/* Do this step. */
gal_threads_spin_off(clumps_find_make_sn_table, &clprm,
- p->ltl.tottiles, p->cp.numthreads);
+ p->ltl.tottiles, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
/* Set the extension name. */
switch(clprm.step)
@@ -658,7 +659,8 @@ clumps_true_find_sn_thresh(struct segmentparams *p)
{
clprm.step=0;
gal_threads_spin_off(clumps_find_make_sn_table, &clprm,
- p->ltl.tottiles, p->cp.numthreads);
+ p->ltl.tottiles, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
}
diff --git a/bin/segment/segment.c b/bin/segment/segment.c
index bbfcfa4..8f88554 100644
--- a/bin/segment/segment.c
+++ b/bin/segment/segment.c
@@ -934,7 +934,8 @@ segment_detections(struct segmentparams *p)
/* (Re-)do everything until this step. */
gal_threads_spin_off(segment_on_threads, &clprm,
- p->numdetections, p->cp.numthreads);
+ p->numdetections, p->cp.numthreads,
+ p->cp.minmapsize, p->cp.quietmmap);
/* Set the extension name. */
switch(clprm.step)
@@ -1061,7 +1062,8 @@ segment_detections(struct segmentparams *p)
{
clprm.step=0;
gal_threads_spin_off(segment_on_threads, &clprm, p->numdetections,
- p->cp.numthreads);
+ p->cp.numthreads, p->cp.minmapsize,
+ p->cp.quietmmap);
}
diff --git a/bin/statistics/sky.c b/bin/statistics/sky.c
index a307aec..df1c437 100644
--- a/bin/statistics/sky.c
+++ b/bin/statistics/sky.c
@@ -190,7 +190,8 @@ sky(struct statisticsparams *p)
/* Find the Sky and Sky standard deviation on the tiles. */
if(!cp->quiet) gettimeofday(&t1, NULL);
- gal_threads_spin_off(sky_on_thread, p, tl->tottiles, cp->numthreads);
+ gal_threads_spin_off(sky_on_thread, p, tl->tottiles, cp->numthreads,
+ cp->minmapsize, cp->quietmmap);
if(!cp->quiet)
{
num=gal_statistics_number(p->sky_t);
diff --git a/bin/warp/warp.c b/bin/warp/warp.c
index 13cd919..fc2e566 100644
--- a/bin/warp/warp.c
+++ b/bin/warp/warp.c
@@ -31,6 +31,7 @@ along with Gnuastro. If not, see
<http://www.gnu.org/licenses/>.
#include <gnuastro/wcs.h>
#include <gnuastro/fits.h>
#include <gnuastro/polygon.h>
+#include <gnuastro/pointer.h>
#include "main.h"
#include "warp.h"
@@ -520,6 +521,7 @@ warp(struct warpparams *p)
{
int err;
pthread_t t; /* All thread ids saved in this, not used. */
+ char *mmapname;
pthread_attr_t attr;
pthread_barrier_t b;
struct iwpparams *iwp;
@@ -540,7 +542,9 @@ warp(struct warpparams *p)
/* Distribute the output pixels into the threads: */
- gal_threads_dist_in_threads(p->output->size, nt, &indexs, &thrdcols);
+ mmapname=gal_threads_dist_in_threads(p->output->size, nt,
+ p->cp.minmapsize, p->cp.quietmmap,
+ &indexs, &thrdcols);
/* Start the warp. */
@@ -586,8 +590,9 @@ warp(struct warpparams *p)
printf(" Output: %s\n", p->cp.output);
- /* Free the allocated spaces: */
+ /* Free the allocated spaces, note that 'indexs' may be memory-mapped. */
free(iwp);
- free(indexs);
gal_data_free(p->output);
+ if(mmapname) gal_pointer_mmap_free(&mmapname, p->cp.quietmmap);
+ else free(indexs);
}
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index 2bf3f22..bfce4e5 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -7653,60 +7653,64 @@ If you are writing your own program, you can use the
@code{gal_data_copy_to_new_
@cindex Memory management
@cindex Non-volatile memory
@cindex Memory, non-volatile
-In this section we'll review how Gnuastro manages your input data in the
available memory of your system.
-But before that, let's have a short basic introduction to the types of memory
most relevant to this discussion.
+In this section we'll review how Gnuastro manages your input data in your
system's memory.
+Knowing this can help you optimize your usage (in speed and memory
consumption) when the data volume ls large and approaches, or exceeds, your
available RAM (usually in various calls to multiple programs simultaneously).
+But before diving into the details, let's have a short basic introduction to
memory in general and in particular the types of memory most relevant to this
discussion.
Input datasets (that are later fed into programs for analysis) are commonly
first stored in @emph{non-volatile memory}.
-This is a type of memory that doesn't need any constant power to keep the
data, like HDDs or SSDs and is primarily aimed for long-term storage.
+This is a type of memory that doesn't need a constant power supply to keep the
data and is therefore primarily aimed for long-term storage, like HDDs or SSDs.
So data in this type of storage is preserved when you turn off your computer.
-But non-volatile memory is much slower in reading or writing than the speeds
that CPUs can process the data.
+But by its nature, non-volatile memory is much slower, in reading or writing,
than the speeds that CPUs can process the data.
Thus relying on this type of memory alone would create a bad bottleneck in the
input/output (I/O) phase of any processing.
@cindex RAM
@cindex Volatile memory
@cindex Memory, volatile
-The first step to decrease this bottleneck is to have a faster storage space,
but with more limited volume.
+The first step to decrease this bottleneck is to have a faster storage space,
but with a much limited storage volume.
For this type of storage, computers have a Random Access Memory (or RAM).
-RAM is classified as a @emph{volatile memory} because it needs a constant
power source to keep the information.
-In other words, the moment power is cut-off, all the stored information in it
is gone (hence the 'volatile' name).
-By assuming that sufficient power is always available, volatile memory is much
faster than non-volatile memory.
+RAM is classified as a @emph{volatile memory} because it needs a constant flow
of electricity to keep the information.
+In other words, the moment power is cut-off, all the stored information in
your RAM is gone (hence the 'volatile' name).
+But thanks to that constant supply of power, it can access any random address
with equal (and very high!) speed.
-Hence, the general/simplistic way that programs deal with memory is the
following (this is general to all programs, not just Gnuastro's):
+Hence, the general/simplistic way that programs deal with memory is the
following (this is general to almost all programs, not just Gnuastro's):
1) Load/copy the input data from the non-volatile memory into RAM.
-2) Use the copy in RAM as input for all the internal processing as well as the
intermediatea data that is necessary during the processing.
-3) Finally, when the analyis is complete, write the final output data back
into non-volatile memory, and delete all the used space in the RAM (the initial
copy and all the intermediate data).
-The RAM is most important for the data of the intermediate steps (that you
never see as a user of a program!).
+2) Use the copy of the data in RAM as input for all the internal processing as
well as the intermediate data that is necessary during the processing.
+3) Finally, when the analyis is complete, write the final output data back
into non-volatile memory, and free/delete all the used space in the RAM (the
initial copy and all the intermediate data).
+Usually the RAM is most important for the data of the intermediate steps (that
you never see as a user of a program!).
-When the datasets are small (compared to the available space in your system's
RAM when they are run) the steps above are roughly accurate in Gnuastro's
programs and libraries.
+When the input dataset(s) to a program are small (compared to the available
space in your system's RAM at the moment it is run) Gnuastro's programs and
libraries follow the standard series of steps above.
The only exception is that deleting the intermediate data is not only done at
the end of the program.
-As soon as an intermediate dataset is no longer necessary for the next
internal steps, it is deleted.
-This allows Gnuastro programs to minimize their usage of your system's RAM.
+As soon as an intermediate dataset is no longer necessary for the next
internal steps, the space it occupied is deleted/freed.
+This allows Gnuastro programs to minimize their usage of your system's RAM
over the full running time.
-The situation gets complicated when the datasets are large (compared to your
available RAM when they are run).
-For example if a dataset if half the size of your system's available RAM, and
the program's analysis needs three or more intermediately processed versions of
it in a certain phase of its analysis, there won't be enough RAM to keep those
higher-level versions.
+The situation gets complicated when the datasets are large (compared to your
available RAM when the program is run).
+For example if a dataset if half the size of your system's available RAM, and
the program's internal analysis needs three or more intermediately processed
copies of it at one moment in its analysis.
+There won't be enough RAM to keep those higher-level intermediate data.
In such cases, programs that don't do any memory management will crash.
But fortunately Gnuastro's programs do have a memory management plans for such
situations.
@cindex Memory-mapped file
-When the necessary amount of space for an intermediate dataset cannot be
allocated in the RAM, instead of crashing, Gnuastro's programs will not use the
RAM at all.
-They will use the ``memory-mapped file'' concept in modern operating systems
to create a randomly-named file in your non-volatile memory and use that
instead.
+When the necessary amount of space for an intermediate dataset cannot be
allocated in the RAM, Gnuastro's programs will not use the RAM at all.
+They will use the ``memory-mapped file'' concept in modern operating systems
to create a randomly-named file in your non-volatile memory and use that
instead of the RAM.
That file will have the exact size (in bytes) of that intermediate dataset.
Any time the program needs that intermediate dataset, the operating system
will directly go to that file, and bypass your RAM.
As soon as that file is no longer necessary for the analysis, it will be
deleted.
But as mentioned above, non-volatile memory has much slower I/O speed than the
RAM.
-Hence in such situations, the programs will become noticably slower.
+Hence in such situations, the programs will become noticably slower (sometimes
by factors of 10 times slower, depending on your non-volatile memory speeds).
-By default, Gnuastro's programs and libraries will notify you with a statement
like below, the moment that an intermediate dataset is memory-mapped (can
happen in any phase of their analysis).
-It shows the location of the memory-mapped file and its size, complemented
with a small description of the cause and a pointer to this section of the book
for more information on how to deal with it (if necessary).
+Because of the drop in I/O speed (and thus the speed of your running program),
the moment that any to-be-allocated dataset is memory-mapped, Gnuastro's
programs and libraries will notify you with a descriptive statement like below
(can happen in any phase of their analysis).
+It shows the location of the memory-mapped file, its size, complemented with a
small description of the cause, a pointer to this section of the book for more
information on how to deal with it (if necessary), and what to do to suppress
it.
@example
-astarithmetic: ./.gnuastro_mmap/B1QgVf: temporary memory-mapped file
-(XXXXX bytes) for intermediate data that is not stored in RAM (see
-the "Memory management" section of Gnuastro's manual)
+astarithmetic: ./.gnuastro_mmap/Fu7Dhs: temporary memory-mapped file
+(XXXXXXXXXXX bytes) created for intermediate data that is not stored
+in RAM (see the "Memory management" section of Gnuastro's manual for
+optimizing your project's memory management, and thus speed). To
+disable this warning, please use the option '--quiet-mmap'
@end example
@noindent
-Finally, when the intermediate dataset is no longer necessary, the program
will automatically delete it and let you know with a statement like this:
+Finally, when the intermediate dataset is no longer necessary, the program
will automatically delete it and notify you with a statement like this:
@example
astarithmetic: ./.gnuastro_mmap/B1QgVf: deleted
@@ -7716,54 +7720,57 @@ astarithmetic: ./.gnuastro_mmap/B1QgVf: deleted
To disable these messages, you can run the program with @code{--quietmmap}, or
set the @code{quietmmap} variable in the allocating library function to be
non-zero.
An important component of these messages is the name of the memory-mapped file.
-Knowing that the file has been deleted is important for the user if the
program crashes for any reason: internal (for example a parameter is given
wrongly) or (for example you mistakenly kill the running job) reason.
+Knowing that the file has been deleted is important for the user if the
program crashes for any reason: internally (for example a parameter is given
wrongly) or externally (for example you mistakenly kill the running job).
In the event of a crash, the memory-mapped files will not be deleted and you
have to manually delete them because they are usually large and they may soon
fill your full storage if not deleted in a long time due to successive crashes.
-This brings us to managing the memory-mapped files in your non-volatile
memory: knowing where they are saved, placing them in different places of your
file system, or deleting them when necessary.
-As the examples above show, memory-mapped files are stored in a hidden
sub-directory of the the running directory: @file{.gnuastro_mmap}.
-If this directory doesn't exist, Gnuastro will automatically create it if it
needs to memory-map an internal dataset.
-However, If the @file{.gnuastro_mmap} sub-directory exists and isn't writable,
or it can't be created, then the file will be created in the running directory
with a @file{.gnuastro_mmap_} prefix.
+This brings us to managing the memory-mapped files in your non-volatile memory.
+In other words: knowing where they are saved, or intentionally placing them in
different places of your file system, or deleting them when necessary.
+As the examples above show, memory-mapped files are stored in a hidden
sub-directory of the the running directory called @file{.gnuastro_mmap}.
+If this directory doesn't exist, Gnuastro will automatically create it when
memory mapping becomes necessary.
+Alternatively, it may happen that the @file{.gnuastro_mmap} sub-directory
exists and isn't writable, or it can't be created.
+In such cases, the memory-mapped file for each dataset will be created in the
running directory with a @file{.gnuastro_mmap_} prefix.
-Therefore one easy way to delete all memory-mapped files is to delete
everything within this directory:
+Therefore one easy way to delete all memory-mapped files in case of a crash,
is to delete everything within the sub-directory (first command below), or all
files stating with this prefix:
@example
rm -f .gnuastro_mmap/*
+rm -f .gnuastro_mmap_*
@end example
-A much more common issue when dealing with memory-mapped files is their
location.
-For example you may be running a program in a partition that is hosted by a
HDD.
+A much more common issue in dealing with memory-mapped files is their location.
+For example you may be running a program in a partition that is hosted by an
HDD.
But you also have another partition on an SSD (which has much faster I/O).
So you want your memory-mapped files to be created in the SSD to speed up your
processing.
-Another common scenario is this: you want your project source directory to
only contain your plain-text scripts and you want your project's built products
(even the temporary memory-mapped files) to be built in a different location
because they are large and hard to maintain with the valuable scripts.
+In this scenario, you want your project source directory to only contain your
plain-text scripts and you want your project's built products (even the
temporary memory-mapped files) to be built in a different location because they
are large; thus I/O speed becomes important.
-To do host the memory-mapped files in another location, you can set
@file{.gnuastro_mmap} to be a symbolic link to that location.
+To host the memory-mapped files in another location, you can set the hidden
subdirectory (@file{.gnuastro_mmap}) to be a symbolic link to the location with
fast I/O.
For example, let's assume you want your memory-mapped files to be stored in
@file{/path/to/dir/for/mmap}.
-All you have to do is to run following command before your Gnuastro analysis
command:
+All you have to do is to run the following command before your Gnuastro
analysis command(s).
@example
ln -s /path/to/dir/for/mmap .gnuastro_mmap
@end example
-The programs will delete the memory-mapped file when it is no longer needed,
but they won't delete the @file{.gnuastro_mmap} directory that hosts them.
-So if your project involves many Gnuastro programs and you want your
memory-mapped files to be in a different location, you just have to make the
symbolic link above once.
+The programs will delete a memory-mapped file when it is no longer needed, but
they won't delete the @file{.gnuastro_mmap} directory that hosts them.
+So if your project involves many Gnuastro programs (possibly called in
parallel) and you want your memory-mapped files to be in a different location,
you just have to make the symbolic link above once at the start, and all the
programs will use it if necessary.
Another memory-management scenario that may happen is this: you don't want a
Gnuastro program to allocate internal datasets in the RAM at all.
-For example the speed of your Gnuastro-related project doesn't matter at that
moment, and you have higher-priority jobs that are being run at the same time.
+For example the speed of your Gnuastro-related project doesn't matter at that
moment, and you have higher-priority jobs that are being run at the same time
which need to have RAM available.
In such cases, you can use the @option{--minmapsize} option that is available
in all Gnuastro programs (see @ref{Processing options}).
Any intermediate dataset that has a size larger than the value of this option
will be memory-mapped, even if there is space available in your RAM.
-For example if you want any intermediate dataset larger than 1 megabyte to be
memory-mapped, use @option{--minmapsize=1000000}.
+For example if you want any dataset larger than 100 megabytes to be
memory-mapped, use @option{--minmapsize=100000000} (8 zeros!).
@cindex Linux kernel
@cindex Kernel, Linux
-You shouldn't set the value of @option{--minmapsize} to be too small,
otherwise even small intermediate values (that are very numerous) in the
program will be memory-mapped.
+You shouldn't set the value of @option{--minmapsize} to be too small,
otherwise even small intermediate values (that are usually very numerous) in
the program will be memory-mapped.
However the kernel can only host a limited number of memory-mapped files at
every moment (by all running programs combined).
For example in the default@footnote{If you need to host more memory-mapped
files at one moment, you need to build your own customized Linux kernel.} Linux
kernel on GNU/Linux operating systems this limit is roughly 64000.
If the total number of memory-mapped files exceeds this number, all the
programs using them will crash.
-Small/numerous intermediate values will rarely exceed a mega byte, so based on
our experience, this is a good value in such scenarios.
+Gnuastro's programs will warn you if your given value is too small and may
cause a problem later.
Actually, the default behavior for Gnuastro's programs (to only use
memory-mapped files when there isn't enough RAM) is a side-effect of
@option{--minmapsize}.
The pre-defined value to this option is an extremely large value in the
lowest-level Gnuastro configuration file (the installed @file{gnuastro.conf}
described in @ref{Configuration file precedence}).
-This value is larger than the largest available RAM.
+This value is larger than the largest possible available RAM.
You can check by running any Gnuastro program with a @option{-P} option.
Because no dataset will be larger than this, by default the programs will
first attempt to use the RAM for temporary storage.
But if writing in the RAM fails (for any reason, maily due to lack of
available space), then a memory-mapped file will be created.
@@ -20162,31 +20169,20 @@ strange and apparently random errors that are
extremely hard to debug.
@subsubsection Gnuastro's thread related functions
@cindex POSIX Threads
-The POSIX Threads functions offered in the C library are very low-level and
-offer a great range of control over the properties of the threads. So if
-you are interested in customizing your tools for complicated thread
-applications, it is strongly encouraged to get a nice familiarity with
-them. Some resources were introduced in @ref{Multithreaded
-programming}.
+The POSIX Threads functions offered in the C library are very low-level and
offer a great range of control over the properties of the threads.
+So if you are interested in customizing your tools for complicated thread
applications, it is strongly encouraged to get a nice familiarity with them.
+Some resources were introduced in @ref{Multithreaded programming}.
-However, in many cases used in astronomical data analysis, you don't need
-communication between threads and each target operation can be done
-independently. Since such operations are very common, Gnuastro provides the
-tools below to facilitate the creation and management of jobs without any
-particular knowledge of POSIX Threads for such operations. The most
-interesting high-level functions of this section are the
-@code{gal_threads_number} and @code{gal_threads_spin_off} that identify the
-number of threads on the system and spin-off threads. You can see a
-demonstration of using these functions in @ref{Library demo -
-multi-threaded operation}.
+However, in many cases used in astronomical data analysis, you don't need
communication between threads and each target operation can be done
independently.
+Since such operations are very common, Gnuastro provides the tools below to
facilitate the creation and management of jobs without any particular knowledge
of POSIX Threads for such operations.
+The most interesting high-level functions of this section are the
@code{gal_threads_number} and @code{gal_threads_spin_off} that identify the
number of threads on the system and spin-off threads.
+You can see a demonstration of using these functions in @ref{Library demo -
multi-threaded operation}.
@deftp {C @code{struct}} gal_threads_params
-Structure keeping the parameters of each thread. When each thread is
-created, a pointer to this structure is passed to it. The @code{params}
-element can be the pointer to a structure defined by the user which
-contains all the necessary parameters to pass onto the worker function. The
-rest of the elements within this structure are set internally by
-@code{gal_threads_spin_off} and are relevant to the worker function.
+Structure keeping the parameters of each thread.
+When each thread is created, a pointer to this structure is passed to it.
+The @code{params} element can be the pointer to a structure defined by the
user which contains all the necessary parameters to pass onto the worker
function.
+The rest of the elements within this structure are set internally by
@code{gal_threads_spin_off} and are relevant to the worker function.
@example
struct gal_threads_params
@{
@@ -20199,60 +20195,82 @@ struct gal_threads_params
@end deftp
@deftypefun size_t gal_threads_number ()
-Return the number of threads that the operating system has available for
-your program. This number is usually fixed for a single machine and doesn't
-change. So this function is useful when you want to run your program on
-different machines (with different CPUs).
+Return the number of threads that the operating system has available for your
program.
+This number is usually fixed for a single machine and doesn't change.
+So this function is useful when you want to run your program on different
machines (with different CPUs).
@end deftypefun
-@deftypefun void gal_threads_spin_off (void @code{*(*worker)(void *)}, void
@code{*caller_params}, size_t @code{numactions}, size_t @code{numthreads})
-Distribute @code{numactions} jobs between @code{numthreads} threads and
-spin-off each thread by calling the @code{worker} function. The
-@code{caller_params} pointer will also be passed to @code{worker} as part
-of the @code{gal_threads_params} structure. For a fully working example of
-this function, please see @ref{Library demo - multi-threaded operation}.
+@deftypefun void gal_threads_spin_off (void @code{*(*worker)(void *)}, void
@code{*caller_params}, size_t @code{numactions}, size_t @code{numthreads},
size_t @code{minmapsize}, int @code{quietmmap})
+Distribute @code{numactions} jobs between @code{numthreads} threads and
spin-off each thread by calling the @code{worker} function.
+The @code{caller_params} pointer will also be passed to @code{worker} as part
of the @code{gal_threads_params} structure.
+For a fully working example of this function, please see @ref{Library demo -
multi-threaded operation}.
+
+If there are many jobs (millions or billions) to organize, memory issues may
become important.
+With @code{minmapsize} you can specify the minimum byte-size to allocate the
necessary space in a memory-mapped file or alternatively in RAM.
+If @code{quietmmap} is non-zero, then a warning will be printed upon creating
a memory-mapped file.
+For more on Gnuastro's memory management, see @ref{Memory management}.
@end deftypefun
@deftypefun void gal_threads_attr_barrier_init (pthread_attr_t @code{*attr},
pthread_barrier_t @code{*b}, size_t @code{limit})
@cindex Detached threads
-This is a low-level function in case you don't want to use
-@code{gal_threads_spin_off}. It will initialize the general thread
-attribute @code{attr} and the barrier @code{b} with @code{limit} threads to
-wait behind the barrier. For maximum efficiency, the threads initialized
-with this function will be detached. Therefore no communication is
-possible between these threads and in particular @code{pthread_join} won't
-work on these threads. You have to use the barrier constructs to wait for
-all threads to finish.
-@end deftypefun
-
-@deftypefun void gal_threads_dist_in_threads (size_t @code{numactions}, size_t
@code{numthreads}, size_t @code{**outthrds}, size_t @code{*outthrdcols})
-This is a low-level function in case you don't want to use
-@code{gal_threads_spin_off}. Identify the ``index''es (starting from 0) of
-the actions to be done on each thread in the @code{outthrds}
-array. @code{outthrds} is treated as a 2D array with @code{numthreads} rows
-and @code{outthrdcols} columns. The indexs in each row, identify the
-actions that should be done by one thread. Please see the explanation below
-to understand the purpose of this operation.
-
-Let's assume you have @mymath{A} actions (where there is only one function
-and the input values differ for each action) and @mymath{T} threads
-available to the system with @mymath{A>T} (common values for these two
-would be @mymath{A>1000} and @mymath{T<10}). Spinning off a thread is not a
-cheap job and requires a significant number of CPU cycles. Therefore,
-creating @mymath{A} threads is not the best way to address such a
-problem. The most efficient way to manage the actions is such that only
-@mymath{T} threads are created, and each thread works on a list of actions
-identified for it in series (one after the other). This way your CPU will
-get all the actions done with minimal overhead.
-
-The purpose of this function is to do what we explained above: each row in
-the @code{outthrds} array contains the indexs of actions which must be done
-by one thread. @code{outthrds} contains @code{outthrdcols} columns. In
-using @code{outthrds}, you don't have to know the number of columns. The
-@code{GAL_BLANK_SIZE_T} macro has a role very similar to a
-string's @code{\0}: every row finishes with this macro, so can easily stop
-parsing the indexes in the row when you confront it. Please see the example
-program in @file{tests/lib/multithread.c} for a demonstration.
+This is a low-level function in case you don't want to use
@code{gal_threads_spin_off}.
+It will initialize the general thread attribute @code{attr} and the barrier
@code{b} with @code{limit} threads to wait behind the barrier.
+For maximum efficiency, the threads initialized with this function will be
detached.
+Therefore no communication is possible between these threads and in particular
@code{pthread_join} won't work on these threads.
+You have to use the barrier constructs to wait for all threads to finish.
+@end deftypefun
+
+@deftypefun {char *} gal_threads_dist_in_threads (size_t @code{numactions},
size_t @code{numthreads}, size_t @code{minmapsize}, int @code{quietmmap},
size_t @code{**indexs}, size_t @code{*icols})
+This is a low-level function in case you don't want to use
@code{gal_threads_spin_off}.
+The job of this function is to distribute @code{numactions} jobs/actions in
@code{numthreads} threads.
+To do this, it will assign each job an ID, ranging from 0 to
@code{numactions}-1.
+The output is the allocated @code{*indexs} array and the @code{*icols} number.
+In memory, its just a simple 1D array that has @code{numthreads}
@mymath{\times} @code{*icols} elements.
+But you can visualize it as a 2D array with @code{numthreads} rows and
@code{*icols} columns.
+For more on the logic of the distribution, see below.
+
+@cindex RAM
+@cindex Memory management
+When you have millions/billions of jobs to distribute, @code{indexs} will
become very large.
+For memory management (when to use a memory-mapped file, and when to use RAM),
you need to specify the @code{minmapsize} and @code{quietmmap} arguments.
+For more on memory management, see @ref{Memory management}.
+In general, if your distributed jobs will not be on the scale of billions (and
you want everything to always be written in RAM), just set @code{minmapsize=-1}
and @code{quietmmap=1}.
+
+When @code{indexs} is actually in a memory-mapped file, this function will
return a string containing the name of the file (that you can later give to
@code{gal_pointer_mmap_free} to free/delete).
+When @code{indexs} is in RAM, this function will return a @code{NULL} pointer.
+So after you are finished with @code{indexs}, you can free it like this:
+
+@example
+char *mmapname;
+int quietmmap=1;
+size_t *indexs, thrdcols;
+size_t numactions=5000, minmapsize=-1;
+size_t numthreads=gal_threads_number();
+
+/* Distribute the jobs. */
+mmapname=gal_threads_dist_in_threads(numactions, numthreads,
+ minmapsize, quietmmap,
+ &indexs, &thrdcols);
+
+/* Do any processing you want... */
+
+/* Free the 'indexs' array. */
+if(mmapname) gal_pointer_mmap_free(&mmapname, quietmmap);
+else free(indexs);
+@end example
+
+Here is a brief description of the reasoning behind the @code{indexs} array
and how the jobs are distributed.
+Let's assume you have @mymath{A} actions (where there is only one function and
the input values differ for each action) and @mymath{T} threads available to
the system with @mymath{A>T} (common values for these two would be
@mymath{A>1000} and @mymath{T<10}).
+Spinning off a thread is not a cheap job and requires a significant number of
CPU cycles.
+Therefore, creating @mymath{A} threads is not the best way to address such a
problem.
+The most efficient way to manage the actions is such that only @mymath{T}
threads are created, and each thread works on a list of actions identified for
it in series (one after the other).
+This way your CPU will get all the actions done with minimal overhead.
+
+The purpose of this function is to do what we explained above: each row in the
@code{indexs} array contains the indexs of actions which must be done by one
thread (so it has @code{numthreads} rows with @code{*icols} columns).
+However, when using @code{indexs}, you don't have to know the number of
columns.
+It is guaranteed that all the rows finish with @code{GAL_BLANK_SIZE_T} (see
@ref{Library blank values}).
+The @code{GAL_BLANK_SIZE_T} macro plays a role very similar to a string's
@code{\0}: every row finishes with this macro, so can easily stop parsing the
indexes in the row as soon as you confront @code{GAL_BLANK_SIZE_T}.
+For some real examples, please see the example program in
@file{tests/lib/multithread.c} for a demonstration.
@end deftypefun
@@ -20594,42 +20612,38 @@ types}).
@end deftypefun
@deftypefun {void *} gal_pointer_allocate (uint8_t @code{type}, size_t
@code{size}, int @code{clear}, const char @code{*funcname}, const char
@code{*varname})
-Allocate an array of type @code{type} with @code{size} elements in RAM (for
-the type codes, see @ref{Library data types}). If @code{clear!=0}, then the
-allocated space is set to zero (cleared). This is effectively just a
-wrapper around C's @code{malloc} or @code{calloc} functions but takes
-Gnuastro's integer type codes and will also abort with a clear error if
-there the allocation was not successful.
+Allocate an array of type @code{type} with @code{size} elements in RAM (for
the type codes, see @ref{Library data types}).
+If @code{clear!=0}, then the allocated space is set to zero (cleared).
+This is effectively just a wrapper around C's @code{malloc} or @code{calloc}
functions but takes Gnuastro's integer type codes and will also abort with a
clear error if there the allocation was not successful.
@cindex C99
-When space cannot be allocated, this function will abort the program with a
-message containing the reason for the failure. @code{funcname} (name of the
-function calling this function) and @code{varname} (name of variable that
-needs this space) will be used in this error message if they are not
-@code{NULL}. In most modern compilers, you can use the generic
-@code{__func__} variable for @code{funcname}. In this way, you don't have
-to manually copy and paste the function name or worry about it changing
-later (@code{__func__} was standardized in C99).
-@end deftypefun
-
-@deftypefun {void *} gal_pointer_allocate_mmap (size_t @code{size}, uint8_t
@code{type}, int @code{clear}, char @code{**mmapname})
-Allocate the necessary space to keep @code{size} elements of type
-@code{type} in HDD/SSD (a file, not in RAM). for the type codes, see
-@ref{Library data types}. If @code{clear!=0}, then the allocated space will
-also be cleared. The allocation is done using C's @code{mmap} function. The
-name of the file containing the allocated space is an allocated string that
-will be put in @code{*mmapname}.
-
-Note that the kernel doesn't allow an infinite number of memory mappings to
-files. So it is not recommended to use this function with every
-allocation. The best case scenario to use this function is for large arrays
-that are very large and can fill up the RAM. Keep the smaller arrays in
-RAM, which is faster and can have a (theoretically) unlimited number of
-allocations.
-
-When you are done with the dataset and don't need it anymore, don't use
-@code{free} (the dataset isn't in RAM). Just delete the file (and the
-allocated space for the filename) with the commands below:
+When space cannot be allocated, this function will abort the program with a
message containing the reason for the failure.
+@code{funcname} (name of the function calling this function) and
@code{varname} (name of variable that needs this space) will be used in this
error message if they are not @code{NULL}.
+In most modern compilers, you can use the generic @code{__func__} variable for
@code{funcname}.
+In this way, you don't have to manually copy and paste the function name or
worry about it changing later (@code{__func__} was standardized in C99).
+@end deftypefun
+
+@deftypefun {void *} gal_pointer_allocate_ram_or_mmap (uint8_t @code{type},
size_t @code{size}, int @code{clear}, size_t @code{minmapsize}, char
@code{**mmapname}, int @code{quietmmap}, const char @code{*funcname}, const
char @code{*varname})
+Allocate the given space either in RAM or in a memory-mapped file.
+This function is just a high-level wrapper to @code{gal_pointer_allocate} (to
allocate in RAM) or @code{gal_pointer_mmap_allocate} (to use a memory-mapped
file).
+For more on memory management in Gnuastro, please see @ref{Memory management}.
+The various arguments are more fully explained in the two functions above.
+@end deftypefun
+
+@deftypefun {void *} gal_pointer_mmap_allocate (size_t @code{size}, uint8_t
@code{type}, int @code{clear}, char @code{**mmapname})
+Allocate the necessary space to keep @code{size} elements of type @code{type}
in HDD/SSD (a file, not in RAM).
+For the type codes, see @ref{Library data types}.
+If @code{clear!=0}, then the allocated space will also be cleared.
+The allocation is done using C's @code{mmap} function.
+The name of the file containing the allocated space is an allocated string
that will be put in @code{*mmapname}.
+
+Note that the kernel doesn't allow an infinite number of memory mappings to
files.
+So it is not recommended to use this function with every allocation.
+The best case scenario to use this function is for large arrays that are very
large and can fill up the RAM.
+Keep the smaller arrays in RAM, which is faster and can have a (theoretically)
unlimited number of allocations.
+
+When you are done with the dataset and don't need it anymore, don't use
@code{free} (the dataset isn't in RAM).
+Just delete the file (and the allocated space for the filename) with the
commands below, or simply use @code{gal_pointer_mmap_free}.
@example
remove(mmapname);
@@ -20637,6 +20651,12 @@ free(mmapname);
@end example
@end deftypefun
+@deftypefun void gal_pointer_mmap_free (char @code{**mmapname}, int
@code{quietmmap})
+``Free'' (actually delete) the memory-mapped file that is named
@code{*mmapname}, then free the string.
+If @code{quietmmap} is non-zero, then a warning will be printed for the user
to know that the given file has been deleted.
+@end deftypefun
+
+
@node Library blank values, Library data container, Pointers, Gnuastro library
@subsection Library blank values (@file{blank.h})
When the position of an element in a dataset is important (for example a
@@ -27154,47 +27174,29 @@ main(void)
@node Library demo - multi-threaded operation, Library demo - reading and
writing table columns, Library demo - inspecting neighbors, Library demo
programs
@subsection Library demo - multi-threaded operation
-The following simple program shows how to use Gnuastro to simplify spinning
-off threads and distributing different jobs between the threads. The
-relevant thread-related functions are defined in @ref{Gnuastro's thread
-related functions}. For easy linking/compilation of this program, along
-with a first run, see Gnuastro's @ref{BuildProgram}. Before running, also
-change the @code{filename} and @code{hdu} variable values to specify an
-existing FITS file and/or extension/HDU.
-
-This is a very simple program to open a FITS image, distribute its pixels
-between different threads and print the value of each pixel and the thread
-it was assigned to. The actual operation is very simple (and would not
-usually be done with threads in a real-life program). It is intentionally
-chosen to put more focus on the important steps in spinning of threads and
-how the worker function (which is called by each thread) can identify the
-job-IDs it should work on.
-
-For example, instead of an array of pixels, you can define an array of
-tiles or any other context-specific structures as separate targets. The
-important thing is that each action should have its own unique ID (counting
-from zero, as is done in an array in C). You can then follow the process
-below and use each thread to work on all the targets that are assigned to
-it. Recall that spinning-off threads is its self an expensive process and
-we don't want to spin-off one thread for each target (see the description
-of @code{gal_threads_dist_in_threads} in @ref{Gnuastro's thread related
-functions}.
-
-There are many (more complicated, real-world) examples of using
-@code{gal_threads_spin_off} in Gnuastro's actual source code, you can see
-them by searching for the @code{gal_threads_spin_off} function from the top
-source (after unpacking the tarball) directory (for example with this
-command):
+The following simple program shows how to use Gnuastro to simplify spinning
off threads and distributing different jobs between the threads.
+The relevant thread-related functions are defined in @ref{Gnuastro's thread
related functions}.
+For easy linking/compilation of this program, along with a first run, see
Gnuastro's @ref{BuildProgram}.
+Before running, also change the @code{filename} and @code{hdu} variable values
to specify an existing FITS file and/or extension/HDU.
+
+This is a very simple program to open a FITS image, distribute its pixels
between different threads and print the value of each pixel and the thread it
was assigned to.
+The actual operation is very simple (and would not usually be done with
threads in a real-life program).
+It is intentionally chosen to put more focus on the important steps in
spinning of threads and how the worker function (which is called by each
thread) can identify the job-IDs it should work on.
+
+For example, instead of an array of pixels, you can define an array of tiles
or any other context-specific structures as separate targets.
+The important thing is that each action should have its own unique ID
(counting from zero, as is done in an array in C).
+You can then follow the process below and use each thread to work on all the
targets that are assigned to it.
+Recall that spinning-off threads is its self an expensive process and we don't
want to spin-off one thread for each target (see the description of
@code{gal_threads_dist_in_threads} in @ref{Gnuastro's thread related functions}.
+
+There are many (more complicated, real-world) examples of using
@code{gal_threads_spin_off} in Gnuastro's actual source code, you can see them
by searching for the @code{gal_threads_spin_off} function from the top source
(after unpacking the tarball) directory (for example with this command):
@example
$ grep -r gal_threads_spin_off ./
@end example
@noindent
-The code of this demonstration program is shown below. This program was
-also built and run when you ran @code{make check} during the building of
-Gnuastro (@code{tests/lib/multithread.c}, so it is already tested for your
-system and you can safely use it as a guide.
+The code of this demonstration program is shown below.
+This program was also built and run when you ran @code{make check} during the
building of Gnuastro (@code{tests/lib/multithread.c}, so it is already tested
for your system and you can safely use it as a guide.
@example
#include <stdio.h>
@@ -27262,12 +27264,16 @@ main(void)
char *filename="input.fits", *hdu="1";
size_t numthreads=gal_threads_number();
+ /* We are using * `-1' for `minmapsize' to ensure that the image is
+ read into * memory and `1' for `quietmmap' (which can also be
+ zero), see the "Memory management" section in the book. */
+ int quietmmap=1;
+ size_t minmapsize=-1;
+
- /* Read the image into memory as a float32 data type. We are using
- * `-1' for `minmapsize' to ensure that the image is read into
- * memory and `1' for `quietmmap' (which can also be zero). */
+ /* Read the image into memory as a float32 data type. */
p.image=gal_fits_img_read_to_type(filename, hdu, GAL_TYPE_FLOAT32,
- -1, 1);
+ minmapsize, quietmmap);
/* Print some basic information before the actual contents: */
@@ -27287,7 +27293,8 @@ main(void)
/* Spin-off the threads and do the processing on each thread. */
- gal_threads_spin_off(worker_on_thread, &p, p.image->size, numthreads);
+ gal_threads_spin_off(worker_on_thread, &p, p.image->size, numthreads,
+ minmapsize, quietmmap);
/* Clean up and return. */
diff --git a/lib/arithmetic.c b/lib/arithmetic.c
index 6d0f5a2..607b61f 100644
--- a/lib/arithmetic.c
+++ b/lib/arithmetic.c
@@ -1206,7 +1206,6 @@ struct multioperandparams
TYPE b, **a; \
gal_data_t *tmp; \
size_t i=0, tind; \
- \
/* Allocate space to keep the pointers to the arrays of each. */ \
/* Input data structure. The operators will increment these */ \
/* pointers while parsing them. */ \
@@ -1446,7 +1445,8 @@ arithmetic_multioperand(int operator, int flags,
gal_data_t *list,
p.dnum=dnum;
p.operator=operator;
p.hasblank=hasblank;
- gal_threads_spin_off(multioperand_on_thread, &p, out->size, numthreads);
+ gal_threads_spin_off(multioperand_on_thread, &p, out->size, numthreads,
+ list->minmapsize, list->quietmmap);
/* Clean up and return. Note that the operation might have been done in
diff --git a/lib/checkset.c b/lib/checkset.c
index 4891e6c..c36199f 100644
--- a/lib/checkset.c
+++ b/lib/checkset.c
@@ -85,6 +85,196 @@ gal_checkset_gsl_rng(uint8_t envseed_bool, const char
**name,
+/* On the Linux kernel, due to "overcommitting" (which is activated by
+ default), malloc will not return NULL when we allocate more memory than
+ the physically available memory. It is possible to disable overcommiting
+ with root permissions, but I have not been able to find any way to do
+ this as a normal user. So the only way is to look into the
+ '/proc/meminfo' file (constantly filled by the Linux kernel) and read
+ the available memory from that.
+
+ Note that this overcommiting apparently only occurs on Linux. From what
+ I have read, other kernels are much more strict and 'malloc' will indeed
+ return NULL if there isn't any physical RAM to support it. So if the
+ '/proc/meminfo' doesn't exist, we can assume that 'malloc' works as
+ expected, until its inverse is proven. */
+size_t
+gal_checkset_ram_available(int quietmmap)
+{
+ FILE *file;
+ int keyfound=0;
+ size_t *freemem=NULL;
+ size_t linelen=80, out=GAL_BLANK_SIZE_T;
+ char *token, *line, *linecp, *saveptr, delimiters[] = " ";
+ char *meminfo="/proc/meminfo", *keyname="MemAvailable", *units="kB";
+
+ /* If /proc/meminfo exists, read it. Otherwise, don't bother doing
+ anything. */
+ if ((file = fopen(meminfo, "r")))
+ {
+ /* Allocate space to read the line. */
+ errno=0;
+ line=malloc(linelen*sizeof *line);
+ if(line==NULL)
+ error(EXIT_FAILURE, errno, "%s: allocating %zu bytes for line",
+ __func__, linelen*sizeof *line);
+
+ /* Read it line-by-line until you find 'MemAvailable'. */
+ while( getline(&line, &linelen, file) != -1 )
+ if( !strncmp(line, keyname, 12) )
+ {
+ /* Necessary for final check: */
+ keyfound=1;
+
+ /* We need to work on a copied line to avoid messing up the
+ contents of the actual line. */
+ gal_checkset_allocate_copy(line, &linecp);
+
+ /* The first token (which we don't need). */
+ token=strtok_r(linecp, delimiters, &saveptr);
+
+ /* The second token (which is the actual number we want). */
+ token=strtok_r(NULL, delimiters, &saveptr);
+ if(token)
+ {
+ /* Read the token as a number. */
+ if( gal_type_from_string((void **)(&freemem), token,
+ GAL_TYPE_SIZE_T) )
+ error(EXIT_SUCCESS, 0, "WARNING: %s: value of '%s' "
+ "keyword couldn't be read as an integer. Hence "
+ "the amount of available RAM couldn't be "
+ "determined. If a large volume of data is "
+ "provided, the program may crash. Please contact "
+ "us at '%s' to fix the problem",
+ meminfo, keyname, PACKAGE_BUGREPORT);
+ else
+ {
+ /* The third token should be the units ('kB'). If it
+ isn't, there should be an error because we currently
+ assume kilobytes. */
+ token=strtok_r(NULL, delimiters, &saveptr);
+ if(token)
+ {
+ /* The units should be 'kB' (for kilobytes). */
+ if( !strncmp(token, units, 2) )
+ out=freemem[0]*1000;
+ else
+ error(EXIT_SUCCESS, 0, "WARNING: %s: the units of "
+ "the value of '%s' keyword is (usually 'kB') "
+ "isn't recognized. Hence the amount of "
+ "available RAM couldn't be determined. If a "
+ "large volume of data is provided, the "
+ "program may crash. Please contact us at "
+ "'%s' to fix the problem", meminfo, keyname,
+ PACKAGE_BUGREPORT);
+ }
+ else
+ error(EXIT_SUCCESS, 0, "WARNING: %s: the units of the "
+ "value of '%s' keyword (usually 'kB') couldn't "
+ "be read as an integer. Hence the amount of "
+ "available RAM couldn't be determined. If a "
+ "large volume of data is provided, the program "
+ "may crash. Please contact us at '%s' to fix "
+ "the problem", meminfo, keyname,
PACKAGE_BUGREPORT);
+ }
+
+ /* Clean up. */
+ if(freemem) free(freemem);
+ }
+ else
+ error(EXIT_SUCCESS, 0, "WARNING: %s: line with the '%s' "
+ "keyword didn't have a value. Hence the amount of "
+ "available RAM couldn't be determined. If a large "
+ "volume of data is provided, the program may crash. "
+ "Please contact us at '%s' to fix the problem",
+ meminfo, keyname, PACKAGE_BUGREPORT);
+
+ /* Clean up. */
+ free(linecp);
+ }
+
+ /* The file existed but a keyname couldn't be found. In this case we
+ should inform the user to be aware that we can't automatically
+ determine the available memory.*/
+ if(keyfound==0 && quietmmap==0)
+ error(EXIT_FAILURE, 0, "WARNING: %s: didn't contain a '%s' keyword "
+ "hence the amount of available RAM couldn't be determined. "
+ "If a large volume of data is provided, the program may "
+ "crash. Please contact us at '%s' to fix the problem",
+ meminfo, keyname, PACKAGE_BUGREPORT);
+
+ /* Close the opened file and free the line. */
+ free(line);
+ fclose(file);
+ }
+
+ /* Return the final value. */
+ return out;
+}
+
+
+
+
+
+int
+gal_checkset_need_mmap(size_t bytesize, size_t minmapsize, int quietmmap)
+{
+ int needmmap=0;
+ size_t availableram;
+ size_t minimumtommap=10000000;
+ size_t mustremainfree=750000000;
+
+ /* In case the given minmapsize is smaller than the default value of
+ 'minimumtomap', then correct 'minimumtomap' to be the same as
+ 'minmapsize' (the user has to have full control to over-write the
+ default value, but let them know in a warning that this is not
+ good). */
+ if(minmapsize < minimumtommap)
+ {
+ /* Let the user know that this is not a good choice and can cause
+ other problems. */
+ if(!quietmmap)
+ error(EXIT_SUCCESS, 0, "it is recommended that minmapsize have "
+ "a value larger than %zu (it is currently %zu), see "
+ "\"Memory management\" section in the Gnuastro book for "
+ "more. To disable this warning, please use the option "
+ "'--quiet-mmap'", minimumtommap, minmapsize);
+
+ /* Set the variable. */
+ minimumtommap=minmapsize;
+ }
+
+ /* Memory mapping is only relevant here if the byte-size of the dataset
+ is larger than 'minimumtommap'. This is primarily because checking the
+ available memory can be expensive. */
+ if( bytesize >= minimumtommap )
+ {
+ /* Find the available RAM space (only relevant for Linux). */
+ availableram=gal_checkset_ram_available(quietmmap);
+
+ /* For a check:
+ printf("check: %zu (bs), %zu (ar), %zu (nu)\n",
+ bytesize, availableram, mustremainfree);
+ */
+
+ /* If the final size is larger than the user's maximum,
+ or is larger than the available memory minus 500Mb (to
+ leave the system some breathing space!), then read the
+ array into disk using memory-mapping (HDD/SSD). */
+ if( bytesize >= minmapsize
+ || availableram < mustremainfree
+ || bytesize > (availableram-mustremainfree) )
+ needmmap=1;
+ }
+
+ /* Return the final choice. */
+ return needmmap;
+}
+
+
+
+
+
diff --git a/lib/convolve.c b/lib/convolve.c
index 8013058..ab3c41a 100644
--- a/lib/convolve.c
+++ b/lib/convolve.c
@@ -549,7 +549,8 @@ gal_convolve_spatial_general(gal_data_t *tiles, gal_data_t
*kernel,
/* Do the spatial convolution on threads. */
gal_threads_spin_off(convolve_spatial_on_thread, ¶ms,
- gal_list_data_number(tiles), numthreads);
+ gal_list_data_number(tiles), numthreads,
+ tiles->minmapsize, tiles->quietmmap);
/* Clean up and return the output array. */
diff --git a/lib/data.c b/lib/data.c
index e22b50e..bc812e8 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -92,137 +92,6 @@ gal_data_alloc(void *array, uint8_t type, size_t ndim,
size_t *dsize,
-/* On the Linux kernel, due to "overcommitting" (which is activated by
- default), malloc will not return NULL when we allocate more memory than
- the physically available memory. It is possible to disable overcommiting
- with root permissions, but I have not been able to find any way to do
- this as a normal user. So the only way is to look into the
- '/proc/meminfo' file (constantly filled by the Linux kernel) and read
- the available memory from that.
-
- Note that this overcommiting apparently only occurs on Linux. From what
- I have read, other kernels are much more strict and 'malloc' will indeed
- return NULL if there isn't any physical RAM to support it. So if the
- '/proc/meminfo' doesn't exist, we can assume that 'malloc' works as
- expected, until its inverse is proven. */
-static size_t
-data_available_ram()
-{
- FILE *file;
- int keyfound=0;
- size_t *freemem=NULL;
- size_t linelen=80, out=GAL_BLANK_SIZE_T;
- char *token, *line, *linecp, *saveptr, delimiters[] = " ";
- char *meminfo="/proc/meminfo", *keyname="MemAvailable", *units="kB";
-
- /* If /proc/meminfo exists, read it. Otherwise, don't bother doing
- anything. */
- if ((file = fopen(meminfo, "r")))
- {
- /* Allocate space to read the line. */
- errno=0;
- line=malloc(linelen*sizeof *line);
- if(line==NULL)
- error(EXIT_FAILURE, errno, "%s: allocating %zu bytes for line",
- __func__, linelen*sizeof *line);
-
- /* Read it line-by-line until you find 'MemAvailable'. */
- while( getline(&line, &linelen, file) != -1 )
- if( !strncmp(line, keyname, 12) )
- {
- /* Necessary for final check: */
- keyfound=1;
-
- /* We need to work on a copied file to avoid messing up the
- contents of the actual line. */
- gal_checkset_allocate_copy(line, &linecp);
-
- /* The first token (which we don't need). */
- token=strtok_r(linecp, delimiters, &saveptr);
-
- /* The second token (which is the actual number we want). */
- token=strtok_r(NULL, delimiters, &saveptr);
- if(token)
- {
- /* Read the token as a number. */
- if( gal_type_from_string((void **)(&freemem), token,
- GAL_TYPE_SIZE_T) )
- error(EXIT_SUCCESS, 0, "WARNING: %s: value of '%s' "
- "keyword couldn't be read as an integer. Hence "
- "the amount of available RAM couldn't be "
- "determined. If a large volume of data is "
- "provided, the program may crash. Please contact "
- "us at '%s' to fix the problem",
- meminfo, keyname, PACKAGE_BUGREPORT);
- else
- {
- /* The third token should be the units ('kB'). If it
- isn't, there should be an error because we currently
- assume kilobytes. */
- token=strtok_r(NULL, delimiters, &saveptr);
- if(token)
- {
- /* The units should be 'kB' (for kilobytes). */
- if( !strncmp(token, units, 2) )
- out=freemem[0]*1000;
- else
- error(EXIT_SUCCESS, 0, "WARNING: %s: the units of "
- "the value of '%s' keyword is (usually 'kB') "
- "isn't recognized. Hence the amount of "
- "available RAM couldn't be determined. If a "
- "large volume of data is provided, the "
- "program may crash. Please contact us at "
- "'%s' to fix the problem", meminfo, keyname,
- PACKAGE_BUGREPORT);
- }
- else
- error(EXIT_SUCCESS, 0, "WARNING: %s: the units of the "
- "value of '%s' keyword (usually 'kB') couldn't "
- "be read as an integer. Hence the amount of "
- "available RAM couldn't be determined. If a "
- "large volume of data is provided, the program "
- "may crash. Please contact us at '%s' to fix "
- "the problem", meminfo, keyname,
PACKAGE_BUGREPORT);
- }
-
- /* Clean up. */
- if(freemem) free(freemem);
- }
- else
- error(EXIT_SUCCESS, 0, "WARNING: %s: line with the '%s' "
- "keyword didn't have a value. Hence the amount of "
- "available RAM couldn't be determined. If a large "
- "volume of data is provided, the program may crash. "
- "Please contact us at '%s' to fix the problem",
- meminfo, keyname, PACKAGE_BUGREPORT);
-
- /* Clean up. */
- free(linecp);
- }
-
- /* The file existed but a keyname couldn't be found. In this case we
- should inform the user to be aware that we can't automatically
- determine the available memory.*/
- if(keyfound==0)
- error(EXIT_FAILURE, 0, "WARNING: %s: didn't contain a '%s' keyword "
- "hence the amount of available RAM couldn't be determined. "
- "If a large volume of data is provided, the program may "
- "crash. Please contact us at '%s' to fix the problem",
- meminfo, keyname, PACKAGE_BUGREPORT);
-
- /* Close the opened file and free the line. */
- free(line);
- fclose(file);
- }
-
- /* Return the final value. */
- return out;
-}
-
-
-
-
-
/* Initialize the data structure.
Some notes:
@@ -246,8 +115,7 @@ gal_data_initialize(gal_data_t *data, void *array, uint8_t
type,
int clear, size_t minmapsize, int quietmmap,
char *name, char *unit, char *comment)
{
- size_t nouseram=1500000000;
- size_t i, bytesize, availableram;
+ size_t i;
size_t data_size_limit = (size_t)(-1);
/* Do the simple copying cases. For the display elements, set them all to
@@ -322,46 +190,9 @@ gal_data_initialize(gal_data_t *data, void *array, uint8_t
type,
{
/* If a size wasn't given, just set a NULL pointer. */
if(data->size)
- {
- /* Find the available RAM space (only relevant for Linux). */
- availableram=data_available_ram();
- bytesize=gal_type_sizeof(type)*data->size;
-
- /* For a check:
- printf("check: %zu (data), %zu (ram)\n",
- bytesize, availableram-nouseram);
- */
- /* If the final size is larger than the user's maximum, or is
- larger than the available memory minus 500Mb (to leave the
- system some breathing space!), then read the array into
- disk using memory-mapping (HDD/SSD). */
- if( bytesize > 1000000
- && ( bytesize > minmapsize
- || availableram < nouseram
- || bytesize > (availableram-nouseram) ) )
- data->array=gal_pointer_allocate_mmap(data->type, data->size,
- clear, &data->mmapname,
- quietmmap);
- else
- {
- /* Allocate the necessary space in the RAM. */
- data->array = ( clear
- ? calloc( data->size, gal_type_sizeof(data->type) )
- : malloc( data->size * gal_type_sizeof(data->type) ) );
-
- /* If the array is NULL. */
- if(data->array==NULL)
- data->array=gal_pointer_allocate_mmap(data->type,
- data->size,
- clear,
- &data->mmapname,
- quietmmap);
-
- /* The 'errno' is re-set to zero just incase 'malloc'
- changed it, which may cause problems later. */
- errno=0;
- }
- }
+ data->array=gal_pointer_allocate_ram_or_mmap(data->type,
+ data->size, clear, minmapsize,
+ &data->mmapname, quietmmap, __func__, "");
else data->array=NULL; /* The given size was zero! */
}
}
@@ -418,24 +249,14 @@ gal_data_free_contents(gal_data_t *data)
for(i=0;i<data->size;++i) if(strarr[i]) free(strarr[i]);
}
- /* Free the array. */
- if(data->mmapname)
+ /* Free the array (if it was separately allocated: not part of a block),
+ then set the 'array' to NULL. */
+ if(data->array && data->block==NULL)
{
- /* Delete the file keeping the array. */
- remove(data->mmapname);
-
- /* Inform the user. */
- if(!data->quietmmap)
- error(EXIT_SUCCESS, 0, "%s: deleted", data->mmapname);
-
- /* Free the file name space. */
- free(data->mmapname);
-
- /* Set the name pointer to NULL since it has been freed. */
- data->mmapname=NULL;
+ if(data->mmapname)
+ gal_pointer_mmap_free(&data->mmapname, data->quietmmap);
+ else free(data->array);
}
- else
- if(data->array && data->block==NULL) free(data->array);
data->array=NULL;
}
diff --git a/lib/fits.c b/lib/fits.c
index cccfd32..89daf98 100644
--- a/lib/fits.c
+++ b/lib/fits.c
@@ -2170,11 +2170,13 @@ gal_fits_img_write_to_ptr(gal_data_t *input, char
*filename)
if( gal_fits_name_is_fits(filename)==0 )
error(EXIT_FAILURE, 0, "%s: not a FITS suffix", filename);
+
/* If the input is a tile (isn't a contiguous region of memory), then
copy it into a contiguous region. */
towrite = input==block ? input : gal_data_copy(input);
hasblank=gal_blank_present(towrite, 0);
+
/* Allocate the naxis area. */
naxes=gal_pointer_allocate( ( sizeof(long)==8
? GAL_TYPE_INT64
@@ -2220,7 +2222,6 @@ gal_fits_img_write_to_ptr(gal_data_t *input, char
*filename)
fits_create_img(fptr, LONGLONG_IMG, ndim, naxes, &status);
gal_fits_io_error(status, NULL);
-
/* Write the image into the file. */
fits_write_img(fptr, datatype, fpixel, i64data->size, i64data->array,
&status);
diff --git a/lib/gnuastro-internal/checkset.h b/lib/gnuastro-internal/checkset.h
index 9ad3f2e..1439863 100644
--- a/lib/gnuastro-internal/checkset.h
+++ b/lib/gnuastro-internal/checkset.h
@@ -57,6 +57,13 @@ gsl_rng *
gal_checkset_gsl_rng(uint8_t envseed_bool, const char **name,
unsigned long int *seed);
+size_t
+gal_checkset_ram_available(int quietmmap);
+
+int
+gal_checkset_need_mmap(size_t bytesize, size_t minmapsize, int quietmmap);
+
+
/**************************************************************/
diff --git a/lib/gnuastro/pointer.h b/lib/gnuastro/pointer.h
index f44ac8f..519b885 100644
--- a/lib/gnuastro/pointer.h
+++ b/lib/gnuastro/pointer.h
@@ -59,10 +59,17 @@ gal_pointer_allocate(uint8_t type, size_t size, int clear,
const char *funcname, const char *varname);
void *
-gal_pointer_allocate_mmap(uint8_t type, size_t size, int clear,
+gal_pointer_mmap_allocate(uint8_t type, size_t size, int clear,
char **filename, int quiet);
+void
+gal_pointer_mmap_free(char **mmapname, int quietmmap);
+void *
+gal_pointer_allocate_ram_or_mmap(uint8_t type, size_t size, int clear,
+ size_t minmapsize, char **mmapname,
+ int quietmmap, const char *funcname,
+ const char *varname);
diff --git a/lib/gnuastro/threads.h b/lib/gnuastro/threads.h
index c03bd5f..007d692 100644
--- a/lib/gnuastro/threads.h
+++ b/lib/gnuastro/threads.h
@@ -102,8 +102,9 @@ pthread_barrier_destroy(pthread_barrier_t *b);
size_t
gal_threads_number();
-void
+char *
gal_threads_dist_in_threads(size_t numactions, size_t numthreads,
+ size_t minmapsize, int quietmmap,
size_t **outthrds, size_t *outthrdcols);
void
@@ -126,7 +127,8 @@ struct gal_threads_params
void
gal_threads_spin_off(void *(*worker)(void *), void *caller_params,
- size_t numactions, size_t numthreads);
+ size_t numactions, size_t numthreads,
+ size_t minmapsize, int quietmmap);
__END_C_DECLS /* From C++ preparations */
diff --git a/lib/interpolate.c b/lib/interpolate.c
index ecbf6fb..545f817 100644
--- a/lib/interpolate.c
+++ b/lib/interpolate.c
@@ -459,7 +459,8 @@ gal_interpolate_neighbors(gal_data_t *input,
/* Spin off the threads. */
gal_threads_spin_off(interpolate_neighbors_on_thread, &prm,
- input->size, numthreads);
+ input->size, numthreads, input->minmapsize,
+ input->quietmmap);
/* If the values were permuted for the interpolation, then re-order the
diff --git a/lib/pointer.c b/lib/pointer.c
index 7dfb287..e6b2da6 100644
--- a/lib/pointer.c
+++ b/lib/pointer.c
@@ -105,7 +105,7 @@ gal_pointer_allocate(uint8_t type, size_t size, int clear,
void *
-gal_pointer_allocate_mmap(uint8_t type, size_t size, int clear,
+gal_pointer_mmap_allocate(uint8_t type, size_t size, int clear,
char **filename, int quietmmap)
{
void *out;
@@ -156,8 +156,10 @@ gal_pointer_allocate_mmap(uint8_t type, size_t size, int
clear,
/* Inform the user. */
if(!quietmmap)
error(EXIT_SUCCESS, 0, "%s: temporary memory-mapped file (%zu bytes) "
- "for intermediate data that is not stored in RAM (see "
- "the \"Memory management\" section of Gnuastro's manual)",
+ "created for intermediate data that is not stored in RAM (see "
+ "the \"Memory management\" section of Gnuastro's manual for "
+ "optimizing your project's memory management, and thus speed). "
+ "To disable this warning, please use the option '--quiet-mmap'",
*filename, bsize);
@@ -198,3 +200,70 @@ gal_pointer_allocate_mmap(uint8_t type, size_t size, int
clear,
/* Return the mmap'd pointer and save the file name. */
return out;
}
+
+
+
+
+
+void
+gal_pointer_mmap_free(char **mmapname, int quietmmap)
+{
+ /* Delete the file keeping the array. */
+ remove(*mmapname);
+
+ /* Inform the user. */
+ if(!quietmmap)
+ error(EXIT_SUCCESS, 0, "%s: deleted", *mmapname);
+
+ /* Free the file name space. */
+ free(*mmapname);
+
+ /* Set the name pointer to NULL since it has been freed. */
+ *mmapname=NULL;
+}
+
+
+
+
+
+void *
+gal_pointer_allocate_ram_or_mmap(uint8_t type, size_t size, int clear,
+ size_t minmapsize, char **mmapname,
+ int quietmmap, const char *funcname,
+ const char *varname)
+{
+ void *out;
+ size_t bytesize=gal_type_sizeof(type)*size;
+
+ /* See if the requested size is larger than 1MB (otherwise,
+ its not worth checking RAM, which involves reading a text
+ file, we won't try memory-mapping anyway). */
+
+ /* If it is decided to do memory-mapping, then do it. */
+ if( gal_checkset_need_mmap(bytesize, minmapsize, quietmmap) )
+ out=gal_pointer_mmap_allocate(type, size, clear, mmapname,
+ quietmmap);
+ else
+ {
+ /* Allocate the necessary space in the RAM. */
+ errno=0;
+ out = ( clear
+ ? calloc( size, gal_type_sizeof(type) )
+ : malloc( size * gal_type_sizeof(type) ) );
+
+ /* If the array is NULL (there was no RAM left: on
+ systems other than Linux, 'malloc' will actually
+ return NULL, Linux doesn't do this unfortunately so we
+ need to read the available RAM). */
+ if(out==NULL)
+ out=gal_pointer_mmap_allocate(type, size, clear,
+ mmapname, quietmmap);
+
+ /* The 'errno' is re-set to zero just incase 'malloc'
+ changed it, which may cause problems later. */
+ errno=0;
+ }
+
+ /* Return the allocated dataset. */
+ return out;
+}
diff --git a/lib/threads.c b/lib/threads.c
index 5f09d06..148cea4 100644
--- a/lib/threads.c
+++ b/lib/threads.c
@@ -29,6 +29,7 @@ along with Gnuastro. If not, see
<http://www.gnu.org/licenses/>.
#include <stdlib.h>
#include <gnuastro/threads.h>
+#include <gnuastro/pointer.h>
#include <nproc.h> /* from Gnulib, in Gnuastro's source */
@@ -184,18 +185,20 @@ gal_threads_number()
images for each thread is 1. The results will be saved in a 2D
array of 'outthrdcols' columns and each row will finish with a
(size_t) -1, which is larger than any possible index!. */
-void
+char *
gal_threads_dist_in_threads(size_t numactions, size_t numthreads,
+ size_t minmapsize, int quietmmap,
size_t **outthrds, size_t *outthrdcols)
{
size_t *sp, *fp;
+ char *mmapname=NULL;
size_t i, *thrds, thrdcols;
*outthrdcols = thrdcols = numactions/numthreads+2;
- errno=0;
- thrds=*outthrds=malloc(numthreads*thrdcols*sizeof *thrds);
- if(thrds==NULL)
- error(EXIT_FAILURE, errno, "%s: allocating thrds", __func__);
+ /* Allocate the space to keep the identifiers. */
+ thrds=*outthrds=gal_pointer_allocate_ram_or_mmap(GAL_TYPE_SIZE_T,
+ numthreads*thrdcols, 0, minmapsize, &mmapname,
+ 0, __func__, "thrds");
/* Initialize all the elements to NONINDEX. */
fp=(sp=thrds)+numthreads*thrdcols;
@@ -217,6 +220,9 @@ gal_threads_dist_in_threads(size_t numactions, size_t
numthreads,
}
exit(0);
*/
+
+ /* Return the name of the possibly memory-mapped file. */
+ return mmapname;
}
@@ -328,10 +334,12 @@ gal_threads_attr_barrier_init(pthread_attr_t *attr,
pthread_barrier_t *b,
*/
void
gal_threads_spin_off(void *(*worker)(void *), void *caller_params,
- size_t numactions, size_t numthreads)
+ size_t numactions, size_t numthreads,
+ size_t minmapsize, int quietmmap)
{
int err;
pthread_t t; /* All thread ids saved in this, not used. */
+ char *mmapname=NULL;
pthread_attr_t attr;
pthread_barrier_t b;
struct gal_threads_params *prm;
@@ -356,7 +364,8 @@ gal_threads_spin_off(void *(*worker)(void *), void
*caller_params,
}
/* Distribute the actions into the threads: */
- gal_threads_dist_in_threads(numactions, numthreads, &indexs, &thrdcols);
+ mmapname=gal_threads_dist_in_threads(numactions, numthreads, minmapsize,
+ quietmmap, &indexs, &thrdcols);
/* Do the job: when only one thread is necessary, there is no need to
spin off one thread, just call the workerfunction directly (spinning
@@ -402,7 +411,13 @@ gal_threads_spin_off(void *(*worker)(void *), void
*caller_params,
pthread_barrier_destroy(&b);
}
+ /* If 'mmapname' is NULL, then 'indexs' is in RAM and we can safely
+ 'free' it. However, when its not NULL, then the space for 'indexs' has
+ been memory-mapped (its not in RAM) so special treatment is necessary
+ to delete it through the proper function. */
+ if(mmapname) gal_pointer_mmap_free(&mmapname, quietmmap);
+ else free(indexs);
+
/* Clean up. */
free(prm);
- free(indexs);
}
diff --git a/lib/tile.c b/lib/tile.c
index 39b5964..3433192 100644
--- a/lib/tile.c
+++ b/lib/tile.c
@@ -562,7 +562,8 @@ gal_tile_block_blank_flag(gal_data_t *tile_ll, size_t
numthreads)
{
/* Go over all the tiles and update their blank flag. */
gal_threads_spin_off(tile_block_blank_flag, tile_ll,
- gal_list_data_number(tile_ll), numthreads);
+ gal_list_data_number(tile_ll), numthreads,
+ tile_ll->minmapsize, tile_ll->quietmmap);
}
diff --git a/tests/lib/multithread.c b/tests/lib/multithread.c
index d16474b..f59978a 100644
--- a/tests/lib/multithread.c
+++ b/tests/lib/multithread.c
@@ -90,12 +90,15 @@ int
main(void)
{
struct params p;
+ int quietmmap=1;
+ size_t minmapsize=-1;
char *filename="psf.fits", *hdu="1";
size_t numthreads=gal_threads_number();
/* Read the image into memory as a float32 data type. */
- p.image=gal_fits_img_read_to_type(filename, hdu, GAL_TYPE_FLOAT32, -1, 1);
+ p.image=gal_fits_img_read_to_type(filename, hdu, GAL_TYPE_FLOAT32,
+ minmapsize, quietmmap);
/* Print some basic information before the actual contents: */
@@ -114,7 +117,8 @@ main(void)
/* Spin-off the threads and do the processing on each thread. */
- gal_threads_spin_off(worker_on_thread, &p, p.image->size, numthreads);
+ gal_threads_spin_off(worker_on_thread, &p, p.image->size, numthreads,
+ minmapsize, quietmmap);
/* Clean up and return. */
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [gnuastro-commits] master 99ab4e9: Library and book: memory management by checking available RAM,
Mohammad Akhlaghi <=