diff --git a/dist/builddist b/dist/builddist index 3b31b829..b3a1bf38 100755 --- a/dist/builddist +++ b/dist/builddist @@ -5,7 +5,7 @@ print_usage() { echo "" echo "Tags:" echo " main - build tarball of latest" - echo " v0.11.1 - build tarball of v0.11.1" + echo " v0.12 - build tarball of v0.12" } # check that we got an argument or print usage @@ -23,13 +23,13 @@ if [ "$1" == "main" ] ; then "libcircle" "hpc" "master" "mpifileutils" "hpc" "master" ) -elif [ "$1" == "v0.11.1" ] ; then +elif [ "$1" == "v0.12" ] ; then # to build from latest branch of all repos ORGS=( "lwgrp" "llnl" "v1.0.4" "dtcmp" "llnl" "v1.1.4" "libcircle" "hpc" "v0.3" - "mpifileutils" "hpc" "v0.11.1" + "mpifileutils" "hpc" "v0.12" ) else echo "Error: unknown tag: $1" diff --git a/doc/rst/build.rst b/doc/rst/build.rst index 81bdf76b..4f113b43 100644 --- a/doc/rst/build.rst +++ b/doc/rst/build.rst @@ -44,13 +44,13 @@ To be certain of compatibility, it is recommended that one install libarchive-3. To build on PowerPC, one may need to add :code:`--build=powerpc64le-redhat-linux-gnu` to the configure command. Assuming libarchive has been installed to an `install` directory as shown above, -one can then build mpiFileUtils from a release like v0.11.1 with commands like the following: +one can then build mpiFileUtils from a release like v0.12 with commands like the following: .. code-block:: Bash - wget https://github.com/hpc/mpifileutils/releases/download/v0.11.1/mpifileutils-v0.11.1.tgz - tar -zxf mpifileutils-v0.11.1.tgz - cd mpifileutils-v0.11.1 + wget https://github.com/hpc/mpifileutils/releases/download/v0.12/mpifileutils-v0.12.tgz + tar -zxf mpifileutils-v0.12.tgz + cd mpifileutils-v0.12 mkdir build cd build cmake .. \ diff --git a/doc/rst/conf.py b/doc/rst/conf.py index 8c0b9d5c..ad211a83 100644 --- a/doc/rst/conf.py +++ b/doc/rst/conf.py @@ -54,9 +54,9 @@ # built documents. # # The short X.Y version. -version = u'0.11.1' +version = u'0.12' # The full version, including alpha/beta/rc tags. -release = u'0.11.1' +release = u'0.12' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/man/dbcast.1 b/man/dbcast.1 index 263de20a..7b6cc7f8 100644 --- a/man/dbcast.1 +++ b/man/dbcast.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DBCAST" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DBCAST" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dbcast \- distributed broadcast . @@ -56,7 +56,7 @@ number of MPI processes. .TP .B \-s, \-\-size SIZE The chunk size in bytes used to segment files during the broadcast. -Units like “MB” and “GB” should be immediately follow the number +Units like "MB" and "GB" should be immediately follow the number without spaces (ex. 2MB). The default size is 1MB. It is recommended to use the stripe size of a file if this is known. .UNINDENT diff --git a/man/dbz2.1 b/man/dbz2.1 index e09d49b5..08608ac9 100644 --- a/man/dbz2.1 +++ b/man/dbz2.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DBZ2" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DBZ2" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dbz2 \- distributed bz2 compression . @@ -64,7 +64,7 @@ Overwrite the output file, if it exists. .TP .B \-b, \-\-blocksize SIZE Set the compression block size, from 1 to 9. -Where 1=100kB … and 9=900kB. Default is 9. +Where 1=100kB ... and 9=900kB. Default is 9. .UNINDENT .INDENT 0.0 .TP diff --git a/man/dchmod.1 b/man/dchmod.1 index 7babf687..b47d1ad2 100644 --- a/man/dchmod.1 +++ b/man/dchmod.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DCHMOD" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DCHMOD" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dchmod \- distributed tool to set permissions and group . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdchmod [OPTION] PATH …\fP +\fBdchmod [OPTION] PATH ...\fP .SH DESCRIPTION .sp Parallel MPI application to recursively change permissions and/or group @@ -62,8 +62,8 @@ Change group to specified GROUP name or numeric group id. .TP .B \-m, \-\-mode MODE The mode to apply to each item. MODE may be octal or symbolic syntax -similar to \fBchmod(1)\fP\&. In symbolic notation, “ugoa” are supported -as are “rwxX”. As with chmod, if no leading letter “ugoa” is provided, +similar to \fBchmod(1)\fP\&. In symbolic notation, "ugoa" are supported +as are "rwxX". As with chmod, if no leading letter "ugoa" is provided, mode bits are combined with umask to determine the actual mode. .UNINDENT .INDENT 0.0 @@ -73,7 +73,7 @@ Attempt to change every item. By default, dchmod avoids unncessary chown and chmod calls, for example trying to change the group on an item that already has the correct group, or trying to change the group on an item that is not owned by the user running the tool. -With –force, dchmod executes chown/chmod calls on every item. +With \-\-force, dchmod executes chown/chmod calls on every item. .UNINDENT .INDENT 0.0 .TP @@ -96,7 +96,7 @@ Only modify items whose full path matches REGEX, processed by .INDENT 0.0 .TP .B \-n, \-\-name -Change –exclude and –match to apply to item name rather than its +Change \-\-exclude and \-\-match to apply to item name rather than its full path. .UNINDENT .INDENT 0.0 @@ -151,7 +151,7 @@ regex: .sp \fBmpirun \-np 128 dchmod \-\-name \-\-exclude ‘afilename’ \-\-mode u+rw /directory\fP .sp -Note: You can use –match to change file permissions on all of the +Note: You can use \-\-match to change file permissions on all of the files/directories that match the regex. .SH SEE ALSO .sp diff --git a/man/dcmp.1 b/man/dcmp.1 index 4ccf673b..59ffbe39 100644 --- a/man/dcmp.1 +++ b/man/dcmp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DCMP" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DCMP" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dcmp \- distributed compare . @@ -54,17 +54,17 @@ in which case, each option should provide a different output file name. .INDENT 0.0 .TP .B \-t, \-\-text -Change –output to write files in text format rather than binary. +Change \-\-output to write files in text format rather than binary. .UNINDENT .INDENT 0.0 .TP .B \-b, \-\-base -Enable base checks and normal stdout results when –output is used. +Enable base checks and normal stdout results when \-\-output is used. .UNINDENT .INDENT 0.0 .TP .B \-\-bufsize SIZE -Set the I/O buffer to be SIZE bytes. Units like “MB” and “GB” may +Set the I/O buffer to be SIZE bytes. Units like "MB" and "GB" may immediately follow the number without spaces (e.g. 8MB). The default bufsize is 4MB. .UNINDENT @@ -72,8 +72,8 @@ bufsize is 4MB. .TP .B \-\-chunksize SIZE Multiple processes copy a large file in parallel by dividing it into chunks. -Set chunk to be at minimum SIZE bytes. Units like “MB” and -“GB” can immediately follow the number without spaces (e.g. 64MB). +Set chunk to be at minimum SIZE bytes. Units like "MB" and +"GB" can immediately follow the number without spaces (e.g. 64MB). The default chunksize is 4MB. .UNINDENT .INDENT 0.0 @@ -91,6 +91,11 @@ Use O_DIRECT to avoid caching file data. .UNINDENT .INDENT 0.0 .TP +.B \-\-open\-noatime +Open files with O_NOATIME flag. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-progress N Print progress message to stdout approximately every N seconds. The number of seconds must be a non\-negative integer. @@ -125,7 +130,7 @@ Print the command usage, and the list of options available. .SH EXPRESSIONS .sp An expression is made up of one or more conditions, where each condition specifies a field and a state. -A single condition consists of a field name, an ‘=’ sign, and a state name. +A single condition consists of a field name, an \(aq=\(aq sign, and a state name. .sp Valid fields are listed below, along with the property of the entry that is checked. .TS @@ -288,7 +293,7 @@ CONTENT=COMMON => EXISTS=COMMON@TYPE=COMMON@SIZE=COMMON@CONTENT=COMMON .sp A successful check on any other field also implies that EXIST=COMMON. .sp -When used with the \-o option, one must also specify a file name at the end of the expression, separated with a ‘:’. +When used with the \-o option, one must also specify a file name at the end of the expression, separated with a \(aq:\(aq. The list of any entries that match the expression are written to the named file. For example, to list any entries matching the above expression to a file named outfile1, one should use the following option: @@ -303,7 +308,7 @@ one should use the following option: .UNINDENT .UNINDENT .sp -If the –base option is given or when no output option is specified, +If the \-\-base option is given or when no output option is specified, the following expressions are checked and numeric results are reported to stdout: .INDENT 0.0 .INDENT 3.5 diff --git a/man/dcp.1 b/man/dcp.1 index 0123e7d5..2cb69f66 100644 --- a/man/dcp.1 +++ b/man/dcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DCP" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DCP" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dcp \- distributed copy . @@ -46,7 +46,7 @@ file system, and it splits large file copies across multiple processes. .INDENT 0.0 .TP .B \-\-bufsize SIZE -Set the I/O buffer to be SIZE bytes. Units like “MB” and “GB” may +Set the I/O buffer to be SIZE bytes. Units like "MB" and "GB" may immediately follow the number without spaces (e.g. 8MB). The default bufsize is 4MB. .UNINDENT @@ -54,14 +54,14 @@ bufsize is 4MB. .TP .B \-\-chunksize SIZE Multiple processes copy a large file in parallel by dividing it into chunks. -Set chunk to be at minimum SIZE bytes. Units like “MB” and -“GB” can immediately follow the number without spaces (e.g. 64MB). +Set chunk to be at minimum SIZE bytes. Units like "MB" and +"GB" can immediately follow the number without spaces (e.g. 64MB). The default chunksize is 4MB. .UNINDENT .INDENT 0.0 .TP .B \-\-xattrs WHICH -Copy extended attributes (“xattrs”) from source files to target files. +Copy extended attributes ("xattrs") from source files to target files. WHICH determines which xattrs are copied. Options are to copy no xattrs, all xattrs, xattrs not excluded by /etc/xattr.conf, or all xattrs except those which have special meaning to Lustre. Certain xattrs control Lustre @@ -104,7 +104,7 @@ that each symbolic link refers to. .B \-P, \-\-no\-dereference Do not follow symbolic links in source paths. Effectviely allows symbolic links to be copied when the link target is not valid -or there is not permission to read the link’s target. +or there is not permission to read the link\(aqs target. .UNINDENT .INDENT 0.0 .TP @@ -118,6 +118,11 @@ Use O_DIRECT to avoid caching file data. .UNINDENT .INDENT 0.0 .TP +.B \-\-open\-noatime +Open files with O_NOATIME flag. +.UNINDENT +.INDENT 0.0 +.TP .B \-S, \-\-sparse Create sparse files when possible. .UNINDENT @@ -130,6 +135,18 @@ A value of 0 disables progress messages. .UNINDENT .INDENT 0.0 .TP +.B \-G, \-\-gid GID +Set the effective group ID to perform the copy operation. The copy +may fail if the group does not have sufficient privileges. +.UNINDENT +.INDENT 0.0 +.TP +.B \-U, \-\-uid UID +Set the effective user ID to perform the copy operation. The copy +may fail if the user does not have sufficient privileges. +.UNINDENT +.INDENT 0.0 +.TP .B \-v, \-\-verbose Run in verbose mode. .UNINDENT diff --git a/man/ddup.1 b/man/ddup.1 index f5fb3041..d11c00dd 100644 --- a/man/ddup.1 +++ b/man/ddup.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DDUP" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DDUP" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME ddup \- report files with identical content . @@ -45,6 +45,11 @@ Multiple sets of duplicate files can be matched using this final reported hash. .SH OPTIONS .INDENT 0.0 .TP +.B \-\-open\-noatime +Open files with O_NOATIME flag, if possible. +.UNINDENT +.INDENT 0.0 +.TP .B \-d, \-\-debug LEVEL Set verbosity level. LEVEL can be one of: fatal, err, warn, info, dbg. .UNINDENT diff --git a/man/dfind.1 b/man/dfind.1 index d76e449a..701d9239 100644 --- a/man/dfind.1 +++ b/man/dfind.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DFIND" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DFIND" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dfind \- distributed file filtering . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdfind [OPTION] [EXPRESSION] PATH …\fP +\fBdfind [OPTION] [EXPRESSION] PATH ...\fP .SH DESCRIPTION .sp Parallel MPI application to filter a list of files according to an expression. @@ -57,7 +57,7 @@ Write the processed list to a file. .INDENT 0.0 .TP .B \-t, \-\-text -Must be used with the –output option. Write processed list of files to +Must be used with the \-\-output option. Write processed list of files to FILE in ascii text format. .UNINDENT .INDENT 0.0 @@ -123,22 +123,22 @@ File was last accessed N days ago. .INDENT 0.0 .TP .B \-\-cmin N -File’s status was last changed N minutes ago. +File\(aqs status was last changed N minutes ago. .UNINDENT .INDENT 0.0 .TP .B \-\-cnewer FILE -File’s status was last changed more recently than FILE was modified. +File\(aqs status was last changed more recently than FILE was modified. .UNINDENT .INDENT 0.0 .TP .B \-\-ctime N -File’s status was last changed N days ago. +File\(aqs status was last changed N days ago. .UNINDENT .INDENT 0.0 .TP .B \-\-mmin N -File’s data was last modified N minutes ago. +File\(aqs data was last modified N minutes ago. .UNINDENT .INDENT 0.0 .TP @@ -148,12 +148,12 @@ File was modified more recently than FILE. .INDENT 0.0 .TP .B \-\-mtime N -File’s data was last modified N days ago. +File\(aqs data was last modified N days ago. .UNINDENT .INDENT 0.0 .TP .B \-\-gid N -File’s numeric group ID is N. +File\(aqs numeric group ID is N. .UNINDENT .INDENT 0.0 .TP @@ -163,7 +163,7 @@ File belongs to group NAME. .INDENT 0.0 .TP .B \-\-uid N -File’s numeric user ID is N. +File\(aqs numeric user ID is N. .UNINDENT .INDENT 0.0 .TP @@ -188,7 +188,7 @@ Full path to file matches POSIX regular expression REGEX. Regular expressions p .INDENT 0.0 .TP .B \-\-size N -File size is N bytes. Units can be used like ‘KB’, ‘MB’, ‘GB’. +File size is N bytes. Units can be used like \(aqKB\(aq, \(aqMB\(aq, \(aqGB\(aq. .UNINDENT .INDENT 0.0 .TP @@ -251,7 +251,7 @@ Print file name to stdout. .INDENT 0.0 .TP .B \-\-exec CMD ; -Execute command CMD on file. All following arguments are taken as arguments to the command until ‘;’ is encountered. The string ‘{}’ is replaced by the current file name. +Execute command CMD on file. All following arguments are taken as arguments to the command until \(aq;\(aq is encountered. The string \(aq{}\(aq is replaced by the current file name. .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/man/dreln.1 b/man/dreln.1 index 953adb4e..8ea9418b 100644 --- a/man/dreln.1 +++ b/man/dreln.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DRELN" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DRELN" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dreln \- distributed relink . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdreln [OPTION] OLDPATH NEWPATH PATH …\fP +\fBdreln [OPTION] OLDPATH NEWPATH PATH ...\fP .SH DESCRIPTION .sp Parallel MPI application to recursively update symlinks within a diff --git a/man/drm.1 b/man/drm.1 index 174f9dc4..ecd152d4 100644 --- a/man/drm.1 +++ b/man/drm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DRM" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DRM" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME drm \- distributed remove . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdrm [OPTION] PATH…\fP +\fBdrm [OPTION] PATH...\fP .SH DESCRIPTION .sp Parallel MPI application to recursively delete a directory and its @@ -45,11 +45,11 @@ drm behaves like \fIrm \-rf\fP, but it is faster. .INDENT 0.0 .INDENT 3.5 DO NOT USE SHELL REGEX!!! -The –match and –exclude options use POSIX regex syntax. Because of +The \-\-match and \-\-exclude options use POSIX regex syntax. Because of this make sure that the shell does not try to interpret your regex before it gets passed to the program. You can generally use quotes around your regex to prevent the shell from expanding. An example of this using the -–match option with –dryrun would be: +\-\-match option with \-\-dryrun would be: .sp \fBmpirun \-np 128 drm \-\-dryrun \-v \-\-name \-\-match \(aqfile_.*\(aq /path/to/dir/*\fP .UNINDENT @@ -65,12 +65,12 @@ from the mpiFileUtils suite. .TP .B \-o, \-\-output FILE Write the list of items drm attempts to delete to FILE in mpiFileUtils format. -Format can be changed with –text option. +Format can be changed with \-\-text option. .UNINDENT .INDENT 0.0 .TP .B \-t, \-\-text -Must be used with the –output option. Write list of items drm attempts +Must be used with the \-\-output option. Write list of items drm attempts to delete to FILE in ascii text format. .UNINDENT .INDENT 0.0 @@ -97,22 +97,22 @@ Only remove items whose full path matches REGEX, processed by .INDENT 0.0 .TP .B \-\-name -Change –exclude and match to apply to item name rather than its +Change \-\-exclude and match to apply to item name rather than its full path. .UNINDENT .INDENT 0.0 .TP .B \-\-dryrun Print a list of files that \fBwould\fP be deleted without deleting -them. This is useful to check list of items satisfying –exclude or -–match options before actually deleting anything. +them. This is useful to check list of items satisfying \-\-exclude or +\-\-match options before actually deleting anything. .UNINDENT .INDENT 0.0 .TP .B \-\-aggressive This option will delete files during the walk phase, and then delete directories by level after the walk in drm. You cannot -use this option with –dryrun. +use this option with \-\-dryrun. .UNINDENT .INDENT 0.0 .TP diff --git a/man/dstripe.1 b/man/dstripe.1 index cfe6589c..8fd91744 100644 --- a/man/dstripe.1 +++ b/man/dstripe.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DSTRIPE" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DSTRIPE" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dstripe \- restripe files on underlying storage . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdstripe [OPTION] PATH…\fP +\fBdstripe [OPTION] PATH...\fP .SH DESCRIPTION .sp Parallel MPI application to restripe files. @@ -55,16 +55,16 @@ the lustre file system default is used. The default stripe count is .INDENT 0.0 .TP .B \-s, \-\-size STRIPE_SIZE -The stripe size to use during file restriping. Units like “MB” and -“GB” can immediately follow the number without spaces (ex. 2MB). The +The stripe size to use during file restriping. Units like "MB" and +"GB" can immediately follow the number without spaces (ex. 2MB). The default stripe size is 1MB. .UNINDENT .INDENT 0.0 .TP .B \-m, \-\-minsize SIZE The minimum size a file must be to be a candidate for restriping. -Files smaller than SIZE will not be restriped. Units like “MB” and -“GB” can immediately follow the number without spaces (ex. 2MB). The +Files smaller than SIZE will not be restriped. Units like "MB" and +"GB" can immediately follow the number without spaces (ex. 2MB). The default minimum file size is 0MB. .UNINDENT .INDENT 0.0 diff --git a/man/dsync.1 b/man/dsync.1 index 39380bcd..1a9569e7 100644 --- a/man/dsync.1 +++ b/man/dsync.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DSYNC" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DSYNC" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dsync \- synchronize directory trees . @@ -57,7 +57,7 @@ Batch files into groups of up to size N during copy operation. .INDENT 0.0 .TP .B \-\-bufsize SIZE -Set the I/O buffer to be SIZE bytes. Units like “MB” and “GB” may +Set the I/O buffer to be SIZE bytes. Units like "MB" and "GB" may immediately follow the number without spaces (e.g. 8MB). The default bufsize is 4MB. .UNINDENT @@ -65,14 +65,14 @@ bufsize is 4MB. .TP .B \-\-chunksize SIZE Multiple processes copy a large file in parallel by dividing it into chunks. -Set chunk to be at minimum SIZE bytes. Units like “MB” and -“GB” can immediately follow the number without spaces (e.g. 64MB). +Set chunk to be at minimum SIZE bytes. Units like "MB" and +"GB" can immediately follow the number without spaces (e.g. 64MB). The default chunksize is 4MB. .UNINDENT .INDENT 0.0 .TP .B \-\-xattrs WHICH -Copy extended attributes (“xattrs”) from source files to target files. +Copy extended attributes ("xattrs") from source files to target files. WHICH determines which xattrs are copied. Options are to copy no xattrs, all xattrs, xattrs not excluded by /etc/xattr.conf, or all xattrs except those which have special meaning to Lustre. Certain xattrs control Lustre @@ -110,7 +110,7 @@ that each symbolic link refers to. .B \-P, \-\-no\-dereference Do not follow symbolic links in source paths. Effectviely allows symbolic links to be copied when the link target is not valid -or there is not permission to read the link’s target. +or there is not permission to read the link\(aqs target. .UNINDENT .INDENT 0.0 .TP @@ -119,6 +119,11 @@ Use O_DIRECT to avoid caching file data. .UNINDENT .INDENT 0.0 .TP +.B \-\-open\-noatime +Open files with O_NOATIME flag. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-link\-dest DIR Create hardlink in DEST to files in DIR when file is unchanged rather than create a new file. One can use this option to conserve diff --git a/man/dtar.1 b/man/dtar.1 index 739f7d7b..83cc4b21 100644 --- a/man/dtar.1 +++ b/man/dtar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DTAR" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DTAR" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dtar \- create and extract a tar archive . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdtar [OPTION] \-c \-f ARCHIVE SOURCE…\fP +\fBdtar [OPTION] \-c \-f ARCHIVE SOURCE...\fP .sp \fBdtar [OPTION] \-x \-f ARCHIVE\fP .SH DESCRIPTION @@ -59,8 +59,8 @@ during extraction to benefit subsequent extractions of the same archive file. When extracting an archive, dtar skips the entry corresponding to its index. If other tools, like tar, are used to extract the archive, the index entry is extracted as a regular file that is placed in the current working directory -with a file extension of “.dtaridx” and having the same basename as the original archive file. -For an archive that was named “file.tar” when it was created, the dtar index file is named “file.tar.dtaridx”. +with a file extension of ".dtaridx" and having the same basename as the original archive file. +For an archive that was named "file.tar" when it was created, the dtar index file is named "file.tar.dtaridx". .SH LIMITATIONS .sp dtar only supports directories, regular files, and symlinks. @@ -137,13 +137,18 @@ Default does not record or extract xattrs. .UNINDENT .INDENT 0.0 .TP +.B \-\-open\-noatime +Open source files with O_NOATIME flag when creating archive. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-fsync Call fsync before closing files after writing. .UNINDENT .INDENT 0.0 .TP .B \-\-bufsize SIZE -Set the I/O buffer to be SIZE bytes. Units like “MB” and “GB” may +Set the I/O buffer to be SIZE bytes. Units like "MB" and "GB" may immediately follow the number without spaces (e.g. 8MB). The default bufsize is 4MB. .UNINDENT @@ -151,8 +156,8 @@ bufsize is 4MB. .TP .B \-\-chunksize SIZE Multiple processes copy a large file in parallel by dividing it into chunks. -Set chunk to be at minimum SIZE bytes. Units like “MB” and -“GB” can immediately follow the number without spaces (e.g. 64MB). +Set chunk to be at minimum SIZE bytes. Units like "MB" and +"GB" can immediately follow the number without spaces (e.g. 64MB). The default chunksize is 4MB. .UNINDENT .INDENT 0.0 @@ -161,7 +166,7 @@ The default chunksize is 4MB. Set the memory limit to be SIZE bytes when reading archive files. For some archives, dtar can distribute the file across processes to store segments of the archive in memory for faster processing. -Units like “MB” and “GB” may immediately follow the number +Units like "MB" and "GB" may immediately follow the number without spaces (eg. 8MB). The default is 256MB. .UNINDENT .INDENT 0.0 diff --git a/man/dwalk.1 b/man/dwalk.1 index 3f58d2d5..a1d6cf4d 100644 --- a/man/dwalk.1 +++ b/man/dwalk.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "DWALK" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "DWALK" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME dwalk \- distributed walk and list . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBdwalk [OPTION] PATH …\fP +\fBdwalk [OPTION] PATH ...\fP .SH DESCRIPTION .sp Parallel MPI application to recursively walk and list contents in a @@ -57,12 +57,12 @@ from the mpiFileUtils suite. .TP .B \-o, \-\-output FILE Write the processed list to FILE in binary format. Format can be changed -With –text option. +With \-\-text option. .UNINDENT .INDENT 0.0 .TP .B \-t, \-\-text -Must be used with the –output option. Write processed list of files to +Must be used with the \-\-output option. Write processed list of files to FILE in ascii text format. .UNINDENT .INDENT 0.0 diff --git a/man/mpifileutils.1 b/man/mpifileutils.1 index bc9ab3db..8181c8e7 100644 --- a/man/mpifileutils.1 +++ b/man/mpifileutils.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MPIFILEUTILS" "1" "Feb 04, 2022" "0.11.1" "mpiFileUtils" +.TH "MPIFILEUTILS" "1" "May 31, 2024" "0.12" "mpiFileUtils" .SH NAME mpifileutils \- mpiFileUtils Documentation . @@ -41,7 +41,7 @@ delivering orders of magnitude in performance speedup over their single\-process Furthermore, the libmfu library packages common functionality to simplify the creation of new tools, and it can even be invoked directly from within HPC applications. .sp -Video Overview: \fI\%“Scalable Management of HPC Datasets with mpiFileUtils”\fP, HPCKP‘20. +Video Overview: \fI\%"Scalable Management of HPC Datasets with mpiFileUtils"\fP, HPCKP\(aq20. .sp The figure below, taken from the above presentation, illustrates the potential performance improvement that one can achieve when scaling a tool like dcp to utilize more compute resources. @@ -89,7 +89,7 @@ To be certain of compatibility, it is recommended that one install libarchive\-3 mkdir install installdir=\(gapwd\(ga/install -wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive\-3.5.1.tar.gz +wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive\-3.5.1.tar.gz tar \-zxf libarchive\-3.5.1.tar.gz cd libarchive\-3.5.1 ./configure \-\-prefix=$installdir @@ -103,15 +103,15 @@ cd .. To build on PowerPC, one may need to add \fB\-\-build=powerpc64le\-redhat\-linux\-gnu\fP to the configure command. .sp Assuming libarchive has been installed to an \fIinstall\fP directory as shown above, -one can then build mpiFileUtils from a release like v0.11.1 with commands like the following: +one can then build mpiFileUtils from a release like v0.12 with commands like the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -wget https://github.com/hpc/mpifileutils/releases/download/v0.11.1/mpifileutils\-v0.11.1.tgz -tar \-zxf mpifileutils\-v0.11.1.tgz -cd mpifileutils\-v0.11.1 +wget https://github.com/hpc/mpifileutils/releases/download/v0.12/mpifileutils\-v0.12.tgz +tar \-zxf mpifileutils\-v0.12.tgz +cd mpifileutils\-v0.12 mkdir build cd build cmake .. \e @@ -136,6 +136,8 @@ Additional CMake options: .IP \(bu 2 \fB\-DENABLE_GPFS=[ON/OFF]\fP : specialization for GPFS, defaults to \fBOFF\fP .IP \(bu 2 +\fB\-DENABLE_HPSS=[ON/OFF]\fP : specialization for HPSS, defaults to \fBOFF\fP +.IP \(bu 2 \fB\-DENABLE_EXPERIMENTAL=[ON/OFF]\fP : build experimental tools, defaults to \fBOFF\fP .UNINDENT .SS DAOS support @@ -179,7 +181,7 @@ If HDF5 is installed under a standard system path then specifying the HDF5 path .sp To use \fI\%Spack\fP, it is recommended that one first create a \fIpackages.yaml\fP file to list system\-provided packages, like MPI. Without doing this, Spack will fetch and install an MPI library that may not work on your system. -Make sure that you’ve set up spack in your shell (see \fI\%these instructions\fP). +Make sure that you\(aqve set up Spack in your shell (see \fI\%these instructions\fP). .sp Once Spack has been configured, mpiFileUtils can be installed as: .INDENT 0.0 @@ -199,7 +201,7 @@ or to enable all features: .sp .nf .ft C -spack install mpifileutils +lustre +gpfs +experimental +spack install mpifileutils +lustre +gpfs +hpss +experimental .ft P .fi .UNINDENT @@ -223,7 +225,7 @@ cd deps wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle\-0.3.0.tar.gz wget https://github.com/llnl/lwgrp/releases/download/v1.0.4/lwgrp\-1.0.4.tar.gz wget https://github.com/llnl/dtcmp/releases/download/v1.1.4/dtcmp\-1.1.4.tar.gz - wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive\-3.5.1.tar.gz + wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive\-3.5.1.tar.gz tar \-zxf libcircle\-0.3.0.tar.gz cd libcircle\-0.3.0 @@ -306,7 +308,7 @@ cmake ../mpifileutils .UNINDENT .UNINDENT .sp -The other way to use spack is to create a “view” to the installed dependencies. +The other way to use Spack is to create a "view" to the installed dependencies. Details on this are coming soon. .SS Project Design Principles .sp @@ -331,7 +333,7 @@ careful to learn the options of each tool. .SS Portability .sp The tools are intended to support common file systems used in HPC centers, like -Lustre, GPFS, and NFS. Additionally, methods in the library should be portable +Lustre, GPFS, NFS, and HPSS. Additionally, methods in the library should be portable and efficient across multiple file systems. Tool and library users can rely on mpiFileUtils to provide portable and performant implementations. .SS Composability @@ -351,31 +353,31 @@ mpirun. The tools do not currently checkpoint, so one must be careful that an invocation of the tool has sufficient time to complete before it is killed. .INDENT 0.0 .IP \(bu 2 -dbcast \- Broadcast a file to each compute node. +\fI\%dbcast\fP \- Broadcast a file to each compute node. .IP \(bu 2 -dbz2 \- Compress and decompress a file with bz2. +\fI\%dbz2\fP \- Compress and decompress a file with bz2. .IP \(bu 2 -dchmod \- Change owner, group, and permissions on files. +\fI\%dchmod\fP \- Change owner, group, and permissions on files. .IP \(bu 2 -dcmp \- Compare contents between directories or files. +\fI\%dcmp\fP \- Compare contents between directories or files. .IP \(bu 2 -dcp \- Copy files. +\fI\%dcp\fP \- Copy files. .IP \(bu 2 -ddup \- Find duplicate files. +\fI\%ddup\fP \- Find duplicate files. .IP \(bu 2 -dfind \- Filter files. +\fI\%dfind\fP \- Filter files. .IP \(bu 2 -dreln \- Update symlinks to point to a new path. +\fI\%dreln\fP \- Update symlinks to point to a new path. .IP \(bu 2 -drm \- Remove files. +\fI\%drm\fP \- Remove files. .IP \(bu 2 -dstripe \- Restripe files (Lustre). +\fI\%dstripe\fP \- Restripe files (Lustre). .IP \(bu 2 -dsync \- Synchronize source and destination directories or files. +\fI\%dsync\fP \- Synchronize source and destination directories or files. .IP \(bu 2 -dtar \- Create and extract tape archive files. +\fI\%dtar\fP \- Create and extract tape archive files. .IP \(bu 2 -dwalk \- List, sort, and profile files. +\fI\%dwalk\fP \- List, sort, and profile files. .UNINDENT .SS Experimental Utilities .sp @@ -403,7 +405,7 @@ One must launch the job using the MPI job launcher like mpirun or mpiexec. One .IP \(bu 2 Most tools do not checkpoint their progress. Be sure to request sufficient time in your allocation to allow the job to complete. One may need to start over from the beginning if a tool is interrupted. .IP \(bu 2 -One cannot pipe output of one tool to the input of another. However, the –input and –output file options are good approximations. +One cannot pipe output of one tool to the input of another. However, the \-\-input and \-\-output file options are good approximations. .IP \(bu 2 One cannot easily check the return codes of tools. Instead, inspect stdout and stderr output for errors. .UNINDENT @@ -538,7 +540,7 @@ dchmod \-\-group grp1 \-\-mode g+rw /path/to/walk .UNINDENT .UNINDENT .sp -drm is like “rm \-rf” but in parallel: +drm is like "rm \-rf" but in parallel: .INDENT 0.0 .INDENT 3.5 .sp @@ -575,7 +577,7 @@ dsync /path/src /path/dest .UNINDENT .UNINDENT .sp -For large directory trees, the –batch\-files option offers a type of checkpoint. +For large directory trees, the \-\-batch\-files option offers a type of checkpoint. It moves files in batches, and if interrupted, a restart picks up from the last completed batch.: .INDENT 0.0 .INDENT 3.5 @@ -588,7 +590,7 @@ dsync \-\-batch\-files 100000 /path/src /path/dest .UNINDENT .UNINDENT .sp -The tools can be composed in various ways using the –input and –output options. +The tools can be composed in various ways using the \-\-input and \-\-output options. For example, the following sequence of commands executes a purge operation, which deletes any file that has not been accessed in the past 180 days.: .INDENT 0.0 @@ -648,9 +650,9 @@ entries from one location to another or compare corresponding entries across two different lists. A file list can be serialized and written to or read from a file. .sp -Each MPI rank “owns” a portion of the list, and there are routines to step +Each MPI rank "owns" a portion of the list, and there are routines to step through the entries owned by that process. This portion is referred to as the -“local” list. Functions exist to get and set properties of the items in the +"local" list. Functions exist to get and set properties of the items in the local list, for example to get the path name, type, and size of a file. Functions dealing with the local list can be called by the MPI process independently of other MPI processes. diff --git a/mpifileutils.spec b/mpifileutils.spec index 94619f00..acd3290c 100644 --- a/mpifileutils.spec +++ b/mpifileutils.spec @@ -1,5 +1,5 @@ Name: mpifileutils -Version: 0.11.1 +Version: 0.12 Release: 1%{?dist} Summary: File utilities designed for scalability and performance.