-
Notifications
You must be signed in to change notification settings - Fork 78
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce incremental index write down in XML format (#466)
- Loading branch information
Atsushi Abe
authored
Jun 21, 2024
1 parent
06367df
commit 8c8f6cc
Showing
21 changed files
with
1,320 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # Unit Tests for incremental index | ||
|
|
||
| This is a directory for having unit tests for incremental index feature. | ||
|
|
||
| ## How to run | ||
|
|
||
| ### Basic operation test | ||
|
|
||
| 1. `cd` to this directory | ||
| 2. Run the basic test with `./ut-basic.sh [mount_point]` | ||
| - The test script formats a (filebackend) tape under `/tmp/ltfstape`, start ltfs and stop ltfs automatically. | ||
| - If `[mount_point]` is not specified, the script uses `/tmp/mnt` | ||
| - You can pass the specific `ltfs` binary directory with teh environmental value `LTFS_BIN_PATH` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| #!/bin/sh | ||
|
|
||
| source ./utils.sh | ||
|
|
||
| MOUNTPOINT='/tmp/mnt' | ||
| TAPE_PATH='/tmp/ltfstape' | ||
|
|
||
| if [ $# == 1 ]; then | ||
| MOUNTPOINT=$1 | ||
| elif [ $# -gt 2 ]; then | ||
| MOUNTPOINT=$1 | ||
| TAPE_PATH=$2 | ||
| fi | ||
|
|
||
| # Format LTFS | ||
| FormatLTFS ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Launch LTFS | ||
| LaunchLTFS ${MOUNTPOINT} ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 1. CREATE DIRS | ||
| # Create a few dirs and files but all objects are handles by new dirs | ||
| echo "1. CREATE DIRS" | ||
| mkdir -p ${MOUNTPOINT}/dir1/dir11 | ||
| mkdir -p ${MOUNTPOINT}/dir1/dir12 | ||
| mkdir -p ${MOUNTPOINT}/dir2/dir21 | ||
| mkdir -p ${MOUNTPOINT}/dir2/dir22 | ||
| echo "AAA" > ${MOUNTPOINT}/dir1/file11 | ||
| echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file111 | ||
| echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file112 | ||
| IncrementalSync ${MOUNTPOINT} '1.CREATE_DIRS' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 2. CREATE FILES | ||
| # Create files for checking file creation and directory traverse | ||
| echo "2. CREATE FILES" | ||
| echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file113 | ||
| echo "AAA" > ${MOUNTPOINT}/dir1/dir12/file121 | ||
| echo "AAA" > ${MOUNTPOINT}/dir2/dir22/file221 | ||
| echo "AAA" > ${MOUNTPOINT}/dir2/file21 | ||
| IncrementalSync ${MOUNTPOINT} '2.CREATE_FILES' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 3. MODIFY FILES | ||
| # Modify contents of files. Need to check /dir2 doesn't have meta-data on the incremental index | ||
| echo "3. MODIFY FILES" | ||
| echo "BBB" > ${MOUNTPOINT}/dir1/dir11/file111 | ||
| echo "BBB" > ${MOUNTPOINT}/dir2/dir22/file221 | ||
| echo "BBB" > ${MOUNTPOINT}/dir1/file11 | ||
| IncrementalSync ${MOUNTPOINT} '3.MODIFY_FILES' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 4. MODIFY DIRS | ||
| # Modify directory's meta-data. Need to check both /dir1 and /dir1/dir11 has meta-data | ||
| # on the incremental index | ||
| echo "4. MODIFY DIRS" | ||
| AddXattr ${MOUNTPOINT}/dir1 'ut-attr1' 'val1' | ||
| echo "CCC" > ${MOUNTPOINT}/dir1/dir11/file111 | ||
| IncrementalSync ${MOUNTPOINT} '4.MODIFY_DIRS' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 5. DELETE FILES | ||
| echo "5. DELETE FILES" | ||
| rm ${MOUNTPOINT}/dir1/dir11/* | ||
| IncrementalSync ${MOUNTPOINT} '5.DELETE_FILES' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # 6. DELETE DIR | ||
| echo "5. DELETE DIR" | ||
| rm -rf ${MOUNTPOINT}/dir1/dir11 | ||
| IncrementalSync ${MOUNTPOINT} '6.DELETE_DIR' | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Stop LTFS | ||
| StopLTFS ${MOUNTPOINT} ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| exit 1 | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,197 @@ | ||
| #!/bin/sh | ||
|
|
||
| PLATFORM=`uname` | ||
| ECHO='/bin/echo' | ||
|
|
||
| if [ "x${LTFS_BIN_PATH}" == 'x' ]; then | ||
| LTFS_BIN_PATH='/usr/local/bin' | ||
| fi | ||
|
|
||
| AddXattr() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| "Need to a target to set xattr" | ||
| return 1 | ||
| else | ||
| TARGET=$1 | ||
| fi | ||
|
|
||
| if [ "x$2" == "x" ]; then | ||
| "Need to a name to set xattr" | ||
| return 1 | ||
| else | ||
| NAME=$2 | ||
| fi | ||
|
|
||
| if [ "x$3" == "x" ]; then | ||
| "Need to a value to set xattr" | ||
| return 1 | ||
| else | ||
| VAL=$2 | ||
| fi | ||
|
|
||
| ${ECHO} -n "Setting a xattr ${NAME} to ${TARGET} ... " | ||
| if [ "$PLATFORM" == "Darwin" ]; then | ||
| /usr/bin/xattr -w ${NAME} ${VAL} ${TARGET} | ||
| else | ||
| /usr/bin/attr -s ${NAME} -V ${VAL} ${TARGET} | ||
| fi | ||
|
|
||
| if [ $? == 0 ]; then | ||
| ${ECHO} "Done" | ||
| return 0 | ||
| else | ||
| ${ECHO} "Failed" | ||
| return 1 | ||
| fi | ||
|
|
||
| } | ||
|
|
||
| IncrementalSync() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| MOUNTPOINT='/tmp/mnt' | ||
| else | ||
| MOUNTPOINT=$1 | ||
| fi | ||
|
|
||
| if [ "x$2" == "x" ]; then | ||
| MSG='Incremental Sync' | ||
| else | ||
| MSG=$2 | ||
| fi | ||
|
|
||
| ${ECHO} -n "Syncing LTFS (Incremental) ... " | ||
| if [ "$PLATFORM" == "Darwin" ]; then | ||
| /usr/bin/xattr -w ltfs.vendor.IBM.IncrementalSync ${MSG} ${MOUNTPOINT} | ||
| else | ||
| /usr/bin/attr -s ltfs.vendor.IBM.IncrementalSync -V ${MSG} ${MOUNTPOINT} | ||
| fi | ||
|
|
||
| if [ $? == 0 ]; then | ||
| ${ECHO} "Done" | ||
| return 0 | ||
| else | ||
| ${ECHO} "Failed" | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| FullSync() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| MOUNTPOINT='/tmp/mnt' | ||
| else | ||
| MOUNTPOINT=$1 | ||
| fi | ||
|
|
||
| if [ "x$2" == "x" ]; then | ||
| MSG='Full Sync' | ||
| else | ||
| MSG=$2 | ||
| fi | ||
|
|
||
| ${ECHO} "Syncing LTFS (Full) ... " | ||
| if [ "$PLATFORM" == "Darwin" ]; then | ||
| /usr/bin/xattr -w ltfs.vendor.IBM.FullSync ${MSG} ${MOUNTPOINT} | ||
| else | ||
| /usr/bin/attr -s ltfs.vendor.IBM.FullSync -V ${MSG} ${MOUNTPOINT} | ||
| fi | ||
|
|
||
| if [ $? == 0 ]; then | ||
| ${ECHO} "Done" | ||
| return 0 | ||
| else | ||
| ${ECHO} "Failed" | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| FormatLTFS() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| TAPE_PATH='/tmp/ltfstape' | ||
| else | ||
| TAPE_PATH=$1 | ||
| fi | ||
|
|
||
| if [ ! -d ${TAPE_PATH} ]; then | ||
| ${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}" | ||
| mkdir -p ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to create a tape path: ${TAPE_PATH}" | ||
| return 1 | ||
| fi | ||
| fi | ||
|
|
||
| ${ECHO} "Formatting tape directory with the file backend on ${TAPE_PATH} ... " | ||
| ${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to format a tape path: ${TAPE_PATH}" | ||
| return 1 | ||
| fi | ||
|
|
||
| ${ECHO} "Formatted the file backend on ${TAPE_PATH}" | ||
| return 0 | ||
| } | ||
|
|
||
| LaunchLTFS() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| MOUNTPOINT='/tmp/mnt' | ||
| else | ||
| MOUNTPOINT=$1 | ||
| fi | ||
|
|
||
| if [ "x$2" == "x" ]; then | ||
| TAPE_PATH='/tmp/ltfstape' | ||
| else | ||
| TAPE_PATH=$2 | ||
| fi | ||
|
|
||
| if [ ! -d ${MOUNTPOINT} ]; then | ||
| ${ECHO} "Creating mount point for LTFS: ${MOUNTPOINT}" | ||
| mkdir -p ${MOUNTPOINT} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to create a mount point" | ||
| return 1 | ||
| fi | ||
| fi | ||
|
|
||
| if [ ! -d ${TAPE_PATH} ]; then | ||
| ${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}" | ||
| mkdir -p ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to create a tape path: ${TAPE_PATH}" | ||
| return 1 | ||
| fi | ||
|
|
||
| ${ECHO} "Formatting tape directory with the file backend" | ||
| ${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to format a tape path: ${TAPE_PATH}" | ||
| return 1 | ||
| fi | ||
| fi | ||
|
|
||
| ${ECHO} "Launching LTFS with the file backend" | ||
| ${LTFS_BIN_PATH}/ltfs -o tape_backend=file -o sync_type=unmount -o devname=${TAPE_PATH} ${MOUNTPOINT} | ||
| if [ $? != 0 ]; then | ||
| ${ECHO} "Failed to launch LTFS on ${MOUNTPOINT}" | ||
| return 1 | ||
| fi | ||
|
|
||
| ${ECHO} "LTFS is launched on ${MOUNTPOINT}" | ||
| return 0 | ||
| } | ||
|
|
||
| StopLTFS() | ||
| { | ||
| if [ "x$1" == "x" ]; then | ||
| MOUNTPOINT='/tmp/mnt' | ||
| else | ||
| MOUNTPOINT=$1 | ||
| fi | ||
|
|
||
| sudo umount ${MOUNTPOINT} | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.