mover diff 6.11 to 6.12

Created Diff never expires
37 removals
149 lines
3 additions
131 lines
#!/bin/bash
#!/bin/bash
#Copyright 2005-2020, Lime Technology
#Copyright 2005-2020, Lime Technology
#License: GPLv2 only
#License: GPLv2 only


# This is the 'mover' script used for moving files between a pool and the main unRAID array.
# This is the 'mover' script used for moving files between a pool and the main unRAID array.
# It is typically invoked via cron.
# It is typically invoked via cron.


# First we check if it's valid for this script run: pool use in shfs must be enabled and
# First we check if it's valid for this script run: pool use in shfs must be enabled and
# an instance of the script must not already be running.
# an instance of the script must not already be running.


# Next, check each of the top-level directories (shares) on each pool.
# Next, check each of the top-level directories (shares) on each pool.
# If, and only if, the 'Use Cache' setting for the share is set to "yes", we use 'find' to
# If, and only if, the 'Use Cache' setting for the share is set to "yes", we use 'find' to
# list the objects (files and directories) of that share directory, moving them to the array.
# list the objects (files and directories) of that share directory, moving them to the array.
# Next, we check each of the top-level directories (shares) on each array disk (in sorted order).
# Next, we check each of the top-level directories (shares) on each array disk (in sorted order).
# If, and only if, the 'Use Cache' setting for the share is set to "prefer", we use 'find' to
# If, and only if, the 'Use Cache' setting for the share is set to "prefer", we use 'find' to
# list the objects (files and directories) of that share directory, moving them to the pool
# list the objects (files and directories) of that share directory, moving them to the pool
# associted with the share.
# associted with the share.


# The script is set up so that hidden directories (i.e., directory names beginning with a '.'
# The script is set up so that hidden directories (i.e., directory names beginning with a '.'
# character) at the topmost level of a pool or an array disk are not moved. This behavior
# character) at the topmost level of a pool or an array disk are not moved. This behavior
# can be turned off by uncommenting the following line:
# can be turned off by uncommenting the following line:
# shopt -s dotglob
# shopt -s dotglob


# Files at the top level of a pool or an array disk are never moved.
# Files at the top level of a pool or an array disk are never moved.


# The 'find' command generates a list of all files and directories of a share.
# The 'find' command generates a list of all files and directories of a share.
# For each file, if the file is not "in use" by any process (as detected by 'in_use' command),
# For each file, if the file is not "in use" by any process (as detected by 'in_use' command),
# then the file is moved, and upon success, deleted from the source disk. If the file already
# then the file is moved, and upon success, deleted from the source disk. If the file already
# exists on the target, it is not moved and the source is not deleted. All meta-data of moved
# exists on the target, it is not moved and the source is not deleted. All meta-data of moved
# files/directories is preserved: permissions, ownership, extended attributes, and access/modified
# files/directories is preserved: permissions, ownership, extended attributes, and access/modified
# timestamps.
# timestamps.


# If an error occurs in copying a file, the partial file, if present, is deleted and the
# If an error occurs in copying a file, the partial file, if present, is deleted and the
# operation continues on to the next file.
# operation continues on to the next file.


PIDFILE="/var/run/mover.pid"
PIDFILE="/var/run/mover.pid"
CFGFILE="/boot/config/share.cfg"
CFGFILE="/boot/config/share.cfg"
LOGLEVEL=0
DEBUGGING=""


start() {
start() {
echo "mover: lets move some stuff"

if [ -f $PIDFILE ]; then
if [ -f $PIDFILE ]; then
if ps h $(cat $PIDFILE) | grep mover ; then
if ps h $(cat $PIDFILE) | grep mover ; then
echo "mover: already running"
echo "mover: already running"
exit 1
exit 1
fi
fi
fi
fi


if [ -f $CFGFILE ]; then
if [ -f $CFGFILE ]; then
# Only start if shfs includes pools
# Only start if shfs includes pools
if ! grep -qs 'shareCacheEnabled="yes"' $CFGFILE ; then
if ! grep -qs 'shareCacheEnabled="yes"' $CFGFILE ; then
echo "mover: cache not enabled"
echo "mover: cache not enabled"
exit 2
exit 2
fi
fi
if grep -qs 'shareMoverLogging="yes"' $CFGFILE ; then
LOGLEVEL=1
fi
fi
fi
if ! mountpoint -q /mnt/user0 ; then
if ! mountpoint -q /mnt/user0 ; then
echo "mover: array devices not mounted"
echo "mover: array devices not mounted"
exit 3
exit 3
fi
fi


echo $$ >/var/run/mover.pid
echo $$ >/var/run/mover.pid
[[ $LOGLEVEL -gt 0 ]] && echo "mover: started"
echo "mover: started"


shopt -s nullglob
shopt -s nullglob


# This script was modified to ignore a single share, `data` but respect all other
# Unraid share's that want to move data from any cache pool to the array
# Context: `data` contains multimedia that we want to remain on the cache
# Context: and rely on Unraid's plugin mover tuning to only run mover when cache is >70% used
# Context: without forcing all other shares to leave data on cache during that time period
# Check for objects to move from pools to array
# Check for objects to move from pools to array
for POOL in /boot/config/pools/*.cfg ; do
for POOL in /boot/config/pools/*.cfg ; do
for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do
for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do
SHARE=$(basename "$SHAREPATH")
SHARE=$(basename "$SHAREPATH")

if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then
if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then
if [[ $SHARE == 'data' ]]
find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING
then
echo "- Skipping share: $SHARE"
continue
fi
echo "+ Processing share: $SHARE"
find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL
fi
fi
done
done
done
done


# Check for objects to move from array to pools
# Check for objects to move from array to pools
ls -dvc1 /mnt/disk[0-9]*/*/ | while read SHAREPATH ; do
ls -dvc1 /mnt/disk[0-9]*/*/ | while read SHAREPATH ; do
SHARE=$(basename "$SHAREPATH")
SHARE=$(basename "$SHAREPATH")
echo $SHARE
if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then
if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then
eval $(grep -s shareCachePool "/boot/config/shares/${SHARE}.cfg" | tr -d '\r')
eval $(grep -s shareCachePool "/boot/config/shares/${SHARE}.cfg" | tr -d '\r')
if [[ -z "$shareCachePool" ]]; then
if [[ -z "$shareCachePool" ]]; then
shareCachePool="cache"
shareCachePool="cache"
fi
fi
if [[ -d "/mnt/$shareCachePool" ]]; then
if [[ -d "/mnt/$shareCachePool" ]]; then
find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING
find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL
fi
fi
fi
fi
done
done


rm -f $PIDFILE
rm -f $PIDFILE
[[ $LOGLEVEL -gt 0 ]] && echo "mover: finished"
echo "mover: finished"
}
}


killtree() {
killtree() {
local pid=$1 child
local pid=$1 child
for child in $(pgrep -P $pid); do
for child in $(pgrep -P $pid); do
killtree $child
killtree $child
done
done
[ $pid -ne $$ ] && kill -TERM $pid
[ $pid -ne $$ ] && kill -TERM $pid
}
}


# Caution: stopping mover like this can lead to partial files on the destination
# Caution: stopping mover like this can lead to partial files on the destination
# and possible incomplete hard link transfer. Not recommended to do this.
# and possible incomplete hard link transfer. Not recommended to do this.
stop() {
stop() {
if [ ! -f $PIDFILE ]; then
if [ ! -f $PIDFILE ]; then
echo "mover: not running"
echo "mover: not running"
exit 0
exit 0
fi
fi
killtree $(cat $PIDFILE)
killtree $(cat $PIDFILE)
sleep 2
sleep 2
rm -f $PIDFILE
rm -f $PIDFILE
echo "mover: stopped"
echo "mover: stopped"
}
}


case $1 in
case $1 in
start)
start)
start
start
;;
;;
stop)
stop)
stop
stop
;;
;;
status)
status)
[ -f $PIDFILE ]
[ -f $PIDFILE ]
;;
;;
*)
*)
# Default is "start"
# Default is "start"
# echo "Usage: $0 (start|stop|status)"
# echo "Usage: $0 (start|stop|status)"
start
start
;;
;;
esac
esac