mirror of
https://github.com/sbopkg/sbopkg
synced 2024-09-19 22:40:19 +03:00
sqg: Parallelize building of all packages by option -j.
Parallelization requires GNU parallel to be installed. Signed-off-by: Marcel Saegebarth <marc@mos6581.de> Signed-off-by: Willy Sudiarto Raharjo <willysr@sbopkg.org>
This commit is contained in:
Marcel Saegebarth
Willy Sudiarto Raharjo
parent
e0aad3b78f
commit
1eb39ee8eb
@ -5,7 +5,7 @@ SBOPKG COPYRIGHT/LICENSE
|
||||
Copyright 2007-2010 Chess Griffin <chess@chessgriffin.com>
|
||||
Copyright 2009-2011 Mauro Giachero <mauro.giachero@gmail.com>
|
||||
slakmagik <slakmagik@gmail.com>
|
||||
Copyright 2016 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
Copyright 2016-2017 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
|
||||
Sbopkg is released under a one-clause BSD license. See the script
|
||||
itself for the text.
|
||||
|
||||
@ -87,3 +87,5 @@ We moved to github, so revision number no longer used
|
||||
pyllyukko
|
||||
Dhaby Xiloj
|
||||
Chris Abela
|
||||
Sergey V.
|
||||
Marcel Saegebarth
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
# Copyright 2007-2010,2013 Chess Griffin <chess@chessgriffin.com>
|
||||
# Copyright 2009-2011 Mauro Giachero <mauro.giachero@gmail.com>
|
||||
# Copyright 2009-2013 slakmagik <slakmagik@gmail.com>
|
||||
# Copyright 2015-2016 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
# Copyright 2015-2017 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
#
|
||||
# Redistribution and use of this script, with or without modification, is
|
||||
# permitted provided that the following conditions are met:
|
||||
|
||||
151
src/usr/sbin/sqg
151
src/usr/sbin/sqg
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/bin/sh
|
||||
|
||||
# $Id$
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
|
||||
# Copyright 2013 Chess Griffin <chess@chessgriffin.com> and
|
||||
# slakmagik <slakmagik@gmail.com>
|
||||
# Copyright 2016 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
# Copyright 2016-2017 Willy Sudiarto Raharjo <willysr@sbopkg.org>
|
||||
|
||||
# Redistribution and use of this script, with or without modification, is
|
||||
# permitted provided that the following conditions are met:
|
||||
@ -27,8 +27,6 @@
|
||||
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
SCRIPT=${0##*/}
|
||||
|
||||
### OPTIONAL CONFIGURATION BELOW ###
|
||||
|
||||
SBOPKG_CONF=${SBOPKG_CONF:-/etc/sbopkg/sbopkg.conf}
|
||||
@ -44,138 +42,31 @@ SBOPKG_CONF=${SBOPKG_CONF:-/etc/sbopkg/sbopkg.conf}
|
||||
# contain the package name in the queuefile since there are no REQUIRES).
|
||||
|
||||
#QUEUEDIR=${QUEUEDIR:-/var/lib/sbopkg/queues}
|
||||
REPO_ROOT=${REPO_ROOT:-/var/lib/sbopkg}
|
||||
REPO_NAME=${REPO_NAME:-SBo}
|
||||
REPO_BRANCH=${REPO_BRANCH:-14.2}
|
||||
REPO_BRANCH=${REPO_BRANCH:-$(cat /etc/slackware-version | awk '{print $2}')}
|
||||
#SKIP_EMPTY=${SKIP_EMPTY:-NO}
|
||||
|
||||
### NO CHANGES SHOULD BE NECESSARY BELOW THIS LINE ###
|
||||
|
||||
sanity_checks () {
|
||||
if [[ ! -e $SBOPKG_CONF ]]; then
|
||||
echo "$SBOPKG_CONF not found."
|
||||
echo "Check the configurable variables at the top of the script."
|
||||
exit 1
|
||||
else
|
||||
. $SBOPKG_CONF
|
||||
fi
|
||||
if [ -d $REPO_ROOT/$REPO_NAME/.git ]; then
|
||||
REPO_SUBPATH=$REPO_NAME
|
||||
else
|
||||
REPO_SUBPATH=$REPO_NAME/$REPO_BRANCH
|
||||
fi
|
||||
REPO_DIR=$REPO_ROOT/$REPO_SUBPATH
|
||||
if [[ ! -w $QUEUEDIR || ! -d $REPO_DIR ]]; then
|
||||
echo "ERROR: $QUEUEDIR or $REPO_DIR do not exist or are not writable."
|
||||
echo "Check the configurable variables at the top of the script."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
# source all sqg functions
|
||||
. /usr/share/sbopkg/sqg/functions
|
||||
|
||||
usage() {
|
||||
cat << EOF
|
||||
Usage: $SCRIPT [-a] to build all queuefiles or [-p <package>] for individual
|
||||
packages. Multiple packages can be passed with quotes, e.g. -p "pkg1" "pkg2".
|
||||
Packages with an empty REQUIRES= line will be skipped unless SKIP_EMPTY is
|
||||
uncommented at the top of the script. Package names are case-sensitive. Use
|
||||
'sbopkg -g pkg' to search if needed.
|
||||
|
||||
This script will overwrite existing queuefiles in \$QUEUEDIR so back up any
|
||||
existing queuefiles or local modifications. Check the top of the script for
|
||||
configurable variables.
|
||||
EOF
|
||||
exit
|
||||
}
|
||||
|
||||
search_package () {
|
||||
local SRCHAPP="$1"
|
||||
|
||||
cd $REPO_DIR
|
||||
PKGPATH=( $(find -type d -mindepth 2 -maxdepth 2 -name "$SRCHAPP" | sort) )
|
||||
if [[ -z $PKGPATH ]]; then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
parse_queuefile_requires () {
|
||||
local PARSEAPP="$1"
|
||||
local DEPLIST DEP
|
||||
|
||||
if search_package $PARSEAPP; then
|
||||
. $REPO_DIR/$PKGPATH/$PARSEAPP.info
|
||||
DEPLIST=($REQUIRES)
|
||||
for DEP in "${DEPLIST[@]}"; do
|
||||
if search_package $DEP; then
|
||||
sed -i "/^$DEP$/ d" $QUEUEFILE
|
||||
echo "$DEP" >> $QUEUEFILE
|
||||
parse_queuefile_requires $DEP
|
||||
elif [[ "$DEP" == "%README%" ]]; then
|
||||
echo "# %README%: see the $PARSEAPP README file. " >> $QUEUEFILE
|
||||
fi
|
||||
done
|
||||
else
|
||||
continue
|
||||
fi
|
||||
}
|
||||
|
||||
build_queuefile () {
|
||||
OLDFILE=$QUEUEDIR/$PRGNAM.sqf
|
||||
QUEUEFILE=$QUEUEDIR/$PRGNAM.sqf.tmp
|
||||
CATFILE=$QUEUEDIR/tmp.sqf
|
||||
touch $QUEUEFILE
|
||||
echo "$PRGNAM" > $QUEUEFILE
|
||||
parse_queuefile_requires $PRGNAM
|
||||
tac $QUEUEFILE > $CATFILE
|
||||
if cmp -s "$CATFILE" "$OLDFILE"; then
|
||||
rm $QUEUEFILE $CATFILE
|
||||
else
|
||||
mv $CATFILE $OLDFILE
|
||||
rm $QUEUEFILE
|
||||
fi
|
||||
}
|
||||
|
||||
main_loop () {
|
||||
local MAINAPP="$1" INFO
|
||||
if [[ $MAINAPP == "_all" ]]; then
|
||||
printf "Processing all SlackBuilds in the $REPO_SUBPATH repo..."
|
||||
for INFO in $(find $REPO_DIR -name *.info); do
|
||||
. $INFO
|
||||
if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then
|
||||
continue
|
||||
else
|
||||
printf "."
|
||||
build_queuefile
|
||||
fi
|
||||
done
|
||||
else
|
||||
for MAINAPP in $PKG; do
|
||||
if search_package $MAINAPP; then
|
||||
. $REPO_DIR/$PKGPATH/$MAINAPP.info
|
||||
if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then
|
||||
echo "$MAINAPP: empty REQUIRES line. No queuefile generated."
|
||||
echo "Uncomment SKIP_EMPTY to change this."
|
||||
continue
|
||||
else
|
||||
echo "Processing $MAINAPP."
|
||||
build_queuefile
|
||||
fi
|
||||
else
|
||||
echo "$MAINAPP: not found."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
case $1 in
|
||||
-a) shift; PKG="_all" ;;
|
||||
-p) shift; PKG="$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
SCRIPT=${0##*/}
|
||||
|
||||
sanity_checks
|
||||
main_loop "$PKG"
|
||||
|
||||
case $1 in
|
||||
-a)
|
||||
if [ "$2" == "-j" ] && [ -n $3 ]; then
|
||||
get_jobs $3
|
||||
JOBS=$?
|
||||
fi
|
||||
shift;
|
||||
execute_build "$REPO_DIR" "$QUEUEDIR" "" "yes" $JOBS ;;
|
||||
-p)
|
||||
shift;
|
||||
execute_build "$REPO_DIR" "$QUEUEDIR" "$@" "no" $JOBS ;;
|
||||
*) usage $SCRIPT ;;
|
||||
esac
|
||||
|
||||
echo "Done."
|
||||
exit 0
|
||||
|
||||
180
src/usr/share/sbopkg/sqg/functions
Normal file
180
src/usr/share/sbopkg/sqg/functions
Normal file
@ -0,0 +1,180 @@
|
||||
#!/bin/sh
|
||||
|
||||
sanity_checks () {
|
||||
if [ ! -e "$SBOPKG_CONF" ]; then
|
||||
echo "$SBOPKG_CONF not found."
|
||||
echo "Check the configurable variables at the top of the script."
|
||||
exit 1
|
||||
else
|
||||
. $SBOPKG_CONF
|
||||
fi
|
||||
|
||||
if [ -d $REPO_ROOT/$REPO_NAME/.git ]; then
|
||||
REPO_SUBPATH=$REPO_NAME
|
||||
else
|
||||
REPO_SUBPATH=$REPO_NAME/$REPO_BRANCH
|
||||
fi
|
||||
|
||||
REPO_DIR=$REPO_ROOT/$REPO_SUBPATH
|
||||
if [ ! -w "$QUEUEDIR" ] || [ ! -d "$REPO_DIR" ]; then
|
||||
echo "ERROR: $QUEUEDIR or $REPO_DIR do not exist or are not writable."
|
||||
echo "Check the configurable variables at the top of the script."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
has_parallel () {
|
||||
parallel --help &> /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
usage () {
|
||||
local SCRIPT="$1"
|
||||
|
||||
cat << EOF
|
||||
Usage: $SCRIPT -p <packagename(s)> | -a [-j]
|
||||
Options are:
|
||||
-p package(s) Creates queuefile(s) for individual package(s).
|
||||
Multiple packages can be passed with quotes,
|
||||
e.g. -p "pkg1 pkg2".
|
||||
Packages with an empty REQUIRES= line will be skipped unless
|
||||
SKIP_EMPTY is uncommented at the top of the script.
|
||||
Package names are case-sensitive.
|
||||
-a Builds queuefiles for all packages.
|
||||
-j Number of concurrent queuefile generation processes.
|
||||
This could heavily raise disk I/O so use with care.
|
||||
Requires: GNU parallel
|
||||
Default: 1
|
||||
|
||||
This script will overwrite existing queuefiles in \$QUEUEDIR so back up any
|
||||
existing queuefile(s) or local modifications. Check the top of the script for
|
||||
configurable variables.
|
||||
EOF
|
||||
exit
|
||||
}
|
||||
|
||||
get_jobs () {
|
||||
local JOBS=${1:-1}
|
||||
|
||||
if [ ! $JOBS -ge 1 ]; then
|
||||
echo "ERROR: Value for -j must be an unsigned integer > 0."
|
||||
exit 1
|
||||
else
|
||||
JOBS=$1
|
||||
fi
|
||||
|
||||
return $JOBS
|
||||
}
|
||||
|
||||
search_package () {
|
||||
local REPO_DIR="$1"
|
||||
local SRCHAPP="$2"
|
||||
|
||||
cd $REPO_DIR
|
||||
PKGPATH=($(find -type d -mindepth 2 -maxdepth 2 -name "$SRCHAPP" | sort))
|
||||
if [ -z "$PKGPATH" ]; then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
parse_queuefile_requires () {
|
||||
local REPO_DIR="$1"
|
||||
local PARSEAPP="$2"
|
||||
local QUEUEFILE="$3"
|
||||
local DEPLIST DEP RESPONSE
|
||||
|
||||
if search_package "$REPO_DIR" "$PARSEAPP"; then
|
||||
. $REPO_DIR/$PKGPATH/$PARSEAPP.info
|
||||
DEPLIST=($REQUIRES)
|
||||
for DEP in "${DEPLIST[@]}"; do
|
||||
if [ "$DEP" == "%README%" ]; then
|
||||
echo "# %README%: see the $PARSEAPP README file. " >> $QUEUEFILE
|
||||
continue
|
||||
fi
|
||||
|
||||
if search_package "$REPO_DIR" "$DEP"; then
|
||||
sed -i "/^$DEP$/ d" $QUEUEFILE
|
||||
echo "$DEP" >> $QUEUEFILE
|
||||
parse_queuefile_requires "$REPO_DIR" "$DEP" "$QUEUEFILE"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "$PARSEAPP: not found."
|
||||
read -s -p ' Do you want to continue? [y/N]' RESPONSE
|
||||
RESPONSE=$(echo "$RESPONSE" | awk '{print tolower($0)}')
|
||||
|
||||
if [ "$RESPONSE" =~ ^\(yes|y\)$ ]; then
|
||||
continue
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
build_queuefile () {
|
||||
local REPO_DIR="$1"
|
||||
local QUEUEDIR="$2"
|
||||
local PRGNAM="$3"
|
||||
local OLDFILE=$QUEUEDIR/$PRGNAM.sqf
|
||||
local QUEUEFILE=$QUEUEDIR/$PRGNAM.sqf.tmp
|
||||
local CATFILE=$QUEUEDIR/$PRGNAM.tmp.sqf
|
||||
|
||||
touch $QUEUEFILE
|
||||
echo "$PRGNAM" > $QUEUEFILE
|
||||
parse_queuefile_requires "$REPO_DIR" "$PRGNAM" "$QUEUEFILE"
|
||||
tac $QUEUEFILE > $CATFILE
|
||||
if cmp -s "$CATFILE" "$OLDFILE"; then
|
||||
rm $QUEUEFILE $CATFILE
|
||||
else
|
||||
mv $CATFILE $OLDFILE
|
||||
rm $QUEUEFILE
|
||||
fi
|
||||
}
|
||||
|
||||
execute_build () {
|
||||
local REPO_DIR="$1"
|
||||
local QUEUEDIR="$2"
|
||||
local PKGS="$3"
|
||||
local ALL=${4:-"no"}
|
||||
local JOBS=${5:-1}
|
||||
local PKGSNEW=()
|
||||
local VERBOSE="no"
|
||||
local PKG INFOPATH
|
||||
|
||||
if [ "$ALL" == "yes" ]; then
|
||||
printf "Processing all SlackBuilds in the $REPO_SUBPATH repository..."
|
||||
PKGSNEW=($(find "$REPO_DIR" -name *.info -print0 | xargs -r0))
|
||||
else
|
||||
for PKG in ${PKGS[@]}; do
|
||||
INFOPATH=$(find "$REPO_DIR" -name ${PKG}.info)
|
||||
|
||||
if [ -z "$INFOPATH" ]; then
|
||||
echo "$PKG: Not found."
|
||||
exit
|
||||
fi
|
||||
|
||||
PKGSNEW+=($INFOPATH)
|
||||
done
|
||||
|
||||
VERBOSE="yes"
|
||||
fi
|
||||
|
||||
has_parallel
|
||||
if [ $? -eq 1 ]; then
|
||||
printf '%s\n' "${PKGSNEW[@]}" | \
|
||||
parallel --eta --will-cite --jobs $JOBS \
|
||||
/usr/share/sbopkg/sqg/sqg-build-queuefile \
|
||||
"$REPO_DIR" "$QUEUEDIR" {} "$SKIP_EMPTY" $VERBOSE
|
||||
else
|
||||
for PKG in "${PKGSNEW[@]}"; do
|
||||
/usr/share/sbopkg/sqg/sqg-build-queuefile \
|
||||
"$REPO_DIR" "$QUEUEDIR" "$PKG" "$SKIP_EMPTY" $VERBOSE
|
||||
done
|
||||
fi
|
||||
}
|
||||
27
src/usr/share/sbopkg/sqg/sqg-build-queuefile
Executable file
27
src/usr/share/sbopkg/sqg/sqg-build-queuefile
Executable file
@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
|
||||
# source all sqg functions
|
||||
. /usr/share/sbopkg/sqg/functions
|
||||
|
||||
REPO_DIR="$1"
|
||||
QUEUEDIR="$2"
|
||||
INFO="$3"
|
||||
SKIP_EMPTY=${4:-""}
|
||||
VERBOSE=${5:-"no"}
|
||||
|
||||
. $INFO
|
||||
|
||||
if [ "$VERBOSE" == "yes" ]; then
|
||||
INFO_BASENAME=$(basename $INFO .info)
|
||||
|
||||
if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then
|
||||
echo "$INFO_BASENAME: empty REQUIRES line. No queuefile generated."
|
||||
echo "Uncomment SKIP_EMPTY to change this."
|
||||
fi
|
||||
|
||||
echo "Processing $INFO_BASENAME."
|
||||
fi
|
||||
|
||||
if [[ ! -z $REQUIRES || $SKIP_EMPTY == "NO" ]]; then
|
||||
build_queuefile "$REPO_DIR" "$QUEUEDIR" "$PRGNAM"
|
||||
fi
|
||||
@ -1,5 +1,15 @@
|
||||
SBOPKG NEWS
|
||||
|
||||
sbopkg-dev (2017-01-02 12:51 UTC)
|
||||
FEATURES
|
||||
* sqg: Parallelize building of all packages by option -j.
|
||||
Requires GNU Parallel.
|
||||
Patch by Marcel Saegebarth.
|
||||
|
||||
MODIFICATIONS
|
||||
* bash-completion: Complete -i and -b when git repo is used
|
||||
Patch by Sergey V.
|
||||
|
||||
sbopkg 0.38.1 (2016-09-01 13:42 UTC)
|
||||
FEATURES
|
||||
* Added option to download from third party source repository
|
||||
|
||||
Loading…
Reference in New Issue
Block a user