From 1eb39ee8ebbf8387b45bade396b6de5daa74da4a Mon Sep 17 00:00:00 2001 From: Marcel Saegebarth Date: Sat, 19 Nov 2016 17:39:30 +0100 Subject: [PATCH] sqg: Parallelize building of all packages by option -j. Parallelization requires GNU parallel to be installed. Signed-off-by: Marcel Saegebarth Signed-off-by: Willy Sudiarto Raharjo --- src/usr/doc/README | 2 +- src/usr/doc/THANKS | 2 + src/usr/sbin/sbopkg | 2 +- src/usr/sbin/sqg | 157 +++------------- src/usr/share/sbopkg/sqg/functions | 180 +++++++++++++++++++ src/usr/share/sbopkg/sqg/sqg-build-queuefile | 27 +++ tools/ChangeLog-latest.txt | 10 ++ 7 files changed, 245 insertions(+), 135 deletions(-) create mode 100644 src/usr/share/sbopkg/sqg/functions create mode 100755 src/usr/share/sbopkg/sqg/sqg-build-queuefile diff --git a/src/usr/doc/README b/src/usr/doc/README index 2d86c0c..b9fdebf 100644 --- a/src/usr/doc/README +++ b/src/usr/doc/README @@ -5,7 +5,7 @@ SBOPKG COPYRIGHT/LICENSE Copyright 2007-2010 Chess Griffin Copyright 2009-2011 Mauro Giachero slakmagik - Copyright 2016 Willy Sudiarto Raharjo + Copyright 2016-2017 Willy Sudiarto Raharjo Sbopkg is released under a one-clause BSD license. See the script itself for the text. diff --git a/src/usr/doc/THANKS b/src/usr/doc/THANKS index 5cc1df2..9f63a63 100644 --- a/src/usr/doc/THANKS +++ b/src/usr/doc/THANKS @@ -87,3 +87,5 @@ We moved to github, so revision number no longer used pyllyukko Dhaby Xiloj Chris Abela + Sergey V. + Marcel Saegebarth diff --git a/src/usr/sbin/sbopkg b/src/usr/sbin/sbopkg index 2521661..f9506cb 100755 --- a/src/usr/sbin/sbopkg +++ b/src/usr/sbin/sbopkg @@ -6,7 +6,7 @@ # Copyright 2007-2010,2013 Chess Griffin # Copyright 2009-2011 Mauro Giachero # Copyright 2009-2013 slakmagik -# Copyright 2015-2016 Willy Sudiarto Raharjo +# Copyright 2015-2017 Willy Sudiarto Raharjo # # Redistribution and use of this script, with or without modification, is # permitted provided that the following conditions are met: diff --git a/src/usr/sbin/sqg b/src/usr/sbin/sqg index fc21735..8377144 100755 --- a/src/usr/sbin/sqg +++ b/src/usr/sbin/sqg @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # $Id$ @@ -8,7 +8,7 @@ # Copyright 2013 Chess Griffin and # slakmagik -# Copyright 2016 Willy Sudiarto Raharjo +# Copyright 2016-2017 Willy Sudiarto Raharjo # Redistribution and use of this script, with or without modification, is # permitted provided that the following conditions are met: @@ -27,155 +27,46 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -SCRIPT=${0##*/} - ### OPTIONAL CONFIGURATION BELOW ### SBOPKG_CONF=${SBOPKG_CONF:-/etc/sbopkg/sbopkg.conf} # If necessary, override SBOPKG_CONF above or directly from the environment. # The user may want to change QUEUEDIR to a local temporary directory in $HOME -# since queuefiles in QUEUEDIR will be overwritten by this script. QUEUEDIR +# since queuefiles in QUEUEDIR will be overwritten by this script. QUEUEDIR # is commented out on purpose to get the user to review these variables. -# QUEUEDIR must be writable by the user executing the script. SKIP_EMPTY +# QUEUEDIR must be writable by the user executing the script. SKIP_EMPTY # means the script will not create queuefiles for packages with an empty -# REQUIRES= line in the .info file. Uncomment or export a value to create +# REQUIRES= line in the .info file. Uncomment or export a value to create # queuefiles for packages with an empty REQUIRES= line (which will only # contain the package name in the queuefile since there are no REQUIRES). #QUEUEDIR=${QUEUEDIR:-/var/lib/sbopkg/queues} -REPO_ROOT=${REPO_ROOT:-/var/lib/sbopkg} -REPO_NAME=${REPO_NAME:-SBo} -REPO_BRANCH=${REPO_BRANCH:-14.2} +REPO_BRANCH=${REPO_BRANCH:-$(cat /etc/slackware-version | awk '{print $2}')} #SKIP_EMPTY=${SKIP_EMPTY:-NO} ### NO CHANGES SHOULD BE NECESSARY BELOW THIS LINE ### -sanity_checks () { - if [[ ! -e $SBOPKG_CONF ]]; then - echo "$SBOPKG_CONF not found." - echo "Check the configurable variables at the top of the script." - exit 1 - else - . $SBOPKG_CONF - fi - if [ -d $REPO_ROOT/$REPO_NAME/.git ]; then - REPO_SUBPATH=$REPO_NAME - else - REPO_SUBPATH=$REPO_NAME/$REPO_BRANCH - fi - REPO_DIR=$REPO_ROOT/$REPO_SUBPATH - if [[ ! -w $QUEUEDIR || ! -d $REPO_DIR ]]; then - echo "ERROR: $QUEUEDIR or $REPO_DIR do not exist or are not writable." - echo "Check the configurable variables at the top of the script." - exit 1 - fi -} +# source all sqg functions +. /usr/share/sbopkg/sqg/functions -usage() { -cat << EOF -Usage: $SCRIPT [-a] to build all queuefiles or [-p ] for individual -packages. Multiple packages can be passed with quotes, e.g. -p "pkg1" "pkg2". -Packages with an empty REQUIRES= line will be skipped unless SKIP_EMPTY is -uncommented at the top of the script. Package names are case-sensitive. Use -'sbopkg -g pkg' to search if needed. - -This script will overwrite existing queuefiles in \$QUEUEDIR so back up any -existing queuefiles or local modifications. Check the top of the script for -configurable variables. -EOF -exit -} - -search_package () { - local SRCHAPP="$1" - - cd $REPO_DIR - PKGPATH=( $(find -type d -mindepth 2 -maxdepth 2 -name "$SRCHAPP" | sort) ) - if [[ -z $PKGPATH ]]; then - return 1 - else - return 0 - fi -} - -parse_queuefile_requires () { - local PARSEAPP="$1" - local DEPLIST DEP - - if search_package $PARSEAPP; then - . $REPO_DIR/$PKGPATH/$PARSEAPP.info - DEPLIST=($REQUIRES) - for DEP in "${DEPLIST[@]}"; do - if search_package $DEP; then - sed -i "/^$DEP$/ d" $QUEUEFILE - echo "$DEP" >> $QUEUEFILE - parse_queuefile_requires $DEP - elif [[ "$DEP" == "%README%" ]]; then - echo "# %README%: see the $PARSEAPP README file. " >> $QUEUEFILE - fi - done - else - continue - fi -} - -build_queuefile () { - OLDFILE=$QUEUEDIR/$PRGNAM.sqf - QUEUEFILE=$QUEUEDIR/$PRGNAM.sqf.tmp - CATFILE=$QUEUEDIR/tmp.sqf - touch $QUEUEFILE - echo "$PRGNAM" > $QUEUEFILE - parse_queuefile_requires $PRGNAM - tac $QUEUEFILE > $CATFILE - if cmp -s "$CATFILE" "$OLDFILE"; then - rm $QUEUEFILE $CATFILE - else - mv $CATFILE $OLDFILE - rm $QUEUEFILE - fi -} - -main_loop () { - local MAINAPP="$1" INFO - if [[ $MAINAPP == "_all" ]]; then - printf "Processing all SlackBuilds in the $REPO_SUBPATH repo..." - for INFO in $(find $REPO_DIR -name *.info); do - . $INFO - if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then - continue - else - printf "." - build_queuefile - fi - done - else - for MAINAPP in $PKG; do - if search_package $MAINAPP; then - . $REPO_DIR/$PKGPATH/$MAINAPP.info - if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then - echo "$MAINAPP: empty REQUIRES line. No queuefile generated." - echo "Uncomment SKIP_EMPTY to change this." - continue - else - echo "Processing $MAINAPP." - build_queuefile - fi - else - echo "$MAINAPP: not found." - exit 1 - fi - done - fi -} - -case $1 in - -a) shift; PKG="_all" ;; - -p) shift; PKG="$@" ;; - *) usage ;; -esac +SCRIPT=${0##*/} sanity_checks -main_loop "$PKG" + +case $1 in + -a) + if [ "$2" == "-j" ] && [ -n $3 ]; then + get_jobs $3 + JOBS=$? + fi + shift; + execute_build "$REPO_DIR" "$QUEUEDIR" "" "yes" $JOBS ;; + -p) + shift; + execute_build "$REPO_DIR" "$QUEUEDIR" "$@" "no" $JOBS ;; + *) usage $SCRIPT ;; +esac + echo "Done." exit 0 diff --git a/src/usr/share/sbopkg/sqg/functions b/src/usr/share/sbopkg/sqg/functions new file mode 100644 index 0000000..19d2bc1 --- /dev/null +++ b/src/usr/share/sbopkg/sqg/functions @@ -0,0 +1,180 @@ +#!/bin/sh + +sanity_checks () { + if [ ! -e "$SBOPKG_CONF" ]; then + echo "$SBOPKG_CONF not found." + echo "Check the configurable variables at the top of the script." + exit 1 + else + . $SBOPKG_CONF + fi + + if [ -d $REPO_ROOT/$REPO_NAME/.git ]; then + REPO_SUBPATH=$REPO_NAME + else + REPO_SUBPATH=$REPO_NAME/$REPO_BRANCH + fi + + REPO_DIR=$REPO_ROOT/$REPO_SUBPATH + if [ ! -w "$QUEUEDIR" ] || [ ! -d "$REPO_DIR" ]; then + echo "ERROR: $QUEUEDIR or $REPO_DIR do not exist or are not writable." + echo "Check the configurable variables at the top of the script." + exit 1 + fi +} + +has_parallel () { + parallel --help &> /dev/null + if [ $? -eq 0 ]; then + return 1 + fi + + return 0 +} + +usage () { + local SCRIPT="$1" + +cat << EOF +Usage: $SCRIPT -p | -a [-j] +Options are: + -p package(s) Creates queuefile(s) for individual package(s). + Multiple packages can be passed with quotes, + e.g. -p "pkg1 pkg2". + Packages with an empty REQUIRES= line will be skipped unless + SKIP_EMPTY is uncommented at the top of the script. + Package names are case-sensitive. + -a Builds queuefiles for all packages. + -j Number of concurrent queuefile generation processes. + This could heavily raise disk I/O so use with care. + Requires: GNU parallel + Default: 1 + +This script will overwrite existing queuefiles in \$QUEUEDIR so back up any +existing queuefile(s) or local modifications. Check the top of the script for +configurable variables. +EOF +exit +} + +get_jobs () { + local JOBS=${1:-1} + + if [ ! $JOBS -ge 1 ]; then + echo "ERROR: Value for -j must be an unsigned integer > 0." + exit 1 + else + JOBS=$1 + fi + + return $JOBS +} + +search_package () { + local REPO_DIR="$1" + local SRCHAPP="$2" + + cd $REPO_DIR + PKGPATH=($(find -type d -mindepth 2 -maxdepth 2 -name "$SRCHAPP" | sort)) + if [ -z "$PKGPATH" ]; then + return 1 + else + return 0 + fi +} + +parse_queuefile_requires () { + local REPO_DIR="$1" + local PARSEAPP="$2" + local QUEUEFILE="$3" + local DEPLIST DEP RESPONSE + + if search_package "$REPO_DIR" "$PARSEAPP"; then + . $REPO_DIR/$PKGPATH/$PARSEAPP.info + DEPLIST=($REQUIRES) + for DEP in "${DEPLIST[@]}"; do + if [ "$DEP" == "%README%" ]; then + echo "# %README%: see the $PARSEAPP README file. " >> $QUEUEFILE + continue + fi + + if search_package "$REPO_DIR" "$DEP"; then + sed -i "/^$DEP$/ d" $QUEUEFILE + echo "$DEP" >> $QUEUEFILE + parse_queuefile_requires "$REPO_DIR" "$DEP" "$QUEUEFILE" + fi + done + else + echo "$PARSEAPP: not found." + read -s -p ' Do you want to continue? [y/N]' RESPONSE + RESPONSE=$(echo "$RESPONSE" | awk '{print tolower($0)}') + + if [ "$RESPONSE" =~ ^\(yes|y\)$ ]; then + continue + else + exit 1 + fi + fi +} + +build_queuefile () { + local REPO_DIR="$1" + local QUEUEDIR="$2" + local PRGNAM="$3" + local OLDFILE=$QUEUEDIR/$PRGNAM.sqf + local QUEUEFILE=$QUEUEDIR/$PRGNAM.sqf.tmp + local CATFILE=$QUEUEDIR/$PRGNAM.tmp.sqf + + touch $QUEUEFILE + echo "$PRGNAM" > $QUEUEFILE + parse_queuefile_requires "$REPO_DIR" "$PRGNAM" "$QUEUEFILE" + tac $QUEUEFILE > $CATFILE + if cmp -s "$CATFILE" "$OLDFILE"; then + rm $QUEUEFILE $CATFILE + else + mv $CATFILE $OLDFILE + rm $QUEUEFILE + fi +} + +execute_build () { + local REPO_DIR="$1" + local QUEUEDIR="$2" + local PKGS="$3" + local ALL=${4:-"no"} + local JOBS=${5:-1} + local PKGSNEW=() + local VERBOSE="no" + local PKG INFOPATH + + if [ "$ALL" == "yes" ]; then + printf "Processing all SlackBuilds in the $REPO_SUBPATH repository..." + PKGSNEW=($(find "$REPO_DIR" -name *.info -print0 | xargs -r0)) + else + for PKG in ${PKGS[@]}; do + INFOPATH=$(find "$REPO_DIR" -name ${PKG}.info) + + if [ -z "$INFOPATH" ]; then + echo "$PKG: Not found." + exit + fi + + PKGSNEW+=($INFOPATH) + done + + VERBOSE="yes" + fi + + has_parallel + if [ $? -eq 1 ]; then + printf '%s\n' "${PKGSNEW[@]}" | \ + parallel --eta --will-cite --jobs $JOBS \ + /usr/share/sbopkg/sqg/sqg-build-queuefile \ + "$REPO_DIR" "$QUEUEDIR" {} "$SKIP_EMPTY" $VERBOSE + else + for PKG in "${PKGSNEW[@]}"; do + /usr/share/sbopkg/sqg/sqg-build-queuefile \ + "$REPO_DIR" "$QUEUEDIR" "$PKG" "$SKIP_EMPTY" $VERBOSE + done + fi +} diff --git a/src/usr/share/sbopkg/sqg/sqg-build-queuefile b/src/usr/share/sbopkg/sqg/sqg-build-queuefile new file mode 100755 index 0000000..1934f39 --- /dev/null +++ b/src/usr/share/sbopkg/sqg/sqg-build-queuefile @@ -0,0 +1,27 @@ +#!/bin/sh + +# source all sqg functions +. /usr/share/sbopkg/sqg/functions + +REPO_DIR="$1" +QUEUEDIR="$2" +INFO="$3" +SKIP_EMPTY=${4:-""} +VERBOSE=${5:-"no"} + +. $INFO + +if [ "$VERBOSE" == "yes" ]; then + INFO_BASENAME=$(basename $INFO .info) + + if [[ -z $REQUIRES && $SKIP_EMPTY == "" ]]; then + echo "$INFO_BASENAME: empty REQUIRES line. No queuefile generated." + echo "Uncomment SKIP_EMPTY to change this." + fi + + echo "Processing $INFO_BASENAME." +fi + +if [[ ! -z $REQUIRES || $SKIP_EMPTY == "NO" ]]; then + build_queuefile "$REPO_DIR" "$QUEUEDIR" "$PRGNAM" +fi diff --git a/tools/ChangeLog-latest.txt b/tools/ChangeLog-latest.txt index 23b4ba8..fb86fae 100644 --- a/tools/ChangeLog-latest.txt +++ b/tools/ChangeLog-latest.txt @@ -1,5 +1,15 @@ SBOPKG NEWS +sbopkg-dev (2017-01-02 12:51 UTC) + FEATURES + * sqg: Parallelize building of all packages by option -j. + Requires GNU Parallel. + Patch by Marcel Saegebarth. + + MODIFICATIONS + * bash-completion: Complete -i and -b when git repo is used + Patch by Sergey V. + sbopkg 0.38.1 (2016-09-01 13:42 UTC) FEATURES * Added option to download from third party source repository