Re: Parallelism a la make -j <n> / GNU parallel

[John Kearney]

Am 03.05.2012 22:30, schrieb Greg Wooledge:
> On Thu, May 03, 2012 at 10:12:17PM +0200, John Kearney wrote:
>>     function runJobParrell {
>>         local mjobCnt=${1} && shift
>>         jcnt=0
>>         function WrapJob {
>>             "${@}"
>>             kill -s USR2 $$
>>         }
>>         function JobFinised {
>>             jcnt=$((${jcnt}-1))
>>         }
>>         trap JobFinised USR2
>>         while [ $# -gt 0 ] ; do
>>             while [ ${jcnt} -lt ${mjobCnt} ]; do
>>                 jcnt=$((${jcnt}+1))
>>                 echo WrapJob "${1}" "${2}"
>>                 WrapJob "${1}" "${2}" &
>>                 shift 2
>>         done
>>         sleep 1
>>         done
>>     }
>>     function testProcess {
>>         echo "${*}"
>>         sleep 1
>>     }
>>     runJobParrell 2  testProcess "jiji#" testProcess "jiji#" testProcess
>> "jiji#"
>>
>>     tends to work well enough.
>> it gets a bit more complex if you want to recover output but not too much.
> The real issue here is that there is no generalizable way to store an
> arbitrary command for later execution.  Your example assumes that each
> pair of arguments constitutes one simple command, which is fine if that's
> all you need it to do.  But the next guy asking for this will want to
> schedule arbitrarily complex shell pipelines and complex commands with
> here documents and brace expansions and ....
>


:)
A more complex/flexible example. More like what I actually use.




  CNiceLevel=$(nice)
    declare -a JobArray
    function PushAdvancedCmd {
        local IFS=$'\v'
        JobArray+=("${*}")
    }
    function PushSimpleCmd {
        PushAdvancedCmd  WrapJob ${CNiceLevel} "${@}"
    }
    function PushNiceCmd {
        PushAdvancedCmd  WrapJob "${@}"
    }
    function UnpackCmd {
        local IFS=$'\v'
        set -o noglob
        _RETURN=( .${1}. )  
        set +o noglob
        _RETURN[0]="${_RETURN[0]#.}"
        local -i le=${#_RETURN[@]}-1
        _RETURN[${le}]="${_RETURN[${le}]%.}"
    }
    function runJobParrell {
        local mjobCnt=${1} && shift
        jcnt=0
        function WrapJob {
            [ ${1} -le ${CNiceLevel} ] || renice -n ${1}
            local Buffer=$("${@:2}")
            echo "${Buffer}"
            kill -s USR2 $$
        }
        function JobFinised {
            jcnt=$((${jcnt}-1))
        }
        trap JobFinised USR2
        while [ $# -gt 0 ] ; do
            while [ ${jcnt} -lt ${mjobCnt} ]; do
                jcnt=$((${jcnt}+1))
                UnpackCmd "${1}"
                "${_RETURN[@]}" &
                shift
        done
        sleep 1
        done
    }



    function testProcess {
        echo "${*}"
        sleep 1
    }
    #  So standard variable args can be handled in 2 ways 1
    #  encode them as such
    PushSimpleCmd testProcess "jiji#" dfds dfds dsfsd
    PushSimpleCmd testProcess "jiji#" dfds dfds
    PushNiceCmd 20 testProcess "jiji#" dfds
    PushSimpleCmd testProcess "jiji#"
    PushSimpleCmd testProcess "jiji#" "*" s
    # more complex things just wrap them in a function and call it
    function DoComplexMagicStuff1 {
        echo "${@}" >&2
    }
    # Or more normally just do a hybrid of both.
    PushSimpleCmd DoComplexMagicStuff1 "jiji#"

    #
   
    runJobParrell 1 "${JobArray[@]}"



Note there is another level of complexity where I start a JobQueue
Process and issues it commands using a fifo.