shell的分批并发操作的简单实现

发表于 更新于 分类于 阅读次数：

很多情况下，计算资源是有限的。我们需要在任务时长和资源消耗之前取得一种平衡。因此需要进行并发和控制。

  • 示例一

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    # cat packetConcurrency.sh

    #!/bin/bash

    # 并发数
    threads=3
    # 任务数
    tasks=9
    function task()
    {
    	r=$(( $RANDOM % 30 + 1 ))
    	echo "Start $1."
    	sleep $r
    	echo "Finished $1,took ${r}s."
    }

    counter=1
    for sn in $(seq 1 ${tasks})
    do
        task ${sn} &
        if [ "$((${counter} % ${threads}))" -eq "0" ]; then
    	time wait
    	echo "==================="
        fi
        counter=$((${counter} + 1))
    done

    #######################################################
    执行结果：
    # sh packetConcurrency.sh
    Start 2.
    Start 3.
    Start 1.
    Finished 2,took 3s.
    Finished 1,took 14s.
    Finished 3,took 15s.

    real	0m15.014s       #每一批任务的耗时和该批任务中耗时最长的任务消耗的时间一致。
    user	0m0.021s
    sys	0m0.059s
    ===================
    Start 5.
    Start 6.
    Start 4.
    Finished 5,took 1s.
    Finished 6,took 22s.
    Finished 4,took 26s.

    real	0m26.065s       #每一批任务的耗时和该批任务中耗时最长的任务消耗的时间一致。
    user	0m0.021s
    sys	0m0.050s
    ===================
    Start 8.
    Start 9.
    Start 7.
    Finished 8,took 6s.
    Finished 7,took 9s.
    Finished 9,took 22s.

    real	0m22.101s       #每一批任务的耗时和该批任务中耗时最长的任务消耗的时间一致。
    user	0m0.088s
    sys	0m0.099s
    ===================

  • 示例二

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    # cat groupExec.sh
    #!/bin/bash

    groups=5
    tasks=20
    function groupExec()
    {
    for group in $(seq 0 $((${groups} - 1 )))
    do
    	echo "======第$((${group} + 1))组开始：======"
            for task in $(seq 0 $((${tasks} - 1 )))
            do
                    if [ "$((${task} / $((${tasks} / ${groups}))))" -eq "${group}" ]; then
                            { echo "task:$(( ${task} + 1 ))";sleep $(($RANDOM % 10)); } &
                    else
                            continue
                    fi
            done
            wait
    	echo "======第$((${group} + 1))组完成。======"
    done
    }
    groupExec

    # sh groupExec.sh
    ======第1组开始：======
    task:1
    task:4
    task:3
    task:2
    ======第1组完成。======
    ======第2组开始：======
    task:5
    task:7
    task:8
    task:6
    ======第2组完成。======
    ======第3组开始：======
    task:9
    task:11
    task:12
    task:10
    ======第3组完成。======
    ======第4组开始：======
    task:15
    task:16
    task:13
    task:14
    ======第4组完成。======
    ======第5组开始：======
    task:19
    task:17
    task:18
    task:20
    ======第5组完成。======