Running various commands in bash with limits

Running various commands in bash with limits

# Run various commands at sametime in bash
#!/bin/bash

NUM_MAX_THREADS=200;
IN_FILE="filein.txt";
OUT_FILE="fileout.txt";
CURR_LINE=0;
SECONDS=0;
SECONDS_AVG=0;
TOTAL_LINES=$(cat $IN_FILE | wc -l);
LINE_START_AT="$1";

# echo the time statistics
time (
# read the in file
cat $IN_FILE |
# get the row,
while read LINE;
do
# jumps at the start line
if [[ $CURR_LINE -lt $LINE_START_AT ]]; then
echo "Jumping line... $(( CURR_LINE++ ))";
continue;
fi;
# if don't got processed
if [[ $(grep $LINE $OUT_FILE|wc -l) -eq 0 ]]; then
# check the number of threads running
THREADS_IN_USE=$( ps -Afe | grep wget | grep 'YOURCOMMAND' | wc -l );
# if over the limits, wait to free...
while [[ $THREADS_IN_USE -gt $NUM_MAX_THREADS ]];
do echo "WAITING THREADS TO GET DONE ($THREADS_IN_USE RUNNING)...";
sleep 1;
THREADS_IN_USE=$( ps -Afe | grep wget | grep 'YOURCOMMAND' | wc -l );
done;
# runs the command
{
echo $( YOURCOMMAND "$LINE" ) >> $OUT_FILE;
} &
fi;
# TIP, use my estimate.sh to get the remaining time.
# get the process average per second.
NUM_LINES_DONE=$(cat $OUT_FILE | wc -l);
SECONDS=$(date "+%s");
if [[ $SECONDS -gt $(( SECONDS_AVG + 10 )) ]]; then
SECONDS_ACTUAL=$(( SECONDS - SECONDS_AVG ));
NUM_LINES_DONE_MEDIA=$(( NUM_LINES_DONE - NUM_LINES_DONE_ANT ));
NUM_LINES_DONE_ANT=$NUM_LINES_DONE;
SECONDS_AVG=$SECONDS;
MEDIA=$(( NUM_LINES_DONE_MEDIA / SECONDS_ACTUAL ));
NUM_LINES_REMAINING=$(( TOTAL_LINES - NUM_LINES_DONE ));
REMAINING_TIME=$(( NUM_LINES_REMAINING * MEDIA ));
fi;
# echoes the status and qtd of active threads
echo "Threads running: ${THREADS_IN_USE}, Lines done: $NUM_LINES_DONE, processing line: $(( CURR_LINE++)), MEDIA: $MEDIA Lines/Second, Remaining time: $REMAINING_TIME SECONDS;";
done;
);