added headers

gaboshlib.include

@@ -2,10 +2,6 @@
 
 . /etc/profile
 
-# export all functions
-set -a
-
-
 ## Include functions
 for bashfunc in $(find /etc/bash/gaboshlib -type f -name "g_*.bashfunc" -o -name "g_*.sh")
 do
@@ -14,6 +10,9 @@ do
 done
 ##
 
+# WGET Options
+g_wget_opts="--timeout 10 --tries=2 --user-agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'"
+
 # if runnign directly from shell
 if echo $0 | egrep -q '^-bash$|^-su$'
 then
@@ -30,8 +29,8 @@ g_tmp="$g_tmp/g_$g_scriptname-$$"
 [ -d "$g_tmp" ] || mkdir -p "$g_tmp"
 
 # START and EXIT Notification
-g_trap_exit="g_logger EXITING $g_scriptname ; rm -r $g_tmp ; g_kill_all_background_jobs"
-trap "$g_trap_exit" EXIT
+g_trap_exit="g_logger EXITING $g_scriptname ; rm -r $g_tmp ; g_kill_all_background_jobs >/dev/null 2>&1"
+trap "$g_trap_exit" INT TERM EXIT 
 g_syslogtag="g_bash-script:$g_scriptname[$$]"
 [ $g_scriptname = "bash" ] || g_logger STARTING $g_scriptname
 

gaboshlib/g_array.sh

@@ -0,0 +1,20 @@
+function g_array {
+  local g_filename=$1
+  local g_arrayname=$2
+  local g_delimeter=$3
+  unset -v g_array
+
+  [ -n "$g_delimeter" ] && g_delimeter="-d${g_delimeter}"
+  if [ -f "$g_filename" ]
+  then
+    mapfile $g_delimeter -tn 0 g_array < "$g_filename"
+  else
+    mapfile $g_delimeter -tn 0 g_array <<< "$g_filename"
+  fi
+  
+  # remove newlines
+  g_array=("${g_array[@]%$'\n'}")
+  declare -ng $g_arrayname=g_array
+
+}
+

gaboshlib/g_backup.bashfunc

@@ -61,8 +61,6 @@ dwhelper
 *.del
 *.bak
 *.old
-*backup/
-*backups/
 *.backup
 *.backups
 *Backup/
@@ -74,6 +72,7 @@ vpn
 .config/Nextcloud/
 nextcloud-test.*/
 .VirtualBox/
+VirtualBox/
 DMS/
 tmp
 .zoom/
@@ -145,7 +144,7 @@ var-lib-docker
       g_runcmd $sshopts mkdir -p "$BACKUPSRVPATH_WITH_DATE_INCOMPLETE" || continue
 
       # Create hardlinks before rsync because of some strange behaviour with "rsync --link-dest="
-      [ -n "$BACKUPSRVPATH_LINK_DEST" ] && g_runcmd $sshopts cp -rlfTP "$BACKUPSRVPATH_LINK_DEST"/ "$BACKUPSRVPATH_WITH_DATE_INCOMPLETE"/ || continue
+      [ -n "$BACKUPSRVPATH_LINK_DEST" ] && g_runcmd $sshopts cp -rlfTP "$BACKUPSRVPATH_LINK_DEST"/ "$BACKUPSRVPATH_WITH_DATE_INCOMPLETE"/
 
     else
       BACKUPSRVPATH_WITH_DATE_INCOMPLETE="$BACKUPSRVPATH"

gaboshlib/g_basename.sh

@@ -1,5 +1,5 @@
 function g_basename {
-  g_basename_result="${1##*/}"
-  g_basename_result="${g_basename_result:-/}"
+  g_basename_result=${1##*/}
+  g_basename_result=${g_basename_result:-/}
 }
 

gaboshlib/g_calc.sh

@@ -1,56 +1,58 @@
 function g_calc {
 
+  # function for calculating via bc running in the background (much faster then starting bc every time)
+  # $g_fd_bc_in $fd_bc_out have to be global
+
   unset g_calc_result
 
-  # Use bc in backround for multiple bc's running much faster
-  if [ -z "${g_bc_running}" ]
+  local g_jobs
+  mapfile -t g_jobs < <(jobs -r)
+
+  # check if bc is already running
+  if [[ -z "$g_fd_bc_in" || -z "$g_fd_bc_out" || ${g_jobs[*]} != *bc* ]]
   then
-    g_rnd=$$
-    mkdir -p ${g_tmp}/${g_rnd}
-    [ -p ${g_tmp}/${g_rnd}/bc-in ] || mkfifo ${g_tmp}/${g_rnd}/bc-in
-    [ -p ${g_tmp}/${g_rnd}/bc-out ] || mkfifo ${g_tmp}/${g_rnd}/bc-out
-    [ -p ${g_tmp}/${g_rnd}/bc-sed-in ] || mkfifo ${g_tmp}/${g_rnd}/bc-sed-in
-    [ -p ${g_tmp}/${g_rnd}/bc-sed-out ] || mkfifo ${g_tmp}/${g_rnd}/bc-sed-out
-
-    # bc stream channel
-    { bc -ql <${g_tmp}/${g_rnd}/bc-in 2>&1 >${g_tmp}/${g_rnd}/bc-out & } 2>/dev/null
-    exec 3>${g_tmp}/${g_rnd}/bc-in 4<${g_tmp}/${g_rnd}/bc-out
-
-    # sed stream channel
-    { sed -su 's/^\./0./; s/^-\./-0./' <${g_tmp}/${g_rnd}/bc-sed-in >${g_tmp}/${g_rnd}/bc-sed-out & } 2>/dev/null
-    exec 5>${g_tmp}/${g_rnd}/bc-sed-in 6<${g_tmp}/${g_rnd}/bc-sed-out
-
-    g_bc_running="true"
+    local bc_input="$g_tmp/$$-g_bc_input"
+    local bc_output="$g_tmp/$$-g_bc_output"
+    # create fifo pipe
+    [ -p "$bc_input" ] || mkfifo "$bc_input"
+    [ -p "$bc_output" ] || mkfifo "$bc_output"
+    # run bc in background und switch i/o to pipes
+    bc -ql < "$bc_input" > "$bc_output" 2>&1 &
+    # store in filedescriptiors
+    exec {g_fd_bc_in}> "$bc_input"
+    exec {g_fd_bc_out}< "$bc_output"
   fi
+  # send calculation and read result
+  echo "$1" >&${g_fd_bc_in}
+  read -u ${g_fd_bc_out} g_calc_result
 
-  # do bc
-  echo "scale=8; $@" >&3
-  local g_bc_out
-  if ! read -t 0.1 g_bc_out <&4
+  # check if there is a output
+  if [ -z "$g_calc_result" ]
   then
-    g_traceback "$@"
-    ( exec 3<&- ) 2>/dev/null
-    ( exec 4<&- ) 2>/dev/null
-    ( exec 5<&- ) 2>/dev/null
-    ( exec 6<&- ) 2>/dev/null
-    unset g_bc_running
+    echo "${FUNCNAME} $@" 1>&2
+    unset g_calc_result
     return 1
   fi 
 
-  # do sed
-  echo ${g_bc_out} >&5
-  local g_sed_out
-  if ! read -t 0.1 g_sed_out <&6
+  # fix bc output -> change for example .224 to 0.224 and -.224 to -0.224
+  [[ $g_calc_result  == "."* ]] && g_calc_result="0$g_calc_result"
+  [[ $g_calc_result  == "-."* ]] && g_calc_result="-0.${g_calc_result#-.}"
+
+  # remove ending 0 if for exabple 4.54300000
+  while [[ $g_calc_result =~ [.] && ${g_calc_result: -1} == "0" ]]
+  do
+    g_calc_result=${g_calc_result: : -1}
+  done
+
+  # remove ending . for example "100." -> 100
+  [[ $g_calc_result  == *"." ]] && g_calc_result=${g_calc_result%?}
+
+  # check output
+  if ! g_num_valid_number "$g_calc_result"
   then
-    g_traceback "$@"
-    ( exec 3<&- ) 2>/dev/null
-    ( exec 4<&- ) 2>/dev/null
-    ( exec 5<&- ) 2>/dev/null
-    ( exec 6<&- ) 2>/dev/null
-    unset g_bc_running
+    echo "${FUNCNAME} $@" 1>&2
+    unset g_calc_result
     return 1
   fi
-
-  # store result
-  g_calc_result="${g_sed_out}"
 }
+

gaboshlib/g_compress_video.sh

@@ -5,7 +5,7 @@ function g_compress_video {
  local g_remotedockerffmpeg=$2
 
  # Datei OK und noch da?
- ffmpeg -i "$g_vid" 2>&1 | perl -pe 's/\[0x...\]//g' >"$g_tmp"/vidinfo
+ ffmpeg -i "$g_vid" 2>&1 | perl -pe 's/\[0x[0-9]+\]//g' >"$g_tmp"/vidinfo
  if egrep -q "Invalid data found when processing input|No such file or directory" "$g_tmp"/vidinfo
  then
   g_echo_warn "Video $g_vid existiert nicht (mehr) oder ist defekt."
@@ -19,7 +19,7 @@ function g_compress_video {
   local g_lines=`cat "$g_tmp"/vidinfo | egrep -v "WARNING: | configuration: |Side data\:|audio service type\: main|Video\:.+hevc.+1280x|Video\:.+hevc.+960x|Video\:.+hevc.+[1-8][0-9][0-9]x|Audio\:.+HE-AAC\).+mono|Audio\:.+HE-AAC.+stereo|Audio\:.+ac3.+5\.1.side.|Input .+mp4.+from|vendor_id|^ +lib[a-z]+ " | wc -l`
   # Max DVD Quali
   [ -e "$g_tmp"/VID-SD ] && g_lines=`cat "$g_tmp"/vidinfo | egrep -v "WARNING: | configuration: |Side data\:|audio service type\: main|Video\:.+hevc.+720x|Video\:.+hevc.+[1-6][0-9][0-9]x|Audio\:.+HE-AAC\).+mono|Audio\:.+HE-AAC.+stereo|Audio\:.+ac3.+5\.1.side.|Input .+mp4.+from|vendor_id|^ +lib[a-z]+ " | wc -l`
-  if [ $g_lines -eq '13' ]
+  if [ $g_lines -eq '14' ]
   then
    g_echo "Video $g_vid bereits bearbeitet!"
    return 1
@@ -174,7 +174,7 @@ function g_compress_video {
  sshstream="ssh -p33 ${g_remotedockerffmpeg}"
  [ -z ${g_remotedockerffmpeg} ] && sshstream="sh -c"
  g_echo "Baue MP4 ($g_vid) ${g_remotedockerffmpeg}"
- echo "cat \"${g_viddone}-streamable\"| $sshstream 'cat | docker run -i --rm linuxserver/ffmpeg -loglevel warning -stats -i pipe: -f mp4 -map_metadata -1 -map_chapters -1 -map $g_vidstream -map $g_audstream -filter:v \"${g_ass}${g_vidscale}\" -c:v libx265 -crf 25 -x265-params \"vbv-maxrate=${g_vidmaxratenew}:vbv-bufsize=${g_vidmaxratenew}:log-level=warning\" -pix_fmt yuv420p -max_muxing_queue_size 9999 $g_audionew -threads 1 -movflags +faststart+empty_moov -f mp4 pipe:' >\"${g_viddone}-stream\"" >>"$g_tmp"/cmd
+ echo "cat \"${g_viddone}-streamable\"| $sshstream 'cat | docker run -i --rm linuxserver/ffmpeg -loglevel warning -stats -i pipe: -f mp4 -map_metadata -1 -map_chapters -1 -map $g_vidstream -map $g_audstream -filter:v \"${g_ass}${g_vidscale}\" -c:v libx265 -crf 25 -x265-params \"vbv-maxrate=${g_vidmaxratenew}:vbv-bufsize=${g_vidmaxratenew}:log-level=warning\" -pix_fmt yuv420p -max_muxing_queue_size 9999 $g_audionew -threads 1 -movflags +faststart+empty_moov+delay_moov -f mp4 pipe:' >\"${g_viddone}-stream\"" >>"$g_tmp"/cmd
  echo "ffmpeg -loglevel warning -stats -i \"${g_viddone}-stream\" -c:v copy -c:a copy  -movflags +faststart -f mp4 \"$g_viddone\" < /dev/null 2>&1" >>"$g_tmp"/cmd
 
  cat "$g_tmp"/cmd
@@ -188,6 +188,7 @@ function g_compress_video {
   cat $g_tmp/cmd
   sh $g_tmp/cmd
   g_try=$((g_try+1))
+  rm /tmp/"$g_vid_md5".g_progressing
   [ "$g_try" -gt "3" ] && return 1
  done
  local g_timestamp=$(ls --time-style='+%Y%m%d%H%M' -l "$g_vid" | cut -d" " -f6)

gaboshlib/g_json.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# thanks to dylanaraps - https://github.com/dylanaraps/nosj
+
+# function g_json gives back the associative array $g_json
+# parsed json by stdin or file in $1
+#
+# To get keys use: ${!g_json[@]}
+# To get values use ${g_json[@]}
+# To get specific key use ${g_json[keyname]}
+
+function g_json {
+  IFS= g_json_tokenize < "${1:-/dev/stdin}"
+  IFS= g_json_parse
+}
+
+function g_json_tokenize {
+  local j str
+  while read -rN 1; do
+    case $REPLY in
+      [\{\}\[\],])
+        [[ $str ]] && j+=$REPLY
+        [[ $str ]] || { tokens+=("$j" "$REPLY"); j=; }
+      ;;
+      :)
+        [[ $str ]] && j+=:
+        [[ $str ]] || j+='\ '
+      ;;
+         [[:space:]])
+         [[ $str ]] && j+=$REPLY
+      ;;
+         [\"\'])
+         [[ $str ]] && str= || str=1
+         [[ ${j: -1} == \\ ]] && { str=1; j+=$REPLY; }
+       ;;
+       *) j+=$REPLY ;;
+     esac
+  done
+}
+
+function g_json_parse {
+  local i key key_plain objects o val out
+  declare -Ag g_json
+  for ((i=0;i<${#tokens[@]};i++)) {
+    case ${tokens[i]} in
+      \{|\[)
+        objects+=("${tokens[i-1]}")
+      ;;
+      \}|\])
+        unset 'objects[-1]'
+      ;;
+      *\\\ *)
+        key=${objects[*]//\\ /.}${tokens[i]/\\ *}
+        key_plain=index_${key//[^A-Za-z0-9]/_}
+        val=${tokens[i]/*\\ }
+        if [[ -n ${g_json[$key]} ]]
+        then
+          [[ -n ${g_json[${key}[0]]} ]] || {
+            g_json["${key}[0]"]=${g_json[$key]}
+            printf -v o 'g_json[%q]=%q' "${key}[0]" "${g_json[$key]}"
+            out+=("$o")
+          }
+          declare -i "$key_plain+=1"
+          printf -v o 'g_json[%q]=%q' "${key}[${!key_plain}]" "$val"
+          out+=("$o")
+        elif [[ $val ]]; then
+          g_json["$key"]=$val
+          printf -v o 'g_json[%q]=%q' "$key" "$val"
+          out+=("$o")
+        fi
+      ;;
+    esac
+  }
+}
+

gaboshlib/g_kill_all_background_jobs.sh

@@ -1,4 +1,23 @@
 function g_kill_all_background_jobs {
-  kill -9 $(jobs -p) >/dev/null 2>&1
-
+  [ -z "$1" ] &&
+  sleep 0.1
+  g_array "$(jobs -p)" g_pids
+  if [ -n "${g_pids[0]}" ]
+  then
+    if [ -z "$1" ]
+    then
+      kill -9 ${g_pids[*]} >/dev/null 2>&1
+    else
+      local g_cmdline="$1"
+      local g_proc
+      local g_pid
+      for g_pid in "${g_pids[@]}"
+      do
+        read g_proc < <(tr "\0" " " < /proc/${g_pid}/cmdline)
+        [ "$g_proc" = "$g_cmdline" ] && kill -9 $g_pid >/dev/null 2>&1
+      done
+    fi
+  fi
+  return 0
 }
+

gaboshlib/g_median.sh

@@ -0,0 +1,26 @@
+function g_median {
+  unset g_median_result
+
+  # Array with numbers
+  local g_numbers=("$@")
+
+  # sort array
+  local g_sorted_numbers=($(printf "%s\n" "${g_numbers[@]}" | sort -n))
+  
+  # number of elements
+  local g_num_elements=${#g_sorted_numbers[@]}
+   
+  # calculate the middle
+  local g_middle=$(($g_num_elements/2))
+
+  # even/odd number 
+  if (($g_num_elements % 2 == 1))
+  then
+    # odd number
+    g_median_result="${g_sorted_numbers[$g_middle]}"
+  else
+    # even number
+    g_calc "(${g_sorted_numbers[$g_middle - 1]} + ${g_sorted_numbers[$g_middle]}) / 2"
+    g_median_result="$g_calc_result"
+  fi
+}

gaboshlib/g_num_exponential2normal.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+function g_num_exponential2normal {
+  [ -z "$1" ] && return 1
+  # if there is a exponential number (for example 9.881e-05) convert it to "normal" notation
+  if [[ "$1" =~ e-[0-9] ]]
+  then
+    # convert
+    printf -v g_num_exponential2normal_result -- "%.12f" "$1"
+    # remove ending 0
+    if [[ $g_num_exponential2normal_result =~ \. ]]
+    then
+      g_num_exponential2normal_result=${g_num_exponential2normal_result%%+(0)}
+      g_num_exponential2normal_result=${g_num_exponential2normal_result%%.}
+    fi
+    return 0
+  else
+    g_num_exponential2normal_result=$1
+    return 2
+  fi
+}

gaboshlib/g_num_exponential2normal_file.sh

@@ -0,0 +1,11 @@
+function g_num_exponential2normal_file {
+  # changes expionential numbers in normal notation in given file
+  local g_file=$1
+  local g_substitution g_exnum
+  for g_exnum in $(egrep -o -i "[0-9]+\.[0-9]+e[\+\-][0-9]+" "$g_file" | sort -u)
+  do
+    g_num_exponential2normal $g_exnum && g_substitution="${g_substitution}s/${g_exnum}/${g_num_exponential2normal_result}/g;"
+  done
+  sed -i "${g_substitution}" "$g_file" && return 0
+}
+

gaboshlib/g_num_is_between.sh

@@ -5,19 +5,19 @@ function g_num_is_between {
   local f_between2=$3
 
   # Check for integer (can be done with bash itself)
-  if [[ ${f_num} =~ ^[0-9]+$ ]] && [[ ${f_between1} =~ ^[0-9]+$ ]] && [[ ${f_between2} =~ ^[0-9]+$ ]]
+  if [[ $f_num =~ ^[0-9]+$ ]] && [[ $f_between1 =~ ^[0-9]+$ ]] && [[ $f_between2 =~ ^[0-9]+$ ]]
   then
     # Check which is the low (from) and the high (to) number
-    if [ "${f_between1}" -lt "${f_between2}" ]
+    if [ $f_between1 -lt $f_between2 ]
     then
-      local f_from=${f_between1}
-      local f_to=${f_between2}
+      local f_from=$f_between1
+      local f_to=$f_between2
     else
-      local f_from=${f_between2}
-      local f_to=${f_between1}
+      local f_from=$f_between2
+      local f_to=$f_between1
     fi
     # Check if given number is in or out range
-    if [ ${f_num} -lt ${f_from} ] || [ ${f_num} -gt ${f_to} ]
+    if [ $f_num -lt $f_from ] || [ $f_num -gt $f_to ]
     then
       return 1
     else
@@ -29,21 +29,21 @@ function g_num_is_between {
   g_num_valid_number "$f_num" "$f_between1" "$f_between2" || return 1
 
   # Check which is the low (from) and the high (to) number
-  g_calc "${f_between1} < ${f_between2}"
-  if [ ${g_calc_result} -ne 0 ]
+  g_calc "$f_between1 < $f_between2"
+  if [ "$g_calc_result" -ne 0 ]
   then
-    local f_from=${f_between1}
-    local f_to=${f_between2}
+    local f_from=$f_between1
+    local f_to=$f_between2
   else
-    local f_from=${f_between2}
-    local f_to=${f_between1}
+    local f_from=$f_between2
+    local f_to=$f_between1
   fi
   # Check if given number is in or out range
-  g_calc "${f_num} < ${f_from}"
-  local g_calc_result_from="${g_calc_result}"
-  g_calc "${f_num} > ${f_to}"
-  local g_calc_result_to="${g_calc_result}"
-  if [ ${g_calc_result_from} -ne 0 ] || [ ${g_calc_result_to} -ne 0 ]
+  g_calc "$f_num < $f_from"
+  local g_calc_result_from="$g_calc_result"
+  g_calc "$f_num > $f_to"
+  local g_calc_result_to="$g_calc_result"
+  if [ "$g_calc_result_from" -ne 0 ] || [ "$g_calc_result_to" -ne 0 ]
   then
     return 1
   else

gaboshlib/g_num_is_higher.sh

@@ -19,7 +19,7 @@ function g_num_is_higher_equal {
   g_num_valid_number "$f_num" "$f_checkhigher" || return 1
  
   g_calc "${f_num} >= ${f_checkhigher}"
-  if [ ${g_calc_result} -ne 0 ]
+  if [ "${g_calc_result}" -ne 0 ]
   then
     return 0
   else
@@ -48,7 +48,7 @@ function g_num_is_higher {
   g_num_valid_number "$f_num" "$f_checkhigher" || return 1
   
   g_calc "${f_num} > ${f_checkhigher}"
-  if [ ${g_calc_result} -ne 0 ]
+  if [ "${g_calc_result}" -ne 0 ]
   then
     return 0
   else

gaboshlib/g_num_is_lower.sh

@@ -19,7 +19,7 @@ function g_num_is_lower_equal {
   g_num_valid_number "$f_num" "$f_checklower" || return 1
  
   g_calc "${f_num} <= ${f_checklower}"
-  if [ ${g_calc_result} -ne 0 ]
+  if [ "${g_calc_result}" -ne 0 ]
   then
     return 0
   else
@@ -49,7 +49,7 @@ function g_num_is_lower {
   g_num_valid_number "$f_num" "$f_checklower" || return 1
  
   g_calc "${f_num} < ${f_checklower}"
-  if [ ${g_calc_result} -ne 0 ]
+  if [ "${g_calc_result}" -ne 0 ]
   then
     return 0
   else

gaboshlib/g_num_valid_number.sh

@@ -1,13 +1,18 @@
 function g_num_valid_number {
-  [ -z "$1" ] && return 1
-  local f_arg=( "0" "$@" )
-  for (( i=0; i<=$#; i++ ))
+  local arg
+  for arg in "$@"
   do
-    if ! [[ ${f_arg[$i]} =~ ^(-)?(\.)?[0-9]+(\.)?([0-9]+)?$ ]]
+    if ! [[ $arg =~ ^(-)?(\.)?[0-9]+(\.)?([0-9]+)?$ ]]
     then
-      echo "\"${f_arg[$i]}\": Not a valid number" 1>&2
+      echo "\"$arg\": Not a valid number" 1>&2
       g_traceback
       return 1
     fi
   done
+  if [ -z "$1" ]
+  then
+    echo "No argument given" 1>&2
+    g_traceback
+    return 2
+  fi
 }

gaboshlib/g_percentage-diff.bashfunc

@@ -5,11 +5,18 @@ function g_percentage-diff {
   local g_to=$2
   unset g_percentage_diff_result
   g_num_valid_number "$g_from" "$g_to" || return 1
-  g_calc "${g_from} == 0"
-  [ ${g_calc_result} -eq 1 ] && return 1
-  g_calc "${g_to} == 0"
-  [ ${g_calc_result} -eq 1 ] && return 1
-  g_calc "100/${g_from}*(${g_to}-${g_from})"
-  #g_percentage_diff_result=$(printf -- "%.2f" "${g_calc_result}")
-  printf -v g_percentage_diff_result -- "%.2f" "${g_calc_result}"
+  g_calc "$g_from == 0"
+  if [ $g_calc_result -eq 1 ] 
+  then
+    g_echo_note "$0: ERR: g_from=$g_from"
+    return 2
+  fi
+  g_calc "$g_to == 0"
+  if [ $g_calc_result -eq 1 ]
+  then
+    g_echo_note "$0: ERR: g_to=$g_to"
+    return 3
+  fi
+  g_calc "100 / $g_from * ( $g_to - ( $g_from ) )" || return 4
+  printf -v g_percentage_diff_result -- "%.2f" $g_calc_result
 }

gaboshlib/g_python.sh

@@ -1,53 +1,122 @@
 function g_python {
 
+  # function for running python in the background
+  # $g_fd_python_in $g_fd_python_out have to be global
+
   unset g_python_result
-  g_tmp=/tmp
-  g_rnd=$$
-  # Use python in backround for multiple python commands running much faster
-  if [ -z "${g_python_running}" ]
-  then
-    mkdir -p ${g_tmp}/${g_rnd}
-    [ -p ${g_tmp}/${g_rnd}/python-in ] || mkfifo ${g_tmp}/${g_rnd}/python-in
-    [ -p ${g_tmp}/${g_rnd}/python-out ] || mkfifo ${g_tmp}/${g_rnd}/python-out
-    echo "while 1:
-  exec(open(\"${g_tmp}/${g_rnd}/python-in\").read())
-" >${g_tmp}/${g_rnd}/python-pipeexec.py
 
-    # python stream channel
-    { python3 -u ${g_tmp}/${g_rnd}/python-pipeexec.py >${g_tmp}/${g_rnd}/python-out 2>${g_tmp}/${g_rnd}/python-error & }
-    g_python_running="true"
+  local g_jobs
+  mapfile -t g_jobs < <(jobs -r)
+  # check if python is already running
+  if [[ -z $g_fd_python_in || -z $g_fd_python_out || ${g_jobs[*]} != *python3* ]]
+  then
+    local python_input="$g_tmp/$$-g_python_input"
+    local python_output="$g_tmp/$$-g_python_output"
+    local python_error="$g_tmp/$$-g_python_error"
+    # create fifo pipe
+    [ -p "$python_input" ] || mkfifo "$python_input"
+    [ -p "$python_output" ] || mkfifo "$python_output"
+    [ -p "$python_error" ] || mkfifo "$python_error"
+    # run bc in background und switch i/o to pipes
+    python3 -iuq  < "$python_input" > "$python_output" 2> "$python_error" &
+    # store in filedescriptiors
+    exec {g_fd_python_in}> "$python_input"
+    exec {g_fd_python_out}< "$python_output"
+    exec {g_fd_python_error}< "$python_error"
   fi
  
-  # do python
-  echo "$@" >${g_tmp}/${g_rnd}/python-in &
-  sleep 0.02
+  # read potential old output
+  read -t 0.001 -u ${g_fd_python_out} g_python_result_old
  
-  local g_python_jobs
-  mapfile -t g_python_jobs < <(jobs -r)
-  if ! [[ $g_python_jobs =~ python-pipeexec.py ]]
-  then
-    g_echo_error "$(cat ${g_tmp}/${g_rnd}/python-error)"
-    unset g_python_running
-    return 1
-  fi
+  # send python command
+  echo "$@
+print('')" >&${g_fd_python_in}
 
-  if [ -n "$g_python_waitforoutput" ]
-  then
-    until [ -n "$g_python_result" ]
-    do
-      read -t 0.1 g_python_result <${g_tmp}/${g_rnd}/python-out
-      [ -s "${g_tmp}/${g_rnd}/python-error" ] && break
-    done
-    unset g_python_waitforoutput
-  else
-    read -t0.2 g_python_result <${g_tmp}/${g_rnd}/python-out
-  fi
+  # read output
+  read -u ${g_fd_python_out} g_python_result
 
-  if [ -s "${g_tmp}/${g_rnd}/python-error" ]
+  local g_errline
+  local g_err_msg
+
+  # look for error
+  while read -t 0.001 -u ${g_fd_python_error} g_errline
+  do
+    [[ "$g_errline" =~ \>\>\>$ ]] && break
+    [ -z "$g_errline" ] && break
+    g_err_msg="$g_err_msg
+$g_errline"
+  done
+  if [ -n "$g_err_msg" ]
   then
-    g_echo_error "$(cat ${g_tmp}/${g_rnd}/python-error)"
+    g_echo_error "$g_err_msg"
     return 1
   fi
 
 }
 
+
+function g_python_old {
+
+  local g_python_tmp=${g_tmp}/$$
+  g_python_out=${g_python_tmp}/python-out
+  local g_python_in=${g_python_tmp}/python-in
+  local g_python_error=${g_python_tmp}/python-error
+  local g_python_jobs
+  mapfile -t g_python_jobs < <(jobs -r)
+  unset g_python_result
+
+  # Use python in backround for multiple python commands running much faster
+  #if [ -z "${g_python_running}" ]
+  if [[ ${g_python_jobs[*]} != *python-pipeexec.py* ]]
+  then
+    mkdir -p ${g_python_tmp}
+    if [ -s ${g_python_error} ]
+    then
+      g_echo_error "From last python run: $(cat ${g_python_error})"
+    fi
+    [ -p ${g_python_in} ] || mkfifo ${g_python_in}
+    echo "while 1:
+  exec(open(\"${g_python_in}\").read())
+  print('DONE')
+" >${g_python_tmp}/python-pipeexec.py
+
+    # python stream channel
+    { python3 -u ${g_python_tmp}/python-pipeexec.py >>${g_python_out} 2>>${g_python_error} & }
+    g_python_running="true"
+  fi
+
+  # do python
+  >${g_python_out}
+  >${g_python_error}
+  echo $@ >${g_python_in}
+
+  while true
+  do
+   
+    # Check for output
+    if [ -s ${g_python_out} ]
+    then
+      mapfile -t g_python_result <${g_python_out}
+      if [[ ${g_python_result[-1]} == DONE ]]
+      then
+        # remove the DONE output (last array element
+        unset g_python_result[-1]
+        break
+      fi
+    fi
+
+    # Check for error
+    mapfile -t g_python_jobs < <(jobs -r)
+    if [ -s "${g_python_error}" ] || [[ ${g_python_jobs[*]} != *python-pipeexec.py* ]]
+    then
+      g_echo_error "Python Progress not running:
+$(cat ${g_python_error})"
+      return 1
+    fi
+
+    # sleep a short time
+    sleep 0.1
+  done
+
+}
+

gaboshlib/g_read_csv.sh

@@ -0,0 +1,98 @@
+function g_read_csv {
+  unset v_csv_array_associative
+  unset v_csv_array_associative_reverse
+  unset g_csv_array
+
+  local g_csvfile="$1"
+  local g_last_lines=$2
+  local g_headline=$3
+  local g_separator=$4
+
+  local g_headline_item i l r g_csv_headline_array
+
+  # check for individual separator
+  [ -z "$g_separator" ] && g_separator=","
+  # only one/first character
+  g_separator=${g_separator:0:1}
+
+  # check given file
+  if ! [ -s "$g_csvfile" ]
+  then
+    g_echo_error "${g_csvfile} does not exist or is empty"
+    return 1
+  fi
+
+  # check for ,
+  if ! grep -q $g_separator "$g_csvfile"
+  then
+    g_echo_error "${g_csvfile} does not contain \"$g_separator\""
+    return 1
+  fi
+
+  # is there a headline file
+  if [ -z "$g_headline" ]
+  then
+    [ -s "${g_csvfile}.headline" ] && g_headline=$(<"${g_csvfile}.headline")
+  fi
+
+  # if no given headline and no headlinfile use forst line
+  [ -z "$g_headline" ] && g_headline=$(head -n1 "${g_csvfile}")
+
+  # read the headline
+  unset g_csv_headline_array_ref
+  g_array "$g_headline" g_csv_headline_array_ref $g_separator
+
+  # prepare varnames from headline(s)
+  for g_headline_item in "${g_csv_headline_array_ref[@]}"
+  do
+    g_csv_headline_array+=("${g_headline_item//[^a-zA-Z0-9_]/}")
+  done
+
+  g_basename $g_csvfile
+  local g_csvfile_base=${g_basename_result/\.history*.csv/}
+  g_csvfile_base=${g_csvfile_base//[^a-zA-Z0-9_]/}
+  g_csvfile_base=${g_csvfile_base//ECONOMY*/}
+
+  # read last lines if defined or complete csv file
+  if [ -n "$g_last_lines" ]
+  then
+    tail -n $g_last_lines "$g_csvfile" >"${g_tmp}/g_csv_tmp.csv"
+    g_csvfile="${g_tmp}/g_csv_tmp.csv"
+  fi
+
+  # read csv file to array
+  g_array "$g_csvfile" g_csv_array_ref
+  g_csv_array=("${g_csv_array_ref[@]}")
+
+  # create associative arrays forward and reverse and superarray v/vr
+  declare -Ag v_csv_array_associative
+  declare -Ag v_csv_array_associative_reverse
+  declare -Ag v
+  declare -Ag vr
+  l=0  
+  for (( r=${#g_csv_array[@]}-1 ; r>=0 ; r-- ))
+  do 
+    g_array "${g_csv_array[r]}" g_csv_line_array ,
+    i=0
+    # put headlines to vars with reverse numbers (last line 0)
+    for g_headline_item in "${g_csv_headline_array[@]}"
+    do
+      #declare -g v_${l}_${g_headline_item}="${g_csv_line_array[i]}"
+      # last line to vars without number
+      [ "$l" = 0 ] && declare -g v_${g_headline_item}="${g_csv_line_array[i]}"
+      v_csv_array_associative[${g_headline_item}_${r}]="${g_csv_line_array[i]}"
+      v_csv_array_associative_reverse[${g_headline_item}_${l}]="${g_csv_line_array[i]}"
+      if [ -z "${g_csvfile_base}" ]
+      then
+        v[${g_headline_item}_${r}]=${g_csv_line_array[i]}
+        vr[${g_headline_item}_${l}]=${g_csv_line_array[i]}
+      else
+        v[${g_csvfile_base}_${g_headline_item}_${r}]=${g_csv_line_array[i]}
+        vr[${g_csvfile_base}_${g_headline_item}_${l}]=${g_csv_line_array[i]}
+      fi
+      ((i++))
+    done
+    ((l++))
+  done
+}
+

gaboshlib/g_traceback.sh

@@ -1,5 +1,6 @@
 function g_traceback {
   local deptn=${#FUNCNAME[@]}
+  local i
   for ((i=1; i<$deptn; i++)); do
     local func="${FUNCNAME[$i]}"
     local line="${BASH_LINENO[$((i-1))]}"
@@ -7,4 +8,5 @@ function g_traceback {
     printf '%*s' $i '' # indent
     echo "at: $func, $src, line $line" 1>&2
   done
+  echo "$@" 1>&2
 }

gaboshlib/g_wget.sh

@@ -0,0 +1,7 @@
+function g_wget {
+  wget -T 10 -t 2 \
+   --header="User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" \
+   --header="Content-Type: application/json" \
+   --header="Accept-Language: en-US,en," \
+   $@
+}