percentage change year for specific values like CPI, added calc OHLCV, speed ptimization, fixes

dabo/functions/get_indicators.sh

@@ -26,7 +26,8 @@ function get_indicators_all {
   
   local f_histfile f_symbol
 
-  find asset-histories -maxdepth 1 -name "ECONOMY-*.history.[0-5][5dhwm]*.csv" | sort | while read f_histfile
+  # ECONOMY and MARKETDATA
+  find asset-histories -maxdepth 1 -name "ECONOMY-*.history.[0-5][5dhwm]*.csv" -o -name "MARKETDATA_*.history.[0-5][5dhwm]*.csv" | sort | while read f_histfile
   do
     if [ -s "${f_histfile}.fetching" ] || [ -s "${f_histfile}.indicators-calculating" ]
     then
@@ -69,7 +70,7 @@ function get_indicators_all {
       fi
     done
   done
-  shopt +s nullglob
+  shopt -u nullglob
 
 }
 
@@ -82,17 +83,27 @@ function get_indicators {
   local f_fill_missing_ohlcv_intervals=$3
   local f_line 
 
+  # check if the job is already done
+  if [ $(wc -l <"${f_histfile}") -gt 801 ] 
+  then
+    if ! tail +801 "${f_histfile}" | egrep -vq "^....-..-..*,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.\-]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.\-]+,[0-9\.\-]+,[0-9\.\-]+,[a-z]*,[0-9\.\-]+,[0-9\.\-]+"
+    then
+      g_echo_note "${f_histfile} seems to be indicator-complete"
+      return 0
+    fi
+  fi
+  
   # history
   local f_columns="date,open,high,low,close,volume,change,ath,ema12,ema26,ema50,ema100,ema200,ema400,ema800,rsi5,rsi14,rsi21,macd,macd_ema9_signal,macd_histogram,macd_histogram_signal,macd_histogram_max,macd_histogram_strength"
   local f_emas="12 26 50 100 200 400 800"
   local f_rsis="5 14 21"
-  local f_ema f_change f_changed f_line f_valid_data f_ath
+  local f_ema f_change f_changed f_line f_valid_data f_ath f_last_year
   local f_columns_space="${f_columns//,/ }"
   g_read_csv "${f_histfile}" "${f_last_intervals}" "$f_columns"
   for ((i=0; i<=${#g_csv_array[@]}-1; i++))
   do
 
-    if [ -z "${v_csv_array_associative[date_${i}]}" ] 
+    if [ -z "${v_csv_array_associative[date_${i}]}" ] || [ -z "${v_csv_array_associative[open_${i}]}" ]
     then
       g_echo_note "No data $f_histfile:${v_csv_array_associative[date_${i}]}"
       return 0
@@ -105,16 +116,42 @@ function get_indicators {
     ### check for unfilled fields
     f_change=""
 
+    # check olhc data
+    if [ -z "${v_csv_array_associative[high_${i}]}" ] && [ -z "${v_csv_array_associative[low_${i}]}" ] && [ -z "${v_csv_array_associative[close_${i}]}" ]
+    then
+      g_echo_note "fixing OHLC Data"
+      # open is close
+      v_csv_array_associative[close_${i}]=${v_csv_array_associative[open_${i}]}
+      # open is previous close
+      [ -n "${v_csv_array_associative[close_${p}]}" ] && v_csv_array_associative[open_${i}]="${v_csv_array_associative[close_${p}]}"
+      # calc high/low from open/close 
+      if g_num_is_higher_equal ${v_csv_array_associative[open_${i}]}  ${v_csv_array_associative[close_${i}]}
+      then
+        v_csv_array_associative[high_${i}]=${v_csv_array_associative[open_${i}]}
+        v_csv_array_associative[low_${i}]=${v_csv_array_associative[close_${i}]}
+      else
+        v_csv_array_associative[high_${i}]=${v_csv_array_associative[close_${i}]}
+        v_csv_array_associative[low_${i}]=${v_csv_array_associative[open_${i}]}
+      fi
+      f_change=1
+    fi
+    
     # check for missing percentage change
     if [ -z "${v_csv_array_associative[change_${i}]}" ]
     then
-      #if ! [ $p -lt 0 ] 
+      # special for changes watched per year like CPI,...
-      #then
+      if [[ $f_histfile = "asset-histories/MARKETDATA_US_CONSUMER_PRICE_INDEX_CPI.history.1d.csv" ]] || \
-        #echo "g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]}"
+       [[ $f_histfile = "asset-histories/MARKETDATA_US_UNEMPLOYMENT_RATE.history.1d.csv" ]]
-        #g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]} && f_change=1
+      then
-        g_percentage-diff ${v_csv_array_associative[open_${i}]} ${v_csv_array_associative[close_${i}]} && f_change=1
+        if [ $i -ge 12 ]
-        v_csv_array_associative[change_${i}]=${g_percentage_diff_result}
+        then
-      #fi
+          f_last_year=$((i-12))
+          g_percentage-diff ${v_csv_array_associative[open_${f_last_year}]} ${v_csv_array_associative[close_${i}]}
+          v_csv_array_associative[year_change_${i}]=${g_percentage_diff_result}
+        fi
+      fi
+      g_percentage-diff ${v_csv_array_associative[open_${i}]} ${v_csv_array_associative[close_${i}]} && f_change=1
+      v_csv_array_associative[change_${i}]=${g_percentage_diff_result}
     fi
     
     # ath (all-time-high) of present data
@@ -135,10 +172,20 @@ function get_indicators {
     # check for missing EMAs
     for f_ema_column in $f_emas
     do
-      # check for enough values/lines to calculate EMA
+      # check for enough values/lines to calculate EMA if no previous EMA given
-      [ $i -ge $f_ema_column ] || continue
+      if [ -z "${v_csv_array_associative[ema${f_ema_column}_${p}]}" ]
+      then
+        if ! [ $i -ge $f_ema_column ] 
+        then
+          #echo "not enough lines $i -ge $f_ema_column"
+          continue
+        fi
+      fi
       # calculate EMA
-      [ -z "${v_csv_array_associative[ema${f_ema_column}_${i}]}" ] && calc_ema ${f_ema_column} close && f_change=1
+      if [ -z "${v_csv_array_associative[ema${f_ema_column}_${i}]}" ] 
+      then
+        calc_ema ${f_ema_column} close && f_change=1
+      fi
     done
 
     # check for missing RSI
@@ -164,6 +211,11 @@ function get_indicators {
       # build line
       for f_column in $f_columns
       do
+        # special for changes watched per year like CPI,...
+        if [[ $f_column = change ]]
+        then
+          [ -n "${v_csv_array_associative[year_change_${i}]}" ] && v_csv_array_associative[change_${i}]=${v_csv_array_associative[year_change_${i}]}
+        fi
         if [ -z "$f_line" ]
         then
           f_line="${v_csv_array_associative[${f_column}_${i}]}"