From fa4fec677bf18bdf08ec7710f935278678906192 Mon Sep 17 00:00:00 2001 From: olli Date: Tue, 5 Nov 2024 12:43:24 +0100 Subject: [PATCH] percentage change year for specific values like CPI, added calc OHLCV, speed ptimization, fixes --- dabo/functions/get_indicators.sh | 80 ++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/dabo/functions/get_indicators.sh b/dabo/functions/get_indicators.sh index 3603e50..40f6a27 100644 --- a/dabo/functions/get_indicators.sh +++ b/dabo/functions/get_indicators.sh @@ -26,7 +26,8 @@ function get_indicators_all { local f_histfile f_symbol - find asset-histories -maxdepth 1 -name "ECONOMY-*.history.[0-5][5dhwm]*.csv" | sort | while read f_histfile + # ECONOMY and MARKETDATA + find asset-histories -maxdepth 1 -name "ECONOMY-*.history.[0-5][5dhwm]*.csv" -o -name "MARKETDATA_*.history.[0-5][5dhwm]*.csv" | sort | while read f_histfile do if [ -s "${f_histfile}.fetching" ] || [ -s "${f_histfile}.indicators-calculating" ] then @@ -69,7 +70,7 @@ function get_indicators_all { fi done done - shopt +s nullglob + shopt -u nullglob } @@ -82,17 +83,27 @@ function get_indicators { local f_fill_missing_ohlcv_intervals=$3 local f_line + # check if the job is already done + if [ $(wc -l <"${f_histfile}") -gt 801 ] + then + if ! tail +801 "${f_histfile}" | egrep -vq "^....-..-..*,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.\-]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.]+,[0-9\.\-]+,[0-9\.\-]+,[0-9\.\-]+,[a-z]*,[0-9\.\-]+,[0-9\.\-]+" + then + g_echo_note "${f_histfile} seems to be indicator-complete" + return 0 + fi + fi + # history local f_columns="date,open,high,low,close,volume,change,ath,ema12,ema26,ema50,ema100,ema200,ema400,ema800,rsi5,rsi14,rsi21,macd,macd_ema9_signal,macd_histogram,macd_histogram_signal,macd_histogram_max,macd_histogram_strength" local f_emas="12 26 50 100 200 400 800" local f_rsis="5 14 21" - local f_ema f_change f_changed f_line f_valid_data f_ath + local f_ema f_change f_changed f_line f_valid_data f_ath f_last_year local f_columns_space="${f_columns//,/ }" g_read_csv "${f_histfile}" "${f_last_intervals}" "$f_columns" for ((i=0; i<=${#g_csv_array[@]}-1; i++)) do - if [ -z "${v_csv_array_associative[date_${i}]}" ] + if [ -z "${v_csv_array_associative[date_${i}]}" ] || [ -z "${v_csv_array_associative[open_${i}]}" ] then g_echo_note "No data $f_histfile:${v_csv_array_associative[date_${i}]}" return 0 @@ -105,16 +116,42 @@ function get_indicators { ### check for unfilled fields f_change="" + # check olhc data + if [ -z "${v_csv_array_associative[high_${i}]}" ] && [ -z "${v_csv_array_associative[low_${i}]}" ] && [ -z "${v_csv_array_associative[close_${i}]}" ] + then + g_echo_note "fixing OHLC Data" + # open is close + v_csv_array_associative[close_${i}]=${v_csv_array_associative[open_${i}]} + # open is previous close + [ -n "${v_csv_array_associative[close_${p}]}" ] && v_csv_array_associative[open_${i}]="${v_csv_array_associative[close_${p}]}" + # calc high/low from open/close + if g_num_is_higher_equal ${v_csv_array_associative[open_${i}]} ${v_csv_array_associative[close_${i}]} + then + v_csv_array_associative[high_${i}]=${v_csv_array_associative[open_${i}]} + v_csv_array_associative[low_${i}]=${v_csv_array_associative[close_${i}]} + else + v_csv_array_associative[high_${i}]=${v_csv_array_associative[close_${i}]} + v_csv_array_associative[low_${i}]=${v_csv_array_associative[open_${i}]} + fi + f_change=1 + fi + # check for missing percentage change if [ -z "${v_csv_array_associative[change_${i}]}" ] then - #if ! [ $p -lt 0 ] - #then - #echo "g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]}" - #g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]} && f_change=1 - g_percentage-diff ${v_csv_array_associative[open_${i}]} ${v_csv_array_associative[close_${i}]} && f_change=1 - v_csv_array_associative[change_${i}]=${g_percentage_diff_result} - #fi + # special for changes watched per year like CPI,... + if [[ $f_histfile = "asset-histories/MARKETDATA_US_CONSUMER_PRICE_INDEX_CPI.history.1d.csv" ]] || \ + [[ $f_histfile = "asset-histories/MARKETDATA_US_UNEMPLOYMENT_RATE.history.1d.csv" ]] + then + if [ $i -ge 12 ] + then + f_last_year=$((i-12)) + g_percentage-diff ${v_csv_array_associative[open_${f_last_year}]} ${v_csv_array_associative[close_${i}]} + v_csv_array_associative[year_change_${i}]=${g_percentage_diff_result} + fi + fi + g_percentage-diff ${v_csv_array_associative[open_${i}]} ${v_csv_array_associative[close_${i}]} && f_change=1 + v_csv_array_associative[change_${i}]=${g_percentage_diff_result} fi # ath (all-time-high) of present data @@ -135,10 +172,20 @@ function get_indicators { # check for missing EMAs for f_ema_column in $f_emas do - # check for enough values/lines to calculate EMA - [ $i -ge $f_ema_column ] || continue + # check for enough values/lines to calculate EMA if no previous EMA given + if [ -z "${v_csv_array_associative[ema${f_ema_column}_${p}]}" ] + then + if ! [ $i -ge $f_ema_column ] + then + #echo "not enough lines $i -ge $f_ema_column" + continue + fi + fi # calculate EMA - [ -z "${v_csv_array_associative[ema${f_ema_column}_${i}]}" ] && calc_ema ${f_ema_column} close && f_change=1 + if [ -z "${v_csv_array_associative[ema${f_ema_column}_${i}]}" ] + then + calc_ema ${f_ema_column} close && f_change=1 + fi done # check for missing RSI @@ -164,6 +211,11 @@ function get_indicators { # build line for f_column in $f_columns do + # special for changes watched per year like CPI,... + if [[ $f_column = change ]] + then + [ -n "${v_csv_array_associative[year_change_${i}]}" ] && v_csv_array_associative[change_${i}]=${v_csv_array_associative[year_change_${i}]} + fi if [ -z "$f_line" ] then f_line="${v_csv_array_associative[${f_column}_${i}]}"