From 927102b2e9908d0bd2aef534d009df14f9bedce7 Mon Sep 17 00:00:00 2001 From: olli Date: Sun, 7 Jul 2024 16:39:38 +0200 Subject: [PATCH] indicators, levels, fixes,... --- dabo/calc-indicators-hist.sh | 17 ++ dabo/dabo-bot.sh | 33 +-- dabo/dabo-prep.sh | 3 - dabo/fetch-ohlcv-candles-indicators.sh | 16 ++ dabo/fetch-orders.sh | 2 +- ...etch-assets.sh => fetch-symbols_ticker.sh} | 4 +- dabo/fetch-transaction-history.sh | 2 +- dabo/functions/calc_ema.sh | 70 ++++++ dabo/functions/calc_macd.sh | 81 +++++++ dabo/functions/calc_rsi.sh | 87 +++++++ dabo/functions/ccxt.sh | 45 ++-- dabo/functions/get_assets.sh | 49 ---- dabo/functions/get_balance.sh | 2 +- dabo/functions/get_indicators.sh | 149 ++++++++++++ dabo/functions/get_levels.sh | 121 ++++++++++ dabo/functions/get_macd_indicator.sh | 123 ---------- ...ic.sh => get_marketdata_yahoo_historic.sh} | 49 +++- dabo/functions/get_ohlcv-candle.sh | 182 ++++++++++++++ dabo/functions/get_orders.sh | 8 +- dabo/functions/get_positions.sh | 2 +- dabo/functions/get_symbols.sh | 19 -- dabo/functions/get_symbols_ticker.sh | 63 +++++ dabo/functions/get_transactions.sh | 36 ++- dabo/functions/position_close.sh | 2 +- dabo/functions/transactions_overview.sh | 4 +- dabo/functions/webpage.sh | 224 +++++------------- docker-compose.yml | 58 ++++- 27 files changed, 993 insertions(+), 458 deletions(-) create mode 100755 dabo/calc-indicators-hist.sh create mode 100755 dabo/fetch-ohlcv-candles-indicators.sh rename dabo/{fetch-assets.sh => fetch-symbols_ticker.sh} (86%) create mode 100644 dabo/functions/calc_ema.sh create mode 100644 dabo/functions/calc_macd.sh create mode 100644 dabo/functions/calc_rsi.sh delete mode 100644 dabo/functions/get_assets.sh create mode 100644 dabo/functions/get_indicators.sh create mode 100644 dabo/functions/get_levels.sh delete mode 100644 dabo/functions/get_macd_indicator.sh rename dabo/functions/{get_marketdata_historic.sh => get_marketdata_yahoo_historic.sh} (62%) create mode 100644 dabo/functions/get_ohlcv-candle.sh delete mode 100644 dabo/functions/get_symbols.sh create mode 100644 dabo/functions/get_symbols_ticker.sh diff --git a/dabo/calc-indicators-hist.sh b/dabo/calc-indicators-hist.sh new file mode 100755 index 0000000..14c1c81 --- /dev/null +++ b/dabo/calc-indicators-hist.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +. /dabo/dabo-prep.sh + +rm asset-histories/*.history.*.csv.indicators-calculating + +while true +do + sleep 300 + # Reload Config + . ../../dabo-bot.conf + . ../../dabo-bot.override.conf + # get orders + get_indicators_all + rm asset-histories/*.history.*.csv.indicators-calculating +done + diff --git a/dabo/dabo-bot.sh b/dabo/dabo-bot.sh index 3ac258d..8ea8d32 100755 --- a/dabo/dabo-bot.sh +++ b/dabo/dabo-bot.sh @@ -15,7 +15,7 @@ echo $0 | grep -q "dabo-bot\.sh" && BOT=1 find asset-histories -name "*.csv" -type f | while read csv_file do csv_timestamp=$(ls --time-style='+%Y%m%d%H%M' -l "${csv_file}" | cut -d" " -f6) - sed -i "/[0-9]$(date +%Y)-/d" ${csv_file} + sed -i "/[0-9]$(date +%Y)-/d" "${csv_file}" touch -t ${csv_timestamp} "${csv_file}" done @@ -75,20 +75,6 @@ do ## watch some manual defined assets #watch_assets - # transactions overview - #[ ${FULL_LOOP} = 1 ] && transactions_overview - - # stock data - #if [ "${STOCK_EXCHANGE}" = "BINANCE" ] - #then - # # command for current token infos (function for setting var QUANTITY_LOT_CUT - # TOKEN_INFO_CMD="binance_get_token_info" - # # command for buying/selling a token - # TRADE_CMD='binance-api-call POST /api/v3/order "&symbol=TOKEN"eOrderQty=QUANTITY&side=ACTION&type=MARKET"' - #elif [ "${STOCK_EXCHANGE}" = "ONETRADING" ] - #then - # TOKEN_INFO_CMD="onetrading_get_token_info" - # TRADE_CMD='onetrading-api-call POST public/v1/account/orders "--header \"Content-Type: application/json\" --data \"{\\\"instrument_code\\\":\\\"TOKEN\\\",\\\"side\\\":\\\"ACTION\\\",\\\"type\\\":\\\"MARKET\\\",\\\"amount\\\":\\\"QUANTITY\\\"}\""' if [ "${STOCK_EXCHANGE}" = "NONE" ] then ## stop here if STOCK_EXCHANGE not present @@ -96,11 +82,8 @@ do fi # Get current symbols - [ ${FULL_LOOP} = 1 ] && get_symbols + [ ${FULL_LOOP} = 1 ] && get_symbols_ticker - ## Get current assets - #get_assets || continue - # Sell something? #check_for_sell @@ -110,16 +93,8 @@ do # Get current positions [ ${FULL_LOOP} = 1 ] && get_positions || continue - # Buy something? - [ ${FULL_LOOP} = 1 ] && check_for_buy - - # Update webpage - #if jobs | egrep -q "Running.+webpage" && [ ${FULL_LOOP} = 1 ] - #then - # g_echo_note "webpage already running" - #else - # webpage & - #fi + ## Buy something? + #[ ${FULL_LOOP} = 1 ] && check_for_buy done diff --git a/dabo/dabo-prep.sh b/dabo/dabo-prep.sh index b44bcdc..a005a25 100644 --- a/dabo/dabo-prep.sh +++ b/dabo/dabo-prep.sh @@ -1,7 +1,5 @@ # functions -# Export all functions and vars -set -a BASEPATH=/dabo/htdocs # load functions @@ -21,6 +19,5 @@ set +a # prepare directories mkdir -p ${BASEPATH}/botdata/asset-histories -mkdir -p ${BASEPATH}/botdata/trade-histories cd ${BASEPATH}/botdata diff --git a/dabo/fetch-ohlcv-candles-indicators.sh b/dabo/fetch-ohlcv-candles-indicators.sh new file mode 100755 index 0000000..74f73d3 --- /dev/null +++ b/dabo/fetch-ohlcv-candles-indicators.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +. /dabo/dabo-prep.sh + +while true +do + # Reload Config + . ../../dabo-bot.conf + . ../../dabo-bot.override.conf + # Timestamp + export f_timestamp=$(g_date_print) + # get orders + get_ohlcv-candles + sleep 10 +done + diff --git a/dabo/fetch-orders.sh b/dabo/fetch-orders.sh index 8e863e3..82e756f 100755 --- a/dabo/fetch-orders.sh +++ b/dabo/fetch-orders.sh @@ -10,7 +10,7 @@ do # Timestamp export f_timestamp=$(g_date_print) # get orders - get_symbols + get_symbols_ticker get_orders sleep 3600 done diff --git a/dabo/fetch-assets.sh b/dabo/fetch-symbols_ticker.sh similarity index 86% rename from dabo/fetch-assets.sh rename to dabo/fetch-symbols_ticker.sh index d92d3a3..f2393f4 100755 --- a/dabo/fetch-assets.sh +++ b/dabo/fetch-symbols_ticker.sh @@ -13,7 +13,7 @@ do # Timestamp export f_timestamp=$(g_date_print) # get assets - get_assets - sleep 30 + get_symbols_ticker refetchonly || g_echo_warn "Error while refetching tickers from ${STOCK_EXCHANGE}" + sleep 2 done diff --git a/dabo/fetch-transaction-history.sh b/dabo/fetch-transaction-history.sh index 3cbc9af..bafdf5e 100755 --- a/dabo/fetch-transaction-history.sh +++ b/dabo/fetch-transaction-history.sh @@ -2,7 +2,7 @@ . /dabo/dabo-prep.sh -sleep 1800 +#sleep 1800 while true do transactions_overview diff --git a/dabo/functions/calc_ema.sh b/dabo/functions/calc_ema.sh new file mode 100644 index 0000000..5187ecb --- /dev/null +++ b/dabo/functions/calc_ema.sh @@ -0,0 +1,70 @@ +function calc_ema { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # - needs array ${v_csv_array_associative[${f_column}_${i}] for example from g_read_csv + # - needs $i as position + + local f_period=$1 # integer! + local f_column=$2 # column in "$v_csv_array_associative" from which ema should be calculated + local f_target_column=$3 # column with previus EMAs - if not given "ema$f_period" is used + + local f_position=$i # position + + # check if there is a position (i from loop through array) + [ -z "$f_position" ] && return 1 + + # check if there is a period (i from loop through array) + [ -z "$f_period" ] && return 2 + + # check if there is a column (i from loop through array) + [ -z "$f_column" ] && return 3 + + # check for enough positions/values to calculate (enough values) + [ $f_position -ge $f_period ] || return 0 + + # get ema column + [ -z "$f_target_column" ] && local f_target_column="ema$f_period" + + local f_last_value=${v_csv_array_associative[${f_column}_${f_position}]} + [ -z "$f_target_column" ] && return 4 + + local v + + # reset old ema var + unset f_ema + + # find last EMA + local f_last_ema_position=$((f_position-1)) + local f_last_ema=${v_csv_array_associative[${f_target_column}_${f_last_ema_pos}]} + + # check if last EMA is given + if [ -n "$f_last_ema" ] + then + # calc EMA with previous EMA if given + g_calc "scale=10; ${f_last_value}*(2/(${f_period}+1))+${f_last_ema}*(1-(2/(${f_period}+1)))" + else + ## calc SMA if previous EMA is not given (only needed on first EMA calc) + # get last $f_period values + local f_last_period_values_from=$((f_position-$f_period+1)) + local f_last_period_values + for ((v=$f_last_period_values_from; v<=${f_position}; v++)) + do + if [ -z ${f_last_period_values} ] + then + f_last_period_values=${v_csv_array_associative[${f_column}_${v}]} + else + f_last_period_values="$f_last_period_values+${v_csv_array_associative[${f_column}_${v}]}" + fi + done + # calc SMA (EMA=SMA in this special first case) + g_calc "($f_last_period_values)/$f_period" + fi + + # write back EMA + v_csv_array_associative[${f_target_column}_${f_position}]=$g_calc_result + f_ema=$g_calc_result + + return 0 +} + diff --git a/dabo/functions/calc_macd.sh b/dabo/functions/calc_macd.sh new file mode 100644 index 0000000..022c4dd --- /dev/null +++ b/dabo/functions/calc_macd.sh @@ -0,0 +1,81 @@ +#!/bin/bash +function calc_macd { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # - needs array ${v_csv_array_associative} for example from g_read_csv + # - needs $i as position + # - needs $p as previous position + + local f_period=$1 # integer! + local f_ema12=$2 # ema12 - if not given "${v_csv_array_associative[ema12_${f_position}]}" is used + local f_ema26=$3 # ema26 - if not given "${v_csv_array_associative[ema26_${f_position}]}" is used + local f_target_column=$4 # column with previus RSIs - if not given "rsi$f_period" is used + + local f_position=$i # position + + # check if there is a position (i from loop through array) + [ -z "$f_position" ] && return 1 + + # check for EMA12 and 26 + [ -z "$f_ema12" ] && f_ema12="${v_csv_array_associative[ema12_${f_position}]}" + [ -z "$f_ema12" ] && return 2 + [ -z "$f_ema26" ] && f_ema26="${v_csv_array_associative[ema26_${f_position}]}" + [ -z "$f_ema26" ] && return 3 + + # get rsi column + [ -z "$f_target_column" ] && f_target_column="macd" + + local f_macd f_macd_ema9_signal f_macd_signal f_macd_histogram_relation f_macd_histogram f_macd_histogram_max f_macd_histogram_strength + + g_calc "${f_ema12}-${f_ema26}" + f_macd=$g_calc_result + v_csv_array_associative[macd_${f_position}]=$g_calc_result + + # calc MACD Signal + calc_ema 9 macd macd_ema9_signal + [ -z "${v_csv_array_associative[macd_ema9_signal_${f_position}]}" ] && return 5 + f_macd_ema9_signal=${v_csv_array_associative[macd_ema9_signal_${f_position}]} + + # calc MACD Histogram + g_calc "${f_macd}-(${f_macd_ema9_signal})" + f_macd_histogram=$g_calc_result + v_csv_array_associative[macd_histogram_${f_position}]=$g_calc_result + + # check for MACD signal up- or downtrend and buy or sell if switched histogram from - to + or + to - + f_last_histogram=${v_csv_array_associative[macd_histogram_${p}]} + if [ -n "$f_last_histogram" ] + then + f_macd_signal=uptrend + [[ $f_macd_histogram =~ ^- ]] && f_macd_signal=downtrend + [[ $f_macd_histogram =~ ^- ]] && [[ $f_last_histogram =~ ^[0-9] ]] && f_macd_signal=sell + [[ $f_macd_histogram =~ ^[0-9] ]] && [[ $f_last_histogram =~ ^- ]] && f_macd_signal=buy + v_csv_array_associative[macd_histogram_signal_${f_position}]=$f_macd_signal + fi + + # check if there is a new macd max value to calculate the strength of the trend + f_macd_histogram_positive=${f_macd_histogram//-/} + f_macd_histogram_max=${v_csv_array_associative[macd_histogram_max_${p}]} + if [ -z "$f_macd_histogram_max" ] + then + # define max for the first time + v_csv_array_associative[macd_histogram_max_${f_position}]=$f_macd_histogram_positive + f_macd_histogram_max=$f_macd_histogram_positive + else + if g_num_is_higher $f_macd_histogram_positive $f_macd_histogram_max + then + v_csv_array_associative[macd_histogram_max_${f_position}]=$f_macd_histogram_positive + f_macd_histogram_max=$f_macd_histogram_positive + else + v_csv_array_associative[macd_histogram_max_${f_position}]=$f_macd_histogram_max + fi + fi + + # calculate relative trend strength (percentage 100 = strongest in history) + g_percentage-diff ${f_macd_histogram_max} ${f_macd_histogram_positive} + [ -z "$g_percentage_diff_result" ] && g_percentage_diff_result=0 + g_calc "100+(${g_percentage_diff_result})" + f_macd_histogram_strength=${g_calc_result} + v_csv_array_associative[macd_histogram_strength_${f_position}]=$f_macd_histogram_strength + +} diff --git a/dabo/functions/calc_rsi.sh b/dabo/functions/calc_rsi.sh new file mode 100644 index 0000000..93e9539 --- /dev/null +++ b/dabo/functions/calc_rsi.sh @@ -0,0 +1,87 @@ +function calc_rsi { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # - needs array ${v_csv_array_associative[${f_column}_${i}] for example from g_read_csv + # - needs $i as position + + local f_period=$1 # integer! + local f_column=$2 # column in "$v_csv_array_associative" from which rsi should be calculated + local f_target_column=$3 # column with previus RSIs - if not given "rsi$f_period" is used + + local f_position=$i # position + + # check if there is a position (i from loop through array) + [ -z "$f_position" ] && return 1 + + # check if there is a period if not default to 14 + [ -z "$f_period" ] && f_period=14 + + # check if there is a column (i from loop through array) + [ -z "$f_column" ] && return 3 + + # check for enough positions/values to calculate (enough values) + [ $f_position -ge $f_period ] || return 0 + + # get rsi column + [ -z "$f_target_column" ] && f_target_column="rsi$f_period" + + local f_last_value=${v_csv_array_associative[${f_column}_${f_position}]} + [ -z "$f_target_column" ] && return 4 + + local v + + # reset old rsi sar + unset f_rsi + + # get last $f_period values + local f_last_period_values_from=$((f_position-$f_period+1)) + local f_last_period_values_positive=0 + local f_last_period_values_negative=0 + local f_last_period_num_positive=0 + local f_last_period_num_negative=0 + for ((v=$f_last_period_values_from; v<=${f_position}; v++)) + do + if [[ ${v_csv_array_associative[${f_column}_${v}]} =~ ^- ]] + then + ((f_last_period_num_negative++)) + f_last_period_values_negative="$f_last_period_values_negative+(${v_csv_array_associative[${f_column}_${v}]})" + else + ((f_last_period_num_positive++)) + f_last_period_values_positive="$f_last_period_values_positive+${v_csv_array_associative[${f_column}_${v}]}" + fi + done + + # add positive and negative values + g_calc "$f_last_period_values_positive" + local f_positive_sum=$g_calc_result + g_calc "$f_last_period_values_negative" + local f_negative_sum=${g_calc_result//-} + + # if one of both is "0" then fix results + [ ${f_negative_sum} = "0" ] && f_rsi=100 + [ ${f_positive_sum} = "0" ] && f_rsi=0 + + # calculate RSI + if [ -z "$f_rsi" ] + then + # calculate positive/negative change averages + g_calc "${f_negative_sum}/${f_last_period_num_negative}" + local f_negative_sum_average=$g_calc_result + g_calc "${f_positive_sum}/${f_last_period_num_positive}" + local f_positive_sum_average=$g_calc_result + + # calculate RS + g_calc "${f_positive_sum_average}/${f_negative_sum_average}" + local f_rs=$g_calc_result + + # calculate RSI + g_calc "100-(100/(1+${f_rs}))" + printf -v f_rsi "%.0f" $g_calc_result + + fi + + v_csv_array_associative[${f_target_column}_${f_position}]=$f_rsi + +} + diff --git a/dabo/functions/ccxt.sh b/dabo/functions/ccxt.sh index e912ab4..abffe3f 100644 --- a/dabo/functions/ccxt.sh +++ b/dabo/functions/ccxt.sh @@ -1,5 +1,8 @@ function f_ccxt { - + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # remove old result unset f_ccxt_result @@ -15,52 +18,62 @@ function f_ccxt { fi # Initialize ccxt in python if not initialized - mapfile -t g_ccxt_jobs < <(jobs -r) - [[ ${g_ccxt_jobs[*]} =~ *python-pipeexec.py* ]] || unset f_ccxt_initialized if [ -z "$f_ccxt_initialized" ] then + g_echo_note "Initializing ccxt" g_python 'import os' || return 1 g_python 'import sys' || return 1 g_python 'sys.path.append("/ccxt/python")' || return 1 g_python 'import ccxt' || return 1 fi + + # Initialize exchange in ccxt if not initialized if ! [[ "$f_ccxt_initialized" =~ $STOCK_EXCHANGE ]] then + g_echo_note "Initializing exchange ${STOCK_EXCHANGE} in ccxt" local f_exchange_type="swap" [ -z "$LEVERAGE" ] && f_exchange_type="spot" g_python "${STOCK_EXCHANGE} = ccxt.${STOCK_EXCHANGE}({'apiKey': '${API_KEY}','secret': '${API_SECRET}','enableRateLimit': True,'options': {'defaultType': '${f_exchange_type}',},})" || return 1 - g_python "${STOCK_EXCHANGE}.load_markets()" + g_python "${STOCK_EXCHANGE}markets=${STOCK_EXCHANGE}.load_markets()" || return 1 f_ccxt_initialized="${f_ccxt_initialized}${STOCK_EXCHANGE}," fi - + # send and receive ccxt command in python - on error kill progress - if ! g_python "$@" + if ! g_python "$@" then - kill -9 $(jobs -l | grep python-pipeexec.py | cut -d" " -f 2) + g_echo_warn "Resetting CCXT!!!" + g_kill_all_background_jobs "python3 -iuq" + unset f_ccxt_initialized return 1 fi + # reference result to python-result declare -ng f_ccxt_result=g_python_result # Check for json output or empty json output - f_ccxt_json_out="" + unset f_ccxt_json_out if ! [ "$f_ccxt_result" = '[]' ] then [[ $f_ccxt_result =~ ^\[ ]] && [[ $f_ccxt_result =~ \]$ ]] && f_ccxt_json_out=1 [[ $f_ccxt_result =~ ^\{ ]] && [[ $f_ccxt_result =~ \}$ ]] && f_ccxt_json_out=1 fi - # make the output jq-conform if json poutput - # avoids errors like: "parse error: Invalid numeric literal at" - if [ -n "$f_ccxt_json_out" ] + if [ -z "$f_ccxt_json_out" ] then - f_ccxt_result=${f_ccxt_result//\'/\"} - f_ccxt_result=${f_ccxt_result// None/ null} - f_ccxt_result=${f_ccxt_result// True/ true} - f_ccxt_result=${f_ccxt_result// False/ false} - f_ccxt_result=${f_ccxt_result//,,/,} + return 1 + else + # make the output jq-conform if json poutput + # avoids errors like: "parse error: Invalid numeric literal at" + f_ccxt_result=$(echo $f_ccxt_result | sed "s/'/\"/g; s/ None/ null/g; s/ True/ true/g; s/ False/ false/g; s/,,/,/g") + # sed is needed here because bash parameter substitution like down here hands with 100% cpu usage if the variable is large. Noticed with output about ~2.5M + #f_ccxt_result=${f_ccxt_result//\'/\"} + #f_ccxt_result=${f_ccxt_result// None/ null} + #f_ccxt_result=${f_ccxt_result// True/ true} + #f_ccxt_result=${f_ccxt_result// False/ false} + #f_ccxt_result=${f_ccxt_result//,,/,} fi return 0 } + diff --git a/dabo/functions/get_assets.sh b/dabo/functions/get_assets.sh deleted file mode 100644 index af1371d..0000000 --- a/dabo/functions/get_assets.sh +++ /dev/null @@ -1,49 +0,0 @@ -function get_assets { - - g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - - ## determine assets with prices - if [ ${STOCK_EXCHANGE} = "NONE" ] - then - f_no_exchange=1 - return 0 - else - f_ccxt "print(${STOCK_EXCHANGE}.fetch_tickers())" && [ -n "$f_ccxt_json_out" ] && echo $f_ccxt_result >CCXT_TICKERS_RAW - fi - jq -r '.[] | .symbol + "," + (.last|tostring)' CCXT_TICKERS_RAW | sed "s/:${CURRENCY},/,/; s/\///" | grep "${CURRENCY}," >CCXT_TICKERS - - # Write file with asset list and ignore marketcap under LARGEST_MARKETCAP - cat CCXT_TICKERS | cut -d"," -f1 | while read f_ASSET - do - # ignore marketcap under LARGEST_MARKETCAP - f_assetwocurrency=$(echo ${f_ASSET} | sed "s/$CURRENCY$//") - if ! egrep -q "^${f_assetwocurrency}$" SORTMARKETCAP - then - #g_echo_note "Ignoring $f_ASSET because of lower marketcap then Top ${LARGEST_MARKETCAP}" - continue - fi - echo ${f_ASSET} >>ASSETS.tmp - done - mv ASSETS.tmp ASSETS - - ## write histfiles parallel - #local f_ASSET - #local f_parallel_arg - #echo -n "parallel -j3 bash -c --" >/tmp/parallel - #for f_ASSET in $(cat ASSETS | egrep -v "${BLACKLIST}") - #do - # #get_asset "${f_ASSET}" - # echo -n " \"get_asset ${f_ASSET}\"" >>/tmp/parallel - #done - #export f_timestamp - #export csv_headline - #. /tmp/parallel - - ## alternatively single jobs (for debugging!?) - for f_ASSET in $(cat ASSETS | egrep -v "${BLACKLIST}") - do - get_asset "${f_ASSET}" - done - -} - diff --git a/dabo/functions/get_balance.sh b/dabo/functions/get_balance.sh index 784ec66..1b8e58e 100644 --- a/dabo/functions/get_balance.sh +++ b/dabo/functions/get_balance.sh @@ -2,7 +2,7 @@ function get_balance { g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - f_ccxt "print(${STOCK_EXCHANGE}.fetch_balance ({\"currency\": \"$CURRENCY\"}))" && [ -n "$f_ccxt_json_out" ] && echo $f_ccxt_result >CCXT_BALANCE + f_ccxt "print(${STOCK_EXCHANGE}.fetch_balance ({\"currency\": \"$CURRENCY\"}))" && echo $f_ccxt_result >CCXT_BALANCE # get current investmentbalance f_CURRENCY_BALANCE=$(jq -r ".${CURRENCY}.free" CCXT_BALANCE) diff --git a/dabo/functions/get_indicators.sh b/dabo/functions/get_indicators.sh new file mode 100644 index 0000000..144946e --- /dev/null +++ b/dabo/functions/get_indicators.sh @@ -0,0 +1,149 @@ +function get_indicators_all { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_last_intervals="$1" + + local f_histfile f_symbol + + shopt -s nullglob + + # find all hitory files + get_symbols_ticker + for f_symbol in "${f_symbols_array[@]}" + do + f_symbol=${f_symbol%%:*} + f_symbol=${f_symbol//\/} + + for f_histfile in "asset-histories/${f_symbol}.history."[0-5][5dhwm]*.csv + do + if [ -s "$f_histfile" ] + then + # check for already running jobs + if [ -s "${f_histfile}.fetching" ] + then + g_echo_note "Fetching active on ${f_histfile}" + continue + fi + if [ -s "${f_histfile}.indicators-calculating" ] + then + g_echo_note "Indicators-Calculating already active on ${f_histfile}" + continue + fi + + printf '%(%Y-%m-%d %H:%M:%S)T' >"${f_histfile}.indicators-calculating" + get_indicators "${f_histfile}" ${f_last_intervals} && printf '%(%Y-%m-%d %H:%M:%S)T' >>"$f_histfile.indicators-calculated" + rm -f "${f_histfile}.indicators-calculating" + + fi + done + done + +} + + +function get_indicators { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + local f_histfile="$1" + local f_last_intervals="$2" + local f_line + + # history + local f_columns="date,open,high,low,close,volume,change,ath,ema12,ema26,ema50,ema100,ema200,ema400,ema800,rsi5,rsi14,rsi21,macd,macd_ema9_signal,macd_histogram,macd_histogram_signal,macd_histogram_max,macd_histogram_strength" + local f_emas="12 26 50 100 200 400 800" + local f_rsis="5 14 21" + local f_ema f_change f_changed f_line f_valid_data f_ath + local f_columns_space="${f_columns//,/ }" + g_read_csv "${f_histfile}" "${f_last_intervals}" "$f_columns" + for ((i=0; i<=${#g_csv_array[@]}-1; i++)) + do + + if [ -z "${v_csv_array_associative[date_${i}]}" ] + then + g_echo_note "No data $f_histfile:${v_csv_array_associative[date_${i}]}" + return 0 + fi + #g_echo_note "=== $0 for $f_histfile:${v_csv_array_associative[date_${i}]},$f_histfile:${v_csv_array_associative[close_${i}]}" + + # get previous position + p=$((i-1)) + + ### check for unfilled fields + f_change="" + + # check for missing percentage change + if [ -z "${v_csv_array_associative[change_${i}]}" ] + then + if ! [ $p -lt 0 ] + then + #echo "g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]}" + g_percentage-diff ${v_csv_array_associative[close_${p}]} ${v_csv_array_associative[close_${i}]} && f_change=1 + v_csv_array_associative[change_${i}]=${g_percentage_diff_result} + fi + fi + + # ath (all-time-high) of present data + if [ -z "${v_csv_array_associative[ath_${p}]}" ] + then + # define max for the first time + v_csv_array_associative[ath_${i}]=${v_csv_array_associative[high_${i}]} + else + #echo "g_num_is_higher ${v_csv_array_associative[high_${i}]} ${v_csv_array_associative[ath_${p}]}" + if g_num_is_higher ${v_csv_array_associative[high_${i}]} ${v_csv_array_associative[ath_${p}]} + then + v_csv_array_associative[ath_${i}]=${v_csv_array_associative[high_${i}]} + else + v_csv_array_associative[ath_${i}]=${v_csv_array_associative[ath_${p}]} + fi + fi + + # check for missing EMAs + for f_ema_column in $f_emas + do + # check for enough values/lines to calculate EMA + [ $i -ge $f_ema_column ] || continue + # calculate EMA + [ -z "${v_csv_array_associative[ema${f_ema_column}_${i}]}" ] && calc_ema ${f_ema_column} close && f_change=1 + done + + # check for missing RSI + for f_rsi_column in $f_rsis + do + # check for enough values/lines to calculate RSI + [ $i -ge $f_rsi_column ] || continue + # calculate RSI + [ -z "${v_csv_array_associative[rsi${f_rsi_column}_${i}]}" ] && calc_rsi ${f_rsi_column} change && f_change=1 + done + + # check for missing macd + [ $i -ge 26 ] && [ -z "${v_csv_array_associative[macd_${i}]}" ] && calc_macd && f_change=1 + + # write to file if change is provided + if [ -n "$f_change" ] + then + # find line by date + f_line_date="${v_csv_array_associative[date_${i}]}" + oldIFS=$IFS + IFS=, + f_line="" + # build line + for f_column in $f_columns + do + if [ -z "$f_line" ] + then + f_line="${v_csv_array_associative[${f_column}_${i}]}" + else + f_line="$f_line,${v_csv_array_associative[${f_column}_${i}]}" + fi + done + g_echo_note "Changing values with date ${v_csv_array_associative[date_${i}]} in \"${f_histfile}\"" + # replace line by date + sed -i "s/$f_line_date,.*/$f_line/" "$f_histfile" + IFS=$oldIFS + fi + + done + +} + diff --git a/dabo/functions/get_levels.sh b/dabo/functions/get_levels.sh new file mode 100644 index 0000000..4822761 --- /dev/null +++ b/dabo/functions/get_levels.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +function get_levels_all { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + local f_histfile f_level f_symbol + + get_symbols_ticker + for f_symbol in "${f_symbols_array[@]}" + do + f_symbol=${f_symbol%%:*} + f_symbol=${f_symbol//\/} + + for f_histfile in "asset-histories/${f_symbol}.history.1d.csv" "asset-histories/${f_symbol}.history.4h.csv" "asset-histories/${f_symbol}.history.15m.csv" + do + [ -s "$f_histfile" ] || continue 2 + done + f_histfile="asset-histories/${f_symbol}.history.csv" + printf '%(%Y-%m-%d %H:%M:%S)T' >"${f_histfile}.levels-calculating" + g_echo_note "Estimating relevant levels of $f_symbol" + mapfile -t f_prices < <(cut -d, -f3,4,5 "asset-histories/${f_symbol}.history.1d.csv" "asset-histories/${f_symbol}.history.4h.csv" "asset-histories/${f_symbol}.history.15m.csv" | sed 's/,/\n/g' | sort -rnu) + get_levels && printf '%(%Y-%m-%d %H:%M:%S)T' + rm -f "${f_histfile}.levels-calculating" + + for f_level in "$g_levels[@]" + do + echo $f_level + done >"${f_histfile}.levels.new" + mv "${f_histfile}.levels.new" "${f_histfile}.levels" + done +} + + + +function get_levels { + + # estimates the relevant levels from price list from array f_prices and put then in arra f_levels + # - needs array $f_prices with prices sorted from low to high and no duplicates (sort -nru) to analyze + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # if there is not enough or no price data + [ -z "$f_prices[100]" ] && return 1 + + # reset old levels var + unset f_levels + + local f_min_occurrences i j f_level f_level_count f_level_prices f_level_first_price f_baseprice f_threshold_test + + # some key points + local f_lowest_price=${f_prices[-1]} + local f_highest_price=${f_prices[1]} + local f_number_of_prices=${#f_prices[@]} + # calc percentual price range + g_percentage-diff $f_highest_price $f_lowest_price + + # calc threshold (avarage of percentual price difference) + local f_price_range_percentage=${g_percentage_diff_result//-/} + g_calc "$f_price_range_percentage / $f_number_of_prices" + local f_threshold=$g_calc_result + # calc threshold in range (1/100 of percentual range) + g_calc "$f_price_range_percentage / 100" + local f_threshold_in_range=$g_calc_result + + # how much occurencies / same prices have so show up for a defined level - percentage from number of prices + local f_min_occurrences=3 + + + # Loop through the f_prices and compare each number with the next + for ((i=0; i<${#f_prices[@]}-1; i++)) + do + #echo "$i of ${#f_prices[@]}" + + # pair this and next elemtn + j=$((i+1)) + + f_threshold_test=$f_threshold + f_baseprice=${f_prices[i]} + # if we are in a level use first price of level + if [ -n "$f_level_count" ] + then + f_baseprice=$f_level_first_price + f_threshold_test=$f_threshold_in_range + fi + + # pair similiar? + if g_num_is_approx ${f_prices[j]} $f_baseprice $f_threshold_test $f_threshold_test + then + # first number of similars? + if [ -z "$f_level_count" ] + then + # new level + f_level_count=2 + f_level_prices="${f_prices[i]}+${f_prices[j]}" + f_level_first_price=${f_prices[i]} + else + # add values to level + f_level_count=$((f_level_count+1)) + f_level_prices="$f_level_prices+${f_prices[j]}" + fi + #echo "level ($f_level_count): $f_level_prices" + else + if [ -n "$f_level_count" ] + then + # end of level + if [ "$f_level_count" -ge "$f_min_occurrences" ] + then + g_calc "($f_level_prices)/$f_level_count" + f_levels+=("$g_calc_result") + #echo "ending relevant level $g_calc_result after $f_level_count times" + fi + f_level_prices="" + f_level_count="" + f_level_first_price="" + fi + fi + done + +} + + diff --git a/dabo/functions/get_macd_indicator.sh b/dabo/functions/get_macd_indicator.sh deleted file mode 100644 index db57c15..0000000 --- a/dabo/functions/get_macd_indicator.sh +++ /dev/null @@ -1,123 +0,0 @@ -function get_macd_indicator { - - #g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - # get histfile - local f_hist_file="$1" - - # define or clean gloval macdvars - f_macd_ema12="" - f_macd_ema26="" - f_macd="" - f_macd_ema9_signal="" - f_macd_signal="" - - if ! [ -s "$f_hist_file" ] - then - g_echo_warn "${FUNCNAME} $@: Histfile $f_hist_file does not exist or is empty" - return 1 - fi - - # read last two lines - local f_lastline=$(tail -n1 "$f_hist_file" | grep ^2) - - # force min 15min 2 last lines - local f_second_lastline=$(tail -n2 "$f_hist_file" | head -n1 | grep ^2) - - # Try to get current MACD values - local f_macd_lastprice=$(echo "$f_lastline" | cut -d, -f2) - f_macd_ema12=$(echo "$f_lastline" | cut -d, -f4) - f_macd_ema26=$(echo "$f_lastline" | cut -d, -f5) - f_macd=$(echo "$f_lastline" | cut -d, -f6) - f_macd_ema9_signal=$(echo "$f_lastline" | cut -d, -f7) - - # if data is complete - if [ -n "${f_macd_ema12}" ] && [ -n "${f_macd_ema26}" ] && [ -n "${f_macd}" ] && [ -n "${f_macd_ema9_signal}" ] - then - g_echo_note "${FUNCNAME} $@: Looks like current MACD was already calculated" - return 1 - fi - - # Try to get last MACD values - f_macd_last_ema12=$(echo "$f_second_lastline" | cut -d, -f4) - f_macd_last_ema26=$(echo "$f_second_lastline" | cut -d, -f5) - f_macd_last=$(echo "$f_second_lastline" | cut -d, -f6) - f_macd_last_ema9_signal=$(echo "$f_second_lastline" | cut -d, -f7) - f_macd_last_histogram=$(echo "$f_second_lastline" | cut -d, -f8) - - # calc EMA12 - get_ema "${f_hist_file}" 2 12 "${f_macd_last_ema12}" "${f_macd_lastprice}" - if [ -z "${f_ema}" ] - then - g_echo_note "${FUNCNAME} $@: Not enough data for calculating EMA12 - waiting for more values" - echo -n ",,,,,," >>"${f_hist_file}" - return 0 - fi - f_macd_ema12=${f_ema} - echo -n ",${f_macd_ema12}" >>"${f_hist_file}" - - # calc EMA26 - if [ -z "${f_macd_last_ema12}" ] - then - echo -n ",,,,," >>"${f_hist_file}" - return 0 - fi - get_ema "${f_hist_file}" 2 26 "${f_macd_last_ema26}" "${f_macd_lastprice}" - if [ -z "${f_ema}" ] - then - g_echo_note "${FUNCNAME} $@: Not enough data for calculating EMA26 - waiting for more values" - echo -n ",,,,," >>"${f_hist_file}" - return 0 - fi - f_macd_ema26=${f_ema} - echo -n ",${f_macd_ema26}" >>"${f_hist_file}" - - # calc MACD - if [ -z "${f_macd_ema26}" ] - then - echo -n ",,,," >>"${f_hist_file}" - return 0 - fi - #[ -z "${f_macd_ema12}" ] && return 0 - f_macd=$(echo "scale=8; ${f_macd_ema12}-${f_macd_ema26}" | bc | sed 's/^\./0./; s/^-\./-0./') - echo -n ",${f_macd}" >>"${f_hist_file}" - - # calc MACD Signal - if [ -z "${f_macd}" ] - then - echo -n ",,," >>"${f_hist_file}" - return 0 - fi - get_ema "${f_hist_file}" 6 9 "${f_macd_last_ema9_signal}" "${f_macd}" - if [ -z "${f_ema}" ] - then - g_echo_note "${FUNCNAME} $@: Not enough data for calculating EMA9 Signal - waiting for more values" - echo -n ",,," >>"${f_hist_file}" - return 0 - fi - f_macd_ema9_signal=${f_ema} - echo -n ",${f_macd_ema9_signal}" >>"${f_hist_file}" - - # calc MACD Histogram - f_macd_histogram=$(echo "scale=10; ${f_macd}-(${f_macd_ema9_signal})" | bc | sed 's/^\./0./; s/^-\./-0./') - echo -n ",${f_macd_histogram}" >>"${f_hist_file}" - - # check for MACD buy or sell signal - echo ${f_macd_histogram} | grep -q "^-" && echo ${f_macd_last_histogram} | grep -q "^[0-9]" && f_macd_signal="sell" - echo ${f_macd_histogram} | grep -q "^[0-9]" && echo ${f_macd_last_histogram} | grep -q "^-" && f_macd_signal="buy" - - # calculate MACD Histogram relation - if [ $(tail -n36 "${f_hist_file}" | wc -l) -ge 35 ] - then - local f_macd_histogram_max=$(tail -n350 "${f_hist_file}" | cut -d"," -f8 | egrep "^[0-9]|^-[0-9]" | sed 's/^-//' | sort -n | tail -n1) - g_percentage-diff ${f_macd_histogram_max} ${f_macd_histogram} - #local f_macd_histogram_relation=$(echo "100+$(g_percentage-diff ${f_macd_histogram_max} ${f_macd_histogram})" | bc | sed 's/^\./0./; s/^-\./-0./' | cut -d\. -f1) - g_calc "100+${g_percentage_diff_result}" - local f_macd_histogram_relation=${g_calc_result} - fi - - echo -n ",${f_macd_histogram_relation}|${f_macd_signal}" >>"${f_hist_file}" - - - -} - diff --git a/dabo/functions/get_marketdata_historic.sh b/dabo/functions/get_marketdata_yahoo_historic.sh similarity index 62% rename from dabo/functions/get_marketdata_historic.sh rename to dabo/functions/get_marketdata_yahoo_historic.sh index ea3253d..db19eb2 100644 --- a/dabo/functions/get_marketdata_historic.sh +++ b/dabo/functions/get_marketdata_yahoo_historic.sh @@ -1,12 +1,37 @@ function get_marketdata_yahoo_historic { - #g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" local f_item="$1" local f_name="$2" + local f_timeframe="$3" + local f_targetcsv="asset-histories/${f_name}.history-yahoo.csv" local f_targetcsvtmp="${g_tmp}/${f_name}.history-yahoo.csv" - + [ -n "$f_timeframe" ] && f_targetcsv="${g_tmp}/${f_name}.history-yahoo.${f_timeframe}.csv" + f_histfile_yahoo="$f_targetcsv" + + [ "$f_timeframe" = "1w" ] && f_timeframe="1wk" + [ -z "$f_timeframe" ] && f_timeframe="1d" + + + # transform CCXT symbols to Yahoo symbol + if [[ $f_item =~ / ]] + then + # change / to - + f_item=${f_item////-} + # remove :* (:USDT in contract markets) + f_item=${f_item//:*} + # remove spaces + f_item=${f_item/ /} + fi + + # USDT to USD + f_item=${f_item//USDT/USD} + # BUSD to USD + f_item=${f_item//BUSD/USD} + + # special names of some coins/currencies of yahoo finance [[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X" [[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X" [[ $f_item = "ARB-USD" ]] && f_item="ARB11841-USD" @@ -20,22 +45,26 @@ function get_marketdata_yahoo_historic { [[ $f_item = "BEER-USD" ]] && f_item="BEER31337-USD" [[ $f_item = "TAI-USD" ]] && f_item="TAI20605-USD" [[ $f_item = "DEFI-USD" ]] && f_item="DEFI29200-USD" + [[ $f_item = "TON-USD" ]] && f_item="TON11419-USD" + [[ $f_item = "BRETT-USD" ]] && f_item="BRETT29743-USD" + [[ $f_item = "ADS-USD" ]] && f_item="%24ADS-USD" + [[ $f_item = "PT-USD" ]] && f_item="PT28582-USD" # end if already failed the last hour - if [ -f FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD ] + if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ] then find "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" -mmin +60 -delete - if [ -f FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD ] + if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ] then #g_echo_note "${f_targetcsv} already failed to downloaded within last hour" return 1 fi fi - # end if already exists and modified under 1 day - if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mtime -1 | grep -q "${f_targetcsv}" + # end if already exists and modified under given time + if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}" then - #g_echo_note "${f_targetcsv} has already been downloaded within a day" + #g_echo_note "${f_targetcsv} has already been downloaded within 2 minutes" return 0 fi @@ -46,7 +75,7 @@ function get_marketdata_yahoo_historic { # Download historical data from yahoo #g_echo_note "Fetching Yahoo-Historical data of $f_name" - g_wget -O ${f_targetcsvtmp} "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history" 2>"${f_targetcsvtmp}".err + g_wget -O ${f_targetcsvtmp} "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history" 2>"${f_targetcsvtmp}".err if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ] then egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" "${f_targetcsv}" | sort -u >"${f_targetcsv}.tmp" @@ -58,7 +87,7 @@ function get_marketdata_yahoo_historic { # report error # g_echo_note "Fetching historical data of $f_name from Yahoo failed! #CMD: -#wget -q -O ${f_targetcsvtmp} ${g_wget_opts} \"https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history\" +#wget -q -O ${f_targetcsvtmp} ${g_wget_opts} \"https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history\" # #OUT: #$(cat "${f_targetcsvtmp}") @@ -67,7 +96,7 @@ function get_marketdata_yahoo_historic { #$(cat "${f_targetcsvtmp}".err) #" mkdir -p FAILED_YAHOO - mv ${f_targetcsvtmp}.err FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD + mv "${f_targetcsvtmp}.err" "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" return 1 fi } diff --git a/dabo/functions/get_ohlcv-candle.sh b/dabo/functions/get_ohlcv-candle.sh new file mode 100644 index 0000000..e627284 --- /dev/null +++ b/dabo/functions/get_ohlcv-candle.sh @@ -0,0 +1,182 @@ +function get_ohlcv-candles { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_asset f_histfile f_histfile_week f_symbol f_timeframe + local f_timeframes="1w 1d 4h 1h 15m 5m" + + get_symbols_ticker + + for f_symbol in "${f_symbols_array[@]}" + do + # fetch only single symbols (for debugging) + #[ "$f_symbol" = "BTC/USDT:USDT" ] || continue + echo "=== Fetching/Refreshing $f_symbol ===" + for f_timeframe in $f_timeframes + do + f_asset="${f_symbol//:*}" + f_asset="${f_asset///}" + f_histfile="asset-histories/$f_asset.history.$f_timeframe.csv" + f_histfile_week="asset-histories/$f_asset.history.1w.csv" + + if [ -s "${f_histfile}.indicators-calculating" ] + then + g_echo_note "Indicators calculating active on ${f_histfile}" + continue + fi + + # get data + printf '%(%Y-%m-%d %H:%M:%S)T' >"${f_histfile}.fetching" + get_ohlcv-candle "$f_symbol" $f_timeframe "${f_histfile}" "${f_histfile_week}" && printf '%(%Y-%m-%d %H:%M:%S)T' >>"$f_histfile.fetched" + + # refresh latest indicators + get_indicators "${f_histfile}" 801 + + rm -f "${f_histfile}.fetching" + + done + done +} + +function get_ohlcv-candle { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_symbol="$1" + local f_timeframe=$2 + local f_histfile="$3" + local f_histfile_week="$4" + + local f_yahoo f_date f_unit_date f_data f_data_array f_data_unit f_open f_high f_low f_close f_volume + + # fetch >=1d from yahoo finance + if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "1w" ] || [ "$f_timeframe" = "1mo" ] + then + get_marketdata_yahoo_historic "$f_symbol" "$f_asset" $f_timeframe && f_yahoo=1 + fi + + # fetch OHLCV data (loop because of multiple chunks on exchanges) + while true + do + # fetch data + if [ -z "$f_yahoo" ] + then + # find latest time which is not fetched already create f_since + get_ohlcv-candle-latest "$f_symbol" "$f_histfile" "$f_histfile_week" + [ -z $f_since ] && break + + # from exchange + g_echo_note "Get $f_symbol OHLCV-candle $f_timeframe data since $f_since_date" + f_ccxt "print($STOCK_EXCHANGE.fetchOHLCV(symbol='$f_symbol', timeframe='$f_timeframe', since=$f_since))" || return 1 + # parse the result to array f_data_array + f_data=${f_ccxt_result//[} + f_data=${f_data//, /,} + f_data=${f_data//]]} + f_data=${f_data//],/+} + g_array $f_data f_data_ref + + else + # from yahoo finance + g_array $f_histfile_yahoo f_data_ref + fi + + f_data_array=("${f_data_ref[@]}") + + #echo "+++ $f_data" + #echo "--- ${f_data_array[-1]}" + + # check if last data already in history file and end if already present + g_array ${f_data_array[-1]} f_last_data_unit_ref , + [ -z "$f_yahoo" ] && printf -v f_last_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_last_data_unit_ref[0]::-3} + [ -n "$f_yahoo" ] && f_last_unit_date="${f_last_data_unit_ref[0]}" + + #echo "grep -q ^\"$f_last_unit_date\" \"$f_histfile\"" + [ -s "$f_histfile" ] && grep -q ^"${f_last_unit_date}," "$f_histfile" && break + + # go through data and write to history file if new units available + for f_data_unit in "${f_data_array[@]}" + do + + # use array for each unit and assignt values to vars + g_array $f_data_unit f_data_unit_ref , + [ -z "$f_yahoo" ] && printf -v f_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_data_unit_ref[0]::-3} + [ -n "$f_yahoo" ] && f_unit_date="${f_last_data_unit_ref[0]}" + + # check if date is already in history file + [ -s "$f_histfile" ] && grep -q ^"$f_unit_date" "$f_histfile" && continue + + # define field vars and convert exponential number (for example 9.881e-05) to "normal" notation + f_open=${f_data_unit_ref[1]} + g_num_exponential2normal "$f_open" && f_open=$f_g_num_exponential2normal_result + f_high=${f_data_unit_ref[2]} + g_num_exponential2normal "$f_high" && f_high=$f_g_num_exponential2normal_result + f_low=${f_data_unit_ref[3]} + g_num_exponential2normal "$f_low" && f_low=$f_g_num_exponential2normal_result + f_close=${f_data_unit_ref[4]} + g_num_exponential2normal "$f_close" && f_close=$f_g_num_exponential2normal_result + f_volume=${f_data_unit_ref[5]} + [ -n "$f_yahoo" ] && f_volume=${f_data_unit_ref[6]} + g_num_exponential2normal "$f_volume" && f_volume=$f_g_num_exponential2normal_result + + # check date for valid date + if ! [[ $f_unit_date =~ ^2[0-1][0-9][0-9]-[0-1][0-9]-[0-3][0-9]( [0-2][0-9]:[0-5][0-9]:[0-5][0-9])?$ ]] + then + g_echo_warn "Date $f_unit_date \"$f_data_unit\" seems to be invalid @$f_histfile:$f_data_unit" + break + fi + + # check vars for valid numbers + if ! g_num_valid_number "$f_open" "$f_high" "$f_low" "$f_close" "$f_volume" + then + echo "$f_open $f_high $f_low $f_close $f_volume" + g_echo_warn "Data in \"$f_data_unit\" seems to be invalid @$f_histfile:$f_unit_date" + break + fi + + # write history file + #echo "$f_unit_date,$f_open,$f_high,$f_low,$f_close,$f_volume" + echo "$f_unit_date,$f_open,$f_high,$f_low,$f_close,$f_volume" >>"$f_histfile" + + done + + # end if yahoo (complete file and not time chunks) + [ -n "$f_yahoo" ] && break + + # end if lates refresh is this day + printf -v f_date '%(%Y-%m-%d)T\n' + #echo "[ $f_date = $f_since_date ]" + if [ $f_date = $f_since_date ] + then + break + fi + + done +} + +function get_ohlcv-candle-latest { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_symbol="$1" + local f_histfile="$2" + local f_histfile_week="$3" + + # find latest time which is not fetched already + if [ -s "$f_histfile" ] + then + # get latest date from histfile if it exists + f_since=$(date -d "$(grep ^[0-9] "$f_histfile" | tail -n1 | cut -d, -f1)" +%s000) + else + # if hist does not exist + # get week to find the oldest point in time available in exchange + f_ccxt "print($STOCK_EXCHANGE.fetchOHLCV(symbol='$f_symbol', timeframe='1w'))" || return 0 + # parse oldest point in time from json output + f_since=${f_ccxt_result//[} + f_since=${f_since//, /,} + f_since=${f_since//]*} + f_since=${f_since//,*} + fi + + # get the date + printf -v f_since_date '%(%Y-%m-%d)T\n' ${f_since::-3} + +} diff --git a/dabo/functions/get_orders.sh b/dabo/functions/get_orders.sh index 0e99886..137226d 100644 --- a/dabo/functions/get_orders.sh +++ b/dabo/functions/get_orders.sh @@ -30,17 +30,15 @@ function get_orders { g_echo_note "Getting orders from $f_symbol to \"CCXT_OPEN_ORDERS_$f_symbol_file\"" if f_ccxt "print($STOCK_EXCHANGE.fetchOpenOrders(symbol='${f_symbol}'))" then - if [ -z "$f_ccxt_json_out" ] - then - rm -f "CCXT_OPEN_ORDERS_${f_symbol_file}_RAW" "CCXT_OPEN_ORDERS_${f_symbol_file}" - continue - fi echo $f_ccxt_result | tee "CCXT_OPEN_ORDERS_${f_symbol_file}_RAW" | jq -r " .[] | select(.status==\"open\") | .symbol + \",\" + .type + \",\" + .side + \",\" + (.price|tostring) + \",\" + (.stopPrice|tostring) + \",\" + (.amount |tostring) " >"CCXT_OPEN_ORDERS_${f_symbol_file}" + else + rm -f "CCXT_OPEN_ORDERS_${f_symbol_file}_RAW" "CCXT_OPEN_ORDERS_${f_symbol_file}" + continue fi done } diff --git a/dabo/functions/get_positions.sh b/dabo/functions/get_positions.sh index 928a780..371244f 100644 --- a/dabo/functions/get_positions.sh +++ b/dabo/functions/get_positions.sh @@ -16,7 +16,7 @@ function get_positions { done [ -z "$f_symbols" ] && return 1 - f_ccxt "print($STOCK_EXCHANGE.fetchPositions(symbols=[${f_symbols}]))" && [ -n "$f_ccxt_json_out" ] && echo $f_ccxt_result >CCXT_POSITIONS_RAW + f_ccxt "print($STOCK_EXCHANGE.fetchPositions(symbols=[${f_symbols}]))" && echo $f_ccxt_result >CCXT_POSITIONS_RAW jq -r " .[] | diff --git a/dabo/functions/get_symbols.sh b/dabo/functions/get_symbols.sh deleted file mode 100644 index d3648f4..0000000 --- a/dabo/functions/get_symbols.sh +++ /dev/null @@ -1,19 +0,0 @@ -function get_symbols { - - g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - - local f_symbols="" - [ -s CCXT_TICKERS_RAW ] && f_symbols=$(jq -r '.[] | .symbol' CCXT_TICKERS_RAW) - f_symbols=${f_symbols//$'\n'/'+'} - - if [ -z "$f_symbols" ] - then - g_echo_warn "Could not get symbols list - empty" - return 1 - fi - - g_array "$f_symbols" f_symbols_array_ref + - f_symbols_array=("${g_array[@]}") - printf '%s\n' "${f_symbols_array[@]}" >SYMBOLS-$STOCK_EXCHANGE - -} diff --git a/dabo/functions/get_symbols_ticker.sh b/dabo/functions/get_symbols_ticker.sh new file mode 100644 index 0000000..50cb2f4 --- /dev/null +++ b/dabo/functions/get_symbols_ticker.sh @@ -0,0 +1,63 @@ +function get_symbols_ticker { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_fetch=$1 + local f_symbols + + ## determine assets with prices + [ ${STOCK_EXCHANGE} = "NONE" ] && return 0 + + # refetch from exchange + if [ "$f_fetch" = "refetchonly" ] + then + # fetch from exchange + f_ccxt "print(${STOCK_EXCHANGE}.fetch_tickers())" && echo $f_ccxt_result >CCXT_TICKERS_RAW + + # parse relevant tokens + local f_grep="${CURRENCY}," + [ -n "$LEVERAGE" ] && f_grep="${CURRENCY}:${CURRENCY}," + [ -s CCXT_TICKERS_RAW ] && jq -r '.[] | .symbol + "," + (.last|tostring)' CCXT_TICKERS_RAW | grep "${f_grep}" >CCXT_TICKERS + + if [ -s CCXT_TICKERS ] + then + cut -d, -f1 CCXT_TICKERS >CCXT_SYMBOLS + cat CCXT_SYMBOLS >CCXT_SYMBOLS-$STOCK_EXCHANGE + + ## get symbols by volume from history files and check with CCXT_SYMBOLS-$STOCK_EXCHANGE + [ -n "$LEVERAGE" ] && f_naming="${CURRENCY}:${CURRENCY}" + tail -n1 asset-histories/*.history.1w.csv 2>/dev/null \ + | perl -pe "s/<==\n//; s/==> //; s/\.csv /,/g; s/\//./; s/$CURRENCY\.history\.1w/\/${f_naming}./" \ + | grep ",$(date +%Y)-" \ + | sort -r -n -t, -k7 \ + | cut -d. -f2 \ + >CCXT_SYMBOLS-$STOCK_EXCHANGE-by-volume + # add mising (not yet fetched) symbols + awk 'NR==FNR{a[$0];next} !($0 in a)' CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume CCXT_SYMBOLS-$STOCK_EXCHANGE >>CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume + # remove (no more) existing symbols + awk 'NR==FNR{a[$0];next} !($0 in a)' CCXT_SYMBOLS-$STOCK_EXCHANGE CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume \ + | while read f_remove_symbol + do + sed -i "\#${f_remove_symbol}#d" CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume + done + + else + return 1 + fi + return 0 + fi + + g_array CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume f_symbols_array_ref + f_symbols_array=("${f_symbols_array_ref[@]}") + + [ -s CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume ] && f_symbols=$(cat CCXT_SYMBOLS-${STOCK_EXCHANGE}-by-volume) + f_symbols=${f_symbols//$'\n'/'+'} + + if [ -z "$f_symbols" ] + then + g_echo_warn "Could not get symbols list - empty" + return 1 + fi + +} + diff --git a/dabo/functions/get_transactions.sh b/dabo/functions/get_transactions.sh index 504baf9..45ab72d 100644 --- a/dabo/functions/get_transactions.sh +++ b/dabo/functions/get_transactions.sh @@ -15,16 +15,17 @@ function get_transactions { [[ $f_exchange = bitpanda ]] && continue [[ $f_exchange = onetrading ]] && continue - + g_echo_note "Exchange: $f_exchange" - get_symbols - #f_ccxt "print(${STOCK_EXCHANGE}.symbols)" - #f_symbols=${f_ccxt_result} - #f_symbols=${f_symbols//\"} - #f_symbols=${f_symbols//, /+} - #f_symbols=${f_symbols//\[} - #f_symbols=${f_symbols//\]} + # refetch symbols if not default exchange + if ! [[ $DEFAULT_STOCK_EXCHANGE = $STOCK_EXCHANGE ]] + then + get_symbols_ticker refetchonly + fi + + # load symbols to array + get_symbols_ticker # transfer-dir mkdir -p "TRANSACTIONS-$f_exchange" @@ -37,10 +38,8 @@ function get_transactions { #for f_symbol in $f_symbols for f_symbol in "${f_symbols_array[@]}" do - #echo $f_symbol - #[[ $f_symbol =~ ETH|BTC ]] || continue - #IFS=$f_orig_IFS - # binance does not allow derivate trading in germany so ignore because of 400-Error + + # binance does not allow derivate trading in many countries so ignore because of 400-Error [[ $f_symbol =~ : ]] && [[ $f_exchange = binance ]] && continue f_symbol_file="TRANSACTIONS-$f_exchange/${f_symbol//\/}" @@ -50,15 +49,14 @@ function get_transactions { # fetch only if not exists [ -f "$f_symbol_file" ] && continue g_echo_note "fetching closed orders of $f_symbol on $f_exchange" - f_ccxt "print(${STOCK_EXCHANGE}.fetchMyTrades(symbol='$f_symbol', limit=500, params={'paginate': True}))" + f_ccxt "print(${STOCK_EXCHANGE}.fetchMyTrades(symbol='$f_symbol', limit=500, params={'paginate': True}))" || continue # write to file - #cat ${g_python_out} >"$f_symbol_file" - [[ $f_ccxt_result = \[\] ]] && f_ccxt_result="" + #[[ $f_ccxt_result = \[\] ]] && f_ccxt_result="" echo -n $f_ccxt_result >"$f_symbol_file" # continue if no trade - [ -z "$f_ccxt_result" ] && continue + #[ -z "$f_ccxt_result" ] && continue # get f_asset+f_currency from symbol (BTC/USDT) g_array $f_symbol f_symbol_array / @@ -77,8 +75,7 @@ function get_transactions { f_leverage="leverage-" # get funding fees - f_ccxt "print(${STOCK_EXCHANGE}.fetchFundingHistory('$f_symbol', limit=200, params={'paginate': True}))" - [[ $f_ccxt_result = \[\] ]] || echo -n $f_ccxt_result >"${f_symbol_file}.FundingFees" + f_ccxt "print(${STOCK_EXCHANGE}.fetchFundingHistory('$f_symbol', limit=200, params={'paginate': True}))" && echo -n $f_ccxt_result >"${f_symbol_file}.FundingFees" cat ${f_symbol_file}.FundingFees | jq -r " .[] | .datetime + \",fundingfee,$f_asset,0,\" + .code + \",0\" + \",$f_exchange,\" + .code + \",\" + (.amount|tostring) @@ -152,8 +149,7 @@ function get_transactions { [ -s "${f_convert_file}.csv" ] && continue - f_ccxt "print(${f_exchange}.fetchConvertTradeHistory(since=${f_start_date}, params={'until': ${f_end_date}}))" - [[ $f_ccxt_result = \[\] ]] && f_ccxt_result="" + f_ccxt "print(${f_exchange}.fetchConvertTradeHistory(since=${f_start_date}, params={'until': ${f_end_date}}))" || f_ccxt_result="" echo -n $f_ccxt_result >"$f_convert_file" if [ -s "$f_convert_file" ] diff --git a/dabo/functions/position_close.sh b/dabo/functions/position_close.sh index a2cf4b8..5841ce5 100644 --- a/dabo/functions/position_close.sh +++ b/dabo/functions/position_close.sh @@ -5,7 +5,7 @@ function position_close { local f_symbol=$1 local f_position - get_symbols + get_symbols_ticker get_positions get_position_array diff --git a/dabo/functions/transactions_overview.sh b/dabo/functions/transactions_overview.sh index b7bc3bb..859011b 100644 --- a/dabo/functions/transactions_overview.sh +++ b/dabo/functions/transactions_overview.sh @@ -2,12 +2,12 @@ function transactions_overview { g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + get_transactions + get_bitpanda_api_transactions get_justtrade_csv_transactions get_onetrading_csv_transactions - get_transactions - >ALL_TRANSACTIONS_OVERVIEW.csv.tmp >ALL_TRANSACTIONS_OVERVIEW_WARN.csv.tmp >TRANSACTIONS_OVERVIEW-trade-result_tax_german_eur.tmp diff --git a/dabo/functions/webpage.sh b/dabo/functions/webpage.sh index 93086ed..0771219 100644 --- a/dabo/functions/webpage.sh +++ b/dabo/functions/webpage.sh @@ -26,6 +26,8 @@ function webpage { local f_USED_BALANCE=$(tail -n1 "asset-histories/BALANCEUSED${CURRENCY}.history.csv" | cut -d, -f2) local f_COMPLETE_BALANCE=$(tail -n1 "asset-histories/BALANCECOMPLETE${CURRENCY}.history.csv" | cut -d, -f2) + g_calc "$f_COMPLETE_BALANCE-$f_USED_BALANCE" + printf -v CURRENCY_BALANCE %.2f $g_calc_result echo '

Overview

' >>../index.html.tmp echo " @@ -48,7 +50,7 @@ function webpage { for f_balance_date in Day Week Month 3Month Year do f_balance_at_date=$(grep "^$(date -d "1 $f_balance_date ago" +"%Y-%m-%d ")" asset-histories/BALANCECOMPLETE${CURRENCY}.history.csv | head -n1 | cut -d, -f2) - if g_num_valid_number "$f_balance_at_date" + if g_num_valid_number "$f_balance_at_date" 2>/dev/null then printf -v f_balance_at_date %.2f $f_balance_at_date g_calc "$f_COMPLETE_BALANCE-$f_balance_at_date" @@ -62,11 +64,16 @@ function webpage { echo '

Open Positions

' >>../index.html.tmp - echo "
" >>../index.html.tmp + echo "
SymbolAountEntry PriceCurrent PriceProfit/LossNotes
" >>../index.html.tmp + get_position_array for f_position in "${f_get_positions_array[@]}" do get_position_line_vars "$f_position" - echo "" >>../index.html.tmp + printf -v f_position_currency_amount %.2f $f_position_currency_amount + printf -v f_position_entry_price %.2f $f_position_entry_price + printf -v f_position_current_price %.2f $f_position_current_price + printf -v f_position_pnl %.2f $f_position_pnl + echo "" >>../index.html.tmp done echo "
SymbolAmountEntry PriceCurrent PriceProfit/LossNotes
$f_position_symbol$f_position_currency_amount$f_position_entry_price$f_position_current_price$f_position_pnl ( ${f_position_pnl_percentage}%)$f_position_side ${f_position_leverage}X
$f_position_symbol${CURRENCY} $f_position_currency_amount${CURRENCY} $f_position_entry_price${CURRENCY} $f_position_current_price${CURRENCY} $f_position_pnl ( ${f_position_pnl_percentage}%)$f_position_side ${f_position_leverage}x
" >>../index.html.tmp @@ -80,7 +87,9 @@ function webpage { local f_currency_amount_complete=0 local f_sold_complete=0 local f_result_percent_complete=0 - local f_asset f_exchange f_amount f_spent f_sold f_currency_amount f_result_percent + local f_asset f_exchange f_amount f_spent f_sold f_currency_amount f_result_percent + if [ -s ALL_TRANSACTIONS_OVERVIEW.csv ] + then for f_asset_per_exchange in $(cat ALL_TRANSACTIONS_OVERVIEW.csv | cut -d, -f2,4 | sort -u) do mapfile -d, -t f_asset_per_exchange_array < <(echo $f_asset_per_exchange) @@ -95,9 +104,11 @@ function webpage { currency_converter $f_amount $f_asset $TRANSFER_CURRENCY || continue f_currency_amount=$f_currency_converter_result g_calc "$f_currency_amount+($f_sold)" - f_currency_amount=$g_calc_result + #f_currency_amount=$g_calc_result + printf -v f_currency_amount %.2f $g_calc_result g_calc "$f_spent-($f_sold)" - f_spent=$g_calc_result + #f_spent=$g_calc_result + printf -v f_spent %.2f $g_calc_result if [ "$f_spent" = 0 ] then @@ -107,20 +118,25 @@ function webpage { f_result_percent=$g_percentage_diff_result fi g_calc "$f_currency_amount-($f_spent)" - f_result=$g_calc_result + printf -v f_result %.2f $g_calc_result + #f_result=$g_calc_result ### Calc Complete values g_calc "$f_result_complete+($f_result)" - f_result_complete=$g_calc_result + #f_result_complete=$g_calc_result + printf -v f_result_complete %.2f $g_calc_result g_calc "$f_currency_amount_complete+($f_currency_amount)" - f_currency_amount_complete=$g_calc_result + #f_currency_amount_complete=$g_calc_result + printf -v f_currency_amount_complete %.2f $g_calc_result g_calc "$f_spent_complete+($f_spent)" - f_spent_complete=$g_calc_result + #f_spent_complete=$g_calc_result + printf -v f_spent_complete %.2f $g_calc_result g_calc "$f_sold_complete+($f_sold)" - f_sold_complete=$g_calc_result + #f_sold_complete=$g_calc_result + printf -v f_sold_complete %.2f $g_calc_result echo "$f_currency_amount $TRANSFER_CURRENCY$f_spent $TRANSFER_CURRENCY$f_sold $TRANSFER_CURRENCY$f_result $TRANSFER_CURRENCY ( ${f_result_percent}%)$f_amount $f_asset$f_exchange" >>../index.html.tmp.tmp fi @@ -133,6 +149,7 @@ function webpage { sort -n -k3 -t'>' -r ../index.html.tmp.tmp >>../index.html.tmp rm ../index.html.tmp.tmp echo "" >>../index.html.tmp + fi echo "

Market Performance ( $(cat MARKET_PERFORMANCE_LATEST)%)

" >>../index.html.tmp #echo "
Charts" >>../index.html.tmp @@ -175,170 +192,37 @@ function webpage { echo "
" >>../index.html.tmp echo "Complete list" >>../index.html.tmp - echo '

Latest trades

' >>../index.html.tmp - echo '

Open

' >>../index.html.tmp - - echo "" >>../index.html.tmp - echo " - - - - - - " >>../index.html.tmp - local f_trade_file - for f_trade_file in $(ls -t trade-histories/trade-*-open.history.csv 2>/dev/null) - do - local tradeline=$(tail -n1 ${f_trade_file}) - local asset=$(echo ${f_trade_file} | cut -d. -f3 | cut -d- -f1) - local interimfile=$(echo ${f_trade_file} | sed 's/open\.history\.csv/interim\.history\.csv/') - echo " - - - - - - - " >>../index.html.tmp - echo "" >>../index.html.tmp - done - echo "
DateAssetAction${CURRENCY} Quantity${CURRENCY} PriceCommissionComment
$(echo ${tradeline} | cut -d, -f1)${asset} 🔗$(echo ${tradeline} | cut -d, -f2)$(echo ${tradeline} | cut -d, -f4)$(echo ${tradeline} | cut -d, -f5)$(echo ${tradeline} | cut -d, -f6)$(echo ${tradeline} | cut -d, -f7,8,9,10,11,12,13,14,15)
Chart" >>../index.html.tmp - genchart "$interimfile" >>../index.html.tmp - echo "
" >>../index.html.tmp - - echo '

Closed

' >>../index.html.tmp - - echo "" >>../index.html.tmp - echo " - - - - - - " >>../index.html.tmp - - for f_trade_file in $(ls -t trade-histories/trade-*-closed.history.csv 2>/dev/null | head -n 50) - do - local tradeline - cat ${f_trade_file} | while read tradeline - do - local f_action=$(echo ${tradeline} | cut -d, -f2) - local f_price=$(echo ${tradeline} | cut -d, -f5) - local tradedate=$(echo ${tradeline} | cut -d, -f1 | perl -pe 's/_([0-9][0-9])-([0-9][0-9])-([0-9][0-9])/ $1:$2/') - - if echo ${f_action} | grep -q buy - then - echo ${f_price} >${g_tmp}/buyprice - local tradedatebuy=${tradedate} - fi - - if echo ${f_action} | grep -q sell - then - if [ -s "${f_trade_file}.result" ] - then - local f_profit=$(cat "${f_trade_file}.result") - f_price="${f_price} ( ${f_profit}%)" - else - g_percentage-diff $(cat ${g_tmp}/buyprice) ${f_price} - local f_profit=${g_percentage_diff_result} - f_price="${f_price} ( ${f_profit}%)" - echo "${f_profit}" >"${f_trade_file}.result" - fi - fi - local asset=$(echo ${f_trade_file} | cut -d. -f3 | cut -d- -f1) - interimfile=$(echo ${f_trade_file} | sed 's/closed\.history\.csv/interim\.history\.csv/') - echo " - - - - - - ">>../index.html.tmp - if echo ${f_action} | grep -q sell - then - echo "" >>../index.html.tmp - else - echo "" >>../index.html.tmp - fi - done - done - echo "
DateAssetAction${CURRENCY} Quantity${CURRENCY} Price (result)CommissionComment
${tradedate}${asset} 🔗${f_action}$(echo ${tradeline} | cut -d, -f4)${f_price}$(echo ${tradeline} | cut -d, -f6)$(echo ${tradeline} | cut -d, -f7,8,9,10,11,12,13,14,15)
Charts" >>../index.html.tmp - local tradeintervals=$(cat ${interimfile} | wc -l) - head -n1 asset-histories/${asset}.history.csv >${g_tmp}/trade.csv - grep -A ${tradeintervals} "${tradedatebuy}:" asset-histories/${asset}.history.csv >>${g_tmp}/trade.csv - # cache old trade charts - if ! [ -s "${interimfile}.chart" ] - then - echo "Price, EMA, Levels" >${interimfile}.chart - genchart "${g_tmp}/trade.csv" ${tradeintervals} 2,25,26,27,28,29,30,31,32,33,34,35,4,36,37,38,39 green,DarkSlateGrey,DarkSlateGrey,Gold,DarkOrange,DarkOrange,GoldenRod,GoldenRod,GoldenRod,GoldenRod,DarkOrange,DarkOrange,MidnightBlue,Indigo,DarkSlateBlue,DodgerBlue,DeepSkyBlue >>${interimfile}.chart - echo "MACD" >>${interimfile}.chart - genchart "${g_tmp}/trade.csv" ${tradeintervals} 8,6,7 >>${interimfile}.chart - echo "RSIs" >>${interimfile}.chart - genchart "${g_tmp}/trade.csv" ${tradeintervals} 10,11,12,14,15,16,17,13 >>${interimfile}.chart - fi - cat "${interimfile}.chart" >>../index.html.tmp - echo "
" >>../index.html.tmp - - - - #echo "

Current config

" >>../index.html.tmp - #echo "
$(cat ../../dabo-bot.conf | perl -pe 's/\/>/g;')
" >>../index.html.tmp - - - - echo '

Available Assets and histories

' >>../index.html.tmp - echo "" >>../index.html.tmp - echo "" >>../index.html.tmp - local asset - cat ASSETS | egrep -v "${BLACKLIST}" | sort | while read asset - do - [ -s asset-histories/${asset}.history.csv ] || continue - echo "" >>../index.html.tmp - kcurrency=$(echo ${asset} | sed "s/${CURRENCY}//") - #get_rate_percentage_min_before_and_now ${kcurrency} ${CURRENCY} 1440 - local assetin=($(cat asset-histories/${asset}.history.csv | egrep -v "0.00000000$" | tail -n2 | head -n1 | sed 's/,/ /g')) - echo " - - - " >>../index.html.tmp - - done - echo "
AssetDatePrice ${CURRENCY}24h change (USD)
${asset} 🔗${assetin[0]} ${assetin[1]}${CURRENCY} ${assetin[2]} $(grep "^$kcurrency," ASSET_PRICE_CHANGE_PERCENTAGE_24H | cut -d, -f2)%
" >>../index.html.tmp - echo "
${asset} Charts" >>../index.html.tmp - echo "Price, EMA, Levels" >>../index.html.tmp - genchart asset-histories/${asset}.history.csv 50 2,25,26,27,28,29,30,31,32,33,34,35,5,36,37,38 green,DarkSlateGrey,DarkSlateGrey,Gold,DarkOrange,DarkOrange,GoldenRod,GoldenRod,GoldenRod,GoldenRod,DarkOrange,DarkOrange,MidnightBlue,Indigo,DarkSlateBlue,DodgerBlue,DeepSkyBlue >>../index.html.tmp - echo "MACD" >>../index.html.tmp - genchart asset-histories/${asset}.history.csv 50 8,6,7 >>../index.html.tmp - echo "RSIs" >>../index.html.tmp - genchart asset-histories/${asset}.history.csv 50 10,11,12,14,15,16,17,13 >>../index.html.tmp - echo "
" >>../index.html.tmp +# echo '

Available Assets and histories

' >>../index.html.tmp +# echo "" >>../index.html.tmp +# echo "" >>../index.html.tmp +# local asset +# cat ASSETS | egrep -v "${BLACKLIST}" | sort | while read asset +# do +# [ -s asset-histories/${asset}.history.csv ] || continue +# echo "" >>../index.html.tmp +# kcurrency=$(echo ${asset} | sed "s/${CURRENCY}//") +# #get_rate_percentage_min_before_and_now ${kcurrency} ${CURRENCY} 1440 +# local assetin=($(cat asset-histories/${asset}.history.csv | egrep -v "0.00000000$" | tail -n2 | head -n1 | sed 's/,/ /g')) +# echo " +# +# +# " >>../index.html.tmp +# +# done +# echo "
AssetDatePrice ${CURRENCY}24h change (USD)
${asset} 🔗${assetin[0]} ${assetin[1]}${CURRENCY} ${assetin[2]} $(grep "^$kcurrency," ASSET_PRICE_CHANGE_PERCENTAGE_24H | cut -d, -f2)%
" >>../index.html.tmp +# echo "
${asset} Charts" >>../index.html.tmp +# echo "Price, EMA, Levels" >>../index.html.tmp +# genchart asset-histories/${asset}.history.csv 50 2,25,26,27,28,29,30,31,32,33,34,35,5,36,37,38 green,DarkSlateGrey,DarkSlateGrey,Gold,DarkOrange,DarkOrange,GoldenRod,GoldenRod,GoldenRod,GoldenRod,DarkOrange,DarkOrange,MidnightBlue,Indigo,DarkSlateBlue,DodgerBlue,DeepSkyBlue >>../index.html.tmp +# echo "MACD" >>../index.html.tmp +# genchart asset-histories/${asset}.history.csv 50 8,6,7 >>../index.html.tmp +# echo "RSIs" >>../index.html.tmp +# genchart asset-histories/${asset}.history.csv 50 10,11,12,14,15,16,17,13 >>../index.html.tmp +# echo "
" >>../index.html.tmp - echo "

Top/Flop

" >>../index.html.tmp - ls -1 trade-histories/*.result | cut -d . -f3-6 | sort -u | while read f_asset_results - do - f_asset=$(echo ${f_asset_results} | cut -d\- -f1) - echo $(cat trade-histories/*${f_asset_results} | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM)}") >trade-histories/${f_asset}.complete_result - done - echo "

Top 10

" >>../index.html.tmp - grep [0-9] trade-histories/*.complete_result | sort -t: -k2 -rn | head -n 10 | cut -d/ -f2 | perl -pe 's/\.complete_result:/ /; s/$/%
/' >>../index.html.tmp - - echo "

Flop 10

" >>../index.html.tmp - grep [0-9] trade-histories/*.complete_result | sort -t: -k2 -n | head -n 10 | cut -d/ -f2 | perl -pe 's/\.complete_result:/ /; s/$/%
/' >>../index.html.tmp - - - echo '

Complete trading histories

' >>../index.html.tmp - echo "" >>../index.html.tmp - find trade-histories -type f -name *.history.csv | cut -d/ -f2 | cut -d. -f1 | sort | while read asset - do - [ -s trade-histories/${asset}.history.csv ] || continue - echo "" >>../index.html.tmp - done - echo "
${asset} 🔗
" >>../index.html.tmp # color magic cat ../index.html.tmp | perl -pe 's/ (\-[0-9]+\.[0-9]+\%)/$1<\/font>/g; s/ ([0-9]+\.[0-9]+\%)/$1<\/font>/g;' >../index.html - #mv ../index.html.tmp ../index.html g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@ finished" diff --git a/docker-compose.yml b/docker-compose.yml index d984652..d67b03d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: cpus: '2' memory: 1024M - dabo-assets: + dabo-symbols_ticker: build: context: . dockerfile: Dockerfile @@ -38,13 +38,60 @@ services: - /usr/local/bin/notify.sh:/usr/local/bin/notify.sh:ro - /usr/local/etc/notify.conf:/usr/local/etc/notify.conf:ro - /etc/localtime:/etc/localtime:ro - entrypoint: /dabo/fetch-assets.sh + entrypoint: /dabo/fetch-symbols_ticker.sh deploy: resources: limits: cpus: '1' memory: 512M + dabo-ohlcv-candles-indicators: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + user: 10000:10000 + volumes: + - ./dabo:/dabo:ro + - ./strategies:/dabo/strategies:ro + - ./dabo-bot.conf:/dabo/dabo-bot.override.conf + - ./watch-assets.csv:/dabo/watch-assets.csv + - ./data:/dabo/htdocs:rw + - ./home:/dabo/home:rw + - /usr/local/bin/notify.sh:/usr/local/bin/notify.sh:ro + - /usr/local/etc/notify.conf:/usr/local/etc/notify.conf:ro + - /etc/localtime:/etc/localtime:ro + entrypoint: /dabo/fetch-ohlcv-candles-indicators.sh + deploy: + resources: + limits: + cpus: '1' + memory: 512M + + dabo-calc-indicators-hist: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + user: 10000:10000 + volumes: + - ./dabo:/dabo:ro + - ./strategies:/dabo/strategies:ro + - ./dabo-bot.conf:/dabo/dabo-bot.override.conf + - ./watch-assets.csv:/dabo/watch-assets.csv + - ./data:/dabo/htdocs:rw + - ./home:/dabo/home:rw + - /usr/local/bin/notify.sh:/usr/local/bin/notify.sh:ro + - /usr/local/etc/notify.conf:/usr/local/etc/notify.conf:ro + - /etc/localtime:/etc/localtime:ro + entrypoint: /dabo/calc-indicators-hist.sh + cpu_shares: 128 + deploy: + resources: + limits: + cpus: '1' + memory: 1024M + dabo-orders: build: context: . @@ -66,7 +113,7 @@ services: resources: limits: cpus: '0.5' - memory: 128M + memory: 512M dabo-transaction-history: build: @@ -85,11 +132,12 @@ services: - /usr/local/etc/notify.conf:/usr/local/etc/notify.conf:ro - /etc/localtime:/etc/localtime:ro entrypoint: /dabo/fetch-transaction-history.sh + cpu_shares: 128 deploy: resources: limits: - cpus: '1' - memory: 128M + cpus: '0.5' + memory: 512M dabo-webpage: build: