diff --git a/README.md b/README.md index d6a3727..304c049 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ dabo is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY You should have received a copy of the GNU General Public License along with dabo (see COPYING file). If not, see . ## Data sources -Various data sources such as finance.yahoo.com and crypto exchanges available via ccxt are used. please check whether this is legal in your region before use. +Various data sources such as finance.yahoo.com and crypto exchanges available via ccxt are used. Please check whether this is legal in your region before use. - query1.finance.yahoo.com (economic data,...) +- api.coinmarketcap.com (crypto data) - 30rates.com (forecast) - fapi.binance.com (OpenInterest,...) - api.alternative.me (Fear and Greed) @@ -88,7 +89,8 @@ Each part runs parallel to the others in its own docker-container. OHLCV = Open, High, Low, Close and Volume of a time unit - time units 1w, 1d, 4h, 1h, 15m and 5m - 4h, 1h, 15m and 5m from the respective stock exchange -- 1d and 1w data from yahoo finace to have longer terms +- 1d and 1w data from coinmarketcap to have longer terms +- economic data from yahoo finance ### Dabo Indicators - data per time unit - time units 1w, 1d, 4h, 1h, 15m and 5m @@ -98,6 +100,7 @@ OHLCV = Open, High, Low, Close and Volume of a time unit - self-calculated significant levels (support/resist) ### Dabo Market Data - Yahoo Finance +- CoinMarketCyp ### Dabo Orders ### Dabo Transaction History - Support of additional Exchnages/Brokers JustTrade (CSV-Import) and Bitpanda (API+CSV-Import) diff --git a/dabo/fetch-coinmarketcapids.sh b/dabo/fetch-coinmarketcapids.sh new file mode 100755 index 0000000..2c79877 --- /dev/null +++ b/dabo/fetch-coinmarketcapids.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (c) 2022-2024 olli +# +# This file is part of dabo (crypto bot). +# +# dabo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# dabo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with dabo. If not, see . + + + +. /dabo/dabo-prep.sh + +while true +do + g_echo_note "Next loop" + [ -s COINMARKETCAPIDS ] || get_marketdata_coinmarketcap_ids + sleeptime=$(($(TZ=UTC date +%s -d "next monday 0:00") - $(date +%s) +2 )) + g_echo_note "Waiting $sleeptime seconds until next run" + sleep $sleeptime + get_marketdata_coinmarketcap_ids + +done + diff --git a/dabo/fetch-ohlcv-candles-indicators.sh b/dabo/fetch-ohlcv-candles-indicators.sh index 4c8be10..9136356 100755 --- a/dabo/fetch-ohlcv-candles-indicators.sh +++ b/dabo/fetch-ohlcv-candles-indicators.sh @@ -22,7 +22,7 @@ . /dabo/dabo-prep.sh interval=$1 -[ -z "$interval" ] && return 1 +[ -z "$interval" ] && exit 1 seconds=$2 while true diff --git a/dabo/functions/currency_converter.sh b/dabo/functions/currency_converter.sh index b67e358..f3b4c83 100644 --- a/dabo/functions/currency_converter.sh +++ b/dabo/functions/currency_converter.sh @@ -91,15 +91,15 @@ function currency_converter { fi # try direct pair - get_marketdata_yahoo "${f_currency_target}-${f_currency}" "${f_currency_target}${f_currency}" || get_marketdata_yahoo "${f_currency}-${f_currency_target}" "${f_currency}${f_currency_target}" + [ "${f_currency}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency_target}-${f_currency}" "${f_currency_target}${f_currency}" + [ "${f_currency_target}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency}-${f_currency_target}" "${f_currency}${f_currency_target}" local f_histfile_default="${f_asset_histories}${f_currency_target}${f_currency}.history" - local f_histfile_yahoo="${f_asset_histories}${f_currency_target}${f_currency}.history" + local f_histfile_coinmarketcap="${f_asset_histories}${f_currency_target}${f_currency}.history" # reverse as backup local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history" - local f_histfile_yahoo_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history" - + local f_histfile_coinmarketcap_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history" # search for rate by date - for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_yahoo" "$f_histfile_yahoo_reverse" + for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse" do # histfile has to exist #if [ -s "${f_histfile}*.csv" ] @@ -136,7 +136,7 @@ function currency_converter { return $? fi fi - g_echo_note "didn't find rate for ${f_currency}-${f_currency_target} - '${FUNCNAME} $@'" + g_echo_error "didn't find rate for ${f_currency}-${f_currency_target} - '${FUNCNAME} $@'" return 1 fi diff --git a/dabo/functions/get_indicators.sh b/dabo/functions/get_indicators.sh index 8fc6aa7..3603e50 100644 --- a/dabo/functions/get_indicators.sh +++ b/dabo/functions/get_indicators.sh @@ -38,7 +38,7 @@ function get_indicators_all { printf '%(%Y-%m-%d %H:%M:%S)T' >"${f_histfile}.indicators-calculating" get_indicators "${f_histfile}" ${f_last_intervals} && printf '%(%Y-%m-%d %H:%M:%S)T' >>"$f_histfile.indicators-calculated" # add missing intervals for example from weekends from non-24h-assets like economic data - interval from filename - f_add_missing_ohlcv_intervals "${f_histfile}" + #f_add_missing_ohlcv_intervals "${f_histfile}" rm -f "${f_histfile}.indicators-calculating" done diff --git a/dabo/functions/get_marketdata_coinmarketcap.sh b/dabo/functions/get_marketdata_coinmarketcap.sh new file mode 100644 index 0000000..72b35b1 --- /dev/null +++ b/dabo/functions/get_marketdata_coinmarketcap.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +# Copyright (c) 2022-2024 olli +# +# This file is part of dabo (crypto bot). +# +# dabo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# dabo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with dabo. If not, see . + + +function get_marketdata_coinmarketcap { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + local f_item="$1" + local f_name="$2" + local f_timeframe="$3" + + local f_targetcsvtmp="${g_tmp}/${f_name}.history-coinmarketcap.csv" + local f_targetjsontmp="${g_tmp}/${f_name}.history-coinmarketcap.json" + rm -f "$f_targetcsvtmp" "$f_targetjsontmp" + + [ -z "$f_timeframe" ] && f_timeframe="1d" + local f_targetcsv="asset-histories/${f_name}.history-coinmarketcap.${f_timeframe}.csv" + [ "$f_timeframe" = "1w" ] && f_timeframe="7d" + f_histfile_coinmarketcap="$f_targetcsv" + + # use EUR EURC stable coin fo EUR + f_item=${f_item//EUR-/EURC-} + + # remove - + f_item=${f_item//-//} + + # USDT to USD + f_item=${f_item//USDT/USD} + # BUSD to USD + f_item=${f_item//BUSD/USD} + + + if ! [[ $f_item =~ /USD ]] + then + g_echo_error "${FUNCNAME} $@: Only USD supported" + return 1 + fi + + # transform CCXT symbols to CoinmarketCap IDs + if [[ $f_item =~ / ]] + then + # remove /* + f_item=${f_item///*/} + ## remove :* (for example :USDT in contract markets) + #f_item=${f_item//:*} + # remove spaces + #f_item=${f_item/ /} + fi + + local f_id + f_id=$(egrep "^${f_item}," COINMARKETCAPIDS | head -n1 | cut -d, -f2) + if [ -z "$f_id" ] + then + g_echo_error "${FUNCNAME} $@: No CoinMarketCap ID for $f_item" + return 1 + fi + + # end if already failed the last 5 minutes + if [ -f "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" ] + then + find "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" -mmin +5 -delete + if [ -f "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" ] + then + return 1 + fi + fi + + # end if already exists and modified under given time + if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}" + then + return 0 + fi + + # cleanup + rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err ${f_targetjsontmp} "${f_targetjsontmp}".err + + if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "7d" ] + then + # Download data from coinmarketcap + g_wget -O "${f_targetjsontmp}" "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=${f_timeframe}" 2>"${f_targetjsontmp}".err + jq -r '.data.quotes[] | .quote.timestamp[0:10] + "," + (.quote.open|tostring) + "," + (.quote.high|tostring) + "," + (.quote.low|tostring) + "," + (.quote.close|tostring) + "," + (.quote.volume|tostring)' "${f_targetjsontmp}" | egrep -v ',0$|,$' >"${f_targetcsvtmp}" 2>"${f_targetjsontmp}".err + else + g_echo_error "${FUNCNAME} $@: Timeframe $f_timeframe in CoinMarketCap not supported." + return 1 + fi + + # error if no csvfile available + if ! [ -s "${f_targetcsvtmp}" ] + then + mkdir -p FAILED_COINMARKETCAP + cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>&1 + f_get_marketdata_coinmarketcap_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null) + return 1 + fi + + # put the csvs together + if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ] + then + egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9].*,[0-9]" "${f_targetcsv}" "${f_targetcsvtmp}" | sort -k1,2 -t, -u | sort -k1,1 -t, -u >"${f_targetcsv}.tmp" + mv "${f_targetcsv}.tmp" "${f_targetcsv}" + else + egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9].*,[0-9]" "${f_targetcsvtmp}" | sort -k1,2 -t, -u >"$f_targetcsv" + fi + +} + +function get_marketdata_coinmarketcap_ids { + + # get symbol ids from coinmarketcap + + local f_target=COINMARKETCAPIDS + local f_target_tmp="${f_target}.tmp" + local f_target_loop=$f_target_tmp + + # write direct to target if not exists or empty + [ -s "$f_target" ] || f_target_loop=$f_target + + for f_id in $(seq 1 50000) + do + curl -s --request GET --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" | jq -r '.[] | .symbol + "," + (.id|tostring)' | head -n1 + sleep 0.3 + done | egrep '^.+,[0-9]*$' >"$f_target_loop" + + if [ -s "$f_target_tmp" ] + then + cp -p "$f_target" "${f_target}.old" + sort -u "$f_target_tmp" "${f_target}.old" >"$f_target" + rm "$f_target_tmp" + fi +} + diff --git a/dabo/functions/get_marketdata_yahoo.sh b/dabo/functions/get_marketdata_yahoo.sh index 31b957f..e5af7da 100644 --- a/dabo/functions/get_marketdata_yahoo.sh +++ b/dabo/functions/get_marketdata_yahoo.sh @@ -65,27 +65,6 @@ function get_marketdata_yahoo { [[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X" [[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X" - # special names of some coins/currencies of yahoo finance - [[ $f_item = "ARB-USD" ]] && f_item="ARB11841-USD" - [[ $f_item = "DUEL-USD" ]] && f_item="DUEL28868-USD" - [[ $f_item = "GMX-USD" ]] && f_item="GMX11857-USD" - [[ $f_item = "MEW-USD" ]] && f_item="MEW30126-USD" - [[ $f_item = "TAO-USD" ]] && f_item="TAO22974-USD" - [[ $f_item = "UNI-USD" ]] && f_item="UNI7083-USD" - [[ $f_item = "SUI-USD" ]] && f_item="SUI20947-USD" - [[ $f_item = "BLAZE-USD" ]] && f_item="BLAZE30179-USD" - [[ $f_item = "BEER-USD" ]] && f_item="BEER31337-USD" - [[ $f_item = "TAI-USD" ]] && f_item="TAI20605-USD" - [[ $f_item = "DEFI-USD" ]] && f_item="DEFI29200-USD" - [[ $f_item = "TON-USD" ]] && f_item="TON11419-USD" - [[ $f_item = "BRETT-USD" ]] && f_item="BRETT29743-USD" - #[[ $f_item = "ADS-USD" ]] && f_item="%24ADS-USD" - [[ $f_item = "AIT-USD" ]] && f_item="AIT28882-USD" - [[ $f_item = "PT-USD" ]] && f_item="PT28582-USD" - [[ $f_item = "BLAST-USD" ]] && f_item="BLAST28480-USD" - [[ $f_item = "GRT-USD" ]] && f_item="GRT6719-USD" - [[ $f_item = "ARTY-USD" ]] && f_item="ARTY23751-USD" - # end if already failed the last 5 minutes if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ] then @@ -109,12 +88,15 @@ function get_marketdata_yahoo { if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "1wk" ] || [ "$f_timeframe" = "1mo" ] then - # restrict to finished candles - [ "$f_timeframe" = "1d" ] && f_sec=$(TZ=US/NY date -d 'last day 0:00' '+%s') - [ "$f_timeframe" = "1wk" ] && f_sec=$(TZ=US/NY date -d 'last monday 0:00' '+%s') - [ "$f_timeframe" = "1mo" ] && f_sec=$(TZ=US/NY date -d "$(date -d "@$(date -d "last month" +%s)" +'%Y-%m-01')" +%s) - # Download historical data from yahoo - g_wget -O "${f_targetcsvtmp}" "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history" 2>"${f_targetcsvtmp}".err + ### DEPRECATED - Yahoo Finance deactivated public API 2024-09 !!! ### + g_echo_note "DEPRECATED - Yahoo Finance deactivated public API 2024-09 !!!" + return 1 + ## restrict to finished candles + #[ "$f_timeframe" = "1d" ] && f_sec=$(TZ=US/NY date -d 'last day 0:00' '+%s') + #[ "$f_timeframe" = "1wk" ] && f_sec=$(TZ=US/NY date -d 'last monday 0:00' '+%s') + #[ "$f_timeframe" = "1mo" ] && f_sec=$(TZ=US/NY date -d "$(date -d "@$(date -d "last month" +%s)" +'%Y-%m-01')" +%s) + ## Download historical data from yahoo + #g_wget -O "${f_targetcsvtmp}" "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history" 2>"${f_targetcsvtmp}".err else # Download data from yahoo g_wget -O "${f_targetjsontmp}" "https://query1.finance.yahoo.com/v7/finance/chart/${f_item}?interval=${f_timeframe}&period2=${f_sec}" 2>"${f_targetjsontmp}".err @@ -134,7 +116,7 @@ function get_marketdata_yahoo { lastlow=$low lastclose=$close echo "$date_time,$open,$high,$low,$close,$volume" - done < "${f_targetcsvtmp}.unixtime" | grep ":00," >${f_targetcsvtmp} + done < "${f_targetcsvtmp}.unixtime" | grep ":00," >${f_targetcsvtmp} fi # error if no csvfile available diff --git a/dabo/functions/get_ohlcv-candle.sh b/dabo/functions/get_ohlcv-candle.sh index 7196fc9..badf461 100644 --- a/dabo/functions/get_ohlcv-candle.sh +++ b/dabo/functions/get_ohlcv-candle.sh @@ -35,11 +35,17 @@ function get_ohlcv-candles { g_echo_note "Fetching/Refreshing $f_eco_asset $f_timeframe" f_histfile="asset-histories/ECONOMY-${f_eco_asset}.history.${f_timeframe}.csv" - # 4h timefrage does not exist on yahoo finance so calc from 1h + # 4h timefrage does not exist on coinmarketcap finance so calc from 1h if [ "$f_timeframe" = "4h" ] then f_1h_histfile="asset-histories/ECONOMY-${f_eco_asset}.history.1h.csv" [ -s "$f_1h_histfile" ] && convert_ohlcv_1h_to_4h "$f_1h_histfile" "$f_histfile" + #f_add_missing_ohlcv_intervals "$f_histfile" 4h + elif [ "$f_timeframe" = "1d" ] + then + f_1h_histfile="asset-histories/ECONOMY-${f_eco_asset}.history.1h.csv" + [ -s "$f_1h_histfile" ] && convert_ohlcv_1h_to_1d "$f_1h_histfile" "$f_histfile" + #f_add_missing_ohlcv_intervals "$f_histfile" 1d else get_ohlcv-candle "${f_eco_asset}" ${f_timeframe} "${f_histfile}" "ECONOMY-${f_eco_asset}" fi @@ -86,28 +92,45 @@ function get_ohlcv-candle { g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - local f_yahoo f_date f_unit_date f_data f_data_array f_data_unit f_open f_high f_low f_close f_volume f_last_unit_date f_last_unit_close + local f_extdata f_date f_unit_date f_data f_data_array f_data_unit f_open f_high f_low f_close f_volume f_last_unit_date f_last_unit_close local f_symbol="$1" local f_timeframe=$2 local f_histfile="$3" local f_asset=$4 - unset f_histfile_yahoo - #[ -n "$f_asset" ] && f_yahoo=1a + unset f_histfile_yahoo f_histfile_coinmarketcap + #[ -n "$f_asset" ] && f_extdata=1 #local f_histfile_week="$4" - # fetch >=1d from yahoo finance + # fetch >=1d from coinmarketcap if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "1w" ] || [ "$f_timeframe" = "1mo" ] || [ -n "$f_asset" ] then - f_yahoo=1 + f_extdata=1 if [ -z "$f_asset" ] then f_asset=${f_symbol///} f_asset=${f_asset//:*} fi - if ! get_marketdata_yahoo "$f_symbol" "$f_asset" $f_timeframe + + if [[ $f_asset =~ ^ECONOMY- ]] then - g_echo_error "$f_get_marketdata_yahoo_error" - return 1 + # economy from yahoo finance + if [ "$f_timeframe" = "1h" ] || [ "$f_timeframe" = "15m" ] || [ "$f_timeframe" = "15m" ] + then + if ! get_marketdata_yahoo "$f_symbol" "$f_asset" $f_timeframe + then + g_echo_error "$f_get_marketdata_coinmarketcap_error" + return 1 + fi + fi + f_histfile_extdata=$f_histfile_yahoo + else + # crypto from coinmarketcap + if ! get_marketdata_coinmarketcap "$f_symbol" "$f_asset" $f_timeframe + then + g_echo_error "$f_get_marketdata_coinmarketcap_error" + return 1 + fi + f_histfile_extdata=$f_histfile_coinmarketcap fi fi @@ -115,7 +138,7 @@ function get_ohlcv-candle { while true do # fetch data - if [ -z "$f_yahoo" ] + if [ -z "$f_extdata" ] then # find latest time which is not fetched already create f_since get_ohlcv-candle-latest "$f_symbol" "$f_histfile" @@ -131,16 +154,16 @@ function get_ohlcv-candle { f_data=${f_data//],/+} g_array $f_data f_data_ref + else - # from yahoo finance - g_array "$f_histfile_yahoo" f_data_ref + # from coinmarketcap + g_array "$f_histfile_extdata" f_data_ref fi f_data_array=("${f_data_ref[@]}") # check if last data already in history file and end if already present g_array "${f_data_array[-1]}" f_last_data_unit_ref , - [ -z "$f_yahoo" ] && printf -v f_last_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_last_data_unit_ref[0]::-3} - [ -n "$f_yahoo" ] && f_last_unit_date="${f_last_data_unit_ref[0]}" + [ -z "$f_extdata" ] && printf -v f_last_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_last_data_unit_ref[0]::-3} + [ -n "$f_extdata" ] && f_last_unit_date="${f_last_data_unit_ref[0]}" #echo "grep -q ^\"$f_last_unit_date\" \"$f_histfile\"" [ -s "$f_histfile" ] && grep -q ^"${f_last_unit_date}," "$f_histfile" && break @@ -151,8 +174,8 @@ function get_ohlcv-candle { # use array for each unit and assigned values to vars g_array "$f_data_unit" f_data_unit_ref , - [ -z "$f_yahoo" ] && printf -v f_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_data_unit_ref[0]::-3} - [ -n "$f_yahoo" ] && f_unit_date="${f_last_data_unit_ref[0]}" + [ -z "$f_extdata" ] && printf -v f_unit_date '%(%Y-%m-%d %H:%M:%S)T' ${f_data_unit_ref[0]::-3} + [ -n "$f_extdata" ] && f_unit_date="${f_last_data_unit_ref[0]}" # check if date is already in history file [ -s "$f_histfile" ] && grep -q ^"$f_unit_date" "$f_histfile" && continue @@ -172,7 +195,7 @@ function get_ohlcv-candle { g_num_exponential2normal "$f_close" && f_close=$g_num_exponential2normal_result f_last_unit_close=$f_close f_volume=${f_data_unit_ref[5]} - # yahoo historic volume col 6 + # coinmarketcap historic volume col 6 [ -n "${f_data_unit_ref[6]}" ] && f_volume=${f_data_unit_ref[6]} g_num_exponential2normal "$f_volume" && f_volume=$g_num_exponential2normal_result @@ -197,8 +220,8 @@ function get_ohlcv-candle { done - # end if yahoo (complete file and not time chunks) - [ -n "$f_yahoo" ] && break + # end if coinmarketcap (complete file and not time chunks) + [ -n "$f_extdata" ] && break # end if lates refresh is this day printf -v f_date '%(%Y-%m-%d)T\n' @@ -245,6 +268,9 @@ function get_ohlcv-candle-latest { function convert_ohlcv_1h_to_4h { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + local f_input_file="$1" local f_output_file="$2" @@ -328,6 +354,104 @@ function convert_ohlcv_1h_to_4h { } +function convert_ohlcv_1h_to_1d { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_input_file="$1" + local f_output_file="$2" + + local f_latestdate f_nextdate f_mytimezone f_line f_date f_open f_high f_low f_close f_volume f_inday + + if ! [ -s "$f_input_file" ] + then + g_echo_error "$f_input_file" + return 0 + fi + + # crypto timezone UTC + local f_target_timezone=UTC + # US economy timezone America/New_York + [[ $f_input_file =~ ECONOMY ]] && f_target_timezone="America/New_York" + + [ -s "$f_output_file" ] && f_latestdate=$(tail -n1 "$f_output_file" | cut -d, -f1) + [ -z "$f_latestdate" ] && f_latestdate=$(date -d "$(head -n1 "$f_input_file" | cut -d, -f1)" +%Y-%m-%d) + f_latestdate=$(TZ="$f_target_timezone" date -d "$f_latestdate $f_mytimezone" "+%Y-%m-%d") + f_nextdate=$(date -d "$f_latestdate +1day" "+%Y-%m-%d") + + echo $f_latestdate + echo $f_nextdate + + # mytimezone, respecting summer/winter time + f_mytimezone=$(date -d "$_latestdate" +%Z) + + local f_today=$(TZ="$f_target_timezone" date "+%Y-%m-%d") + + grep -A9999 -B24 "^$f_latestdate" "$f_input_file" | grep ':00:00,' | cut -d, -f1,2,3,4,5,6 | while read f_line + do + g_array "$f_line" g_line_array , + # calculate day in target timezone + g_line_array[0]=$(TZ="$f_target_timezone" date -d "${g_line_array[0]} $f_mytimezone" "+%Y-%m-%d") + [[ ${g_line_array[0]} = $f_today ]] && break + echo "${g_line_array[0]},${g_line_array[1]},${g_line_array[2]},${g_line_array[3]},${g_line_array[4]},${g_line_array[5]}" + done >"${f_output_file}.tmp" + + cat "${f_output_file}.tmp" | while read f_line + do + g_array "$f_line" g_line_array , + [[ ${g_line_array[0]} = $f_today ]] && break + + # wait untin next day in target file reached + if [[ ${g_line_array[0]} = $f_nextdate ]] + then + f_end_reached=1 + else + [ -z $f_end_reached ] && continue + fi + + # if dayend + if [ -n "$f_inday" ] && [[ $f_latestdate != ${g_line_array[0]} ]] + then + #echo "day end $f_date" 1>&2 + # day end + f_close=${g_line_array[4]} + echo "$f_date,$f_open,$f_high,$f_low,$f_close,$f_volume" + f_inday="" + fi + + # calc values if inday + if [ -n "$f_inday" ] + then + #echo "in day $f_date" 1>&2 + # in day + # add volume + g_calc "$f_volume+${g_line_array[5]}" + f_volume=$g_calc_result + # look for higher high + g_num_is_higher ${g_line_array[2]} $f_high && f_high=${g_line_array[2]} + # look for lower low + g_num_is_lower ${g_line_array[3]} $f_low && f_low=${g_line_array[3]} + fi + + # if newday + if [ -z "$f_inday" ] + then + #echo "day begin ${g_line_array[0]}" 1>&2 + # day begin + f_inday=1 + f_date=${g_line_array[0]} + f_latestdate=$f_date + f_open=${g_line_array[1]} + f_high=${g_line_array[2]} + f_low=${g_line_array[3]} + f_volume=${g_line_array[5]} + fi + + done >>"$f_output_file" + +} + + function f_add_missing_ohlcv_intervals { g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" @@ -354,12 +478,16 @@ function f_add_missing_ohlcv_intervals { [[ $f_interval = 1w ]] && return 0 [[ $f_histfile =~ \.1w\. ]] && return 0 - local f_prev_date f_prev_vals f_curr_date f_curr_vals f_missing_date f_open f_high f_low f_close f_volume f_percent + + local f_prev_date f_prev_vals f_curr_date f_curr_vals f_missing_date f_open f_high f_low f_close f_volume f_percent f_open f_counter # go through csv per line while IFS=',' read -r f_curr_date f_open f_high f_low f_close f_volume f_percent f_curr_vals do - # Wegnn das vorherige Datum nicht leer ist + + echo "$f_curr_date" 1>&2 + + # if prev date is not empty if [ -z "$f_prev_date" ] then f_prev_date=$f_curr_date @@ -367,24 +495,38 @@ function f_add_missing_ohlcv_intervals { continue fi - while true + echo "$f_curr_date x" 1>&2 + + # only 10 interations to prevelt endless loop + f_counter=0 + while [ $f_counter -lt 10 ] + #while true do + + ((f_counter++)) + echo "$f_curr_date xx $f_counter" 1>&2 + # get second timestamps f_prev_date_in_seconds=$(date -d"$f_prev_date" +%s) f_curr_date_in_seconds=$(date -d"$f_curr_date" +%s) - # calculate/check the next timestamp from previ15ms - # and check for summer/winter time + # echo [ "$f_prev_date_in_seconds" -gt "$f_curr_date_in_seconds" ] # && break + + # calculate/check the next timestamp from previous + # and check for summer/winter time in 4h or greater interval if [ $f_interval -gt 3600 ] then # reduce an hour because of possible summer/winter time change + #g_calc "$f_curr_date_in_seconds - ($f_counter * $f_prev_date_in_seconds - 3600)" g_calc "$f_curr_date_in_seconds - $f_prev_date_in_seconds - 3600" else + #g_calc "$f_curr_date_in_seconds - $f_counter * $f_prev_date_in_seconds" g_calc "$f_curr_date_in_seconds - $f_prev_date_in_seconds" fi if [ $g_calc_result -gt $f_interval ] then # calc missing timestamp in seconds + #f_curr_date_in_seconds=$(( f_prev_date_in_seconds + f_interval * f_counter )) f_curr_date_in_seconds=$(( f_prev_date_in_seconds + f_interval )) # and calculate next timestamp g_calc "$f_curr_date_in_seconds - $f_prev_date_in_seconds" @@ -396,15 +538,28 @@ function f_add_missing_ohlcv_intervals { else f_missing_date=$(date -d"@$f_curr_date_in_seconds" +"%F") fi - + + # prevent endless loop if something goes wrong (strange errors in 1d ohlcv!) + f_missing_date_in_seconds=$(date -d"$f_missing_date" +%s) + if [ $f_missing_date_in_seconds -lt $f_curr_date_in_seconds ] + then + [ -z "$f_curr_vals" ] && echo "$f_curr_date,$f_open,$f_high,$f_low,$f_close,$f_volume,$f_percent" + [ -n "$f_curr_vals" ] && echo "$f_curr_date,$f_open,$f_high,$f_low,$f_close,$f_volume,$f_percent,$f_curr_vals" + f_prev_date=$f_curr_date + break + fi + # write missing line - echo "$f_missing_date,$f_close,$f_close,$f_close,$f_close,0,0.00,$f_curr_vals" + [ -z "$f_curr_vals" ] && echo "$f_missing_date,$f_open,$f_open,$f_open,$f_open,0,0.00" + [ -n "$f_curr_vals" ] && echo "$f_missing_date,$f_open,$f_open,$f_open,$f_open,0,0.00,$f_curr_vals" f_prev_date=$f_missing_date else f_prev_date=$f_curr_date - echo "$f_curr_date,$f_open,$f_high,$f_low,$f_close,$f_volume,$f_percent,$f_curr_vals" + [ -z "$f_curr_vals" ] && echo "$f_curr_date,$f_open,$f_high,$f_low,$f_close,$f_volume,$f_percent" + [ -n "$f_curr_vals" ] && echo "$f_curr_date,$f_open,$f_high,$f_low,$f_close,$f_volume,$f_percent,$f_curr_vals" break fi + done done < "$f_histfile" > $g_tmp/f_add_missing_ohlcv_intervals_result diff --git a/docker-compose.yml b/docker-compose.yml index 8ab4eff..9763fc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -235,6 +235,30 @@ services: cpus: '0.5' memory: 512M + dabo-fetch-coinmarketcapids: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + user: 10000:10000 + volumes: + - ./dabo:/dabo:ro + - ./strategies:/dabo/strategies:ro + - ./dabo-bot.conf:/dabo/dabo-bot.override.conf +# - ./watch-assets.csv:/dabo/watch-assets.csv + - ./data:/dabo/htdocs:rw + - ./home:/dabo/home:rw + - ./notify.local.conf:/usr/local/etc/notify.conf:ro + - /etc/localtime:/etc/localtime:ro + entrypoint: /dabo/fetch-coinmarketcapids.sh + cpu_shares: 128 + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + + # dabo-test: # build: # context: .