diff --git a/dabo/functions/currency_converter.sh b/dabo/functions/currency_converter.sh index f3b4c83..2cd7fe9 100644 --- a/dabo/functions/currency_converter.sh +++ b/dabo/functions/currency_converter.sh @@ -29,7 +29,7 @@ function currency_converter { unset f_currency_converter_result - local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse + local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse f_file f_link_file if [[ $f_currency_target =~ ^20.*-.*: ]] then @@ -54,24 +54,22 @@ function currency_converter { f_currency_date_month=$(date -d "${f_currency_date}" "+%Y-%m") # path to history files for the converting rate - [ -d asset-histories ] && f_asset_histories="asset-histories/" + [ -d asset-histories ] || mkdir asset-histories + f_asset_histories="asset-histories/" # map USD-Stablecoins to USD local f_stablecoins="USDT BUSD" for f_stablecoin in $f_stablecoins do # Link USD Stablecoin files to USD - if [ -s ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ] - then - [ -e ${f_asset_histories}${f_currency}USD.history-raw.csv ] || \ - ln ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ${f_asset_histories}${f_currency}USD.history-raw.csv - fi - - if [ -s ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ] - then - [ -e ${f_asset_histories}USD${f_currency_target}.history-raw.csv ] || \ - ln ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ${f_asset_histories}USD${f_currency_target}.history-raw.csv - fi + cd "$f_asset_histories" + find . -name "*${f_stablecoin}.history.*.csv" | while read f_file + do + f_link_file=${f_file/${f_stablecoin}/USD} + echo "Linkfile $f_link_file" + ln -sf "$f_file" "$f_link_file" + done + cd - # use USD if [[ $f_currency_target = $f_stablecoin ]] then @@ -99,17 +97,25 @@ function currency_converter { local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history" local f_histfile_coinmarketcap_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history" # search for rate by date + + echo "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse" + + echo "$f_currency_date_minute $f_currency_date_hour $f_currency_date_day $f_currency_date_month" + + for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse" do # histfile has to exist #if [ -s "${f_histfile}*.csv" ] #then # search for most precise date + set -x f_line=$(egrep "^$f_currency_date_minute" "$f_histfile".*m.csv 2>/dev/null | tail -n1) [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile".*h.csv 2>/dev/null | tail -n1) [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile".*d.csv 2>/dev/null | tail -n1) [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile"*.csv 2>/dev/null | tail -n1) [ -n "$f_line" ] && f_rate=$(echo "$f_line" | cut -d, -f2) + set +x f_reverse=false if [ -n "$f_rate" ] then diff --git a/dabo/functions/get_marketdata_coinmarketcap.sh b/dabo/functions/get_marketdata_coinmarketcap.sh index ae15caf..a9d8fbc 100644 --- a/dabo/functions/get_marketdata_coinmarketcap.sh +++ b/dabo/functions/get_marketdata_coinmarketcap.sh @@ -86,19 +86,22 @@ function get_marketdata_coinmarketcap { fi # end if already exists and modified under given time + set -x if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}" then + echo XXX + set +x return 0 fi # cleanup rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err ${f_targetjsontmp} "${f_targetjsontmp}".err + if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "7d" ] then # Download data from coinmarketcap g_wget -O "${f_targetjsontmp}" "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=${f_timeframe}" 2>"${f_targetjsontmp}".err - cp "${f_targetjsontmp}" /tmp/xxxx jq -r '.data.quotes[] | .quote.timestamp[0:10] + "," + (.quote.open|tostring) + "," + (.quote.high|tostring) + "," + (.quote.low|tostring) + "," + (.quote.close|tostring) + "," + (.quote.volume|tostring)' "${f_targetjsontmp}" | egrep -v ',0$|,$' >"${f_targetcsvtmp}" 2>"${f_targetjsontmp}".err else g_echo_error "${FUNCNAME} $@: Timeframe $f_timeframe in CoinMarketCap not supported." @@ -109,7 +112,7 @@ function get_marketdata_coinmarketcap { if ! [ -s "${f_targetcsvtmp}" ] then mkdir -p FAILED_COINMARKETCAP - cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>&1 + cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null f_get_marketdata_coinmarketcap_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null) return 1 fi @@ -131,22 +134,30 @@ function get_marketdata_coinmarketcap_ids { local f_target=COINMARKETCAPIDS local f_target_tmp="${f_target}.tmp" - local f_target_loop=$f_target_tmp + local f_target_loop=$f_target_tmp + local f_latest_date + local f_latest_date_seconds + local f_latest_date_seconds_now=$(date -d "now - 8 days" +%s) # write direct to target if not exists or empty [ -s "$f_target" ] || f_target_loop=$f_target for f_id in $(seq 1 50000) do - curl -s --request GET --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" | jq -r '.[] | .symbol + "," + (.id|tostring)' | head -n1 sleep 0.3 - done | egrep '^.+,[0-9]*$' >"$f_target_loop" + curl -s --request GET --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" >"$g_tmp/get_marketdata_coinmarketcap_ids.json" + f_latest_date=$(jq -r '.data.quotes[] | .quote.timestamp[0:10]' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | tail -n1) + f_latest_date_seconds=$(date -d "$f_latest_date" +%s) + if [ $f_latest_date_seconds_now -lt $f_latest_date_seconds ] + then + jq -r '.[] | .symbol + "," + (.id|tostring)' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | head -n 1 + fi + done | egrep --line-buffered '^.+,[0-9]*$' >"$f_target_loop" if [ -s "$f_target_tmp" ] then - cp -p "$f_target" "${f_target}.old" - sort -u "$f_target_tmp" "${f_target}.old" >"$f_target" - rm "$f_target_tmp" + cp -p "$f_target" "${f_target}.$(date +%F)" + mv "$f_target_tmp" "$f_target" fi } diff --git a/dabo/functions/get_marketdata_yahoo.sh b/dabo/functions/get_marketdata_yahoo.sh index e5af7da..38f3088 100644 --- a/dabo/functions/get_marketdata_yahoo.sh +++ b/dabo/functions/get_marketdata_yahoo.sh @@ -123,7 +123,7 @@ function get_marketdata_yahoo { if ! [ -s "${f_targetcsvtmp}" ] then mkdir -p FAILED_YAHOO - cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>&1 + cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null f_get_marketdata_yahoo_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null) return 1 fi