fixes error handling and coinmarketcap ids
This commit is contained in:
parent
ae70ddfb48
commit
d6cefa56d1
@ -29,7 +29,7 @@ function currency_converter {
|
||||
|
||||
unset f_currency_converter_result
|
||||
|
||||
local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse
|
||||
local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse f_file f_link_file
|
||||
|
||||
if [[ $f_currency_target =~ ^20.*-.*: ]]
|
||||
then
|
||||
@ -54,24 +54,22 @@ function currency_converter {
|
||||
f_currency_date_month=$(date -d "${f_currency_date}" "+%Y-%m")
|
||||
|
||||
# path to history files for the converting rate
|
||||
[ -d asset-histories ] && f_asset_histories="asset-histories/"
|
||||
[ -d asset-histories ] || mkdir asset-histories
|
||||
f_asset_histories="asset-histories/"
|
||||
|
||||
# map USD-Stablecoins to USD
|
||||
local f_stablecoins="USDT BUSD"
|
||||
for f_stablecoin in $f_stablecoins
|
||||
do
|
||||
# Link USD Stablecoin files to USD
|
||||
if [ -s ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ]
|
||||
then
|
||||
[ -e ${f_asset_histories}${f_currency}USD.history-raw.csv ] || \
|
||||
ln ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ${f_asset_histories}${f_currency}USD.history-raw.csv
|
||||
fi
|
||||
|
||||
if [ -s ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ]
|
||||
then
|
||||
[ -e ${f_asset_histories}USD${f_currency_target}.history-raw.csv ] || \
|
||||
ln ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ${f_asset_histories}USD${f_currency_target}.history-raw.csv
|
||||
fi
|
||||
cd "$f_asset_histories"
|
||||
find . -name "*${f_stablecoin}.history.*.csv" | while read f_file
|
||||
do
|
||||
f_link_file=${f_file/${f_stablecoin}/USD}
|
||||
echo "Linkfile $f_link_file"
|
||||
ln -sf "$f_file" "$f_link_file"
|
||||
done
|
||||
cd -
|
||||
# use USD
|
||||
if [[ $f_currency_target = $f_stablecoin ]]
|
||||
then
|
||||
@ -99,17 +97,25 @@ function currency_converter {
|
||||
local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
|
||||
local f_histfile_coinmarketcap_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
|
||||
# search for rate by date
|
||||
|
||||
echo "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
|
||||
|
||||
echo "$f_currency_date_minute $f_currency_date_hour $f_currency_date_day $f_currency_date_month"
|
||||
|
||||
|
||||
for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse" "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
|
||||
do
|
||||
# histfile has to exist
|
||||
#if [ -s "${f_histfile}*.csv" ]
|
||||
#then
|
||||
# search for most precise date
|
||||
set -x
|
||||
f_line=$(egrep "^$f_currency_date_minute" "$f_histfile".*m.csv 2>/dev/null | tail -n1)
|
||||
[ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile".*h.csv 2>/dev/null | tail -n1)
|
||||
[ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile".*d.csv 2>/dev/null | tail -n1)
|
||||
[ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile"*.csv 2>/dev/null | tail -n1)
|
||||
[ -n "$f_line" ] && f_rate=$(echo "$f_line" | cut -d, -f2)
|
||||
set +x
|
||||
f_reverse=false
|
||||
if [ -n "$f_rate" ]
|
||||
then
|
||||
|
||||
@ -86,19 +86,22 @@ function get_marketdata_coinmarketcap {
|
||||
fi
|
||||
|
||||
# end if already exists and modified under given time
|
||||
set -x
|
||||
if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}"
|
||||
then
|
||||
echo XXX
|
||||
set +x
|
||||
return 0
|
||||
fi
|
||||
|
||||
# cleanup
|
||||
rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err ${f_targetjsontmp} "${f_targetjsontmp}".err
|
||||
|
||||
|
||||
if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "7d" ]
|
||||
then
|
||||
# Download data from coinmarketcap
|
||||
g_wget -O "${f_targetjsontmp}" "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=${f_timeframe}" 2>"${f_targetjsontmp}".err
|
||||
cp "${f_targetjsontmp}" /tmp/xxxx
|
||||
jq -r '.data.quotes[] | .quote.timestamp[0:10] + "," + (.quote.open|tostring) + "," + (.quote.high|tostring) + "," + (.quote.low|tostring) + "," + (.quote.close|tostring) + "," + (.quote.volume|tostring)' "${f_targetjsontmp}" | egrep -v ',0$|,$' >"${f_targetcsvtmp}" 2>"${f_targetjsontmp}".err
|
||||
else
|
||||
g_echo_error "${FUNCNAME} $@: Timeframe $f_timeframe in CoinMarketCap not supported."
|
||||
@ -109,7 +112,7 @@ function get_marketdata_coinmarketcap {
|
||||
if ! [ -s "${f_targetcsvtmp}" ]
|
||||
then
|
||||
mkdir -p FAILED_COINMARKETCAP
|
||||
cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>&1
|
||||
cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null
|
||||
f_get_marketdata_coinmarketcap_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null)
|
||||
return 1
|
||||
fi
|
||||
@ -131,22 +134,30 @@ function get_marketdata_coinmarketcap_ids {
|
||||
|
||||
local f_target=COINMARKETCAPIDS
|
||||
local f_target_tmp="${f_target}.tmp"
|
||||
local f_target_loop=$f_target_tmp
|
||||
local f_target_loop=$f_target_tmp
|
||||
local f_latest_date
|
||||
local f_latest_date_seconds
|
||||
local f_latest_date_seconds_now=$(date -d "now - 8 days" +%s)
|
||||
|
||||
# write direct to target if not exists or empty
|
||||
[ -s "$f_target" ] || f_target_loop=$f_target
|
||||
|
||||
for f_id in $(seq 1 50000)
|
||||
do
|
||||
curl -s --request GET --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" | jq -r '.[] | .symbol + "," + (.id|tostring)' | head -n1
|
||||
sleep 0.3
|
||||
done | egrep '^.+,[0-9]*$' >"$f_target_loop"
|
||||
curl -s --request GET --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" >"$g_tmp/get_marketdata_coinmarketcap_ids.json"
|
||||
f_latest_date=$(jq -r '.data.quotes[] | .quote.timestamp[0:10]' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | tail -n1)
|
||||
f_latest_date_seconds=$(date -d "$f_latest_date" +%s)
|
||||
if [ $f_latest_date_seconds_now -lt $f_latest_date_seconds ]
|
||||
then
|
||||
jq -r '.[] | .symbol + "," + (.id|tostring)' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | head -n 1
|
||||
fi
|
||||
done | egrep --line-buffered '^.+,[0-9]*$' >"$f_target_loop"
|
||||
|
||||
if [ -s "$f_target_tmp" ]
|
||||
then
|
||||
cp -p "$f_target" "${f_target}.old"
|
||||
sort -u "$f_target_tmp" "${f_target}.old" >"$f_target"
|
||||
rm "$f_target_tmp"
|
||||
cp -p "$f_target" "${f_target}.$(date +%F)"
|
||||
mv "$f_target_tmp" "$f_target"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@ -123,7 +123,7 @@ function get_marketdata_yahoo {
|
||||
if ! [ -s "${f_targetcsvtmp}" ]
|
||||
then
|
||||
mkdir -p FAILED_YAHOO
|
||||
cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>&1
|
||||
cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null
|
||||
f_get_marketdata_yahoo_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null)
|
||||
return 1
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user