fixes error handling and coinmarketcap ids

dabo/functions/currency_converter.sh

@@ -29,7 +29,7 @@ function currency_converter {
   
   unset f_currency_converter_result
 
-  local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse
+  local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse f_file f_link_file
 
   if [[ $f_currency_target =~ ^20.*-.*: ]]
   then
@@ -54,24 +54,22 @@ function currency_converter {
   f_currency_date_month=$(date -d "${f_currency_date}" "+%Y-%m")
 
   # path to history files for the converting rate
-  [ -d asset-histories ] && f_asset_histories="asset-histories/"
+  [ -d asset-histories ] || mkdir asset-histories
+  f_asset_histories="asset-histories/"
 
   # map USD-Stablecoins to USD
   local f_stablecoins="USDT BUSD"
   for f_stablecoin in $f_stablecoins
   do
     # Link USD Stablecoin files to USD
-    if [ -s ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ] 
+    cd "$f_asset_histories"
-    then
+    find . -name "*${f_stablecoin}.history.*.csv" | while read f_file
-      [ -e ${f_asset_histories}${f_currency}USD.history-raw.csv ] || \
+    do
-      ln ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ${f_asset_histories}${f_currency}USD.history-raw.csv
+      f_link_file=${f_file/${f_stablecoin}/USD}
-    fi
+      echo "Linkfile $f_link_file"
-   
+      ln -sf "$f_file" "$f_link_file"
-    if [ -s ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ] 
+    done
-    then
+    cd -
-      [ -e ${f_asset_histories}USD${f_currency_target}.history-raw.csv ] || \
-       ln ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ${f_asset_histories}USD${f_currency_target}.history-raw.csv
-    fi
     # use USD
     if [[ $f_currency_target = $f_stablecoin ]]
     then
@@ -99,17 +97,25 @@ function currency_converter {
   local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
   local f_histfile_coinmarketcap_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
   # search for rate by date
+
+  echo "$f_histfile_default" "$f_histfile_default_reverse"  "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
+
+  echo "$f_currency_date_minute $f_currency_date_hour $f_currency_date_day $f_currency_date_month"
+
+
   for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse"  "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
   do
     # histfile has to exist
     #if [ -s "${f_histfile}*.csv" ]
     #then
       # search for most precise date
+      set -x
       f_line=$(egrep "^$f_currency_date_minute" "$f_histfile".*m.csv 2>/dev/null | tail -n1)
       [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile".*h.csv 2>/dev/null | tail -n1)
       [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile".*d.csv 2>/dev/null | tail -n1)
       [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile"*.csv 2>/dev/null | tail -n1) 
       [ -n "$f_line" ] && f_rate=$(echo "$f_line" | cut -d, -f2)
+      set +x
       f_reverse=false
       if [ -n "$f_rate" ]
       then

dabo/functions/get_marketdata_coinmarketcap.sh

@@ -86,19 +86,22 @@ function get_marketdata_coinmarketcap {
   fi
 
   # end if already exists and modified under given time
+  set -x
   if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}"
   then
+    echo XXX
+    set +x
     return 0
   fi
 
   # cleanup
   rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err ${f_targetjsontmp} "${f_targetjsontmp}".err
 
+
   if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "7d" ]
   then
     # Download data from coinmarketcap
     g_wget -O "${f_targetjsontmp}" "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=${f_timeframe}" 2>"${f_targetjsontmp}".err
-    cp "${f_targetjsontmp}"  /tmp/xxxx
     jq -r '.data.quotes[] | .quote.timestamp[0:10] + "," + (.quote.open|tostring) + "," + (.quote.high|tostring) + "," + (.quote.low|tostring) + "," + (.quote.close|tostring) + "," + (.quote.volume|tostring)' "${f_targetjsontmp}" | egrep -v ',0$|,$' >"${f_targetcsvtmp}" 2>"${f_targetjsontmp}".err
   else
     g_echo_error "${FUNCNAME} $@: Timeframe $f_timeframe in CoinMarketCap not supported."
@@ -109,7 +112,7 @@ function get_marketdata_coinmarketcap {
   if ! [ -s "${f_targetcsvtmp}" ]
   then
     mkdir -p FAILED_COINMARKETCAP
-    cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>&1
+    cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_COINMARKETCAP/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null
     f_get_marketdata_coinmarketcap_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null)
     return 1
   fi
@@ -132,21 +135,29 @@ function get_marketdata_coinmarketcap_ids {
   local f_target=COINMARKETCAPIDS
   local f_target_tmp="${f_target}.tmp"
   local f_target_loop=$f_target_tmp
+  local f_latest_date
+  local f_latest_date_seconds
+  local f_latest_date_seconds_now=$(date -d "now - 8 days" +%s)
 
   # write direct to target if not exists or empty
   [ -s "$f_target" ] || f_target_loop=$f_target
 
   for f_id in $(seq 1 50000)
   do
-    curl -s --request GET  --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" | jq -r '.[] | .symbol + "," + (.id|tostring)' | head -n1
     sleep 0.3
-  done | egrep '^.+,[0-9]*$' >"$f_target_loop"
+    curl -s --request GET  --url "https://api.coinmarketcap.com/data-api/v3.1/cryptocurrency/historical?id=${f_id}&interval=1d" >"$g_tmp/get_marketdata_coinmarketcap_ids.json"
+    f_latest_date=$(jq -r '.data.quotes[] | .quote.timestamp[0:10]' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | tail -n1)
+    f_latest_date_seconds=$(date -d "$f_latest_date" +%s)
+    if [ $f_latest_date_seconds_now -lt $f_latest_date_seconds ]
+    then
+      jq -r '.[] | .symbol + "," + (.id|tostring)' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | head -n 1
+    fi
+  done | egrep --line-buffered '^.+,[0-9]*$' >"$f_target_loop"
   
   if [ -s "$f_target_tmp" ] 
   then
-    cp -p "$f_target" "${f_target}.old"
+    cp -p "$f_target" "${f_target}.$(date +%F)"
-    sort -u "$f_target_tmp" "${f_target}.old" >"$f_target"
+    mv "$f_target_tmp" "$f_target"
-    rm "$f_target_tmp"
   fi
 }
 

dabo/functions/get_marketdata_yahoo.sh

@@ -123,7 +123,7 @@ function get_marketdata_yahoo {
   if ! [ -s "${f_targetcsvtmp}" ]
   then
     mkdir -p FAILED_YAHOO
-    cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>&1
+    cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>/dev/null
     f_get_marketdata_yahoo_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null)
     return 1
   fi