speed optimations

dabo/functions/currency_converter.sh

@@ -29,7 +29,7 @@ function currency_converter {
   
   unset f_currency_converter_result
 
-  local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse f_file f_link_file
+  local f_line f_rate f_histfile f_date_array f_stablecoin f_reverse f_file f_link_file f_timeframe
 
   if [[ $f_currency_target =~ ^20.*-.*: ]]
   then
@@ -63,13 +63,12 @@ function currency_converter {
   do
     # Link USD Stablecoin files to USD
     cd "$f_asset_histories"
-    find . -name "*${f_stablecoin}.history.*.csv" | while read f_file
+    find . -maxdepth 1 -mindepth 1 -name "*${f_stablecoin}.history.*.csv" | while read f_file
     do
       f_link_file=${f_file/${f_stablecoin}/USD}
-      #echo "Linkfile $f_link_file"
       ln -sf "$f_file" "$f_link_file"
     done
-    cd -
+    cd - >/dev/null
     # use USD
     if [[ $f_currency_target = $f_stablecoin ]]
     then
@@ -88,28 +87,35 @@ function currency_converter {
     return 0
   fi
   
-  # try direct pair
+  # define possiblefiles
-  [ "${f_currency}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency_target}-${f_currency}" "${f_currency_target}${f_currency}" 
-  [ "${f_currency_target}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency}-${f_currency_target}" "${f_currency}${f_currency_target}"
   local f_histfile_default="${f_asset_histories}${f_currency_target}${f_currency}.history"
-  local f_histfile_coinmarketcap="${f_asset_histories}${f_currency_target}${f_currency}.history"
-  # reverse as backup
   local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
-  local f_histfile_coinmarketcap_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history"
-  # search for rate by date
 
-  echo "$f_histfile_default" "$f_histfile_default_reverse"  "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
+  for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse"
-
-  echo "$f_currency_date_minute $f_currency_date_hour $f_currency_date_day $f_currency_date_month"
-
-
-  for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse"  "$f_histfile_coinmarketcap" "$f_histfile_coinmarketcap_reverse"
   do
     # search for most precise date
-    f_line=$(egrep "^$f_currency_date_minute" "$f_histfile"*m.csv 2>/dev/null | tail -n1)
+    f_line=$(egrep "^$f_currency_date_minute" "$f_histfile"*m.csv 2>/dev/null | sort | tail -n1)
-    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile"*h.csv 2>/dev/null | tail -n1)
+    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile"*m.csv 2>/dev/null | sort | tail -n1)
-    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile"*d.csv 2>/dev/null | tail -n1)
+    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile"*h.csv 2>/dev/null | sort | tail -n1)
-    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile"*.csv 2>/dev/null | tail -n1) 
+  done
+
+  # download from coinmarketcap if nothing found
+  [ -z "$f_line" ] && for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse" 
+  do
+    # download data from coinmarketcap
+    for f_timeframe in 1d 1w
+    do
+      [ "${f_currency}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency_target}-${f_currency}" "${f_currency_target}${f_currency}" $f_timeframe
+      [ "${f_currency_target}" = "USD" ] && get_marketdata_coinmarketcap "${f_currency}-${f_currency_target}" "${f_currency}${f_currency_target}"
+    done
+    f_line=$(egrep "^$f_currency_date_minute" "$f_histfile"*m.csv 2>/dev/null | sort | tail -n1)
+    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile"*m.csv 2>/dev/null | sort | tail -n1)
+    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile"*h.csv 2>/dev/null | sort | tail -n1)
+    [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile"*.csv 2>/dev/null | sort | tail -n1)
+    [ -n "$f_line" ] && break
+  done  
+
+  # extract rate/price
   [ -n "$f_line" ] && f_rate=$(echo "$f_line" | cut -d, -f2)
   f_reverse=false
   if [ -n "$f_rate" ]
@@ -120,7 +126,6 @@ function currency_converter {
     [ $f_currency_target = "EUR" ] && [ $f_currency = "USD" ] &&  f_reverse=true
     [[ $f_line =~ ^$f_currency_date_hour ]] && break
   fi
-  done
 
   # end if no rate found
   if [ -z "$f_rate" ]
@@ -147,3 +152,6 @@ function currency_converter {
 
 }
 
+
+
+

dabo/functions/get_bitpanda_api_transactions.sh

@@ -58,7 +58,7 @@ function get_bitpanda_api_transactions {
         select(.effective_leverage!= null) |
         .time.date_iso8601 + ",leverage-" + .type + "," + .cryptocoin_symbol + "," + .amount_cryptocoin + ",EUR," + .amount_fiat + ",Bitpanda"
       ' BITPANDA_trades.json >>BITPANDA.csv.tmp
-     # Workaround fpr staking-rewards (not availabpe per API yet (https://help.blockpit.io/hc/de-at/articles/360011790820-Wie-importiere-ich-Daten-mittels-Bitpanda-API-Key)
+     # Workaround for staking-rewards (not availabpe per API yet (https://help.blockpit.io/hc/de-at/articles/360011790820-Wie-importiere-ich-Daten-mittels-Bitpanda-API-Key)
      [ -s bitpanda-export.csv ] && cat bitpanda-export.csv  | grep reward,incoming | awk -F, '{print $2",reward-staking,"$8","$7",EUR,"$5",Bitpanda"}' >>BITPANDA.csv.tmp
 
      cat BITPANDA.csv.tmp | grep -v ",reward.best," | sort >TRANSACTIONS-BITPANDA.csv

dabo/functions/get_marketdata_coinmarketcap.sh

@@ -60,14 +60,11 @@ function get_marketdata_coinmarketcap {
   then
     # remove /*
     f_item=${f_item///*/}
-    ## remove :* (for example :USDT in contract markets)
-    #f_item=${f_item//:*}
-    # remove spaces
-    #f_item=${f_item/ /}
   fi
 
   local f_id
-  f_id=$(egrep "^${f_item},[1-9]" COINMARKETCAPIDS | head -n1 | cut -d, -f2)
+  # get id -> If multiple take the one with the largest marketcap
+  f_id=$(egrep "^${f_item},[1-9]" COINMARKETCAPIDS | sort -n -t, -k4 | tail -n1 | cut -d, -f2)
   [[ $f_item = EURC ]] && f_id=20641
   if [ -z "$f_id" ]
   then
@@ -155,8 +152,7 @@ function get_marketdata_coinmarketcap_ids {
     f_latest_date_seconds=$(date -d "$f_latest_date" +%s)
     if [ $f_latest_date_seconds_now -lt $f_latest_date_seconds ]
     then
-      jq -r '.[] | .symbol + "," + (.id|tostring) + "," + .name' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | head -n 1
+      jq -r '.data | .symbol + "," + (.id|tostring) + "," + .name + "," + (.quotes[].quote|.marketCap|tostring)' "$g_tmp/get_marketdata_coinmarketcap_ids.json"  | head -n 1
-     # jq -r '.[] | .symbol + "," + (.id|tostring) + "," + .name' "$g_tmp/get_marketdata_coinmarketcap_ids.json" | head -n 1 1>&2
     fi
   done | egrep --line-buffered '^.+,[0-9]*,' >"$f_target_loop"
   

dabo/functions/transactions_overview.sh

@@ -154,6 +154,7 @@ function transactions_overview {
       [[ $f_type =~ stake ]] && continue
 
       # what did I spent on asset in currency
+      #if [[ $f_type =~ buy|leverage-buy ]]
       if [[ $f_type =~ buy|leverage-buy|reward-staking|instant_trade_bonus|giveaway ]]
       then
         g_calc "$f_currency_spent+$f_currency_amount"