fixes

.gitignore

@@ -4,6 +4,7 @@
 /data/botdata
 /data/index.html
 /data/index.html.tmp
+/data/TRANSACTIONS*
 /data/charts.min.css
 /dabo/.*-secrets
 /home/.ssh

dabo/dabo-bot.sh

@@ -98,7 +98,10 @@ do
 
 
   ## watch some manual defined assets
-  watch_assets
+  #watch_assets
+  
+  # transactions overview
+  #[ ${FULL_LOOP} = 1 ] && transactions_overview
 
   # stock data
   if [ "${STOCK_EXCHANGE}" = "BINANCE" ]
@@ -124,7 +127,7 @@ do
   check_for_sell
 
   # Get current balances 
-  get_ccxt_balances
+  [ ${FULL_LOOP} = 1 ] && get_ccxt_balances || continue
   [ ${FULL_LOOP} = 1 ] && get_balances || continue
 
   # Buy something?

dabo/functions/ccxt.sh

@@ -1,30 +1,55 @@
 function f_ccxt {
-  if [ -s /dabo/.${STOCK_EXCHANGE,,}-secrets ]
+  
+  # remove old result
+  unset f_ccxt_result
+
+  # lower case
+  STOCK_EXCHANGE=${STOCK_EXCHANGE,,}
+
+  if [ -s /dabo/.${STOCK_EXCHANGE}-secrets ]
   then
-    . /dabo/.${STOCK_EXCHANGE,,}-secrets
+    . /dabo/.${STOCK_EXCHANGE}-secrets
   else
-    g_echo_error "No secrets found (/dabo/.${STOCK_EXCHANGE,,}-secrets) found"
+    g_echo_error "No secrets found (/dabo/.${STOCK_EXCHANGE}-secrets) found"
     return 1
   fi
   
   unset g_ccxt_jobs
   local g_ccxt_jobs
   mapfile -t g_ccxt_jobs < <(jobs -r)
-  # Initialize ccxt in python
+  # Initialize ccxt in python if not initialized
-  if [[ ${g_ccxt_jobs[*]} != *python-pipeexec.py* ]] || [ -z "$f_ccxt_initialized" ]
+  [[ ${g_ccxt_jobs[*]} != *python-pipeexec.py* ]] || unset f_ccxt_initialized
+  if [ -z "$f_ccxt_initialized" ]
   then
-    unset f_ccxt_initialized
     g_python 'import os' || return 1
     g_python 'import sys' || return 1
     g_python 'sys.path.append("/ccxt/python")' || return 1
     g_python 'import ccxt' || return 1
+  fi
+  if ! [[ "$f_ccxt_initialized" =~ $STOCK_EXCHANGE ]]
+  then
     local f_exchange_type="swap"
     [ -z "$LEVERAGE" ] && f_exchange_type="spot"
-    g_python "exchange = ccxt.${STOCK_EXCHANGE,,}({'apiKey': '${API_KEY}','secret': '${API_SECRET}','options': {'defaultType': '${f_exchange_type}',},})" || return 1
+    g_python "${STOCK_EXCHANGE} = ccxt.${STOCK_EXCHANGE}({'apiKey': '${API_KEY}','secret': '${API_SECRET}','enableRateLimit': True,'options': {'defaultType': '${f_exchange_type}',},})" || return 1
-    f_ccxt_initialized=true
+    g_python "${STOCK_EXCHANGE}.load_markets()"
+    f_ccxt_initialized="${f_ccxt_initialized}${STOCK_EXCHANGE},"
   fi
   
-  g_python "$@" || return 1
+  # send and receive ccxt command in python - on error kill progress
-  unset f_ccxt_result
+  if ! g_python "$@" 
-  f_ccxt_result=${g_python_result[*]}
+  then
+    kill -9 $(jobs -l | grep python-pipeexec.py | cut -d" " -f 2)
+    return 1
+  fi
+  # reference result to python-result
+  declare -ng f_ccxt_result=g_python_result
+
+  # make the output jq-conform
+  # avoids errors like: "parse error: Invalid numeric literal at"
+  f_ccxt_result=${f_ccxt_result//\'/\"}
+  f_ccxt_result=${f_ccxt_result// None/ null}
+  f_ccxt_result=${f_ccxt_result// True/ true}
+  f_ccxt_result=${f_ccxt_result// False/ false}
+  f_ccxt_result=${f_ccxt_result//,,/,}
+  
 }

dabo/functions/currency_converter.sh

@@ -0,0 +1,118 @@
+function currency_converter {
+
+  #g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+
+  local f_currency_amount=$1
+  local f_currency=$2
+  local f_currency_target=$3
+  local f_currency_date=$4
+  
+  unset f_currency_converter_result 
+  local f_reverse=false
+  local f_line f_rate f_histfile f_date_array f_stablecoin
+
+  # get current date if none given
+  [ -z "$f_currency_date" ] && printf -v f_currency_date '%(%Y-%m-%d %H:%M:%S)T'
+ 
+  # rate per minute
+  f_currency_date_minute=$(date -d "${f_currency_date}" "+%Y-%m-%d.%H:%M")
+
+  # hour failback
+  f_currency_date_hour=$(date -d "${f_currency_date}" "+%Y-%m-%d.%H")
+  
+  # day failback
+  f_currency_date_day=$(date -d "${f_currency_date}" "+%Y-%m-%d")
+
+  # month failback
+  f_currency_date_month=$(date -d "${f_currency_date}" "+%Y-%m")
+
+  # path to history files for the converting rate
+  [ -d asset-histories ] && f_asset_histories="asset-histories/"
+
+  # map USD-Stablecoins to USD
+  local f_stablecoins="USDT BUSD"
+  for f_stablecoin in $f_stablecoins
+  do
+    # Link USD Stablecoin files to USD
+    if [ -s ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ] 
+    then
+      [ -e ${f_asset_histories}${f_currency}USD.history-raw.csv ] || \
+      ln ${f_asset_histories}${f_currency}${f_stablecoin}.history-raw.csv ${f_asset_histories}${f_currency}USD.history-raw.csv
+    fi
+   
+    if [ -s ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ] 
+    then
+      [ -e ${f_asset_histories}USD${f_currency_target}.history-raw.csv ] || \
+       ln ${f_asset_histories}${f_currency_target}${f_stablecoin}.history-raw.csv ${f_asset_histories}USD${f_currency_target}.history-raw.csv
+    fi
+    # use USD
+    if [[ $f_currency_target = $f_stablecoin ]]
+    then
+      f_currency_target=USD
+    fi
+    if [[ $f_currency = $f_stablecoin ]] 
+    then
+      f_currency=USD
+    fi
+  done
+
+  # if there is no currency change (USD USD or USDT USD)
+  if [[ $f_currency == $f_currency_target ]]
+  then
+    f_currency_converter_result=$f_currency_amount
+    return 0
+  fi
+
+  # try direct pair
+  get_marketdata_yahoo_historic "${f_currency_target}-${f_currency}" "${f_currency_target}${f_currency}" || get_marketdata_yahoo_historic "${f_currency}-${f_currency_target}" "${f_currency}${f_currency_target}"
+  local f_histfile_default="${f_asset_histories}${f_currency_target}${f_currency}.history-raw.csv"
+  local f_histfile_yahoo="${f_asset_histories}${f_currency_target}${f_currency}.history-yahoo.csv"
+  # reverse as backup
+  local f_histfile_default_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history-raw.csv"
+  local f_histfile_yahoo_reverse="${f_asset_histories}${f_currency}${f_currency_target}.history-yahoo.csv"
+  
+  # search for rate by date
+  for f_histfile in "$f_histfile_default" "$f_histfile_default_reverse"  "$f_histfile_yahoo" "$f_histfile_yahoo_reverse"
+  do
+    # histfile has to exist
+    if [ -s "$f_histfile" ]
+    then
+      # search for most precise date
+      f_line=$(egrep "^$f_currency_date_minute" "$f_histfile" | tail -n1)
+      [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_hour" "$f_histfile" | tail -n1)
+      [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_day" "$f_histfile" | tail -n1)
+      [ -z "$f_line" ] && f_line=$(egrep "^$f_currency_date_month" "$f_histfile" | tail -n1) 
+      [ -n "$f_line" ] && f_rate=$(echo "$f_line" | cut -d, -f2)
+      if [ -n "$f_rate" ]
+      then
+        [[ $f_histfile =~ ${f_currency}${f_currency_target} ]] && f_reverse=true
+        [[ $f_line =~ ^$f_currency_date_hour ]] && break
+      fi
+    fi
+  done
+
+  # end if no rate found
+  if [ -z "$f_rate" ]
+  then
+    # try workaround over USD if EUR
+    if [[ ${f_currency_target} = EUR ]] && [[ $f_currency != USD ]]
+    then
+      #g_echo_note "trying way over USD (workaround)"
+      if currency_converter $f_currency_amount $f_currency USD $f_currency_date
+      then
+        f_currency_amount=$f_currency_converter_result
+        currency_converter $f_currency_amount USD EUR $f_currency_date
+        return $?
+      fi
+    fi
+    g_echo_warn "didn't find rate for ${f_currency}-${f_currency_target} - '${FUNCNAME} $@'"
+    return 1
+  fi
+ 
+  # calculate converted currency and store result
+  [[ $f_reverse = true ]] && g_calc "${f_currency_amount}*${f_rate}"
+  [[ $f_reverse = false ]] && g_calc "1/${f_rate}*${f_currency_amount}"
+  f_currency_converter_result=$g_calc_result
+
+}
+

dabo/functions/get_balances.sh

@@ -15,6 +15,8 @@ function get_balances {
   then
     onetrading-api-call GET public/v1/account/balances || return 1
     cat ${g_tmp}/API_CMD_OUT | jq -r '.balances[] | .currency_code + "," + .available' | egrep -v ",0$|,0[\.][0]*$|${TRANSFER_CURRENCY}" | sort >${f_filename}_OUT.tmp_raw
+  else 
+    return 0
   fi
 
   if ! [ -s ${f_filename}_OUT.tmp_raw ] && egrep -q "^[A-Z]+,[0-9]*\.[0-9]+$" ${f_filename}_OUT.tmp_raw

dabo/functions/get_bitpanda_api_transactions.sh

@@ -6,7 +6,7 @@ function get_bitpanda_api_transactions {
   if [ -s /dabo/.bitpanda-secrets ]
   then
     source /dabo/.bitpanda-secrets
-    g_echo "Bitpanda API-Key found. Getting data from Biotpanda API"
+    g_echo "Bitpanda API-Key found. Getting data from Bitpanda API"
     curl -s -X GET "https://api.bitpanda.com/v1/wallets/transactions?page_size=999999" -H "X-Api-Key: ${BITPANDA_API_KEY}" >BITPANDA_wallets_transactions.csv.tmp \
       && mv BITPANDA_wallets_transactions.csv.tmp BITPANDA_wallets_transactions.json
     curl -s -X GET "https://api.bitpanda.com/v1/fiatwallets/transactions?page_size=999999" -H "X-Api-Key: ${BITPANDA_API_KEY}" >BITPANDA_fiatwallets_transactions.csv.tmp \
@@ -14,20 +14,23 @@ function get_bitpanda_api_transactions {
     curl -s -X GET "https://api.bitpanda.com/v1/trades?page_size=999999" -H "X-Api-Key: ${BITPANDA_API_KEY}" >BITPANDA_trades.csv.tmp \
       && mv BITPANDA_trades.csv.tmp BITPANDA_trades.json
     unset BITPANDA_API_KEY
+    # Trades
     jq -r '
       .data[].attributes |
         select(.status=="finished") |
         select(.type=="sell" or .type=="buy") |
         select(.cryptocoin_symbol!= null) |
-        .time.date_iso8601 + "," + .type + "," + .cryptocoin_symbol + "," + .amount + ",EUR," + .amount_eur + ",Bitpanda"
+        .time.date_iso8601 + "," + .type + "," + .cryptocoin_symbol + "," + .amount + ",EUR," + .amount_eur_incl_fee + ",Bitpanda"
       ' BITPANDA_wallets_transactions.json >BITPANDA.csv.tmp
+    # Giveaways
     jq -r '
       .data[].attributes |
         select(.status=="finished") |
         select(.type=="transfer") |
         select(.cryptocoin_symbol!= null) |
-        .time.date_iso8601 + "," + .tags[].attributes.short_name + "," + .cryptocoin_symbol + "," + .amount + ",EUR," + .amount_eur + ",Bitpanda"
+        .time.date_iso8601 + "," + .tags[].attributes.short_name + "," + .cryptocoin_symbol + "," + .amount + ",EUR," + .amount_eur_incl_fee + ",Bitpanda"
       ' BITPANDA_wallets_transactions.json | sed 's/,reward,/,giveaway,/' >>BITPANDA.csv.tmp
+    # Leverage-Trades 
     jq -r '
       .data[].attributes |
         select(.status=="finished") |

dabo/functions/get_ccxt_balances.sh

@@ -4,8 +4,8 @@ function get_ccxt_balances {
   
   local f_filename="CCXT_GET_BALANCES_CMD"
   
-  #f_ccxt 'print(exchange.fetch_balance ({"currency": "USDT"}))' && echo $f_ccxt_result >CCXT_BALANCES_RAW
+  #f_ccxt "print(${STOCK_EXCHANGE}.fetch_balance ({"currency": "USDT"}))" && echo $f_ccxt_result >CCXT_BALANCES_RAW
-  f_ccxt 'print(exchange.fetch_balance ())' && echo $f_ccxt_result >CCXT_BALANCES_RAW
+  f_ccxt "print(${STOCK_EXCHANGE}.fetch_balance ())" && echo $f_ccxt_result >CCXT_BALANCES_RAW
   return 0
 
 
@@ -16,9 +16,9 @@ function get_ccxt_balances {
     binance-api-call GET sapi/v1/capital/config/getall || return 1
     # parse outout
     cat ${g_tmp}/API_CMD_OUT | jq -r '.[] | .coin + "," + .free' | egrep -v ",0$|,0[\.][0]*$|${TRANSFER_CURRENCY}" | sort >${f_filename}_OUT.tmp_raw
-  elif [ ${STOCK_EXCHANGE} = "BITPANDA" ]
+  elif [ ${STOCK_EXCHANGE} = "ONETRADING" ]
   then
-    bitpanda-api-call GET public/v1/account/balances || return 1
+    onetrading-api-call GET public/v1/account/balances || return 1
     cat ${g_tmp}/API_CMD_OUT | jq -r '.balances[] | .currency_code + "," + .available' | egrep -v ",0$|,0[\.][0]*$|${TRANSFER_CURRENCY}" | sort >${f_filename}_OUT.tmp_raw
   fi
 

dabo/functions/get_justtrade_csv_transactions.sh

@@ -2,7 +2,6 @@ function get_justtrade_csv_transactions {
 
   g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
 
-  # Check for JustTrade API Key - If there is data from JustTrade csv
   if [ -s justtrade-export.csv ] 
   then
     cat justtrade-export.csv | sed 's/\"//g' | egrep '^[0-9]' | egrep    ',otc,justtrade,.+Z,EUR,' | awk -F, '{print $4",sell,"$7","$8","$5","$6",JustTrade"}' >JUSTTRADE-sell.csv.tmp

dabo/functions/get_marketdata.sh

@@ -1,5 +1,7 @@
 function get_marketdata {
  
+  g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+ 
   get_marketdata_from_url YM=F DOW-JONES-INDEX
   get_marketdata_from_url ES=F SP500-INDEX
   get_marketdata_from_url NQ=F NASDAQ-INDEX
@@ -15,24 +17,6 @@ function get_marketdata {
   get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX
   get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX
   get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX
-  #get_marketdata_from_url https://www.boerse.de/indizes/Dow-Jones/US2605661048 DOW-JONES-INDEX
-  #get_marketdata_from_url https://www.boerse.de/indizes/SundP-500/US78378X1072 SP500-INDEX
-  #get_marketdata_from_url https://www.boerse.de/indizes/Nasdaq-100/US6311011026 NASDAQ-INDEX
-  #get_marketdata_from_url https://www.investing.com/indices/msci-eafe MSCI-EAFE-INDEX
-  #get_marketdata_from_url https://www.investing.com/indices/10-year-treasury-yield 10-YEAR-TREASURY-YIELD-INDEX
-  #get_marketdata_from_url https://www.boerse.de/rohstoffe/WTI-Rohoelpreis/XD0015948363 OIL-INVERTED-INDEX
-  #get_marketdata_from_url https://www.boerse.de/rohstoffe/Goldpreis/XC0009655157 GOLD-INDEX
-  ##get_marketdata https://www.boerse.de/aktien/Nvidia-Aktie/US67066G1040 NVIDIA-INDEX
-  ##get_marketdata https://www.boerse.de/aktien/Tesla-Aktie/US88160R1014 TESLA-INDEX
-  ##get_marketdata https://www.boerse.de/aktien/Apple-Aktie/US0378331005 APPLE-INDEX
-  #get_marketdata_from_url https://www.boerse.de/realtime-kurse/MSCI-World/XC0009692739 MSCI-WORLD-INDEX
-  #get_marketdata_from_url https://www.boerse.de/indizes/DJ-AMER-OIL-und-GAS/XC0006886516 OIL-GAS-INVERTED-INDEX
-  #get_marketdata_from_url https://www.boerse.de/fonds/SPDR-SundP-Regional-Banking-ETF/US78464A6982 KRE-BANKING-INDEX
-  ##get_marketdata https://www.boerse.de/indizes/Dax/DE0008469008 DAX-INDEX
-  #get_marketdata_from_url https://www.investing.com/indices/usdollar DXY-INVERTED-INDEX
-  #get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX
-  #get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX
-  #get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX
   get_marketdata_from_url '"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=5m" | jq  -r .[].longShortRatio | tail -n1' BINANCE-BTC-GlobalLongShortAccountRatio
   get_marketdata_from_url '"https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=5m" | jq -r .[].buySellRatio | tail -n1' BINANCE-BTC-TakerLongShortRatio
   get_marketdata_from_url '"https://fapi.binance.com/futures/data/openInterestHist?symbol=BTCUSDT&period=5m" | jq -r .[].sumOpenInterest | tail -n1' BINANCE-BTC-OpenInterest
@@ -42,6 +26,8 @@ function get_marketdata {
 }
 
 function get_marketdata_from_url {
+  g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+
   local f_url="$1"
   local f_name="$2"
 
@@ -70,6 +56,7 @@ function get_marketdata_from_url {
   else 
   # default to Yahoo Finace Symbols via API
     echo "wget ${g_wget_opts} -q -O - https://query1.finance.yahoo.com/v8/finance/chart/${f_url} | jq -r '.[].result[].meta.regularMarketPrice'" >MARKET_DATA_CMD
+    get_marketdata_yahoo_historic "$f_url" "$f_name"
   fi
 
   g_runcmd g_retrycmd sh MARKET_DATA_CMD >MARKET_DATA_CMD_OUT-${f_name}.tmp 2>MARKET_DATA_CMD_OUT-${f_name}.tmp.err

dabo/functions/get_marketdata_historic.sh

@@ -0,0 +1,61 @@
+function get_marketdata_yahoo_historic {
+
+  #g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+
+  local f_item="$1"
+  local f_name="$2"
+  local f_targetcsv="asset-histories/${f_name}.history-yahoo.csv"
+  local f_targetcsvtmp="${g_tmp}/${f_name}.history-yahoo.csv"
+ 
+  [[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X"
+  [[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X"
+ 
+  # end if already failed the last hour
+  if [ -f FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD ]
+  then
+    find "FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD" -mmin +60 -delete
+    if [ -f FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD ]
+    then
+      #g_echo_note "${f_targetcsv} already failed to downloaded within last hour"
+      return 1
+    fi
+  fi
+
+  # end if already exists and modified under 1 day
+  if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mtime -1 | grep -q "${f_targetcsv}"
+  then
+    #g_echo_note "${f_targetcsv} has already been downloaded within a day"
+    return 0
+  fi
+
+  local f_sec
+  printf -v f_sec '%(%s)T'
+  # cleanup
+  rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err
+
+  # Download historical data from yahoo
+  #g_echo_note "Fetching Yahoo-Historical data of $f_name"
+  g_wget -O ${f_targetcsvtmp} "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history" 2>"${f_targetcsvtmp}".err
+  if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ] 
+  then
+    egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" "${f_targetcsv}" | sort -u >"${f_targetcsv}.tmp"
+    mv "${f_targetcsv}.tmp" "${f_targetcsv}"
+  elif [ -s "${f_targetcsvtmp}" ]
+  then
+    egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" >"$f_targetcsv"
+  else
+    # report error
+#    g_echo_note "Fetching historical data of $f_name from Yahoo failed! 
+#CMD: 
+#wget -q -O ${f_targetcsvtmp} ${g_wget_opts} \"https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history\"
+#
+#OUT:
+#$(cat "${f_targetcsvtmp}")
+#
+#ERR:
+#$(cat "${f_targetcsvtmp}".err)
+#"
+    mv ${f_targetcsvtmp}.err FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD
+    return 1
+  fi
+}

dabo/functions/get_onetrading_csv_transactions.sh

@@ -0,0 +1,18 @@
+function get_onetrading_csv_transactions {
+
+  g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+
+  if [ -s onetrading-export.csv ] 
+  then
+
+    cat onetrading-export.csv | sed 's/\"//g' | grep ',SELL,' | awk -F, '{print $12","tolower($3)","$6","$5","$8","$9",OneTrading"}' | sort >ONETRADING-sell.csv.tmp
+
+    cat onetrading-export.csv | sed 's/\"//g' | grep ',BUY,' | awk -F, '{print $12","tolower($3)","$11","$9","$8","$5",OneTrading"}' | sort >ONETRADING-buy.csv.tmp
+
+    cat ONETRADING-buy.csv.tmp ONETRADING-sell.csv.tmp | sort >TRANSACTIONS-ONETRADING.csv
+    rm -f ONETRADING-buy.csv.tmp ONETRADING-sell.csv.tmp
+
+  fi
+
+}
+

dabo/functions/get_transactions.sh

@@ -0,0 +1,205 @@
+function get_transactions {
+
+  g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
+
+  local f_exchange f_symbols f_symbol
+  local DEFAULT_STOCK_EXCHANGE=$STOCK_EXCHANGE
+
+  for f_exchange in /dabo/.*-secrets;
+  do
+    # extract ccxt exchange from filename
+    g_basename $f_exchange
+    f_exchange=${g_basename_result/-secrets}
+    f_exchange=${f_exchange/.}
+    STOCK_EXCHANGE=$f_exchange
+    
+    [[ $f_exchange = bitpanda ]] && continue
+    [[ $f_exchange = onetrading ]] && continue
+  
+    g_echo_note "Exchange: $f_exchange"
+
+    f_ccxt "print(${STOCK_EXCHANGE}.symbols)"
+    f_symbols=${f_ccxt_result}
+    f_symbols=${f_symbols//\"}
+    f_symbols=${f_symbols//, /+}
+    f_symbols=${f_symbols//\[}
+    f_symbols=${f_symbols//\]}
+    
+    # transfer-dir
+    mkdir -p "TRANSACTIONS-$f_exchange"
+    # create timestamp file
+    touch --time=mtime -t $(date -d "now -1 day" +%Y%m%d%H%M) TRANSACTIONS-TIMESTAMP
+
+    # go through symbols
+    local f_orig_IFS=$IFS
+    IFS=+
+    for f_symbol in $f_symbols
+    do
+      #echo $f_symbol
+      #[[ $f_symbol =~ ETH|BTC ]] || continue
+      IFS=$f_orig_IFS
+      # binance does not allow derivate trading in germany so ignore because of 400-Error
+      [[ $f_symbol =~ : ]] && [[ $f_exchange = binance ]] && continue
+      f_symbol_file="TRANSACTIONS-$f_exchange/${f_symbol//\/}"
+      
+      # remove file older then 1 day and refetch
+      [ "$f_symbol_file" -ot TRANSACTIONS-TIMESTAMP ] && rm -f "$f_symbol_file"
+      
+      # fetch only if not exists
+      [ -f "$f_symbol_file" ] && continue
+      g_echo_note "fetchng closed orders of $f_symbol"
+      f_ccxt "print(${STOCK_EXCHANGE}.fetchMyTrades(symbol='$f_symbol', limit=500, params={'paginate': True}))"
+      
+      # write to file
+      #cat ${g_python_out} >"$f_symbol_file"
+      [[ $f_ccxt_result = \[\] ]] && f_ccxt_result=""
+      echo -n $f_ccxt_result >"$f_symbol_file"
+
+      # continue if no trade
+      [ -z "$f_ccxt_result" ] && continue
+      
+      # get f_asset+f_currency from symbol (BTC/USDT)
+      g_array $f_symbol f_symbol_array /
+      f_asset=${f_symbol_array[0]}
+      f_currency=${f_symbol_array[1]}
+
+      f_symbol_file_csv_tmp="${f_symbol_file}.csv.tmp"
+      f_symbol_file_csv="${f_symbol_file}.csv"      
+      >"${f_symbol_file_csv_tmp}"
+
+      # Check for contract/leverage Trade
+      f_leverage=""
+      if [[ $f_currency =~ : ]]
+      then
+        # mark
+        f_leverage="leverage-"
+
+        # get funding fees
+        f_ccxt "print(${STOCK_EXCHANGE}.fetchFundingHistory('$f_symbol', limit=200, params={'paginate': True}))"
+        [[ $f_ccxt_result = \[\] ]] || echo -n $f_ccxt_result >"${f_symbol_file}.FundingFees"
+        cat ${f_symbol_file}.FundingFees | jq -r "
+.[] |
+.datetime + \",fundingfee,$f_asset,0,\" + .code  + \",0\" + \",$f_exchange,\" + .code  + \",\" + (.amount|tostring)
+" >>$f_symbol_file_csv_tmp
+        
+        # remove the ':' in f_currency
+        g_array "$f_currency" f_currency_array :
+        f_currency=${f_currency_array[0]}
+      fi
+
+      # generate csv
+      # get longs (posSide=="1")
+      cat "$f_symbol_file" | jq -r "
+.[] |
+ select(.side==\"buy\" or .side==\"sell\") |
+ select(.info.posSide==\"1\") |
+ select(.symbol!= null) |
+.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency  + \",\" +  (.fee.cost|tostring)
+" >>"$f_symbol_file_csv_tmp"
+
+      # get shorts (posSide=="2") sell first, then buy (https://github.com/ccxt/ccxt/issues/22518)
+      cat "$f_symbol_file" | jq -r "
+.[] |
+ select(.side==\"sell\") |
+ select(.info.posSide==\"2\") |
+ select(.symbol!= null) |
+ .side = \"sell\" |
+.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency  + \",\" +  (.fee.cost|tostring) + \",short\"
+" >>"$f_symbol_file_csv_tmp"
+
+      cat "$f_symbol_file" | jq -r "
+.[] |
+ select(.side==\"buy\") |
+ select(.info.posSide==\"2\") |
+ select(.symbol!= null) |
+ .side = \"buy\" |
+.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency  + \",\" +  (.fee.cost|tostring) + \",short\"
+" >>"$f_symbol_file_csv_tmp"
+
+      if [ -s "$f_symbol_file_csv_tmp" ] 
+      then
+        if [ -s "$f_symbol_file_csv" ]
+        then
+          cat "$f_symbol_file_csv_tmp" "$f_symbol_file_csv" | sort -u >"${f_symbol_file_csv}.sorted"
+          mv "${f_symbol_file_csv}.sorted" "$f_symbol_file_csv"
+          rm "$f_symbol_file_csv_tmp"
+        else
+          mv "$f_symbol_file_csv_tmp" "$f_symbol_file_csv"
+        fi
+      else
+        rm "$f_symbol_file_csv_tmp"
+      fi
+
+    done
+
+    ## Get converts on supported exchanges since 2022
+    if [[ $f_exchange = binance ]]
+    then
+    printf -v f_convert_end_year '%(%Y)T'
+    for ((y=2022;y<=$f_convert_end_year;y++))
+    do
+      f_convert_end_month=12
+      [[ $y == $f_convert_end_year ]] && printf -v f_convert_end_month '%(%m)T'
+      for ((m=1;m<=$f_convert_end_month;m++))
+      do
+        f_start_date="$(date -d "$y-$m-1" +%s)001"
+        em=$((m+1))
+        [ $em = 13 ] && em=1
+        f_end_date="$(date -d "$y-$em-1" +%s)000"
+        f_convert_file="TRANSACTIONS-$f_exchange/CONVERT-$y-$m"
+
+        [ -s "${f_convert_file}.csv" ] && continue
+
+        f_ccxt "print(${f_exchange}.fetchConvertTradeHistory(since=${f_start_date}, params={'until': ${f_end_date}}))"
+        [[ $f_ccxt_result = \[\] ]] && f_ccxt_result=""
+        echo -n $f_ccxt_result >"$f_convert_file"
+
+        if [ -s "$f_convert_file" ]
+        then
+          cat "$f_convert_file" | jq -r "
+.[] |
+ select(.info.side==\"BUY\") |
+ .datetime + \",\" + (.info.side|ascii_downcase) + \",\" + .toCurrency + \",\" + (.toAmount|tostring) + \",\" + .fromCurrency + \",\" + (.fromAmount|tostring) + \",$f_exchange\"
+" >"${f_convert_file}.csv"
+          cat "$f_convert_file" | jq -r "
+.[] |
+ select(.info.side==\"SELL\") |
+ .datetime + \",\" + (.info.side|ascii_downcase) + \",\" + .fromCurrency + \",\" + (.fromAmount|tostring) + \",\" + .toCurrency + \",\" + (.toAmount|tostring) + \",$f_exchange\"
+" >>"${f_convert_file}.csv"
+
+        fi
+      done
+    done
+    fi
+
+    # put all sorted n one file
+    if [ -s TRANSACTIONS-$f_exchange.csv ] 
+    then
+      cat "TRANSACTIONS-$f_exchange/"*.csv TRANSACTIONS-$f_exchange.csv | sort -u >TRANSACTIONS-$f_exchange.csv.tmp
+      mv TRANSACTIONS-$f_exchange.csv.tmp TRANSACTIONS-$f_exchange.csv
+    else
+      cat "TRANSACTIONS-$f_exchange/"*.csv | sort -u >TRANSACTIONS-$f_exchange.csv
+    fi
+
+    # Switch sides if Fiat is in Krypto side
+    local f_fiats="USD EUR"
+    local f_fiat
+    for f_fiat in $f_fiats
+    do
+      [ -f TRANSACTIONS-$f_exchange.csv.tmp ] && rm TRANSACTIONS-$f_exchange.csv.tmp
+      grep -h ",sell,$f_fiat," TRANSACTIONS-$f_exchange.csv | awk -F, '{print $1",buy,"$5","$6","$3","$4","$7","$8","$9}' >>TRANSACTIONS-$f_exchange.csv.tmp
+      grep -h ",buy,$f_fiat," TRANSACTIONS-$f_exchange.csv | awk -F, '{print $1",sell,"$5","$6","$3","$4","$7","$8","$9}' >>TRANSACTIONS-$f_exchange.csv.tmp
+      if [ -s TRANSACTIONS-$f_exchange.csv.tmp ]
+      then
+        g_echo_note "Switched some fiat/krypto sides"
+        #cat TRANSACTIONS-$f_exchange.csv.tmp
+        cat TRANSACTIONS-$f_exchange.csv | egrep -v ",sell,$f_fiat,|,buy,$f_fiat," >>TRANSACTIONS-$f_exchange.csv.tmp
+        cat TRANSACTIONS-$f_exchange.csv.tmp | sort >TRANSACTIONS-$f_exchange.csv
+      fi
+    done
+
+  done
+  STOCK_EXCHANGE=$DEFAULT_STOCK_EXCHANGE
+
+}
+

dabo/functions/transactions_overview.sh

@@ -4,33 +4,44 @@ function transactions_overview {
 
   get_bitpanda_api_transactions
   get_justtrade_csv_transactions
+  get_onetrading_csv_transactions
+
+  #get_transactions
 
   >ALL_TRANSACTIONS_OVERVIEW.csv.tmp
 
-  local f_exchange f_transactions_array f_transaction 
+  local f_exchange f_asset f_transactions_array f_transaction f_result f_asset_quantity f_asset_quantity_sold f_currency_quantity f_currency_quantity_sold f_currency_spent f_date f_type f_asset_amount f_currency f_currency_amount f_fee_currency f_fee_amount f_sell_result f_taxable f_tax_type f_one_year_ago f_currency_amount_eur f_currency_spent_eur f_currency_quantity_sold_eur
 
-  #f_seconds_one_year_ago=$(date -d "1 year ago" +%s)
+  f_assets_per_exchange=$(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$|^$' TRANSACTIONS-*.csv | cut -d, -f3,7 | sort -u | grep phemex)
-
-  f_tax_rate="32"
-
-  f_assets_per_exchange=$(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$|^$' TRANSACTIONS-*.csv | cut -d, -f3,7 | sort -u)
-
-  f_trade_result=0
 
   for f_asset_per_exchange in ${f_assets_per_exchange}
   do
-    f_asset_quantity=0
-    f_currency_spent=0
-    f_result=0
-    f_asset_amount_tax_free=0
-    f_asset_amount_tax_able=0
-    f_trade_result_asset=0
     mapfile -d, -t f_asset_per_exchange_array < <(echo $f_asset_per_exchange)
     f_asset=${f_asset_per_exchange_array[0]}
     f_exchange=${f_asset_per_exchange_array[1]%$'\n'}
-    echo "=== Asset $f_asset on Exchange $f_exchange"
+    
-    mapfile -t f_transactions_array < <(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$' TRANSACTIONS-*.csv | egrep ",${f_asset},.+,${f_exchange}" | sort)
+
-    for f_transaction in ${f_transactions_array[@]}
+    # Ignore stableCoins, EUR, USD
+     [[ $f_asset = USDT ]] && continue
+     [[ $f_asset = USDC ]] && continue
+     [[ $f_asset = BUSD ]] && continue
+     [[ $f_asset = USD ]] && continue
+     [[ $f_asset = EUR ]] && continue
+
+    echo -e "\n\n=== Asset $f_asset on Exchange $f_exchange"
+
+    f_result=0
+    f_result_eur=0
+    f_asset_quantity=0
+    f_asset_quantity_sold=0
+    f_currency_quantity=0
+    f_currency_quantity_sold=0
+    f_currency_quantity_sold_eur=0
+    f_currency_spent=0
+    f_currency_spent_eur=0
+
+    mapfile -t f_transactions_array < <(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$' TRANSACTIONS-*.csv | sort -u | egrep ",${f_asset},.+,.+,${f_exchange}" | sort)
+    for f_transaction in "${f_transactions_array[@]}"
     do 
       mapfile -d, -t f_transaction_array < <(echo $f_transaction)
       f_date=${f_transaction_array[0]}
@@ -38,21 +49,73 @@ function transactions_overview {
       f_asset_amount=${f_transaction_array[3]}
       f_currency=${f_transaction_array[4]}
       f_currency_amount=${f_transaction_array[5]}
+      f_currency_amount_eur=0
+      f_fee_currency=${f_transaction_array[7]}
+      f_fee_amount=""
+      [ -n "$f_fee_currency" ] && printf -v f_fee_amount %.20f ${f_transaction_array[8]}
       f_sell_result=0
+      f_sell_result_eur=0
       f_taxable=0
       f_tax_type=""
       f_one_year_ago=""
-      f_staking_rewards_asset=0
-      f_giveaway_asset=0
-      f_instant_trade_bonus_asset=0
-      f_leverage_result_asset=0
-      f_asset_amount_tax_able=0
-      f_asset_amount_tax_free=0
 
-      # Ignore USDT
+      # if there is a fee change to f_currency_amount and deduct f_currency_amount
-      [[ $f_asset =~ USDT ]] && continue
+      if [ -n "$f_fee_amount" ]
+      then
+        if [[ $f_fee_currency != $f_currency ]]
+        then
+          #mapfile -dT -t f_date_array < <(echo $f_date)
+          #if currency_converter $f_fee_amount $f_fee_currency $f_currency ${f_date_array[0]}
+          if currency_converter $f_fee_amount $f_fee_currency $f_currency "$f_date"
+          then
+            f_fee_amount=$f_currency_converter_result
+            f_fee_currency=$f_currency
+          else
+            g_echo_warn "!!!!!! Could not convert currency $f_fee_currency to $f_currency"
+            continue
+          fi
+        fi
+        # deduct fee
+        if [[ $f_type =~ sell ]]
+        then
+           g_calc "$f_currency_amount-($f_fee_amount)"
+           f_currency_amount=$g_calc_result
+        elif [[ $f_type =~ fundingfee ]]
+        then
+           g_calc "$f_currency_spent+($f_fee_amount)"
+           f_currency_spent=$g_calc_result
+           f_currency_amount=$f_fee_amount
+           f_tax_type="Note: Included in purchase price"
+        else
+           g_calc "$f_currency_amount+($f_fee_amount)"
+           f_currency_amount=$g_calc_result
+        fi
+      fi
 
-      # If moveng to stake -> irrelevant/ignore
+      # get f_currency_amount in EUR for german tax declaration
+      if [[ $f_currency != EUR ]]
+      then
+        if currency_converter $f_currency_amount $f_currency EUR "$f_date"
+        then
+          f_currency_amount_eur=$f_currency_converter_result
+        else
+          g_echo_error "!!!!!! Could not convert currency $f_currency to EUR"
+          return 1
+        fi
+      else
+        f_currency_amount_eur=$f_currency_amount
+      fi
+      
+      # round fiat numbers to 2 decimal places
+      printf -v f_currency_amount_eur %.2f $f_currency_amount_eur
+      local f_fiats="USD USDT BUSD"
+      local f_fiat
+      for f_fiat in $f_fiats
+      do
+        [ "$f_fiat" = "$f_currency" ] && printf -v f_currency_amount %.2f $f_currency_amount
+      done
+
+      # If transfer to stake -> irrelevant/ignore
       [[ $f_type =~ stake ]] && continue
 
       # what did I spent on asset in currency
@@ -60,6 +123,9 @@ function transactions_overview {
       then
         g_calc "$f_currency_spent+$f_currency_amount"
         f_currency_spent=$g_calc_result
+
+        g_calc "$f_currency_spent_eur+$f_currency_amount_eur"
+        f_currency_spent_eur=$g_calc_result
       fi
 
       # what did I spent on asset
@@ -73,70 +139,172 @@ function transactions_overview {
       # rise result if reward-staking|instant_trade_bonus|giveaway
       if [[ $f_type =~ reward-staking|instant_trade_bonus|giveaway ]]
       then
-        g_calc "$f_result+$f_currency_amount"
+      #  # Calculate/change to Euro if not
-        f_result=$g_calc_result
+      #  if [[ $f_currency != EUR ]]
+      #  then
+      #    if currency_converter $f_currency_amount $f_currency EUR "${f_date}"
+      #    then
+      #      f_currency_amount=$f_currency_converter_result
+      #      f_currency="EUR"
+      #    else
+      #      g_echo_warn "!!!!!! Could not convert currency $f_currency to EUR"
+      #      continue
+      #    fi
+      #  fi
 
         if [[ $f_type =~ reward-staking ]]
         then
-          f_staking_rewards_asset=$f_currency_amount
+          f_taxable=$f_currency_amount_eur
-          f_taxable=$f_currency_amount
           f_tax_type="Staking Reward (Einkommenssteuersatz)"
         fi
         if [[ $f_type =~ instant_trade_bonus ]]
         then
-          f_instant_trade_bonus_asset=$f_currency_amount
+          f_taxable=$f_currency_amount_eur
-          f_taxable=$f_currency_amount
           f_tax_type="Instand Trade Bonus (Einkommenssteuersatz)"
         fi
         if [[ $f_type =~ giveaway ]]
         then
-          f_giveaway_asset=$f_currency_amount
+          f_taxable=$f_currency_amount_eur
-          f_taxable=$f_currency_amount
           f_tax_type="Giveaway (Einkommenssteuersatz)"
         fi
 
       fi
 
       # calculate result and tax (if taxable) on sale
-      if [[ $f_type =~ sell|leverage-sell ]]
+      if [[ $f_type =~ sell ]]
       then
 
-        # how much in percentage is sold?
+        ## Some validity checks
-        g_calc "100/$f_asset_quantity*$f_asset_amount"
+        # not for leverage because of short-trade
-        printf -v f_sold_percentage %.2f $g_calc_result
+        if [ $f_type = sell ]
-        #echo "Sold percentage: $f_sold_percentage" 
+        then
+          # if sell on never buyed!?
+          if [ $f_currency_spent = 0 ]
+          then
+            g_echo_warn "!!!!!! Sell never buyed!? Spent currency on $f_asset is 0"
+            continue
+          fi
+          # if sell wahats not exists!?
+          if [ $f_asset_quantity = 0 ]
+          then
+            g_echo_warn "!!!!!! Sell never buyed!? Buyed asset $f_asset is 0"
+            continue
+          fi
+        fi
 
-        # what is the currency amount with result (profit/loss)?
+        # summarize sold asset quantity
-        g_calc "$f_currency_amount/$f_sold_percentage*100"
+        g_calc "$f_asset_quantity_sold+$f_asset_amount"
-        f_curency_amount_with_result=$g_calc_result
+        f_asset_quantity_sold=$g_calc_result
-        #echo "f_curency_amount_with_result: $f_curency_amount_with_result"
        
-        # what is the result of this sell in currency (profit/loss)?
+        # remaining quantity after sell
-        g_calc "$f_curency_amount_with_result-$f_currency_spent"
+        g_calc "$f_asset_quantity-$f_asset_quantity_sold"
+        f_asset_quantity_remaining=$g_calc_result
+
+        # summarize sold currency quantity
+        g_calc "$f_currency_quantity_sold+$f_currency_amount"
+        f_currency_quantity_sold=$g_calc_result
+
+        # summarize sold currency quantity in EUR
+        g_calc "$f_currency_quantity_sold_eur+$f_currency_amount_eur"
+        f_currency_quantity_sold_eur=$g_calc_result
+
+        ## Check for ended trade (asset-quantity=0 or dust)
+        # if all is sold trade ended and calculate PNL
+        local f_trade_end=0
+        local f_dust=0
+        local f_dust_eur=0
+        g_calc "$f_asset_quantity_sold==$f_asset_quantity"
+        [ ${g_calc_result} -eq 1 ] && f_trade_end=1
+ 
+        # Alterntively check for remaining dust only to find end of trade and calculate PNL
+        if [ ${g_calc_result} -eq 0 ]
+        then
+          echo currency_converter $f_asset_quantity_remaining $f_asset $f_currency "${f_date}"
+          currency_converter $f_asset_quantity_remaining $f_asset $f_currency "${f_date}"
+          local f_currency_remaining=$f_currency_converter_result
+          if g_num_is_between $f_currency_remaining -5 5
+          then
+            f_dust=$f_currency_remaining
+            currency_converter $f_asset_quantity_remaining $f_asset EUR "${f_date}"
+            f_dust_eur=$f_currency_converter_result
+            g_echo_note "Quantity ($f_asset_quantity $f_asset - $f_dust (USD) looks like dust - Ending trade"
+            f_trade_end=1
+
+            # add to sold quantity
+            g_calc "$f_currency_quantity_sold+($f_dust)"
+            f_currency_quantity_sold=$g_calc_result
+            # add to sold quantity EUR
+            g_calc "$f_currency_quantity_sold_eur+($f_dust_eur)"
+            f_currency_quantity_sold_eur=$g_calc_result
+          else
+            g_echo_note "Tade not closed - partial sale!? Remaining $f_asset_quantity_remaining $f_asset ($f_currency_remaining $f_currency)!?"
+            f_tax_type="Note: Trade not closed - partial sale. Remaining $f_asset_quantity_remaining $f_asset ($f_currency_remaining $f_currency)"
+          fi
+        fi
+
+        if [ ${f_trade_end} -eq 1 ]
+        then
+
+          echo "Buy price: $f_currency_spent $f_currency ($f_currency_spent_eur EUR)"
+          echo "Sell price: $f_currency_quantity_sold $f_currency ($f_currency_quantity_sold_eur EUR)"
+
+          # calculate result of trade 
+          g_calc "$f_currency_quantity_sold-$f_currency_spent"
           f_sell_result=$g_calc_result
-        #echo "Result: $f_sell_result"
+          g_calc "$f_currency_quantity_sold_eur-$f_currency_spent_eur"
+          f_sell_result_eur=$g_calc_result
+          echo "Sell result: $f_sell_result $f_currency ($f_sell_result_eur EUR)"
+
+          g_percentage-diff $f_currency_spent $f_currency_quantity_sold
+          f_sell_result_percentage=$g_percentage_diff_result
+          g_percentage-diff $f_currency_spent_eur $f_currency_quantity_sold_eur
+          f_sell_result_percentage_eur=$g_percentage_diff_result
+          echo "Sell result percentage: ${f_sell_result_percentage}% (EUR ${f_sell_result_percentage_eur}%)"
+
+          # calculate complete result
+          g_calc "$f_result+($f_sell_result)"
+          f_result=$g_calc_result
+
+          # calculate complete result EUR
+          g_calc "$f_result_eur+($f_sell_result_eur)"
+          f_result_eur=$g_calc_result
+   
+          # reset vars
+          f_asset_quantity=0
+          f_asset_quantity_sold=0
+          f_asset_quantity_remaining=0
+          f_currency_spent=0
+          f_currency_spent_eur=0
+          f_currency_quantity_sold=0
+          f_currency_quantity_sold_eur=0
+        fi
 
         # at leverage always full taxable
-        if [[ $f_type == leverage-sell ]]
+        if [ "$f_type" = "leverage-sell" ] && [ ${f_trade_end} -eq 1 ]
         then
           f_taxable=$f_sell_result
           f_tax_type="Verkauf gehebelter Position (Kapitalertragssteuer)"
         # at no leverage-sell
-        else
+        elif [ ${f_trade_end} -eq 1 ]
+        then
           ## taxable - one year?
           f_seconds_sell_ago=$(date -d "$f_date 1 year ago" +%s)
 
           oldIFS=$IFS
           IFS=$'\n'
-          # add all but sell to f_asset_amount_tax_able or f_asset_amount_tax_free
+          ## catculate f_asset_amount_tax_able and/or f_asset_amount_tax_free for current history
-          mapfile -t f_assetlines_array < <(egrep ",$f_exchange,[a-z]+,$f_asset," ALL_TRANSACTIONS_OVERVIEW.csv.tmp | grep -v ',sell,')
+          f_asset_amount_tax_able=0
-          for f_assetline in ${f_assetlines_array[@]}
+          f_asset_amount_tax_free=0
+          mapfile -t f_assetlines_array < <(egrep ",$f_exchange,[a-z]+,$f_asset," ALL_TRANSACTIONS_OVERVIEW.csv.tmp)
+          for f_assetline in "${f_assetlines_array[@]}"
           do
             mapfile -d, -t f_assetline_array < <(echo $f_assetline)
             f_assetline_date=${f_assetline_array[0]}
             f_assetline_type=${f_assetline_array[2]}
             f_assetline_amount=${f_assetline_array[4]}
             f_seconds_transfer_ago=$(date -d "$f_assetline_date" +%s)
+            if [[ $f_assetline_type =~ buy|reward-staking|instant_trade_bonus|giveaway ]]
+            then
               if [ $f_seconds_sell_ago -gt $f_seconds_transfer_ago ]
               then
                 # buy from sell one year ago
@@ -147,25 +315,22 @@ function transactions_overview {
                 g_calc "$f_asset_amount_tax_able+$f_assetline_amount"
                 f_asset_amount_tax_able=$g_calc_result
               fi
-          done
+            elif [[ $f_assetline_type = sell ]]
-
+            then
-          # add sell to f_asset_amount_tax_able or f_asset_amount_tax_free
-          mapfile -t f_assetlines_sell_array < <(egrep ",$f_exchange,sell,$f_asset," ALL_TRANSACTIONS_OVERVIEW.csv.tmp)
-          for f_assetlinesell in ${f_assetlines_sell_array[@]}
-          do
-            mapfile -d, -t f_assetline_sell_array < <(echo $f_assetlinesell)
-            f_assetline_amount=${f_assetline_sell_array[4]}
               g_calc "$f_asset_amount_tax_free-$f_assetline_amount"
               f_assetline_amount_left=$g_calc_result
               if g_num_is_lower_equal $f_assetline_amount_left 0
               then
                 g_calc "$f_asset_amount_tax_able+$f_assetline_amount_left"
                 f_asset_amount_tax_able=$g_calc_result
+                f_asset_amount_tax_free=0
                else 
                 f_asset_amount_tax_free=$f_assetline_amount_left
               fi
+            fi
           done
           IFS=$oldIFS
+          #g_echo_note "Still in Stock: Taxable: $f_asset_amount_tax_able; Tax free: $f_asset_amount_tax_free"
 
           if g_num_is_higher_equal $f_asset_amount_tax_free $f_asset_amount
           then
@@ -179,7 +344,7 @@ function transactions_overview {
           then
             ## complete taxable if under 1 year
             f_one_year_ago="no"
-            f_taxable=$f_sell_result
+            f_taxable=$f_sell_result_eur
             f_tax_type="Verkauf (Einkommenssteuersatz)"
             g_calc "$f_asset_amount_tax_able-$f_asset_amount"
             f_asset_amount_tax_able=$g_calc_result
@@ -195,7 +360,7 @@ function transactions_overview {
             f_percentage_taxable=$g_calc_result
             
             # calculate part of sell_result from percentage from sell sum
-            g_calc "$f_sell_result/100*$f_percentage_taxable"
+            g_calc "$f_sell_result_eur/100*$f_percentage_taxable"
             f_tax_able=$g_calc_result            
 
             f_tax_type="Verkauf (Einkommenssteuersatz)"
@@ -203,20 +368,38 @@ function transactions_overview {
 
           fi
         fi
-        
-        # refesh amount on sell
-        g_calc "$f_asset_quantity-$f_asset_amount"
-        f_asset_quantity=$g_calc_result
-        g_calc "$f_currency_spent-($f_currency_spent/100*$f_sold_percentage)"
-        f_currency_spent=$g_calc_result
-
-        #g_calc "$f_trade_result_asset+$f_sell_result"
-        f_trade_result_asset=$g_calc_result
-
       fi
 
-      echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable" >>ALL_TRANSACTIONS_OVERVIEW.csv.tmp
+
-      echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable"
+      ## Fields
+      # 1 date
+      # 2 exchange
+      # 3 type
+      # 4 krypto asset
+      # 5 krypto asset amount
+      # 6 mutual currency
+      # 7 mutual currency amount
+      # 8 tax 1 year ago yes/no/partially?
+      # 9 totally mutal currency amount (spent money for trade)
+      # 10 totally krypto asset amount (got krypto in trade)
+      # 11 total result profit/loss for exchange/krypto asset
+      # 12 total result of trade (from/until krypto asset amount is 0 again)
+      # 13 type of tax if to pay
+      # 14 taxable amount of trade
+      # 15 mutual currency amount in EUR
+      # 16 total result profit/loss for exchange/krypto asset calculated in EUR
+      # 17 total result of trade (from/until krypto asset amount is 0 again) calculated in EUR
+      # 18 remaining krypto asset amount
+      echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable,$f_currency_amount_eur,$f_result_eur,$f_sell_result_eur,$f_asset_quantity_remaining" | tee -a ALL_TRANSACTIONS_OVERVIEW.csv.tmp
+
+      #printf -v f_currency_amount_rounded %.2f $f_currency_amount
+      #printf -v f_sell_result_rounded %.2f $f_sell_result
+      #printf -v f_taxable_rounded %.2f $f_taxable
+
+      if [[ $f_type =~ sell|leverage-sell ]]
+      then 
+        echo -e "\n"
+      fi 
 
     done
 
@@ -224,41 +407,118 @@ function transactions_overview {
     f_staking_rewards_asset=$(cat ALL_TRANSACTIONS_OVERVIEW.csv.tmp | egrep ",$f_exchange,reward-staking,$f_asset," | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
     f_giveaway_asset=$(cat ALL_TRANSACTIONS_OVERVIEW.csv.tmp | egrep ",$f_exchange,giveaway,$f_asset," | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
     f_instant_trade_bonus_asset=$(cat ALL_TRANSACTIONS_OVERVIEW.csv.tmp | egrep ",$f_exchange,instant_trade_bonus,$f_asset," | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
+    f_tax_total=$(cat ALL_TRANSACTIONS_OVERVIEW.csv.tmp | egrep ",$f_exchange,.+,$f_asset," | cut -d, -f14 | awk "{ SUM += \$1} END { printf(\"%.10f\", SUM) }")
 
-    echo "Straking Rewards: $f_staking_rewards_asset"
-    echo "Giveaways: $f_giveaway_asset"
-    echo "Instand Trade Bonus: $f_instant_trade_bonus_asset"
-    echo ""
 
-    #echo "$f_asset $f_exchange" | grep "ETH JustTrade" && exit 0
+    echo -e "\n $f_exchange $f_asset Results:"
+    echo " Remaining $f_asset_quantity_remaining $f_asset"
+    echo " Straking Rewards: $f_staking_rewards_asset $f_currency"
+    echo " Giveaways: $f_giveaway_asset $f_currency"
+    echo " Instand Trade Bonus: $f_instant_trade_bonus_asset $f_currency"
+    echo " Result: $f_result $f_currency ($f_result_eur EUR)"
+    echo " German Tax: $f_tax_total EUR"
+    echo "======================================"
+  
 
   done
 
   mv ALL_TRANSACTIONS_OVERVIEW.csv.tmp ALL_TRANSACTIONS_OVERVIEW.csv
 
+}
+
+function transactions_summary {
+
+
   # calculate totals from csv
-  f_instant_trade_bonus=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',instant_trade_bonus,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
+  local f_instant_trade_bonus=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',instant_trade_bonus,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
-  f_staking_rewards=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',reward-staking,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
+  local f_staking_rewards=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',reward-staking,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
-  f_giveaway=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',giveaway,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
+  local f_giveaway=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep ',giveaway,' | cut -d, -f7 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
   
   
-  echo "========== Total Results ==========="
+  echo -e "\n\n========== Total Results ==========="
-  echo "Trade Result: $f_trade_result"
+  #echo "Trade Result: $f_trade_result EUR" 
-  echo "Staking Result: $f_staking_rewards"
+  echo "Staking Result: $f_staking_rewards EUR"
-  echo "Giveaway Result: $f_giveaway"
+  echo "Giveaway Result: $f_giveaway EUR"
-  echo "Instand Trade Bonus: $f_instant_trade_bonus"
+  echo -e "Instand Trade Bonus: $f_instant_trade_bonus EUR\n"
-  echo ""
+  
+  rm -f ${g_tmp}/tax_summary_*
 
   # generate and go through list of years and Exchange/Tax-Types
+  local f_tax_year
   cat ALL_TRANSACTIONS_OVERVIEW.csv | cut -d- -f1 | sort -u  | while read f_tax_year
   do
     echo "========== Tax year $f_tax_year (German Tax Law) =========="
-    cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^$f_tax_year-" | egrep -v ',,0' | cut -d, -f  2,13 | sort -u | while read f_exchange_tax_type
+    local f_exchange_tax_type
+    cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^$f_tax_year-" | cut -d, -f  2,13 | sort -u | egrep -v ',$' | grep -v "Note: " | while read f_exchange_tax_type
     do
-      f_tax=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^$f_tax_year-" | egrep -v ',,0' | cut -d, -f  2,13,14 | grep "$f_exchange_tax_type" | cut -d, -f3 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
+      local f_exchange=$(echo $f_exchange_tax_type | cut -d, -f1)
+      local f_tax_type=$(echo $f_exchange_tax_type | cut -d, -f2)
+
+      local f_tax=$(cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^$f_tax_year-" | cut -d, -f  2,13,14  | egrep -v ',,0$' | grep "$f_exchange_tax_type" | cut -d, -f3 | awk "{ SUM += \$1} END { printf(\"%.2f\", SUM) }")
       echo "$f_exchange_tax_type: $f_tax"
+
+      echo "$f_tax_type: $f_tax EUR<br>" >>${g_tmp}/tax_summary_$f_exchange-$f_tax_year
+
+      echo "<html>
+<head>
+  <meta charset='UTF-8'>
+  <meta http-equiv='refresh' content='${INTERVAL}'> 
+  <meta name='viewport' content='width=device-width, initial-scale=1'>
+  <link rel='stylesheet' type='text/css' href='/browser.css'>
+  <link rel='stylesheet' type='text/css' href='/charts.min.css'>
+  <title>Detailed Transactions on $f_exchange from ${f_tax_year}</title>
+</head>
+<body>
+  <h1>Detailed Transactions on $f_exchange from ${f_tax_year}</h1>
+<h2>Summary</h2>
+$(cat ${g_tmp}/tax_summary_$f_exchange-$f_tax_year)
+<h2>List of trades</h2>
+- Fees included<br>
+- EUR values were calculated using the exchange rate at the time of trading<br>
+- Fiat rounded to two decimal places. Internally, further decimal places are used in the calculation<br>
+<table>
+<tr><td>Date</td><td>Type of transaction</td><td>Crypto value</td><td>Fiat value</td><td>Result</td><td>Tax type</td><td>Tax amount</td></tr>
+" >TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp
+
+      cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^${f_tax_year}-" | grep ",${f_exchange}," |  awk -F, '
+{printf "<tr><td>"$1"</td><td>"$3"</td><td>"$5" "$4"</td><td>"}
+{printf("%.2f", $7)}
+{printf " EUR </td><td>"}
+{printf("%.2f", $16)}
+{printf " EUR</td><td>"$13"</td><td>"}
+{printf("%.2f", $14)}
+{print " EUR</td></tr>"}' >>TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp
+
+      echo "</table></body></html>" >>TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp
+      mv TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp ../TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html
+
     done
     echo ""
   done
 
+  ## Overview over Overviews
+
+  echo "<html>
+<head>
+  <meta charset='UTF-8'>
+  <meta http-equiv='refresh' content='${INTERVAL}'> 
+  <meta name='viewport' content='width=device-width, initial-scale=1'>
+  <link rel='stylesheet' type='text/css' href='/browser.css'>
+  <link rel='stylesheet' type='text/css' href='/charts.min.css'>
+  <title>Trading Overview</title>
+</head>
+<body>
+  <h1>Transaction Overviews</h1>" >../TRANSACTIONS_OVERVIEWS.html
+
+
+  local f_html f_name
+  ls ../TRANSACTIONS_OVERVIEW-* | while read f_html
+  do
+    f_html=$(basename $f_html)
+    f_name=$(echo $f_html | cut -d- -f2,3 | cut -d. -f1)
+    echo "<a href='${f_html}'>$f_name</a><br>" >>../TRANSACTIONS_OVERVIEWS.html
+  done
+
+  echo "</body></html>" >>../TRANSACTIONS_OVERVIEWS.htm
+
 }

dabo/functions/watch_assets.sh

@@ -21,6 +21,9 @@ function watch_assets {
   <h1>Dabo-Bot WATCH ASSETS - ${URL} (ReadOnly)</h1>
   <h1>Last update $(date '+%F %T')</h1>"
 
+  .watch_assets.sh.swp
+
+  return 0
  
   mapfile -t f_watch_assets_array < <(egrep -v '^ASSET,ALERTS,BUYPRICE,BUYDATE,BUYQUANTITY,SELLPRICE,SELLDATE,SELLQUANTITY|^#|^$|^ +$' /dabo/watch-assets.csv)
   for f_line in "${f_watch_assets_array[@]}"