From 6aec58b05ef48d87140bf257e211dc7ca65bd6b2 Mon Sep 17 00:00:00 2001 From: olli Date: Thu, 27 Feb 2025 10:44:55 +0100 Subject: [PATCH] pnl, tax, transactions --- dabo/fetch-transaction-history.sh | 13 +- dabo/functions/calc_fifo_pnl.sh | 483 ++++++++++++++++++ dabo/functions/get_marketdata.sh | 6 + dabo/functions/get_phemex_csv_transactions.sh | 50 ++ dabo/functions/get_transactions.sh | 72 ++- dabo/functions/transactions_overview.sh | 38 +- dabo/functions/webpage_transactions.sh | 21 +- 7 files changed, 616 insertions(+), 67 deletions(-) create mode 100644 dabo/functions/calc_fifo_pnl.sh create mode 100644 dabo/functions/get_phemex_csv_transactions.sh diff --git a/dabo/fetch-transaction-history.sh b/dabo/fetch-transaction-history.sh index 5d62988..9182ae6 100755 --- a/dabo/fetch-transaction-history.sh +++ b/dabo/fetch-transaction-history.sh @@ -24,8 +24,19 @@ sleep 1800 while true do + >ALL_TRANSACTIONS_OVERVIEW.csv.tmp g_echo_note "Next loop" - transactions_overview + get_bitpanda_api_transactions + get_justtrade_csv_transactions + get_onetrading_csv_transactions + get_phemex_csv_transactions + get_transactions + for transaction_csv in TRANSACTIONS-*.csv + do + calc_fifo_pnl "$transaction_csv" + done + mv ALL_TRANSACTIONS_OVERVIEW.csv.tmp ALL_TRANSACTIONS_OVERVIEW.csv + webpage_transactions sleep 3600 done diff --git a/dabo/functions/calc_fifo_pnl.sh b/dabo/functions/calc_fifo_pnl.sh new file mode 100644 index 0000000..6d3897a --- /dev/null +++ b/dabo/functions/calc_fifo_pnl.sh @@ -0,0 +1,483 @@ +function calc_fifo_pnl { + # Initialize variables + local f_csv_file="$1" + local f_current_year=$(date +%Y) + declare -A f_holdings + local f_date f_action f_symbol f_crypto_amount f_fiat_currency f_fiat_amount f_exchange f_fee_currency f_fee_amount f_note f_fiat_amount_tax_currency + + # Read CSV file line by line + while IFS=',' read -r f_date f_action f_symbol f_crypto_amount f_fiat_currency f_fiat_amount f_exchange f_fee_currency f_fee_amount f_note + do + + ## Debug + #[ "$f_symbol" == "ETH" ] || continue + #[ "$f_note" == "short" ] && continue + + # ignore stable coins + [[ $f_symbol == USDT || $f_symbol == EUR ]] && continue + + # Extract year from date + local f_year=${f_date:0:4} + + ## Debug + #[ "$f_year" == "2024" ] || continue + #[ "$f_action" == "fundingfee" ] && continue + + # add exchange + [[ "$f_exchanges" != *"$f_exchange "* ]] && f_exchanges+="$f_exchange " + + # prevent exponential numbers + g_num_exponential2normal $f_crypto_amount + f_crypto_amount=$g_num_exponential2normal_result + g_num_exponential2normal $f_fiat_amount + f_fiat_amount=$g_num_exponential2normal_result + g_num_exponential2normal $f_fee_amount + f_fee_amount=$g_num_exponential2normal_result + + echo "f_fiat_amount=$f_fiat_amount" + + # convert f_fiat_currency/f_fiat_amount to TRANSFER_CURRENCY/f_fiat_amount_tax_currency if they are not equal + if ! [ "$f_fiat_currency" == "$TRANSFER_CURRENCY" ] && [ "$f_fiat_amount" != "0" ] + then + currency_converter $f_fiat_amount "$f_fiat_currency" $TRANSFER_CURRENCY "$f_date" >/dev/null + f_fiat_amount_tax_currency=$f_currency_converter_result + echo "currency_converter: $f_fiat_amount "$f_fiat_currency" $TRANSFER_CURRENCY "$f_date" -> $f_currency_converter_result" + else + f_fiat_amount_tax_currency=$f_fiat_amount + fi + + echo "f_fiat_amount_tax_currency=$f_fiat_amount_tax_currency" + + # convert f_fee_currency/f_fee_amount to TRANSFER_CURRENCY/f_fiat_amount_tax_currency if present + if [ -n "$f_fee_amount" ] + then + currency_converter $f_fee_amount $f_fee_currency $TRANSFER_CURRENCY "$f_date" >/dev/null + f_fee_amount=$f_currency_converter_result + [[ $f_action == "buy" || $f_action == "leverage-buy" ]] && g_calc "$f_fiat_amount_tax_currency + $f_fee_amount" + [[ $f_action == "sell" || $f_action == "leverage-sell" || $f_action == "liquidation" ]] && g_calc "$f_fiat_amount_tax_currency - $f_fee_amount" + f_fiat_amount_tax_currency=$g_calc_result + fi + + echo "f_fiat_amount_tax_currency=$f_fiat_amount_tax_currency" + + # no space in date (prevent problems mit $f_holdings) + f_date="${f_date/ /T}" + + # get current holfings for determining if this is a long or short trade (f_holdings_amount) + get_holdings_amount + + echo "f_crypto_amount=$f_crypto_amount" + + # Process fundingfee action + if [[ $f_action == "fundingfee" ]] + then + process_fundingfee "$f_symbol" "$f_crypto_amount" "$f_fee_amount" "$f_date" "$f_year" + # Process buy actions + elif [[ $f_action == "buy" || $f_action == "leverage-buy" || $f_action == "reward-staking" || $f_action == "giveaway" || $f_action == "instant_trade_bonus" ]] + then + if g_num_is_higher_equal $f_holdings_amount 0 + then + # long + echo process_buy "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" + process_buy "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" + elif g_num_is_lower_equal "$f_holdings_amount" "-$f_crypto_amount" + then + # short + echo process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" "$f_year" short + process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" "$f_year" short + else + # long+short (partial) + # calc long/short parts + g_percentage-diff "-$f_crypto_amount" "$f_holdings_amount" + g_calc "$f_fiat_amount_tax_currency/100*($g_percentage_diff_result+100)" + f_fiat_amount_tax_currency_long=$g_calc_result + g_calc "$f_fiat_amount_tax_currency/100*(($g_percentage_diff_result*-1))" + f_fiat_amount_tax_currency_short=$g_calc_result + g_calc "$f_crypto_amount/100*($g_percentage_diff_result+100)" + f_crypto_amount_long=$g_calc_result + g_calc "$f_crypto_amount/100*($g_percentage_diff_result*-1)" + f_crypto_amount_short=$g_calc_result + # part short-sell + # part long-sell + echo PART: process_sell process_sell "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + process_sell "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + + echo PART: process_buy "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + process_buy "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + fi + # Process sell actions + elif [[ $f_action == "sell" || $f_action == "leverage-sell" || $f_action == "liquidation" ]] + then + # check for long or short or log+short + if g_num_is_higher_equal "$f_holdings_amount" "$f_crypto_amount" + then + # long + echo process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" "$f_year" + process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" "$f_year" + elif g_num_is_higher "$f_holdings_amount" 0 + then + # long+short (partial) + # calc long/short parts + g_percentage-diff "$f_crypto_amount" "$f_holdings_amount" + g_calc "$f_fiat_amount_tax_currency/100*($g_percentage_diff_result+100)" + f_fiat_amount_tax_currency_long=$g_calc_result + g_calc "$f_fiat_amount_tax_currency/100*(($g_percentage_diff_result*-1))" + f_fiat_amount_tax_currency_short=$g_calc_result + g_calc "$f_crypto_amount/100*($g_percentage_diff_result+100)" + f_crypto_amount_long=$g_calc_result + g_calc "$f_crypto_amount/100*($g_percentage_diff_result*-1)" + f_crypto_amount_short=$g_calc_result + # part long-sell + echo PART: process_sell "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + process_sell "$f_symbol" "$f_crypto_amount_long" "$f_fiat_amount_tax_currency_long" "$f_date" "$f_year" + # part short-sell + echo PART: process_buy "$f_symbol" "$f_crypto_amount_short" "$f_fiat_amount_tax_currency_short" "$f_date" short + process_buy "$f_symbol" "$f_crypto_amount_short" "$f_fiat_amount_tax_currency_short" "$f_date" short + elif [[ $f_action == "liquidation" ]] + then + # short sell/liquidation + echo process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" $f_year short + process_sell "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" $f_year short + else + # short buy + echo process_buy "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" short + process_buy "$f_symbol" "$f_crypto_amount" "$f_fiat_amount_tax_currency" "$f_date" short + fi + fi + + ## DEBUG output + get_holdings_amount + echo "f_holdings_amount=$f_holdings_amount" + echo "============================" + + done < "$f_csv_file" +} + +function process_buy { + local f_symbol="$1" f_amount="$2" f_price="$3" f_date="$4" f_short="$5" + local f_tax_type f_trade_tax + # Add to holdings + # long + [ -z "$f_short" ] && f_holdings[$f_symbol]+="$f_amount:$f_price:$f_date " + # short + if [ -n "$f_short" ] + then + f_holdings[$f_symbol]+="-$f_amount:$f_price:$f_date " + f_action="${f_action}-short" + echo ACTION:$f_action + elif [[ $f_action == "reward-staking" ]] + then + f_tax_type="Sonst-Einkünfte-Staking" + elif [[ $f_action == "giveaway" ]] + then + f_tax_type="Sonst-Einkünfte-Giveaway" + elif [[ $f_action == "instant_trade_bonus" ]] + then + f_tax_type="Kapitalertrag-Instant-Trade-Bonus" + fi + + if [ -n "$f_tax_type" ] + then + f_trade_tax=$f_price + f_fiat_amount=0 + f_fiat_amount_tax_currency=0 + fi + + + echo "$f_date,$f_exchange,$f_action,$f_symbol,$f_amount,$f_fiat_currency,-$f_price,,,,,,$f_tax_type,$f_trade_tax,-$f_price,,,,,,,,," >>ALL_TRANSACTIONS_OVERVIEW.csv.tmp +} + +function process_sell { + local f_symbol="$1" f_sell_amount="$2" f_sell_price="$3" f_sell_date="$4" f_year="$5" f_short="$6" + f_remaining_sell=$f_sell_amount + local f_profit=0 f_loss=0 f_profit_tax=0 f_loss_tax=0 f_trade_tax=0 + local f_pnl + + # define tax type + local f_tax_type="Kapitalertrag-Derivat" + local f_trade_result + [[ $f_action == "sell" ]] && f_tax_type="Veräußerungsgeschäft" + local f_buy_amount f_buy_price f_buy_date + + # Process each holding using FIFO + while [[ $f_remaining_sell > 0 && -n "${f_holdings[$f_symbol]}" ]] + do + + IFS=':' read -r f_buy_amount f_buy_price f_buy_date < <(echo "${f_holdings[$f_symbol]%% *}") + + # Calculate amount to sell from this holding + f_sell_from_holding=$f_buy_amount + [ -n "$f_short" ] && f_remaining_sell=-${f_remaining_sell#-} + [ -z "$f_short" ] && g_num_is_lower $f_remaining_sell $f_buy_amount && f_sell_from_holding=$f_remaining_sell + [ -n "$f_short" ] && g_num_is_higher $f_remaining_sell $f_buy_amount && f_sell_from_holding=$f_remaining_sell + + # calculate sell percentage of buy trade + echo "f_sell_from_holding=$f_sell_from_holding" + [ -z "$f_short" ] && g_percentage-diff $f_buy_amount $f_sell_from_holding + [ -n "$f_short" ] && g_percentage-diff $f_buy_amount $f_sell_from_holding + g_calc "100+$g_percentage_diff_result" + f_percentage_of_buy=${g_calc_result#-} + + echo "f_percentage_of_buy=$f_percentage_of_buy" + + # Calculate profit/loss (pnl) + echo "f_sell_price=$f_sell_price" + echo "f_buy_price=$f_buy_price" + # if not first iteration (f_pnl is already set from previous iteration) and partial sell + if [ -n "$f_pnl" ] + then + # on multiple iteration partial sell + g_calc "$f_pnl - ($f_buy_price/100*$f_percentage_of_buy)" + else + # on first interation partial sell + g_calc "$f_sell_price - ($f_buy_price/100*$f_percentage_of_buy)" + fi + local f_pnl=$g_calc_result + echo "f_pnl=$f_pnl" + + # Check if trade is tax-free (held for more than a year) + local f_is_taxable=true + if [ "$f_tax_type" == "Veräußerungsgeschäft" ] + then + local f_days_held=$(( ($(date -d "$f_sell_date" +%s) - $(date -d "$f_buy_date" +%s)) / 86400 )) + [[ $f_days_held -gt 365 && ${f_tax_type} == "Veräußerungsgeschäft" ]] && f_is_taxable=false + fi + + # Update remaining sell amount and holdings + g_calc "$f_remaining_sell - $f_sell_from_holding" + f_remaining_sell=$g_calc_result + echo "f_remaining_sell=$f_remaining_sell" + echo "HOLDINGS1: ${f_holdings[$f_symbol]}" + f_holdings[$f_symbol]="${f_holdings[$f_symbol]#* }" + + # If there's remaining amount in the holding, add it back + [ -z "$f_short" ] && g_calc "$f_buy_amount - $f_sell_from_holding" + [ -n "$f_short" ] && g_calc "$f_buy_amount + $f_sell_from_holding" + g_calc "$f_buy_amount - $f_sell_from_holding" + local f_remaining_buy_amount=$g_calc_result + echo "f_remaining_buy_amount=$g_calc_result" + #if g_num_is_higher $f_remaining_buy_amount 0 + if [ "$f_remaining_buy_amount" != "0" ] + then + g_calc "$f_buy_price/100*(100-$f_percentage_of_buy)" + f_remaining_buy_price=$g_calc_result + f_holdings[$f_symbol]="$f_remaining_buy_amount:$f_remaining_buy_price:$f_buy_date ${f_holdings[$f_symbol]}" + fi + echo "HOLDINGS2: ${f_holdings[$f_symbol]}" + done + + # Update profit/loss + [ -n "$f_short" ] && g_calc "$f_pnl * -1" && f_pnl=$g_calc_result + if g_num_is_higher $f_pnl 0 + then + g_calc "$f_profit + $f_pnl" + f_profit=$g_calc_result + else + g_calc "$f_loss - $f_pnl" + f_loss=$g_calc_result + fi + + # calculate result of trade + g_calc "$f_profit - $f_loss" + f_trade_result=$g_calc_result + + # calculate taxable part of trade + if [[ $f_is_taxable == true ]] + then + g_calc "$f_trade_tax + $f_pnl" + f_trade_tax=$g_calc_result + fi + + ## DEBUG output + get_holdings_amount + echo "f_holdings_amount=$f_holdings_amount" + echo "Result: $f_trade_result ; taxable=$f_is_taxable ; REMAINING: $f_holdings_amount" + + # write to csv + if [ -n "$f_short" ] + then + f_action="${f_action}-short" + echo ACTION:$f_action + fi + [ "$f_trade_tax" == "0" ] && [ "$f_tax_type" == "Veräußerungsgeschäft" ] && f_tax_type="Veräußerungsgeschäft Spekulationsfrist > 1 Jahr" + echo "$f_date,$f_exchange,$f_action,$f_symbol,-$f_sell_amount,$f_fiat_currency,$f_sell_price,,,,,,$f_tax_type,$f_trade_tax,$f_sell_price,,$f_trade_result,,,,,,," >>ALL_TRANSACTIONS_OVERVIEW.csv.tmp + + [ -z "$f_trade_result" ] && g_echo_error "No trade result!!! Someting wrong $f_date,$f_symbol,$f_action $f_short" + +} + +function get_holdings_amount { + local block first_value + f_holdings_amount=0 + + # Durch jeden Block iterieren + IFS=" " + for block in ${f_holdings[$f_symbol]} + do + IFS=$origIFS + # Den ersten Wert vor dem Doppelpunkt extrahieren + first_value=${block%%:*} + + # Zum Gesamtwert addieren + g_calc "$f_holdings_amount + $first_value" + f_holdings_amount=$g_calc_result + done + IFS=$origIFS +} + +function process_fundingfee { + local f_symbol="$1" f_amount="$2" f_fiat_amount_tax_currency="$3" f_date="$4" f_year="$5" + + echo "adding fundingfee: $f_fiat_amount_tax_currency" + + ## add fundingfee + [[ $f_fiat_amount_tax_currency == -* ]] && f_tax="${f_fiat_amount_tax_currency#-}" + [[ $f_fiat_amount_tax_currency == -* ]] || f_tax="-${f_fiat_amount_tax_currency}" + + echo "$f_date,$f_exchange,$f_action,$f_symbol,$f_amount,,,,,,,,Kapitalertrag-Derivat,$f_tax,$f_tax,,$f_tax,,,,,,," >>ALL_TRANSACTIONS_OVERVIEW.csv.tmp +} + +function transaction_csv_validity_ckecks { + local f_buy f_sell f_liquidation f_liquidation_short + local f_complete_result=0 + declare -A transaction_csv_validity_ckeck_buy_sell_diff + + f_symbols=$(cut -d, -f3 $f_csv_file | sort -u) + local f_buy_amount f_sell_amount f_tax_type + + # go through symbols and male some pre-checks + for f_symbol in $f_symbols + do + + ## check asset amount + g_echo_note "Initial checks for $f_symbol" + # add all buys and sells of a symbols amount + f_buy=$(\ + egrep "buy,${f_symbol},|,reward-staking,${f_symbol}|,giveaway,${f_symbol},instant_trade_bonus,${f_symbol}" "$f_csv_file" | \ + cut -d, -f4 | \ + awk '{ SUM += $1} END { printf("%.12f\n", SUM) }' \ + ) + f_sell=$(\ + egrep "sell,${f_symbol}," "$f_csv_file" | \ + cut -d, -f4 | \ + awk '{ SUM += $1} END { printf("%.12f\n", SUM) }' \ + ) + f_liquidation=$(\ + egrep "liquidation,${f_symbol}," "$f_csv_file" | \ + grep -v ",short" | \ + cut -d, -f4 | \ + awk '{ SUM += $1} END { printf("%.12f\n", SUM) }' \ + ) + f_liquidation_short=$(\ + egrep "liquidation,${f_symbol},.+,short" "$f_csv_file" | \ + cut -d, -f4 | \ + awk '{ SUM += $1} END { printf("%.12f\n", SUM) }' \ + ) + + # add liquidations to sell + # long + g_calc "$f_sell + $f_liquidation - $f_liquidation_short" + f_sell=$g_calc_result + + # buy should be same as sell sum to be fine - if not: + g_calc "$f_buy == $f_sell" + if ! [[ $g_calc_result == 1 ]] + then + g_echo_note "buy ($f_buy) and sell ($f_sell) amount sums are different for ${f_symbol}. Open Positions!?" + g_calc "$f_sell - ($f_buy)" + transaction_csv_validity_ckecks[$f_symbol]=$g_calc_result + else + transaction_csv_validity_ckeck_buy_sell_diff[$f_symbol]=0 + fi + + done +} + +function print_results { + + local f_csv=ALL_TRANSACTIONS_OVERVIEW.csv + local f_exchange_symbol f_exchange_symbol_year_tax f_amount f_result + #transaction_csv_validity_ckecks + + echo "" + echo "Open Positions:" + echo "===============" + + local f_exchanges_symbols=$(cut -d, -f 2,4 "$f_csv" | sort -u) + for f_exchange_symbol in $f_exchanges_symbols + do + f_exchange=${f_exchange_symbol%%,*} + f_symbol=${f_exchange_symbol#*,} + f_amount=$(\ + egrep ",${f_exchange},.+,$f_symbol" "$f_csv" | \ + cut -d, -f5 | \ + awk '{ SUM += $1} END { printf("%.12f\n", SUM) }' \ + ) + + f_result=$(\ + egrep ",${f_exchange},.*sell,$f_symbol|,${f_exchange},.*buy,$f_symbol" "$f_csv" | \ + #egrep "${f_exchange},.+,$f_symbol" "$f_csv" \ + cut -d, -f17 | \ + awk '{ SUM += $1} END { printf("%.2f\n", SUM) }' \ + ) + + g_calc "$f_amount == 0" + if ! [[ $g_calc_result == 1 ]] + then + echo "$f_exchange/$f_symbol: $f_amount" + fi + + done + + echo "" + echo "Profit and Loss (Tax):" + echo "======================" + + declare -A f_taxes f_pnls + local f_total_tax + local f_exchanges_symbols_years_tax=$(sed 's/-/,/' "$f_csv" | cut -d, -f 1,3,5,14 | sort -u) + for f_exchange_symbol_year_tax in $f_exchanges_symbols_years_tax + do + IFS=',' read -r f_year f_exchange f_symbol f_tax_type < <(echo "$f_exchange_symbol_year_tax") + [ -z "$f_tax_type" ] && continue + f_tax=$(\ + egrep "^$f_year-.+,${f_exchange},.+,${f_symbol},.+,$f_tax_type" "$f_csv" | \ + cut -d, -f14 | \ + awk '{ SUM += $1} END { printf("%.2f\n", SUM) }' \ + ) + f_pnl=$(\ + egrep "^$f_year-.+,${f_exchange},.+,${f_symbol},.+,$f_tax_type" "$f_csv" | \ + cut -d, -f17 | \ + awk '{ SUM += $1} END { printf("%.2f\n", SUM) }' \ + ) + echo "$f_year/$f_exchange/$f_symbol/$f_tax_type: $f_tax $TRANSFER_CURRENCY" + [ -z "${f_taxes[${f_year}_${f_exchange}_${f_tax_type}]}" ] && f_taxes[${f_year}_${f_exchange}_${f_tax_type}]=0 + [ -z "${f_pnls[${f_year}_${f_exchange}]}" ] && f_pnls[${f_year}_${f_exchange}]=0 + g_calc "${f_taxes[${f_year}_${f_exchange}_${f_tax_type}]} + ($f_tax)" + f_taxes[${f_year}_${f_exchange}_${f_tax_type}]=$g_calc_result + g_calc "${f_pnls[${f_year}_${f_exchange}]} + ($f_pnl)" + f_pnls[${f_year}_${f_exchange}]=$g_calc_result + + done + + echo "" + echo "Profit and Loss (Tax per exchange):" + echo "===================================" + + for f_tax_year in "${!f_taxes[@]}" + do + echo "$f_tax_year: ${f_taxes[$f_tax_year]} $TRANSFER_CURRENCY" + done | sort + + echo "" + echo "Profit and Loss:" + echo "================" + + for f_pnl_year in "${!f_pnls[@]}" + do + echo "$f_pnl_year: ${f_pnls[$f_pnl_year]} $TRANSFER_CURRENCY" + done | sort + +} + + diff --git a/dabo/functions/get_marketdata.sh b/dabo/functions/get_marketdata.sh index c420517..c870b6b 100644 --- a/dabo/functions/get_marketdata.sh +++ b/dabo/functions/get_marketdata.sh @@ -26,9 +26,15 @@ function get_marketdata_all { # FEAR_AND_GREED_ALTERNATIVEME get_marketdata FEAR_AND_GREED_ALTERNATIVEME 'https://api.alternative.me/fng/?limit=0&format=json' '.data[] | (.timestamp | tonumber | strftime("%Y-%m-%d")) + "," + .value + ",,,,0"' "" 1d + # FEAR AND GREED COINMARKETCAP + get_marketdata FEAR_AND_GREED_COINMARKETCAP "https://api.coinmarketcap.com/data-api/v3/fear-greed/chart?start=1&end=$(date +%s)" '.data.dataList[] | (.timestamp | tonumber | strftime("%Y-%m-%d")) + "," + (.score|tostring) + ",,,,0"' + # FEAR_AND_GREED_CNN get_marketdata FEAR_AND_GREED_CNN 'https://production.dataviz.cnn.io/index/fearandgreed/graphdata' '.fear_and_greed_historical.data[] | (.x/1000 | strftime("%Y-%m-%d")) + "," + (.y|tostring) + ",,,,0"' "" 1d + # Altcoin-Saison-Index COINMARKETCAP Top 100 Altcoins + get_marketdata ALTCOIN_SEASON_INDEX_COINMARKETCAP "https://api.coinmarketcap.com/data-api/v3/altcoin-season/chart?start=1&end=$(date +%s)" '.data.points[:-1][] | (.timestamp | tonumber | strftime("%Y-%m-%d")) + "," + (.altcoinIndex|tostring) + ",,,,0"' "" 1d + # monthly US consumer price index CPI data get_marketdata US_CONSUMER_PRICE_INDEX_CPI "https://api.bls.gov/publicAPI/v2/timeseries/data/CUUR0000SA0?startyear=$(date -d 'now -8 years' '+%Y')&endyear=$(date '+%Y')" '.Results.series[0].data[] | .year + "-" + (.period | gsub("M"; "")) + "-01," + .value + ",,,,0"' "" 1d diff --git a/dabo/functions/get_phemex_csv_transactions.sh b/dabo/functions/get_phemex_csv_transactions.sh new file mode 100644 index 0000000..9543e00 --- /dev/null +++ b/dabo/functions/get_phemex_csv_transactions.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright (c) 2022-2024 olli +# +# This file is part of dabo (crypto bot). +# +# dabo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# dabo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with dabo. If not, see . + + +function get_phemex_csv_transactions { + + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + # PHEMEX Export format: + # Time (UTC),Symbol,Exec Type,Exec. Size,Direction,Exec. Price,Order Size,Order Price,Exec Value,Fee Rate,Fee Paid,Type,"ID" + + if [ -s phemex-export.csv ] + then + # explicit long + cat phemex-export.csv | egrep '^[0-9].+,Trade,.+ Long,' | sort | sed 's/,Long,/,Open Long,/; s/Open Long/leverage-buy/; s/Close Long/leverage-sell/; s/ /,/g' | awk -F, '{print $1" "$2","$7","$6","$5","$13","$12",phemex,"$16","$15","$18}' >TRANSACTIONS-phemex-LONG.csv.tmp + # explicit short + cat phemex-export.csv | egrep '^[0-9].+,Trade,.+ Short,' | sort | sed 's/,Short,/,Open Short,/; s/Open Short/leverage-sell/; s/Close Short/leverage-buy/; s/ /,/g' | awk -F, '{print $1" "$2","$7","$6","$5","$13","$12",phemex,"$16","$15","$18}' >TRANSACTIONS-phemex-SHORT.csv.tmp + # buy/sell + cat phemex-export.csv | egrep '^[0-9].+,Trade,.+,Short,|^[0-9].+,Trade,.+,Long,' | sort | sed 's/Short/leverage-sell/; s/Long/leverage-buy/; s/ /,/g' | awk -F, '{print $1" "$2","$7","$6","$5","$13","$12",phemex,"$16","$15","$18}' >TRANSACTIONS-phemex.csv.tmp + # liquidations long + cat phemex-export.csv | egrep '^[0-9].+,Liquidation,.+Long,' | sort | sed 's/ /,/g' | awk -F, '{print $1" "$2",liquidation,"$6","$5","$14","$13",phemex,"$17","$16","$19}' >TRANSACTIONS-phemex-liquidations-LONG.csv.tmp + # liquidations short + cat phemex-export.csv | egrep '^[0-9].+,Liquidation,.+Short,' | sort | sed 's/ /,/g' | awk -F, '{print $1" "$2",liquidation,"$6","$5","$14","$13",phemex,"$17","$16","$19}' >TRANSACTIONS-phemex-liquidations-SHORT.csv.tmp + # fundingfees seeem to be included in sell + cat TRANSACTIONS-phemex/*.csv | egrep 'funding' | sort -u >>TRANSACTIONS-phemex.csv.tmp + + # put together + cat TRANSACTIONS-phemex-LONG.csv.tmp TRANSACTIONS-phemex.csv.tmp TRANSACTIONS-phemex-liquidations-LONG.csv.tmp | sort >TRANSACTIONS-phemex.csv + cat TRANSACTIONS-phemex-SHORT.csv.tmp TRANSACTIONS-phemex-liquidations-SHORT.csv.tmp | sort >>TRANSACTIONS-phemex.csv + # cleanup + rm -f TRANSACTIONS-phemex-LONG.csv.tmp TRANSACTIONS-phemex-SHORT.csv.tmp TRANSACTIONS-phemex.csv.tmp TRANSACTIONS-phemex-liquidations-LONG.csv.tmp TRANSACTIONS-phemex-liquidations-SHORT.csv.tmp + fi +} + diff --git a/dabo/functions/get_transactions.sh b/dabo/functions/get_transactions.sh index b22ad1d..bee176a 100644 --- a/dabo/functions/get_transactions.sh +++ b/dabo/functions/get_transactions.sh @@ -113,7 +113,7 @@ function get_transactions { f_ccxt "print(${STOCK_EXCHANGE}.fetchFundingHistory('$f_symbol', limit=200, params={'paginate': True}))" && echo -n $f_ccxt_result >"${f_symbol_file}.FundingFees" cat ${f_symbol_file}.FundingFees | jq -r " .[] | -.datetime + \",fundingfee,$f_asset,0,\" + .code + \",0\" + \",$f_exchange,\" + .code + \",\" + (.amount|tostring) +.datetime + \",fundingfee,$f_asset,0,\" + .code + \",0\" + \",$f_exchange,\" + .code + \",\" + (.amount|tostring) " >>$f_symbol_file_csv_tmp # remove the ':' in f_currency @@ -127,46 +127,26 @@ function get_transactions { cat "$f_symbol_file" | jq -r " .[] | select(.side==\"buy\" or .side==\"sell\") | - select(.info.posSide=null) | select(.symbol!= null) | -.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) -" >>"$f_symbol_file_csv_tmp" - - # get longs (posSide=="1") - cat "$f_symbol_file" | jq -r " -.[] | - select(.side==\"buy\" or .side==\"sell\") | - select(.info.posSide=\"1\") | - select(.symbol!= null) | -.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) -" >>"$f_symbol_file_csv_tmp" - - # get shorts (posSide=="2") sell first, then buy (https://github.com/ccxt/ccxt/issues/22518) - cat "$f_symbol_file" | jq -r " -.[] | - select(.side==\"buy\" or .side==\"sell\") | - select(.info.posSide==\"2\") | - select(.symbol!=null) | -.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",short\" +.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",\" + .id " >>"$f_symbol_file_csv_tmp" +# # get longs (posSide=="1") +# cat "$f_symbol_file" | jq -r " +#.[] | +# select(.side==\"buy\" or .side==\"sell\") | +# select(.info.posSide=\"1\") | +# select(.symbol!= null) | +#.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",\" + .id +#" >>"$f_symbol_file_csv_tmp" +# # # get shorts (posSide=="2") sell first, then buy (https://github.com/ccxt/ccxt/issues/22518) # cat "$f_symbol_file" | jq -r " #.[] | -# select(.side==\"sell\") | +# select(.side==\"buy\" or .side==\"sell\") | # select(.info.posSide==\"2\") | # select(.symbol!=null) | -# .side = \"sell\" | -#.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",short\" -#" >>"$f_symbol_file_csv_tmp" -# -# cat "$f_symbol_file" | jq -r " -#.[] | -# select(.side==\"buy\") | -# select(.info.posSide==\"2\") | -# select(.symbol!=null) | -# .side = \"buy\" | -#.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",short\" +#.datetime + \",$f_leverage\" + .side + \",$f_asset,\" + (.amount|tostring) + \",$f_currency,\" + (.cost|tostring) + \",$f_exchange,\" + .fee.currency + \",\" + (.fee.cost|tostring) + \",\" + .id #" >>"$f_symbol_file_csv_tmp" if [ -s "$f_symbol_file_csv_tmp" ] @@ -226,14 +206,22 @@ function get_transactions { done fi - # put all sorted n one file - if [ -s TRANSACTIONS-$f_exchange.csv ] - then - cat "TRANSACTIONS-$f_exchange/"*.csv TRANSACTIONS-$f_exchange.csv | sort -u >TRANSACTIONS-$f_exchange.csv.tmp - mv TRANSACTIONS-$f_exchange.csv.tmp TRANSACTIONS-$f_exchange.csv - else - cat "TRANSACTIONS-$f_exchange/"*.csv | sort -u >TRANSACTIONS-$f_exchange.csv - fi + # put all sorted n one file by id + local f_line f_id + touch "TRANSACTIONS-$f_exchange.csv" + cat "TRANSACTIONS-$f_exchange/"*.csv | while read f_line + do + echo $f_line + f_id=$(echo "$line" | cut -d, -f10) + grep -q "$f_id" "TRANSACTIONS-$f_exchange.csv" || echo $f_line >>"TRANSACTIONS-$f_exchange.csv" + done + #if [ -s TRANSACTIONS-$f_exchange.csv ] + #then + # cat "TRANSACTIONS-$f_exchange/"*.csv TRANSACTIONS-$f_exchange.csv | sort -u >TRANSACTIONS-$f_exchange.csv.tmp + # mv TRANSACTIONS-$f_exchange.csv.tmp TRANSACTIONS-$f_exchange.csv + #else + # cat "TRANSACTIONS-$f_exchange/"*.csv | sort -u >TRANSACTIONS-$f_exchange.csv + #fi # Switch sides if Fiat is in Krypto side f_fiats="USD EUR" @@ -247,7 +235,7 @@ function get_transactions { g_echo_note "Switched some fiat/krypto sides" #cat TRANSACTIONS-$f_exchange.csv.tmp cat TRANSACTIONS-$f_exchange.csv | egrep -v ",sell,$f_fiat,|,buy,$f_fiat," >>TRANSACTIONS-$f_exchange.csv.tmp - cat TRANSACTIONS-$f_exchange.csv.tmp | sort >TRANSACTIONS-$f_exchange.csv + cat TRANSACTIONS-$f_exchange.csv.tmp >TRANSACTIONS-$f_exchange.csv fi done diff --git a/dabo/functions/transactions_overview.sh b/dabo/functions/transactions_overview.sh index 62b55e1..ddde4d8 100644 --- a/dabo/functions/transactions_overview.sh +++ b/dabo/functions/transactions_overview.sh @@ -27,6 +27,7 @@ function transactions_overview { get_bitpanda_api_transactions get_justtrade_csv_transactions get_onetrading_csv_transactions + get_phemex_csv_transactions >ALL_TRANSACTIONS_OVERVIEW.csv.tmp >ALL_TRANSACTIONS_OVERVIEW_WARN.csv.tmp @@ -34,7 +35,7 @@ function transactions_overview { local f_exchange f_asset f_transactions_array f_transaction f_result f_asset_quantity f_asset_quantity_sold f_currency_quantity f_currency_quantity_sold f_currency_spent f_date f_type f_asset_amount f_currency f_currency_amount f_fee_currency f_fee_amount f_sell_result f_taxable f_tax_type f_one_year_ago f_currency_amount_eur f_currency_spent_eur f_currency_quantity_sold_eur f_note f_asset_quantity_remaining f_currency_remaining f_year f_currency_spent_eur_tax f_currency_quantity_sold_eur_tax f_sell_result_percentage f_sell_result_percentage_eur - f_assets_per_exchange=$(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$' TRANSACTIONS-*.csv | cut -d, -f3,7 | sort -u) + f_assets_per_exchange=$(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$' TRANSACTIONS-*.csv | cut -d, -f3,7 | sort -u | grep JustTrade) for f_asset_per_exchange in ${f_assets_per_exchange} do @@ -70,6 +71,7 @@ function transactions_overview { mapfile -t f_transactions_array < <(egrep -h -v '^DATE,TYPE,ASSET,ASSET_AMOUNT,CURRENCY,CURRENCY_AMOUNT,EXCHANGE|^#|^$|^ +$' TRANSACTIONS-*.csv | sort -u | egrep ",${f_asset},.+,.+,${f_exchange}" | sort) for f_transaction in "${f_transactions_array[@]}" do + g_echo_note "Transaction: $f_transaction" mapfile -d, -t f_transaction_array < <(echo $f_transaction) f_date=${f_transaction_array[0]} f_type=${f_transaction_array[1]} @@ -211,7 +213,7 @@ function transactions_overview { echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable,$f_currency_amount_eur,$f_result_eur,$f_sell_result_eur,$f_asset_quantity_remaining,$f_note,Sell never buyed!? Spent currency on $f_asset is 0" 1>&2 continue fi - # if sell wahats not exists!? + # if sell what not exists!? if [ $f_asset_quantity = 0 ] then #g_echo_warn "!!!!!! Sell never buyed!? Buyed asset $f_asset is 0" @@ -239,6 +241,7 @@ function transactions_overview { ## Check for ended trade (asset-quantity=0 or tttt) # if all is sold trade ended and calculate PNL local f_trade_end=0 + local f_trade_partial_end=0 local f_dust=0 local f_dust_eur=0 g_calc "$f_asset_quantity_sold==$f_asset_quantity" @@ -274,10 +277,11 @@ function transactions_overview { else g_echo_note "Tade not closed - partial sale!? Remaining $f_asset_quantity_remaining $f_asset ($f_currency_remaining $f_currency)!?" >>ALL_TRANSACTIONS_OVERVIEW.log f_note="Trade not closed - partial sale. Remaining $f_asset_quantity_remaining $f_asset ($f_currency_remaining $f_currency)" + f_trade_partial_end=1 fi fi - if [ ${f_trade_end} -eq 1 ] + if [ ${f_trade_end} -eq 1 ] || [ ${f_trade_partial_end} -eq 1 ] then echo "Buy price: $f_currency_spent $f_currency ($f_currency_spent_eur EUR)" >>ALL_TRANSACTIONS_OVERVIEW.log @@ -303,17 +307,20 @@ function transactions_overview { # calculate complete result EUR g_calc "$f_result_eur+($f_sell_result_eur)" f_result_eur=$g_calc_result - - # reset vars - f_asset_quantity=0 - f_asset_quantity_sold=0 - f_asset_quantity_remaining=0 - f_currency_spent=0 - printf -v f_currency_spent_eur_tax %.2f $f_currency_spent_eur - f_currency_spent_eur=0 - f_currency_quantity_sold=0 - printf -v f_currency_quantity_sold_eur_tax %.2f $f_currency_quantity_sold_eur - f_currency_quantity_sold_eur=0 + + if [ ${f_trade_end} -eq 1 ] + then + # reset vars + f_asset_quantity=0 + f_asset_quantity_sold=0 + f_asset_quantity_remaining=0 + f_currency_spent=0 + printf -v f_currency_spent_eur_tax %.2f $f_currency_spent_eur + f_currency_spent_eur=0 + f_currency_quantity_sold=0 + printf -v f_currency_quantity_sold_eur_tax %.2f $f_currency_quantity_sold_eur + f_currency_quantity_sold_eur=0 + fi fi # at leverage always full taxable @@ -374,6 +381,7 @@ function transactions_overview { ## completely tax free if over 1 year f_taxable=0 f_one_year_ago="yes" + f_tax_type="Kauf >1 Jahr zurück" # reduce tax free volume g_calc "$f_asset_amount_tax_free-$f_asset_amount" f_asset_amount_tax_free=$g_calc_result @@ -437,6 +445,8 @@ function transactions_overview { # 24 f_sell_result_percentage_eur echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable,$f_currency_amount_eur,$f_result_eur,$f_sell_result_eur,$f_asset_quantity_remaining,$f_note,$f_currency_spent_eur,$f_currency_quantity_sold,$f_currency_quantity_sold_eur,$f_sell_result_percentage,$f_sell_result_percentage_eur" | tee -a ALL_TRANSACTIONS_OVERVIEW.csv.tmp >>ALL_TRANSACTIONS_OVERVIEW.log + echo "$f_date,$f_exchange,$f_type,$f_asset,$f_asset_amount,$f_currency,$f_currency_amount,$f_one_year_ago,$f_currency_spent,$f_asset_quantity,$f_result,$f_sell_result,$f_tax_type,$f_taxable,$f_currency_amount_eur,$f_result_eur,$f_sell_result_eur,$f_asset_quantity_remaining,$f_note,$f_currency_spent_eur,$f_currency_quantity_sold,$f_currency_quantity_sold_eur,$f_sell_result_percentage,$f_sell_result_percentage_eur" + if [[ $f_type =~ sell|leverage-sell ]] then echo -e "\n" >>ALL_TRANSACTIONS_OVERVIEW.log diff --git a/dabo/functions/webpage_transactions.sh b/dabo/functions/webpage_transactions.sh index 346d34f..7631988 100644 --- a/dabo/functions/webpage_transactions.sh +++ b/dabo/functions/webpage_transactions.sh @@ -27,7 +27,7 @@ function webpage_transactions { #echo -e "\n\n========== Total Results ===========" - ##echo "Trade Result: $f_trade_result EUR" + #echo "Trade Result: $f_trade_result EUR" #echo "Staking Result: $f_staking_rewards EUR" #echo "Giveaway Result: $f_giveaway EUR" #echo -e "Instand Trade Bonus: $f_instant_trade_bonus EUR\n" @@ -41,6 +41,7 @@ function webpage_transactions { local f_exchange_tax_type cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^$f_tax_year-" | cut -d, -f 2,13 | sort -u | egrep -v ',$' | grep -v "Note: " | while read f_exchange_tax_type do + #echo "$f_exchange_tax_type" local f_exchange=$(echo $f_exchange_tax_type | cut -d, -f1) local f_tax_type=$(echo $f_exchange_tax_type | cut -d, -f2) @@ -70,14 +71,14 @@ $(cat ${g_tmp}/tax_summary_$f_exchange-$f_tax_year) DateType of transactionCrypto valueFiat valueResultTax typeTax amount " >TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp - cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^${f_tax_year}-" | grep ",${f_exchange}," | awk -F, ' -{printf ""$1""$3""$5" "$4""} -{printf("%.2f", $15)} -{printf " EUR "} -{printf("%.2f", $17)} -{printf " EUR"$13""} -{printf("%.2f", $14)} -{print " EUR"}' >>TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp +cat ALL_TRANSACTIONS_OVERVIEW.csv | grep "^${f_tax_year}-" | grep ",${f_exchange}," | awk -F, ' +{ + printf "%s%s%s %s%.2f EUR", $1, $3, $5, $4, $15 + if ($17 != "") { + printf "%.2f EUR", $17 + } + printf "%s%.2f EUR\n", $13, $14 +}' >>TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp echo "" >>TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp mv TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html.tmp ../TRANSACTIONS_OVERVIEW-${f_exchange}-${f_tax_year}.html @@ -85,7 +86,7 @@ $(cat ${g_tmp}/tax_summary_$f_exchange-$f_tax_year) done - echo "" + #echo "" done }