From c9cc555b4226dcb1bca604f4bcd3cc845af61192 Mon Sep 17 00:00:00 2001 From: olli Date: Tue, 5 Nov 2024 12:45:16 +0100 Subject: [PATCH] cleanup marketdata download: FEAR_AND_GREED_ALTERNATIVEME FEAR_AND_GREED_CNN US_CONSUMER_PRICE_INDEX_CPI US_UNEMPLOYMENT_RATE US_FED_FUNDS_RATE BINANCE_LONG_SHORT_RATIO_ACCOUNT BINANCE_LONG_SHORT_RATIO_TAKER BINANCE_OPEN_INTEREST --- dabo/functions/get_marketdata.sh | 191 +++++++++++++++++++------------ 1 file changed, 119 insertions(+), 72 deletions(-) diff --git a/dabo/functions/get_marketdata.sh b/dabo/functions/get_marketdata.sh index 596cac1..1568560 100644 --- a/dabo/functions/get_marketdata.sh +++ b/dabo/functions/get_marketdata.sh @@ -17,95 +17,142 @@ # You should have received a copy of the GNU General Public License # along with dabo. If not, see . +function get_marketdata_all { + local f_interval=$1 + + # daily garketdata jobs + if [[ $f_interval = 1d ]] + then + # FEAR_AND_GREED_ALTERNATIVEME + get_marketdata FEAR_AND_GREED_ALTERNATIVEME 'https://api.alternative.me/fng/?limit=0&format=json' '.data[] | (.timestamp | tonumber | strftime("%Y-%m-%d")) + "," + .value + ",,,,0"' "" 1d + + # FEAR_AND_GREED_CNN + get_marketdata FEAR_AND_GREED_CNN 'https://production.dataviz.cnn.io/index/fearandgreed/graphdata' '.fear_and_greed_historical.data[] | (.x/1000 | strftime("%Y-%m-%d")) + "," + (.y|tostring) + ",,,,0"' "" 1d + + # monthly US consumer price index CPI data + get_marketdata US_CONSUMER_PRICE_INDEX_CPI "https://api.bls.gov/publicAPI/v2/timeseries/data/CUUR0000SA0?startyear=$(date -d 'now -8 years' '+%Y')&endyear=$(date '+%Y')" '.Results.series[0].data[] | .year + "-" + (.period | gsub("M"; "")) + "-01," + .value + ",,,,0"' "" 1d + + # monthly US unemployment rate + get_marketdata US_UNEMPLOYMENT_RATE "https://api.bls.gov/publicAPI/v2/timeseries/data/LNU03000000?startyear=$(date -d 'now -8 years' '+%Y')&endyear=$(date '+%Y')" '.Results.series[0].data[] | .year + "-" + (.period | gsub("M"; "")) + "-01," + .value + ",,,,0"' "" 1d + + # US FED funds rate + get_marketdata US_FED_FUNDS_RATE 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=DFF' "" "" 1d + + fi + + # Binance Long Short Ration Account / Taker and Open Interest per symbol + get_symbols_ticker + local f_symbol f_asset f_time + for f_symbol in BTC/$CURRENCY "${f_symbols_array_trade[@]}" + do + f_asset=${f_symbol//:$CURRENCY/} + f_asset=${f_asset//\//} + + # week not available + [[ $f_interval = 1w ]] && continue + + f_time='%Y-%m-%d %H:%M:00' + [[ $f_interval = 1d ]] && f_time='%Y-%m-%d' + + # BINANCE_LONG_SHORT_RATIO_ACCOUNT per symbol + get_marketdata BINANCE_LONG_SHORT_RATIO_ACCOUNT_$f_asset "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .longShortRatio + \",,,,0\"" "" ${f_interval} + + # BINANCE_LONG_SHORT_RATIO_Taker per symbol + get_marketdata BINANCE_LONG_SHORT_RATIO_TAKER_$f_asset "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .buySellRatio + \",,,,0\"" "" ${f_interval} + + # BINANCE_OPEN_INTEREST per symbol + get_marketdata BINANCE_OPEN_INTEREST_$f_asset "https://fapi.binance.com/futures/data/openInterestHist?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .sumOpenInterest + \",,,,0\"" "" ${f_interval} + + done +} + function get_marketdata { - + g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + local f_name=$1 + local f_wget=$2 + local f_jq=$3 + local f_other=$4 + local f_timeframe=$5 + [ -z "$f_timeframe" ] && f_timeframe=1d + local f_histfile="asset-histories/MARKETDATA_${f_name}.history.1d.csv" + local f_dataline f_failed -### CPI Data: curl hapi.bls.gov/publicAPI/v2/timeseries/data/cpi | jq -a -### release dates: https://www.bls.gov/schedule/news_release/bls.ics - - get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX - get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX - get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX - get_marketdata_from_url '"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=5m" | jq -r .[].longShortRatio | tail -n1' BINANCE-BTC-GlobalLongShortAccountRatio - get_marketdata_from_url '"https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=5m" | jq -r .[].buySellRatio | tail -n1' BINANCE-BTC-TakerLongShortRatio - get_marketdata_from_url '"https://fapi.binance.com/futures/data/openInterestHist?symbol=BTCUSDT&period=5m" | jq -r .[].sumOpenInterest | tail -n1' BINANCE-BTC-OpenInterest - get_marketdata_from_url '"https://api.alternative.me/fng/?limit=1&format=json" | jq -r .data[].value' CRYPTO_FEAR_AND_GREED-INDEX - # clear old stuff - find asset-histories/*INDEX* -type f -mtime +6 -delete -} - -function get_marketdata_from_url { - g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" - - local f_url="$1" - local f_name="$2" - - ## get data for analysis - if find asset-histories/${f_name}.history.csv -mmin -${INTERVAL_MIN} 2>/dev/null | grep -q "asset-histories/${f_name}.history.csv" + # download + g_wget -O "${f_histfile}.wget.tmp" $f_wget 2>"${f_histfile}.err.tmp" || f_failed=wget + [ -s "${f_histfile}.wget.tmp" ] || f_failed=wget + if [ -n "$f_failed" ] then - g_echo_note "asset-histories/${f_name}.history.csv already downloaded in the last ${INTERVAL_MIN} minutes" - f_get_marketdata_price=$(cat MARKET_DATA_CMD_OUT-${f_name}) - return 0 + echo "g_wget -O \"${f_histfile}.wget.tmp\" $f_wget 2>\"${f_histfile}.err\"" >"${f_histfile}.err" fi - # check source platform for parsing parameters, prepare and run wget command - >MARKET_DATA_CMD - if echo "${f_url}" | grep -q "boerse.de" + # jd + if [ -z "$f_failed" ] && [ -n "$f_jq" ] then - echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'itemprop=\"price\" content=\"[0-9]+\.[0-9]+\"' | sed s'#\"#\n#g' | egrep '^[0-9]+\.[0-9]+'" >MARKET_DATA_CMD - elif echo "${f_url}" | egrep -q "investing.com.+economic-calendar" - then - echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'Actual.+Forecast.+Previous' | cut -d'>' -f7,11 | cut -d'<' -f1,2 | sed 's#,##g' | sed 's/\%//g' | sed 's##,#' | grep '[0-9]' | cut -d, -f1" >MARKET_DATA_CMD - elif echo "${f_url}" | egrep -q "investing.com.+indices" - then - echo "wget ${g_wget_opts} -q -O - ${f_url} | sed 's##\n#g' | grep 'text-5xl.*font-bold.*md:text-' | sed 's#^.*>##; s#,##g' | grep '[0-9]'" >MARKET_DATA_CMD - elif echo "${f_url}" | egrep -q '^"https://' - then - echo "wget -q -O - ${g_wget_opts} ${f_url}" >MARKET_DATA_CMD - else - # default to Yahoo Finace Symbols via API - echo "wget ${g_wget_opts} -q -O - https://query1.finance.yahoo.com/v8/finance/chart/${f_url} | jq -r '.[].result[].meta.regularMarketPrice'" >MARKET_DATA_CMD - fi - - g_runcmd g_retrycmd sh MARKET_DATA_CMD >MARKET_DATA_CMD_OUT-${f_name}.tmp 2>MARKET_DATA_CMD_OUT-${f_name}.tmp.err - - # check output - local f_get_marketdata_price_tmp=$(cat MARKET_DATA_CMD_OUT-${f_name}.tmp) - if g_num_valid_number ${f_get_marketdata_price_tmp} - then - if egrep -q "^0\.00" MARKET_DATA_CMD_OUT-${f_name}.tmp + if ! jq -r "$f_jq" "${f_histfile}.wget.tmp" >"${f_histfile}.tmp" 2>"${f_histfile}.err.tmp" then - g_echo_note "Ignoring ${f_name} $(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp) - maybe out of business day" + echo jq -r "$f_jq" "${f_histfile}.wget.tmp" >"${f_histfile}.err" + f_failed=jq else - g_echo_note "${f_name}: ${f_get_marketdata_price_tmp}" - mv MARKET_DATA_CMD_OUT-${f_name}.tmp MARKET_DATA_CMD_OUT-${f_name} + mv "${f_histfile}.tmp" "${f_histfile}.wget.tmp" + fi + fi + + # other/additional processing + if [ -z "$f_failed" ] && [ -n "$f_other" ] + then + if ! cat "${f_histfile}.wget.tmp" | eval $f_other + then + echo "cat \"${f_histfile}.wget.tmp\" | $f_other" >"${f_histfile}.err" + f_failed=other fi else - g_echo_warn "MARKET_DATA_CMD_OUT-${f_name}.tmp has wrong Syntax. - Not updating ${f_name} Index. - -CMD: -$(tail -n 10 MARKET_DATA_CMD) - -Output: -$(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp | cat -t) - -Error: -$(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp.err | cat -t)" + mv "${f_histfile}.wget.tmp" "${f_histfile}.tmp" fi - if ! [ -e "MARKET_DATA_CMD_OUT-${f_name}" ] + + + # cleanup + rm -f "${f_histfile}.wget.tmp" "${f_histfile}.err.tmp" + + # error if no csvfile available + if [ -n "$f_failed" ] || ! [ -s "${f_histfile}.tmp" ] then - local f_old_value=$(tail -n 1 asset-histories/${f_name}.history.csv | cut -d, -f2) - if echo ${f_old_value} | egrep -q "^[0-9]*\.[0-9]+" + cat "${f_histfile}.err.tmp" >>"${f_histfile}.err" + cat "${f_histfile}.wget.tmp" >>"${f_histfile}.err" + cat "${f_histfile}.err" 1>&2 + mkdir -p FAILED_MARKETDATA + mv "${f_histfile}.err" "FAILED_MARKETDATA/MARKETDATA-${f_name}" 2>/dev/null + return 1 + fi + + # on first download + if ! [ -s "${f_histfile}" ] + then + cat "${f_histfile}.tmp" | sort -k1,1 -t, -u >"${f_histfile}" + else + # merge data + egrep -h ^[0-9][0-9][0-9][0-9]-[0-9][0-9] "${f_histfile}" "${f_histfile}.tmp" | sort -k1,1 -t, -u >"${g_tmp}/${FUNCNAME}.tmp" + + # if there is new dataline add it + if ! cmp -s "${g_tmp}/${FUNCNAME}.tmp" "${f_histfile}" then - echo ${f_old_value} >MARKET_DATA_CMD_OUT-${f_name} - else - echo 0 >MARKET_DATA_CMD_OUT-${f_name} + cat "${g_tmp}/${FUNCNAME}.tmp" >"${f_histfile}" fi fi - f_get_marketdata_price=$(cat MARKET_DATA_CMD_OUT-${f_name}) - echo "${f_timestamp},${f_get_marketdata_price}" >>asset-histories/${f_name}.history.csv + rm "${f_histfile}.tmp" + + # calc indicators ans if 1d then generate 1w histfile + if [[ $f_interval = 1d ]] + then + get_indicators "${f_histfile}" + convert_ohlcv_1d_to_1w "${f_histfile}" "${f_histfile/.1d./.1w.}" + get_indicators "${f_histfile/.1d./.1w.}" + else + get_indicators "${f_histfile}" 999 + fi } +#https://production.dataviz.cnn.io/index/fearandgreed/graphdata +