cleanup marketdata download: FEAR_AND_GREED_ALTERNATIVEME FEAR_AND_GREED_CNN US_CONSUMER_PRICE_INDEX_CPI US_UNEMPLOYMENT_RATE US_FED_FUNDS_RATE BINANCE_LONG_SHORT_RATIO_ACCOUNT BINANCE_LONG_SHORT_RATIO_TAKER BINANCE_OPEN_INTEREST
This commit is contained in:
parent
fa4fec677b
commit
c9cc555b42
@ -17,95 +17,142 @@
|
|||||||
# You should have received a copy of the GNU General Public License
|
# You should have received a copy of the GNU General Public License
|
||||||
# along with dabo. If not, see <http://www.gnu.org/licenses/>.
|
# along with dabo. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
function get_marketdata_all {
|
||||||
|
local f_interval=$1
|
||||||
|
|
||||||
|
# daily garketdata jobs
|
||||||
|
if [[ $f_interval = 1d ]]
|
||||||
|
then
|
||||||
|
# FEAR_AND_GREED_ALTERNATIVEME
|
||||||
|
get_marketdata FEAR_AND_GREED_ALTERNATIVEME 'https://api.alternative.me/fng/?limit=0&format=json' '.data[] | (.timestamp | tonumber | strftime("%Y-%m-%d")) + "," + .value + ",,,,0"' "" 1d
|
||||||
|
|
||||||
|
# FEAR_AND_GREED_CNN
|
||||||
|
get_marketdata FEAR_AND_GREED_CNN 'https://production.dataviz.cnn.io/index/fearandgreed/graphdata' '.fear_and_greed_historical.data[] | (.x/1000 | strftime("%Y-%m-%d")) + "," + (.y|tostring) + ",,,,0"' "" 1d
|
||||||
|
|
||||||
|
# monthly US consumer price index CPI data
|
||||||
|
get_marketdata US_CONSUMER_PRICE_INDEX_CPI "https://api.bls.gov/publicAPI/v2/timeseries/data/CUUR0000SA0?startyear=$(date -d 'now -8 years' '+%Y')&endyear=$(date '+%Y')" '.Results.series[0].data[] | .year + "-" + (.period | gsub("M"; "")) + "-01," + .value + ",,,,0"' "" 1d
|
||||||
|
|
||||||
|
# monthly US unemployment rate
|
||||||
|
get_marketdata US_UNEMPLOYMENT_RATE "https://api.bls.gov/publicAPI/v2/timeseries/data/LNU03000000?startyear=$(date -d 'now -8 years' '+%Y')&endyear=$(date '+%Y')" '.Results.series[0].data[] | .year + "-" + (.period | gsub("M"; "")) + "-01," + .value + ",,,,0"' "" 1d
|
||||||
|
|
||||||
|
# US FED funds rate
|
||||||
|
get_marketdata US_FED_FUNDS_RATE 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=DFF' "" "" 1d
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Binance Long Short Ration Account / Taker and Open Interest per symbol
|
||||||
|
get_symbols_ticker
|
||||||
|
local f_symbol f_asset f_time
|
||||||
|
for f_symbol in BTC/$CURRENCY "${f_symbols_array_trade[@]}"
|
||||||
|
do
|
||||||
|
f_asset=${f_symbol//:$CURRENCY/}
|
||||||
|
f_asset=${f_asset//\//}
|
||||||
|
|
||||||
|
# week not available
|
||||||
|
[[ $f_interval = 1w ]] && continue
|
||||||
|
|
||||||
|
f_time='%Y-%m-%d %H:%M:00'
|
||||||
|
[[ $f_interval = 1d ]] && f_time='%Y-%m-%d'
|
||||||
|
|
||||||
|
# BINANCE_LONG_SHORT_RATIO_ACCOUNT per symbol
|
||||||
|
get_marketdata BINANCE_LONG_SHORT_RATIO_ACCOUNT_$f_asset "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .longShortRatio + \",,,,0\"" "" ${f_interval}
|
||||||
|
|
||||||
|
# BINANCE_LONG_SHORT_RATIO_Taker per symbol
|
||||||
|
get_marketdata BINANCE_LONG_SHORT_RATIO_TAKER_$f_asset "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .buySellRatio + \",,,,0\"" "" ${f_interval}
|
||||||
|
|
||||||
|
# BINANCE_OPEN_INTEREST per symbol
|
||||||
|
get_marketdata BINANCE_OPEN_INTEREST_$f_asset "https://fapi.binance.com/futures/data/openInterestHist?symbol=${f_asset}&limit=500&period=${f_interval}" ".[] | (.timestamp/1000 | strftime(\"${f_time}\")) + \",\" + .sumOpenInterest + \",,,,0\"" "" ${f_interval}
|
||||||
|
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
function get_marketdata {
|
function get_marketdata {
|
||||||
|
|
||||||
g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
|
g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
|
||||||
|
|
||||||
|
local f_name=$1
|
||||||
|
local f_wget=$2
|
||||||
|
local f_jq=$3
|
||||||
|
local f_other=$4
|
||||||
|
local f_timeframe=$5
|
||||||
|
[ -z "$f_timeframe" ] && f_timeframe=1d
|
||||||
|
local f_histfile="asset-histories/MARKETDATA_${f_name}.history.1d.csv"
|
||||||
|
local f_dataline f_failed
|
||||||
|
|
||||||
### CPI Data: curl hapi.bls.gov/publicAPI/v2/timeseries/data/cpi | jq -a
|
# download
|
||||||
### release dates: https://www.bls.gov/schedule/news_release/bls.ics
|
g_wget -O "${f_histfile}.wget.tmp" $f_wget 2>"${f_histfile}.err.tmp" || f_failed=wget
|
||||||
|
[ -s "${f_histfile}.wget.tmp" ] || f_failed=wget
|
||||||
get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX
|
if [ -n "$f_failed" ]
|
||||||
get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX
|
|
||||||
get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX
|
|
||||||
get_marketdata_from_url '"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=5m" | jq -r .[].longShortRatio | tail -n1' BINANCE-BTC-GlobalLongShortAccountRatio
|
|
||||||
get_marketdata_from_url '"https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=5m" | jq -r .[].buySellRatio | tail -n1' BINANCE-BTC-TakerLongShortRatio
|
|
||||||
get_marketdata_from_url '"https://fapi.binance.com/futures/data/openInterestHist?symbol=BTCUSDT&period=5m" | jq -r .[].sumOpenInterest | tail -n1' BINANCE-BTC-OpenInterest
|
|
||||||
get_marketdata_from_url '"https://api.alternative.me/fng/?limit=1&format=json" | jq -r .data[].value' CRYPTO_FEAR_AND_GREED-INDEX
|
|
||||||
# clear old stuff
|
|
||||||
find asset-histories/*INDEX* -type f -mtime +6 -delete
|
|
||||||
}
|
|
||||||
|
|
||||||
function get_marketdata_from_url {
|
|
||||||
g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
|
|
||||||
|
|
||||||
local f_url="$1"
|
|
||||||
local f_name="$2"
|
|
||||||
|
|
||||||
## get data for analysis
|
|
||||||
if find asset-histories/${f_name}.history.csv -mmin -${INTERVAL_MIN} 2>/dev/null | grep -q "asset-histories/${f_name}.history.csv"
|
|
||||||
then
|
then
|
||||||
g_echo_note "asset-histories/${f_name}.history.csv already downloaded in the last ${INTERVAL_MIN} minutes"
|
echo "g_wget -O \"${f_histfile}.wget.tmp\" $f_wget 2>\"${f_histfile}.err\"" >"${f_histfile}.err"
|
||||||
f_get_marketdata_price=$(cat MARKET_DATA_CMD_OUT-${f_name})
|
|
||||||
return 0
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# check source platform for parsing parameters, prepare and run wget command
|
# jd
|
||||||
>MARKET_DATA_CMD
|
if [ -z "$f_failed" ] && [ -n "$f_jq" ]
|
||||||
if echo "${f_url}" | grep -q "boerse.de"
|
|
||||||
then
|
then
|
||||||
echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'itemprop=\"price\" content=\"[0-9]+\.[0-9]+\"' | sed s'#\"#\n#g' | egrep '^[0-9]+\.[0-9]+'" >MARKET_DATA_CMD
|
if ! jq -r "$f_jq" "${f_histfile}.wget.tmp" >"${f_histfile}.tmp" 2>"${f_histfile}.err.tmp"
|
||||||
elif echo "${f_url}" | egrep -q "investing.com.+economic-calendar"
|
|
||||||
then
|
|
||||||
echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'Actual.+Forecast.+Previous' | cut -d'>' -f7,11 | cut -d'<' -f1,2 | sed 's#,##g' | sed 's/\%//g' | sed 's#</div>#,#' | grep '[0-9]' | cut -d, -f1" >MARKET_DATA_CMD
|
|
||||||
elif echo "${f_url}" | egrep -q "investing.com.+indices"
|
|
||||||
then
|
|
||||||
echo "wget ${g_wget_opts} -q -O - ${f_url} | sed 's#</div>#\n#g' | grep 'text-5xl.*font-bold.*md:text-' | sed 's#^.*>##; s#,##g' | grep '[0-9]'" >MARKET_DATA_CMD
|
|
||||||
elif echo "${f_url}" | egrep -q '^"https://'
|
|
||||||
then
|
|
||||||
echo "wget -q -O - ${g_wget_opts} ${f_url}" >MARKET_DATA_CMD
|
|
||||||
else
|
|
||||||
# default to Yahoo Finace Symbols via API
|
|
||||||
echo "wget ${g_wget_opts} -q -O - https://query1.finance.yahoo.com/v8/finance/chart/${f_url} | jq -r '.[].result[].meta.regularMarketPrice'" >MARKET_DATA_CMD
|
|
||||||
fi
|
|
||||||
|
|
||||||
g_runcmd g_retrycmd sh MARKET_DATA_CMD >MARKET_DATA_CMD_OUT-${f_name}.tmp 2>MARKET_DATA_CMD_OUT-${f_name}.tmp.err
|
|
||||||
|
|
||||||
# check output
|
|
||||||
local f_get_marketdata_price_tmp=$(cat MARKET_DATA_CMD_OUT-${f_name}.tmp)
|
|
||||||
if g_num_valid_number ${f_get_marketdata_price_tmp}
|
|
||||||
then
|
|
||||||
if egrep -q "^0\.00" MARKET_DATA_CMD_OUT-${f_name}.tmp
|
|
||||||
then
|
then
|
||||||
g_echo_note "Ignoring ${f_name} $(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp) - maybe out of business day"
|
echo jq -r "$f_jq" "${f_histfile}.wget.tmp" >"${f_histfile}.err"
|
||||||
|
f_failed=jq
|
||||||
else
|
else
|
||||||
g_echo_note "${f_name}: ${f_get_marketdata_price_tmp}"
|
mv "${f_histfile}.tmp" "${f_histfile}.wget.tmp"
|
||||||
mv MARKET_DATA_CMD_OUT-${f_name}.tmp MARKET_DATA_CMD_OUT-${f_name}
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
g_echo_warn "MARKET_DATA_CMD_OUT-${f_name}.tmp has wrong Syntax. - Not updating ${f_name} Index.
|
|
||||||
|
|
||||||
CMD:
|
|
||||||
$(tail -n 10 MARKET_DATA_CMD)
|
|
||||||
|
|
||||||
Output:
|
|
||||||
$(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp | cat -t)
|
|
||||||
|
|
||||||
Error:
|
|
||||||
$(tail -n 10 MARKET_DATA_CMD_OUT-${f_name}.tmp.err | cat -t)"
|
|
||||||
fi
|
|
||||||
if ! [ -e "MARKET_DATA_CMD_OUT-${f_name}" ]
|
|
||||||
then
|
|
||||||
local f_old_value=$(tail -n 1 asset-histories/${f_name}.history.csv | cut -d, -f2)
|
|
||||||
if echo ${f_old_value} | egrep -q "^[0-9]*\.[0-9]+"
|
|
||||||
then
|
|
||||||
echo ${f_old_value} >MARKET_DATA_CMD_OUT-${f_name}
|
|
||||||
else
|
|
||||||
echo 0 >MARKET_DATA_CMD_OUT-${f_name}
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
f_get_marketdata_price=$(cat MARKET_DATA_CMD_OUT-${f_name})
|
|
||||||
echo "${f_timestamp},${f_get_marketdata_price}" >>asset-histories/${f_name}.history.csv
|
# other/additional processing
|
||||||
|
if [ -z "$f_failed" ] && [ -n "$f_other" ]
|
||||||
|
then
|
||||||
|
if ! cat "${f_histfile}.wget.tmp" | eval $f_other
|
||||||
|
then
|
||||||
|
echo "cat \"${f_histfile}.wget.tmp\" | $f_other" >"${f_histfile}.err"
|
||||||
|
f_failed=other
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
mv "${f_histfile}.wget.tmp" "${f_histfile}.tmp"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# cleanup
|
||||||
|
rm -f "${f_histfile}.wget.tmp" "${f_histfile}.err.tmp"
|
||||||
|
|
||||||
|
# error if no csvfile available
|
||||||
|
if [ -n "$f_failed" ] || ! [ -s "${f_histfile}.tmp" ]
|
||||||
|
then
|
||||||
|
cat "${f_histfile}.err.tmp" >>"${f_histfile}.err"
|
||||||
|
cat "${f_histfile}.wget.tmp" >>"${f_histfile}.err"
|
||||||
|
cat "${f_histfile}.err" 1>&2
|
||||||
|
mkdir -p FAILED_MARKETDATA
|
||||||
|
mv "${f_histfile}.err" "FAILED_MARKETDATA/MARKETDATA-${f_name}" 2>/dev/null
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# on first download
|
||||||
|
if ! [ -s "${f_histfile}" ]
|
||||||
|
then
|
||||||
|
cat "${f_histfile}.tmp" | sort -k1,1 -t, -u >"${f_histfile}"
|
||||||
|
else
|
||||||
|
# merge data
|
||||||
|
egrep -h ^[0-9][0-9][0-9][0-9]-[0-9][0-9] "${f_histfile}" "${f_histfile}.tmp" | sort -k1,1 -t, -u >"${g_tmp}/${FUNCNAME}.tmp"
|
||||||
|
|
||||||
|
# if there is new dataline add it
|
||||||
|
if ! cmp -s "${g_tmp}/${FUNCNAME}.tmp" "${f_histfile}"
|
||||||
|
then
|
||||||
|
cat "${g_tmp}/${FUNCNAME}.tmp" >"${f_histfile}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm "${f_histfile}.tmp"
|
||||||
|
|
||||||
|
# calc indicators ans if 1d then generate 1w histfile
|
||||||
|
if [[ $f_interval = 1d ]]
|
||||||
|
then
|
||||||
|
get_indicators "${f_histfile}"
|
||||||
|
convert_ohlcv_1d_to_1w "${f_histfile}" "${f_histfile/.1d./.1w.}"
|
||||||
|
get_indicators "${f_histfile/.1d./.1w.}"
|
||||||
|
else
|
||||||
|
get_indicators "${f_histfile}" 999
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#https://production.dataviz.cnn.io/index/fearandgreed/graphdata
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user