103 lines
3.6 KiB
Bash
103 lines
3.6 KiB
Bash
function get_marketdata_yahoo_historic {
|
|
|
|
g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
|
|
|
|
local f_item="$1"
|
|
local f_name="$2"
|
|
local f_timeframe="$3"
|
|
|
|
local f_targetcsv="asset-histories/${f_name}.history-yahoo.csv"
|
|
local f_targetcsvtmp="${g_tmp}/${f_name}.history-yahoo.csv"
|
|
[ -n "$f_timeframe" ] && f_targetcsv="${g_tmp}/${f_name}.history-yahoo.${f_timeframe}.csv"
|
|
f_histfile_yahoo="$f_targetcsv"
|
|
|
|
[ "$f_timeframe" = "1w" ] && f_timeframe="1wk"
|
|
[ -z "$f_timeframe" ] && f_timeframe="1d"
|
|
|
|
|
|
# transform CCXT symbols to Yahoo symbol
|
|
if [[ $f_item =~ / ]]
|
|
then
|
|
# change / to -
|
|
f_item=${f_item////-}
|
|
# remove :* (:USDT in contract markets)
|
|
f_item=${f_item//:*}
|
|
# remove spaces
|
|
f_item=${f_item/ /}
|
|
fi
|
|
|
|
# USDT to USD
|
|
f_item=${f_item//USDT/USD}
|
|
# BUSD to USD
|
|
f_item=${f_item//BUSD/USD}
|
|
|
|
# special names of some coins/currencies of yahoo finance
|
|
[[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X"
|
|
[[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X"
|
|
[[ $f_item = "ARB-USD" ]] && f_item="ARB11841-USD"
|
|
[[ $f_item = "DUEL-USD" ]] && f_item="DUEL28868-USD"
|
|
[[ $f_item = "GMX-USD" ]] && f_item="GMX11857-USD"
|
|
[[ $f_item = "MEW-USD" ]] && f_item="MEW30126-USD"
|
|
[[ $f_item = "TAO-USD" ]] && f_item="TAO22974-USD"
|
|
[[ $f_item = "UNI-USD" ]] && f_item="UNI7083-USD"
|
|
[[ $f_item = "SUI-USD" ]] && f_item="SUI20947-USD"
|
|
[[ $f_item = "BLAZE-USD" ]] && f_item="BLAZE30179-USD"
|
|
[[ $f_item = "BEER-USD" ]] && f_item="BEER31337-USD"
|
|
[[ $f_item = "TAI-USD" ]] && f_item="TAI20605-USD"
|
|
[[ $f_item = "DEFI-USD" ]] && f_item="DEFI29200-USD"
|
|
[[ $f_item = "TON-USD" ]] && f_item="TON11419-USD"
|
|
[[ $f_item = "BRETT-USD" ]] && f_item="BRETT29743-USD"
|
|
[[ $f_item = "ADS-USD" ]] && f_item="%24ADS-USD"
|
|
[[ $f_item = "PT-USD" ]] && f_item="PT28582-USD"
|
|
|
|
# end if already failed the last hour
|
|
if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ]
|
|
then
|
|
find "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" -mmin +60 -delete
|
|
if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ]
|
|
then
|
|
#g_echo_note "${f_targetcsv} already failed to downloaded within last hour"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
# end if already exists and modified under given time
|
|
if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}"
|
|
then
|
|
#g_echo_note "${f_targetcsv} has already been downloaded within 2 minutes"
|
|
return 0
|
|
fi
|
|
|
|
local f_sec
|
|
printf -v f_sec '%(%s)T'
|
|
# cleanup
|
|
rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err
|
|
|
|
# Download historical data from yahoo
|
|
#g_echo_note "Fetching Yahoo-Historical data of $f_name"
|
|
g_wget -O ${f_targetcsvtmp} "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history" 2>"${f_targetcsvtmp}".err
|
|
if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ]
|
|
then
|
|
egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" "${f_targetcsv}" | sort -u >"${f_targetcsv}.tmp"
|
|
mv "${f_targetcsv}.tmp" "${f_targetcsv}"
|
|
elif [ -s "${f_targetcsvtmp}" ]
|
|
then
|
|
egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" >"$f_targetcsv"
|
|
else
|
|
# report error
|
|
# g_echo_note "Fetching historical data of $f_name from Yahoo failed!
|
|
#CMD:
|
|
#wget -q -O ${f_targetcsvtmp} ${g_wget_opts} \"https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history\"
|
|
#
|
|
#OUT:
|
|
#$(cat "${f_targetcsvtmp}")
|
|
#
|
|
#ERR:
|
|
#$(cat "${f_targetcsvtmp}".err)
|
|
#"
|
|
mkdir -p FAILED_YAHOO
|
|
mv "${f_targetcsvtmp}.err" "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD"
|
|
return 1
|
|
fi
|
|
}
|