132 lines
5.3 KiB
Bash
132 lines
5.3 KiB
Bash
function get_marketdata_yahoo {
|
|
|
|
g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@"
|
|
local f_item="$1"
|
|
local f_name="$2"
|
|
local f_timeframe="$3"
|
|
|
|
#local f_targetcsv="asset-histories/${f_name}.history-yahoo.csv"
|
|
local f_targetcsvtmp="${g_tmp}/${f_name}.history-yahoo.csv"
|
|
local f_targetjsontmp="${g_tmp}/${f_name}.history-yahoo.json"
|
|
rm -f "$f_targetcsvtmp" "$f_targetjsontmp"
|
|
|
|
[ -z "$f_timeframe" ] && f_timeframe="1d"
|
|
local f_targetcsv="asset-histories/${f_name}.history-yahoo.${f_timeframe}.csv"
|
|
[ "$f_timeframe" = "1w" ] && f_timeframe="1wk"
|
|
f_histfile_yahoo="$f_targetcsv"
|
|
|
|
# transform CCXT symbols to Yahoo symbol
|
|
if [[ $f_item =~ / ]]
|
|
then
|
|
# change / to -
|
|
f_item=${f_item////-}
|
|
# remove :* (:USDT in contract markets)
|
|
f_item=${f_item//:*}
|
|
# remove spaces
|
|
f_item=${f_item/ /}
|
|
fi
|
|
|
|
# USDT to USD
|
|
f_item=${f_item//USDT/USD}
|
|
# BUSD to USD
|
|
f_item=${f_item//BUSD/USD}
|
|
|
|
# special names of some economy data/indexes of yahoo finance
|
|
[[ $f_item = "DXY" ]] && f_item="DX=F"
|
|
[[ $f_item = "DOWJONES" ]] && f_item="YM=F"
|
|
[[ $f_item = "SP500" ]] && f_item="ES=F"
|
|
[[ $f_item = "NASDAQ" ]] && f_item="NQ=F"
|
|
[[ $f_item = "MSCIEAFE" ]] && f_item="MFS=F"
|
|
[[ $f_item = "MSCIWORLD" ]] && f_item="IWDA.AS"
|
|
[[ $f_item = "10YRTREASURY" ]] && f_item="ZB=F"
|
|
[[ $f_item = "OIL" ]] && f_item="MCL=F"
|
|
[[ $f_item = "GOLD" ]] && f_item="GC=F"
|
|
[[ $f_item = "OILGAS" ]] && f_item="IEO"
|
|
[[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X"
|
|
[[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X"
|
|
|
|
# special names of some coins/currencies of yahoo finance
|
|
[[ $f_item = "ARB-USD" ]] && f_item="ARB11841-USD"
|
|
[[ $f_item = "DUEL-USD" ]] && f_item="DUEL28868-USD"
|
|
[[ $f_item = "GMX-USD" ]] && f_item="GMX11857-USD"
|
|
[[ $f_item = "MEW-USD" ]] && f_item="MEW30126-USD"
|
|
[[ $f_item = "TAO-USD" ]] && f_item="TAO22974-USD"
|
|
[[ $f_item = "UNI-USD" ]] && f_item="UNI7083-USD"
|
|
[[ $f_item = "SUI-USD" ]] && f_item="SUI20947-USD"
|
|
[[ $f_item = "BLAZE-USD" ]] && f_item="BLAZE30179-USD"
|
|
[[ $f_item = "BEER-USD" ]] && f_item="BEER31337-USD"
|
|
[[ $f_item = "TAI-USD" ]] && f_item="TAI20605-USD"
|
|
[[ $f_item = "DEFI-USD" ]] && f_item="DEFI29200-USD"
|
|
[[ $f_item = "TON-USD" ]] && f_item="TON11419-USD"
|
|
[[ $f_item = "BRETT-USD" ]] && f_item="BRETT29743-USD"
|
|
[[ $f_item = "ADS-USD" ]] && f_item="%24ADS-USD"
|
|
[[ $f_item = "PT-USD" ]] && f_item="PT28582-USD"
|
|
|
|
# end if already failed the last 5 minutes
|
|
if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ]
|
|
then
|
|
find "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" -mmin +5 -delete
|
|
if [ -f "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" ]
|
|
then
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
# end if already exists and modified under given time
|
|
if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mmin -2 | grep -q "${f_targetcsv}"
|
|
then
|
|
return 0
|
|
fi
|
|
|
|
local f_sec
|
|
printf -v f_sec '%(%s)T'
|
|
# cleanup
|
|
rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err ${f_targetjsontmp} "${f_targetjsontmp}".err
|
|
|
|
if [ "$f_timeframe" = "1d" ] || [ "$f_timeframe" = "1wk" ] || [ "$f_timeframe" = "1mo" ]
|
|
then
|
|
# Download historical data from yahoo
|
|
g_wget -O "${f_targetcsvtmp}" "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=${f_timeframe}&events=history" 2>"${f_targetcsvtmp}".err
|
|
else
|
|
# Download data from yahoo
|
|
g_wget -O "${f_targetjsontmp}" "https://query1.finance.yahoo.com/v7/finance/chart/${f_item}?interval=${f_timeframe}&period2=${f_sec}" 2>"${f_targetjsontmp}".err
|
|
jq -r '.chart.result[0] as $result | range(0; $result.timestamp | length) | [$result.timestamp[.], $result.indicators.quote[0].open[.], $result.indicators.quote[0].high[.], $result.indicators.quote[0].low[.], $result.indicators.quote[0].close[.], $result.indicators.quote[0].volume[.]] | @csv' "${f_targetjsontmp}" >"${f_targetcsvtmp}.unixtime" 2>"${f_targetjsontmp}".err
|
|
|
|
# change unix time to human readable and fill unfilled lines, ignore lines not with 00 secolds (last line)
|
|
local date_time open high low close lastopen lasthigh lastlow lastclose volume
|
|
while IFS=, read -r timestamp open high low close volume; do
|
|
date_time=$(printf "%(%Y-%m-%d %H:%M:%S)T" $timestamp)
|
|
[ -z "$open" ] && open=$lastopen
|
|
[ -z "$high" ] && high=$lasthigh
|
|
[ -z "$low" ] && low=$lastlow
|
|
[ -z "$close" ] && close=$lastclose
|
|
[ -z "$volume" ] && volume=0
|
|
lastopen=$open
|
|
lasthigh=$high
|
|
lastlow=$low
|
|
lastclose=$close
|
|
echo "$date_time,$open,$high,$low,$close,$volume"
|
|
done < "${f_targetcsvtmp}.unixtime" | grep ":00," >${f_targetcsvtmp}
|
|
fi
|
|
|
|
# error if no csvfile available
|
|
if ! [ -s "${f_targetcsvtmp}" ]
|
|
then
|
|
mkdir -p FAILED_YAHOO
|
|
cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" > "FAILED_YAHOO/${f_name}_HISTORIC_DOWNLOAD" 2>&1
|
|
f_get_marketdata_yahoo_error=$(cat "${f_targetcsvtmp}.err" "${f_targetjsontmp}.err" 2>/dev/null)
|
|
return 1
|
|
fi
|
|
|
|
# put the csvs together
|
|
if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ]
|
|
then
|
|
egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9].*,[0-9]" "${f_targetcsvtmp}" "${f_targetcsv}" | sort -k1,2 -t, -u >"${f_targetcsv}.tmp"
|
|
mv "${f_targetcsv}.tmp" "${f_targetcsv}"
|
|
else
|
|
egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9].*,[0-9]" "${f_targetcsvtmp}" | sort -k1,2 -t, -u >"$f_targetcsv"
|
|
fi
|
|
|
|
}
|
|
|