diff --git a/dabo/functions/get_marketdata_historic.sh b/dabo/functions/get_marketdata_historic.sh new file mode 100644 index 0000000..cf81482 --- /dev/null +++ b/dabo/functions/get_marketdata_historic.sh @@ -0,0 +1,61 @@ +function get_marketdata_yahoo_historic { + + #g_echo_note "RUNNING FUNCTION ${FUNCNAME} $@" + + local f_item="$1" + local f_name="$2" + local f_targetcsv="asset-histories/${f_name}.history-yahoo.csv" + local f_targetcsvtmp="${g_tmp}/${f_name}.history-yahoo.csv" + + [[ $f_item = "USD-EUR" ]] && f_item="USDEUR=X" + [[ $f_item = "EUR-USD" ]] && f_item="EURUSD=X" + + # end if already failed the last hour + if [ -f FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD ] + then + find "FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD" -mmin +60 -delete + if [ -f FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD ] + then + #g_echo_note "${f_targetcsv} already failed to downloaded within last hour" + return 1 + fi + fi + + # end if already exists and modified under 1 day + if [ -s "${f_targetcsv}" ] && find "${f_targetcsv}" -mtime -1 | grep -q "${f_targetcsv}" + then + #g_echo_note "${f_targetcsv} has already been downloaded within a day" + return 0 + fi + + local f_sec + printf -v f_sec '%(%s)T' + # cleanup + rm -f "$f_targetcsvtmp" "${f_targetcsvtmp}".err + + # Download historical data from yahoo + #g_echo_note "Fetching Yahoo-Historical data of $f_name" + g_wget -O ${f_targetcsvtmp} "https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history" 2>"${f_targetcsvtmp}".err + if [ -s "${f_targetcsv}" ] && [ -s "${f_targetcsvtmp}" ] + then + egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" "${f_targetcsv}" | sort -u >"${f_targetcsv}.tmp" + mv "${f_targetcsv}.tmp" "${f_targetcsv}" + elif [ -s "${f_targetcsvtmp}" ] + then + egrep -h "^[1-9][0-9][0-9][0-9]-[0-1][0-9]-[0-9][0-9],[0-9]" "${f_targetcsvtmp}" >"$f_targetcsv" + else + # report error +# g_echo_note "Fetching historical data of $f_name from Yahoo failed! +#CMD: +#wget -q -O ${f_targetcsvtmp} ${g_wget_opts} \"https://query1.finance.yahoo.com/v7/finance/download/${f_item}?period1=0&period2=${f_sec}&interval=1d&events=history\" +# +#OUT: +#$(cat "${f_targetcsvtmp}") +# +#ERR: +#$(cat "${f_targetcsvtmp}".err) +#" + mv ${f_targetcsvtmp}.err FAILED_YAHOO_${f_name}_HISTORIC_DOWNLOAD + return 1 + fi +}