From d338afc2e50ef9d4f7a27650cbb082fb08eed95e Mon Sep 17 00:00:00 2001 From: olli Date: Wed, 6 Mar 2024 17:55:33 +0100 Subject: [PATCH] switch from boerse.de to Yahoo Finance API --- dabo/dabo-bot.sh | 2 +- dabo/functions/get_marketdata.sh | 53 +++++++++++++++++++++----------- dabo/functions/watch_assets.sh | 3 +- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/dabo/dabo-bot.sh b/dabo/dabo-bot.sh index 74fc7ce..0cd9d6a 100755 --- a/dabo/dabo-bot.sh +++ b/dabo/dabo-bot.sh @@ -10,7 +10,7 @@ BASEPATH=/dabo/htdocs g_tries=13 g_tries_delay=23 -g_wget_opts="--timeout 10 --tries=2" +g_wget_opts="--timeout 10 --tries=2 --user-agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'" ### FUNCTIONS ### for bashfunc in $(find ${BASEPATH}/../functions -type f -name "*.sh") diff --git a/dabo/functions/get_marketdata.sh b/dabo/functions/get_marketdata.sh index a0215a2..c75066b 100644 --- a/dabo/functions/get_marketdata.sh +++ b/dabo/functions/get_marketdata.sh @@ -1,22 +1,37 @@ function get_marketdata { - get_marketdata_from_url https://www.boerse.de/indizes/Dow-Jones/US2605661048 DOW-JONES-INDEX - get_marketdata_from_url https://www.boerse.de/indizes/SundP-500/US78378X1072 SP500-INDEX - get_marketdata_from_url https://www.boerse.de/indizes/Nasdaq-100/US6311011026 NASDAQ-INDEX - get_marketdata_from_url https://www.investing.com/indices/msci-eafe MSCI-EAFE-INDEX - get_marketdata_from_url https://www.investing.com/indices/10-year-treasury-yield 10-YEAR-TREASURY-YIELD-INDEX - get_marketdata_from_url https://www.boerse.de/rohstoffe/WTI-Rohoelpreis/XD0015948363 OIL-INVERTED-INDEX - get_marketdata_from_url https://www.boerse.de/rohstoffe/Goldpreis/XC0009655157 GOLD-INDEX - #get_marketdata https://www.boerse.de/aktien/Nvidia-Aktie/US67066G1040 NVIDIA-INDEX - #get_marketdata https://www.boerse.de/aktien/Tesla-Aktie/US88160R1014 TESLA-INDEX - #get_marketdata https://www.boerse.de/aktien/Apple-Aktie/US0378331005 APPLE-INDEX - get_marketdata_from_url https://www.boerse.de/realtime-kurse/MSCI-World/XC0009692739 MSCI-WORLD-INDEX - get_marketdata_from_url https://www.boerse.de/indizes/DJ-AMER-OIL-und-GAS/XC0006886516 OIL-GAS-INVERTED-INDEX - get_marketdata_from_url https://www.boerse.de/fonds/SPDR-SundP-Regional-Banking-ETF/US78464A6982 KRE-BANKING-INDEX - #get_marketdata https://www.boerse.de/indizes/Dax/DE0008469008 DAX-INDEX - get_marketdata_from_url https://www.investing.com/indices/usdollar DXY-INVERTED-INDEX + + get_marketdata_from_url YM=F DOW-JONES-INDEX + get_marketdata_from_url ES=F SP500-INDEX + get_marketdata_from_url NQ=F NASDAQ-INDEX + get_marketdata_from_url MFS=F MSCI-EAFE-INDEX + get_marketdata_from_url ZB=F 10-YEAR-TREASURY-YIELD-INDEX + get_marketdata_from_url MCL=F OIL-INVERTED-INDEX + get_marketdata_from_url GC=F GOLD-INDEX + get_marketdata_from_url MWL=F MSCI-WORLD-INDEX + get_marketdata_from_url IEO OIL-GAS-INVERTED-INDEX + get_marketdata_from_url KRE KRE-BANKING-INDEX + get_marketdata_from_url DX=F DXY-INVERTED-INDEX get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX + #get_marketdata_from_url https://www.boerse.de/indizes/Dow-Jones/US2605661048 DOW-JONES-INDEX + #get_marketdata_from_url https://www.boerse.de/indizes/SundP-500/US78378X1072 SP500-INDEX + #get_marketdata_from_url https://www.boerse.de/indizes/Nasdaq-100/US6311011026 NASDAQ-INDEX + #get_marketdata_from_url https://www.investing.com/indices/msci-eafe MSCI-EAFE-INDEX + #get_marketdata_from_url https://www.investing.com/indices/10-year-treasury-yield 10-YEAR-TREASURY-YIELD-INDEX + #get_marketdata_from_url https://www.boerse.de/rohstoffe/WTI-Rohoelpreis/XD0015948363 OIL-INVERTED-INDEX + #get_marketdata_from_url https://www.boerse.de/rohstoffe/Goldpreis/XC0009655157 GOLD-INDEX + ##get_marketdata https://www.boerse.de/aktien/Nvidia-Aktie/US67066G1040 NVIDIA-INDEX + ##get_marketdata https://www.boerse.de/aktien/Tesla-Aktie/US88160R1014 TESLA-INDEX + ##get_marketdata https://www.boerse.de/aktien/Apple-Aktie/US0378331005 APPLE-INDEX + #get_marketdata_from_url https://www.boerse.de/realtime-kurse/MSCI-World/XC0009692739 MSCI-WORLD-INDEX + #get_marketdata_from_url https://www.boerse.de/indizes/DJ-AMER-OIL-und-GAS/XC0006886516 OIL-GAS-INVERTED-INDEX + #get_marketdata_from_url https://www.boerse.de/fonds/SPDR-SundP-Regional-Banking-ETF/US78464A6982 KRE-BANKING-INDEX + ##get_marketdata https://www.boerse.de/indizes/Dax/DE0008469008 DAX-INDEX + #get_marketdata_from_url https://www.investing.com/indices/usdollar DXY-INVERTED-INDEX + #get_marketdata_from_url https://www.investing.com/economic-calendar/unemployment-rate-300/ US-UNEMPLOYMENT-INDEX + #get_marketdata_from_url https://www.investing.com/economic-calendar/cpi-733 US-CONSUMER-PRICE-INDEX + #get_marketdata_from_url https://www.investing.com/indices/fed-funds-composite-interest-rate-opinion US-FED-FEDERAL-FUNDS-RATE-INVERTED-INDEX # clear old stuff find asset-histories/*INDEX* -type f -mtime +6 -delete } @@ -35,10 +50,12 @@ function get_marketdata_from_url { # check source platform for parsing parameters, prepare and run wget command >MARKET_DATA_CMD - echo ${f_url} | grep -q "boerse.de" && echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'itemprop=\"price\" content=\"[0-9]+\.[0-9]+\"' | sed s'#\"#\n#g' | egrep '^[0-9]+\.[0-9]+'" >MARKET_DATA_CMD - echo ${f_url} | egrep -q "investing.com.+economic-calendar" && echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'Actual.+Forecast.+Previous' | cut -d'>' -f7,11 | cut -d'<' -f1,2 | sed 's#,##g' | sed 's#\%##g' | sed 's##,#' | grep '[0-9]'" >MARKET_DATA_CMD + echo ${f_url} | grep -q "finance.yahoo.com" && + echo ${f_url} | grep -q "boerse.de" && echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'itemprop=\"price\" content=\"[0-9]+\.[0-9]+\"' | sed s'#\"#\n#g' | egrep '^[0-9]+\.[0-9]+'" >MARKET_DATA_CMD + echo ${f_url} | egrep -q "investing.com.+economic-calendar" && echo "wget ${g_wget_opts} -q -O - ${f_url} | egrep 'Actual.+Forecast.+Previous' | cut -d'>' -f7,11 | cut -d'<' -f1,2 | sed 's#,##g' | sed 's/\%//g' | sed 's##,#' | grep '[0-9]'" >MARKET_DATA_CMD echo ${f_url} | egrep -q "investing.com.+indices" && echo "wget ${g_wget_opts} -q -O - ${f_url} | sed 's##\n#g' | grep 'text-5xl.*font-bold.*md:text-' | sed 's#^.*>##; s#,##g' | grep '[0-9]'" >MARKET_DATA_CMD - [ -s MARKET_DATA_CMD ] || echo "wget ${g_wget_opts} -q -O - ${f_url}" >MARKET_DATA_CMD + # default to Yahoo Finace Symbols via API + [ -s MARKET_DATA_CMD ] || echo "wget ${g_wget_opts} -q -O - https://query1.finance.yahoo.com/v8/finance/chart/${f_url} | jq -r '.[].result[].meta.regularMarketPrice'" >MARKET_DATA_CMD g_runcmd g_retrycmd sh MARKET_DATA_CMD >MARKET_DATA_CMD_OUT-${f_name}.tmp 2>MARKET_DATA_CMD_OUT-${f_name}.tmp.err diff --git a/dabo/functions/watch_assets.sh b/dabo/functions/watch_assets.sh index 0c881d2..f8d0b85 100644 --- a/dabo/functions/watch_assets.sh +++ b/dabo/functions/watch_assets.sh @@ -43,11 +43,12 @@ function watch_assets { # get current asset price and last price f_price="" - if [[ ${f_asset} =~ ^https ]] + if [[ ${f_asset} =~ ^https ]] || [[ ${f_asset} =~ " " ]] then # get asset price from get_marketdata_from_url get_marketdata_from_url ${f_asset} f_price=${f_get_marketdata_price} + ### 2do USD->EUR Umrechnugn wenn f_currency=eur readarray -d " " -t f_asset_array < <(echo -n "${f_asset}") f_asset=${f_asset_array[1]} else