From 3c477b347a0de599fe97f8edbf23e27edefbe7f1 Mon Sep 17 00:00:00 2001 From: Santiago Crespo Date: Sun, 30 Oct 2022 20:51:50 +0000 Subject: [PATCH] Create run_pipelines.sh --- src/run_pipelines.sh | 149 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 src/run_pipelines.sh diff --git a/src/run_pipelines.sh b/src/run_pipelines.sh new file mode 100644 index 0000000..4430b2d --- /dev/null +++ b/src/run_pipelines.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# Run all the pipelines +# Una hora y media con calles-OSM, Direcciones, other-OSM, Ayto-MINT, Fuentes, EMT bus stops, Aseos + +TIMESTART=`date` + +echo "Processing calles-OSM..." +mkdir -p ~/pipelines/calles-OSM +cd ~/pipelines/calles-OSM +wget https://cartobase.es/madrid_streets_extract.sh -O madrid_streets_extract.sh +chmod +x madrid_streets_extract.sh +./madrid_streets_extract.sh && cp current/callesmadrid* /var/www/html/ +echo "callesmadrid-OSM.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + +echo "Processing other OSM features..." +mkdir -p ~/pipelines/OSM +cd ~/pipelines/OSM +wget https://cartobase.es/get_madrid_features_from_osm.sh -O get_madrid_features_from_osm.sh +chmod +x get_madrid_features_from_osm.sh +wget https://cartobase.es/process_madrid_features_from_osm.sh -O process_madrid_features_from_osm.sh +chmod +x process_madrid_features_from_osm.sh +./process_madrid_features_from_osm.sh +cp current/* /var/www/html/ +ls -1 current > /tmp/listaOSM +while IFS= read -r line +do + echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles +done < /tmp/listaOSM + +echo "Processing Ayto-MINT..." +pip install --upgrade ogr2osm +PATH=$PATH:~/.local/bin +mkdir -p ~/pipelines/Ayto-MINT +cd ~/pipelines/Ayto-MINT +wget https://cartobase.es/Ayto-MINT.7z -O Ayto-MINT.7z +7z x -aoa Ayto-MINT.7z +mv Ayto-MINT/* . && rmdir Ayto-MINT && rm Ayto-MINT.7z +chmod +x ./process_madrid_mint_features.sh +./process_madrid_mint_features.sh +cp data/* /var/www/html/ +ls -1 data > /tmp/listaMINT +while IFS= read -r line +do + echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles +done < /tmp/listaMINT + +# 202202: fail, seguramente habrá cambiado algo en la fuente. Uso el anterior. +#echo "Processing Fuentes..." +#mkdir -p ~/pipelines/fuentes +#cd ~/pipelines/fuentes +#wget https://cartobase.es/fuentesaguapotablemadrid.sh -O fuentesaguapotablemadrid.sh +#chmod +x fuentesaguapotablemadrid.sh +#./fuentesaguapotablemadrid.sh +#cp current/fuentes-madrid.osm /var/www/html/ +#echo "fuentes-madrid.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + +echo "Processing EMT bus stops..." +mkdir -p ~/pipelines/EMT +cd ~/pipelines/EMT +wget https://cartobase.es/emt_bus_stop.sh -O emt_bus_stop.sh +chmod +x emt_bus_stop.sh +wget https://cartobase.es/reproyecta_coordenadas_emt_crtm.sh -O reproyecta_coordenadas_emt_crtm.sh +chmod +x reproyecta_coordenadas_emt_crtm.sh +./emt_bus_stop.sh +cp emt_stops.osm /var/www/html/ +echo "emt_stops.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + +echo "Processing mobiliario (aseos)..." +mkdir -p ~/pipelines/mobiliario +cd ~/pipelines/mobiliario +wget https://cartobase.es/procesa_aseos.sh -O procesa_aseos.sh +chmod +x procesa_aseos.sh +./procesa_aseos.sh +cp aseos.osm /var/www/html/ +echo "aseos.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + +echo "Processing addresses..." +mkdir -p ~/pipelines/direcciones +cd ~/pipelines/direcciones +wget https://cartobase.es/madridaddr2osm.sh -O madridaddr2osm.sh +chmod +x madridaddr2osm.sh +./madridaddr2osm.sh +cp current/direccionesmadrid.osm /var/www/html/ +echo "direccionesmadrid.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + +echo "Ayto-API..." +sudo apt-get install xsltproc -y +mkdir -p ~/pipelines/Ayto-API +cd ~/pipelines +wget https://cartobase.es/Ayto-API.7z -O Ayto-API.7z +7z -aoa x Ayto-API.7z +cd ~/pipelines/Ayto-API +chmod +x get_madrid_features_from_ayto-API.sh +chmod +x process_api.csv.sh +./process_api.csv.sh +cd data +rm tourisminformation-201105-0-informacion-turismo-Ayto-Madrid-API.osm # Tiene datos que ya están en tourisminformation-206577-0-oficinas-turismo-Ayto-Madrid-API.osm +rm placeyesFIXME-202180-0-instalaciones-accesibles-no-muni-Ayto-Madrid-API.osm placeyesFIXME-202162-0-instalaciones-accesibles-municip-Ayto-Madrid-API.osm # Tienen datos de colegios, museos, ministerios, Oficinas municipales, salas de exposiciones... +rm leisurepark-200761-0-parques-jardines-Ayto-Madrid-API.osm # mejor OSM +rm cemeteryyes-205026-0-cementerios-Ayto-Madrid-API.osm # Mejor OSM +cd .. +cp data/*osm /var/www/html/ +ls -1 data > /tmp/listaOSM +while IFS= read -r line +do + echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles +done < /tmp/listaOSM + +echo "Processing BiciMAD..." +mkdir -p ~/pipelines/bicimad +cd ~/pipelines/bicimad +wget https://cartobase.es/procesa_bicimad.sh -O procesa_bicimad.sh +chmod +x procesa_bicimad.sh +./procesa_bicimad.sh +cp "current/amenity-bicycle_rental-bicimad_stations_EMT.osm" /var/www/html/ +echo "amenity-bicycle_rental-bicimad_stations_EMT.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + + +#echo "Processing Árboles..." # Desactivado, hace falta JOSM de por medio. Y son 533M de archivo cuando ahora tenemos 546M. Revisar (Filomena?) +#sudo apt install virtualenv npm -y +##npm install -g geojsontoosm +##sudo npm install geojson2osm +#sudo npm install -g json2csv +#sudo wget "https://raw.githubusercontent.com/OSMBrasil/csv2osm/master/csv2osm.py" -O /usr/local/bin/csv2osm.py && sudo chmod +x /usr/local/bin/csv2osm.py +#mkdir -p ~/pipelines/arboles +#cd ~/pipelines/arboles +#wget https://cartobase.es/scrape-trees.sh -O scrape-trees.sh +#chmod +x scrape-trees.sh +#wget https://cartobase.es/procesa_arbolado_viario_scrapeado.sh -O procesa_arbolado_viario_scrapeado.sh +#chmod +x procesa_arbolado_viario_scrapeado.sh +#./scrape-trees.sh # Una media hora +#./procesa_arbolado_viario_scrapeado.sh +#cp Arbolado_Viario.osm /var/www/html/ +#echo "Arbolado_Viario.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles + + + + +# TODO: Datos sin actualizar desde 2017, está a medias +#echo "Ayto-carto..." +#mkdir -p ~/pipelines/Ayto-carto +#cd ~/pipelines/Ayto-carto +#wget https://cartobase.es/descargacartomadrid1-1000.sh -O descargacartomadrid1-1000.sh +#chmod +x descargacartomadrid1-1000.sh + +echo "Start time: $TIMESTART" >> /tmp/pipelines.log +echo "Finish: `date`" >> /tmp/pipelines.log + +