parent
10aeff5144
commit
3c477b347a
@ -0,0 +1,149 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# Run all the pipelines |
||||||
|
# Una hora y media con calles-OSM, Direcciones, other-OSM, Ayto-MINT, Fuentes, EMT bus stops, Aseos |
||||||
|
|
||||||
|
TIMESTART=`date` |
||||||
|
|
||||||
|
echo "Processing calles-OSM..." |
||||||
|
mkdir -p ~/pipelines/calles-OSM |
||||||
|
cd ~/pipelines/calles-OSM |
||||||
|
wget https://cartobase.es/madrid_streets_extract.sh -O madrid_streets_extract.sh |
||||||
|
chmod +x madrid_streets_extract.sh |
||||||
|
./madrid_streets_extract.sh && cp current/callesmadrid* /var/www/html/ |
||||||
|
echo "callesmadrid-OSM.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
echo "Processing other OSM features..." |
||||||
|
mkdir -p ~/pipelines/OSM |
||||||
|
cd ~/pipelines/OSM |
||||||
|
wget https://cartobase.es/get_madrid_features_from_osm.sh -O get_madrid_features_from_osm.sh |
||||||
|
chmod +x get_madrid_features_from_osm.sh |
||||||
|
wget https://cartobase.es/process_madrid_features_from_osm.sh -O process_madrid_features_from_osm.sh |
||||||
|
chmod +x process_madrid_features_from_osm.sh |
||||||
|
./process_madrid_features_from_osm.sh |
||||||
|
cp current/* /var/www/html/ |
||||||
|
ls -1 current > /tmp/listaOSM |
||||||
|
while IFS= read -r line |
||||||
|
do |
||||||
|
echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
done < /tmp/listaOSM |
||||||
|
|
||||||
|
echo "Processing Ayto-MINT..." |
||||||
|
pip install --upgrade ogr2osm |
||||||
|
PATH=$PATH:~/.local/bin |
||||||
|
mkdir -p ~/pipelines/Ayto-MINT |
||||||
|
cd ~/pipelines/Ayto-MINT |
||||||
|
wget https://cartobase.es/Ayto-MINT.7z -O Ayto-MINT.7z |
||||||
|
7z x -aoa Ayto-MINT.7z |
||||||
|
mv Ayto-MINT/* . && rmdir Ayto-MINT && rm Ayto-MINT.7z |
||||||
|
chmod +x ./process_madrid_mint_features.sh |
||||||
|
./process_madrid_mint_features.sh |
||||||
|
cp data/* /var/www/html/ |
||||||
|
ls -1 data > /tmp/listaMINT |
||||||
|
while IFS= read -r line |
||||||
|
do |
||||||
|
echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
done < /tmp/listaMINT |
||||||
|
|
||||||
|
# 202202: fail, seguramente habrá cambiado algo en la fuente. Uso el anterior. |
||||||
|
#echo "Processing Fuentes..." |
||||||
|
#mkdir -p ~/pipelines/fuentes |
||||||
|
#cd ~/pipelines/fuentes |
||||||
|
#wget https://cartobase.es/fuentesaguapotablemadrid.sh -O fuentesaguapotablemadrid.sh |
||||||
|
#chmod +x fuentesaguapotablemadrid.sh |
||||||
|
#./fuentesaguapotablemadrid.sh |
||||||
|
#cp current/fuentes-madrid.osm /var/www/html/ |
||||||
|
#echo "fuentes-madrid.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
echo "Processing EMT bus stops..." |
||||||
|
mkdir -p ~/pipelines/EMT |
||||||
|
cd ~/pipelines/EMT |
||||||
|
wget https://cartobase.es/emt_bus_stop.sh -O emt_bus_stop.sh |
||||||
|
chmod +x emt_bus_stop.sh |
||||||
|
wget https://cartobase.es/reproyecta_coordenadas_emt_crtm.sh -O reproyecta_coordenadas_emt_crtm.sh |
||||||
|
chmod +x reproyecta_coordenadas_emt_crtm.sh |
||||||
|
./emt_bus_stop.sh |
||||||
|
cp emt_stops.osm /var/www/html/ |
||||||
|
echo "emt_stops.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
echo "Processing mobiliario (aseos)..." |
||||||
|
mkdir -p ~/pipelines/mobiliario |
||||||
|
cd ~/pipelines/mobiliario |
||||||
|
wget https://cartobase.es/procesa_aseos.sh -O procesa_aseos.sh |
||||||
|
chmod +x procesa_aseos.sh |
||||||
|
./procesa_aseos.sh |
||||||
|
cp aseos.osm /var/www/html/ |
||||||
|
echo "aseos.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
echo "Processing addresses..." |
||||||
|
mkdir -p ~/pipelines/direcciones |
||||||
|
cd ~/pipelines/direcciones |
||||||
|
wget https://cartobase.es/madridaddr2osm.sh -O madridaddr2osm.sh |
||||||
|
chmod +x madridaddr2osm.sh |
||||||
|
./madridaddr2osm.sh |
||||||
|
cp current/direccionesmadrid.osm /var/www/html/ |
||||||
|
echo "direccionesmadrid.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
echo "Ayto-API..." |
||||||
|
sudo apt-get install xsltproc -y |
||||||
|
mkdir -p ~/pipelines/Ayto-API |
||||||
|
cd ~/pipelines |
||||||
|
wget https://cartobase.es/Ayto-API.7z -O Ayto-API.7z |
||||||
|
7z -aoa x Ayto-API.7z |
||||||
|
cd ~/pipelines/Ayto-API |
||||||
|
chmod +x get_madrid_features_from_ayto-API.sh |
||||||
|
chmod +x process_api.csv.sh |
||||||
|
./process_api.csv.sh |
||||||
|
cd data |
||||||
|
rm tourisminformation-201105-0-informacion-turismo-Ayto-Madrid-API.osm # Tiene datos que ya están en tourisminformation-206577-0-oficinas-turismo-Ayto-Madrid-API.osm |
||||||
|
rm placeyesFIXME-202180-0-instalaciones-accesibles-no-muni-Ayto-Madrid-API.osm placeyesFIXME-202162-0-instalaciones-accesibles-municip-Ayto-Madrid-API.osm # Tienen datos de colegios, museos, ministerios, Oficinas municipales, salas de exposiciones... |
||||||
|
rm leisurepark-200761-0-parques-jardines-Ayto-Madrid-API.osm # mejor OSM |
||||||
|
rm cemeteryyes-205026-0-cementerios-Ayto-Madrid-API.osm # Mejor OSM |
||||||
|
cd .. |
||||||
|
cp data/*osm /var/www/html/ |
||||||
|
ls -1 data > /tmp/listaOSM |
||||||
|
while IFS= read -r line |
||||||
|
do |
||||||
|
echo "$line" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
done < /tmp/listaOSM |
||||||
|
|
||||||
|
echo "Processing BiciMAD..." |
||||||
|
mkdir -p ~/pipelines/bicimad |
||||||
|
cd ~/pipelines/bicimad |
||||||
|
wget https://cartobase.es/procesa_bicimad.sh -O procesa_bicimad.sh |
||||||
|
chmod +x procesa_bicimad.sh |
||||||
|
./procesa_bicimad.sh |
||||||
|
cp "current/amenity-bicycle_rental-bicimad_stations_EMT.osm" /var/www/html/ |
||||||
|
echo "amenity-bicycle_rental-bicimad_stations_EMT.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
|
||||||
|
#echo "Processing Árboles..." # Desactivado, hace falta JOSM de por medio. Y son 533M de archivo cuando ahora tenemos 546M. Revisar (Filomena?) |
||||||
|
#sudo apt install virtualenv npm -y |
||||||
|
##npm install -g geojsontoosm |
||||||
|
##sudo npm install geojson2osm |
||||||
|
#sudo npm install -g json2csv |
||||||
|
#sudo wget "https://raw.githubusercontent.com/OSMBrasil/csv2osm/master/csv2osm.py" -O /usr/local/bin/csv2osm.py && sudo chmod +x /usr/local/bin/csv2osm.py |
||||||
|
#mkdir -p ~/pipelines/arboles |
||||||
|
#cd ~/pipelines/arboles |
||||||
|
#wget https://cartobase.es/scrape-trees.sh -O scrape-trees.sh |
||||||
|
#chmod +x scrape-trees.sh |
||||||
|
#wget https://cartobase.es/procesa_arbolado_viario_scrapeado.sh -O procesa_arbolado_viario_scrapeado.sh |
||||||
|
#chmod +x procesa_arbolado_viario_scrapeado.sh |
||||||
|
#./scrape-trees.sh # Una media hora |
||||||
|
#./procesa_arbolado_viario_scrapeado.sh |
||||||
|
#cp Arbolado_Viario.osm /var/www/html/ |
||||||
|
#echo "Arbolado_Viario.osm" >> /var/www/html/sourcefiles ; sort -u /var/www/html/sourcefiles > /tmp/sf ; mv /tmp/sf /var/www/html/sourcefiles |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: Datos sin actualizar desde 2017, está a medias |
||||||
|
#echo "Ayto-carto..." |
||||||
|
#mkdir -p ~/pipelines/Ayto-carto |
||||||
|
#cd ~/pipelines/Ayto-carto |
||||||
|
#wget https://cartobase.es/descargacartomadrid1-1000.sh -O descargacartomadrid1-1000.sh |
||||||
|
#chmod +x descargacartomadrid1-1000.sh |
||||||
|
|
||||||
|
echo "Start time: $TIMESTART" >> /tmp/pipelines.log |
||||||
|
echo "Finish: `date`" >> /tmp/pipelines.log |
||||||
|
|
||||||
|
|
||||||
Loading…
Reference in new issue