Visualizing public transport

import requests
import shutil
import os
import zipfile

req = requests.get('http://www.bkk.hu/gtfs/budapest_gtfs.zip', stream=True)
with open('budapest_gtfs.zip', 'wb') as file:
    shutil.copyfileobj(req.raw, file)
    
if not os.path.exists('./data'):
    os.mkdir('./data')
    
with zipfile.ZipFile('budapest_gtfs.zip', 'r') as file:
    file.extractall('./data')

os.listdir('./data')

import pandas as pd

trips = pd.read_csv('data/trips.txt', low_memory=False)
shapes = pd.read_csv('data/shapes.txt', low_memory=False)

trips.head()

shapes.head()

num_trips_by_shape = trips.groupby('shape_id').aggregate({'route_id': 'count'}).reset_index()
num_trips_by_shape.rename({'route_id': 'trip_count'}, axis='columns', inplace=True)

shapes_with_nums = shapes.merge(num_trips_by_shape, on='shape_id')
shapes_with_nums.head()

from datashader.utils import lnglat_to_meters
import numpy as np

shapes_with_nums['coord_x'], shapes_with_nums['coord_y'] = \
    lnglat_to_meters(shapes_with_nums['shape_pt_lon'], shapes_with_nums['shape_pt_lat'])

def split_df_by(df, byvar):
    df_parts = []
    for level, df_part in df.groupby(byvar):
        empty = pd.DataFrame([[level if colname == byvar else np.NaN for colname in df.columns]],
                             columns=df.columns)
        df_parts.append(df_part.append(empty))
    return pd.concat(df_parts) 

sep_shapes = split_df_by(shapes_with_nums, 'shape_id')

import bokeh.plotting as bp
from bokeh.models.tiles import WMTSTileSource
import datashader as ds
import datashader.transfer_functions as tf
from datashader.bokeh_ext import InteractiveImage
from datashader.utils import export_image
import colorcet as cc

bp.output_notebook()

x_range=(2.101e6, 2.155e6)
y_range=(5.994e6, 6.052e6)

if not os.path.exists('./img'):
    os.mkdir('./img')

p = bp.figure(tools='pan,wheel_zoom,reset',
              plot_width=int(600),
              plot_height=int(600),
              x_range=x_range,
              y_range=y_range)

p.axis.visible = False
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None

url = "https://cartodb-basemaps-b.global.ssl.fastly.net/dark_all/{z}/{x}/{y}.png"
tile_renderer = p.add_tile(WMTSTileSource(url=url))
tile_renderer.alpha = 1

def image_callback(x_range, y_range, w, h, color_fn=tf.shade):
    cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range)
    agg = cvs.line(sep_shapes, 'coord_x', 'coord_y', agg=ds.sum('trip_count'))
    image = tf.shade(agg, cmap=cc.fire, how='eq_hist')
    return image

export_image(image_callback(x_range=x_range, y_range=y_range, w=2000, h=2000),
             filename="BUD_schedule_fire", background='black')
InteractiveImage(p, image_callback)

futar = pd.read_csv('C:/bkkdata/vfl_data.csv', encoding='cp1250')
print(len(futar))
futar.head()

lat_lon_df = futar.location.str.split(",", expand=True)
lats = lat_lon_df[0].str.replace("{'lat': ", "").astype(np.float64)
lons = lat_lon_df[1].str.replace(" 'lon': ", "").str.replace('}', '').astype(np.float64)

x_coords, y_coords = lnglat_to_meters(lons, lats)
gps_data = pd.DataFrame({'coord_x': x_coords, 'coord_y': y_coords})
gps_data.head()

p = bp.figure(tools='pan,wheel_zoom,reset',
              plot_width=int(600),
              plot_height=int(600),
              x_range=x_range,
              y_range=y_range)
              
p.axis.visible = False
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None

url = "https://cartodb-basemaps-b.global.ssl.fastly.net/dark_all/{z}/{x}/{y}.png"
tile_renderer = p.add_tile(WMTSTileSource(url=url))
tile_renderer.alpha = 1

def image_callback(x_range, y_range, w, h, color_fn=tf.shade):
    cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range)
    agg = cvs.points(gps_data, 'coord_x', 'coord_y', agg=ds.count())
    image = tf.shade(agg, cmap=cc.fire, how='eq_hist')
    return tf.dynspread(image, threshold=0.1, max_px=20)

export_image(image_callback(x_range=x_range, y_range=y_range, w=2000, h=2000),
             filename="BUD_gps_fire", background='black')
InteractiveImage(p, image_callback)

data_bus = gps_data[futar.vehicleRouteType == 'BUS']
data_tram = gps_data[futar.vehicleRouteType == 'TRAM']
data_trolleybus = gps_data[futar.vehicleRouteType == 'TROLLEYBUS']
data_ferry = gps_data[futar.vehicleRouteType == 'FERRY']
futar.vehicleRouteType.value_counts()

p = bp.figure(tools='pan,wheel_zoom,reset',
              plot_width=int(600),
              plot_height=int(600),
              x_range=(2.101e6, 2.155e6),
              y_range=(6012796, 6038876))
              
p.axis.visible = False
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None

url = "https://cartodb-basemaps-b.global.ssl.fastly.net/dark_all/{z}/{x}/{y}.png"
tile_renderer = p.add_tile(WMTSTileSource(url=url))
tile_renderer.alpha = 1

def image_callback(x_range, y_range, w, h, color_fn=tf.shade):
    cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range)
    agg_bus = cvs.points(data_bus, 'coord_x', 'coord_y', agg=ds.count())
    agg_tram = cvs.points(data_tram, 'coord_x', 'coord_y', agg=ds.count())
    agg_trolleybus = cvs.points(data_trolleybus, 'coord_x', 'coord_y', agg=ds.count())
    agg_ferry = cvs.points(data_ferry, 'coord_x', 'coord_y', agg=ds.count())
    image_bus = tf.shade(agg_bus, cmap=[(4, 71, 111), (8, 119, 188)], how='eq_hist')
    image_tram = tf.shade(agg_tram, cmap=[(115, 72, 0), (248, 155, 0)], how='eq_hist')
    image_trolleybus = tf.shade(agg_trolleybus, cmap=[(136, 0, 6), (255, 33, 42)], how='eq_hist')
    image_ferry = tf.shade(agg_ferry, cmap=[(112, 0, 67), (214, 0, 128)], how='eq_hist')
    image = tf.stack(image_bus, image_tram, image_trolleybus, image_ferry)
    return tf.dynspread(image, threshold=0.1, max_px=20)

export_image(image_callback(x_range=x_range, y_range=y_range, w=2000, h=2000),
             filename="BUD_gps_by_type", background='black')
InteractiveImage(p, image_callback)

import os
import requests
import time
import datetime
import pandas as pd

def vehicles_for_location(radius=None,
                          lat_span=None,
                          lon_span=None,
                          lat=47.498412,
                          lon=19.04037,
                          key='test',
                          version=3):
    
    if lat_span is None and lon_span is None and radius is None:
        raise TypeError("Either radius or lon_span and lat_span must be supplied")
    
    params = {'lat': lat,
              'lon': lon,
              'key': key,
              'version': version}
    if radius is not None:
        params['radius'] = radius
    else:
        params['latSpan'] = lat_span
        params['lonSpan'] = lon_span
    
    base_url = "http://futar.bkk.hu/bkk-utvonaltervezo-api/ws/otp/api/where/vehicles-for-location.json"
    req = requests.get(base_url, params=params)
    req.raise_for_status()
    
    if req.ok:
        print('Successful request at {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    
    return req.json()

def store_vfl(vfl_req, path):
    
    df = pd.DataFrame(vfl_req['data']['list'])

    if not os.path.isfile(path):
        with open(path, 'w') as file:
                df.to_csv(file, header=True)
    else:
        with open(path, 'a') as file:
            df.to_csv(file, header=False)

def run_scraper(interval,
                run_until,
                path,
                min_interval=2,
                radius=None,
                lat_span=None,
                lon_span=None):

    while datetime.datetime.now() < run_until:
        t0 = time.time()
        
        try:
            vfl = vehicles_for_location(radius=radius, lat_span=lat_span, lon_span=lon_span)
        except (KeyboardInterrupt, SystemExit):
            raise
        except requests.exceptions.HTTPError as e:
            print('\nUnuccessful request at {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
        except Exception as e:
            print('\nSomething went wrong at {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
        store_vfl(vfl, path)
        
        sleep_time = max(min_interval, t0 + interval - time.time())
        time.sleep(sleep_time)

# if __name__ == "__main__":
    # run_until = datetime.datetime(2018, 10, 12, 22, 10, 00)
    # path = '/bkkdata/vfl_data.csv'
    # run_scraper(10, run_until, path, 100)