diff --git a/fasttrips/Assignment.py b/fasttrips/Assignment.py index b4eee61b..3eae573f 100644 --- a/fasttrips/Assignment.py +++ b/fasttrips/Assignment.py @@ -2,6 +2,7 @@ from __future__ import division from past.builtins import execfile from future import standard_library + standard_library.install_aliases() from builtins import zip from builtins import str @@ -9,7 +10,7 @@ from builtins import object __copyright__ = "Copyright 2015-2017 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -22,6 +23,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import configparser import datetime import math @@ -50,150 +52,150 @@ class Assignment(object): """ #: Configuration file for fasttrips - CONFIGURATION_FILE = None + CONFIGURATION_FILE = None #: Configuration functions - CONFIGURATION_FUNCTIONS_FILE = None + CONFIGURATION_FUNCTIONS_FILE = None #: Output copy of the configuration file in case anything got overridden #: (Hmm naming conventions are a bit awkward here) - CONFIGURATION_OUTPUT_FILE = 'ft_output_config.txt' + CONFIGURATION_OUTPUT_FILE = 'ft_output_config.txt' #: Configuration: Input network directory - INPUT_NETWORK_ARCHIVE = None + INPUT_NETWORK_ARCHIVE = None #: Configuration: Input demand directory - INPUT_DEMAND_DIR = None + INPUT_DEMAND_DIR = None #: Configuration: Pathweight parameters - INPUT_WEIGHTS = None + INPUT_WEIGHTS = None #: Configuration: Run Configuration - OUTPUT_DIR = None + OUTPUT_DIR = None #: Configuration: Maximum number of iterations to remove capacity violations. When #: the transit system is not crowded or when capacity constraint is #: relaxed the model will terminate after the first iteration - MAX_ITERATIONS = None - MAX_PF_ITERATIONS = 10 - CONVERGENCE_GAP = None + MAX_ITERATIONS = None + MAX_PF_ITERATIONS = 10 + CONVERGENCE_GAP = None - NETWORK_BUILD_DATE = datetime.datetime.today() - NETWORK_BUILD_DATE_START_TIME = datetime.datetime.combine(NETWORK_BUILD_DATE, datetime.time()) + NETWORK_BUILD_DATE = datetime.datetime.today() + NETWORK_BUILD_DATE_START_TIME = datetime.datetime.combine(NETWORK_BUILD_DATE, datetime.time()) #: Find paths deterministically, using shortest path search based on travel time. - PATHFINDING_TYPE_DETERMINISTIC = 'deterministic' + PATHFINDING_TYPE_DETERMINISTIC = 'deterministic' #: Find paths stochastically using trip-based hyperpath - PATHFINDING_TYPE_STOCHASTIC = 'stochastic' + PATHFINDING_TYPE_STOCHASTIC = 'stochastic' #: Don't find paths; read :py:attr:`Passenger.PF_PATHS_CSV` and :py:attr:`Passenger.PF_LINKS_CSV`. - PATHFINDING_TYPE_READ_FILE = 'file' + PATHFINDING_TYPE_READ_FILE = 'file' #: Configuration: Pathfinding Type. Should be one of `Deterministic`, `Stochastic` or `File` - PATHFINDING_TYPE = None + PATHFINDING_TYPE = None #: Configuration: Do simulation? It should be True for iterative assignment. In a one shot #: assignment with simulation flag off, the passengers are assigned to #: paths but are not loaded to the network. Boolean. - SIMULATION = None + SIMULATION = None #: Configuration: Passenger trajectory output flag. Passengers' path and time will be #: reported if this flag is on. Note that the simulation flag should be on for #: passengers' time. Boolean. - OUTPUT_PASSENGER_TRAJECTORIES = None + OUTPUT_PASSENGER_TRAJECTORIES = None #: Configuration: If true, outputs pathset every simulation iteration. If false, #: outputs pathset every path-finding iteration. - OUTPUT_PATHSET_PER_SIM_ITER = None + OUTPUT_PATHSET_PER_SIM_ITER = None #: Configuration: Path time-window. This is the time in which the paths are generated. #: E.g. with a typical 30 min window, any path within 30 min of the #: departure time will be checked. A :py:class:`datetime.timedelta` instance. - TIME_WINDOW = None + TIME_WINDOW = None #: Configuration: Create skims flag. This is specific to the travel demand models #: (not working in this version). Boolean. - CREATE_SKIMS = None + CREATE_SKIMS = None #: Configuration: Beginning of the time period for which the skim is required. #: (specify as 'HH:MM'). A :py:class:`datetime.datetime` instance. - SKIM_START_TIME = None + SKIM_START_TIME = None #: Configuration: End of the time period for which the skim is required #: (specify as 'HH:MM'). A :py:class:`datetime.datetime` instance. - SKIM_END_TIME = None + SKIM_END_TIME = None #: Route choice configuration: Max number of paths in a pathset. #: Used in conjuntion with :py:attr:`Assignment.MIN_PATH_PROBABILITY` - MAX_NUM_PATHS = None + MAX_NUM_PATHS = None #: Route choice configuration: Minimum path probability for the path to be used. #: Used in conjucntion with :py:attr:`Assignment.MAX_NUM_PATHS`, so it only #: kicks in if that is specified AND we hit it, then we start dropping using #: this threshhold. - MIN_PATH_PROBABILITY = None + MIN_PATH_PROBABILITY = None #: Route choice configuration: Dispersion parameter in the logit function. #: Higher values result in less stochasticity. Must be nonnegative. #: If unknown use a value between 0.5 and 1. Float. - STOCH_DISPERSION = None + STOCH_DISPERSION = None #: Stop labeling configuration: Multiplies the utilities by this factor #: so that there are not negative costs labels which can result in #: lengthy and ineffective path-finding. Must be positive; should be greater #: than 1.0. Double. - UTILS_CONVERSION = None + UTILS_CONVERSION = None #: In path-finding, suppress trying to adjust fares using transfer fare rules. #: This is for performance testing. TRANSFER_FARE_IGNORE_PATHFINDING = None #: In path-enumeration, suppress trying to adjust fares using transfer fare rules. #: This is for performance testing. - TRANSFER_FARE_IGNORE_PATHENUM = None + TRANSFER_FARE_IGNORE_PATHENUM = None #: Route choice configuration: How many times max times should we process a stop #: during labeling? Use -1 to specify no max. Int. #: Setting this to a positive value may increase runtime but may decrease #: pathset quality. (Todo: test/quantify this.) - STOCH_MAX_STOP_PROCESS_COUNT = None + STOCH_MAX_STOP_PROCESS_COUNT = None #: Route choice configuration: How many stochastic paths will we generate #: (not necessarily unique) to define a path choice set? Int. - STOCH_PATHSET_SIZE = None + STOCH_PATHSET_SIZE = None #: Route choice configuration: Use vehicle capacity constraints. Boolean. - CAPACITY_CONSTRAINT = None + CAPACITY_CONSTRAINT = None #: Debug mode: only run trace passengers - DEBUG_TRACE_ONLY = False + DEBUG_TRACE_ONLY = False #: Debug mode: only run this number of trips, -1 to run all. Int. - DEBUG_NUM_TRIPS = -1 + DEBUG_NUM_TRIPS = -1 #: Debug: include debug columns in output - DEBUG_OUTPUT_COLUMNS = False + DEBUG_OUTPUT_COLUMNS = False #: Fare zone symmetry. If True, will assume fare zone symmetry. That is, if fare_id X is # configured from origin zone A to destination zone B, and there is no fare configured # from zone B to zone A, we'll assume that fare_id X also applies. - FARE_ZONE_SYMMETRY = False + FARE_ZONE_SYMMETRY = False #: Skip these passengers - SKIP_PERSON_IDS = None + SKIP_PERSON_IDS = None #: Trace these persons/person trips (a list of tuples) - TRACE_IDS = [] + TRACE_IDS = [] #: Prepend the route id to the trip id? This is for readability in debugging, since #: route IDs are typically more readable and trip ids are inscrutable - PREPEND_ROUTE_ID_TO_TRIP_ID = False + PREPEND_ROUTE_ID_TO_TRIP_ID = False #: Number of processes to use for path finding (via :py:mod:`multiprocessing`) #: Set to 1 to run everything in this process #: Set to less than 1 to use the result of :py:func:`multiprocessing.cpu_count` #: Set to positive integer greater than 1 to set a fixed number of processes - NUMBER_OF_PROCESSES = None + NUMBER_OF_PROCESSES = None #: Extra time so passengers don't get bumped (?). A :py:class:`datetime.timedelta` instance. - BUMP_BUFFER = None + BUMP_BUFFER = None #: This is the only simulation state that exists across iterations #: It's a dictionary of (trip_id, stop_id) -> earliest time a bumped passenger started waiting - bump_wait = {} - bump_wait_df = None + bump_wait = {} + bump_wait_df = None #: Simulation: bump one stop at a time (slower, more accurate) #: @@ -207,60 +209,60 @@ class Assignment(object): #: may not be necessary. Thus, the more accurate (but slower) method is to bump passengers from #: each (trip,stop) at a time, in order of the full vehicle arrival time, and then recalculate #: loads, and iterate until we have no capacity issues. Boolean. - BUMP_ONE_AT_A_TIME = None + BUMP_ONE_AT_A_TIME = None #: MSA the results that affect the next iteration to avoid oscillation: boards, alights, overcap onboard at stops - MSA_RESULTS = False + MSA_RESULTS = False #: Are we finding paths for everyone right now? Or just un-arrived folks? - PATHFINDING_EVERYONE = True + PATHFINDING_EVERYONE = True #: How many Simulation Iterations should we do before going back to path-finding? - MAX_SIMULATION_ITERS = 10 + MAX_SIMULATION_ITERS = 10 #: Column names for simulation - SIM_COL_PAX_BOARD_TIME = 'board_time' #: Board time on the transit vehicle - SIM_COL_PAX_ALIGHT_TIME = 'alight_time' #: Alight time from the transit vehicle - - SIM_COL_PAX_ALIGHT_DELAY_MIN = 'alight_delay_min' #: Delay in alight_time from original pathfinding understanding of alight time - SIM_COL_PAX_A_TIME = 'new_A_time' #: Time of arrival at A - SIM_COL_PAX_B_TIME = 'new_B_time' #: Time of arrival at B - SIM_COL_PAX_LINK_TIME = 'new_linktime' #: Link time (SIM_COL_PAX_B_TIME - SIM_COL_PAX_A_TIME) - SIM_COL_PAX_WAIT_TIME = 'new_waittime' #: Wait time - SIM_COL_PAX_MISSED_XFER = 'missed_xfer' #: Is this link a missed transfer - - SIM_COL_PAX_OVERCAP = Trip.SIM_COL_VEH_OVERCAP #: - SIM_COL_PAX_OVERCAP_FRAC = Trip.SIM_COL_VEH_OVERCAP_FRAC #: If board at an overcap stop, fraction of boards that are overcap - SIM_COL_PAX_BUMP_ITER = 'bump_iter' - SIM_COL_PAX_BOARD_STATE = 'board_state' #: NaN if not relevent, 1 if lucky enough to board at an at- or over-capacity stop, 0 if bumped. Set by :py:meth:`Assignment.flag_bump_overcap_passengers` - SIM_COL_PAX_DISTANCE = "distance" #: Link distance - SIM_COL_PAX_FARE = "fare" #: Link fare in currency - SIM_COL_PAX_FARE_PERIOD = "fare_period" #: Fare period id - SIM_COL_PAX_FREE_TRANSFER = "free_transfer" #: Free transfer? NaN, 0.0 or 1.0, only free based on `fare_attributes_ft.txt` - SIM_COL_PAX_COST = 'sim_cost' #: Link cost. (Cannot be `cost` because it collides with TAZ.DRIVE_ACCESS_COLUMN_COST) - SIM_COL_PAX_LNPS = 'ln_PS' #: log(PathSize) - SIM_COL_PAX_PROBABILITY = 'probability' #: Probability of this path - SIM_COL_PAX_LOGSUM = 'logsum' #: Logsum of all paths + SIM_COL_PAX_BOARD_TIME = 'board_time' #: Board time on the transit vehicle + SIM_COL_PAX_ALIGHT_TIME = 'alight_time' #: Alight time from the transit vehicle + + SIM_COL_PAX_ALIGHT_DELAY_MIN = 'alight_delay_min' #: Delay in alight_time from original pathfinding understanding of alight time + SIM_COL_PAX_A_TIME = 'new_A_time' #: Time of arrival at A + SIM_COL_PAX_B_TIME = 'new_B_time' #: Time of arrival at B + SIM_COL_PAX_LINK_TIME = 'new_linktime' #: Link time (SIM_COL_PAX_B_TIME - SIM_COL_PAX_A_TIME) + SIM_COL_PAX_WAIT_TIME = 'new_waittime' #: Wait time + SIM_COL_PAX_MISSED_XFER = 'missed_xfer' #: Is this link a missed transfer + + SIM_COL_PAX_OVERCAP = Trip.SIM_COL_VEH_OVERCAP #: + SIM_COL_PAX_OVERCAP_FRAC = Trip.SIM_COL_VEH_OVERCAP_FRAC #: If board at an overcap stop, fraction of boards that are overcap + SIM_COL_PAX_BUMP_ITER = 'bump_iter' + SIM_COL_PAX_BOARD_STATE = 'board_state' #: NaN if not relevent, 1 if lucky enough to board at an at- or over-capacity stop, 0 if bumped. Set by :py:meth:`Assignment.flag_bump_overcap_passengers` + SIM_COL_PAX_DISTANCE = "distance" #: Link distance + SIM_COL_PAX_FARE = "fare" #: Link fare in currency + SIM_COL_PAX_FARE_PERIOD = "fare_period" #: Fare period id + SIM_COL_PAX_FREE_TRANSFER = "free_transfer" #: Free transfer? NaN, 0.0 or 1.0, only free based on `fare_attributes_ft.txt` + SIM_COL_PAX_COST = 'sim_cost' #: Link cost. (Cannot be `cost` because it collides with TAZ.DRIVE_ACCESS_COLUMN_COST) + SIM_COL_PAX_LNPS = 'ln_PS' #: log(PathSize) + SIM_COL_PAX_PROBABILITY = 'probability' #: Probability of this path + SIM_COL_PAX_LOGSUM = 'logsum' #: Logsum of all paths #: Is this link/path a missed transfer? #: Set in both pathset links and pathset paths, this is a 1 or 0 - SIM_COL_MISSED_XFER = 'missed_xfer' + SIM_COL_MISSED_XFER = 'missed_xfer' #: Values for :py:attr:`Assignment.SIM_COL_PAX_BOARD_STATE` column - BOARD_STATE_CATEGORICAL = [ \ - "board_easy", #: path chosen and no capacity problems - "boarded", #: path chosen and lucky enough to board an at-capacity or over-capacity vehicle - "bumped", #: path chosen but bumped due to capacity problems - "bumped_othertrip", #: path invalidated due to being bumped on another link of this person's trip + BOARD_STATE_CATEGORICAL = [ \ + "board_easy", #: path chosen and no capacity problems + "boarded", #: path chosen and lucky enough to board an at-capacity or over-capacity vehicle + "bumped", #: path chosen but bumped due to capacity problems + "bumped_othertrip", #: path invalidated due to being bumped on another link of this person's trip "bumped_unchosen"] #: path invalidated before ever chosen due to capacity problems #: Chosen status for path - SIM_COL_PAX_CHOSEN = 'chosen' + SIM_COL_PAX_CHOSEN = 'chosen' #: categories for SIM_COL_PAX_CHOSEN - CHOSEN_NOT_CHOSEN_YET = "unchosen" - CHOSEN_REJECTED = "rejected" + CHOSEN_NOT_CHOSEN_YET = "unchosen" + CHOSEN_REJECTED = "rejected" #: These will be ordered, so to select chosen, choose those > CHOSEN_NOT_CHOSEN_YET - CHOSEN_CATEGORIES = [CHOSEN_REJECTED,CHOSEN_NOT_CHOSEN_YET] + CHOSEN_CATEGORIES = [CHOSEN_REJECTED, CHOSEN_NOT_CHOSEN_YET] def __init__(self): """ @@ -285,141 +287,147 @@ def read_configuration(config_fullpath): """ Read the configuration parameters from :py:attr:`Assignment.CONFIGURATION_FILE` """ - pd.set_option('display.width', 1000) + pd.set_option('display.width', 1000) # pd.set_option('display.height', 1000) - pd.set_option('display.max_rows', 1000) + pd.set_option('display.max_rows', 1000) pd.set_option('display.max_columns', 100) parser = configparser.RawConfigParser( - defaults={'max_iterations' :1, - 'max_pf_iterations' :1, - 'network_build_date' : datetime.date.today().strftime("%m/%d/%Y"), - 'simulation' :'True', - 'learning_convergence' :'False', - 'learning_rate' : 0.01, - 'convergence_gap' : 0.001, - 'output_pathset_per_sim_iter' :'False', - 'output_passenger_trajectories' :'True', - 'create_skims' :'False', - 'skim_start_time' :'5:00', - 'skim_end_time' :'10:00', - 'capacity_constraint' :'False', - 'skip_person_ids' :'None', - 'trace_ids' :[], - 'debug_trace_only' :'False', - 'debug_num_trips' :-1, - 'debug_output_columns' :'False', - 'fare_zone_symmetry' :'False', - 'prepend_route_id_to_trip_id' :'False', - 'number_of_processes' :0, - 'bump_buffer' :5, - 'bump_one_at_a_time' :'False', + defaults={'max_iterations': 1, + 'max_pf_iterations': 1, + 'network_build_date': datetime.date.today().strftime("%m/%d/%Y"), + 'simulation': 'True', + 'learning_convergence': 'False', + 'learning_rate': 0.01, + 'convergence_gap': 0.001, + 'output_pathset_per_sim_iter': 'False', + 'output_passenger_trajectories': 'True', + 'create_skims': 'False', + 'skim_start_time': '5:00', + 'skim_end_time': '10:00', + 'capacity_constraint': 'False', + 'skip_person_ids': 'None', + 'trace_ids': [], + 'debug_trace_only': 'False', + 'debug_num_trips': -1, + 'debug_output_columns': 'False', + 'fare_zone_symmetry': 'False', + 'prepend_route_id_to_trip_id': 'False', + 'number_of_processes': 0, + 'bump_buffer': 5, + 'bump_one_at_a_time': 'False', # pathfinding - 'max_num_paths' :-1, - 'min_path_probability' :0.005, - 'min_transfer_penalty' :0.1, - 'overlap_chunk_size' :500, - 'overlap_scale_parameter' :1.0, - 'overlap_split_transit' :'False', - 'overlap_variable' :'count', - 'pathfinding_type' :Assignment.PATHFINDING_TYPE_STOCHASTIC, - 'pathweights_fixed_width' :'False', - 'utils_conversion_factor' :1.0, - 'stochastic_dispersion' :1.0, - 'stochastic_max_stop_process_count':20, - 'stochastic_pathset_size' :1000, - 'time_window' :30, - 'transfer_fare_ignore_pathfinding' :'False', - 'transfer_fare_ignore_pathenum' :'False', - 'user_class_function' :'generic_user_class', - 'arrive_late_allowed_min' : 0, - 'depart_early_allowed_min' : 0, - }) + 'max_num_paths': -1, + 'min_path_probability': 0.005, + 'min_transfer_penalty': 0.1, + 'overlap_chunk_size': 500, + 'overlap_scale_parameter': 1.0, + 'overlap_split_transit': 'False', + 'overlap_variable': 'count', + 'pathfinding_type': Assignment.PATHFINDING_TYPE_STOCHASTIC, + 'pathweights_fixed_width': 'False', + 'utils_conversion_factor': 1.0, + 'stochastic_dispersion': 1.0, + 'stochastic_max_stop_process_count': 20, + 'stochastic_pathset_size': 1000, + 'time_window': 30, + 'transfer_fare_ignore_pathfinding': 'False', + 'transfer_fare_ignore_pathenum': 'False', + 'user_class_function': 'generic_user_class', + 'arrive_late_allowed_min': 0, + 'depart_early_allowed_min': 0, + }) # Read configuration from specified configuration directory FastTripsLogger.info("Reading configuration file %s" % config_fullpath) parser.read(config_fullpath) - Assignment.MAX_ITERATIONS = parser.getint ('fasttrips','max_iterations') - Assignment.MAX_PF_ITERATIONS = parser.getint ('fasttrips','max_pf_iterations') - Assignment.NETWORK_BUILD_DATE = datetime.datetime.strptime( - parser.get('fasttrips', 'network_build_date'), '%m/%d/%Y').date() - Assignment.NETWORK_BUILD_DATE_START_TIME = datetime.datetime.combine(Assignment.NETWORK_BUILD_DATE, datetime.time()) - Assignment.SIMULATION = parser.getboolean('fasttrips','simulation') - PathSet.LEARN_ROUTES = parser.getboolean('fasttrips', 'learning_convergence') - PathSet.LEARN_ROUTES_RATE = parser.getfloat('fasttrips', 'learning_rate') - Assignment.CONVERGENCE_GAP = parser.getfloat('fasttrips', 'convergence_gap') - Assignment.OUTPUT_PASSENGER_TRAJECTORIES = parser.getboolean('fasttrips','output_passenger_trajectories') - Assignment.OUTPUT_PATHSET_PER_SIM_ITER = parser.getboolean('fasttrips','output_pathset_per_sim_iter') - Assignment.CREATE_SKIMS = parser.getboolean('fasttrips','create_skims') + Assignment.MAX_ITERATIONS = parser.getint('fasttrips', 'max_iterations') + Assignment.MAX_PF_ITERATIONS = parser.getint('fasttrips', 'max_pf_iterations') + Assignment.NETWORK_BUILD_DATE = datetime.datetime.strptime( + parser.get('fasttrips', 'network_build_date'), '%m/%d/%Y').date() + Assignment.NETWORK_BUILD_DATE_START_TIME = datetime.datetime.combine(Assignment.NETWORK_BUILD_DATE, + datetime.time()) + Assignment.SIMULATION = parser.getboolean('fasttrips', 'simulation') + PathSet.LEARN_ROUTES = parser.getboolean('fasttrips', 'learning_convergence') + PathSet.LEARN_ROUTES_RATE = parser.getfloat('fasttrips', 'learning_rate') + Assignment.CONVERGENCE_GAP = parser.getfloat('fasttrips', 'convergence_gap') + Assignment.OUTPUT_PASSENGER_TRAJECTORIES = parser.getboolean('fasttrips', 'output_passenger_trajectories') + Assignment.OUTPUT_PATHSET_PER_SIM_ITER = parser.getboolean('fasttrips', 'output_pathset_per_sim_iter') + Assignment.CREATE_SKIMS = parser.getboolean('fasttrips', 'create_skims') Assignment.SKIM_START_TIME = datetime.datetime.strptime( - parser.get ('fasttrips','skim_start_time'),'%H:%M') - Assignment.SKIM_END_TIME = datetime.datetime.strptime( - parser.get ('fasttrips','skim_end_time'),'%H:%M') - Assignment.CAPACITY_CONSTRAINT = parser.getboolean('fasttrips','capacity_constraint') - Assignment.SKIP_PERSON_IDS = eval(parser.get ('fasttrips','skip_person_ids')) + parser.get('fasttrips', 'skim_start_time'), '%H:%M') + Assignment.SKIM_END_TIME = datetime.datetime.strptime( + parser.get('fasttrips', 'skim_end_time'), '%H:%M') + Assignment.CAPACITY_CONSTRAINT = parser.getboolean('fasttrips', 'capacity_constraint') + Assignment.SKIP_PERSON_IDS = eval(parser.get('fasttrips', 'skip_person_ids')) try: - Assignment.TRACE_IDS = eval(parser.get ('fasttrips','trace_ids')) + Assignment.TRACE_IDS = eval(parser.get('fasttrips', 'trace_ids')) except: e = "Must have a TRACE_IDS line in config_ft.txt; even if it is trace_ids = []" print(e) FastTripsLogger.error(e) raise - Assignment.DEBUG_TRACE_ONLY = parser.getboolean('fasttrips','debug_trace_only') - Assignment.DEBUG_NUM_TRIPS = parser.getint ('fasttrips','debug_num_trips') - Assignment.DEBUG_OUTPUT_COLUMNS = parser.getboolean('fasttrips','debug_output_columns') - Assignment.FARE_ZONE_SYMMETRY = parser.getboolean('fasttrips','fare_zone_symmetry') - Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID = parser.getboolean('fasttrips','prepend_route_id_to_trip_id') - Assignment.NUMBER_OF_PROCESSES = parser.getint ('fasttrips','number_of_processes') + Assignment.DEBUG_TRACE_ONLY = parser.getboolean('fasttrips', 'debug_trace_only') + Assignment.DEBUG_NUM_TRIPS = parser.getint('fasttrips', 'debug_num_trips') + Assignment.DEBUG_OUTPUT_COLUMNS = parser.getboolean('fasttrips', 'debug_output_columns') + Assignment.FARE_ZONE_SYMMETRY = parser.getboolean('fasttrips', 'fare_zone_symmetry') + Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID = parser.getboolean('fasttrips', 'prepend_route_id_to_trip_id') + Assignment.NUMBER_OF_PROCESSES = parser.getint('fasttrips', 'number_of_processes') Assignment.BUMP_BUFFER = datetime.timedelta( - minutes = parser.getfloat ('fasttrips','bump_buffer')) - Assignment.BUMP_ONE_AT_A_TIME = parser.getboolean('fasttrips','bump_one_at_a_time') + minutes=parser.getfloat('fasttrips', 'bump_buffer')) + Assignment.BUMP_ONE_AT_A_TIME = parser.getboolean('fasttrips', 'bump_one_at_a_time') # pathfinding - Assignment.MAX_NUM_PATHS = parser.getint ('pathfinding','max_num_paths') - Assignment.MIN_PATH_PROBABILITY = parser.getfloat ('pathfinding','min_path_probability') - PathSet.MIN_TRANSFER_PENALTY = parser.getfloat ('pathfinding','min_transfer_penalty') - PathSet.OVERLAP_CHUNK_SIZE = parser.getint ('pathfinding','overlap_chunk_size') - PathSet.OVERLAP_SCALE_PARAMETER = parser.getfloat ('pathfinding','overlap_scale_parameter') - PathSet.OVERLAP_SPLIT_TRANSIT = parser.getboolean('pathfinding','overlap_split_transit') - PathSet.OVERLAP_VARIABLE = parser.get ('pathfinding','overlap_variable') - Assignment.PATHFINDING_TYPE = parser.get ('pathfinding','pathfinding_type') - PathSet.WEIGHTS_FIXED_WIDTH = parser.getboolean('pathfinding','pathweights_fixed_width') - Assignment.STOCH_DISPERSION = parser.getfloat ('pathfinding','stochastic_dispersion') - Assignment.UTILS_CONVERSION = parser.getfloat ('pathfinding','utils_conversion_factor') - Assignment.STOCH_MAX_STOP_PROCESS_COUNT = parser.getint ('pathfinding','stochastic_max_stop_process_count') - Assignment.STOCH_PATHSET_SIZE = parser.getint ('pathfinding','stochastic_pathset_size') + Assignment.MAX_NUM_PATHS = parser.getint('pathfinding', 'max_num_paths') + Assignment.MIN_PATH_PROBABILITY = parser.getfloat('pathfinding', 'min_path_probability') + PathSet.MIN_TRANSFER_PENALTY = parser.getfloat('pathfinding', 'min_transfer_penalty') + PathSet.OVERLAP_CHUNK_SIZE = parser.getint('pathfinding', 'overlap_chunk_size') + PathSet.OVERLAP_SCALE_PARAMETER = parser.getfloat('pathfinding', 'overlap_scale_parameter') + PathSet.OVERLAP_SPLIT_TRANSIT = parser.getboolean('pathfinding', 'overlap_split_transit') + PathSet.OVERLAP_VARIABLE = parser.get('pathfinding', 'overlap_variable') + Assignment.PATHFINDING_TYPE = parser.get('pathfinding', 'pathfinding_type') + PathSet.WEIGHTS_FIXED_WIDTH = parser.getboolean('pathfinding', 'pathweights_fixed_width') + Assignment.STOCH_DISPERSION = parser.getfloat('pathfinding', 'stochastic_dispersion') + Assignment.UTILS_CONVERSION = parser.getfloat('pathfinding', 'utils_conversion_factor') + Assignment.STOCH_MAX_STOP_PROCESS_COUNT = parser.getint('pathfinding', 'stochastic_max_stop_process_count') + Assignment.STOCH_PATHSET_SIZE = parser.getint('pathfinding', 'stochastic_pathset_size') Assignment.TIME_WINDOW = datetime.timedelta( - minutes = parser.getfloat ('pathfinding','time_window')) + minutes=parser.getfloat('pathfinding', 'time_window')) - Assignment.TRANSFER_FARE_IGNORE_PATHFINDING = parser.getboolean('pathfinding','transfer_fare_ignore_pathfinding') - Assignment.TRANSFER_FARE_IGNORE_PATHENUM = parser.getboolean('pathfinding','transfer_fare_ignore_pathenum') - PathSet.USER_CLASS_FUNCTION = parser.get ('pathfinding','user_class_function') - PathSet.DEPART_EARLY_ALLOWED_MIN = datetime.timedelta( - minutes = parser.getfloat('pathfinding', 'depart_early_allowed_min')) - PathSet.ARRIVE_LATE_ALLOWED_MIN = datetime.timedelta( - minutes = parser.getfloat ('pathfinding','arrive_late_allowed_min')) + Assignment.TRANSFER_FARE_IGNORE_PATHFINDING = parser.getboolean('pathfinding', + 'transfer_fare_ignore_pathfinding') + Assignment.TRANSFER_FARE_IGNORE_PATHENUM = parser.getboolean('pathfinding', 'transfer_fare_ignore_pathenum') + PathSet.USER_CLASS_FUNCTION = parser.get('pathfinding', 'user_class_function') + PathSet.DEPART_EARLY_ALLOWED_MIN = datetime.timedelta( + minutes=parser.getfloat('pathfinding', 'depart_early_allowed_min')) + PathSet.ARRIVE_LATE_ALLOWED_MIN = datetime.timedelta( + minutes=parser.getfloat('pathfinding', 'arrive_late_allowed_min')) if Assignment.PATHFINDING_TYPE not in [Assignment.PATHFINDING_TYPE_STOCHASTIC, \ Assignment.PATHFINDING_TYPE_DETERMINISTIC, \ Assignment.PATHFINDING_TYPE_READ_FILE]: - msg = "pathfinding type [%s] not available. Expected values: %s" % (Assignment.PATHFINDING_TYPE, str([Assignment.PATHFINDING_TYPE_STOCHASTIC, Assignment.PATHFINDING_TYPE_DETERMINISTIC, Assignment.PATHFINDING_TYPE_READ_FILE])) + msg = "pathfinding type [%s] not available. Expected values: %s" % (Assignment.PATHFINDING_TYPE, str([ + Assignment.PATHFINDING_TYPE_STOCHASTIC, + Assignment.PATHFINDING_TYPE_DETERMINISTIC, + Assignment.PATHFINDING_TYPE_READ_FILE])) FastTripsLogger.fatal(msg) raise ConfigurationError(config_fullpath, msg) if PathSet.OVERLAP_VARIABLE not in PathSet.OVERLAP_VARIABLE_OPTIONS: - msg = "pathfinding.overlap_variable [%s] not defined. Expected values: %s" % (PathSet.OVERLAP_VARIABLE, str(PathSet.OVERLAP_VARIABLE_OPTIONS)) + msg = "pathfinding.overlap_variable [%s] not defined. Expected values: %s" % ( + PathSet.OVERLAP_VARIABLE, str(PathSet.OVERLAP_VARIABLE_OPTIONS)) FastTripsLogger.fatal(msg) raise ConfigurationError(config_fullpath, msg) if PathSet.USER_CLASS_FUNCTION not in PathSet.CONFIGURED_FUNCTIONS: - msg = "User class function [%s] not defined. Please check your function file [%s]" % (PathSet.USER_CLASS_FUNCTION, Assignment.CONFIGURATION_FUNCTIONS_FILE) + msg = "User class function [%s] not defined. Please check your function file [%s]" % ( + PathSet.USER_CLASS_FUNCTION, Assignment.CONFIGURATION_FUNCTIONS_FILE) FastTripsLogger.fatal(msg) raise ConfigurationError(config_fullpath, msg) - @staticmethod - def read_weights(weights_file = INPUT_WEIGHTS): + def read_weights(weights_file=INPUT_WEIGHTS): """ Read the weights from :py:attr:`Assignment.INPUT_WEIGHTS """ @@ -431,16 +439,15 @@ def read_weights(weights_file = INPUT_WEIGHTS): weights = pd.read_fwf(weights_file) weights[PathSet.WEIGHTS_COLUMN_PURPOSE] = weights[PathSet.WEIGHTS_COLUMN_PURPOSE].astype(str) else: - print (PathSet.WEIGHTS_COLUMN_PURPOSE) + print(PathSet.WEIGHTS_COLUMN_PURPOSE) ##LMZ - weights = pd.read_csv(weights_file, dtype={PathSet.WEIGHTS_COLUMN_PURPOSE:'S'}, skipinitialspace=True) + weights = pd.read_csv(weights_file, dtype={PathSet.WEIGHTS_COLUMN_PURPOSE: 'S'}, skipinitialspace=True) PathSet.WEIGHTS_DF = Assignment.process_weight_qualifiers(weights) FastTripsLogger.debug("Weights =\n%s" % str(PathSet.WEIGHTS_DF)) FastTripsLogger.debug("Weight types = \n%s" % str(PathSet.WEIGHTS_DF.dtypes)) - @staticmethod def process_weight_qualifiers(weights): """ @@ -464,8 +471,10 @@ def process_weight_qualifiers(weights): if growth_type.shape[0] == 0: return weights - growth_type[PathSet.WEIGHTS_GROWTH_TYPE] = growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.extract('((?<=\.)\w+)', expand=False) - growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] = growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.extract('(\w+(?=\.))', expand=False) + growth_type[PathSet.WEIGHTS_GROWTH_TYPE] = growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.extract( + '((?<=\.)\w+)', expand=False) + growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] = growth_type[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.extract( + '(\w+(?=\.))', expand=False) weights = pd.concat([weights, growth_type]) if (~weights[PathSet.WEIGHTS_GROWTH_TYPE].isin(PathSet.PENALTY_GROWTH_MODELS)).any(): @@ -495,32 +504,31 @@ def process_weight_qualifiers(weights): weights_df = pd.merge(weights, qualifiers, on=merge_cols, how='left') - #Check for required logarithmic attributes and that they are all non-zero and positive + # Check for required logarithmic attributes and that they are all non-zero and positive if len(weights_df[weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGARITHMIC_GROWTH_MODEL]) > 0: - assert(PathSet.WEIGHTS_GROWTH_LOG_BASE in weights_df) - assert( - len(weights_df[ - (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGARITHMIC_GROWTH_MODEL) & - (weights_df[PathSet.WEIGHTS_GROWTH_LOG_BASE] <= 0) - ]) == 0) + assert (PathSet.WEIGHTS_GROWTH_LOG_BASE in weights_df) + assert ( + len(weights_df[ + (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGARITHMIC_GROWTH_MODEL) & + (weights_df[PathSet.WEIGHTS_GROWTH_LOG_BASE] <= 0) + ]) == 0) if len(weights_df[weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL]) > 0: - assert(PathSet.WEIGHTS_GROWTH_LOGISTIC_MID in weights_df) + assert (PathSet.WEIGHTS_GROWTH_LOGISTIC_MID in weights_df) assert (PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX in weights_df) - assert( - len(weights_df[ - (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL) & - (weights_df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX] <= 0) - ]) == 0) assert ( - len(weights_df[ - (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL) & - (weights_df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MID] <= 0) - ]) == 0) + len(weights_df[ + (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL) & + (weights_df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX] <= 0) + ]) == 0) + assert ( + len(weights_df[ + (weights_df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL) & + (weights_df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MID] <= 0) + ]) == 0) return weights_df - @staticmethod def write_configuration(output_dir): """ @@ -528,59 +536,65 @@ def write_configuration(output_dir): """ parser = configparser.SafeConfigParser() parser.add_section('fasttrips') - parser.set('fasttrips','input_demand_dir', Assignment.INPUT_DEMAND_DIR) - parser.set('fasttrips','input_network_dir', Assignment.INPUT_NETWORK_ARCHIVE) - parser.set('fasttrips','input_weights', Assignment.INPUT_WEIGHTS) + parser.set('fasttrips', 'input_demand_dir', Assignment.INPUT_DEMAND_DIR) + parser.set('fasttrips', 'input_network_dir', Assignment.INPUT_NETWORK_ARCHIVE) + parser.set('fasttrips', 'input_weights', Assignment.INPUT_WEIGHTS) if Assignment.CONFIGURATION_FUNCTIONS_FILE: - parser.set('fasttrips','input_functions', Assignment.CONFIGURATION_FUNCTIONS_FILE) - parser.set('fasttrips','run_config', Assignment.CONFIGURATION_FILE) - - - parser.set('fasttrips','max_iterations', '%d' % Assignment.MAX_ITERATIONS) - parser.set('fasttrips','max_pf_iterations', '%d' % Assignment.MAX_PF_ITERATIONS) - parser.set('fasttrips','simulation', 'True' if Assignment.SIMULATION else 'False') - parser.set('fasttrips','output_dir', Assignment.OUTPUT_DIR) - parser.set('fasttrips','output_passenger_trajectories', 'True' if Assignment.OUTPUT_PASSENGER_TRAJECTORIES else 'False') - parser.set('fasttrips','output_pathset_per_sim_iter', 'True' if Assignment.OUTPUT_PATHSET_PER_SIM_ITER else 'False') - parser.set('fasttrips','create_skims', 'True' if Assignment.CREATE_SKIMS else 'False') - parser.set('fasttrips','skim_start_time', Assignment.SKIM_START_TIME.strftime('%H:%M')) - parser.set('fasttrips','skim_end_time', Assignment.SKIM_END_TIME.strftime('%H:%M')) - parser.set('fasttrips','capacity_constraint', 'True' if Assignment.CAPACITY_CONSTRAINT else 'False') - parser.set('fasttrips','skip_person_ids', '%s' % str(Assignment.SKIP_PERSON_IDS)) - parser.set('fasttrips','trace_ids', '%s' % str(Assignment.TRACE_IDS)) - parser.set('fasttrips','debug_trace_only', 'True' if Assignment.DEBUG_TRACE_ONLY else 'False') - parser.set('fasttrips','debug_num_trips', '%d' % Assignment.DEBUG_NUM_TRIPS) - parser.set('fasttrips','debug_output_columns', 'True' if Assignment.DEBUG_OUTPUT_COLUMNS else 'False') - parser.set('fasttrips','fare_zone_symmetry', 'True' if Assignment.FARE_ZONE_SYMMETRY else 'False') - parser.set('fasttrips','prepend_route_id_to_trip_id', 'True' if Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID else 'False') - parser.set('fasttrips','number_of_processes', '%d' % Assignment.NUMBER_OF_PROCESSES) - parser.set('fasttrips','bump_buffer', '%f' % (Assignment.BUMP_BUFFER.total_seconds()/60.0)) - parser.set('fasttrips','bump_one_at_a_time', 'True' if Assignment.BUMP_ONE_AT_A_TIME else 'False') - - #pathfinding + parser.set('fasttrips', 'input_functions', Assignment.CONFIGURATION_FUNCTIONS_FILE) + parser.set('fasttrips', 'run_config', Assignment.CONFIGURATION_FILE) + + parser.set('fasttrips', 'max_iterations', '%d' % Assignment.MAX_ITERATIONS) + parser.set('fasttrips', 'max_pf_iterations', '%d' % Assignment.MAX_PF_ITERATIONS) + parser.set('fasttrips', 'simulation', 'True' if Assignment.SIMULATION else 'False') + parser.set('fasttrips', 'output_dir', Assignment.OUTPUT_DIR) + parser.set('fasttrips', 'output_passenger_trajectories', + 'True' if Assignment.OUTPUT_PASSENGER_TRAJECTORIES else 'False') + parser.set('fasttrips', 'output_pathset_per_sim_iter', + 'True' if Assignment.OUTPUT_PATHSET_PER_SIM_ITER else 'False') + parser.set('fasttrips', 'create_skims', 'True' if Assignment.CREATE_SKIMS else 'False') + parser.set('fasttrips', 'skim_start_time', Assignment.SKIM_START_TIME.strftime('%H:%M')) + parser.set('fasttrips', 'skim_end_time', Assignment.SKIM_END_TIME.strftime('%H:%M')) + parser.set('fasttrips', 'capacity_constraint', 'True' if Assignment.CAPACITY_CONSTRAINT else 'False') + parser.set('fasttrips', 'skip_person_ids', '%s' % str(Assignment.SKIP_PERSON_IDS)) + parser.set('fasttrips', 'trace_ids', '%s' % str(Assignment.TRACE_IDS)) + parser.set('fasttrips', 'debug_trace_only', 'True' if Assignment.DEBUG_TRACE_ONLY else 'False') + parser.set('fasttrips', 'debug_num_trips', '%d' % Assignment.DEBUG_NUM_TRIPS) + parser.set('fasttrips', 'debug_output_columns', 'True' if Assignment.DEBUG_OUTPUT_COLUMNS else 'False') + parser.set('fasttrips', 'fare_zone_symmetry', 'True' if Assignment.FARE_ZONE_SYMMETRY else 'False') + parser.set('fasttrips', 'prepend_route_id_to_trip_id', + 'True' if Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID else 'False') + parser.set('fasttrips', 'number_of_processes', '%d' % Assignment.NUMBER_OF_PROCESSES) + parser.set('fasttrips', 'bump_buffer', '%f' % (Assignment.BUMP_BUFFER.total_seconds() / 60.0)) + parser.set('fasttrips', 'bump_one_at_a_time', 'True' if Assignment.BUMP_ONE_AT_A_TIME else 'False') + + # pathfinding parser.add_section('pathfinding') - parser.set('pathfinding','max_num_paths', '%d' % Assignment.MAX_NUM_PATHS) - parser.set('pathfinding','min_path_probability', '%f' % Assignment.MIN_PATH_PROBABILITY) - parser.set('pathfinding','min_transfer_penalty', '%f' % PathSet.MIN_TRANSFER_PENALTY) - parser.set('pathfinding','overlap_chunk_size', '%d' % PathSet.OVERLAP_CHUNK_SIZE) - parser.set('pathfinding','overlap_scale_parameter', '%f' % PathSet.OVERLAP_SCALE_PARAMETER) - parser.set('pathfinding','overlap_split_transit', 'True' if PathSet.OVERLAP_SPLIT_TRANSIT else 'False') - parser.set('pathfinding','overlap_variable', '%s' % PathSet.OVERLAP_VARIABLE) - parser.set('pathfinding','pathfinding_type', Assignment.PATHFINDING_TYPE) - parser.set('pathfinding','pathweights_fixed_width', 'True' if PathSet.WEIGHTS_FIXED_WIDTH else 'False') - parser.set('pathfinding','stochastic_dispersion', '%f' % Assignment.STOCH_DISPERSION) - parser.set('pathfinding','utils_conversion_factor', '%f' % Assignment.UTILS_CONVERSION) - parser.set('pathfinding','stochastic_max_stop_process_count', '%d' % Assignment.STOCH_MAX_STOP_PROCESS_COUNT) - parser.set('pathfinding','stochastic_pathset_size', '%d' % Assignment.STOCH_PATHSET_SIZE) - parser.set('pathfinding','time_window', '%f' % (Assignment.TIME_WINDOW.total_seconds()/60.0)) - - parser.set('pathfinding','transfer_fare_ignore_pathfinding', 'True' if Assignment.TRANSFER_FARE_IGNORE_PATHFINDING else 'False') - parser.set('pathfinding','transfer_fare_ignore_pathenum', 'True' if Assignment.TRANSFER_FARE_IGNORE_PATHENUM else 'False') - - parser.set('pathfinding','user_class_function', '%s' % PathSet.USER_CLASS_FUNCTION) - - parser.set('pathfinding','arrive_late_allowed_min', '%f' % (PathSet.ARRIVE_LATE_ALLOWED_MIN.total_seconds()/60.0)) - parser.set('pathfinding','depart_early_allowed_min', '%f' % (PathSet.DEPART_EARLY_ALLOWED_MIN.total_seconds()/60.0)) + parser.set('pathfinding', 'max_num_paths', '%d' % Assignment.MAX_NUM_PATHS) + parser.set('pathfinding', 'min_path_probability', '%f' % Assignment.MIN_PATH_PROBABILITY) + parser.set('pathfinding', 'min_transfer_penalty', '%f' % PathSet.MIN_TRANSFER_PENALTY) + parser.set('pathfinding', 'overlap_chunk_size', '%d' % PathSet.OVERLAP_CHUNK_SIZE) + parser.set('pathfinding', 'overlap_scale_parameter', '%f' % PathSet.OVERLAP_SCALE_PARAMETER) + parser.set('pathfinding', 'overlap_split_transit', 'True' if PathSet.OVERLAP_SPLIT_TRANSIT else 'False') + parser.set('pathfinding', 'overlap_variable', '%s' % PathSet.OVERLAP_VARIABLE) + parser.set('pathfinding', 'pathfinding_type', Assignment.PATHFINDING_TYPE) + parser.set('pathfinding', 'pathweights_fixed_width', 'True' if PathSet.WEIGHTS_FIXED_WIDTH else 'False') + parser.set('pathfinding', 'stochastic_dispersion', '%f' % Assignment.STOCH_DISPERSION) + parser.set('pathfinding', 'utils_conversion_factor', '%f' % Assignment.UTILS_CONVERSION) + parser.set('pathfinding', 'stochastic_max_stop_process_count', '%d' % Assignment.STOCH_MAX_STOP_PROCESS_COUNT) + parser.set('pathfinding', 'stochastic_pathset_size', '%d' % Assignment.STOCH_PATHSET_SIZE) + parser.set('pathfinding', 'time_window', '%f' % (Assignment.TIME_WINDOW.total_seconds() / 60.0)) + + parser.set('pathfinding', 'transfer_fare_ignore_pathfinding', + 'True' if Assignment.TRANSFER_FARE_IGNORE_PATHFINDING else 'False') + parser.set('pathfinding', 'transfer_fare_ignore_pathenum', + 'True' if Assignment.TRANSFER_FARE_IGNORE_PATHENUM else 'False') + + parser.set('pathfinding', 'user_class_function', '%s' % PathSet.USER_CLASS_FUNCTION) + + parser.set('pathfinding', 'arrive_late_allowed_min', + '%f' % (PathSet.ARRIVE_LATE_ALLOWED_MIN.total_seconds() / 60.0)) + parser.set('pathfinding', 'depart_early_allowed_min', + '%f' % (PathSet.DEPART_EARLY_ALLOWED_MIN.total_seconds() / 60.0)) output_file = open(os.path.join(output_dir, Assignment.CONFIGURATION_OUTPUT_FILE), 'w') parser.write(output_file) @@ -614,11 +628,11 @@ def initialize_fasttrips_extension(process_number, output_dir, stop_times_df): Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED, overcap_col]].as_matrix().astype('float64')) - _fasttrips.initialize_parameters(Assignment.TIME_WINDOW.total_seconds()/ 60.0, - Assignment.BUMP_BUFFER.total_seconds()/ 60.0, + _fasttrips.initialize_parameters(Assignment.TIME_WINDOW.total_seconds() / 60.0, + Assignment.BUMP_BUFFER.total_seconds() / 60.0, Assignment.UTILS_CONVERSION, - PathSet.DEPART_EARLY_ALLOWED_MIN.total_seconds()/ 60.0, - PathSet.ARRIVE_LATE_ALLOWED_MIN.total_seconds()/ 60.0, + PathSet.DEPART_EARLY_ALLOWED_MIN.total_seconds() / 60.0, + PathSet.ARRIVE_LATE_ALLOWED_MIN.total_seconds() / 60.0, Assignment.STOCH_PATHSET_SIZE, Assignment.STOCH_DISPERSION, Assignment.STOCH_MAX_STOP_PROCESS_COUNT, @@ -633,7 +647,7 @@ def set_fasttrips_bump_wait(bump_wait_df): Sends the bump wait information to the fasttrips extension """ # send a clear message? - if type(bump_wait_df)==type(None): return + if type(bump_wait_df) == type(None): return if len(bump_wait_df) == 0: return @@ -641,11 +655,12 @@ def set_fasttrips_bump_wait(bump_wait_df): Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.STOPTIMES_COLUMN_STOP_ID_NUM]].as_matrix().astype('int32'), bump_wait_df[Passenger.PF_COL_PAX_A_TIME_MIN].values.astype('float64')) + @staticmethod def write_vehicle_trips(output_dir, iteration, pathfinding_iteration, simulation_iteration, veh_trips_df): """ """ - columns = ["iteration", # we'll add + columns = ["iteration", # we'll add "pathfinding_iteration", "simulation_iteration", Trip.TRIPS_COLUMN_DIRECTION_ID, @@ -681,59 +696,67 @@ def write_vehicle_trips(output_dir, iteration, pathfinding_iteration, simulation if optional_col not in veh_trips_df.columns.values: columns.remove(optional_col) - veh_trips_df[ "iteration"] = iteration + veh_trips_df["iteration"] = iteration veh_trips_df["pathfinding_iteration"] = pathfinding_iteration - veh_trips_df[ "simulation_iteration"] = simulation_iteration + veh_trips_df["simulation_iteration"] = simulation_iteration Util.write_dataframe(veh_trips_df[columns], "veh_trips_df", os.path.join(output_dir, "veh_trips.csv"), - append=(iteration>0 or pathfinding_iteration>0)) - veh_trips_df.drop(["iteration","pathfinding_iteration","simulation_iteration"], axis=1, inplace=True) + append=(iteration > 0 or pathfinding_iteration > 0)) + veh_trips_df.drop(["iteration", "pathfinding_iteration", "simulation_iteration"], axis=1, inplace=True) @staticmethod - def merge_pathsets(pathfind_trip_list_df, pathset_paths_df, pathset_links_df, new_pathset_paths_df, new_pathset_links_df): + def merge_pathsets(pathfind_trip_list_df, pathset_paths_df, pathset_links_df, new_pathset_paths_df, + new_pathset_links_df): """ Merge the given new pathset paths and links into the existing """ - FastTripsLogger.debug("merge_pathsets(): pathset_paths_df len=%d head=\n%s" % (len( pathset_paths_df), pathset_paths_df.head().to_string())) - FastTripsLogger.debug("merge_pathsets(): new_pathset_paths_df len=%d head=\n%s" % (len(new_pathset_paths_df), new_pathset_paths_df.head().to_string())) + FastTripsLogger.debug("merge_pathsets(): pathset_paths_df len=%d head=\n%s" % ( + len(pathset_paths_df), pathset_paths_df.head().to_string())) + FastTripsLogger.debug("merge_pathsets(): new_pathset_paths_df len=%d head=\n%s" % ( + len(new_pathset_paths_df), new_pathset_paths_df.head().to_string())) FastTripsLogger.debug("merge_pathsets() dtypes=\n%s" % str(pathset_paths_df.dtypes)) - FastTripsLogger.debug("merge_pathsets(): pathset_links_df len=%d head=\n%s" % (len( pathset_links_df), pathset_links_df.head().to_string())) - FastTripsLogger.debug("merge_pathsets(): new_pathset_links_df len=%d head=\n%s" % (len(new_pathset_links_df), new_pathset_links_df.head().to_string())) + FastTripsLogger.debug("merge_pathsets(): pathset_links_df len=%d head=\n%s" % ( + len(pathset_links_df), pathset_links_df.head().to_string())) + FastTripsLogger.debug("merge_pathsets(): new_pathset_links_df len=%d head=\n%s" % ( + len(new_pathset_links_df), new_pathset_links_df.head().to_string())) FastTripsLogger.debug("merge_pathsets() dtypes=\n%s" % str(pathset_links_df.dtypes)) - - FastTripsLogger.debug("merge_pathsets(): pathfind_trip_list_df len=%d head=\n%s" % (len( pathfind_trip_list_df), pathfind_trip_list_df.head().to_string())) + FastTripsLogger.debug("merge_pathsets(): pathfind_trip_list_df len=%d head=\n%s" % ( + len(pathfind_trip_list_df), pathfind_trip_list_df.head().to_string())) # TODO: This might be inefficient... # filter out the new pathset person trips from pathset_paths_df - pathset_paths_df = pd.merge(left =pathset_paths_df, - right =pathfind_trip_list_df[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], - how ="left", - indicator=True) - pathset_paths_df = pathset_paths_df.loc[pathset_paths_df["_merge"]=="left_only"] + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=pathfind_trip_list_df[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], + how="left", + indicator=True) + pathset_paths_df = pathset_paths_df.loc[pathset_paths_df["_merge"] == "left_only"] pathset_paths_df.drop(["_merge"], axis=1, inplace=True) FastTripsLogger.debug("Filtered to %d pathset_paths_df rows" % len(pathset_paths_df)) # TODO: error prone, make this cleaner with where it's initialized elsewhere - new_pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN ] = pd.Categorical([Assignment.CHOSEN_NOT_CHOSEN_YET]*len(new_pathset_paths_df), - categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + new_pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + [Assignment.CHOSEN_NOT_CHOSEN_YET] * len(new_pathset_paths_df), + categories=Assignment.CHOSEN_CATEGORIES, ordered=True) new_pathset_paths_df[Assignment.SIM_COL_MISSED_XFER] = 0 # append pathset_paths_df = pd.concat([pathset_paths_df, new_pathset_paths_df], axis=0) FastTripsLogger.debug("Concatenated so pathset_paths_df has %d rows" % len(pathset_paths_df)) # filter out the new pathset person trips from pathset_links_df - pathset_links_df = pd.merge(left =pathset_links_df, - right =pathfind_trip_list_df[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], - how ="left", - indicator=True) - pathset_links_df = pathset_links_df.loc[pathset_links_df["_merge"]=="left_only"] + pathset_links_df = pd.merge(left=pathset_links_df, + right=pathfind_trip_list_df[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], + how="left", + indicator=True) + pathset_links_df = pathset_links_df.loc[pathset_links_df["_merge"] == "left_only"] pathset_links_df.drop(["_merge"], axis=1, inplace=True) FastTripsLogger.debug("Filtered to %d pathset_links_df rows" % len(pathset_links_df)) # append pathset_links_df = pd.concat([pathset_links_df, new_pathset_links_df], axis=0) FastTripsLogger.debug("Concatenated so pathset_links_df has %d rows" % len(pathset_links_df)) - FastTripsLogger.debug("merge_pathsets(): pathset_paths_df len=%d head=\n%s\ntail=\n%s" % (len(pathset_paths_df), pathset_paths_df.head().to_string(),pathset_paths_df.tail().to_string())) - FastTripsLogger.debug("merge_pathsets(): pathset_links_df len=%d head=\n%s\ntail=\n%s" % (len(pathset_links_df), pathset_links_df.head().to_string(),pathset_links_df.tail().to_string())) + FastTripsLogger.debug("merge_pathsets(): pathset_paths_df len=%d head=\n%s\ntail=\n%s" % ( + len(pathset_paths_df), pathset_paths_df.head().to_string(), pathset_paths_df.tail().to_string())) + FastTripsLogger.debug("merge_pathsets(): pathset_links_df len=%d head=\n%s\ntail=\n%s" % ( + len(pathset_links_df), pathset_links_df.head().to_string(), pathset_links_df.tail().to_string())) # done with this return (pathset_paths_df, pathset_links_df) @@ -743,7 +766,8 @@ def number_of_pathsets(pathset_paths_df): """ Counts the number of passenger trips with pathsets and returns it. """ - return len(pathset_paths_df.groupby([Passenger.PERSONS_COLUMN_PERSON_ID,Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID])) + return len( + pathset_paths_df.groupby([Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID])) @staticmethod def assign_paths(output_dir, FT): @@ -754,7 +778,7 @@ def assign_paths(output_dir, FT): _fasttrips.reset() # write the initial load profile, iteration 0 - veh_trips_df = FT.trips.get_full_trips() + veh_trips_df = FT.trips.get_full_trips() pathset_paths_df = None pathset_links_df = None @@ -784,7 +808,7 @@ def assign_paths(output_dir, FT): # write 0-iter vehicle trips Assignment.write_vehicle_trips(output_dir, 0, 0, 0, veh_trips_df) - for iteration in range(1, Assignment.MAX_ITERATIONS+1): + for iteration in range(1, Assignment.MAX_ITERATIONS + 1): for pathfinding_iteration in range(1, Assignment.MAX_PF_ITERATIONS + 1): @@ -795,12 +819,18 @@ def assign_paths(output_dir, FT): else: Assignment.PATHFINDING_EVERYONE = False - FastTripsLogger.info("***************************** ITERATION %d PATHFINDING ITERATION %d **************************************" % (iteration, pathfinding_iteration)) + FastTripsLogger.info( + "***************************** ITERATION %d PATHFINDING ITERATION %d **************************************" % ( + iteration, pathfinding_iteration)) - if (Assignment.PATHFINDING_TYPE == Assignment.PATHFINDING_TYPE_READ_FILE) and (iteration == 1) and (pathfinding_iteration == 1): + if (Assignment.PATHFINDING_TYPE == Assignment.PATHFINDING_TYPE_READ_FILE) and (iteration == 1) and ( + pathfinding_iteration == 1): FastTripsLogger.info("Reading paths from file") FT.performance.record_step_start(iteration, pathfinding_iteration, -1, "pathreading") - (new_pathset_paths_df, new_pathset_links_df) = FT.passengers.read_passenger_pathsets(output_dir, FT.stops, FT.routes.modes_df, include_asgn=False) + (new_pathset_paths_df, new_pathset_links_df) = FT.passengers.read_passenger_pathsets(output_dir, + FT.stops, + FT.routes.modes_df, + include_asgn=False) num_new_paths_found = Assignment.number_of_pathsets(new_pathset_paths_df) # todo: what about subsequent iterations @@ -808,18 +838,28 @@ def assign_paths(output_dir, FT): else: FT.performance.record_step_start(iteration, pathfinding_iteration, -1, "pathfinding") - num_new_paths_found = Assignment.generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, pathfinding_iteration) - (new_pathset_paths_df, new_pathset_links_df) = FT.passengers.setup_passenger_pathsets(iteration, pathfinding_iteration, FT.stops, - FT.trips.trip_id_df, FT.trips.trips_df, FT.routes.modes_df, - FT.transfers, FT.tazs, Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID) + num_new_paths_found = Assignment.generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, + iteration, pathfinding_iteration) + (new_pathset_paths_df, new_pathset_links_df) = FT.passengers.setup_passenger_pathsets(iteration, + pathfinding_iteration, + FT.stops, + FT.trips.trip_id_df, + FT.trips.trips_df, + FT.routes.modes_df, + FT.transfers, + FT.tazs, + Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID) # write pathfinding results to special PF results file Passenger.write_paths(output_dir, iteration, pathfinding_iteration, -1, new_pathset_paths_df, False, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, False) + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + False) Passenger.write_paths(output_dir, iteration, pathfinding_iteration, -1, new_pathset_links_df, True, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, False) + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + False) # write performance info right away in case we crash, quit, etc - FT.performance.write_pathfinding(output_dir, append=((iteration>1) or (pathfinding_iteration>1))) + FT.performance.write_pathfinding(output_dir, + append=((iteration > 1) or (pathfinding_iteration > 1))) # If we found paths for everyone, excellent if Assignment.PATHFINDING_EVERYONE: @@ -827,25 +867,36 @@ def assign_paths(output_dir, FT): pathset_links_df = new_pathset_links_df # Otherwise, merge with those for whom we already have else: - (pathset_paths_df, pathset_links_df) = Assignment.merge_pathsets(FT.passengers.pathfind_trip_list_df, pathset_paths_df, pathset_links_df, new_pathset_paths_df, new_pathset_links_df) + (pathset_paths_df, pathset_links_df) = Assignment.merge_pathsets( + FT.passengers.pathfind_trip_list_df, pathset_paths_df, pathset_links_df, new_pathset_paths_df, + new_pathset_links_df) # if we have new paths, simulate them if num_new_paths_found > 0: - pathset_paths_df, pathset_links_df = Assignment.merge_prior_choices(pathset_paths_df, pathset_links_df, success_df) + pathset_paths_df, pathset_links_df = Assignment.merge_prior_choices(pathset_paths_df, + pathset_links_df, success_df) if Assignment.SIMULATION: - FastTripsLogger.info("***************************** ITERATION %d PATHFINDING ITERATION %d *** SIMULATING ***********************" % (iteration, pathfinding_iteration)) + FastTripsLogger.info( + "***************************** ITERATION %d PATHFINDING ITERATION %d *** SIMULATING ***********************" % ( + iteration, pathfinding_iteration)) FT.performance.record_step_start(iteration, pathfinding_iteration, -1, "simulating") (num_passengers_arrived, pathset_paths_df, pathset_links_df, veh_trips_df) = \ - Assignment.simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, pathset_links_df, veh_trips_df) + Assignment.simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, + pathset_links_df, veh_trips_df) else: # if we're not simulating, we can still calculate costs and choose paths - FastTripsLogger.info("***************************** ITERATION %d PATHFINDING ITERATION %d *****CHOOSING PATHS WITHOUT SIMULATING" % (iteration, pathfinding_iteration)) - FT.performance.record_step_start(iteration, pathfinding_iteration, -1, "choosing_without_simulating") + FastTripsLogger.info( + "***************************** ITERATION %d PATHFINDING ITERATION %d *****CHOOSING PATHS WITHOUT SIMULATING" % ( + iteration, pathfinding_iteration)) + FT.performance.record_step_start(iteration, pathfinding_iteration, -1, + "choosing_without_simulating") (num_passengers_arrived, pathset_paths_df, pathset_links_df) = \ - Assignment.choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, pathset_links_df, veh_trips_df) + Assignment.choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_iteration, + pathset_paths_df, pathset_links_df, veh_trips_df) - FT.performance.record_step_start(iteration, pathfinding_iteration, -1, "output_per_pathfinding_iteration") + FT.performance.record_step_start(iteration, pathfinding_iteration, -1, + "output_per_pathfinding_iteration") # Set new schedule FT.trips.stop_times_df = veh_trips_df @@ -860,14 +911,12 @@ def assign_paths(output_dir, FT): num_paths_found = Assignment.number_of_pathsets(pathset_paths_df) num_bumped_passengers = num_paths_found - num_passengers_arrived - FastTripsLogger.info("") FastTripsLogger.info(" Length of trip list: %10d" % len(FT.passengers.trip_list_df)) FastTripsLogger.info(" Number of pathsets found: %10d" % num_paths_found) FastTripsLogger.info(" ARRIVED PASSENGERS: %10d" % num_passengers_arrived) FastTripsLogger.info(" MISSED PASSENGERS: %10d" % num_bumped_passengers) - FT.performance.record_step_end(iteration, pathfinding_iteration, -1) # if no new paths found, pathfinding_iteration loop is done @@ -882,7 +931,6 @@ def assign_paths(output_dir, FT): FastTripsLogger.info(" CAPACITY GAP: %10.5f" % capacity_gap) FastTripsLogger.info(" NEW CHOICE FROM PRIOR: %10d" % new_choices) - success_df, bump_df = Assignment.save_choices(pathset_paths_df, success_df, bump_df) new_choices, last_chosen_df = Assignment.compare_choices(pathset_paths_df, last_chosen_df) capacity_gap = (1.0 * new_choices + num_bumped_passengers) / len(FT.passengers.trip_list_df) @@ -891,7 +939,6 @@ def assign_paths(output_dir, FT): FastTripsLogger.info(" CAPACITY GAP: %10.5f" % capacity_gap) FastTripsLogger.info(" NEW CHOICE FROM PRIOR: %10d" % new_choices) - success_df, bump_df = Assignment.save_choices(pathset_paths_df, success_df, bump_df) new_choices, last_chosen_df = Assignment.compare_choices(pathset_paths_df, last_chosen_df) capacity_gap = (1.0 * new_choices + num_bumped_passengers) / len(FT.passengers.trip_list_df) @@ -900,7 +947,6 @@ def assign_paths(output_dir, FT): FastTripsLogger.info(" CAPACITY GAP: %10.5f" % capacity_gap) FastTripsLogger.info(" NEW CHOICE FROM PRIOR: %10d" % new_choices) - # end condition for iterations loop if capacity_gap < Assignment.CONVERGENCE_GAP: break @@ -911,8 +957,7 @@ def assign_paths(output_dir, FT): "paths_found": num_paths_found, "passengers_arrived": num_passengers_arrived, "passengers_missed": num_bumped_passengers, - "passengers_demand": len(FT.passengers.trip_list_df) } - + "passengers_demand": len(FT.passengers.trip_list_df)} @staticmethod def compare_choices(pathset_paths_df, prior_choice): @@ -943,7 +988,6 @@ def compare_choices(pathset_paths_df, prior_choice): return new_choices_count, chosen - @staticmethod def save_choices(pathset_paths_df, success_df, bump_df): """ @@ -984,7 +1028,6 @@ def save_choices(pathset_paths_df, success_df, bump_df): return success_df, bump_df - @staticmethod def merge_prior_choices(pathset_paths_df, pathset_links_df, flag_df): """ @@ -995,7 +1038,7 @@ def merge_prior_choices(pathset_paths_df, pathset_links_df, flag_df): :return: pathset_paths_df and pathset_links_df with additional flag column. """ - assert((PathSet.SUCCESS_FLAG_COLUMN in flag_df) or (PathSet.BUMP_FLAG_COLUMN in flag_df)) + assert ((PathSet.SUCCESS_FLAG_COLUMN in flag_df) or (PathSet.BUMP_FLAG_COLUMN in flag_df)) flag_col = PathSet.SUCCESS_FLAG_COLUMN if PathSet.SUCCESS_FLAG_COLUMN in flag_df else PathSet.BUMP_FLAG_COLUMN @@ -1006,15 +1049,15 @@ def merge_prior_choices(pathset_paths_df, pathset_links_df, flag_df): pathset_links_df.drop(labels=[flag_col], axis=1, inplace=True) pathset_paths_df = pd.merge(pathset_paths_df, flag_df, - on = [ + on=[ Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_DESCRIPTION], how = 'left') + Passenger.PF_COL_DESCRIPTION], how='left') pathset_paths_df.loc[pathset_paths_df[flag_col].isnull(), flag_col] = 0 - pathset_links_df = pd.merge(pathset_links_df,pathset_paths_df[ + pathset_links_df = pd.merge(pathset_links_df, pathset_paths_df[ ['trip_list_id_num', Passenger.PF_COL_PATH_NUM, flag_col, ]], - on = ['trip_list_id_num', Passenger.PF_COL_PATH_NUM], how = 'left') + on=['trip_list_id_num', Passenger.PF_COL_PATH_NUM], how='left') return pathset_paths_df, pathset_links_df @@ -1023,24 +1066,29 @@ def filter_trip_list_to_not_arrived(trip_list_df, pathset_paths_df): """ Filter the given trip list to only those that have not arrived according to *pathset_paths_df*. """ - FastTripsLogger.debug("filter_trip_list_to_not_arrived(): trip_list_df len=%d head()=\n%s" % (len(trip_list_df), trip_list_df.head().to_string())) - FastTripsLogger.debug("filter_trip_list_to_not_arrived(): pathset_paths_df len=%d head()=\n%s" % (len(pathset_paths_df), pathset_paths_df.head().to_string())) + FastTripsLogger.debug("filter_trip_list_to_not_arrived(): trip_list_df len=%d head()=\n%s" % ( + len(trip_list_df), trip_list_df.head().to_string())) + FastTripsLogger.debug("filter_trip_list_to_not_arrived(): pathset_paths_df len=%d head()=\n%s" % ( + len(pathset_paths_df), pathset_paths_df.head().to_string())) FastTripsLogger.debug("pathset_paths_df.dtypes") # filter to only the chosen paths - chosen_paths_df = pathset_paths_df.loc[pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET, - [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Assignment.SIM_COL_PAX_CHOSEN]] + chosen_paths_df = pathset_paths_df.loc[ + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET, + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Assignment.SIM_COL_PAX_CHOSEN]] # add chosen index - trip_list_df_to_return = pd.merge(left =trip_list_df, - right =chosen_paths_df, - how ="left") + trip_list_df_to_return = pd.merge(left=trip_list_df, + right=chosen_paths_df, + how="left") # use it to filter to null chosen - trip_list_df_to_return = trip_list_df_to_return.loc[pd.isnull(trip_list_df_to_return[Assignment.SIM_COL_PAX_CHOSEN])] + trip_list_df_to_return = trip_list_df_to_return.loc[ + pd.isnull(trip_list_df_to_return[Assignment.SIM_COL_PAX_CHOSEN])] # remove chosen column trip_list_df_to_return.drop([Assignment.SIM_COL_PAX_CHOSEN], axis=1, inplace=True) - FastTripsLogger.debug("filter_trip_list_to_not_arrived(): trip_list_df_to_return len=%d head()=\n%s" % (len(trip_list_df_to_return), trip_list_df_to_return.head().to_string())) + FastTripsLogger.debug("filter_trip_list_to_not_arrived(): trip_list_df_to_return len=%d head()=\n%s" % ( + len(trip_list_df_to_return), trip_list_df_to_return.head().to_string())) return trip_list_df_to_return @staticmethod @@ -1053,16 +1101,18 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, Returns the number of pathsets found. """ - FastTripsLogger.info("**************************** GENERATING PATHS **********************************************************") - start_time = datetime.datetime.now() - process_dict = {} # workernum -> {"process":process, "alive":alive bool, "done":done bool, "working_on":(person_id, trip_list_num)} - todo_queue = None - done_queue = None + FastTripsLogger.info( + "**************************** GENERATING PATHS **********************************************************") + start_time = datetime.datetime.now() + process_dict = {} # workernum -> {"process":process, "alive":alive bool, "done":done bool, "working_on":(person_id, trip_list_num)} + todo_queue = None + done_queue = None # We only need to do this once if iteration == 1 and pathfinding_iteration == 1: if Assignment.DEBUG_TRACE_ONLY: - FT.passengers.trip_list_df = FT.passengers.trip_list_df.loc[FT.passengers.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True] + FT.passengers.trip_list_df = FT.passengers.trip_list_df.loc[ + FT.passengers.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True] else: if Assignment.DEBUG_NUM_TRIPS > 0 and len(FT.passengers.trip_list_df) > Assignment.DEBUG_NUM_TRIPS: FastTripsLogger.info("Truncating trip list to %d trips" % Assignment.DEBUG_NUM_TRIPS) @@ -1070,7 +1120,8 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, # Skip someone? if Assignment.SKIP_PERSON_IDS and len(Assignment.SKIP_PERSON_IDS) > 0: - FT.passengers.trip_list_df = FT.passengers.trip_list_df.loc[~FT.passengers.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID].isin(Assignment.SKIP_PERSON_IDS)] + FT.passengers.trip_list_df = FT.passengers.trip_list_df.loc[ + ~FT.passengers.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID].isin(Assignment.SKIP_PERSON_IDS)] # these are the trips for which we'll find paths FT.passengers.pathfind_trip_list_df = FT.passengers.trip_list_df @@ -1080,62 +1131,66 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, Trip.reset_onboard(veh_trips_df) else: FastTripsLogger.info("Finding paths for trips for those that haven't arrived yet") - FT.passengers.pathfind_trip_list_df = Assignment.filter_trip_list_to_not_arrived(FT.passengers.trip_list_df, pathset_paths_df) + FT.passengers.pathfind_trip_list_df = Assignment.filter_trip_list_to_not_arrived(FT.passengers.trip_list_df, + pathset_paths_df) - est_paths_to_find = len(FT.passengers.pathfind_trip_list_df) + est_paths_to_find = len(FT.passengers.pathfind_trip_list_df) FastTripsLogger.info("Finding pathsets for %d trips" % est_paths_to_find) if est_paths_to_find == 0: return 0 - info_freq = pow(10, int(math.log(est_paths_to_find+1,10)-1)) + info_freq = pow(10, int(math.log(est_paths_to_find + 1, 10) - 1)) if info_freq < 1: info_freq = 1 # info_freq = 1 # DEBUG CRASH - num_processes = Assignment.NUMBER_OF_PROCESSES - if Assignment.NUMBER_OF_PROCESSES < 1: - num_processes = multiprocessing.cpu_count() + num_processes = Assignment.NUMBER_OF_PROCESSES + if Assignment.NUMBER_OF_PROCESSES < 1: + num_processes = multiprocessing.cpu_count() # it's not worth it unless each process does 3 - if num_processes > est_paths_to_find*3: - num_processes = int(est_paths_to_find//3) + if num_processes > est_paths_to_find * 3: + num_processes = int(est_paths_to_find // 3) # this is probalby time consuming... put in a try block try: # Setup multiprocessing processes if num_processes > 1: - todo_queue = multiprocessing.Queue() - done_queue = multiprocessing.Queue() - for process_idx in range(1, 1+num_processes): + todo_queue = multiprocessing.Queue() + done_queue = multiprocessing.Queue() + for process_idx in range(1, 1 + num_processes): FastTripsLogger.info("Starting worker process %2d" % process_idx) process_dict[process_idx] = { - "process":multiprocessing.Process(target=find_trip_based_paths_process_worker, - args=(iteration, pathfinding_iteration, process_idx, Assignment.INPUT_NETWORK_ARCHIVE, Assignment.INPUT_DEMAND_DIR, - Assignment.CONFIGURATION_FILE, Assignment.CONFIGURATION_FUNCTIONS_FILE, - Assignment.OUTPUT_DIR, todo_queue, done_queue, - Assignment.PATHFINDING_TYPE==Assignment.PATHFINDING_TYPE_STOCHASTIC, - Assignment.bump_wait_df, veh_trips_df)), - "alive":True, - "done":False + "process": multiprocessing.Process(target=find_trip_based_paths_process_worker, + args=(iteration, pathfinding_iteration, process_idx, + Assignment.INPUT_NETWORK_ARCHIVE, + Assignment.INPUT_DEMAND_DIR, + Assignment.CONFIGURATION_FILE, + Assignment.CONFIGURATION_FUNCTIONS_FILE, + Assignment.OUTPUT_DIR, todo_queue, done_queue, + Assignment.PATHFINDING_TYPE == Assignment.PATHFINDING_TYPE_STOCHASTIC, + Assignment.bump_wait_df, veh_trips_df)), + "alive": True, + "done": False } process_dict[process_idx]["process"].start() else: Assignment.initialize_fasttrips_extension(0, output_dir, veh_trips_df) # process tasks or send tasks to workers for processing - num_paths_found_prev = 0 - num_paths_found_now = 0 - num_paths_sought = 0 - path_cols = list(FT.passengers.pathfind_trip_list_df.columns.values) + num_paths_found_prev = 0 + num_paths_found_now = 0 + num_paths_sought = 0 + path_cols = list(FT.passengers.pathfind_trip_list_df.columns.values) for path_tuple in FT.passengers.pathfind_trip_list_df.itertuples(index=False): - path_dict = dict(list(zip(path_cols, path_tuple))) - trip_list_id = path_dict[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM] - person_id = path_dict[Passenger.TRIP_LIST_COLUMN_PERSON_ID] - person_trip_id = path_dict[Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID] - do_trace = path_dict[Passenger.TRIP_LIST_COLUMN_TRACE] + path_dict = dict(list(zip(path_cols, path_tuple))) + trip_list_id = path_dict[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM] + person_id = path_dict[Passenger.TRIP_LIST_COLUMN_PERSON_ID] + person_trip_id = path_dict[Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID] + do_trace = path_dict[Passenger.TRIP_LIST_COLUMN_TRACE] if Assignment.DEBUG_TRACE_ONLY and not do_trace: continue # first iteration -- create path objects - if iteration==1: + if iteration == 1: trip_pathset = PathSet(path_dict) FT.passengers.add_pathset(trip_list_id, trip_pathset) else: @@ -1149,15 +1204,16 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, # continue if num_processes > 1: - todo_queue.put( trip_pathset ) + todo_queue.put(trip_pathset) else: if do_trace: - FastTripsLogger.debug("Tracing assignment of person_id %s and trip %s" % (person_id, person_trip_id)) + FastTripsLogger.debug( + "Tracing assignment of person_id %s and trip %s" % (person_id, person_trip_id)) # do the work (pathdict, perf_dict) = \ Assignment.find_trip_based_pathset(iteration, pathfinding_iteration, trip_pathset, - Assignment.PATHFINDING_TYPE==Assignment.PATHFINDING_TYPE_STOCHASTIC, + Assignment.PATHFINDING_TYPE == Assignment.PATHFINDING_TYPE_STOCHASTIC, trace=do_trace) num_paths_sought += 1 @@ -1169,11 +1225,12 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, if num_paths_sought % info_freq == 0: time_elapsed = datetime.datetime.now() - start_time - FastTripsLogger.info(" %6d paths sought, %6d paths found of %d paths total. Time elapsed: %2dh:%2dm:%2ds" % ( - num_paths_sought, num_paths_found_now, est_paths_to_find, - int( time_elapsed.total_seconds()/ 3600), - int( (time_elapsed.total_seconds() % 3600)/ 60), - time_elapsed.total_seconds() % 60)) + FastTripsLogger.info( + " %6d paths sought, %6d paths found of %d paths total. Time elapsed: %2dh:%2dm:%2ds" % ( + num_paths_sought, num_paths_found_now, est_paths_to_find, + int(time_elapsed.total_seconds() / 3600), + int((time_elapsed.total_seconds() % 3600) / 60), + time_elapsed.total_seconds() % 60)) # multiprocessing follow-up if num_processes > 1: @@ -1186,7 +1243,7 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, while done_procs < len(process_dict): try: - result = done_queue.get(True, 30) + result = done_queue.get(True, 30) worker_num = result[0] # FastTripsLogger.debug("Received %s" % str(result)) @@ -1194,14 +1251,15 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, FastTripsLogger.debug("Received done from process %d" % worker_num) process_dict[worker_num]["done"] = True elif result[1] == "STARTING": - process_dict[worker_num]["working_on"] = (result[2],result[3]) + process_dict[worker_num]["working_on"] = (result[2], result[3]) elif result[1] == "COMPLETED": - trip_list_id = result[2] - pathset = FT.passengers.get_pathset(trip_list_id) - pathset.pathdict= result[3] - perf_dict = result[4] + trip_list_id = result[2] + pathset = FT.passengers.get_pathset(trip_list_id) + pathset.pathdict = result[3] + perf_dict = result[4] - FT.performance.add_info(iteration, pathfinding_iteration, pathset.person_id, pathset.person_trip_id, perf_dict) + FT.performance.add_info(iteration, pathfinding_iteration, pathset.person_id, + pathset.person_trip_id, perf_dict) num_paths_sought += 1 if pathset.path_found(): @@ -1209,11 +1267,12 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, if num_paths_sought % info_freq == 0: time_elapsed = datetime.datetime.now() - start_time - FastTripsLogger.info(" %6d paths sought, %6d paths found of %d paths total. Time elapsed: %2dh:%2dm:%2ds" % ( - num_paths_sought, num_paths_found_now, est_paths_to_find, - int( time_elapsed.total_seconds()/ 3600), - int( (time_elapsed.total_seconds() % 3600)/ 60), - time_elapsed.total_seconds() % 60)) + FastTripsLogger.info( + " %6d paths sought, %6d paths found of %d paths total. Time elapsed: %2dh:%2dm:%2ds" % ( + num_paths_sought, num_paths_found_now, est_paths_to_find, + int(time_elapsed.total_seconds() / 3600), + int((time_elapsed.total_seconds() % 3600) / 60), + time_elapsed.total_seconds() % 60)) del process_dict[worker_num]["working_on"] else: @@ -1244,7 +1303,8 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, FastTripsLogger.info("Process %d appears to have crashed; it was working on %s" % \ (process_idx, str(process_dict[process_idx]["working_on"]))) else: - FastTripsLogger.info("Process %d appears to have crashed; see ft_debug_worker%02d.log" % (process_idx, process_idx)) + FastTripsLogger.info("Process %d appears to have crashed; see ft_debug_worker%02d.log" % ( + process_idx, process_idx)) except (KeyboardInterrupt, SystemExit): exc_type, exc_value, exc_tb = sys.exc_info() @@ -1265,14 +1325,13 @@ def generate_pathsets(FT, pathset_paths_df, veh_trips_df, output_dir, iteration, time_elapsed = datetime.datetime.now() - start_time FastTripsLogger.info("Finished finding %6d passenger paths. Time elapsed: %2dh:%2dm:%2ds" % ( - num_paths_found_now, - int( time_elapsed.total_seconds()/ 3600), - int( (time_elapsed.total_seconds() % 3600)/ 60), - time_elapsed.total_seconds() % 60)) + num_paths_found_now, + int(time_elapsed.total_seconds() / 3600), + int((time_elapsed.total_seconds() % 3600) / 60), + time_elapsed.total_seconds() % 60)) return num_paths_found_now + num_paths_found_prev - @staticmethod def find_trip_based_pathset(iteration, pathfinding_iteration, pathset, hyperpath, trace): """ @@ -1308,26 +1367,29 @@ def find_trip_based_pathset(iteration, pathfinding_iteration, pathset, hyperpath label_iterations, num_labeled_stops, max_label_process_count, ms_labeling, ms_enumerating, bytes_workingset, bytes_privateusage, mem_timestamp) = \ - _fasttrips.find_pathset(iteration, pathfinding_iteration, hyperpath, pathset.person_id, pathset.person_trip_id, - pathset.user_class, pathset.purpose, pathset.access_mode, pathset.transit_mode, pathset.egress_mode, - pathset.o_taz_num, pathset.d_taz_num, - 1 if pathset.outbound else 0, float(pathset.pref_time_min), pathset.vot, - 1 if trace else 0) + _fasttrips.find_pathset(iteration, pathfinding_iteration, hyperpath, pathset.person_id, + pathset.person_trip_id, + pathset.user_class, pathset.purpose, pathset.access_mode, pathset.transit_mode, + pathset.egress_mode, + pathset.o_taz_num, pathset.d_taz_num, + 1 if pathset.outbound else 0, float(pathset.pref_time_min), pathset.vot, + 1 if trace else 0) # FastTripsLogger.debug("C++ extension complete") - FastTripsLogger.debug("Finished finding path for person %s trip %s" % (pathset.person_id, pathset.person_trip_id)) + FastTripsLogger.debug( + "Finished finding path for person %s trip %s" % (pathset.person_id, pathset.person_trip_id)) pathdict = {} - row_num = 0 + row_num = 0 for path_num in range(path_costs.shape[0]): pathdict[path_num] = {} - pathdict[path_num][PathSet.PATH_KEY_COST ] = path_costs[path_num, 0] - pathdict[path_num][PathSet.PATH_KEY_FARE ] = path_costs[path_num, 1] + pathdict[path_num][PathSet.PATH_KEY_COST] = path_costs[path_num, 0] + pathdict[path_num][PathSet.PATH_KEY_FARE] = path_costs[path_num, 1] pathdict[path_num][PathSet.PATH_KEY_PROBABILITY] = path_costs[path_num, 2] - pathdict[path_num][PathSet.PATH_KEY_INIT_COST ] = path_costs[path_num, 3] - pathdict[path_num][PathSet.PATH_KEY_INIT_FARE ] = path_costs[path_num, 4] + pathdict[path_num][PathSet.PATH_KEY_INIT_COST] = path_costs[path_num, 3] + pathdict[path_num][PathSet.PATH_KEY_INIT_FARE] = path_costs[path_num, 4] # List of (stop_id, stop_state) - pathdict[path_num][PathSet.PATH_KEY_STATES ] = [] + pathdict[path_num][PathSet.PATH_KEY_STATES] = [] # print "path_num %d" % path_num @@ -1335,7 +1397,7 @@ def find_trip_based_pathset(iteration, pathfinding_iteration, pathset, hyperpath while (row_num < ret_ints.shape[0]) and (ret_ints[row_num, 0] == path_num): # print row_num - mode = ret_ints[row_num,2] + mode = ret_ints[row_num, 2] # todo if mode == -100: mode = PathSet.STATE_MODE_ACCESS @@ -1347,55 +1409,58 @@ def find_trip_based_pathset(iteration, pathfinding_iteration, pathset, hyperpath mode = Passenger.MODE_GENERIC_TRANSIT_NUM if hyperpath: - pathdict[path_num][PathSet.PATH_KEY_STATES].append( (ret_ints[row_num, 1], [ - ret_doubles[row_num,0], # label, - Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num,1]), # departure/arrival time - mode, # departure/arrival mode - ret_ints[row_num,3], # trip id - ret_ints[row_num,4], # successor/predecessor - ret_ints[row_num,5], # sequence - ret_ints[row_num,6], # sequence succ/pred - datetime.timedelta(minutes=ret_doubles[row_num,2]), # link time - ret_doubles[row_num,3], # link fare - ret_doubles[row_num,4], # link cost - ret_doubles[row_num,5], # link distance - ret_doubles[row_num,6], # cost - Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num,7]) # arrival/departure time - ] ) ) + pathdict[path_num][PathSet.PATH_KEY_STATES].append((ret_ints[row_num, 1], [ + ret_doubles[row_num, 0], # label, + Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num, 1]), + # departure/arrival time + mode, # departure/arrival mode + ret_ints[row_num, 3], # trip id + ret_ints[row_num, 4], # successor/predecessor + ret_ints[row_num, 5], # sequence + ret_ints[row_num, 6], # sequence succ/pred + datetime.timedelta(minutes=ret_doubles[row_num, 2]), # link time + ret_doubles[row_num, 3], # link fare + ret_doubles[row_num, 4], # link cost + ret_doubles[row_num, 5], # link distance + ret_doubles[row_num, 6], # cost + Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num, 7]) + # arrival/departure time + ])) else: - pathdict[path_num][PathSet.PATH_KEY_STATES].append( (ret_ints[row_num, 1], [ - datetime.timedelta(minutes=ret_doubles[row_num,0]), # label, - Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num,1]), # departure/arrival time - mode, # departure/arrival mode - ret_ints[row_num,3], # trip id - ret_ints[row_num,4], # successor/predecessor - ret_ints[row_num,5], # sequence - ret_ints[row_num,6], # sequence succ/pred - datetime.timedelta(minutes=ret_doubles[row_num,2]), # link time - ret_doubles[row_num,3], # link fare - datetime.timedelta(minutes=ret_doubles[row_num,4]), # link cost - ret_doubles[row_num,5], # link dist - datetime.timedelta(minutes=ret_doubles[row_num,6]), # cost - Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num,7]) # arrival/departure time - ] ) ) + pathdict[path_num][PathSet.PATH_KEY_STATES].append((ret_ints[row_num, 1], [ + datetime.timedelta(minutes=ret_doubles[row_num, 0]), # label, + Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num, 1]), + # departure/arrival time + mode, # departure/arrival mode + ret_ints[row_num, 3], # trip id + ret_ints[row_num, 4], # successor/predecessor + ret_ints[row_num, 5], # sequence + ret_ints[row_num, 6], # sequence succ/pred + datetime.timedelta(minutes=ret_doubles[row_num, 2]), # link time + ret_doubles[row_num, 3], # link fare + datetime.timedelta(minutes=ret_doubles[row_num, 4]), # link cost + ret_doubles[row_num, 5], # link dist + datetime.timedelta(minutes=ret_doubles[row_num, 6]), # cost + Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(minutes=ret_doubles[row_num, 7]) + # arrival/departure time + ])) row_num += 1 perf_dict = { \ - Performance.PERFORMANCE_PF_COL_PROCESS_NUM : process_num, - Performance.PERFORMANCE_PF_COL_PATHFINDING_STATUS : pf_returnstatus, - Performance.PERFORMANCE_PF_COL_LABEL_ITERATIONS : label_iterations, - Performance.PERFORMANCE_PF_COL_NUM_LABELED_STOPS : num_labeled_stops, + Performance.PERFORMANCE_PF_COL_PROCESS_NUM: process_num, + Performance.PERFORMANCE_PF_COL_PATHFINDING_STATUS: pf_returnstatus, + Performance.PERFORMANCE_PF_COL_LABEL_ITERATIONS: label_iterations, + Performance.PERFORMANCE_PF_COL_NUM_LABELED_STOPS: num_labeled_stops, Performance.PERFORMANCE_PF_COL_MAX_STOP_PROCESS_COUNT: max_label_process_count, - Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS : ms_labeling, - Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS : ms_enumerating, - Performance.PERFORMANCE_PF_COL_TRACED : trace, - Performance.PERFORMANCE_PF_COL_WORKING_SET_BYTES : bytes_workingset, - Performance.PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES : bytes_privateusage, - Performance.PERFORMANCE_PF_COL_MEM_TIMESTAMP : datetime.datetime.fromtimestamp(mem_timestamp) + Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS: ms_labeling, + Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS: ms_enumerating, + Performance.PERFORMANCE_PF_COL_TRACED: trace, + Performance.PERFORMANCE_PF_COL_WORKING_SET_BYTES: bytes_workingset, + Performance.PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES: bytes_privateusage, + Performance.PERFORMANCE_PF_COL_MEM_TIMESTAMP: datetime.datetime.fromtimestamp(mem_timestamp) } return (pathdict, perf_dict) - @staticmethod def find_passenger_vehicle_times(pathset_links_df, veh_trips_df): """ @@ -1413,7 +1478,8 @@ def find_passenger_vehicle_times(pathset_links_df, veh_trips_df): """ if False and len(Assignment.TRACE_IDS) > 0: FastTripsLogger.debug("find_passenger_vehicle_times(): input pathset_links_df len=%d\n%s" % \ - (len(pathset_links_df), pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string())) + (len(pathset_links_df), pathset_links_df.loc[ + pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string())) if Assignment.SIM_COL_PAX_BOARD_TIME in list(pathset_links_df.columns.values): pathset_links_df.drop([Assignment.SIM_COL_PAX_BOARD_TIME, @@ -1429,26 +1495,27 @@ def find_passenger_vehicle_times(pathset_links_df, veh_trips_df): Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME, - Trip.SIM_COL_VEH_OVERCAP, # TODO: what about msa_overcap? + Trip.SIM_COL_VEH_OVERCAP, # TODO: what about msa_overcap? Trip.SIM_COL_VEH_OVERCAP_FRAC] # this one may not be here -- it's only present during capacity stuff if Trip.SIM_COL_VEH_OVERCAP_FRAC not in list(veh_trips_df.columns.values): veh_trip_cols.remove(Trip.SIM_COL_VEH_OVERCAP_FRAC) - #This is a little long winded, but it cuts down on memory dramatically, but only copying - #what is actually needed during the merges. - intermediate = pd.merge(left=pathset_links_df[[Passenger.PERSONS_COLUMN_PERSON_ID, Trip.STOPTIMES_COLUMN_TRIP_ID,'A_id','A_seq', - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - 'B_id', 'B_seq']], - right=veh_trips_df[veh_trip_cols], - left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, 'A_id', 'A_seq'], - right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, - Trip.STOPTIMES_COLUMN_STOP_ID, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - how ='inner') + # This is a little long winded, but it cuts down on memory dramatically, but only copying + # what is actually needed during the merges. + intermediate = pd.merge( + left=pathset_links_df[[Passenger.PERSONS_COLUMN_PERSON_ID, Trip.STOPTIMES_COLUMN_TRIP_ID, 'A_id', 'A_seq', + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + 'B_id', 'B_seq']], + right=veh_trips_df[veh_trip_cols], + left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, 'A_id', 'A_seq'], + right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_STOP_ID, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + how='inner') intermediate = intermediate.drop(columns=[Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, @@ -1459,31 +1526,33 @@ def find_passenger_vehicle_times(pathset_links_df, veh_trips_df): Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_ARRIVAL_TIME]], - left_on =[Trip.STOPTIMES_COLUMN_TRIP_ID,'B_id','B_seq'], + left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, 'B_id', 'B_seq'], right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - how ='inner',) + how='inner', ) intermediate = intermediate.drop(columns=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - 'B_id','B_seq']) + 'B_id', 'B_seq']) intermediate = intermediate.rename(columns={ - Trip.STOPTIMES_COLUMN_DEPARTURE_TIME:Assignment.SIM_COL_PAX_BOARD_TIME, # transit vehicle depart time (at A) = board time for pax - Trip.STOPTIMES_COLUMN_ARRIVAL_TIME :Assignment.SIM_COL_PAX_ALIGHT_TIME, # transit vehicle arrive time (at B) = alight time for pax + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME: Assignment.SIM_COL_PAX_BOARD_TIME, + # transit vehicle depart time (at A) = board time for pax + Trip.STOPTIMES_COLUMN_ARRIVAL_TIME: Assignment.SIM_COL_PAX_ALIGHT_TIME, + # transit vehicle arrive time (at B) = alight time for pax }) if False and len(Assignment.TRACE_IDS) > 0: FastTripsLogger.debug("find_passenger_vehicle_times(): output pathset_links_df len=%d\n%s" % \ - (len(pathset_links_df), pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string())) + (len(pathset_links_df), pathset_links_df.loc[ + pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string())) return pd.merge(pathset_links_df, intermediate, on=[Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM,], how='left') - + Passenger.PF_COL_LINK_NUM, ], how='left') @staticmethod def put_passengers_on_vehicles(pathset_links_df, veh_trips_df): @@ -1508,12 +1577,14 @@ def put_passengers_on_vehicles(pathset_links_df, veh_trips_df): passengers_df = Passenger.get_chosen_links(pathset_links_df, transit_only=True, copy=False) # only care about trips - #passengers_df = passengers_df.loc[passengers_df[Passenger.PF_COL_ROUTE_ID].notnull()] + # passengers_df = passengers_df.loc[passengers_df[Passenger.PF_COL_ROUTE_ID].notnull()] # Group to boards by counting trip_list_id_nums for a (trip_id, A_id as stop_id) - passenger_trips_boards = passengers_df.loc[passengers_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull(), # unbumped passengers - [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Trip.STOPTIMES_COLUMN_TRIP_ID_NUM,'A_id_num','A_seq']].groupby([Trip.STOPTIMES_COLUMN_TRIP_ID_NUM,'A_id_num','A_seq']).count() + passenger_trips_boards = passengers_df.loc[ + passengers_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull(), # unbumped passengers + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, 'A_id_num', 'A_seq']].groupby( + [Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, 'A_id_num', 'A_seq']).count() passenger_trips_boards.index.names = [Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] @@ -1521,62 +1592,69 @@ def put_passengers_on_vehicles(pathset_links_df, veh_trips_df): # And alights by counting path_ids for a (trip_id, B_id as stop_id) passenger_trips_alights = passengers_df.loc[passengers_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull(), [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Trip.TRIPS_COLUMN_TRIP_ID_NUM,'B_id_num','B_seq']].groupby([Trip.TRIPS_COLUMN_TRIP_ID_NUM,'B_id_num','B_seq']).count() + Trip.TRIPS_COLUMN_TRIP_ID_NUM, 'B_id_num', 'B_seq']].groupby( + [Trip.TRIPS_COLUMN_TRIP_ID_NUM, 'B_id_num', 'B_seq']).count() passenger_trips_alights.index.names = [Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] # Join them to the transit vehicle trips so we can put people on vehicles (boards) - veh_loaded_df = pd.merge(left = veh_trips_df, - right = passenger_trips_boards, - left_on = [Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - right_index = True, - how = 'left') - veh_loaded_df.rename(columns={Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM:Trip.SIM_COL_VEH_BOARDS}, inplace=True) - + veh_loaded_df = pd.merge(left=veh_trips_df, + right=passenger_trips_boards, + left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + right_index=True, + how='left') + veh_loaded_df.rename(columns={Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM: Trip.SIM_COL_VEH_BOARDS}, + inplace=True) # Join for alights - veh_loaded_df = pd.merge(left = veh_loaded_df, - right = passenger_trips_alights, - left_on = [Trip.TRIPS_COLUMN_TRIP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - right_index = True, - how ='left') - veh_loaded_df.rename(columns={Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM:Trip.SIM_COL_VEH_ALIGHTS}, inplace=True) + veh_loaded_df = pd.merge(left=veh_loaded_df, + right=passenger_trips_alights, + left_on=[Trip.TRIPS_COLUMN_TRIP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + right_index=True, + how='left') + veh_loaded_df.rename(columns={Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM: Trip.SIM_COL_VEH_ALIGHTS}, + inplace=True) veh_loaded_df.fillna(value=0, inplace=True) - assert(len(veh_loaded_df)==veh_trips_df_len) + assert (len(veh_loaded_df) == veh_trips_df_len) # these are ints, not floats veh_loaded_df[[Trip.SIM_COL_VEH_BOARDS, Trip.SIM_COL_VEH_ALIGHTS]] = \ veh_loaded_df[[Trip.SIM_COL_VEH_BOARDS, Trip.SIM_COL_VEH_ALIGHTS]].astype(int) - veh_loaded_df.set_index([Trip.TRIPS_COLUMN_TRIP_ID_NUM,Trip.STOPTIMES_COLUMN_STOP_SEQUENCE],inplace=True) - veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD ] = veh_loaded_df[Trip.SIM_COL_VEH_BOARDS ] - veh_loaded_df[Trip.SIM_COL_VEH_ALIGHTS ] + veh_loaded_df.set_index([Trip.TRIPS_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], inplace=True) + veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD] = veh_loaded_df[Trip.SIM_COL_VEH_BOARDS] - veh_loaded_df[ + Trip.SIM_COL_VEH_ALIGHTS] # on board is the cumulative sum of boards - alights trips_cumsum = veh_loaded_df[[Trip.SIM_COL_VEH_ONBOARD]].groupby(level=[0]).cumsum() - veh_loaded_df.drop([Trip.SIM_COL_VEH_ONBOARD], axis=1, inplace=True) # replace with cumsum - veh_loaded_df = pd.merge(left = veh_loaded_df, - right = trips_cumsum, - left_index = True, - right_index = True, - how = 'left') - - assert(len(veh_loaded_df)==veh_trips_df_len) + veh_loaded_df.drop([Trip.SIM_COL_VEH_ONBOARD], axis=1, inplace=True) # replace with cumsum + veh_loaded_df = pd.merge(left=veh_loaded_df, + right=trips_cumsum, + left_index=True, + right_index=True, + how='left') + + assert (len(veh_loaded_df) == veh_trips_df_len) # print veh_trips_df.loc[5123368] veh_loaded_df.reset_index(inplace=True) # overcap = how many people are problematic, or onboard-totalcap. If negative, we have space. # overcap_frac = what percentage of boards are problematic - veh_loaded_df[Trip.SIM_COL_VEH_OVERCAP ] = veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD] - veh_loaded_df[Trip.VEHICLES_COLUMN_TOTAL_CAPACITY] + veh_loaded_df[Trip.SIM_COL_VEH_OVERCAP] = veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD] - veh_loaded_df[ + Trip.VEHICLES_COLUMN_TOTAL_CAPACITY] veh_loaded_df[Trip.SIM_COL_VEH_OVERCAP_FRAC] = 0.0 - veh_loaded_df.loc[veh_loaded_df[Trip.SIM_COL_VEH_BOARDS ]>0, Trip.SIM_COL_VEH_OVERCAP_FRAC] = veh_loaded_df[Trip.SIM_COL_VEH_OVERCAP] / veh_loaded_df[Trip.SIM_COL_VEH_BOARDS] + veh_loaded_df.loc[veh_loaded_df[Trip.SIM_COL_VEH_BOARDS] > 0, Trip.SIM_COL_VEH_OVERCAP_FRAC] = veh_loaded_df[ + Trip.SIM_COL_VEH_OVERCAP] / \ + veh_loaded_df[ + Trip.SIM_COL_VEH_BOARDS] FastTripsLogger.debug("veh_loaded_df with onboard>0: (showing head)\n" + \ - veh_loaded_df.loc[veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD]>0].head().to_string()) + veh_loaded_df.loc[veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD] > 0].head().to_string()) return veh_loaded_df @@ -1623,21 +1701,31 @@ def flag_missed_transfers(pathset_paths_df, pathset_links_df): Assignment.SIM_COL_MISSED_XFER], axis=1, inplace=True) # Set alight delay (min) - FastTripsLogger.debug("flag_missed_transfers() pathset_links_df (%d):\n%s" % (len(pathset_links_df), pathset_links_df.head().to_string())) + FastTripsLogger.debug("flag_missed_transfers() pathset_links_df (%d):\n%s" % ( + len(pathset_links_df), pathset_links_df.head().to_string())) pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = 0.0 - pathset_links_df.loc[pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = \ - ((pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME]-pathset_links_df[Passenger.PF_COL_PAX_B_TIME]) / np.timedelta64(1, 'm')) + pathset_links_df.loc[ + pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = \ + ((pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] - pathset_links_df[ + Passenger.PF_COL_PAX_B_TIME]) / np.timedelta64(1, 'm')) #: todo: is there a more elegant way to take care of this? some trips have times after midnight so they're the next day - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN]>22*60, Assignment.SIM_COL_PAX_BOARD_TIME ] = pathset_links_df[Assignment.SIM_COL_PAX_BOARD_TIME] - np.timedelta64(24, 'h') - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN]>22*60, Assignment.SIM_COL_PAX_ALIGHT_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] - np.timedelta64(24, 'h') - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN]>22*60, Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = \ - ((pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME]-pathset_links_df[Passenger.PF_COL_PAX_B_TIME]) / np.timedelta64(1, 'm')) + pathset_links_df.loc[ + pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] > 22 * 60, Assignment.SIM_COL_PAX_BOARD_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_TIME] - np.timedelta64(24, 'h') + pathset_links_df.loc[ + pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] > 22 * 60, Assignment.SIM_COL_PAX_ALIGHT_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] - np.timedelta64(24, 'h') + pathset_links_df.loc[pathset_links_df[ + Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] > 22 * 60, Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = \ + ((pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] - pathset_links_df[ + Passenger.PF_COL_PAX_B_TIME]) / np.timedelta64(1, 'm')) max_alight_delay_min = pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN].max() FastTripsLogger.debug("Biggest alight_delay = %f" % max_alight_delay_min) if max_alight_delay_min > 0: - FastTripsLogger.debug("\n%s" % pathset_links_df.sort_values(by=Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN, ascending=False).head().to_string()) + FastTripsLogger.debug("\n%s" % pathset_links_df.sort_values(by=Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN, + ascending=False).head().to_string()) # For trips, alight_time is the new B_time # Set A_time for links AFTER trip links by joining to next leg @@ -1648,25 +1736,32 @@ def flag_missed_transfers(pathset_paths_df, pathset_links_df): Passenger.PF_COL_LINK_NUM, Assignment.SIM_COL_PAX_ALIGHT_TIME]].copy() next_trips[Passenger.PF_COL_LINK_NUM] = next_trips[Passenger.PF_COL_LINK_NUM] + 1 - next_trips.rename(columns={Assignment.SIM_COL_PAX_ALIGHT_TIME:Assignment.SIM_COL_PAX_A_TIME}, inplace=True) + next_trips.rename(columns={Assignment.SIM_COL_PAX_ALIGHT_TIME: Assignment.SIM_COL_PAX_A_TIME}, inplace=True) # Add it to passenger trips. Now A time is set for links after trip links (note this will never be a trip link) - pathset_links_df = pd.merge(left =pathset_links_df, - right=next_trips, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]) + pathset_links_df = pd.merge(left=pathset_links_df, + right=next_trips, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]) FastTripsLogger.debug(str(pathset_links_df.dtypes)) # Set the new B time for those links -- link time for access/egress/xfer is travel time since wait times are in trip links - pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] + pathset_links_df[Passenger.PF_COL_LINK_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] + \ + pathset_links_df[Passenger.PF_COL_LINK_TIME] # For trip links, it's alight time - pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRIP, Assignment.SIM_COL_PAX_B_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] + pathset_links_df.loc[ + pathset_links_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP, Assignment.SIM_COL_PAX_B_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_TIME] # For access links, it doesn't change from the original pathfinding result - pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_ACCESS, Assignment.SIM_COL_PAX_A_TIME] = pathset_links_df[Passenger.PF_COL_PAX_A_TIME] - pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_ACCESS, Assignment.SIM_COL_PAX_B_TIME] = pathset_links_df[Passenger.PF_COL_PAX_B_TIME] + pathset_links_df.loc[ + pathset_links_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS, Assignment.SIM_COL_PAX_A_TIME] = \ + pathset_links_df[Passenger.PF_COL_PAX_A_TIME] + pathset_links_df.loc[ + pathset_links_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS, Assignment.SIM_COL_PAX_B_TIME] = \ + pathset_links_df[Passenger.PF_COL_PAX_B_TIME] # Now we only need to set the trip link's A time from the previous link's new_B_time next_trips = pathset_links_df[[ @@ -1676,44 +1771,59 @@ def flag_missed_transfers(pathset_paths_df, pathset_links_df): Passenger.PF_COL_LINK_NUM, Assignment.SIM_COL_PAX_B_TIME]].copy() next_trips[Passenger.PF_COL_LINK_NUM] = next_trips[Passenger.PF_COL_LINK_NUM] + 1 - next_trips.rename(columns={Assignment.SIM_COL_PAX_B_TIME:"new_trip_A_time"}, inplace=True) + next_trips.rename(columns={Assignment.SIM_COL_PAX_B_TIME: "new_trip_A_time"}, inplace=True) # Add it to passenger trips. Now new_trip_A_time is set for trip links - pathset_links_df = pd.merge(left =pathset_links_df, - right =next_trips, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]) - pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRIP, Assignment.SIM_COL_PAX_A_TIME] = pathset_links_df["new_trip_A_time"] + pathset_links_df = pd.merge(left=pathset_links_df, + right=next_trips, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]) + pathset_links_df.loc[ + pathset_links_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP, Assignment.SIM_COL_PAX_A_TIME] = \ + pathset_links_df["new_trip_A_time"] pathset_links_df.drop(["new_trip_A_time"], axis=1, inplace=True) - pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - \ + pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] #: todo: is there a more elegant way to take care of this? some trips have times after midnight so they're the next day #: if the linktime > 22 hours then the trip time is probably off by a day, so it's right after midnight -- back it up - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]/ np.timedelta64(1, 'h') > 22, Assignment.SIM_COL_PAX_B_TIME ] = pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - np.timedelta64(24, 'h') - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]/ np.timedelta64(1, 'h') > 22, Assignment.SIM_COL_PAX_LINK_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] + pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] / np.timedelta64(1, + 'h') > 22, Assignment.SIM_COL_PAX_B_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - np.timedelta64(24, 'h') + pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] / np.timedelta64(1, + 'h') > 22, Assignment.SIM_COL_PAX_LINK_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] #: if the linktime < -22 hours then the trip time is probably off by a day, so it's right before midnight -- back it up - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]/ np.timedelta64(-1, 'h') < -22, Assignment.SIM_COL_PAX_A_TIME ] = pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] - np.timedelta64(24, 'h') - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]/ np.timedelta64(-1, 'h') < -22, Assignment.SIM_COL_PAX_LINK_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] - + pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] / np.timedelta64(-1, + 'h') < -22, Assignment.SIM_COL_PAX_A_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] - np.timedelta64(24, 'h') + pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] / np.timedelta64(-1, + 'h') < -22, Assignment.SIM_COL_PAX_LINK_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] # new wait time - pathset_links_df.loc[pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Assignment.SIM_COL_PAX_WAIT_TIME] = pathset_links_df[Assignment.SIM_COL_PAX_BOARD_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] + pathset_links_df.loc[ + pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Assignment.SIM_COL_PAX_WAIT_TIME] = \ + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] # invalid trips have negative wait time pathset_links_df[Assignment.SIM_COL_MISSED_XFER] = 0 - pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME]/ np.timedelta64(1,'m') < 0, Assignment.SIM_COL_MISSED_XFER] = 1 + pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] / np.timedelta64(1, + 'm') < 0, Assignment.SIM_COL_MISSED_XFER] = 1 # count how many are valid (sum of invalid = 0 for the trip list id + path) - pathset_links_df_grouped = pathset_links_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this - Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM]).aggregate({Assignment.SIM_COL_MISSED_XFER:"sum" }) + pathset_links_df_grouped = pathset_links_df.groupby( + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this + Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM]).aggregate({Assignment.SIM_COL_MISSED_XFER: "sum"}) - pathset_links_df_grouped.loc[pathset_links_df_grouped[Assignment.SIM_COL_MISSED_XFER]> 0, Assignment.SIM_COL_MISSED_XFER] = 1 + pathset_links_df_grouped.loc[ + pathset_links_df_grouped[Assignment.SIM_COL_MISSED_XFER] > 0, Assignment.SIM_COL_MISSED_XFER] = 1 FastTripsLogger.info(" flag_missed_transfers found %d missed transfer trip legs for %d paths" % \ (pathset_links_df[Assignment.SIM_COL_MISSED_XFER].sum(), @@ -1723,10 +1833,14 @@ def flag_missed_transfers(pathset_paths_df, pathset_links_df): if Assignment.SIM_COL_MISSED_XFER in list(pathset_paths_df.columns.values): pathset_paths_df.drop([Assignment.SIM_COL_MISSED_XFER], axis=1, inplace=True) - pathset_paths_df = pd.merge(left =pathset_paths_df, - right =pathset_links_df_grouped.reset_index()[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM, Assignment.SIM_COL_MISSED_XFER]], - how ="left") - FastTripsLogger.debug("flag_missed_transfers() pathset_paths_df (%d):\n%s" % (len(pathset_paths_df), pathset_paths_df.head(30).to_string())) + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=pathset_links_df_grouped.reset_index()[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Assignment.SIM_COL_MISSED_XFER]], + how="left") + FastTripsLogger.debug("flag_missed_transfers() pathset_paths_df (%d):\n%s" % ( + len(pathset_paths_df), pathset_paths_df.head(30).to_string())) return (pathset_paths_df, pathset_links_df) @@ -1781,7 +1895,7 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul Passenger.PF_COL_ROUTE_ID, Passenger.PF_COL_TRIP_ID, Passenger.PF_COL_PAX_A_TIME, - "A_id","A_id_num","A_seq", + "A_id", "A_id_num", "A_seq", Assignment.SIM_COL_PAX_A_TIME, Assignment.SIM_COL_PAX_OVERCAP, Assignment.SIM_COL_PAX_OVERCAP_FRAC, @@ -1790,7 +1904,7 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul Assignment.SIM_COL_PAX_CHOSEN, ] - current_pf_iter = 0.01*pathfinding_iteration + iteration + current_pf_iter = 0.01 * pathfinding_iteration + iteration current_sim_iter = "iter%.2f sim%d" % (current_pf_iter, simulation_iteration) # this will involve looping @@ -1798,78 +1912,92 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul bump_iter = 0 pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP_FRAC] = np.NaN - if simulation_iteration==0: + if simulation_iteration == 0: # For those we just found paths for, no one is bumped or going on overcap vehicles yet # pathset_paths_df.loc[pathset_paths_df[Passenger.PF_COL_PF_ITERATION]==current_pf_iter, Assignment.SIM_COL_PAX_BUMP_ITER ] = np.NaN # pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_PF_ITERATION]==current_pf_iter, Assignment.SIM_COL_PAX_BUMP_ITER ] = np.NaN # pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_PF_ITERATION]==current_pf_iter, Assignment.SIM_COL_PAX_BOARD_STATE ] = np.NaN # anyone can be bumped, including those from previous pathfinding iters. Otherwise, we wouldn't be able to ride to an earlier stop and bump them - pathset_paths_df[ Assignment.SIM_COL_PAX_BUMP_ITER ] = np.NaN - pathset_links_df[ Assignment.SIM_COL_PAX_BUMP_ITER ] = np.NaN - pathset_links_df[ Assignment.SIM_COL_PAX_BOARD_STATE ] = np.NaN + pathset_paths_df[Assignment.SIM_COL_PAX_BUMP_ITER] = np.NaN + pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER] = np.NaN + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] = np.NaN # make sure BOARD_STATE and CHOSEN are categorical - pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN ] = pd.Categorical( pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN ], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ] = pd.Categorical( pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) - pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] = pd.Categorical( pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE], categories=Assignment.BOARD_STATE_CATEGORICAL) + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] = pd.Categorical( + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE], categories=Assignment.BOARD_STATE_CATEGORICAL) - while True: # loop for capacity constraint + while True: # loop for capacity constraint FastTripsLogger.info(" Step 5.1 Put passengers on transit vehicles.") # Put passengers on vehicles, updating the vehicle's boards, alights, onboard, overcap, overcap_frac veh_loaded_df = Assignment.put_passengers_on_vehicles(pathset_links_df, veh_loaded_df) FastTripsLogger.debug("after putting passengers on vehicles, veh_loaded_df with onboard.head(30) = \n%s" % - veh_loaded_df.loc[ veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD]>0, vehicle_trip_debug_columns].head(30).to_string()) + veh_loaded_df.loc[ + veh_loaded_df[Trip.SIM_COL_VEH_ONBOARD] > 0, vehicle_trip_debug_columns].head( + 30).to_string()) if not Assignment.CAPACITY_CONSTRAINT: # We can't do anything about capacity so assume everyone boarded - pathset_links_df.loc[ (pathset_links_df[Passenger.PF_COL_PF_ITERATION]==current_pf_iter)& - (pathset_links_df[Passenger.PF_COL_TRIP_ID].notnull()), - Assignment.SIM_COL_PAX_BOARD_STATE ] = "board_easy" + pathset_links_df.loc[(pathset_links_df[Passenger.PF_COL_PF_ITERATION] == current_pf_iter) & + (pathset_links_df[Passenger.PF_COL_TRIP_ID].notnull()), + Assignment.SIM_COL_PAX_BOARD_STATE] = "board_easy" break FastTripsLogger.info(" Step 5.2 Capacity constraints on transit vehicles.") if bump_iter == 0: - FastTripsLogger.info(" Bumping one at a time? %s" % ("true" if Assignment.BUMP_ONE_AT_A_TIME else "false")) + FastTripsLogger.info( + " Bumping one at a time? %s" % ("true" if Assignment.BUMP_ONE_AT_A_TIME else "false")) # This will update board time, alight time, overcap, overcap_frac pathset_links_df = Assignment.find_passenger_vehicle_times(pathset_links_df, veh_loaded_df) - FastTripsLogger.debug("pathset_links_df.head(20)=\n%s" % pathset_links_df[pax_links_debug_columns].head(20).to_string()) + FastTripsLogger.debug( + "pathset_links_df.head(20)=\n%s" % pathset_links_df[pax_links_debug_columns].head(20).to_string()) # make sure BOARD_STATE and CHOSEN are categorical - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ] = pd.Categorical( pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) - pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] = pd.Categorical( pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE], categories=Assignment.BOARD_STATE_CATEGORICAL) + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] = pd.Categorical( + pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE], categories=Assignment.BOARD_STATE_CATEGORICAL) # CHOSEN: Everyone who can board easily, do so - pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only - pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped - (pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]>Assignment.CHOSEN_NOT_CHOSEN_YET)& # chosen - (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP]<0), # can board - Assignment.SIM_COL_PAX_BOARD_STATE ] = "board_easy" + pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only + pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped + (pathset_links_df[ + Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET) & # chosen + (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP] < 0), # can board + Assignment.SIM_COL_PAX_BOARD_STATE] = "board_easy" # CHOSEN: Everyone who can squeeze in, do so - pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only - pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped - (pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]>Assignment.CHOSEN_NOT_CHOSEN_YET)& # chosen - (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP]==0), # can barely board - Assignment.SIM_COL_PAX_BOARD_STATE ] = "boarded" + pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only + pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped + (pathset_links_df[ + Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET) & # chosen + (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP] == 0), # can barely board + Assignment.SIM_COL_PAX_BOARD_STATE] = "boarded" # UNCHOSEN: paths that are overcap -- nope - pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only - pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped - (pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]==Assignment.CHOSEN_NOT_CHOSEN_YET)& # unchosen - (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP]>=0), # overcap - Assignment.SIM_COL_PAX_BOARD_STATE ] = "bumped_unchosen" - pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only - pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped - (pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]==Assignment.CHOSEN_NOT_CHOSEN_YET)& # unchosen - (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP]>=0), # overcap - Assignment.SIM_COL_PAX_BUMP_ITER ] = bump_iter + pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only + pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped + (pathset_links_df[ + Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET) & # unchosen + (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP] >= 0), # overcap + Assignment.SIM_COL_PAX_BOARD_STATE] = "bumped_unchosen" + pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only + pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped + (pathset_links_df[ + Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET) & # unchosen + (pathset_links_df[Assignment.SIM_COL_PAX_OVERCAP] >= 0), # overcap + Assignment.SIM_COL_PAX_BUMP_ITER] = bump_iter # For those trying to board overcap, choose the winners and losers # These are trips/stops over capacity overcap_df = veh_loaded_df.loc[veh_loaded_df[Trip.SIM_COL_VEH_OVERCAP] > 0] - FastTripsLogger.debug("load_passengers_on_vehicles_with_cap() %d vehicle trip/stops over capacity: (showing head)\n%s" % \ - (len(overcap_df), overcap_df[vehicle_trip_debug_columns].head().to_string())) + FastTripsLogger.debug( + "load_passengers_on_vehicles_with_cap() %d vehicle trip/stops over capacity: (showing head)\n%s" % \ + (len(overcap_df), overcap_df[vehicle_trip_debug_columns].head().to_string())) # If none, we're done if len(overcap_df) == 0: @@ -1878,41 +2006,49 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul # 2) Look at the trip-stops where the *first people* board after we're at capacity (impossible boards) if any bump_stops_df = overcap_df.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID]).aggregate('first').reset_index() - FastTripsLogger.debug("load_passengers_on_vehicles_with_cap() bump_stops_df iter=%d pf_iter=%d sim_iter=%d bump_iter=%d (%d rows, showing head):\n%s" % - (iteration, pathfinding_iteration, simulation_iteration, bump_iter, - len(bump_stops_df), bump_stops_df[vehicle_trip_debug_columns].head().to_string())) + FastTripsLogger.debug( + "load_passengers_on_vehicles_with_cap() bump_stops_df iter=%d pf_iter=%d sim_iter=%d bump_iter=%d (%d rows, showing head):\n%s" % + (iteration, pathfinding_iteration, simulation_iteration, bump_iter, + len(bump_stops_df), bump_stops_df[vehicle_trip_debug_columns].head().to_string())) if Assignment.BUMP_ONE_AT_A_TIME: bump_stops_df.sort_values(by=[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME], inplace=True) bump_stops_df = bump_stops_df.iloc[:1] - FastTripsLogger.info(" Need to bump %d passengers from %d trip-stops" % (bump_stops_df.overcap.sum(), len(bump_stops_df))) + FastTripsLogger.info(" Need to bump %d passengers from %d trip-stops" % ( + bump_stops_df.overcap.sum(), len(bump_stops_df))) # debug -- see the whole trip if True: FastTripsLogger.debug("load_passengers_on_vehicles_with_cap() Trips with bump stops:\n%s\n" % \ - pd.merge( - left=veh_loaded_df[vehicle_trip_debug_columns], - right=bump_stops_df[[Trip.STOPTIMES_COLUMN_TRIP_ID]], - how='inner').to_string()) + pd.merge( + left=veh_loaded_df[vehicle_trip_debug_columns], + right=bump_stops_df[[Trip.STOPTIMES_COLUMN_TRIP_ID]], + how='inner').to_string()) # make sure CHOSEN is categorical - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ] = pd.Categorical( pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN ], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) # join CHOSEN pathset links to bump_stops_df; now passenger links boarding at a bump stop will have Trip.STOPTIMES_COLUMN_STOP_SEQUENCE set - bumpstop_boards = pd.merge(left =pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only - pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped - (pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]>Assignment.CHOSEN_NOT_CHOSEN_YET) ], # chosen - left_on =[Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq"], - right =bump_stops_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]], - right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - how ="left") + bumpstop_boards = pd.merge( + left=pathset_links_df.loc[pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() & # trip links only + pathset_links_df[ + Assignment.SIM_COL_PAX_BUMP_ITER].isnull() & # not already bumped + (pathset_links_df[ + Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET)], + # chosen + left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq"], + right=bump_stops_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]], + right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + how="left") # bump candidates: boarding at bump stops, chosen paths - bumpstop_boards = bumpstop_boards.loc[ bumpstop_boards[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE].notnull(), # board at bump_stops_df stop - pax_links_debug_columns].copy() + bumpstop_boards = bumpstop_boards.loc[ + bumpstop_boards[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE].notnull(), # board at bump_stops_df stop + pax_links_debug_columns].copy() # bump off later arrivals, later trip_list_num bumpstop_boards.sort_values(by=[ \ - Assignment.SIM_COL_PAX_A_TIME, # I think this is correct + Assignment.SIM_COL_PAX_A_TIME, # I think this is correct Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq", Passenger.PF_COL_PAX_A_TIME, @@ -1922,92 +2058,107 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul # For each trip_id, stop_seq, stop_id, we want the first *overcap* rows # group to trip_id, stop_seq, stop_id and count off - bpb_count = bumpstop_boards.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID,"A_seq","A_id_num"]).cumcount() + bpb_count = bumpstop_boards.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq", "A_id_num"]).cumcount() bpb_count.name = 'bump_index' # Add the bump index to our passenger-paths/stops bumpstop_boards = pd.concat([bumpstop_boards, bpb_count], axis=1) # bump or board them - bumpstop_boards[ Assignment.SIM_COL_PAX_BOARD_STATE ] = pd.Categorical(["boarded"]*len(bumpstop_boards), categories=Assignment.BOARD_STATE_CATEGORICAL) - bumpstop_boards.loc[ bumpstop_boards["bump_index"] < bumpstop_boards[Trip.SIM_COL_VEH_OVERCAP], Assignment.SIM_COL_PAX_BOARD_STATE ] = "bumped" # these folks got bumped - bumpstop_boards.loc[ bumpstop_boards["bump_index"] < bumpstop_boards[Trip.SIM_COL_VEH_OVERCAP], Assignment.SIM_COL_PAX_BUMP_ITER ] = bump_iter # these folks got bumped + bumpstop_boards[Assignment.SIM_COL_PAX_BOARD_STATE] = pd.Categorical(["boarded"] * len(bumpstop_boards), + categories=Assignment.BOARD_STATE_CATEGORICAL) + bumpstop_boards.loc[bumpstop_boards["bump_index"] < bumpstop_boards[ + Trip.SIM_COL_VEH_OVERCAP], Assignment.SIM_COL_PAX_BOARD_STATE] = "bumped" # these folks got bumped + bumpstop_boards.loc[bumpstop_boards["bump_index"] < bumpstop_boards[ + Trip.SIM_COL_VEH_OVERCAP], Assignment.SIM_COL_PAX_BUMP_ITER] = bump_iter # these folks got bumped - FastTripsLogger.debug("load_passengers_on_vehicles_with_cap() bumpstop_boards (%d rows, showing head):\n%s" % \ - (len(bumpstop_boards), bumpstop_boards.head(50).to_string())) + FastTripsLogger.debug( + "load_passengers_on_vehicles_with_cap() bumpstop_boards (%d rows, showing head):\n%s" % \ + (len(bumpstop_boards), bumpstop_boards.head(50).to_string())) # filter to unique passengers/paths who got bumped - bump_paths = bumpstop_boards.loc[ bumpstop_boards[ Assignment.SIM_COL_PAX_BOARD_STATE ] == "bumped", - [Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Passenger.PF_COL_PATH_NUM]].drop_duplicates() + bump_paths = bumpstop_boards.loc[bumpstop_boards[Assignment.SIM_COL_PAX_BOARD_STATE] == "bumped", + [Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.PF_COL_PATH_NUM]].drop_duplicates() chosen_paths_bumped = len(bump_paths) # figure when the wait time starts for the bump stops - new_bump_wait = bumpstop_boards[[Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq", "A_id_num", Passenger.PF_COL_PAX_A_TIME]].groupby( \ - [Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq","A_id_num"]).first().reset_index(drop=False) - new_bump_wait.rename(columns={"A_seq" :Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - "A_id_num":Trip.STOPTIMES_COLUMN_STOP_ID_NUM}, inplace=True) + new_bump_wait = bumpstop_boards[ + [Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq", "A_id_num", Passenger.PF_COL_PAX_A_TIME]].groupby( \ + [Trip.STOPTIMES_COLUMN_TRIP_ID, "A_seq", "A_id_num"]).first().reset_index(drop=False) + new_bump_wait.rename(columns={"A_seq": Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, + "A_id_num": Trip.STOPTIMES_COLUMN_STOP_ID_NUM}, inplace=True) # need trip id num - new_bump_wait = trips.add_numeric_trip_id(new_bump_wait, Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_TRIP_ID_NUM) - FastTripsLogger.debug("new_bump_wait (%d rows, showing head):\n%s" % (len(new_bump_wait), new_bump_wait.head().to_string())) + new_bump_wait = trips.add_numeric_trip_id(new_bump_wait, Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_TRIP_ID_NUM) + FastTripsLogger.debug( + "new_bump_wait (%d rows, showing head):\n%s" % (len(new_bump_wait), new_bump_wait.head().to_string())) - # incorporate it into the bump wait df + # incorporate it into the bump wait df if type(Assignment.bump_wait_df) == type(None): Assignment.bump_wait_df = new_bump_wait else: Assignment.bump_wait_df = pd.concat([Assignment.bump_wait_df, new_bump_wait], axis=0) - FastTripsLogger.debug("load_passengers_on_vehicles_with_cap() bump_wait_df (%d rows, showing head):\n%s" % + FastTripsLogger.debug( + "load_passengers_on_vehicles_with_cap() bump_wait_df (%d rows, showing head):\n%s" % (len(Assignment.bump_wait_df), Assignment.bump_wait_df.head().to_string())) Assignment.bump_wait_df.drop_duplicates(subset=[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], inplace=True) # finally, incorporate the board state and bump_iter to the full pathset_links_df - pathset_links_df = pd.merge(left =pathset_links_df, - right =bumpstop_boards[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - Assignment.SIM_COL_PAX_BOARD_STATE, - Assignment.SIM_COL_PAX_BUMP_ITER]], - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - how ="left", - suffixes=[""," bb"], - indicator=True) - pathset_links_df.loc[ pathset_links_df["_merge"]=="both", Assignment.SIM_COL_PAX_BOARD_STATE ] = pathset_links_df["%s bb" % Assignment.SIM_COL_PAX_BOARD_STATE] - pathset_links_df.loc[ pathset_links_df["_merge"]=="both", Assignment.SIM_COL_PAX_BUMP_ITER ] = pathset_links_df["%s bb" % Assignment.SIM_COL_PAX_BUMP_ITER ] - pathset_links_df.drop(["_merge","%s bb" % Assignment.SIM_COL_PAX_BOARD_STATE, "%s bb" % Assignment.SIM_COL_PAX_BUMP_ITER], axis=1, inplace=True) + pathset_links_df = pd.merge(left=pathset_links_df, + right=bumpstop_boards[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + Assignment.SIM_COL_PAX_BOARD_STATE, + Assignment.SIM_COL_PAX_BUMP_ITER]], + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + how="left", + suffixes=["", " bb"], + indicator=True) + pathset_links_df.loc[pathset_links_df["_merge"] == "both", Assignment.SIM_COL_PAX_BOARD_STATE] = \ + pathset_links_df["%s bb" % Assignment.SIM_COL_PAX_BOARD_STATE] + pathset_links_df.loc[pathset_links_df["_merge"] == "both", Assignment.SIM_COL_PAX_BUMP_ITER] = \ + pathset_links_df["%s bb" % Assignment.SIM_COL_PAX_BUMP_ITER] + pathset_links_df.drop( + ["_merge", "%s bb" % Assignment.SIM_COL_PAX_BOARD_STATE, "%s bb" % Assignment.SIM_COL_PAX_BUMP_ITER], + axis=1, inplace=True) FastTripsLogger.debug(pathset_links_df[pax_links_debug_columns].head()) # bump the whole path - bump_paths_df = pathset_links_df.loc[ pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER]==bump_iter, + bump_paths_df = pathset_links_df.loc[pathset_links_df[Assignment.SIM_COL_PAX_BUMP_ITER] == bump_iter, [Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM]].drop_duplicates() - pathset_paths_df = pd.merge(left =pathset_paths_df, - right =bump_paths_df, - how ="left", - indicator=True) - pathset_paths_df.loc[ pathset_paths_df["_merge"]=="both", Assignment.SIM_COL_PAX_BUMP_ITER ] = bump_iter + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=bump_paths_df, + how="left", + indicator=True) + pathset_paths_df.loc[pathset_paths_df["_merge"] == "both", Assignment.SIM_COL_PAX_BUMP_ITER] = bump_iter pathset_paths_df.drop(["_merge"], axis=1, inplace=True) # communicate back to other links in the same path too - pathset_links_df = pd.merge(left =pathset_links_df, - right =bump_paths_df, - how ="left", - indicator=True) - pathset_links_df.loc[ pathset_links_df["_merge"]=="both", Assignment.SIM_COL_PAX_BUMP_ITER ] = bump_iter - pathset_links_df.loc[ (pathset_links_df["_merge"]=="both")& - (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE].isnull()| - (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE]=="boarded")| - (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE]=="board_easy"))& - pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull(), - Assignment.SIM_COL_PAX_BOARD_STATE ] = "bumped_othertrip" + pathset_links_df = pd.merge(left=pathset_links_df, + right=bump_paths_df, + how="left", + indicator=True) + pathset_links_df.loc[pathset_links_df["_merge"] == "both", Assignment.SIM_COL_PAX_BUMP_ITER] = bump_iter + pathset_links_df.loc[(pathset_links_df["_merge"] == "both") & + (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE].isnull() | + (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] == "boarded") | + (pathset_links_df[Assignment.SIM_COL_PAX_BOARD_STATE] == "board_easy")) & + pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull(), + Assignment.SIM_COL_PAX_BOARD_STATE] = "bumped_othertrip" pathset_links_df.drop(["_merge"], axis=1, inplace=True) - FastTripsLogger.info(" -> completed loop bump_iter %d and bumped %d chosen paths" % (bump_iter, chosen_paths_bumped)) + FastTripsLogger.info( + " -> completed loop bump_iter %d and bumped %d chosen paths" % (bump_iter, chosen_paths_bumped)) if chosen_paths_bumped == 0: break @@ -2016,7 +2167,8 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul if type(Assignment.bump_wait_df) == pd.DataFrame and len(Assignment.bump_wait_df) > 0: Assignment.bump_wait_df[Passenger.PF_COL_PAX_A_TIME_MIN] = \ - Assignment.bump_wait_df[Passenger.PF_COL_PAX_A_TIME].map(lambda x: (60.0*x.hour) + x.minute + (x.second/60.0)) + Assignment.bump_wait_df[Passenger.PF_COL_PAX_A_TIME].map( + lambda x: (60.0 * x.hour) + x.minute + (x.second / 60.0)) if type(Assignment.bump_wait_df) == pd.DataFrame and len(Assignment.bump_wait_df) > 0: FastTripsLogger.debug("Bump_wait_df:\n%s" % Assignment.bump_wait_df.to_string()) @@ -2024,13 +2176,14 @@ def load_passengers_on_vehicles_with_cap(iteration, pathfinding_iteration, simul return (pathset_paths_df, pathset_links_df, veh_loaded_df) @staticmethod - def choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, pathset_links_df, veh_trips_df): + def choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, + pathset_links_df, veh_trips_df): """ Given a pathset for each passernger, choose a path (if relevant). That's it. Returns (valid_linked_trips, pathset_paths_df, pathset_links_df) """ - simulation_iteration = 0 + simulation_iteration = 0 num_passengers_arrived = 0 ###################################################################################################### FastTripsLogger.info(" Step 1. Find out board/alight times for all pathset links from vehicle times") @@ -2040,18 +2193,18 @@ def choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_itera # instead of flag_missed_transfers(), set these to pathfinding results pathset_links_df[Assignment.SIM_COL_PAX_ALIGHT_DELAY_MIN] = 0 - pathset_links_df[Assignment.SIM_COL_PAX_A_TIME ] = pathset_links_df[Passenger.PF_COL_PAX_A_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_B_TIME ] = pathset_links_df[Passenger.PF_COL_PAX_B_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME ] = pathset_links_df[Passenger.PF_COL_LINK_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME ] = pathset_links_df[Passenger.PF_COL_WAIT_TIME] - pathset_links_df[Assignment.SIM_COL_PAX_MISSED_XFER ] = 0 + pathset_links_df[Assignment.SIM_COL_PAX_A_TIME] = pathset_links_df[Passenger.PF_COL_PAX_A_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_B_TIME] = pathset_links_df[Passenger.PF_COL_PAX_B_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pathset_links_df[Passenger.PF_COL_LINK_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] = pathset_links_df[Passenger.PF_COL_WAIT_TIME] + pathset_links_df[Assignment.SIM_COL_PAX_MISSED_XFER] = 0 ###################################################################################################### FastTripsLogger.info(" Step 2. Calculate costs and probabilities for all pathset paths") (pathset_paths_df, pathset_links_df) = PathSet.calculate_cost( Assignment.STOCH_DISPERSION, pathset_paths_df, pathset_links_df, FT.veh_trips_df, FT.passengers.trip_list_df, FT.routes, FT.tazs, FT.transfers, stops=FT.stops, - reset_bump_iter=simulation_iteration==0) + reset_bump_iter=simulation_iteration == 0) ###################################################################################################### FastTripsLogger.info(" Step 3. Choose a path for each passenger from their pathset") @@ -2064,21 +2217,27 @@ def choose_paths_without_simulation(FT, output_dir, iteration, pathfinding_itera pathset_paths_df, pathset_links_df) # Write the pathsets - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, False, Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_links_df, True, Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, + False, Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_links_df, + True, Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) # write the final chosen paths for this iteration chosen_links_df = Passenger.get_chosen_links(pathset_links_df) chosen_links_df["iteration"] = iteration - Util.write_dataframe(chosen_links_df, "chosen_links_df", os.path.join(output_dir, "chosenpaths_links.csv"), append=(iteration>1), - drop_debug_columns =not Assignment.DEBUG_OUTPUT_COLUMNS, + Util.write_dataframe(chosen_links_df, "chosen_links_df", os.path.join(output_dir, "chosenpaths_links.csv"), + append=(iteration > 1), + drop_debug_columns=not Assignment.DEBUG_OUTPUT_COLUMNS, drop_pathfinding_columns=not Assignment.DEBUG_OUTPUT_COLUMNS) chosen_links_df.drop(["iteration"], axis=1, inplace=True) chosen_paths_df = Passenger.get_chosen_links(pathset_paths_df) chosen_paths_df["iteration"] = iteration - Util.write_dataframe(chosen_paths_df, "chosen_paths_df", os.path.join(output_dir, "chosenpaths_paths.csv"), append=(iteration>1), - drop_debug_columns =not Assignment.DEBUG_OUTPUT_COLUMNS, + Util.write_dataframe(chosen_paths_df, "chosen_paths_df", os.path.join(output_dir, "chosenpaths_paths.csv"), + append=(iteration > 1), + drop_debug_columns=not Assignment.DEBUG_OUTPUT_COLUMNS, drop_pathfinding_columns=not Assignment.DEBUG_OUTPUT_COLUMNS) chosen_paths_df.drop(["iteration"], axis=1, inplace=True) @@ -2092,11 +2251,12 @@ def simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, Returns (valid_linked_trips, pathset_paths_df, pathset_links_df, veh_loaded_df) """ - simulation_iteration = 0 - num_passengers_arrived = 0 # will get returned from choose_paths + simulation_iteration = 0 + num_passengers_arrived = 0 # will get returned from choose_paths while True: - FT.performance.record_step_start(iteration, pathfinding_iteration, simulation_iteration, "simulation iteration") + FT.performance.record_step_start(iteration, pathfinding_iteration, simulation_iteration, + "simulation iteration") FastTripsLogger.info("Simulation Iteration %d" % simulation_iteration) # for trace_tuple in Assignment.TRACE_PERSON_IDS: # FastTripsLogger.debug("Initial pathset_links_df for %s\n%s" % \ @@ -2127,12 +2287,14 @@ def simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, # Choose path for each passenger -- pathset_paths_df and pathset_links_df will now have # SIM_COL_PAX_CHOSEN >=0 for chosen paths/path links (num_passengers_arrived, num_chosen, pathset_paths_df, pathset_links_df) = Passenger.choose_paths( - Assignment.PATHFINDING_EVERYONE and simulation_iteration==0, # choose for everyone if we just re-found all paths + Assignment.PATHFINDING_EVERYONE and simulation_iteration == 0, + # choose for everyone if we just re-found all paths iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, pathset_links_df) ###################################################################################################### - FastTripsLogger.info(" Step 5. Put passenger paths on transit vehicles to get vehicle boards/alights/load and assess capacity constraints") + FastTripsLogger.info( + " Step 5. Put passenger paths on transit vehicles to get vehicle boards/alights/load and assess capacity constraints") (pathset_paths_df, pathset_links_df, veh_trips_df) = Assignment.load_passengers_on_vehicles_with_cap( iteration, pathfinding_iteration, simulation_iteration, @@ -2141,18 +2303,22 @@ def simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, ###################################################################################################### FastTripsLogger.info(" Step 6. Update dwell and travel times for transit vehicles") # update the trip times -- accel/decel rates + stops affect travel times, and boards/alights affect dwell times - veh_trips_df = Trip.update_trip_times(veh_trips_df, Assignment.MSA_RESULTS) + veh_trips_df = Trip.update_trip_times(veh_trips_df, Assignment.MSA_RESULTS) ###################################################################################################### if Assignment.OUTPUT_PATHSET_PER_SIM_ITER: FastTripsLogger.info(" Step 7. Write pathsets (paths and links)") - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, False, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_links_df, True, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, + pathset_paths_df, False, + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, + pathset_links_df, True, + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) # and vehicle trips - Assignment.write_vehicle_trips(output_dir, iteration, pathfinding_iteration, simulation_iteration, veh_trips_df) - + Assignment.write_vehicle_trips(output_dir, iteration, pathfinding_iteration, simulation_iteration, + veh_trips_df) FT.performance.record_step_end(iteration, pathfinding_iteration, simulation_iteration) simulation_iteration += 1 @@ -2162,33 +2328,41 @@ def simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, break if simulation_iteration > Assignment.MAX_SIMULATION_ITERS: - FastTripsLogger.info(" Maximum simulation iterations reached (%d) => Ending simulation loop" % Assignment.MAX_SIMULATION_ITERS) + FastTripsLogger.info( + " Maximum simulation iterations reached (%d) => Ending simulation loop" % Assignment.MAX_SIMULATION_ITERS) break - FT.performance.record_step_start(iteration, pathfinding_iteration, simulation_iteration, "output_per_simulation") + FT.performance.record_step_start(iteration, pathfinding_iteration, simulation_iteration, + "output_per_simulation") # Write the pathsets (if we haven't been already) if Assignment.OUTPUT_PATHSET_PER_SIM_ITER == False: - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, False, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) - Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_links_df, True, - Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, + False, + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) + Passenger.write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_links_df, + True, + Assignment.OUTPUT_PATHSET_PER_SIM_ITER, not Assignment.DEBUG_OUTPUT_COLUMNS, + not Assignment.DEBUG_OUTPUT_COLUMNS) # write the final chosen paths for this iteration chosen_links_df = Passenger.get_chosen_links(pathset_links_df) - chosen_links_df["iteration"] = iteration + chosen_links_df["iteration"] = iteration chosen_links_df["pathfinding_iteration"] = pathfinding_iteration - Util.write_dataframe(chosen_links_df, "chosen_links_df", os.path.join(output_dir, "chosenpaths_links.csv"), append=((iteration>1) or (pathfinding_iteration>1)), - drop_debug_columns =not Assignment.DEBUG_OUTPUT_COLUMNS, + Util.write_dataframe(chosen_links_df, "chosen_links_df", os.path.join(output_dir, "chosenpaths_links.csv"), + append=((iteration > 1) or (pathfinding_iteration > 1)), + drop_debug_columns=not Assignment.DEBUG_OUTPUT_COLUMNS, drop_pathfinding_columns=not Assignment.DEBUG_OUTPUT_COLUMNS) chosen_links_df.drop(["iteration", "pathfinding_iteration"], axis=1, inplace=True) chosen_paths_df = Passenger.get_chosen_links(pathset_paths_df) - chosen_paths_df["iteration" ] = iteration + chosen_paths_df["iteration"] = iteration chosen_paths_df["pathfinding_iteration"] = pathfinding_iteration - Util.write_dataframe(chosen_paths_df, "chosen_paths_df", os.path.join(output_dir, "chosenpaths_paths.csv"), append=((iteration>1) or (pathfinding_iteration>1)), - drop_debug_columns =not Assignment.DEBUG_OUTPUT_COLUMNS, + Util.write_dataframe(chosen_paths_df, "chosen_paths_df", os.path.join(output_dir, "chosenpaths_paths.csv"), + append=((iteration > 1) or (pathfinding_iteration > 1)), + drop_debug_columns=not Assignment.DEBUG_OUTPUT_COLUMNS, drop_pathfinding_columns=not Assignment.DEBUG_OUTPUT_COLUMNS) chosen_paths_df.drop(["iteration", "pathfinding_iteration"], axis=1, inplace=True) @@ -2197,8 +2371,10 @@ def simulate(FT, output_dir, iteration, pathfinding_iteration, pathset_paths_df, return (num_passengers_arrived, pathset_paths_df, pathset_links_df, veh_trips_df) -def find_trip_based_paths_process_worker(iteration, pathfinding_iteration, worker_num, input_network_dir, input_demand_dir, run_config, func_file, - output_dir, todo_pathset_queue, done_queue, hyperpath, bump_wait_df, stop_times_df): +def find_trip_based_paths_process_worker(iteration, pathfinding_iteration, worker_num, input_network_dir, + input_demand_dir, run_config, func_file, + output_dir, todo_pathset_queue, done_queue, hyperpath, bump_wait_df, + stop_times_df): """ Process worker function. Processes all the paths in queue. @@ -2207,14 +2383,15 @@ def find_trip_based_paths_process_worker(iteration, pathfinding_iteration, worke worker_str = "_worker%02d" % worker_num from .FastTrips import FastTrips - setupLogging(infoLogFilename = None, - debugLogFilename = os.path.join(output_dir, FastTrips.DEBUG_LOG % worker_str), - logToConsole = False, - append = False if ((iteration==1) and (pathfinding_iteration==1)) else True) - FastTripsLogger.info("Iteration %d Pathfinding Iteration %d Worker %2d starting" % (iteration, pathfinding_iteration, worker_num)) + setupLogging(infoLogFilename=None, + debugLogFilename=os.path.join(output_dir, FastTrips.DEBUG_LOG % worker_str), + logToConsole=False, + append=False if ((iteration == 1) and (pathfinding_iteration == 1)) else True) + FastTripsLogger.info( + "Iteration %d Pathfinding Iteration %d Worker %2d starting" % (iteration, pathfinding_iteration, worker_num)) # the child process doesn't have these set so read them - Assignment.CONFIGURATION_FILE = run_config + Assignment.CONFIGURATION_FILE = run_config Assignment.CONFIGURATION_FUNCTIONS_FILE = func_file Assignment.read_functions(func_file) Assignment.read_configuration(run_config) @@ -2232,7 +2409,7 @@ def find_trip_based_paths_process_worker(iteration, pathfinding_iteration, worke # go through my queue -- check if we're done todo = todo_pathset_queue.get() if todo == 'DONE': - done_queue.put( (worker_num, 'DONE') ) + done_queue.put((worker_num, 'DONE')) FastTripsLogger.debug("Received DONE from the todo_pathset_queue") return @@ -2241,18 +2418,20 @@ def find_trip_based_paths_process_worker(iteration, pathfinding_iteration, worke FastTripsLogger.info("Processing person %20s trip %20s" % (pathset.person_id, pathset.person_trip_id)) # communicate it to the parent - done_queue.put( (worker_num, "STARTING", pathset.person_id, pathset.person_trip_id )) + done_queue.put((worker_num, "STARTING", pathset.person_id, pathset.person_trip_id)) trace_person = False if (pathset.person_id, pathset.person_trip_id) in Assignment.TRACE_IDS: - FastTripsLogger.debug("Tracing assignment of person %s trip %s" % (pathset.person_id, pathset.person_trip_id)) + FastTripsLogger.debug( + "Tracing assignment of person %s trip %s" % (pathset.person_id, pathset.person_trip_id)) trace_person = True try: - (pathdict, perf_dict) = Assignment.find_trip_based_pathset(iteration, pathfinding_iteration, pathset, hyperpath, trace=trace_person) - done_queue.put( (worker_num, "COMPLETED", pathset.trip_list_id_num, pathdict, perf_dict) ) + (pathdict, perf_dict) = Assignment.find_trip_based_pathset(iteration, pathfinding_iteration, pathset, + hyperpath, trace=trace_person) + done_queue.put((worker_num, "COMPLETED", pathset.trip_list_id_num, pathdict, perf_dict)) except: FastTripsLogger.exception("Exception") # call it a day - done_queue.put( (worker_num, "EXCEPTION", str(sys.exc_info()) ) ) + done_queue.put((worker_num, "EXCEPTION", str(sys.exc_info()))) return diff --git a/fasttrips/Error.py b/fasttrips/Error.py index e8dce4dd..d6b45095 100644 --- a/fasttrips/Error.py +++ b/fasttrips/Error.py @@ -3,7 +3,7 @@ """ __copyright__ = "Copyright 2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,12 +17,14 @@ limitations under the License. """ + class Error(Exception): """ Base class for exceptions in fast-trips. """ pass + class NetworkInputError(Error): """ Exception raised for errors in the network input. @@ -33,8 +35,9 @@ class NetworkInputError(Error): """ def __init__(self, filename, msg): - self.expr = filename - self.msg = msg + self.expr = filename + self.msg = msg + class DemandInputError(Error): """ @@ -46,8 +49,9 @@ class DemandInputError(Error): """ def __init__(self, filename, msg): - self.expr = filename - self.msg = msg + self.expr = filename + self.msg = msg + class ConfigurationError(Error): """ @@ -59,8 +63,9 @@ class ConfigurationError(Error): """ def __init__(self, filename, msg): - self.expr = filename - self.msg = msg + self.expr = filename + self.msg = msg + class NotImplementedError(Error): """ @@ -69,8 +74,10 @@ class NotImplementedError(Error): Attributes: msg -- explanation of the error """ + def __init__(self, msg): - self.msg = msg + self.msg = msg + class UnexpectedError(Error): """ @@ -79,5 +86,6 @@ class UnexpectedError(Error): Attributes: msg -- explanation of the error """ + def __init__(self, msg): - self.msg = msg + self.msg = msg diff --git a/fasttrips/Examples/Bear/run_bear.py b/fasttrips/Examples/Bear/run_bear.py index 9ee1a0b7..9fe8a4ae 100644 --- a/fasttrips/Examples/Bear/run_bear.py +++ b/fasttrips/Examples/Bear/run_bear.py @@ -2,22 +2,23 @@ from fasttrips import Run network = "cub" -demand = "earlier" -config = "A" +demand = "earlier" +config = "A" out_folder = "bear_cub_earlier_B" -ex_dir = os.path.abspath(os.path.dirname(__file__)) -print "Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:]) +ex_dir = os.path.abspath(os.path.dirname(__file__)) +print +"Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:]) Run.run_fasttrips( - input_network_dir = os.path.join(ex_dir,"networks",network), - input_demand_dir = os.path.join(ex_dir,"demand",demand), - run_config = os.path.join(ex_dir,"configs",config,"config_ft.txt"), - input_weights = os.path.join(ex_dir,"configs",config,"pathweight_ft.txt"), - output_dir = os.path.join(ex_dir,"output"), - output_folder = out_folder, - pathfinding_type = "stochastic", - overlap_variable = "count", - overlap_split_transit = True, - iters = 1, - dispersion = 0.50) + input_network_dir=os.path.join(ex_dir, "networks", network), + input_demand_dir=os.path.join(ex_dir, "demand", demand), + run_config=os.path.join(ex_dir, "configs", config, "config_ft.txt"), + input_weights=os.path.join(ex_dir, "configs", config, "pathweight_ft.txt"), + output_dir=os.path.join(ex_dir, "output"), + output_folder=out_folder, + pathfinding_type="stochastic", + overlap_variable="count", + overlap_split_transit=True, + iters=1, + dispersion=0.50) diff --git a/fasttrips/Examples/Bunny_Hop/run_bunny_hop.py b/fasttrips/Examples/Bunny_Hop/run_bunny_hop.py index e5fd5507..0e63609a 100644 --- a/fasttrips/Examples/Bunny_Hop/run_bunny_hop.py +++ b/fasttrips/Examples/Bunny_Hop/run_bunny_hop.py @@ -2,22 +2,22 @@ from fasttrips import Run network = "bunny_hop" -demand = "forward_bunnies" -config = "base" +demand = "forward_bunnies" +config = "base" out_folder = "small_bunny_hop_fwd" -ex_dir = os.path.abspath(os.path.dirname(__file__)) -print ("Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:])) +ex_dir = os.path.abspath(os.path.dirname(__file__)) +print("Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:])) Run.run_fasttrips( - input_network_dir = os.path.join(ex_dir,"networks",network), - input_demand_dir = os.path.join(ex_dir,"demand",demand), - run_config = os.path.join(ex_dir,"configs",config,"config_ft.txt"), - input_weights = os.path.join(ex_dir,"configs",config,"pathweight_ft.txt"), - output_dir = os.path.join(ex_dir,"output"), - output_folder = out_folder, - pathfinding_type = "stochastic", - overlap_variable = "count", - overlap_split_transit = True, - iters = 1, - dispersion = 0.50) + input_network_dir=os.path.join(ex_dir, "networks", network), + input_demand_dir=os.path.join(ex_dir, "demand", demand), + run_config=os.path.join(ex_dir, "configs", config, "config_ft.txt"), + input_weights=os.path.join(ex_dir, "configs", config, "pathweight_ft.txt"), + output_dir=os.path.join(ex_dir, "output"), + output_folder=out_folder, + pathfinding_type="stochastic", + overlap_variable="count", + overlap_split_transit=True, + iters=1, + dispersion=0.50) diff --git a/fasttrips/Examples/Seattle_Region/run_seattle_region.py b/fasttrips/Examples/Seattle_Region/run_seattle_region.py index d67b5d7f..ce735ae3 100644 --- a/fasttrips/Examples/Seattle_Region/run_seattle_region.py +++ b/fasttrips/Examples/Seattle_Region/run_seattle_region.py @@ -2,23 +2,24 @@ from fasttrips import Run network = "psrc_1_1" -demand = "psrc_1_1" -config = "base" +demand = "psrc_1_1" +config = "base" out_folder = "seattle_base" -ex_dir = os.path.abspath(os.path.dirname(__file__)) -print "Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:]) +ex_dir = os.path.abspath(os.path.dirname(__file__)) +print +"Running Fast-Trips in %s" % (ex_dir.split(os.sep)[-1:]) Run.run_fasttrips( - input_network_dir = os.path.join(ex_dir,"networks",network), - input_demand_dir = os.path.join(ex_dir,"demand",demand), - run_config = os.path.join(ex_dir,"configs",config,"config_ft.txt"), - input_weights = os.path.join(ex_dir,"configs",config,"pathweight_ft.txt"), - input_functions = os.path.join(ex_dir,"configs",config,'config_ft.py'), - output_dir = os.path.join(ex_dir,"output"), - output_folder = out_folder, - pathfinding_type = "stochastic", - overlap_variable = "count", - overlap_split_transit = True, - iters = 1, - dispersion = 0.50) + input_network_dir=os.path.join(ex_dir, "networks", network), + input_demand_dir=os.path.join(ex_dir, "demand", demand), + run_config=os.path.join(ex_dir, "configs", config, "config_ft.txt"), + input_weights=os.path.join(ex_dir, "configs", config, "pathweight_ft.txt"), + input_functions=os.path.join(ex_dir, "configs", config, 'config_ft.py'), + output_dir=os.path.join(ex_dir, "output"), + output_folder=out_folder, + pathfinding_type="stochastic", + overlap_variable="count", + overlap_split_transit=True, + iters=1, + dispersion=0.50) diff --git a/fasttrips/Examples/Springfield/run_springfield.py b/fasttrips/Examples/Springfield/run_springfield.py index 21f68d65..ae4b4486 100644 --- a/fasttrips/Examples/Springfield/run_springfield.py +++ b/fasttrips/Examples/Springfield/run_springfield.py @@ -2,29 +2,30 @@ from fasttrips import Run # DIRECTORY LOCATIONS -EXAMPLE_DIR = os.path.abspath(os.path.dirname(__file__)) +EXAMPLE_DIR = os.path.abspath(os.path.dirname(__file__)) -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -OUTPUT_FOLDER = "general_run" +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +OUTPUT_FOLDER = "general_run" # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') -print "Running Fast-Trips in %s" % (EXAMPLE_DIR.split(os.sep)[-1:]) +print +"Running Fast-Trips in %s" % (EXAMPLE_DIR.split(os.sep)[-1:]) Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = OUTPUT_FOLDER, - pathfinding_type = "stochastic", - overlap_variable = "count", - overlap_split_transit = True, - iters = 3, - dispersion = 0.50) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder=OUTPUT_FOLDER, + pathfinding_type="stochastic", + overlap_variable="count", + overlap_split_transit=True, + iters=3, + dispersion=0.50) diff --git a/fasttrips/FastTrips.py b/fasttrips/FastTrips.py index e901c826..7747d4cc 100644 --- a/fasttrips/FastTrips.py +++ b/fasttrips/FastTrips.py @@ -1,8 +1,9 @@ from __future__ import print_function from builtins import str from builtins import object + __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -15,21 +16,22 @@ See the License for the specific language governing permissions and limitations under the License. """ + import os import sys import partridge as ptg -from .Assignment import Assignment -from .Logger import FastTripsLogger, setupLogging -from .Passenger import Passenger +from .Assignment import Assignment +from .Logger import FastTripsLogger, setupLogging +from .Passenger import Passenger from .Performance import Performance -from .Route import Route -from .Stop import Stop -from .TAZ import TAZ -from .Transfer import Transfer -from .Trip import Trip -from .Util import Util +from .Route import Route +from .Stop import Stop +from .TAZ import TAZ +from .Transfer import Transfer +from .Trip import Trip +from .Util import Util class FastTrips(object): @@ -38,13 +40,13 @@ class FastTrips(object): """ #: Info log filename. Writes brief information about program progression here. - INFO_LOG = "ft_info%s.log" + INFO_LOG = "ft_info%s.log" #: Debug log filename. Detailed output goes here, including trace information. DEBUG_LOG = "ft_debug%s.log" def __init__(self, input_network_archive, input_demand_dir, input_weights, run_config, - output_dir, input_functions=None, logname_append="", appendLog=False): + output_dir, input_functions=None, logname_append="", appendLog=False): """ Constructor. @@ -69,40 +71,40 @@ def __init__(self, input_network_archive, input_demand_dir, input_weights, run_c """ #: :py:class:`collections.OrdederedDict` of :py:class:`fasttrips.Passenger` instances indexed by passenger's path ID - self.passengers = None + self.passengers = None #: :py:class:`dict` with :py:attr:`fasttrips.Stop.stop_id` key and :py:class:`fasttrips.Stop` value - self.stops = None + self.stops = None #: :py:class:`dict` with :py:attr:`fasttrips.Route.route_id` key and :py:class:`fasttrips.Route` value - self.routes = None + self.routes = None #: :py:class:`dict` with :py:attr:`fasttrips.TAZ.taz_id` key and :py:class:`fasttrips.TAZ` value - self.tazs = None + self.tazs = None #: :py:class:`dict` with :py:attr:`fasttrips.Trip.trip_id` key and :py:class:`fasttrips.Trip` value - self.trips = None + self.trips = None #: string representing directory with input network data - Assignment.INPUT_NETWORK_ARCHIVE = input_network_archive + Assignment.INPUT_NETWORK_ARCHIVE = input_network_archive #: string representing directory with input demand data - Assignment.INPUT_DEMAND_DIR = input_demand_dir + Assignment.INPUT_DEMAND_DIR = input_demand_dir - #: string representing directory with input demand data - Assignment.INPUT_WEIGHTS = input_weights + #: string representing directory with input demand data + Assignment.INPUT_WEIGHTS = input_weights - #: string representing directory with input demand data + #: string representing directory with input demand data Assignment.CONFIGURATION_FILE = run_config #: string representing directory with input demand data - Assignment.CONFIGURATION_FUNCTIONS_FILE = input_functions + Assignment.CONFIGURATION_FUNCTIONS_FILE = input_functions #: string representing directory in which to write our output - Assignment.OUTPUT_DIR = output_dir + Assignment.OUTPUT_DIR = output_dir #: transitfeed schedule instance. See https://github.com/google/transitfeed - #self.gtfs_schedule = None + # self.gtfs_schedule = None # setup logging setupLogging(os.path.join(Assignment.OUTPUT_DIR, FastTrips.INFO_LOG % logname_append), @@ -116,30 +118,30 @@ def read_configuration(self): """ Read the fast-trips assignment and path-finding configuration """ - self.performance.record_step_start(-1,-1,-1,"read_configuration") + self.performance.record_step_start(-1, -1, -1, "read_configuration") if Assignment.CONFIGURATION_FUNCTIONS_FILE: - Assignment.read_functions(func_file = Assignment.CONFIGURATION_FUNCTIONS_FILE) - Assignment.read_configuration(config_fullpath = Assignment.CONFIGURATION_FILE) - Assignment.read_weights(weights_file = Assignment.INPUT_WEIGHTS) + Assignment.read_functions(func_file=Assignment.CONFIGURATION_FUNCTIONS_FILE) + Assignment.read_configuration(config_fullpath=Assignment.CONFIGURATION_FILE) + Assignment.read_weights(weights_file=Assignment.INPUT_WEIGHTS) def read_input_files(self): """ Reads in the input network and demand files and initializes the relevant data structures. """ - self.performance.record_step_start(0,0,0,"read_input_files") + self.performance.record_step_start(0, 0, 0, "read_input_files") # Read the gtfs files first FastTripsLogger.info("Reading GTFS schedule") - service_ids_by_date =ptg.read_service_ids_by_date(Assignment.INPUT_NETWORK_ARCHIVE) + service_ids_by_date = ptg.read_service_ids_by_date(Assignment.INPUT_NETWORK_ARCHIVE) service_ids = service_ids_by_date[Assignment.NETWORK_BUILD_DATE] gtfs_feed = ptg.feed(os.path.join(Assignment.INPUT_NETWORK_ARCHIVE), - config=Util.get_fast_trips_config(), view={ - 'trips.txt': { - 'service_id': service_ids - }, - }) + config=Util.get_fast_trips_config(), view={ + 'trips.txt': { + 'service_id': service_ids + }, + }) # Read Stops (gtfs-required) self.stops = Stop(Assignment.INPUT_NETWORK_ARCHIVE, Assignment.OUTPUT_DIR, gtfs_feed, Assignment.NETWORK_BUILD_DATE) @@ -162,11 +164,12 @@ def read_input_files(self): self.stops, self.transfers, self.routes) # Read the demand int passenger_id -> passenger instance - self.passengers = Passenger(Assignment.INPUT_DEMAND_DIR, Assignment.OUTPUT_DIR, Assignment.NETWORK_BUILD_DATE, self.stops, self.routes, Assignment.CAPACITY_CONSTRAINT) + self.passengers = Passenger(Assignment.INPUT_DEMAND_DIR, Assignment.OUTPUT_DIR, Assignment.NETWORK_BUILD_DATE, + self.stops, self.routes, Assignment.CAPACITY_CONSTRAINT) def run_assignment(self, output_dir): - self.performance.record_step_start(-1,-1,-1,"run_assignment") + self.performance.record_step_start(-1, -1, -1, "run_assignment") # Do it! Try it! r = None @@ -181,8 +184,8 @@ def run_assignment(self, output_dir): FastTripsLogger.fatal("Unexpected error: %s" % str(sys.exc_info()[0])) raise - self.performance.record_step_end(-1,-1,-1) + self.performance.record_step_end(-1, -1, -1) self.performance.write(output_dir) FastTripsLogger.info("Successfully completed!") - return r \ No newline at end of file + return r diff --git a/fasttrips/Logger.py b/fasttrips/Logger.py index c6fae366..5d77f604 100644 --- a/fasttrips/Logger.py +++ b/fasttrips/Logger.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -12,15 +12,16 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import multiprocessing - __all__ = ['FastTripsLogger', 'setupLogging'] #: This is the instance of :py:class:`Logger` that gets used for all dta logging needs! FastTripsLogger = multiprocessing.get_logger() + def setupLogging(infoLogFilename, debugLogFilename, logToConsole=True, append=False): """ Sets up the logger. @@ -49,14 +50,15 @@ def setupLogging(infoLogFilename, debugLogFilename, logToConsole=True, append=Fa if debugLogFilename: debugloghandler = logging.StreamHandler(open(debugLogFilename, 'a' if append else 'w')) debugloghandler.setLevel(logging.DEBUG) - debugloghandler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s/%(processName)s] %(message)s', '%Y-%m-%d %H:%M:%S')) + debugloghandler.setFormatter( + logging.Formatter('%(asctime)s [%(levelname)s/%(processName)s] %(message)s', '%Y-%m-%d %H:%M:%S')) FastTripsLogger.addHandler(debugloghandler) FastTripsDebugLogFilename = debugLogFilename if logToConsole: consolehandler = logging.StreamHandler() consolehandler.setLevel(logging.INFO) - consolehandler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s/%(processName)s] %(message)s', '%Y-%m-%d %H:%M:%S')) + consolehandler.setFormatter( + logging.Formatter('%(asctime)s [%(levelname)s/%(processName)s] %(message)s', '%Y-%m-%d %H:%M:%S')) FastTripsLogger.addHandler(consolehandler) FastTripsLogToConsole = logToConsole - diff --git a/fasttrips/Passenger.py b/fasttrips/Passenger.py index dde1efba..588c96aa 100644 --- a/fasttrips/Passenger.py +++ b/fasttrips/Passenger.py @@ -4,7 +4,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import collections import os import sys @@ -24,12 +25,12 @@ import numpy as np import pandas as pd -from .Error import DemandInputError +from .Error import DemandInputError from .Logger import FastTripsLogger -from .Route import Route -from .TAZ import TAZ -from .Trip import Trip -from .Util import Util +from .Route import Route +from .TAZ import TAZ +from .Trip import Trip +from .Util import Util class Passenger(object): @@ -43,110 +44,110 @@ class Passenger(object): """ #: File with households - INPUT_HOUSEHOLDS_FILE = "household.txt" + INPUT_HOUSEHOLDS_FILE = "household.txt" #: Households column: Household ID - HOUSEHOLDS_COLUMN_HOUSEHOLD_ID = 'hh_id' + HOUSEHOLDS_COLUMN_HOUSEHOLD_ID = 'hh_id' #: File with persons - INPUT_PERSONS_FILE = "person.txt" + INPUT_PERSONS_FILE = "person.txt" #: Persons column: Household ID - PERSONS_COLUMN_HOUSEHOLD_ID = HOUSEHOLDS_COLUMN_HOUSEHOLD_ID + PERSONS_COLUMN_HOUSEHOLD_ID = HOUSEHOLDS_COLUMN_HOUSEHOLD_ID #: Persons column: Person ID (string) - PERSONS_COLUMN_PERSON_ID = 'person_id' + PERSONS_COLUMN_PERSON_ID = 'person_id' # ========== Added by fasttrips ======================================================= #: Persons column: Person ID number - PERSONS_COLUMN_PERSON_ID_NUM = 'person_id_num' + PERSONS_COLUMN_PERSON_ID_NUM = 'person_id_num' #: File with trip list - INPUT_TRIP_LIST_FILE = "trip_list.txt" + INPUT_TRIP_LIST_FILE = "trip_list.txt" #: Trip list column: Person ID - TRIP_LIST_COLUMN_PERSON_ID = PERSONS_COLUMN_PERSON_ID + TRIP_LIST_COLUMN_PERSON_ID = PERSONS_COLUMN_PERSON_ID #: Trip list column: Person Trip ID - TRIP_LIST_COLUMN_PERSON_TRIP_ID = "person_trip_id" + TRIP_LIST_COLUMN_PERSON_TRIP_ID = "person_trip_id" #: Trip list column: Origin TAZ ID - TRIP_LIST_COLUMN_ORIGIN_TAZ_ID = "o_taz" + TRIP_LIST_COLUMN_ORIGIN_TAZ_ID = "o_taz" #: Trip list column: Destination TAZ ID - TRIP_LIST_COLUMN_DESTINATION_TAZ_ID = "d_taz" + TRIP_LIST_COLUMN_DESTINATION_TAZ_ID = "d_taz" #: Trip list column: Mode - TRIP_LIST_COLUMN_MODE = "mode" + TRIP_LIST_COLUMN_MODE = "mode" #: Trip list column: Departure Time. DateTime. - TRIP_LIST_COLUMN_DEPARTURE_TIME = 'departure_time' + TRIP_LIST_COLUMN_DEPARTURE_TIME = 'departure_time' #: Trip list column: Arrival Time. DateTime. - TRIP_LIST_COLUMN_ARRIVAL_TIME = 'arrival_time' + TRIP_LIST_COLUMN_ARRIVAL_TIME = 'arrival_time' #: Trip list column: Time Target (either 'arrival' or 'departure') - TRIP_LIST_COLUMN_TIME_TARGET = 'time_target' + TRIP_LIST_COLUMN_TIME_TARGET = 'time_target' # ========== Added by fasttrips ======================================================= #: Trip list column: Unique numeric ID for this passenger/trip - TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM = "trip_list_id_num" + TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM = "trip_list_id_num" #: Trip list column: Origin TAZ Numeric ID - TRIP_LIST_COLUMN_ORIGIN_TAZ_ID_NUM = "o_taz_num" + TRIP_LIST_COLUMN_ORIGIN_TAZ_ID_NUM = "o_taz_num" #: Trip list column: Destination Numeric TAZ ID - TRIP_LIST_COLUMN_DESTINATION_TAZ_ID_NUM = "d_taz_num" + TRIP_LIST_COLUMN_DESTINATION_TAZ_ID_NUM = "d_taz_num" #: Trip list column: Departure Time. Float, minutes after midnight. - TRIP_LIST_COLUMN_DEPARTURE_TIME_MIN = 'departure_time_min' + TRIP_LIST_COLUMN_DEPARTURE_TIME_MIN = 'departure_time_min' #: Trip list column: Departure Time. Float, minutes after midnight. - TRIP_LIST_COLUMN_ARRIVAL_TIME_MIN = 'arrival_time_min' + TRIP_LIST_COLUMN_ARRIVAL_TIME_MIN = 'arrival_time_min' #: Trip list column: Transit Mode - TRIP_LIST_COLUMN_TRANSIT_MODE = "transit_mode" + TRIP_LIST_COLUMN_TRANSIT_MODE = "transit_mode" #: Trip list column: Access Mode - TRIP_LIST_COLUMN_ACCESS_MODE = "access_mode" + TRIP_LIST_COLUMN_ACCESS_MODE = "access_mode" #: Trip list column: Egress Mode - TRIP_LIST_COLUMN_EGRESS_MODE = "egress_mode" + TRIP_LIST_COLUMN_EGRESS_MODE = "egress_mode" #: Trip list column: Outbound (bool), true iff time target is arrival - TRIP_LIST_COLUMN_OUTBOUND = "outbound" + TRIP_LIST_COLUMN_OUTBOUND = "outbound" #: Option for :py:attr:`Passenger.TRIP_LIST_COLUMN_TIME_TARGET` (arrival time) - TIME_TARGET_ARRIVAL = "arrival" + TIME_TARGET_ARRIVAL = "arrival" #: Option for :py:attr:`Passenger.TRIP_LIST_COLUMN_TIME_TARGET` (departure time) - TIME_TARGET_DEPARTURE = "departure" + TIME_TARGET_DEPARTURE = "departure" #: Generic transit. Specify this for mode when you mean walk, any transit modes, walk #: TODO: get rid of this? Maybe user should always specify. - MODE_GENERIC_TRANSIT = "transit" + MODE_GENERIC_TRANSIT = "transit" #: Generic transit - Numeric mode number - MODE_GENERIC_TRANSIT_NUM = 1000 + MODE_GENERIC_TRANSIT_NUM = 1000 #: Minumum Value of Time: 1 dollar shouldn't be worth 180 minutes - MIN_VALUE_OF_TIME = 60.0/180.0 + MIN_VALUE_OF_TIME = 60.0 / 180.0 #: Trip list column: User class. String. - TRIP_LIST_COLUMN_USER_CLASS = "user_class" + TRIP_LIST_COLUMN_USER_CLASS = "user_class" #: Trip list column: Purpose. String. - TRIP_LIST_COLUMN_PURPOSE = "purpose" + TRIP_LIST_COLUMN_PURPOSE = "purpose" #: Trip list column: Value of time. Float. - TRIP_LIST_COLUMN_VOT = "vot" + TRIP_LIST_COLUMN_VOT = "vot" #: Trip list column: Trace. Boolean. - TRIP_LIST_COLUMN_TRACE = "trace" + TRIP_LIST_COLUMN_TRACE = "trace" #: Column names from pathfinding - PF_COL_PF_ITERATION = 'pf_iteration' #: 0.01*pathfinding_iteration + iteration during which this path was found - PF_COL_PAX_A_TIME = 'pf_A_time' #: time path-finder thinks passenger arrived at A - PF_COL_PAX_B_TIME = 'pf_B_time' #: time path-finder thinks passenger arrived at B - PF_COL_LINK_TIME = 'pf_linktime' #: time path-finder thinks passenger spent on link - PF_COL_LINK_FARE = 'pf_linkfare' #: fare path-finder thinks passenger spent on link - PF_COL_LINK_COST = 'pf_linkcost' #: cost (generalized) path-finder thinks passenger spent on link - PF_COL_LINK_DIST = 'pf_linkdist' #: dist path-finder thinks passenger spent on link - PF_COL_WAIT_TIME = 'pf_waittime' #: time path-finder thinks passenger waited for vehicle on trip links - - PF_COL_PATH_NUM = 'pathnum' #: path number, starting from 0 - PF_COL_LINK_NUM = 'linknum' #: link number, starting from access - PF_COL_LINK_MODE = 'linkmode' #: link mode (Access, Trip, Egress, etc) - - PF_COL_MODE = TRIP_LIST_COLUMN_MODE #: supply mode - PF_COL_ROUTE_ID = Trip.TRIPS_COLUMN_ROUTE_ID #: link route ID - PF_COL_TRIP_ID = Trip.TRIPS_COLUMN_TRIP_ID #: link trip ID - PF_COL_DESCRIPTION = 'description' #: path text description + PF_COL_PF_ITERATION = 'pf_iteration' #: 0.01*pathfinding_iteration + iteration during which this path was found + PF_COL_PAX_A_TIME = 'pf_A_time' #: time path-finder thinks passenger arrived at A + PF_COL_PAX_B_TIME = 'pf_B_time' #: time path-finder thinks passenger arrived at B + PF_COL_LINK_TIME = 'pf_linktime' #: time path-finder thinks passenger spent on link + PF_COL_LINK_FARE = 'pf_linkfare' #: fare path-finder thinks passenger spent on link + PF_COL_LINK_COST = 'pf_linkcost' #: cost (generalized) path-finder thinks passenger spent on link + PF_COL_LINK_DIST = 'pf_linkdist' #: dist path-finder thinks passenger spent on link + PF_COL_WAIT_TIME = 'pf_waittime' #: time path-finder thinks passenger waited for vehicle on trip links + + PF_COL_PATH_NUM = 'pathnum' #: path number, starting from 0 + PF_COL_LINK_NUM = 'linknum' #: link number, starting from access + PF_COL_LINK_MODE = 'linkmode' #: link mode (Access, Trip, Egress, etc) + + PF_COL_MODE = TRIP_LIST_COLUMN_MODE #: supply mode + PF_COL_ROUTE_ID = Trip.TRIPS_COLUMN_ROUTE_ID #: link route ID + PF_COL_TRIP_ID = Trip.TRIPS_COLUMN_TRIP_ID #: link trip ID + PF_COL_DESCRIPTION = 'description' #: path text description #: todo replace/rename ?? - PF_COL_PAX_A_TIME_MIN = 'pf_A_time_min' + PF_COL_PAX_A_TIME_MIN = 'pf_A_time_min' #: pathfinding results - PF_PATHS_CSV = r"enumerated_paths.csv" - PF_LINKS_CSV = r"enumerated_links.csv" + PF_PATHS_CSV = r"enumerated_paths.csv" + PF_LINKS_CSV = r"enumerated_links.csv" #: results - PathSets - PATHSET_PATHS_CSV = r"pathset_paths.csv" - PATHSET_LINKS_CSV = r"pathset_links.csv" + PATHSET_PATHS_CSV = r"pathset_paths.csv" + PATHSET_LINKS_CSV = r"pathset_links.csv" def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constraint): """ @@ -159,79 +160,82 @@ def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constra return FastTripsLogger.info("-------- Reading demand --------") - FastTripsLogger.info("Capacity constraint? %x" % capacity_constraint ) - - self.trip_list_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_TRIP_LIST_FILE), - skipinitialspace=True, ##LMZ - dtype={Passenger.TRIP_LIST_COLUMN_PERSON_ID :'S', - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID :'S', - Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID :'S', - Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID:'S', - Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME :'S', - Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME :'S', - Passenger.TRIP_LIST_COLUMN_PURPOSE :'S'}) - trip_list_cols = list(self.trip_list_df.columns.values) - - assert(Passenger.TRIP_LIST_COLUMN_PERSON_ID in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_TIME_TARGET in trip_list_cols) - assert(Passenger.TRIP_LIST_COLUMN_VOT in trip_list_cols) + FastTripsLogger.info("Capacity constraint? %x" % capacity_constraint) + + self.trip_list_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_TRIP_LIST_FILE), + skipinitialspace=True, ##LMZ + dtype={Passenger.TRIP_LIST_COLUMN_PERSON_ID: 'S', + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID: 'S', + Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID: 'S', + Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID: 'S', + Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME: 'S', + Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME: 'S', + Passenger.TRIP_LIST_COLUMN_PURPOSE: 'S'}) + trip_list_cols = list(self.trip_list_df.columns.values) + + assert (Passenger.TRIP_LIST_COLUMN_PERSON_ID in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_TIME_TARGET in trip_list_cols) + assert (Passenger.TRIP_LIST_COLUMN_VOT in trip_list_cols) FastTripsLogger.debug("=========== TRIP LIST ===========\n" + str(self.trip_list_df.head())) - FastTripsLogger.debug("\n"+str(self.trip_list_df.index.dtype)+"\n"+str(self.trip_list_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.trip_list_df.index.dtype) + "\n" + str(self.trip_list_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.trip_list_df), "person trips", Passenger.INPUT_TRIP_LIST_FILE)) # Error on missing person ids or person_trip_ids - missing_person_ids = self.trip_list_df[pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID])| + missing_person_ids = self.trip_list_df[pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID]) | pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID])] - if len(missing_person_ids)>0: + if len(missing_person_ids) > 0: error_msg = "Missing person_id or person_trip_id fields:\n%s\n" % str(missing_person_ids) error_msg += "Use 0 for person_id for trips without corresponding person." FastTripsLogger.fatal(error_msg) raise DemandInputError(Passenger.INPUT_TRIP_LIST_FILE, error_msg) # Drop (warn) on missing origins or destinations - missing_ods = self.trip_list_df[ pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID])| - pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID]) ] - if len(missing_ods)>0: - FastTripsLogger.warn("Missing origin or destination for the following trips. Dropping.\n%s" % str(missing_ods)) - self.trip_list_df = self.trip_list_df.loc[ pd.notnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID ])& - pd.notnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID]) ].reset_index(drop=True) + missing_ods = self.trip_list_df[pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID]) | + pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID])] + if len(missing_ods) > 0: + FastTripsLogger.warn( + "Missing origin or destination for the following trips. Dropping.\n%s" % str(missing_ods)) + self.trip_list_df = self.trip_list_df.loc[ + pd.notnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID]) & + pd.notnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID])].reset_index(drop=True) FastTripsLogger.warn("=> Have %d person trips" % len(self.trip_list_df)) - non_zero_person_ids = len(self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID]!="0"]) + non_zero_person_ids = len(self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_PERSON_ID] != "0"]) if non_zero_person_ids > 0 and os.path.exists(os.path.join(input_dir, Passenger.INPUT_PERSONS_FILE)): - self.persons_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_PERSONS_FILE), - skipinitialspace=True, - dtype={Passenger.PERSONS_COLUMN_PERSON_ID:'S'}) - self.persons_id_df = Util.add_numeric_column(self.persons_df[[Passenger.PERSONS_COLUMN_PERSON_ID]], - id_colname=Passenger.PERSONS_COLUMN_PERSON_ID, - numeric_newcolname=Passenger.PERSONS_COLUMN_PERSON_ID_NUM) - self.persons_df = pd.merge(left=self.persons_df, right=self.persons_id_df, - how="left") - persons_cols = list(self.persons_df.columns.values) + self.persons_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_PERSONS_FILE), + skipinitialspace=True, + dtype={Passenger.PERSONS_COLUMN_PERSON_ID: 'S'}) + self.persons_id_df = Util.add_numeric_column(self.persons_df[[Passenger.PERSONS_COLUMN_PERSON_ID]], + id_colname=Passenger.PERSONS_COLUMN_PERSON_ID, + numeric_newcolname=Passenger.PERSONS_COLUMN_PERSON_ID_NUM) + self.persons_df = pd.merge(left=self.persons_df, right=self.persons_id_df, + how="left") + persons_cols = list(self.persons_df.columns.values) FastTripsLogger.debug("=========== PERSONS ===========\n" + str(self.persons_df.head())) - FastTripsLogger.debug("\n"+str(self.persons_df.index.dtype)+"\n"+str(self.persons_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.persons_df.index.dtype) + "\n" + str(self.persons_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.persons_df), "persons", Passenger.INPUT_PERSONS_FILE)) - self.households_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_HOUSEHOLDS_FILE), skipinitialspace=True) - household_cols = list(self.households_df.columns.values) + self.households_df = pd.read_csv(os.path.join(input_dir, Passenger.INPUT_HOUSEHOLDS_FILE), + skipinitialspace=True) + household_cols = list(self.households_df.columns.values) FastTripsLogger.debug("=========== HOUSEHOLDS ===========\n" + str(self.households_df.head())) - FastTripsLogger.debug("\n"+str(self.households_df.index.dtype)+"\n"+str(self.households_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.households_df.index.dtype) + "\n" + str(self.households_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.households_df), "households", Passenger.INPUT_HOUSEHOLDS_FILE)) else: - self.persons_df = pd.DataFrame() - self.households_df = pd.DataFrame() + self.persons_df = pd.DataFrame() + self.households_df = pd.DataFrame() # make sure that each tuple TRIP_LIST_COLUMN_PERSON_ID, TRIP_LIST_COLUMN_PERSON_TRIP_ID is unique self.trip_list_df["ID_dupes"] = self.trip_list_df.duplicated(subset=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, @@ -239,9 +243,9 @@ def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constra keep=False) if self.trip_list_df["ID_dupes"].sum() > 0: error_msg = "Duplicate IDs (%s, %s) found:\n%s" % \ - (Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - self.trip_list_df.loc[self.trip_list_df["ID_dupes"]==True].to_string()) + (Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + self.trip_list_df.loc[self.trip_list_df["ID_dupes"] == True].to_string()) FastTripsLogger.fatal(error_msg) raise DemandInputError(Passenger.INPUT_TRIP_LIST_FILE, error_msg) @@ -257,55 +261,59 @@ def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constra # float version self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME_MIN] = \ self.trip_list_df[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME].map(lambda x: \ - 60*x.time().hour + x.time().minute + (x.time().second/60.0) ) + 60 * x.time().hour + x.time().minute + ( + x.time().second / 60.0)) self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME_MIN] = \ self.trip_list_df[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME].map(lambda x: \ - 60*x.time().hour + x.time().minute + (x.time().second/60.0) ) + 60 * x.time().hour + x.time().minute + ( + x.time().second / 60.0)) # TODO: validate fields? # value of time must be greater than a threshhold or any fare becomes prohibitively expensive - low_vot = self.trip_list_df.loc[ self.trip_list_df[Passenger.TRIP_LIST_COLUMN_VOT] < Passenger.MIN_VALUE_OF_TIME ] + low_vot = self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_VOT] < Passenger.MIN_VALUE_OF_TIME] if len(low_vot) > 0: - FastTripsLogger.warn("These trips have value of time lower than the minimum threshhhold (%f): raising to minimum.\n%s" % - (Passenger.MIN_VALUE_OF_TIME, str(low_vot) )) - self.trip_list_df.loc[ self.trip_list_df[Passenger.TRIP_LIST_COLUMN_VOT] < Passenger.MIN_VALUE_OF_TIME, - Passenger.TRIP_LIST_COLUMN_VOT] = Passenger.MIN_VALUE_OF_TIME + FastTripsLogger.warn( + "These trips have value of time lower than the minimum threshhhold (%f): raising to minimum.\n%s" % + (Passenger.MIN_VALUE_OF_TIME, str(low_vot))) + self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_VOT] < Passenger.MIN_VALUE_OF_TIME, + Passenger.TRIP_LIST_COLUMN_VOT] = Passenger.MIN_VALUE_OF_TIME if len(self.persons_df) > 0: # Join trips to persons self.trip_list_df = pd.merge(left=self.trip_list_df, right=self.persons_df, - how='left', - on=Passenger.TRIP_LIST_COLUMN_PERSON_ID) + how='left', + on=Passenger.TRIP_LIST_COLUMN_PERSON_ID) # are any null? - no_person_ids = self.trip_list_df.loc[ pd.isnull(self.trip_list_df[Passenger.PERSONS_COLUMN_PERSON_ID_NUM])& - (self.trip_list_df[Passenger.PERSONS_COLUMN_PERSON_ID]!="0")] + no_person_ids = self.trip_list_df.loc[pd.isnull(self.trip_list_df[Passenger.PERSONS_COLUMN_PERSON_ID_NUM]) & + (self.trip_list_df[Passenger.PERSONS_COLUMN_PERSON_ID] != "0")] if len(no_person_ids) > 0: - error_msg = "Even though a person list is given, failed to find person information for %d trips" % len(no_person_ids) + error_msg = "Even though a person list is given, failed to find person information for %d trips" % len( + no_person_ids) FastTripsLogger.fatal(error_msg) FastTripsLogger.fatal("\n%s\n" % no_person_ids.to_string()) raise DemandInputError(Passenger.INPUT_TRIP_LIST_FILE, error_msg) # And then to households self.trip_list_df = pd.merge(left=self.trip_list_df, right=self.households_df, - how='left', - on=Passenger.PERSONS_COLUMN_HOUSEHOLD_ID) + how='left', + on=Passenger.PERSONS_COLUMN_HOUSEHOLD_ID) else: # Give each passenger a unique person ID num self.trip_list_df[Passenger.PERSONS_COLUMN_PERSON_ID_NUM] = self.trip_list_df.index + 1 # add TAZ numeric ids (stored in the stop mapping) self.trip_list_df = stops.add_numeric_stop_id(self.trip_list_df, - id_colname =Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID, - numeric_newcolname=Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID_NUM, - warn =True, - warn_msg ="TAZ numbers configured as origins in demand file are not found in the network") + id_colname=Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID, + numeric_newcolname=Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID_NUM, + warn=True, + warn_msg="TAZ numbers configured as origins in demand file are not found in the network") self.trip_list_df = stops.add_numeric_stop_id(self.trip_list_df, - id_colname =Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID, - numeric_newcolname=Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID_NUM, - warn =True, - warn_msg ="TAZ numbers configured as destinations in demand file are not found in the network") + id_colname=Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID, + numeric_newcolname=Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID_NUM, + warn=True, + warn_msg="TAZ numbers configured as destinations in demand file are not found in the network") # trips with invalid TAZs have been dropped FastTripsLogger.debug("Have %d person trips" % len(self.trip_list_df)) @@ -317,51 +325,57 @@ def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constra else: # count the dashes in the mode - self.trip_list_df['mode_dash_count'] = self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE]\ + self.trip_list_df['mode_dash_count'] = self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE] \ .map(lambda x: x.count('-')) # The only modes allowed are access-transit-egress or MODE_GENERIC_TRANSIT - bad_mode_df = self.trip_list_df.loc[((self.trip_list_df['mode_dash_count']!=2)& - ((self.trip_list_df['mode_dash_count']!=0)| - (self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE]!=Passenger.MODE_GENERIC_TRANSIT)))] + bad_mode_df = self.trip_list_df.loc[((self.trip_list_df['mode_dash_count'] != 2) & + ((self.trip_list_df['mode_dash_count'] != 0) | + (self.trip_list_df[ + Passenger.TRIP_LIST_COLUMN_MODE] != Passenger.MODE_GENERIC_TRANSIT)))] if len(bad_mode_df) > 0: FastTripsLogger.fatal("Could not understand column '%s' in the following: \n%s" % (Passenger.TRIP_LIST_COLUMN_MODE, - bad_mode_df[[Passenger.TRIP_LIST_COLUMN_MODE,'mode_dash_count']].to_string())) + bad_mode_df[[Passenger.TRIP_LIST_COLUMN_MODE, 'mode_dash_count']].to_string())) sys.exit(2) # Take care of the transit generic - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==0, + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 0, Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE] = Passenger.MODE_GENERIC_TRANSIT - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==0, - Passenger.TRIP_LIST_COLUMN_ACCESS_MODE ] = "%s" % TAZ.ACCESS_EGRESS_MODES[0] - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==0, - Passenger.TRIP_LIST_COLUMN_EGRESS_MODE ] = "%s" % TAZ.ACCESS_EGRESS_MODES[0] + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 0, + Passenger.TRIP_LIST_COLUMN_ACCESS_MODE] = "%s" % TAZ.ACCESS_EGRESS_MODES[0] + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 0, + Passenger.TRIP_LIST_COLUMN_EGRESS_MODE] = "%s" % TAZ.ACCESS_EGRESS_MODES[0] # Take care of the access-transit-egress - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==2, - Passenger.TRIP_LIST_COLUMN_ACCESS_MODE] = self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE]\ + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 2, + Passenger.TRIP_LIST_COLUMN_ACCESS_MODE] = self.trip_list_df[ + Passenger.TRIP_LIST_COLUMN_MODE] \ .map(lambda x: "%s" % x[:x.find('-')]) - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==2, - Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE] = self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE]\ - .map(lambda x: x[x.find('-')+1:x.rfind('-')]) - self.trip_list_df.loc[self.trip_list_df['mode_dash_count']==2, - Passenger.TRIP_LIST_COLUMN_EGRESS_MODE] = self.trip_list_df[Passenger.TRIP_LIST_COLUMN_MODE]\ - .map(lambda x: "%s" % x[x.rfind('-')+1:]) + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 2, + Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE] = self.trip_list_df[ + Passenger.TRIP_LIST_COLUMN_MODE] \ + .map(lambda x: x[x.find('-') + 1:x.rfind('-')]) + self.trip_list_df.loc[self.trip_list_df['mode_dash_count'] == 2, + Passenger.TRIP_LIST_COLUMN_EGRESS_MODE] = self.trip_list_df[ + Passenger.TRIP_LIST_COLUMN_MODE] \ + .map(lambda x: "%s" % x[x.rfind('-') + 1:]) # We're done with mode_dash_count, thanks for your service - self.trip_list_df.drop('mode_dash_count', axis=1, inplace=True) # replace with cumsum + self.trip_list_df.drop('mode_dash_count', axis=1, inplace=True) # replace with cumsum # validate time_target - invalid_time_target = self.trip_list_df.loc[ self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET].isin( - [Passenger.TIME_TARGET_ARRIVAL, Passenger.TIME_TARGET_DEPARTURE])==False ] + invalid_time_target = self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET].isin( + [Passenger.TIME_TARGET_ARRIVAL, Passenger.TIME_TARGET_DEPARTURE]) == False] if len(invalid_time_target) > 0: - error_msg = "Invalid value in column %s:\n%s" % (Passenger.TRIP_LIST_COLUMN_TIME_TARGET, str(invalid_time_target)) + error_msg = "Invalid value in column %s:\n%s" % ( + Passenger.TRIP_LIST_COLUMN_TIME_TARGET, str(invalid_time_target)) FastTripsLogger.fatal(error_msg) raise DemandInputError(Passenger.INPUT_TRIP_LIST_FILE, error_msg) # set outbound - self.trip_list_df[Passenger.TRIP_LIST_COLUMN_OUTBOUND] = (self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL) + self.trip_list_df[Passenger.TRIP_LIST_COLUMN_OUTBOUND] = ( + self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL) # Set the user class for each trip from .PathSet import PathSet @@ -373,42 +387,49 @@ def __init__(self, input_dir, output_dir, today, stops, routes, capacity_constra Passenger.TRIP_LIST_COLUMN_PURPOSE, Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE, Passenger.TRIP_LIST_COLUMN_ACCESS_MODE, - Passenger.TRIP_LIST_COLUMN_EGRESS_MODE]].set_index([Passenger.TRIP_LIST_COLUMN_USER_CLASS, Passenger.TRIP_LIST_COLUMN_PURPOSE]) + Passenger.TRIP_LIST_COLUMN_EGRESS_MODE]].set_index( + [Passenger.TRIP_LIST_COLUMN_USER_CLASS, Passenger.TRIP_LIST_COLUMN_PURPOSE]) # stack - so before we have three columns: transit_mode, access_mode, egress_mode # after, we have two columns: demand_mode_type and the value, demand_mode - self.modes_df = self.modes_df.stack().to_frame() - self.modes_df.index.names = [Passenger.TRIP_LIST_COLUMN_USER_CLASS, Passenger.TRIP_LIST_COLUMN_PURPOSE, PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] - self.modes_df.columns = [PathSet.WEIGHTS_COLUMN_DEMAND_MODE] + self.modes_df = self.modes_df.stack().to_frame() + self.modes_df.index.names = [Passenger.TRIP_LIST_COLUMN_USER_CLASS, Passenger.TRIP_LIST_COLUMN_PURPOSE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] + self.modes_df.columns = [PathSet.WEIGHTS_COLUMN_DEMAND_MODE] self.modes_df.reset_index(inplace=True) self.modes_df.drop_duplicates(inplace=True) # fix demand_mode_type since transit_mode is just transit, etc - self.modes_df[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] = self.modes_df[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE].apply(lambda x: x[:-5]) + self.modes_df[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] = self.modes_df[ + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE].apply(lambda x: x[:-5]) FastTripsLogger.debug("Demand mode types by class & purpose: \n%s" % str(self.modes_df)) # Make sure we have all the weights required for these user_class/mode combinations - self.trip_list_df = PathSet.verify_weight_config(self.modes_df, output_dir, routes, capacity_constraint, self.trip_list_df) + self.trip_list_df = PathSet.verify_weight_config(self.modes_df, output_dir, routes, capacity_constraint, + self.trip_list_df) # add column trace from .Assignment import Assignment if len(Assignment.TRACE_IDS) > 0: trace_df = pd.DataFrame.from_records(data=Assignment.TRACE_IDS, - columns=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) + columns=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) trace_df[Passenger.TRIP_LIST_COLUMN_TRACE] = True # combine - self.trip_list_df = pd.merge(left =self.trip_list_df, - right =trace_df, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) + self.trip_list_df = pd.merge(left=self.trip_list_df, + right=trace_df, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) # make nulls into False - self.trip_list_df.loc[pd.isnull(self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE]), Passenger.TRIP_LIST_COLUMN_TRACE] = False + self.trip_list_df.loc[pd.isnull( + self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE]), Passenger.TRIP_LIST_COLUMN_TRACE] = False else: self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE] = False - FastTripsLogger.info("Have %d person trips" % len(self.trip_list_df)) - FastTripsLogger.debug("Final trip_list_df\n"+str(self.trip_list_df.index.dtype)+"\n"+str(self.trip_list_df.dtypes)) - FastTripsLogger.debug("\n"+self.trip_list_df.head().to_string()) + FastTripsLogger.debug( + "Final trip_list_df\n" + str(self.trip_list_df.index.dtype) + "\n" + str(self.trip_list_df.dtypes)) + FastTripsLogger.debug("\n" + self.trip_list_df.head().to_string()) #: Maps trip_list_id to :py:class:`PathSet` instance. Use trip_list_id instead of (person_id, person_trip_id) for simplicity and to iterate sequentially #: in setup_passenger_pathsets() @@ -427,10 +448,10 @@ def get_pathset(self, trip_list_id): return self.id_to_pathset[trip_list_id] def get_person_id(self, trip_list_id): - to_ret = self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]==trip_list_id, - [Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]] - return(to_ret.iloc[0,0], to_ret.iloc[0,1]) + to_ret = self.trip_list_df.loc[self.trip_list_df[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM] == trip_list_id, + [Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]] + return (to_ret.iloc[0, 0], to_ret.iloc[0, 1]) def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=True): """ @@ -448,9 +469,9 @@ def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=Tru # read existing paths paths_file = os.path.join(pathset_dir, Passenger.PATHSET_PATHS_CSV if include_asgn else Passenger.PF_PATHS_CSV) pathset_paths_df = pd.read_csv(paths_file, - skipinitialspace=True, - dtype={Passenger.TRIP_LIST_COLUMN_PERSON_ID :'S', - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID:'S'}) + skipinitialspace=True, + dtype={Passenger.TRIP_LIST_COLUMN_PERSON_ID: 'S', + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID: 'S'}) FastTripsLogger.info("Read %s" % paths_file) FastTripsLogger.debug("pathset_paths_df.dtypes=\n%s" % str(pathset_paths_df.dtypes)) @@ -462,13 +483,13 @@ def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=Tru Assignment.SIM_COL_PAX_ALIGHT_TIME, Assignment.SIM_COL_PAX_A_TIME, Assignment.SIM_COL_PAX_B_TIME]) - links_dtypes = {Passenger.TRIP_LIST_COLUMN_PERSON_ID :'S', - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID:'S', - Trip.TRIPS_COLUMN_TRIP_ID :'S', - "A_id" :'S', - "B_id" :'S', - Passenger.PF_COL_ROUTE_ID :'S', - Passenger.PF_COL_TRIP_ID :'S'} + links_dtypes = {Passenger.TRIP_LIST_COLUMN_PERSON_ID: 'S', + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID: 'S', + Trip.TRIPS_COLUMN_TRIP_ID: 'S', + "A_id": 'S', + "B_id": 'S', + Passenger.PF_COL_ROUTE_ID: 'S', + Passenger.PF_COL_TRIP_ID: 'S'} # read datetimes as string initially for date_col in date_cols: links_dtypes[date_col] = 'S' @@ -484,25 +505,31 @@ def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=Tru # convert time duration columns to time durations link_cols = list(pathset_links_df.columns.values) if Passenger.PF_COL_LINK_TIME in link_cols: - pathset_links_df[Passenger.PF_COL_LINK_TIME] = pd.to_timedelta(pathset_links_df[Passenger.PF_COL_LINK_TIME]) + pathset_links_df[Passenger.PF_COL_LINK_TIME] = pd.to_timedelta(pathset_links_df[Passenger.PF_COL_LINK_TIME]) elif "%s min" % Passenger.PF_COL_LINK_TIME in link_cols: - pathset_links_df[Passenger.PF_COL_LINK_TIME] = pd.to_timedelta(pathset_links_df["%s min" % Passenger.PF_COL_LINK_TIME], unit='m') + pathset_links_df[Passenger.PF_COL_LINK_TIME] = pd.to_timedelta( + pathset_links_df["%s min" % Passenger.PF_COL_LINK_TIME], unit='m') if Passenger.PF_COL_WAIT_TIME in link_cols: - pathset_links_df[Passenger.PF_COL_WAIT_TIME] = pd.to_timedelta(pathset_links_df[Passenger.PF_COL_WAIT_TIME]) + pathset_links_df[Passenger.PF_COL_WAIT_TIME] = pd.to_timedelta(pathset_links_df[Passenger.PF_COL_WAIT_TIME]) elif "%s min" % Passenger.PF_COL_WAIT_TIME in link_cols: - pathset_links_df[Passenger.PF_COL_WAIT_TIME] = pd.to_timedelta(pathset_links_df["%s min" % Passenger.PF_COL_WAIT_TIME], unit='m') + pathset_links_df[Passenger.PF_COL_WAIT_TIME] = pd.to_timedelta( + pathset_links_df["%s min" % Passenger.PF_COL_WAIT_TIME], unit='m') # if simulation results are available if Assignment.SIM_COL_PAX_LINK_TIME in link_cols: - pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pd.to_timedelta(pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]) + pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pd.to_timedelta( + pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME]) elif "%s min" % Assignment.SIM_COL_PAX_WAIT_TIME in link_cols: - pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pd.to_timedelta(pathset_links_df["%s min" % Assignment.SIM_COL_PAX_LINK_TIME], unit='m') + pathset_links_df[Assignment.SIM_COL_PAX_LINK_TIME] = pd.to_timedelta( + pathset_links_df["%s min" % Assignment.SIM_COL_PAX_LINK_TIME], unit='m') if Assignment.SIM_COL_PAX_WAIT_TIME in link_cols: - pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] = pd.to_timedelta(pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME]) + pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] = pd.to_timedelta( + pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME]) elif "%s min" % Assignment.SIM_COL_PAX_WAIT_TIME in link_cols: - pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] = pd.to_timedelta(pathset_links_df["%s min" % Assignment.SIM_COL_PAX_WAIT_TIME], unit='m') + pathset_links_df[Assignment.SIM_COL_PAX_WAIT_TIME] = pd.to_timedelta( + pathset_links_df["%s min" % Assignment.SIM_COL_PAX_WAIT_TIME], unit='m') # and drop the numeric version if "%s min" % Passenger.PF_COL_LINK_TIME in link_cols: @@ -514,29 +541,35 @@ def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=Tru # if A_id_num isn't there, add it if "A_id_num" not in pathset_links_df.columns.values: - pathset_links_df = stops.add_numeric_stop_id(pathset_links_df, id_colname="A_id", numeric_newcolname="A_id_num", - warn=True, warn_msg="read_passenger_pathsets: invalid stop ID", drop_failures=False) + pathset_links_df = stops.add_numeric_stop_id(pathset_links_df, id_colname="A_id", + numeric_newcolname="A_id_num", + warn=True, warn_msg="read_passenger_pathsets: invalid stop ID", + drop_failures=False) if "B_id_num" not in pathset_links_df.columns.values: - pathset_links_df = stops.add_numeric_stop_id(pathset_links_df, id_colname="B_id", numeric_newcolname="B_id_num", - warn=True, warn_msg="read_passenger_pathsets: invalid stop ID", drop_failures=False) + pathset_links_df = stops.add_numeric_stop_id(pathset_links_df, id_colname="B_id", + numeric_newcolname="B_id_num", + warn=True, warn_msg="read_passenger_pathsets: invalid stop ID", + drop_failures=False) # if trip_list_id_num is in trip list and not in these, add it if Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM in self.trip_list_df.columns.values: if Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM not in pathset_paths_df.columns.values: - pathset_paths_df = pd.merge(left =pathset_paths_df, - right =self.trip_list_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], - how ="left") + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=self.trip_list_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], + how="left") if Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM not in pathset_links_df.columns.values: - pathset_links_df = pd.merge(left =pathset_links_df, - right =self.trip_list_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], - how ="left") + pathset_links_df = pd.merge(left=pathset_links_df, + right=self.trip_list_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]], + how="left") # add mode_num if it's not there if Route.ROUTES_COLUMN_MODE_NUM not in pathset_links_df.columns.values: - pathset_links_df = pd.merge(left=pathset_links_df, right=modes_df[[Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE]], how="left") + pathset_links_df = pd.merge(left=pathset_links_df, + right=modes_df[[Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE]], + how="left") FastTripsLogger.info("Read %s" % links_file) FastTripsLogger.debug("pathset_links_df head=\n%s" % str(pathset_links_df.head())) @@ -544,7 +577,6 @@ def read_passenger_pathsets(self, pathset_dir, stops, modes_df, include_asgn=Tru return (pathset_paths_df, pathset_links_df) - def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip_id_df, trips_df, modes_df, transfers, tazs, prepend_route_id_to_trip_id): """ @@ -621,7 +653,7 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip trip_list_id_nums = self.pathfind_trip_list_df[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM].tolist() - for trip_list_id,pathset in self.id_to_pathset.items(): + for trip_list_id, pathset in self.id_to_pathset.items(): # only process if we just did pathfinding for this person trip if trip_list_id not in trip_list_id_nums: continue @@ -664,14 +696,14 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip state_list = pathset.pathdict[pathnum][PathSet.PATH_KEY_STATES] if not pathset.outbound: state_list = list(reversed(state_list)) - pathlist.append([\ + pathlist.append([ \ pathset.person_id, pathset.person_trip_id, trip_list_id, - (pathset.person_id,pathset.person_trip_id) in Assignment.TRACE_IDS, + (pathset.person_id, pathset.person_trip_id) in Assignment.TRACE_IDS, pathset.direction, pathset.mode, - 0.01*pathfinding_iteration+iteration, + 0.01 * pathfinding_iteration + iteration, pathnum, pathset.pathdict[pathnum][PathSet.PATH_KEY_COST], pathset.pathdict[pathnum][PathSet.PATH_KEY_FARE], @@ -680,53 +712,55 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip pathset.pathdict[pathnum][PathSet.PATH_KEY_INIT_FARE] ]) - link_num = 0 + link_num = 0 for (state_id, state) in state_list: - linkmode = state[PathSet.STATE_IDX_DEPARRMODE] - mode_num = None - trip_id = None - waittime = None + linkmode = state[PathSet.STATE_IDX_DEPARRMODE] + mode_num = None + trip_id = None + waittime = None if linkmode in [PathSet.STATE_MODE_ACCESS, PathSet.STATE_MODE_TRANSFER, PathSet.STATE_MODE_EGRESS]: - mode_num = state[PathSet.STATE_IDX_TRIP] + mode_num = state[PathSet.STATE_IDX_TRIP] else: # trip mode_num will need to be joined - trip_id = state[PathSet.STATE_IDX_TRIP] - linkmode = PathSet.STATE_MODE_TRIP + trip_id = state[PathSet.STATE_IDX_TRIP] + linkmode = PathSet.STATE_MODE_TRIP if pathset.outbound: - a_id_num = state_id - b_id_num = state[PathSet.STATE_IDX_SUCCPRED] - a_seq = state[PathSet.STATE_IDX_SEQ] - b_seq = state[PathSet.STATE_IDX_SEQ_SUCCPRED] - b_time = state[PathSet.STATE_IDX_ARRDEP] - a_time = b_time - state[PathSet.STATE_IDX_LINKTIME] - trip_time = state[PathSet.STATE_IDX_ARRDEP] - state[PathSet.STATE_IDX_DEPARR] + a_id_num = state_id + b_id_num = state[PathSet.STATE_IDX_SUCCPRED] + a_seq = state[PathSet.STATE_IDX_SEQ] + b_seq = state[PathSet.STATE_IDX_SEQ_SUCCPRED] + b_time = state[PathSet.STATE_IDX_ARRDEP] + a_time = b_time - state[PathSet.STATE_IDX_LINKTIME] + trip_time = state[PathSet.STATE_IDX_ARRDEP] - state[PathSet.STATE_IDX_DEPARR] else: - a_id_num = state[PathSet.STATE_IDX_SUCCPRED] - b_id_num = state_id - a_seq = state[PathSet.STATE_IDX_SEQ_SUCCPRED] - b_seq = state[PathSet.STATE_IDX_SEQ] - b_time = state[PathSet.STATE_IDX_DEPARR] - a_time = b_time - state[PathSet.STATE_IDX_LINKTIME] - trip_time = state[PathSet.STATE_IDX_DEPARR] - state[PathSet.STATE_IDX_ARRDEP] + a_id_num = state[PathSet.STATE_IDX_SUCCPRED] + b_id_num = state_id + a_seq = state[PathSet.STATE_IDX_SEQ_SUCCPRED] + b_seq = state[PathSet.STATE_IDX_SEQ] + b_time = state[PathSet.STATE_IDX_DEPARR] + a_time = b_time - state[PathSet.STATE_IDX_LINKTIME] + trip_time = state[PathSet.STATE_IDX_DEPARR] - state[PathSet.STATE_IDX_ARRDEP] # trips: linktime includes wait if linkmode == PathSet.STATE_MODE_TRIP: - waittime = state[PathSet.STATE_IDX_LINKTIME] - trip_time + waittime = state[PathSet.STATE_IDX_LINKTIME] - trip_time # two trips in a row -- this shouldn't happen if linkmode == PathSet.STATE_MODE_TRIP and prev_linkmode == PathSet.STATE_MODE_TRIP: - FastTripsLogger.warn("Two trip links in a row... this shouldn't happen. person_id is %s trip is %s\npathnum is %d\nstatelist (%d): %s\n" % (person_id, person_trip_id, pathnum, len(state_list), str(state_list))) + FastTripsLogger.warn( + "Two trip links in a row... this shouldn't happen. person_id is %s trip is %s\npathnum is %d\nstatelist (%d): %s\n" % ( + person_id, person_trip_id, pathnum, len(state_list), str(state_list))) sys.exit() - linklist.append([\ + linklist.append([ \ pathset.person_id, pathset.person_trip_id, trip_list_id, - (pathset.person_id,pathset.person_trip_id) in Assignment.TRACE_IDS, - 0.01*pathfinding_iteration + iteration, + (pathset.person_id, pathset.person_trip_id) in Assignment.TRACE_IDS, + 0.01 * pathfinding_iteration + iteration, pathnum, linkmode, mode_num, @@ -742,21 +776,21 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip state[PathSet.STATE_IDX_LINKCOST], state[PathSet.STATE_IDX_LINKDIST], waittime, - link_num ]) + link_num]) prev_linkmode = linkmode prev_state_id = state_id - link_num += 1 + link_num += 1 FastTripsLogger.debug("setup_passenger_pathsets(): pathlist and linklist constructed") - pathset_paths_df = pd.DataFrame(pathlist, columns=[\ + pathset_paths_df = pd.DataFrame(pathlist, columns=[ \ Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Passenger.TRIP_LIST_COLUMN_TRACE, 'pathdir', # for debugging - 'pathmode', # for output + 'pathmode', # for output Passenger.PF_COL_PF_ITERATION, Passenger.PF_COL_PATH_NUM, PathSet.PATH_KEY_COST, @@ -765,7 +799,7 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip PathSet.PATH_KEY_INIT_COST, PathSet.PATH_KEY_INIT_FARE]) - pathset_links_df = pd.DataFrame(linklist, columns=[\ + pathset_links_df = pd.DataFrame(linklist, columns=[ \ Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, @@ -775,8 +809,8 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip Passenger.PF_COL_LINK_MODE, Route.ROUTES_COLUMN_MODE_NUM, Trip.TRIPS_COLUMN_TRIP_ID_NUM, - 'A_id_num','B_id_num', - 'A_seq','B_seq', + 'A_id_num', 'B_id_num', + 'A_seq', 'B_seq', Passenger.PF_COL_PAX_A_TIME, Passenger.PF_COL_PAX_B_TIME, Passenger.PF_COL_LINK_TIME, @@ -784,56 +818,80 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip Passenger.PF_COL_LINK_COST, Passenger.PF_COL_LINK_DIST, Passenger.PF_COL_WAIT_TIME, - Passenger.PF_COL_LINK_NUM ]) + Passenger.PF_COL_LINK_NUM]) - FastTripsLogger.debug("setup_passenger_pathsets(): pathset_paths_df(%d) and pathset_links_df(%d) dataframes constructed" % (len(pathset_paths_df), len(pathset_links_df))) + FastTripsLogger.debug( + "setup_passenger_pathsets(): pathset_paths_df(%d) and pathset_links_df(%d) dataframes constructed" % ( + len(pathset_paths_df), len(pathset_links_df))) # get A_id and B_id and trip_id - pathset_links_df = stops.add_stop_id_for_numeric_id(pathset_links_df,'A_id_num','A_id') - pathset_links_df = stops.add_stop_id_for_numeric_id(pathset_links_df,'B_id_num','B_id') + pathset_links_df = stops.add_stop_id_for_numeric_id(pathset_links_df, 'A_id_num', 'A_id') + pathset_links_df = stops.add_stop_id_for_numeric_id(pathset_links_df, 'B_id_num', 'B_id') # get A_lat, A_lon, B_lat, B_lon - pathset_links_df = stops.add_stop_lat_lon(pathset_links_df, id_colname="A_id", new_lat_colname="A_lat", new_lon_colname="A_lon") - pathset_links_df = stops.add_stop_lat_lon(pathset_links_df, id_colname="B_id", new_lat_colname="B_lat", new_lon_colname="B_lon") + pathset_links_df = stops.add_stop_lat_lon(pathset_links_df, id_colname="A_id", new_lat_colname="A_lat", + new_lon_colname="A_lon") + pathset_links_df = stops.add_stop_lat_lon(pathset_links_df, id_colname="B_id", new_lat_colname="B_lat", + new_lon_colname="B_lon") # get trip_id - pathset_links_df = Util.add_new_id( input_df=pathset_links_df, id_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM, newid_colname=Trip.TRIPS_COLUMN_TRIP_ID, - mapping_df=trip_id_df, mapping_id_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM, mapping_newid_colname=Trip.TRIPS_COLUMN_TRIP_ID) + pathset_links_df = Util.add_new_id(input_df=pathset_links_df, id_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM, + newid_colname=Trip.TRIPS_COLUMN_TRIP_ID, + mapping_df=trip_id_df, mapping_id_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM, + mapping_newid_colname=Trip.TRIPS_COLUMN_TRIP_ID) # get route id # mode_num will appear in left (for non-transit links) and right (for transit link) both, so we need to consolidate - pathset_links_df = pd.merge(left=pathset_links_df, right=trips_df[[Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_ROUTE_ID, Route.ROUTES_COLUMN_MODE_NUM]], - how="left", on=Trip.TRIPS_COLUMN_TRIP_ID) + pathset_links_df = pd.merge(left=pathset_links_df, right=trips_df[ + [Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_ROUTE_ID, Route.ROUTES_COLUMN_MODE_NUM]], + how="left", on=Trip.TRIPS_COLUMN_TRIP_ID) pathset_links_df[Route.ROUTES_COLUMN_MODE_NUM] = pathset_links_df["%s_x" % Route.ROUTES_COLUMN_MODE_NUM] - pathset_links_df.loc[pd.notnull(pathset_links_df["%s_y" % Route.ROUTES_COLUMN_MODE_NUM]), Route.ROUTES_COLUMN_MODE_NUM] = pathset_links_df["%s_y" % Route.ROUTES_COLUMN_MODE_NUM] + pathset_links_df.loc[ + pd.notnull(pathset_links_df["%s_y" % Route.ROUTES_COLUMN_MODE_NUM]), Route.ROUTES_COLUMN_MODE_NUM] = \ + pathset_links_df["%s_y" % Route.ROUTES_COLUMN_MODE_NUM] pathset_links_df.drop(["%s_x" % Route.ROUTES_COLUMN_MODE_NUM, "%s_y" % Route.ROUTES_COLUMN_MODE_NUM], axis=1, inplace=True) # verify it's always set - FastTripsLogger.debug("Have %d links with no mode number set" % len(pathset_links_df.loc[ pd.isnull(pathset_links_df[Route.ROUTES_COLUMN_MODE_NUM]) ])) + FastTripsLogger.debug("Have %d links with no mode number set" % len( + pathset_links_df.loc[pd.isnull(pathset_links_df[Route.ROUTES_COLUMN_MODE_NUM])])) # get supply mode - pathset_links_df = pd.merge(left=pathset_links_df, right=modes_df[[Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE]], how="left") + pathset_links_df = pd.merge(left=pathset_links_df, + right=modes_df[[Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE]], + how="left") - FastTripsLogger.debug("setup_passenger_pathsets(): pathset_paths_df and pathset_links_df dataframes constructed") + FastTripsLogger.debug( + "setup_passenger_pathsets(): pathset_paths_df and pathset_links_df dataframes constructed") # FastTripsLogger.debug("\n%s" % pathset_links_df.head().to_string()) if len(pathset_paths_df) > 0: # create path description - pathset_links_df[Passenger.PF_COL_DESCRIPTION] = pathset_links_df["A_id"] + " " + pathset_links_df[Route.ROUTES_COLUMN_MODE] + pathset_links_df[Passenger.PF_COL_DESCRIPTION] = pathset_links_df["A_id"] + " " + pathset_links_df[ + Route.ROUTES_COLUMN_MODE] if prepend_route_id_to_trip_id: - pathset_links_df.loc[ pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION ] = pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " + pathset_links_df[Trip.TRIPS_COLUMN_ROUTE_ID] + "_" + pathset_links_df.loc[ + pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION] = \ + pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " + pathset_links_df[ + Trip.TRIPS_COLUMN_ROUTE_ID] + "_" else: - pathset_links_df.loc[ pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION ] = pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " - pathset_links_df.loc[ pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION ] = pathset_links_df[Passenger.PF_COL_DESCRIPTION] + pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID] - pathset_links_df.loc[ pathset_links_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_EGRESS, Passenger.PF_COL_DESCRIPTION ] = pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " + pathset_links_df["B_id"] + pathset_links_df.loc[ + pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION] = \ + pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " + pathset_links_df.loc[ + pd.notnull(pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID]), Passenger.PF_COL_DESCRIPTION] = \ + pathset_links_df[Passenger.PF_COL_DESCRIPTION] + pathset_links_df[Trip.TRIPS_COLUMN_TRIP_ID] + pathset_links_df.loc[pathset_links_df[ + Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS, Passenger.PF_COL_DESCRIPTION] = \ + pathset_links_df[Passenger.PF_COL_DESCRIPTION] + " " + pathset_links_df["B_id"] descr_df = pathset_links_df[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Passenger.PF_COL_PF_ITERATION, Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_DESCRIPTION]].groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.PF_COL_PF_ITERATION, - Passenger.PF_COL_PATH_NUM])[Passenger.PF_COL_DESCRIPTION].apply(lambda x:" ".join(x)) + Passenger.PF_COL_DESCRIPTION]].groupby( + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.PF_COL_PF_ITERATION, + Passenger.PF_COL_PATH_NUM])[Passenger.PF_COL_DESCRIPTION].apply(lambda x: " ".join(x)) descr_df = descr_df.to_frame().reset_index() # join it to pathset_paths and drop from pathset_links pathset_paths_df = pd.merge(left=pathset_paths_df, right=descr_df, how="left") @@ -856,28 +914,30 @@ def setup_passenger_pathsets(self, iteration, pathfinding_iteration, stops, trip return (pathset_paths_df, pathset_links_df) @staticmethod - def write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_df, links, output_pathset_per_sim_iter, drop_debug_columns, drop_pathfinding_columns): + def write_paths(output_dir, iteration, pathfinding_iteration, simulation_iteration, pathset_df, links, + output_pathset_per_sim_iter, drop_debug_columns, drop_pathfinding_columns): """ Write either pathset paths (if links=False) or pathset links (if links=True) as the case may be """ # if simulation_iteration < 0, then this is the pathfinding result if simulation_iteration < 0: - pathset_df[ "iteration"] = iteration + pathset_df["iteration"] = iteration pathset_df["pathfinding_iteration"] = pathfinding_iteration Util.write_dataframe(df=pathset_df, name="pathset_links_df" if links else "pathset_paths_df", - output_file=os.path.join(output_dir, Passenger.PF_LINKS_CSV if links else Passenger.PF_PATHS_CSV), + output_file=os.path.join(output_dir, + Passenger.PF_LINKS_CSV if links else Passenger.PF_PATHS_CSV), append=True if ((iteration > 1) or (pathfinding_iteration > 1)) else False, keep_duration_columns=True, drop_debug_columns=drop_debug_columns, drop_pathfinding_columns=drop_pathfinding_columns) - pathset_df.drop(["iteration","pathfinding_iteration"], axis=1, inplace=True) + pathset_df.drop(["iteration", "pathfinding_iteration"], axis=1, inplace=True) return # otherwise, add columns and write it - pathset_df[ "iteration"] = iteration + pathset_df["iteration"] = iteration pathset_df["pathfinding_iteration"] = pathfinding_iteration - pathset_df[ "simulation_iteration"] = simulation_iteration + pathset_df["simulation_iteration"] = simulation_iteration # mostly we append do_append = True @@ -889,15 +949,16 @@ def write_paths(output_dir, iteration, pathfinding_iteration, simulation_iterati Util.write_dataframe(df=pathset_df, name="pathset_links_df" if links else "pathset_paths_df", - output_file=os.path.join(output_dir, Passenger.PATHSET_LINKS_CSV if links else Passenger.PATHSET_PATHS_CSV), + output_file=os.path.join(output_dir, + Passenger.PATHSET_LINKS_CSV if links else Passenger.PATHSET_PATHS_CSV), append=do_append, drop_debug_columns=drop_debug_columns, drop_pathfinding_columns=drop_pathfinding_columns) - pathset_df.drop(["iteration","pathfinding_iteration","simulation_iteration"], axis=1, inplace=True) - + pathset_df.drop(["iteration", "pathfinding_iteration", "simulation_iteration"], axis=1, inplace=True) @staticmethod - def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, pathset_links_df): + def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulation_iteration, pathset_paths_df, + pathset_links_df): """ Returns the same dataframes as input, but with a new/updated column, :py:attr:`Assignment.SIM_COL_PAX_CHOSEN`. This column is set to: @@ -912,30 +973,35 @@ def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulati Returns (TOTAL num passenger trips chosen, NEW num passenger trips chosen, updated pathset_paths_df, updated pathset_links_df) """ from .Assignment import Assignment - from .PathSet import PathSet - + from .PathSet import PathSet # If choose_for_everyone, we need to do all of them. if choose_for_everyone: - pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical([Assignment.CHOSEN_NOT_CHOSEN_YET]*len(pathset_paths_df), categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + [Assignment.CHOSEN_NOT_CHOSEN_YET] * len(pathset_paths_df), categories=Assignment.CHOSEN_CATEGORIES, + ordered=True) else: # set chosen to ordered categories if needed if pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN].dtype.name != "category": - pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN].astype('category') - pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, ordered=True, inplace=True) + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_paths_df[ + Assignment.SIM_COL_PAX_CHOSEN].astype('category') + pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, + ordered=True, inplace=True) # Otherwise, just choose for those that still need it - rejected_paths = pathset_paths_df.loc[ (pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET)& - (pathset_paths_df[Assignment.SIM_COL_PAX_COST ] >= PathSet.HUGE_COST) ] + rejected_paths = pathset_paths_df.loc[ + (pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET) & + (pathset_paths_df[Assignment.SIM_COL_PAX_COST] >= PathSet.HUGE_COST)] FastTripsLogger.info(" Rejecting %d previously chosen paths for huge costs" % len(rejected_paths)) FastTripsLogger.debug("rejected_paths head(20): \n%s" % rejected_paths.head(20)) # why doesn't this translate to pathset_links_df ? if len(rejected_paths) > 0: # first invalidate any high cost choices - pathset_paths_df.loc[ (pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET)& - (pathset_paths_df[Assignment.SIM_COL_PAX_COST ] >= PathSet.HUGE_COST), - Assignment.SIM_COL_PAX_CHOSEN ] = Assignment.CHOSEN_REJECTED + pathset_paths_df.loc[ + (pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET) & + (pathset_paths_df[Assignment.SIM_COL_PAX_COST] >= PathSet.HUGE_COST), + Assignment.SIM_COL_PAX_CHOSEN] = Assignment.CHOSEN_REJECTED # # do the same to links # rejected_paths.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, Passenger.PF_COL_PATH_NUM]) @@ -948,20 +1014,28 @@ def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulati # group to passenger trips pathset_paths_df_grouped = pathset_paths_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Assignment.SIM_COL_PAX_CHOSEN]].groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).aggregate("max").reset_index() - pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical(pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, ordered=True) + Assignment.SIM_COL_PAX_CHOSEN]].groupby( + [Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).aggregate("max").reset_index() + pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN] = pd.Categorical( + pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN], categories=Assignment.CHOSEN_CATEGORIES, + ordered=True) # if there's no chosen AND one of the unchosen options is choosable then we can choose - num_rejected = len(pathset_paths_df_grouped.loc[ pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN]==Assignment.CHOSEN_REJECTED ]) # everything is rejected - num_unchosen = len(pathset_paths_df_grouped.loc[ pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN]==Assignment.CHOSEN_NOT_CHOSEN_YET ]) - num_chosen = len(pathset_paths_df_grouped) - num_rejected - num_unchosen + num_rejected = len(pathset_paths_df_grouped.loc[pathset_paths_df_grouped[ + Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_REJECTED]) # everything is rejected + num_unchosen = len(pathset_paths_df_grouped.loc[pathset_paths_df_grouped[ + Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET]) + num_chosen = len(pathset_paths_df_grouped) - num_rejected - num_unchosen # count how many passenger trips have pathsets with valid paths (logsum > 0) AND no path chosen (chosen < 0) - pax_choose_df = pathset_paths_df_grouped.loc[ pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN]==Assignment.CHOSEN_NOT_CHOSEN_YET ].copy() - num_unchosen = len(pax_choose_df) + pax_choose_df = pathset_paths_df_grouped.loc[ + pathset_paths_df_grouped[Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET].copy() + num_unchosen = len(pax_choose_df) - FastTripsLogger.info(" Have %6d total passenger-trips, with %6d chosen paths, %6d fully rejected and %6d needing a choice" % (len(pathset_paths_df_grouped), num_chosen, num_rejected, num_unchosen)) + FastTripsLogger.info( + " Have %6d total passenger-trips, with %6d chosen paths, %6d fully rejected and %6d needing a choice" % ( + len(pathset_paths_df_grouped), num_chosen, num_rejected, num_unchosen)) # If we have nothing to do, return if len(pax_choose_df) == 0: @@ -972,30 +1046,32 @@ def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulati # Choose a random number for them now # todo: do this differently? - np.random.seed(iteration*1000 + simulation_iteration) + np.random.seed(iteration * 1000 + simulation_iteration) pax_choose_df["rand"] = np.random.rand(len(pax_choose_df)) # FastTripsLogger.debug("\n%s" % pax_choose_df.head().to_string()) # add to_choose flag and rand to pathset_paths_df - pathset_paths_df = pd.merge(left =pathset_paths_df, - right=pax_choose_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - "to_choose", "rand"]], - how ="left") + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=pax_choose_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + "to_choose", "rand"]], + how="left") # select out just those pathsets we're choosing, and eligible - paths_choose_df = pathset_paths_df.loc[ (pathset_paths_df["to_choose"]==1) & - (pathset_paths_df[Assignment.SIM_COL_PAX_COST] < PathSet.HUGE_COST) & - (pathset_paths_df[Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET) ].copy() + paths_choose_df = pathset_paths_df.loc[(pathset_paths_df["to_choose"] == 1) & + (pathset_paths_df[Assignment.SIM_COL_PAX_COST] < PathSet.HUGE_COST) & + (pathset_paths_df[ + Assignment.SIM_COL_PAX_CHOSEN] == Assignment.CHOSEN_NOT_CHOSEN_YET)].copy() if len(paths_choose_df) == 0: FastTripsLogger.info(" No choosable paths") - pathset_paths_df.drop(["to_choose","rand"], axis=1, inplace=True) + pathset_paths_df.drop(["to_choose", "rand"], axis=1, inplace=True) return (num_chosen, 0, pathset_paths_df, pathset_links_df) # Use updated probability -- create cumulative probability paths_choose_df["prob_cum"] = paths_choose_df.groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID])[Assignment.SIM_COL_PAX_PROBABILITY].cumsum() + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID])[ + Assignment.SIM_COL_PAX_PROBABILITY].cumsum() # verify cumsum is ok # FastTripsLogger.debug("choose_path() paths_choose_df=\n%s\n" % paths_choose_df.head(100).to_string()) @@ -1011,41 +1087,45 @@ def choose_paths(choose_for_everyone, iteration, pathfinding_iteration, simulati chosen_path_df = paths_choose_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, "rand_less"]].groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).idxmax(axis=0).reset_index() - chosen_path_df.rename(columns={"rand_less":"chosen_idx"}, inplace=True) + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).idxmax( + axis=0).reset_index() + chosen_path_df.rename(columns={"rand_less": "chosen_idx"}, inplace=True) # FastTripsLogger.debug("choose_path() chosen_path_df=\n%s\n" % chosen_path_df.head(30).to_string()) num_chosen += len(chosen_path_df) # mark it as chosen pathset_paths_df = pd.merge(left=pathset_paths_df, right=chosen_path_df, how="left") - pathset_paths_df.loc[pathset_paths_df["chosen_idx"]==pathset_paths_df.index, Assignment.SIM_COL_PAX_CHOSEN] = CHOSEN_VALUE + pathset_paths_df.loc[ + pathset_paths_df["chosen_idx"] == pathset_paths_df.index, Assignment.SIM_COL_PAX_CHOSEN] = CHOSEN_VALUE if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("choose_path() pathset_paths_df=\n%s\n" % pathset_paths_df.loc[ pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string()) + FastTripsLogger.debug("choose_path() pathset_paths_df=\n%s\n" % pathset_paths_df.loc[ + pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string()) FastTripsLogger.info(" Chose %d out of %d paths from the pathsets => total chosen %d" % (len(chosen_path_df), len(pathset_paths_df_grouped), num_chosen)) # drop the intermediates - pathset_paths_df.drop(["to_choose","rand","chosen_idx"], axis=1, inplace=True) + pathset_paths_df.drop(["to_choose", "rand", "chosen_idx"], axis=1, inplace=True) # give the chosen index to pathset_links_df if Assignment.SIM_COL_PAX_CHOSEN in list(pathset_links_df.columns.values): pathset_links_df.drop(Assignment.SIM_COL_PAX_CHOSEN, axis=1, inplace=True) pathset_links_df = pd.merge(left=pathset_links_df, - right=pathset_paths_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Assignment.SIM_COL_PAX_CHOSEN]], - how="left") + right=pathset_paths_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Assignment.SIM_COL_PAX_CHOSEN]], + how="left") - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].astype('category') - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, ordered=True, inplace=True) + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].astype( + 'category') + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, ordered=True, + inplace=True) return (num_chosen, len(chosen_path_df), pathset_paths_df, pathset_links_df) - @staticmethod def get_chosen_links(pathset_links_df, transit_only=False, copy=True): """ @@ -1056,10 +1136,12 @@ def get_chosen_links(pathset_links_df, transit_only=False, copy=True): # set to ordered categories if needed if pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].dtype.name != "category": - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].astype('category') - pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, ordered=True, inplace=True) + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].astype( + 'category') + pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN].cat.set_categories(Assignment.CHOSEN_CATEGORIES, + ordered=True, inplace=True) - slicer = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN]>Assignment.CHOSEN_NOT_CHOSEN_YET + slicer = pathset_links_df[Assignment.SIM_COL_PAX_CHOSEN] > Assignment.CHOSEN_NOT_CHOSEN_YET if transit_only: slicer = slicer & pathset_links_df[Passenger.PF_COL_ROUTE_ID].notnull() diff --git a/fasttrips/PathSet.py b/fasttrips/PathSet.py index 93376fca..b20a52df 100644 --- a/fasttrips/PathSet.py +++ b/fasttrips/PathSet.py @@ -4,7 +4,7 @@ from builtins import object __copyright__ = "Copyright 2015-2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import datetime import os import sys @@ -24,86 +25,87 @@ import numpy as np import pandas as pd -from .Error import NotImplementedError, UnexpectedError -from .Logger import FastTripsLogger +from .Error import NotImplementedError, UnexpectedError +from .Logger import FastTripsLogger from .Passenger import Passenger -from .Route import Route -from .TAZ import TAZ -from .Trip import Trip -from .Util import Util +from .Route import Route +from .TAZ import TAZ +from .Trip import Trip +from .Util import Util #: Default user class: just one class called "all" def generic_user_class(row_series): return "all" + class PathSet(object): """ Represents a path set for a passenger from an origin :py:class:`TAZ` to a destination :py:class:`TAZ` through a set of stops. """ #: Paths output file - PATHS_OUTPUT_FILE = 'ft_output_passengerPaths.txt' + PATHS_OUTPUT_FILE = 'ft_output_passengerPaths.txt' #: Path times output file - PATH_TIMES_OUTPUT_FILE = 'ft_output_passengerTimes.txt' + PATH_TIMES_OUTPUT_FILE = 'ft_output_passengerTimes.txt' #: Configured functions, indexed by name - CONFIGURED_FUNCTIONS = { 'generic_user_class':generic_user_class } + CONFIGURED_FUNCTIONS = {'generic_user_class': generic_user_class} #: Path configuration: Name of the function that defines user class - USER_CLASS_FUNCTION = None + USER_CLASS_FUNCTION = None #: File with weights file. Space delimited table. - WEIGHTS_FILE = 'pathweight_ft.txt' + WEIGHTS_FILE = 'pathweight_ft.txt' #: Path weights - WEIGHTS_DF = None + WEIGHTS_DF = None #: Read weights file as fixed-width format. If false, standard CSV format is read. - WEIGHTS_FIXED_WIDTH = False + WEIGHTS_FIXED_WIDTH = False #: Configuration: Minimum transfer penalty. Safeguard against having no transfer penalty #: which can result in terrible paths with excessive transfers. - MIN_TRANSFER_PENALTY = None + MIN_TRANSFER_PENALTY = None #: Configuration: Overlap scale parameter. - OVERLAP_SCALE_PARAMETER = None + OVERLAP_SCALE_PARAMETER = None #: Configuration: Overlap variable. Can be "None", "count", "distance", "time". - OVERLAP_VARIABLE = None + OVERLAP_VARIABLE = None #: Overlap variable option: None. Don't use overlap pathsize correction. - OVERLAP_NONE = "None" + OVERLAP_NONE = "None" #: Overlap variable option: count. Use leg count overlap pathsize correction. - OVERLAP_COUNT = "count" + OVERLAP_COUNT = "count" #: Overlap variable option: distance. Use leg distance overlap pathsize correction. - OVERLAP_DISTANCE = "distance" + OVERLAP_DISTANCE = "distance" #: Overlap variable option: time. Use leg time overlap pathsize correction. - OVERLAP_TIME = "time" + OVERLAP_TIME = "time" #: Valid values for OVERLAP_VARAIBLE - OVERLAP_VARIABLE_OPTIONS = [OVERLAP_NONE, - OVERLAP_COUNT, - OVERLAP_DISTANCE, - OVERLAP_TIME] + OVERLAP_VARIABLE_OPTIONS = [OVERLAP_NONE, + OVERLAP_COUNT, + OVERLAP_DISTANCE, + OVERLAP_TIME] #: Overlap chunk size. How many person's trips to process at a time in overlap calculations #: in python simulation - OVERLAP_CHUNK_SIZE = None + OVERLAP_CHUNK_SIZE = None #: Overlap option: Split transit leg into component parts? e.g. split A-E #: into A-B-C-D-E for overlap calculations? - OVERLAP_SPLIT_TRANSIT = None + OVERLAP_SPLIT_TRANSIT = None - LEARN_ROUTES = False - LEARN_ROUTES_RATE = 0.05 - SUCCESS_FLAG_COLUMN = 'success_flag' - BUMP_FLAG_COLUMN = 'bump_flag' + LEARN_ROUTES = False + LEARN_ROUTES_RATE = 0.05 + SUCCESS_FLAG_COLUMN = 'success_flag' + BUMP_FLAG_COLUMN = 'bump_flag' #: Allow departures and arrivals before / after preferred time - ARRIVE_LATE_ALLOWED_MIN = datetime.timedelta(minutes = 0) - DEPART_EARLY_ALLOWED_MIN = datetime.timedelta(minutes = 0) + ARRIVE_LATE_ALLOWED_MIN = datetime.timedelta(minutes=0) + DEPART_EARLY_ALLOWED_MIN = datetime.timedelta(minutes=0) - CONSTANT_GROWTH_MODEL = 'constant' - EXP_GROWTH_MODEL = 'exponential' - LOGARITHMIC_GROWTH_MODEL = 'logarithmic' - LOGISTIC_GROWTH_MODEL = 'logistic' + CONSTANT_GROWTH_MODEL = 'constant' + EXP_GROWTH_MODEL = 'exponential' + LOGARITHMIC_GROWTH_MODEL = 'logarithmic' + LOGISTIC_GROWTH_MODEL = 'logistic' - PENALTY_GROWTH_MODELS = [ + PENALTY_GROWTH_MODELS = [ CONSTANT_GROWTH_MODEL, EXP_GROWTH_MODEL, LOGARITHMIC_GROWTH_MODEL, @@ -111,33 +113,33 @@ class PathSet(object): ] #: Weights column: User Class - WEIGHTS_COLUMN_USER_CLASS = "user_class" + WEIGHTS_COLUMN_USER_CLASS = "user_class" #: Weights column: Purpose - WEIGHTS_COLUMN_PURPOSE = "purpose" + WEIGHTS_COLUMN_PURPOSE = "purpose" #: Weights column: Demand Mode Type WEIGHTS_COLUMN_DEMAND_MODE_TYPE = "demand_mode_type" #: Weights column: Demand Mode Type - WEIGHTS_COLUMN_DEMAND_MODE = "demand_mode" + WEIGHTS_COLUMN_DEMAND_MODE = "demand_mode" #: Weights column: Supply Mode - WEIGHTS_COLUMN_SUPPLY_MODE = "supply_mode" + WEIGHTS_COLUMN_SUPPLY_MODE = "supply_mode" #: Weights column: Weight Name - WEIGHTS_COLUMN_WEIGHT_NAME = "weight_name" + WEIGHTS_COLUMN_WEIGHT_NAME = "weight_name" #: Weights column: Weight Value - WEIGHTS_COLUMN_WEIGHT_VALUE = "weight_value" + WEIGHTS_COLUMN_WEIGHT_VALUE = "weight_value" #: Weights column: Growth Type - WEIGHTS_GROWTH_TYPE = "growth_type" + WEIGHTS_GROWTH_TYPE = "growth_type" #: Weights column: Log Base for logarithmic growth function - WEIGHTS_GROWTH_LOG_BASE = "log_base" + WEIGHTS_GROWTH_LOG_BASE = "log_base" #: Weights column: Max value for logistic growth function - WEIGHTS_GROWTH_LOGISTIC_MAX = "logistic_max" + WEIGHTS_GROWTH_LOGISTIC_MAX = "logistic_max" #: Weights column: Midpoint value for logistic growth function - WEIGHTS_GROWTH_LOGISTIC_MID = "logistic_mid" + WEIGHTS_GROWTH_LOGISTIC_MID = "logistic_mid" - WEIGHT_NAME_DEPART_EARLY_MIN = "depart_early_min" - WEIGHT_NAME_ARRIVE_LATE_MIN = "arrive_late_min" + WEIGHT_NAME_DEPART_EARLY_MIN = "depart_early_min" + WEIGHT_NAME_ARRIVE_LATE_MIN = "arrive_late_min" - WEIGHT_NAME_DEPART_LATE_MIN = 'depart_late_min' - WEIGHT_NAME_ARRIVE_EARLY_MIN = 'arrive_early_min' + WEIGHT_NAME_DEPART_LATE_MIN = 'depart_late_min' + WEIGHT_NAME_ARRIVE_EARLY_MIN = 'arrive_early_min' WEIGHT_NAME_VALID_NAMES = [ WEIGHT_NAME_DEPART_EARLY_MIN, @@ -148,44 +150,44 @@ class PathSet(object): # ========== Added by fasttrips ======================================================= #: Weights column: Supply Mode number - WEIGHTS_COLUMN_SUPPLY_MODE_NUM = "supply_mode_num" + WEIGHTS_COLUMN_SUPPLY_MODE_NUM = "supply_mode_num" #: File with weights for c++ - OUTPUT_WEIGHTS_FILE = "ft_intermediate_weights.txt" - - DIR_OUTBOUND = 1 #: Trips outbound from home have preferred arrival times - DIR_INBOUND = 2 #: Trips inbound to home have preferred departure times - - PATH_KEY_COST = "pf_cost" #: path cost according to pathfinder - PATH_KEY_FARE = "pf_fare" #: path fare according to pathfinder - PATH_KEY_PROBABILITY = "pf_probability" #: path probability according to pathfinder - PATH_KEY_INIT_COST = "pf_initcost" #: initial cost (in pathfinding, before path was finalized) - PATH_KEY_INIT_FARE = "pf_initfare" #: initial fare (in pathfinding, before path was finalized) - PATH_KEY_STATES = "states" - - STATE_IDX_LABEL = 0 #: :py:class:`datetime.timedelta` instance - STATE_IDX_DEPARR = 1 #: :py:class:`datetime.datetime` instance. Departure if outbound/backwards, arrival if inbound/forwards. - STATE_IDX_DEPARRMODE = 2 #: mode id - STATE_IDX_TRIP = 3 #: trip id - STATE_IDX_SUCCPRED = 4 #: stop identifier or TAZ identifier - STATE_IDX_SEQ = 5 #: sequence (for trip) - STATE_IDX_SEQ_SUCCPRED = 6 #: sequence for successor/predecessor - STATE_IDX_LINKTIME = 7 #: :py:class:`datetime.timedelta` instance - STATE_IDX_LINKFARE = 8 #: fare cost, float - STATE_IDX_LINKCOST = 9 #: link generalized cost, float for hyperpath/stochastic, - STATE_IDX_LINKDIST = 10 #: link distance, float - STATE_IDX_COST = 11 #: cost float, for hyperpath/stochastic assignment - STATE_IDX_ARRDEP = 12 #: :py:class:`datetime.datetime` instance. Arrival if outbound/backwards, departure if inbound/forwards. + OUTPUT_WEIGHTS_FILE = "ft_intermediate_weights.txt" + + DIR_OUTBOUND = 1 #: Trips outbound from home have preferred arrival times + DIR_INBOUND = 2 #: Trips inbound to home have preferred departure times + + PATH_KEY_COST = "pf_cost" #: path cost according to pathfinder + PATH_KEY_FARE = "pf_fare" #: path fare according to pathfinder + PATH_KEY_PROBABILITY = "pf_probability" #: path probability according to pathfinder + PATH_KEY_INIT_COST = "pf_initcost" #: initial cost (in pathfinding, before path was finalized) + PATH_KEY_INIT_FARE = "pf_initfare" #: initial fare (in pathfinding, before path was finalized) + PATH_KEY_STATES = "states" + + STATE_IDX_LABEL = 0 #: :py:class:`datetime.timedelta` instance + STATE_IDX_DEPARR = 1 #: :py:class:`datetime.datetime` instance. Departure if outbound/backwards, arrival if inbound/forwards. + STATE_IDX_DEPARRMODE = 2 #: mode id + STATE_IDX_TRIP = 3 #: trip id + STATE_IDX_SUCCPRED = 4 #: stop identifier or TAZ identifier + STATE_IDX_SEQ = 5 #: sequence (for trip) + STATE_IDX_SEQ_SUCCPRED = 6 #: sequence for successor/predecessor + STATE_IDX_LINKTIME = 7 #: :py:class:`datetime.timedelta` instance + STATE_IDX_LINKFARE = 8 #: fare cost, float + STATE_IDX_LINKCOST = 9 #: link generalized cost, float for hyperpath/stochastic, + STATE_IDX_LINKDIST = 10 #: link distance, float + STATE_IDX_COST = 11 #: cost float, for hyperpath/stochastic assignment + STATE_IDX_ARRDEP = 12 #: :py:class:`datetime.datetime` instance. Arrival if outbound/backwards, departure if inbound/forwards. # these are also the demand_mode_type values - STATE_MODE_ACCESS = "access" - STATE_MODE_EGRESS = "egress" + STATE_MODE_ACCESS = "access" + STATE_MODE_EGRESS = "egress" STATE_MODE_TRANSFER = "transfer" # new - STATE_MODE_TRIP = "transit" # onboard + STATE_MODE_TRIP = "transit" # onboard - BUMP_EXPERIENCED_COST = 999999 + BUMP_EXPERIENCED_COST = 999999 HUGE_COST = 9999 def __init__(self, trip_list_dict): @@ -197,15 +199,16 @@ def __init__(self, trip_list_dict): #: Direction is one of :py:attr:`PathSet.DIR_OUTBOUND` or :py:attr:`PathSet.DIR_INBOUND` #: Preferred time is a datetime.time object if trip_list_dict[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == "arrival": - self.direction = PathSet.DIR_OUTBOUND - self.pref_time = trip_list_dict[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME] + self.direction = PathSet.DIR_OUTBOUND + self.pref_time = trip_list_dict[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME] self.pref_time_min = trip_list_dict[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME_MIN] elif trip_list_dict[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == "departure": - self.direction = PathSet.DIR_INBOUND - self.pref_time = trip_list_dict[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME] + self.direction = PathSet.DIR_INBOUND + self.pref_time = trip_list_dict[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME] self.pref_time_min = trip_list_dict[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME_MIN] else: - raise Exception("Don't understand trip_list %s: %s" % (Passenger.TRIP_LIST_COLUMN_TIME_TARGET, str(trip_list_dict))) + raise Exception( + "Don't understand trip_list %s: %s" % (Passenger.TRIP_LIST_COLUMN_TIME_TARGET, str(trip_list_dict))) #: Dict of path-num -> { cost:, probability:, states: [List of (stop_id, stop_state)]} self.pathdict = {} @@ -214,7 +217,8 @@ def goes_somewhere(self): """ Does this path go somewhere? Does the destination differ from the origin? """ - return (self.__dict__[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID] != self.__dict__[Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID]) + return (self.__dict__[Passenger.TRIP_LIST_COLUMN_ORIGIN_TAZ_ID] != self.__dict__[ + Passenger.TRIP_LIST_COLUMN_DESTINATION_TAZ_ID]) def path_found(self): """ @@ -239,7 +243,8 @@ def set_user_class(trip_list_df, new_colname): """ Adds a column called user_class by applying the configured user class function. """ - trip_list_df[new_colname] = trip_list_df.apply(PathSet.CONFIGURED_FUNCTIONS[PathSet.USER_CLASS_FUNCTION], axis=1) + trip_list_df[new_colname] = trip_list_df.apply(PathSet.CONFIGURED_FUNCTIONS[PathSet.USER_CLASS_FUNCTION], + axis=1) @staticmethod def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip_list_df): @@ -260,12 +265,12 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip # Join - make sure that all demand combinations (user class, purpose, demand mode type and demand mode) are configured weight_check = pd.merge(left=modes_df, - right=PathSet.WEIGHTS_DF, - on=[PathSet.WEIGHTS_COLUMN_USER_CLASS, - PathSet.WEIGHTS_COLUMN_PURPOSE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE], - how='left') + right=PathSet.WEIGHTS_DF, + on=[PathSet.WEIGHTS_COLUMN_USER_CLASS, + PathSet.WEIGHTS_COLUMN_PURPOSE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE], + how='left') FastTripsLogger.debug("demand_modes x weights: \n%s" % weight_check.to_string()) FastTripsLogger.debug("trip_list_df head=\n%s" % str(trip_list_df.head())) @@ -274,7 +279,8 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip null_supply_mode_weights = weight_check.loc[pd.isnull(weight_check[PathSet.WEIGHTS_COLUMN_SUPPLY_MODE])] if len(null_supply_mode_weights) > 0: # warn - FastTripsLogger.warn("The following user_class, demand_mode_type, demand_mode combinations exist in the demand file but are missing from the weight configuration:") + FastTripsLogger.warn( + "The following user_class, demand_mode_type, demand_mode combinations exist in the demand file but are missing from the weight configuration:") FastTripsLogger.warn("\n%s" % null_supply_mode_weights.to_string()) # remove those trips -- need to do it one demand mode type at a time @@ -284,41 +290,43 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip PathSet.WEIGHTS_COLUMN_DEMAND_MODE]] null_supply_mode_weights["to_remove"] = 1 for demand_mode_type in [PathSet.STATE_MODE_ACCESS, PathSet.STATE_MODE_EGRESS, PathSet.STATE_MODE_TRIP]: - remove_trips = null_supply_mode_weights.loc[null_supply_mode_weights[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE]==demand_mode_type].copy() + remove_trips = null_supply_mode_weights.loc[ + null_supply_mode_weights[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] == demand_mode_type].copy() if len(remove_trips) == 0: continue - remove_trips.rename(columns={PathSet.WEIGHTS_COLUMN_DEMAND_MODE:"%s_mode" % demand_mode_type}, inplace=True) + remove_trips.rename(columns={PathSet.WEIGHTS_COLUMN_DEMAND_MODE: "%s_mode" % demand_mode_type}, + inplace=True) remove_trips.drop([PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE], axis=1, inplace=True) FastTripsLogger.debug("Removing for \n%s" % remove_trips) - trip_list_df = pd.merge(left = trip_list_df, - right = remove_trips, - how = "left") + trip_list_df = pd.merge(left=trip_list_df, + right=remove_trips, + how="left") FastTripsLogger.debug("Removing\n%s" % trip_list_df.loc[pd.notnull(trip_list_df["to_remove"])]) # keep only those not flagged to_remove trip_list_df = trip_list_df.loc[pd.isnull(trip_list_df["to_remove"])] trip_list_df.drop(["to_remove"], axis=1, inplace=True) - # demand_mode_type and demand_modes implicit to all travel : xfer walk, xfer wait, initial wait - user_classes = modes_df[[PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE]].drop_duplicates().reset_index() - implicit_df = pd.DataFrame({ PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE:[ 'transfer'], - PathSet.WEIGHTS_COLUMN_DEMAND_MODE :[ 'transfer'], - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE :[ 'transfer'] }) + user_classes = modes_df[ + [PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE]].drop_duplicates().reset_index() + implicit_df = pd.DataFrame({PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE: ['transfer'], + PathSet.WEIGHTS_COLUMN_DEMAND_MODE: ['transfer'], + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE: ['transfer']}) user_classes['key'] = 1 implicit_df['key'] = 1 implicit_df = pd.merge(left=user_classes, right=implicit_df, on='key') - implicit_df.drop(['index','key'], axis=1, inplace=True) + implicit_df.drop(['index', 'key'], axis=1, inplace=True) # FastTripsLogger.debug("implicit_df: \n%s" % implicit_df) weight_check = pd.merge(left=implicit_df, right=PathSet.WEIGHTS_DF, - on=[PathSet.WEIGHTS_COLUMN_USER_CLASS, - PathSet.WEIGHTS_COLUMN_PURPOSE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE, - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE], - how='left') + on=[PathSet.WEIGHTS_COLUMN_USER_CLASS, + PathSet.WEIGHTS_COLUMN_PURPOSE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE, + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE], + how='left') FastTripsLogger.debug("implicit demand_modes x weights: \n%s" % weight_check.to_string()) if pd.isnull(weight_check[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME]).sum() > 0: @@ -327,11 +335,12 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip error_str += "\n\n" # transfer penalty check - tp_index = pd.DataFrame({ PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE:['transfer'], - PathSet.WEIGHTS_COLUMN_DEMAND_MODE :['transfer'], - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE :['transfer'], - PathSet.WEIGHTS_COLUMN_WEIGHT_NAME :['transfer_penalty']}) - uc_purp_index = PathSet.WEIGHTS_DF[[PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE]].drop_duplicates() + tp_index = pd.DataFrame({PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE: ['transfer'], + PathSet.WEIGHTS_COLUMN_DEMAND_MODE: ['transfer'], + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE: ['transfer'], + PathSet.WEIGHTS_COLUMN_WEIGHT_NAME: ['transfer_penalty']}) + uc_purp_index = PathSet.WEIGHTS_DF[ + [PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE]].drop_duplicates() FastTripsLogger.debug("uc_purp_index: \n%s" % uc_purp_index) # these are all the transfer penalties we have @@ -344,10 +353,12 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip # missing transfer penalty if pd.isnull(transfer_penalty_check[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME]).sum() > 0: error_str += "\nThe following user class x purpose are missing a transfer penalty:\n" - error_str += transfer_penalty_check.loc[pd.isnull(transfer_penalty_check[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME])].to_string() + error_str += transfer_penalty_check.loc[ + pd.isnull(transfer_penalty_check[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME])].to_string() error_str += "\n\n" - bad_pen = transfer_penalty_check.loc[transfer_penalty_check[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE] < PathSet.MIN_TRANSFER_PENALTY] + bad_pen = transfer_penalty_check.loc[ + transfer_penalty_check[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE] < PathSet.MIN_TRANSFER_PENALTY] if len(bad_pen) > 0: error_str += "\nThe following user class x purpose path weights have invalid (too small) transfer penalties. MIN=(%f)\n" % PathSet.MIN_TRANSFER_PENALTY error_str += bad_pen.to_string() @@ -357,21 +368,23 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip # to enforce it. if capacity_constraint: # see if it's here already -- we don't know how to handle that... - at_capacity = PathSet.WEIGHTS_DF.loc[ PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "at_capacity" ] + at_capacity = PathSet.WEIGHTS_DF.loc[ + PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "at_capacity"] if len(at_capacity) > 0: error_str += "\nFound at_capacity path weights explicitly set when about to set these for hard capacity constraints.\n" error_str += at_capacity.to_string() error_str += "\n\n" else: # set it for all user_class x transit x demand_mode x supply_mode - transit_weights_df = PathSet.WEIGHTS_DF.loc[PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] == PathSet.STATE_MODE_TRIP, + transit_weights_df = PathSet.WEIGHTS_DF.loc[ + PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] == PathSet.STATE_MODE_TRIP, [PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE, PathSet.WEIGHTS_COLUMN_DEMAND_MODE, PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, PathSet.WEIGHTS_COLUMN_SUPPLY_MODE]].copy() transit_weights_df.drop_duplicates(inplace=True) - transit_weights_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME ] = "at_capacity" + transit_weights_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] = "at_capacity" transit_weights_df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE] = PathSet.HUGE_COST transit_weights_df[PathSet.WEIGHTS_GROWTH_TYPE] = PathSet.CONSTANT_GROWTH_MODEL FastTripsLogger.debug("Adding capacity-constraint weights:\n%s" % transit_weights_df.to_string()) @@ -390,31 +403,32 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip # add mode numbers to weights DF for relevant rows PathSet.WEIGHTS_DF = routes.add_numeric_mode_id(PathSet.WEIGHTS_DF, - id_colname=PathSet.WEIGHTS_COLUMN_SUPPLY_MODE, - numeric_newcolname=PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, - warn=True) # don't fail if some supply modes are configured but not used, they may be for future runs + id_colname=PathSet.WEIGHTS_COLUMN_SUPPLY_MODE, + numeric_newcolname=PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, + warn=True) # don't fail if some supply modes are configured but not used, they may be for future runs FastTripsLogger.debug("PathSet weights: \n%s" % PathSet.WEIGHTS_DF) export_columns = [PathSet.WEIGHTS_COLUMN_USER_CLASS, - PathSet.WEIGHTS_COLUMN_PURPOSE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE, - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, - PathSet.WEIGHTS_COLUMN_WEIGHT_NAME, - PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE, - PathSet.WEIGHTS_GROWTH_TYPE, - PathSet.WEIGHTS_GROWTH_LOG_BASE, - PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX, - PathSet.WEIGHTS_GROWTH_LOGISTIC_MID] - - PathSet.WEIGHTS_DF.reindex(columns=export_columns).to_csv(os.path.join(output_dir,PathSet.OUTPUT_WEIGHTS_FILE), - columns=export_columns, - sep=" ", index=False) + PathSet.WEIGHTS_COLUMN_PURPOSE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE, + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, + PathSet.WEIGHTS_COLUMN_WEIGHT_NAME, + PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE, + PathSet.WEIGHTS_GROWTH_TYPE, + PathSet.WEIGHTS_GROWTH_LOG_BASE, + PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX, + PathSet.WEIGHTS_GROWTH_LOGISTIC_MID] + + PathSet.WEIGHTS_DF.reindex(columns=export_columns).to_csv(os.path.join(output_dir, PathSet.OUTPUT_WEIGHTS_FILE), + columns=export_columns, + sep=" ", index=False) # add placeholder weights (ivt weight) for fares - one for each user_class, purpose, transit demand mode # these will be updated based on the person's value of time in calculate_cost() - fare_weights = PathSet.WEIGHTS_DF.loc[ (PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE]==PathSet.STATE_MODE_TRIP) & - (PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME ]== "in_vehicle_time_min")] + fare_weights = PathSet.WEIGHTS_DF.loc[ + (PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE] == PathSet.STATE_MODE_TRIP) & + (PathSet.WEIGHTS_DF[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min")] fare_weights = fare_weights[[PathSet.WEIGHTS_COLUMN_USER_CLASS, PathSet.WEIGHTS_COLUMN_PURPOSE, PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, @@ -423,12 +437,11 @@ def verify_weight_config(modes_df, output_dir, routes, capacity_constraint, trip PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, PathSet.WEIGHTS_COLUMN_WEIGHT_NAME, PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE]].copy().drop_duplicates() - fare_weights[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME ] = "fare" # SIM_COL_PAX_FARE + fare_weights[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] = "fare" # SIM_COL_PAX_FARE PathSet.WEIGHTS_DF = PathSet.WEIGHTS_DF.append(fare_weights) FastTripsLogger.debug("PathSet.WEIGHTS_DF with fare weights: \n%s" % PathSet.WEIGHTS_DF) return trip_list_df - @staticmethod def verify_weights(weights): # First, verify required columns are found @@ -436,21 +449,21 @@ def verify_weights(weights): weight_cols = list(weights.columns.values) FastTripsLogger.debug("verify_weight_config:\n%s" % weights.to_string()) if (PathSet.WEIGHTS_COLUMN_USER_CLASS not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_USER_CLASS) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_USER_CLASS) if (PathSet.WEIGHTS_COLUMN_PURPOSE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_PURPOSE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_PURPOSE) if (PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE) if (PathSet.WEIGHTS_COLUMN_DEMAND_MODE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_DEMAND_MODE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_DEMAND_MODE) if (PathSet.WEIGHTS_COLUMN_SUPPLY_MODE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_SUPPLY_MODE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_SUPPLY_MODE) if (PathSet.WEIGHTS_COLUMN_WEIGHT_NAME not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_WEIGHT_NAME) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_WEIGHT_NAME) if (PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE) if (PathSet.WEIGHTS_GROWTH_TYPE not in weight_cols): - error_str+='{} not in weight_cols\n'.format(PathSet.WEIGHTS_GROWTH_TYPE) + error_str += '{} not in weight_cols\n'.format(PathSet.WEIGHTS_GROWTH_TYPE) constant_exp_slice = weights.loc[ weights[PathSet.WEIGHTS_GROWTH_TYPE].isin( @@ -485,7 +498,7 @@ def verify_weights(weights): if not pd.notnull(logarithmic_slice.reindex([ PathSet.WEIGHTS_GROWTH_LOG_BASE, ], - axis='columns')).values.all(): + axis='columns')).values.all(): error_str += 'Logarithmic qualifier missing necessary log_base modifier\n' if not pd.notnull(logistic_slice.reindex([ @@ -499,7 +512,6 @@ def verify_weights(weights): return (not error_str), error_str - def __str__(self): """ Readable string version of the path. @@ -507,7 +519,7 @@ def __str__(self): Note: If inbound trip, then the states are in reverse order (egress to access) """ ret_str = "Dict vars:\n" - for k,v in self.__dict__.items(): + for k, v in self.__dict__.items(): ret_str += "%30s => %-30s %s\n" % (str(k), str(v), str(type(v))) # ret_str += PathSet.states_to_str(self.states, self.direction) return ret_str @@ -524,56 +536,61 @@ def write_paths(passengers_df, output_dir): """ # get trip information -- board stops, board trips and alight stops - passenger_trips = passengers_df.loc[passengers_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRIP].copy() - ptrip_group = passenger_trips.groupby([Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) + passenger_trips = passengers_df.loc[passengers_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP].copy() + ptrip_group = passenger_trips.groupby( + [Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) # these are Series - board_stops_str = ptrip_group.A_id.apply(lambda x:','.join(x)) - board_trips_str = ptrip_group.trip_id.apply(lambda x:','.join(x)) - alight_stops_str= ptrip_group.B_id.apply(lambda x:','.join(x)) - board_stops_str.name = 'board_stop_str' - board_trips_str.name = 'board_trip_str' + board_stops_str = ptrip_group.A_id.apply(lambda x: ','.join(x)) + board_trips_str = ptrip_group.trip_id.apply(lambda x: ','.join(x)) + alight_stops_str = ptrip_group.B_id.apply(lambda x: ','.join(x)) + board_stops_str.name = 'board_stop_str' + board_trips_str.name = 'board_trip_str' alight_stops_str.name = 'alight_stop_str' # get walking times - walk_links = passengers_df.loc[(passengers_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_ACCESS )| \ - (passengers_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRANSFER)| \ - (passengers_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_EGRESS )].copy() - walk_links['linktime_str'] = walk_links.pf_linktime.apply(lambda x: "%.2f" % (x/np.timedelta64(1,'m'))) - walklink_group = walk_links[['person_id','trip_list_id_num','linktime_str']].groupby(['person_id','trip_list_id_num']) - walktimes_str = walklink_group.linktime_str.apply(lambda x:','.join(x)) + walk_links = passengers_df.loc[(passengers_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) | \ + (passengers_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRANSFER) | \ + (passengers_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS)].copy() + walk_links['linktime_str'] = walk_links.pf_linktime.apply(lambda x: "%.2f" % (x / np.timedelta64(1, 'm'))) + walklink_group = walk_links[['person_id', 'trip_list_id_num', 'linktime_str']].groupby( + ['person_id', 'trip_list_id_num']) + walktimes_str = walklink_group.linktime_str.apply(lambda x: ','.join(x)) # aggregate to one line per person_id, trip_list_id - print_passengers_df = passengers_df[['person_id','trip_list_id_num','pathmode','A_id','B_id',Passenger.PF_COL_PAX_A_TIME]].groupby(['person_id','trip_list_id_num']).agg( - {'pathmode' :'first', # path mode - 'A_id' :'first', # origin - 'B_id' :'last', # destination - Passenger.PF_COL_PAX_A_TIME :'first' # start time - }) + print_passengers_df = passengers_df[ + ['person_id', 'trip_list_id_num', 'pathmode', 'A_id', 'B_id', Passenger.PF_COL_PAX_A_TIME]].groupby( + ['person_id', 'trip_list_id_num']).agg( + {'pathmode': 'first', # path mode + 'A_id': 'first', # origin + 'B_id': 'last', # destination + Passenger.PF_COL_PAX_A_TIME: 'first' # start time + }) # put them all together print_passengers_df = pd.concat([print_passengers_df, - board_stops_str, - board_trips_str, - alight_stops_str, - walktimes_str], axis=1) + board_stops_str, + board_trips_str, + alight_stops_str, + walktimes_str], axis=1) print_passengers_df.reset_index(inplace=True) print_passengers_df.sort_values(by=['trip_list_id_num'], inplace=True) print_passengers_df.rename(columns= - {'pathmode' :'mode', - 'A_id' :'originTaz', - 'B_id' :'destinationTaz', - Passenger.PF_COL_PAX_A_TIME :'startTime_time', - 'board_stop_str' :'boardingStops', - 'board_trip_str' :'boardingTrips', - 'alight_stop_str' :'alightingStops', - 'linktime_str' :'walkingTimes'}, inplace=True) + {'pathmode': 'mode', + 'A_id': 'originTaz', + 'B_id': 'destinationTaz', + Passenger.PF_COL_PAX_A_TIME: 'startTime_time', + 'board_stop_str': 'boardingStops', + 'board_trip_str': 'boardingTrips', + 'alight_stop_str': 'alightingStops', + 'linktime_str': 'walkingTimes'}, inplace=True) print_passengers_df['startTime'] = print_passengers_df['startTime_time'].apply(Util.datetime64_formatter) - print_passengers_df = print_passengers_df[['trip_list_id_num','person_id','mode','originTaz','destinationTaz','startTime', - 'boardingStops','boardingTrips','alightingStops','walkingTimes']] + print_passengers_df = print_passengers_df[ + ['trip_list_id_num', 'person_id', 'mode', 'originTaz', 'destinationTaz', 'startTime', + 'boardingStops', 'boardingTrips', 'alightingStops', 'walkingTimes']] print_passengers_df.to_csv(os.path.join(output_dir, PathSet.PATHS_OUTPUT_FILE), sep="\t", index=False) # passengerId mode originTaz destinationTaz startTime boardingStops boardingTrips alightingStops walkingTimes @@ -588,40 +605,43 @@ def write_path_times(passengers_df, output_dir): :param output_dir: Output directory :type output_dir: string """ - passenger_trips = passengers_df.loc[passengers_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRIP].copy() + passenger_trips = passengers_df.loc[passengers_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP].copy() ###### TODO: this is really catering to output format; an alternative might be more appropriate from .Assignment import Assignment - passenger_trips.loc[:, 'board_time_str'] = passenger_trips[Assignment.SIM_COL_PAX_BOARD_TIME ].apply(Util.datetime64_formatter) - passenger_trips.loc[:,'arrival_time_str'] = passenger_trips[Passenger.PF_COL_PAX_A_TIME].apply(Util.datetime64_formatter) - passenger_trips.loc[:, 'alight_time_str'] = passenger_trips[Assignment.SIM_COL_PAX_ALIGHT_TIME].apply(Util.datetime64_formatter) + passenger_trips.loc[:, 'board_time_str'] = passenger_trips[Assignment.SIM_COL_PAX_BOARD_TIME].apply( + Util.datetime64_formatter) + passenger_trips.loc[:, 'arrival_time_str'] = passenger_trips[Passenger.PF_COL_PAX_A_TIME].apply( + Util.datetime64_formatter) + passenger_trips.loc[:, 'alight_time_str'] = passenger_trips[Assignment.SIM_COL_PAX_ALIGHT_TIME].apply( + Util.datetime64_formatter) # Aggregate (by joining) across each passenger + path ptrip_group = passenger_trips.groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) # these are Series - board_time_str = ptrip_group['board_time_str' ].apply(lambda x:','.join(x)) - arrival_time_str = ptrip_group['arrival_time_str'].apply(lambda x:','.join(x)) - alight_time_str = ptrip_group['alight_time_str' ].apply(lambda x:','.join(x)) + board_time_str = ptrip_group['board_time_str'].apply(lambda x: ','.join(x)) + arrival_time_str = ptrip_group['arrival_time_str'].apply(lambda x: ','.join(x)) + alight_time_str = ptrip_group['alight_time_str'].apply(lambda x: ','.join(x)) # Aggregate other fields across each passenger + path pax_exp_df = passengers_df.groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).agg( - {# 'pathmode' :'first', # path mode - 'A_id' :'first', # origin - 'B_id' :'last', # destination - Passenger.PF_COL_PAX_A_TIME :'first', # start time - Passenger.PF_COL_PAX_B_TIME :'last', # end time - # TODO: cost needs to be updated for updated dwell & travel time - # 'cost' :'first', # total travel cost is calculated for the whole path + { # 'pathmode' :'first', # path mode + 'A_id': 'first', # origin + 'B_id': 'last', # destination + Passenger.PF_COL_PAX_A_TIME: 'first', # start time + Passenger.PF_COL_PAX_B_TIME: 'last', # end time + # TODO: cost needs to be updated for updated dwell & travel time + # 'cost' :'first', # total travel cost is calculated for the whole path }) # Put them together and return - assert(len(pax_exp_df) == len(board_time_str)) + assert (len(pax_exp_df) == len(board_time_str)) pax_exp_df = pd.concat([pax_exp_df, - board_time_str, - arrival_time_str, - alight_time_str], axis=1) + board_time_str, + arrival_time_str, + alight_time_str], axis=1) # print pax_exp_df.to_string(formatters={'A_time':Assignment.datetime64_min_formatter, # 'B_time':Assignment.datetime64_min_formatter} @@ -635,23 +655,23 @@ def write_path_times(passengers_df, output_dir): # rename columns print_pax_exp_df.rename(columns= - {#'pathmode' :'mode', - 'A_id' :'originTaz', - 'B_id' :'destinationTaz', - 'A_time_str' :'startTime', - 'B_time_str' :'endTime', - 'arrival_time_str' :'arrivalTimes', - 'board_time_str' :'boardingTimes', - 'alight_time_str' :'alightingTimes', - # TODO: cost needs to be updated for updated dwell & travel time - # 'cost' :'travelCost', - }, inplace=True) + { # 'pathmode' :'mode', + 'A_id': 'originTaz', + 'B_id': 'destinationTaz', + 'A_time_str': 'startTime', + 'B_time_str': 'endTime', + 'arrival_time_str': 'arrivalTimes', + 'board_time_str': 'boardingTimes', + 'alight_time_str': 'alightingTimes', + # TODO: cost needs to be updated for updated dwell & travel time + # 'cost' :'travelCost', + }, inplace=True) # reorder print_pax_exp_df = print_pax_exp_df[[ Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - #'mode', + # 'mode', 'originTaz', 'destinationTaz', 'startTime', @@ -660,7 +680,7 @@ def write_path_times(passengers_df, output_dir): 'boardingTimes', 'alightingTimes', # 'travelCost', - ]] + ]] times_out = open(os.path.join(output_dir, PathSet.PATH_TIMES_OUTPUT_FILE), 'w') print_pax_exp_df.to_csv(times_out, @@ -682,7 +702,9 @@ def split_transit_links(pathset_links_df, veh_trips_df, stops): if len(Assignment.TRACE_IDS) > 0: FastTripsLogger.debug("split_transit_links: pathset_links_df (%d) trace\n%s" % (len(pathset_links_df), - pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string())) + pathset_links_df.loc[ + pathset_links_df[ + Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string())) FastTripsLogger.debug("split_transit_links: pathset_links_df columns\n%s" % str(pathset_links_df.dtypes)) veh_links_df = Trip.linkify_vehicle_trips(veh_trips_df, stops) @@ -691,7 +713,7 @@ def split_transit_links(pathset_links_df, veh_trips_df, stops): FastTripsLogger.debug("split_transit_links: veh_links_df\n%s" % veh_links_df.head(20).to_string()) # join the pathset links with the vehicle links - drop_cols = [] + drop_cols = [] merge_cols = [Passenger.PF_COL_LINK_MODE, Route.ROUTES_COLUMN_MODE, Trip.TRIPS_COLUMN_ROUTE_ID, @@ -701,78 +723,87 @@ def split_transit_links(pathset_links_df, veh_trips_df, stops): if Route.ROUTES_COLUMN_MODE_NUM in pathset_links_df.columns.values: merge_cols.append(Route.ROUTES_COLUMN_MODE_NUM) - path2 = pd.merge(left =pathset_links_df, - right =veh_links_df, - on =merge_cols, - how ="left", - suffixes=["","_veh"]) + path2 = pd.merge(left=pathset_links_df, + right=veh_links_df, + on=merge_cols, + how="left", + suffixes=["", "_veh"]) path2["split_first"] = False # delete anything irrelevant -- so keep non-transit links, and transit links WITH valid sequences - path2 = path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]!=Route.MODE_TYPE_TRANSIT) | - ( (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT) & - (path2["A_seq_veh"]>=path2["A_seq"]) & - (path2["B_seq_veh"]<=path2["B_seq"]) ) ] + path2 = path2.loc[(path2[Passenger.PF_COL_LINK_MODE] != Route.MODE_TYPE_TRANSIT) | + ((path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & + (path2["A_seq_veh"] >= path2["A_seq"]) & + (path2["B_seq_veh"] <= path2["B_seq"]))] # These are the new columns -- incorporate them - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["A_seq_veh"]==path2["A_seq"]), "split_first"] = True + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["A_seq_veh"] == path2["A_seq"]), "split_first"] = True # A_arrival_time datetime64[ns] => A time for intermediate links - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["A_id"]!=path2["A_id_veh"]), Assignment.SIM_COL_PAX_A_TIME ] = path2["A_arrival_time"] + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["A_id"] != path2["A_id_veh"]), Assignment.SIM_COL_PAX_A_TIME] = path2["A_arrival_time"] # no waittime, boardtime, missed_xfer except on first link - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["A_id"]!=path2["A_id_veh"]), Assignment.SIM_COL_PAX_WAIT_TIME ] = None - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["A_id"]!=path2["A_id_veh"]), Assignment.SIM_COL_PAX_BOARD_TIME ] = None - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["A_id"]!=path2["A_id_veh"]), Assignment.SIM_COL_PAX_MISSED_XFER] = 0 + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["A_id"] != path2["A_id_veh"]), Assignment.SIM_COL_PAX_WAIT_TIME] = None + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["A_id"] != path2["A_id_veh"]), Assignment.SIM_COL_PAX_BOARD_TIME] = None + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["A_id"] != path2["A_id_veh"]), Assignment.SIM_COL_PAX_MISSED_XFER] = 0 # no alighttime except on last link - path2.loc[ (path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT)&(path2["B_id"]!=path2["B_id_veh"]), Assignment.SIM_COL_PAX_ALIGHT_TIME] = None + path2.loc[(path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT) & ( + path2["B_id"] != path2["B_id_veh"]), Assignment.SIM_COL_PAX_ALIGHT_TIME] = None # route_id_num float64 => ignore # A_id_veh object => A_id - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "A_id" ] = path2["A_id_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "A_id"] = path2["A_id_veh"] # A_id_num_veh float64 => A_id_num - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "A_id_num" ] = path2["A_id_num_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "A_id_num"] = path2["A_id_num_veh"] # A_seq_veh float64 => A_seq - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "A_seq" ] = path2["A_seq_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "A_seq"] = path2["A_seq_veh"] if "A_lat_veh" in path2.columns.values: # A_lat_veh float64 => A_lat - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "A_lat" ] = path2["A_lat_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "A_lat"] = path2["A_lat_veh"] # A_lon_veh float64 => A_lon - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "A_lon" ] = path2["A_lon_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "A_lon"] = path2["A_lon_veh"] # drop these later - drop_cols.extend(["A_lat_veh","A_lon_veh"]) + drop_cols.extend(["A_lat_veh", "A_lon_veh"]) # B_id_veh object => B_id - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "B_id" ] = path2["B_id_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "B_id"] = path2["B_id_veh"] # B_id_num_veh float64 => B_id_num - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "B_id_num" ] = path2["B_id_num_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "B_id_num"] = path2["B_id_num_veh"] # B_seq_veh float64 => B_seq - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "B_seq" ] = path2["B_seq_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "B_seq"] = path2["B_seq_veh"] # B_arrival_time datetime64[ns] => new_B_time - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "new_B_time" ] = path2["B_arrival_time"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "new_B_time"] = path2["B_arrival_time"] # B_departure_time datetime64[ns] => ignore if "B_lat_veh" in path2.columns.values: # B_lat_veh float64 => B_lat - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "B_lat" ] = path2["B_lat_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "B_lat"] = path2["B_lat_veh"] # B_lon_veh float64 => B_lon - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT, "B_lon" ] = path2["B_lon_veh"] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, "B_lon"] = path2["B_lon_veh"] # drop these later - drop_cols.extend(["B_lat_veh","B_lon_veh"]) + drop_cols.extend(["B_lat_veh", "B_lon_veh"]) # update the link time - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT,Assignment.SIM_COL_PAX_LINK_TIME] = path2[Assignment.SIM_COL_PAX_B_TIME] - path2[Assignment.SIM_COL_PAX_A_TIME] + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, Assignment.SIM_COL_PAX_LINK_TIME] = \ + path2[Assignment.SIM_COL_PAX_B_TIME] - path2[Assignment.SIM_COL_PAX_A_TIME] # update transit distance - Util.calculate_distance_miles(path2, "A_lat","A_lon","B_lat","B_lon", "transit_distance") - path2.loc[path2[Passenger.PF_COL_LINK_MODE]==Route.MODE_TYPE_TRANSIT,Assignment.SIM_COL_PAX_DISTANCE ] = path2["transit_distance"] + Util.calculate_distance_miles(path2, "A_lat", "A_lon", "B_lat", "B_lon", "transit_distance") + path2.loc[path2[Passenger.PF_COL_LINK_MODE] == Route.MODE_TYPE_TRANSIT, Assignment.SIM_COL_PAX_DISTANCE] = \ + path2["transit_distance"] # revert these back to ints - path2[["A_id_num","B_id_num","A_seq","B_seq"]] = path2[["A_id_num","B_id_num","A_seq","B_seq"]].astype(int) + path2[["A_id_num", "B_id_num", "A_seq", "B_seq"]] = path2[["A_id_num", "B_id_num", "A_seq", "B_seq"]].astype( + int) # we're done with the fields - drop them drop_cols.extend(["transit_distance", "route_id_num", - "A_id_veh","A_id_num_veh","A_seq_veh","A_arrival_time","A_departure_time", - "B_id_veh","B_id_num_veh","B_seq_veh","B_arrival_time","B_departure_time"]) + "A_id_veh", "A_id_num_veh", "A_seq_veh", "A_arrival_time", "A_departure_time", + "B_id_veh", "B_id_num_veh", "B_seq_veh", "B_arrival_time", "B_departure_time"]) path2.drop(drop_cols, axis=1, inplace=True) # renumber linknum? Let's not bother @@ -780,7 +811,8 @@ def split_transit_links(pathset_links_df, veh_trips_df, stops): # trace if len(Assignment.TRACE_IDS) > 0: FastTripsLogger.debug("split_transit_links: path2 (%d) trace\n%s" % (len(path2), - path2.loc[path2[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string())) + path2.loc[path2[ + Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string())) FastTripsLogger.debug("split_transit_links: path2 columns\n%s" % str(path2.dtypes)) return path2 @@ -813,23 +845,23 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri # leaving this in for writing to CSV for debugging but I could take it out pathset_paths_df.drop(["logsum_component"], axis=1, inplace=True) - if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: pathset_links_df trace\n%s" % str(pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) - FastTripsLogger.debug("calculate_cost: trip_list_df trace\n%s" % str(trip_list_df.loc[trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: pathset_links_df trace\n%s" % str( + pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) + FastTripsLogger.debug("calculate_cost: trip_list_df trace\n%s" % str( + trip_list_df.loc[trip_list_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # Add fares -- need stop zones first if they're not there. # We only need to do this once per pathset. # todo -- could remove non-transit links for this? FastTripsLogger.debug("calculate_cost columns:\n%s" % str(list(pathset_links_df.columns.values))) if "A_zone_id" not in list(pathset_links_df.columns.values): - assert(stops is not None) + assert (stops is not None) pathset_links_df = stops.add_stop_zone_id(pathset_links_df, "A_id", "A_zone_id") pathset_links_df = stops.add_stop_zone_id(pathset_links_df, "B_id", "B_zone_id") # This needs to be done fresh each time since simulation might change the board times and therefore the fare periods pathset_links_df = routes.add_fares(pathset_links_df) - # base this on pathfinding distance pathset_links_df[Assignment.SIM_COL_PAX_DISTANCE] = pathset_links_df[Passenger.PF_COL_LINK_DIST] @@ -840,29 +872,37 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri pathset_links_to_use["split_first"] = True # all transit links are first # First, we need user class, purpose, demand modes, and value of time - pathset_links_cost_df = pd.merge(left =pathset_links_to_use, - right=trip_list_df[[ - Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_USER_CLASS, - Passenger.TRIP_LIST_COLUMN_PURPOSE, - Passenger.TRIP_LIST_COLUMN_VOT, - Passenger.TRIP_LIST_COLUMN_ACCESS_MODE, - Passenger.TRIP_LIST_COLUMN_EGRESS_MODE, - Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE, - ]], - how ="left", - on =[Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) + pathset_links_cost_df = pd.merge(left=pathset_links_to_use, + right=trip_list_df[[ + Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_USER_CLASS, + Passenger.TRIP_LIST_COLUMN_PURPOSE, + Passenger.TRIP_LIST_COLUMN_VOT, + Passenger.TRIP_LIST_COLUMN_ACCESS_MODE, + Passenger.TRIP_LIST_COLUMN_EGRESS_MODE, + Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE, + ]], + how="left", + on=[Passenger.PERSONS_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) # linkmode = demand_mode_type. Set demand_mode for the links pathset_links_cost_df[PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = None - pathset_links_cost_df.loc[ pathset_links_cost_df[Passenger.PF_COL_LINK_MODE]== PathSet.STATE_MODE_ACCESS , PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_ACCESS_MODE ] - pathset_links_cost_df.loc[ pathset_links_cost_df[Passenger.PF_COL_LINK_MODE]== PathSet.STATE_MODE_EGRESS , PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_EGRESS_MODE ] - pathset_links_cost_df.loc[ pathset_links_cost_df[Passenger.PF_COL_LINK_MODE]== PathSet.STATE_MODE_TRIP , PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE] - pathset_links_cost_df.loc[ pathset_links_cost_df[Passenger.PF_COL_LINK_MODE]== PathSet.STATE_MODE_TRANSFER, PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = "transfer" + pathset_links_cost_df.loc[pathset_links_cost_df[ + Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS, PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = \ + pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_ACCESS_MODE] + pathset_links_cost_df.loc[pathset_links_cost_df[ + Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS, PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = \ + pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_EGRESS_MODE] + pathset_links_cost_df.loc[pathset_links_cost_df[ + Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP, PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = \ + pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_TRANSIT_MODE] + pathset_links_cost_df.loc[pathset_links_cost_df[ + Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRANSFER, PathSet.WEIGHTS_COLUMN_DEMAND_MODE] = "transfer" # Verify that it's set for every link missing_demand_mode = pd.isnull(pathset_links_cost_df[PathSet.WEIGHTS_COLUMN_DEMAND_MODE]).sum() - assert(missing_demand_mode == 0) + assert (missing_demand_mode == 0) # drop the individual mode columns, we have what we need pathset_links_cost_df.drop([Passenger.TRIP_LIST_COLUMN_ACCESS_MODE, @@ -874,55 +914,59 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri pathset_links_cost_df[Assignment.SIM_COL_PAX_BUMP_ITER] = -1 if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: pathset_links_cost_df trace\n%s" % str(pathset_links_cost_df.loc[pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: pathset_links_cost_df trace\n%s" % str( + pathset_links_cost_df.loc[pathset_links_cost_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # Inner join with the weights - now each weight has a row - cost_df = pd.merge(left =pathset_links_cost_df, - right =PathSet.WEIGHTS_DF, - left_on =[Passenger.TRIP_LIST_COLUMN_USER_CLASS, - Passenger.TRIP_LIST_COLUMN_PURPOSE, - Passenger.PF_COL_LINK_MODE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE, - Passenger.TRIP_LIST_COLUMN_MODE], - right_on=[Passenger.TRIP_LIST_COLUMN_USER_CLASS, - Passenger.TRIP_LIST_COLUMN_PURPOSE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, - PathSet.WEIGHTS_COLUMN_DEMAND_MODE, - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE], - how ="inner") + cost_df = pd.merge(left=pathset_links_cost_df, + right=PathSet.WEIGHTS_DF, + left_on=[Passenger.TRIP_LIST_COLUMN_USER_CLASS, + Passenger.TRIP_LIST_COLUMN_PURPOSE, + Passenger.PF_COL_LINK_MODE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE, + Passenger.TRIP_LIST_COLUMN_MODE], + right_on=[Passenger.TRIP_LIST_COLUMN_USER_CLASS, + Passenger.TRIP_LIST_COLUMN_PURPOSE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE_TYPE, + PathSet.WEIGHTS_COLUMN_DEMAND_MODE, + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE], + how="inner") # update the fare weight placeholder (ivt pathweight - utils per min)) based on value of time (currency per hour) # since generalized cost is in utils, (ivt utils/min)x(60 min/1 hour)x(hour/vot currency) is the weight (utils/currency) - cost_df.loc[ cost_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME]==Assignment.SIM_COL_PAX_FARE, "weight_value" ] *= (60.0/cost_df[Passenger.TRIP_LIST_COLUMN_VOT]) + cost_df.loc[cost_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == Assignment.SIM_COL_PAX_FARE, "weight_value"] *= ( + 60.0 / cost_df[Passenger.TRIP_LIST_COLUMN_VOT]) if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: cost_df\n%s" % str(cost_df.loc[cost_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].sort_values([ - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.PF_COL_PATH_NUM,Passenger.PF_COL_LINK_NUM]).head(20))) + FastTripsLogger.debug("calculate_cost: cost_df\n%s" % str( + cost_df.loc[cost_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].sort_values([ + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.PF_COL_PATH_NUM, Passenger.PF_COL_LINK_NUM]).head(20))) # NOW we split it into 3 lists -- access/egress, transit, and transfer # This is because they will each be joined to tables specific to those kinds of mode categories, and so we don't want all the transit nulls on the other tables, etc. cost_columns = list(cost_df.columns.values) cost_df["var_value"] = np.nan # This means unset - cost_accegr_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_ACCESS )|(cost_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_EGRESS)] - cost_trip_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRIP )] - cost_transfer_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE]==PathSet.STATE_MODE_TRANSFER)] + cost_accegr_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) | ( + cost_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS)] + cost_trip_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRIP)] + cost_transfer_df = cost_df.loc[(cost_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_TRANSFER)] del cost_df ##################### First, handle Access/Egress link costs - for accegr_type in ["walk","bike","drive"]: + for accegr_type in ["walk", "bike", "drive"]: # make copies; we don't want to mess with originals if accegr_type == "walk": - link_df = tazs.walk_df.copy() + link_df = tazs.walk_df.copy() mode_list = TAZ.WALK_MODE_NUMS elif accegr_type == "bike": mode_list = TAZ.BIKE_MODE_NUMS # not supported yet continue else: - link_df = tazs.drive_df.copy() + link_df = tazs.drive_df.copy() mode_list = TAZ.DRIVE_MODE_NUMS FastTripsLogger.debug("Access/egress link_df %s\n%s" % (accegr_type, link_df.head().to_string())) @@ -933,146 +977,186 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri link_df.reset_index(inplace=True) link_df["A_id_num"] = -1 link_df["B_id_num"] = -1 - link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.ACCESS_MODE_NUMS), "A_id_num"] = link_df[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM ] - link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.ACCESS_MODE_NUMS), "B_id_num"] = link_df[TAZ.WALK_ACCESS_COLUMN_STOP_NUM] - link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "A_id_num"] = link_df[TAZ.WALK_ACCESS_COLUMN_STOP_NUM] - link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "B_id_num"] = link_df[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM ] + link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.ACCESS_MODE_NUMS), "A_id_num"] = \ + link_df[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM] + link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.ACCESS_MODE_NUMS), "B_id_num"] = \ + link_df[TAZ.WALK_ACCESS_COLUMN_STOP_NUM] + link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "A_id_num"] = \ + link_df[TAZ.WALK_ACCESS_COLUMN_STOP_NUM] + link_df.loc[link_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "B_id_num"] = \ + link_df[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM] link_df.drop([TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_STOP_NUM], axis=1, inplace=True) - assert(len(link_df.loc[link_df["A_id_num"] < 0]) == 0) + assert (len(link_df.loc[link_df["A_id_num"] < 0]) == 0) FastTripsLogger.debug("%s link_df =\n%s" % (accegr_type, link_df.head().to_string())) # Merge access/egress with walk|bike|drive access/egress information - cost_accegr_df = pd.merge(left = cost_accegr_df, - right = link_df, - on = ["A_id_num", - PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, - "B_id_num"], - how = "left") + cost_accegr_df = pd.merge(left=cost_accegr_df, + right=link_df, + on=["A_id_num", + PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, + "B_id_num"], + how="left") # rename new columns so it's clear it's for walk|bike|drive - for colname in list(link_df.select_dtypes(include=['float64','int64']).columns.values): + for colname in list(link_df.select_dtypes(include=['float64', 'int64']).columns.values): # don't worry about join columns if colname in ["A_id_num", PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM, "B_id_num"]: continue # rename the rest new_colname = "%s %s" % (colname, accegr_type) - cost_accegr_df.rename(columns={colname:new_colname}, inplace=True) + cost_accegr_df.rename(columns={colname: new_colname}, inplace=True) # use it, if relevant - cost_accegr_df.loc[ (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == colname)& - (cost_accegr_df[PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM].isin(mode_list)), "var_value"] = cost_accegr_df[new_colname] + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == colname) & + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_SUPPLY_MODE_NUM].isin( + mode_list)), "var_value"] = cost_accegr_df[new_colname] # Access/egress needs passenger trip departure, arrival and time_target - cost_accegr_df = pd.merge(left =cost_accegr_df, - right=trip_list_df[[ - Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME, - Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME, - Passenger.TRIP_LIST_COLUMN_TIME_TARGET, - ]], - how ="left", - on =[Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) + cost_accegr_df = pd.merge(left=cost_accegr_df, + right=trip_list_df[[ + Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME, + Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME, + Passenger.TRIP_LIST_COLUMN_TIME_TARGET, + ]], + how="left", + on=[Passenger.PERSONS_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]) # drop links that are irrelevant based on departure time for access links, or arrival time for egress links cost_accegr_df["check_time"] = cost_accegr_df[Assignment.SIM_COL_PAX_A_TIME] # departure time for access - cost_accegr_df.loc[ cost_accegr_df[TAZ.MODE_COLUMN_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "check_time" ] = cost_accegr_df[Assignment.SIM_COL_PAX_B_TIME] # arrival time for egress - cost_accegr_df["check_time"] = (cost_accegr_df["check_time"] - Assignment.NETWORK_BUILD_DATE_START_TIME)/np.timedelta64(1,'m') + cost_accegr_df.loc[cost_accegr_df[TAZ.MODE_COLUMN_MODE_NUM].isin(TAZ.EGRESS_MODE_NUMS), "check_time"] = \ + cost_accegr_df[Assignment.SIM_COL_PAX_B_TIME] # arrival time for egress + cost_accegr_df["check_time"] = (cost_accegr_df[ + "check_time"] - Assignment.NETWORK_BUILD_DATE_START_TIME) / np.timedelta64( + 1, 'm') # it's only drive links we need to check - cost_accegr_df["to_drop"] = False + cost_accegr_df["to_drop"] = False if "%s %s" % (TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN, "drive") in cost_accegr_df.columns.values: - cost_accegr_df.loc[ cost_accegr_df[TAZ.MODE_COLUMN_MODE_NUM].isin(TAZ.DRIVE_MODE_NUMS)& - ((cost_accegr_df["check_time"] < cost_accegr_df["%s %s" % (TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN, "drive")])| - (cost_accegr_df["check_time"] >= cost_accegr_df["%s %s" % (TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN, "drive")])), "to_drop"] = True + cost_accegr_df.loc[cost_accegr_df[TAZ.MODE_COLUMN_MODE_NUM].isin(TAZ.DRIVE_MODE_NUMS) & + ((cost_accegr_df["check_time"] < cost_accegr_df[ + "%s %s" % (TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN, "drive")]) | + (cost_accegr_df["check_time"] >= cost_accegr_df[ + "%s %s" % (TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN, "drive")])), "to_drop"] = True # if len(Assignment.TRACE_IDS) > 0: # FastTripsLogger.debug("cost_accegr_df=\n%s\ndtypes=\n%s" % (cost_accegr_df.loc[cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True]].to_string(), str(cost_accegr_df.dtypes))) FastTripsLogger.debug("Dropping %d rows from cost_accegr_df" % cost_accegr_df["to_drop"].sum()) - cost_accegr_df = cost_accegr_df.loc[ cost_accegr_df["to_drop"]==False ] - cost_accegr_df.drop(["check_time","to_drop"], axis=1, inplace=True) + cost_accegr_df = cost_accegr_df.loc[cost_accegr_df["to_drop"] == False] + cost_accegr_df.drop(["check_time", "to_drop"], axis=1, inplace=True) # penalty for arriving before preferred arrival time. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN )& - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS), "var_value"] = 0.0 - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN) & - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS) & - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'departure'), "var_value"] = 0.0 - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN )& \ - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS)& \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'arrival'), "var_value"] = (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME] - cost_accegr_df[Passenger.PF_COL_PAX_B_TIME])/np.timedelta64(1,'m') + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN) & + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN) & + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS) & + (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'departure'), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN) & \ + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS) & \ + (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'arrival'), "var_value"] = (cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME] - + cost_accegr_df[ + Passenger.PF_COL_PAX_B_TIME]) / np.timedelta64( + 1, 'm') # arrive early is not negative - that would be arriving late - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN)&(cost_accegr_df["var_value"] < 0), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_EARLY_MIN) & ( + cost_accegr_df["var_value"] < 0), "var_value"] = 0.0 # penalty for departing after preferred departure time. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS), "var_value"] = 0.0 - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN )& - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS)& + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS), "var_value"] = 0.0 + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) & (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'arrival'), "var_value"] = 0.0 - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) & - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'departure'), "var_value"] = (cost_accegr_df[Passenger.PF_COL_PAX_A_TIME] - cost_accegr_df[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME])/ np.timedelta64(1, 'm') + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) & + (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == 'departure'), "var_value"] = ( + cost_accegr_df[ + Passenger.PF_COL_PAX_A_TIME] - + cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME]) / np.timedelta64( + 1, 'm') # depart late is not negative - that would be departing early - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN)&(cost_accegr_df["var_value"] < 0), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_LATE_MIN) & ( + cost_accegr_df["var_value"] < 0), "var_value"] = 0.0 # constant growth = exponential growth with 0 percent growth rate # depart before preferred or arrive after preferred means the passenger just missed something important # Arrive late only impacts the egress link, so set the var_value equal to zero for the access link - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN ) & \ - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS), "var_value"] = 0.0 + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS), "var_value"] = 0.0 # Arrive late only impacts those that have a preferred arrival time. If preferred departure time, # set arrive late equal to zero. --This could have been done with previous line, but it would # look ugly mixing and matching 'and' and 'or'. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_DEPARTURE), "var_value"] = 0.0 + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ + (cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_DEPARTURE), "var_value"] = 0.0 # Calculate how late the person arrives after preferred time. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN )& \ - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS)& \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL), "var_value"] = \ - (cost_accegr_df[Passenger.PF_COL_PAX_B_TIME] - cost_accegr_df[Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME])/np.timedelta64(1,'m') + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS) & \ + (cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL), "var_value"] = \ + (cost_accegr_df[Passenger.PF_COL_PAX_B_TIME] - cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_ARRIVAL_TIME]) / np.timedelta64(1, 'm') # If arrived before preferred time, set the arrive late field to zero. You don't get a # discount for arriving early. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ + cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN) & \ (cost_accegr_df['var_value'] < 0), "var_value"] = 0 # preferred delay_min - departure means want to depart after that time # Depart early only impacts the access link, so set the var_value equal to zero for the egress link - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN )& \ - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_EGRESS), "var_value"] = 0.0 # Depart early only impacts those that have a preferred departure time. If preferred arrive time, # set depart early equal to zero. --This could have been done with previous line, but it would # look ugly mixing and matching 'and' and 'or'. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL), "var_value"] = 0.0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ + (cost_accegr_df[ + Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_ARRIVAL), "var_value"] = 0.0 # Calculate how early the person departs before the preferred time. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ - (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) & \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_DEPARTURE), "var_value"] = \ - (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME] - cost_accegr_df[Passenger.PF_COL_PAX_A_TIME])/ np.timedelta64(1, 'm') + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ + (cost_accegr_df[Passenger.PF_COL_LINK_MODE] == PathSet.STATE_MODE_ACCESS) & \ + (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TIME_TARGET] == Passenger.TIME_TARGET_DEPARTURE), "var_value"] = \ + (cost_accegr_df[Passenger.TRIP_LIST_COLUMN_DEPARTURE_TIME] - cost_accegr_df[ + Passenger.PF_COL_PAX_A_TIME]) / np.timedelta64(1, 'm') # If departing after preferred time, set the depart early field to zero. You don't get a # discount for taking your time. - cost_accegr_df.loc[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ - (cost_accegr_df['var_value'] < 0), "var_value"] = 0 + cost_accegr_df.loc[ + (cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == PathSet.WEIGHT_NAME_DEPART_EARLY_MIN) & \ + (cost_accegr_df['var_value'] < 0), "var_value"] = 0 - assert 0 == cost_accegr_df[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].isin([PathSet.WEIGHT_NAME_DEPART_EARLY_MIN, PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN])) & \ + assert 0 == cost_accegr_df[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].isin( + [PathSet.WEIGHT_NAME_DEPART_EARLY_MIN, PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN])) & \ (cost_accegr_df['var_value'].isnull())].shape[0] - assert 0 == cost_accegr_df[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].isin([PathSet.WEIGHT_NAME_DEPART_EARLY_MIN, PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN])) & \ - (cost_accegr_df['var_value']<0)].shape[0] + assert 0 == cost_accegr_df[(cost_accegr_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].isin( + [PathSet.WEIGHT_NAME_DEPART_EARLY_MIN, PathSet.WEIGHT_NAME_ARRIVE_LATE_MIN])) & \ + (cost_accegr_df['var_value'] < 0)].shape[0] if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("cost_accegr_df trace\n%s\ndtypes=\n%s" % (cost_accegr_df.loc[cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string(), str(cost_accegr_df.dtypes))) + FastTripsLogger.debug("cost_accegr_df trace\n%s\ndtypes=\n%s" % ( + cost_accegr_df.loc[cost_accegr_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string(), + str(cost_accegr_df.dtypes))) - missing_accegr_costs = cost_accegr_df.loc[ pd.isnull(cost_accegr_df["var_value"]) ] - error_accegr_msg = "Missing %d out of %d access/egress var_value values" % (len(missing_accegr_costs), len(cost_accegr_df)) + missing_accegr_costs = cost_accegr_df.loc[pd.isnull(cost_accegr_df["var_value"])] + error_accegr_msg = "Missing %d out of %d access/egress var_value values" % ( + len(missing_accegr_costs), len(cost_accegr_df)) FastTripsLogger.debug(error_accegr_msg) if len(missing_accegr_costs) > 0: @@ -1081,53 +1165,71 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri ##################### Next, handle Transit Trip link costs - # set the fare var_values for split_first only - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "fare")&(cost_trip_df["split_first"]==True), "var_value"] = cost_trip_df[Assignment.SIM_COL_PAX_FARE] - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "fare")&(cost_trip_df["split_first"]==False), "var_value"] = 0 + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "fare") & ( + cost_trip_df["split_first"] == True), "var_value"] = cost_trip_df[Assignment.SIM_COL_PAX_FARE] + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "fare") & ( + cost_trip_df["split_first"] == False), "var_value"] = 0 if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("cost_trip_df trace\n%s\ndtypes=\n%s" % (cost_trip_df.loc[cost_trip_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string(), str(cost_trip_df.dtypes))) + FastTripsLogger.debug("cost_trip_df trace\n%s\ndtypes=\n%s" % ( + cost_trip_df.loc[cost_trip_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string(), + str(cost_trip_df.dtypes))) # if there's a board time, in_vehicle_time = new_B_time - board_time # otherwise, in_vehicle_time = B time - A time (for when we split) - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min")&pd.notnull(cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ - (cost_trip_df[Assignment.SIM_COL_PAX_B_TIME] - cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME])/np.timedelta64(1,'m') - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min")& pd.isnull(cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ - (cost_trip_df[Assignment.SIM_COL_PAX_B_TIME] - cost_trip_df[Assignment.SIM_COL_PAX_A_TIME])/np.timedelta64(1,'m') + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min") & pd.notnull( + cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ + (cost_trip_df[Assignment.SIM_COL_PAX_B_TIME] - cost_trip_df[ + Assignment.SIM_COL_PAX_BOARD_TIME]) / np.timedelta64(1, 'm') + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min") & pd.isnull( + cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ + (cost_trip_df[Assignment.SIM_COL_PAX_B_TIME] - cost_trip_df[ + Assignment.SIM_COL_PAX_A_TIME]) / np.timedelta64(1, 'm') # if in vehicle time is less than 0 then off by 1 day error - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min")&(cost_trip_df["var_value"]<0), "var_value"] = cost_trip_df["var_value"] + (24*60) + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "in_vehicle_time_min") & ( + cost_trip_df["var_value"] < 0), "var_value"] = cost_trip_df["var_value"] + (24 * 60) # if there's a board time, wait time = board_time - A time # otherwise, wait time = 0 (for when we split transit links) - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "wait_time_min")&pd.notnull(cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ - (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME] - cost_trip_df[Assignment.SIM_COL_PAX_A_TIME])/np.timedelta64(1,'m') - cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "wait_time_min")& pd.isnull(cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = 0 + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "wait_time_min") & pd.notnull( + cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = \ + (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME] - cost_trip_df[ + Assignment.SIM_COL_PAX_A_TIME]) / np.timedelta64(1, 'm') + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "wait_time_min") & pd.isnull( + cost_trip_df[Assignment.SIM_COL_PAX_BOARD_TIME]), "var_value"] = 0 # which overcap column to use? overcap_col = Trip.SIM_COL_VEH_OVERCAP - if Assignment.MSA_RESULTS and Trip.SIM_COL_VEH_MSA_OVERCAP in list(cost_trip_df.columns.values): overcap_col = Trip.SIM_COL_VEH_MSA_OVERCAP + if Assignment.MSA_RESULTS and Trip.SIM_COL_VEH_MSA_OVERCAP in list( + cost_trip_df.columns.values): overcap_col = Trip.SIM_COL_VEH_MSA_OVERCAP # at cap is a binary, 1 if overcap >= 0 and they're not one of the lucky few that boarded cost_trip_df["at_capacity"] = 0.0 if Assignment.SIM_COL_PAX_BOARD_STATE in list(cost_trip_df.columns.values): - cost_trip_df.loc[ (cost_trip_df[overcap_col] >= 0)& - (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_STATE] != "board_easy")& - (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_STATE] != "boarded"), "at_capacity" ] = 1.0 + cost_trip_df.loc[(cost_trip_df[overcap_col] >= 0) & + (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_STATE] != "board_easy") & + (cost_trip_df[Assignment.SIM_COL_PAX_BOARD_STATE] != "boarded"), "at_capacity"] = 1.0 else: - cost_trip_df.loc[ (cost_trip_df[overcap_col] >= 0) , "at_capacity" ] = 1.0 + cost_trip_df.loc[(cost_trip_df[overcap_col] >= 0), "at_capacity"] = 1.0 - cost_trip_df.loc[cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "at_capacity" , "var_value"] = cost_trip_df["at_capacity"] - cost_trip_df.loc[cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "overcap" , "var_value"] = cost_trip_df[overcap_col] + cost_trip_df.loc[cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "at_capacity", "var_value"] = cost_trip_df[ + "at_capacity"] + cost_trip_df.loc[cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "overcap", "var_value"] = cost_trip_df[ + overcap_col] # overcap shouldn't be negative - cost_trip_df.loc[ (cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "overcap")&(cost_trip_df["var_value"]<0), "var_value"] = 0.0 + cost_trip_df.loc[(cost_trip_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "overcap") & ( + cost_trip_df["var_value"] < 0), "var_value"] = 0.0 if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("cost_trip_df trace\n%s\ndtypes=\n%s" % (cost_trip_df.loc[cost_trip_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True].to_string(), str(cost_trip_df.dtypes))) + FastTripsLogger.debug("cost_trip_df trace\n%s\ndtypes=\n%s" % ( + cost_trip_df.loc[cost_trip_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True].to_string(), + str(cost_trip_df.dtypes))) - missing_trip_costs = cost_trip_df.loc[ pd.isnull(cost_trip_df["var_value"]) ] - error_trip_msg = "Missing %d out of %d transit trip var_value values" % (len(missing_trip_costs), len(cost_trip_df)) + missing_trip_costs = cost_trip_df.loc[pd.isnull(cost_trip_df["var_value"])] + error_trip_msg = "Missing %d out of %d transit trip var_value values" % ( + len(missing_trip_costs), len(cost_trip_df)) FastTripsLogger.debug(error_trip_msg) if len(missing_trip_costs) > 0: @@ -1136,24 +1238,27 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri ##################### Finally, handle Transfer link costs cost_transfer_df = transfers.add_transfer_attributes(cost_transfer_df, pathset_links_df) - cost_transfer_df.loc[cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "walk_time_min", "var_value"] = cost_transfer_df[Passenger.PF_COL_LINK_TIME]/np.timedelta64(1,'m') + cost_transfer_df.loc[cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "walk_time_min", "var_value"] = \ + cost_transfer_df[Passenger.PF_COL_LINK_TIME] / np.timedelta64(1, 'm') # any numeric column can be used - for colname in list(cost_transfer_df.select_dtypes(include=['float64','int64']).columns.values): + for colname in list(cost_transfer_df.select_dtypes(include=['float64', 'int64']).columns.values): FastTripsLogger.debug("Using numeric column %s" % colname) - cost_transfer_df.loc[cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == colname, "var_value"] = cost_transfer_df[colname] + cost_transfer_df.loc[cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == colname, "var_value"] = \ + cost_transfer_df[colname] # make zero walk transfers have default var_values 0 - cost_transfer_df.loc[ (cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] != "transfer_penalty")& - (cost_transfer_df["A_id_num"]==cost_transfer_df["B_id_num"]), "var_value"] = 0.0 + cost_transfer_df.loc[(cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] != "transfer_penalty") & + (cost_transfer_df["A_id_num"] == cost_transfer_df["B_id_num"]), "var_value"] = 0.0 # zero walk transfers have a transfer penalty although they're not otherwise configured - cost_transfer_df.loc[ (cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "transfer_penalty")& - (pd.isnull(cost_transfer_df["var_value"])), "var_value"] = 1.0 + cost_transfer_df.loc[(cost_transfer_df[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == "transfer_penalty") & + (pd.isnull(cost_transfer_df["var_value"])), "var_value"] = 1.0 # FastTripsLogger.debug("cost_transfer_df=\n%s\ndtypes=\n%s" % (cost_transfer_df.head().to_string(), str(cost_transfer_df.dtypes))) - missing_transfer_costs = cost_transfer_df.loc[ pd.isnull(cost_transfer_df["var_value"]) ] - error_transfer_msg = "Missing %d out of %d transfer var_value values" % (len(missing_transfer_costs), len(cost_transfer_df)) + missing_transfer_costs = cost_transfer_df.loc[pd.isnull(cost_transfer_df["var_value"])] + error_transfer_msg = "Missing %d out of %d transfer var_value values" % ( + len(missing_transfer_costs), len(cost_transfer_df)) FastTripsLogger.debug(error_transfer_msg) if len(missing_transfer_costs) > 0: @@ -1162,7 +1267,8 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri # abort here if we're missing anything if len(missing_accegr_costs) + len(missing_trip_costs) + len(missing_transfer_costs) > 0: - abort_error_msg = "\nMissing %d accegr_costs\nMissing %d trip_costs\nMissing %d transfer_costs" % (len(missing_accegr_costs), len(missing_trip_costs), len(missing_transfer_costs)) + abort_error_msg = "\nMissing %d accegr_costs\nMissing %d trip_costs\nMissing %d transfer_costs" % ( + len(missing_accegr_costs), len(missing_trip_costs), len(missing_transfer_costs)) FastTripsLogger.debug(abort_error_msg) raise NotImplementedError("Missing var_values; See log") @@ -1190,10 +1296,10 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri Assignment.SIM_COL_MISSED_XFER, Assignment.SIM_COL_PAX_BUMP_ITER, Assignment.SIM_COL_PAX_FARE] - cost_accegr_df = cost_accegr_df.loc[:, cost_accegr_df.columns.isin(cost_columns)] - cost_trip_df = cost_trip_df.loc[:, cost_trip_df.columns.isin(cost_columns)] + cost_accegr_df = cost_accegr_df.loc[:, cost_accegr_df.columns.isin(cost_columns)] + cost_trip_df = cost_trip_df.loc[:, cost_trip_df.columns.isin(cost_columns)] cost_transfer_df = cost_transfer_df.loc[:, cost_transfer_df.columns.isin(cost_columns)] - cost_df = pd.concat([cost_accegr_df, cost_trip_df, cost_transfer_df], axis=0, ignore_index=True) + cost_df = pd.concat([cost_accegr_df, cost_trip_df, cost_transfer_df], axis=0, ignore_index=True) # FastTripsLogger.debug("calculate_cost: cost_df=\n%s\ndtypes=\n%s" % (cost_df.to_string(), str(cost_df.dtypes))) @@ -1204,11 +1310,13 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM, Passenger.PF_COL_LINK_NUM], inplace=True) - FastTripsLogger.debug("calculate_cost: cost_df\n%s" % str(cost_df.loc[cost_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug( + "calculate_cost: cost_df\n%s" % str(cost_df.loc[cost_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # verify all costs are non-negative if cost_df[Assignment.SIM_COL_PAX_COST].min() < 0: - msg = "calculate_cost: Negative costs found:\n%s" % cost_df.loc[ cost_df[Assignment.SIM_COL_PAX_COST]<0 ].to_string() + msg = "calculate_cost: Negative costs found:\n%s" % cost_df.loc[ + cost_df[Assignment.SIM_COL_PAX_COST] < 0].to_string() FastTripsLogger.fatal(msg) raise UnexpectedError(msg) @@ -1220,26 +1328,28 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri Passenger.PF_COL_PATH_NUM, Passenger.PF_COL_LINK_NUM, Assignment.SIM_COL_PAX_COST]].groupby( - [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this first - Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]).aggregate('sum').reset_index() + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this first + Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRACE, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]).aggregate('sum').reset_index() if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: cost_link_df trace\n%s" % str(cost_link_df.loc[cost_link_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: cost_link_df trace\n%s" % str( + cost_link_df.loc[cost_link_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # join to pathset_links_df - pathset_links_df = pd.merge(left =pathset_links_df, - right=cost_link_df, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]) + pathset_links_df = pd.merge(left=pathset_links_df, + right=cost_link_df, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.TRIP_LIST_COLUMN_TRACE, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]) if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: pathset_links_df trace\n%s" % str(pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: pathset_links_df trace\n%s" % str( + pathset_links_df.loc[pathset_links_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) ###################### overlap calcs full_overlap_df = None @@ -1248,62 +1358,70 @@ def calculate_cost(STOCH_DISPERSION, pathset_paths_df, pathset_links_df, veh_tri ###################### sum linkcost to paths cost_link_df.drop([Passenger.PF_COL_LINK_NUM], axis=1, inplace=True) - cost_path_df = cost_link_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # order by this + cost_path_df = cost_link_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # order by this Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.TRIP_LIST_COLUMN_TRACE, Passenger.PF_COL_PATH_NUM]).aggregate('sum').reset_index() if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: cost_path_df trace\n%s" % str(cost_path_df.loc[cost_path_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: cost_path_df trace\n%s" % str( + cost_path_df.loc[cost_path_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # join to pathset_paths_df - pathset_paths_df = pd.merge(left =pathset_paths_df, - right=cost_path_df, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM]) + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=cost_path_df, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.TRIP_LIST_COLUMN_TRACE, + Passenger.PF_COL_PATH_NUM]) if PathSet.OVERLAP_VARIABLE == PathSet.OVERLAP_NONE: pathset_paths_df[Assignment.SIM_COL_PAX_LNPS] = 0 else: - pathset_paths_df = pd.merge(left =pathset_paths_df, - right=full_overlap_df, - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM]) + pathset_paths_df = pd.merge(left=pathset_paths_df, + right=full_overlap_df, + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.TRIP_LIST_COLUMN_TRACE, + Passenger.PF_COL_PATH_NUM]) if PathSet.LEARN_ROUTES: - #'learn_discount': Exponential decay function - pathset_paths_df['learn_discount'] = np.exp(-PathSet.LEARN_ROUTES_RATE * pathset_paths_df[PathSet.SUCCESS_FLAG_COLUMN]) + # 'learn_discount': Exponential decay function + pathset_paths_df['learn_discount'] = np.exp( + -PathSet.LEARN_ROUTES_RATE * pathset_paths_df[PathSet.SUCCESS_FLAG_COLUMN]) pathset_paths_df['orig_sim_cost'] = pathset_paths_df[Assignment.SIM_COL_PAX_COST] - pathset_paths_df[Assignment.SIM_COL_PAX_COST] = pathset_paths_df[Assignment.SIM_COL_PAX_COST] * pathset_paths_df['learn_discount'] + pathset_paths_df[Assignment.SIM_COL_PAX_COST] = pathset_paths_df[Assignment.SIM_COL_PAX_COST] * \ + pathset_paths_df['learn_discount'] if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: pathset_paths_df trace\n%s" % str(pathset_paths_df.loc[pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: pathset_paths_df trace\n%s" % str( + pathset_paths_df.loc[pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) ###################### logsum and probabilities - pathset_paths_df["logsum_component"] = np.exp((-1.0*pathset_paths_df[Assignment.SIM_COL_PAX_COST] + pathset_paths_df[Assignment.SIM_COL_PAX_LNPS])/STOCH_DISPERSION) + pathset_paths_df["logsum_component"] = np.exp((-1.0 * pathset_paths_df[Assignment.SIM_COL_PAX_COST] + + pathset_paths_df[ + Assignment.SIM_COL_PAX_LNPS]) / STOCH_DISPERSION) # sum across all paths pathset_logsum_df = pathset_paths_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM,"logsum_component"]].groupby( - [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this - Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).aggregate('sum').reset_index() - pathset_logsum_df.rename(columns={"logsum_component":"logsum"}, inplace=True) + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, "logsum_component"]].groupby( + [Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this + Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID]).aggregate('sum').reset_index() + pathset_logsum_df.rename(columns={"logsum_component": "logsum"}, inplace=True) pathset_paths_df = pd.merge(left=pathset_paths_df, - right=pathset_logsum_df, - how="left") - pathset_paths_df[Assignment.SIM_COL_PAX_PROBABILITY] = pathset_paths_df["logsum_component"]/pathset_paths_df["logsum"] + right=pathset_logsum_df, + how="left") + pathset_paths_df[Assignment.SIM_COL_PAX_PROBABILITY] = pathset_paths_df["logsum_component"] / pathset_paths_df[ + "logsum"] if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_cost: pathset_paths_df trace\n%s" % str(pathset_paths_df.loc[pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_cost: pathset_paths_df trace\n%s" % str( + pathset_paths_df.loc[pathset_paths_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # Note: the path finding costs won't match the costs here because missed transfers are already calculated here # It would be good to have some sanity checking that theyre aligned otherwise though to make sure we're @@ -1319,26 +1437,31 @@ def calculate_overlap(pathset_links_to_use): """ from .Assignment import Assignment - FastTripsLogger.debug("calculate_overlap() pathset_links_to_use (%d) head=\n%s" % (len(pathset_links_to_use), str(pathset_links_to_use.head(30)))) + FastTripsLogger.debug("calculate_overlap() pathset_links_to_use (%d) head=\n%s" % ( + len(pathset_links_to_use), str(pathset_links_to_use.head(30)))) # CHUNKING because we run into memory problems # TODO: figure out more sophisticated chunk size - chunk_list = pathset_links_to_use[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]].drop_duplicates().reset_index(drop=True) - num_chunks = (len(chunk_list)//PathSet.OVERLAP_CHUNK_SIZE) + 1 + chunk_list = pathset_links_to_use[[Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM]].drop_duplicates().reset_index( + drop=True) + num_chunks = (len(chunk_list) // PathSet.OVERLAP_CHUNK_SIZE) + 1 chunk_list["chunk_num"] = np.floor_divide(chunk_list.index, PathSet.OVERLAP_CHUNK_SIZE) - FastTripsLogger.debug("calculate_overlap() chunk_list size=%d head=\n%s\ntail=\n%s" % (len(chunk_list), chunk_list.head().to_string(), chunk_list.tail().to_string())) - pathset_links_to_use = pd.merge(left =pathset_links_to_use, - right =chunk_list, - how ='left') - FastTripsLogger.debug("calculate_overlap() mem_use=%s pathset_links_to_use has length %d, head=\n%s" % (Util.get_process_mem_use_str(), - len(pathset_links_to_use), pathset_links_to_use.head().to_string())) + FastTripsLogger.debug("calculate_overlap() chunk_list size=%d head=\n%s\ntail=\n%s" % ( + len(chunk_list), chunk_list.head().to_string(), chunk_list.tail().to_string())) + pathset_links_to_use = pd.merge(left=pathset_links_to_use, + right=chunk_list, + how='left') + FastTripsLogger.debug("calculate_overlap() mem_use=%s pathset_links_to_use has length %d, head=\n%s" % ( + Util.get_process_mem_use_str(), + len(pathset_links_to_use), pathset_links_to_use.head().to_string())) full_overlap_df = pd.DataFrame() for chunk_num in range(num_chunks): # get the person trips in the chunk - overlap_df = pathset_links_to_use.loc[ pathset_links_to_use["chunk_num"] == chunk_num] + overlap_df = pathset_links_to_use.loc[pathset_links_to_use["chunk_num"] == chunk_num] - FastTripsLogger.info(" Calculating overlap for chunk %4d/%4d (len %6d); mem_use=%8s" % (chunk_num+1, num_chunks, len(overlap_df), Util.get_process_mem_use_str())) + FastTripsLogger.info(" Calculating overlap for chunk %4d/%4d (len %6d); mem_use=%8s" % ( + chunk_num + 1, num_chunks, len(overlap_df), Util.get_process_mem_use_str())) overlap_df = overlap_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, @@ -1346,67 +1469,74 @@ def calculate_overlap(pathset_links_to_use): Passenger.TRIP_LIST_COLUMN_TRACE, Passenger.PF_COL_PATH_NUM, Passenger.PF_COL_LINK_NUM, - "A_id_num","B_id_num", + "A_id_num", "B_id_num", Route.ROUTES_COLUMN_MODE, "new_linktime", Assignment.SIM_COL_PAX_DISTANCE]].copy() # get ready to count, time, dist to path and add path sum version to overlap_df -- this is L FastTripsLogger.debug("calculate_overlap() chunk_num %d: mem_use=%s overlap_df has length %d, head=\n%s" % - (chunk_num+1, Util.get_process_mem_use_str(), len(overlap_df), overlap_df.head().to_string())) + (chunk_num + 1, Util.get_process_mem_use_str(), len(overlap_df), + overlap_df.head().to_string())) # sum to path overlap_df["count"] = 1 - overlap_path_df = overlap_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this + overlap_path_df = overlap_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # sort by this Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM]).aggregate({'count':'sum','new_linktime':'sum',Assignment.SIM_COL_PAX_DISTANCE:'sum'}).reset_index(drop=False) - overlap_path_df.rename(columns={"count":"path_count", "new_linktime":"path_time", Assignment.SIM_COL_PAX_DISTANCE:"path_distance"}, inplace=True) + Passenger.PF_COL_PATH_NUM]).aggregate( + {'count': 'sum', 'new_linktime': 'sum', Assignment.SIM_COL_PAX_DISTANCE: 'sum'}).reset_index(drop=False) + overlap_path_df.rename(columns={"count": "path_count", "new_linktime": "path_time", + Assignment.SIM_COL_PAX_DISTANCE: "path_distance"}, inplace=True) overlap_df.drop(["count"], axis=1, inplace=True) - FastTripsLogger.debug("calculate_overlap() chunk_num %d: Added path summed count,time,distance. mem_use=%s overlap_path_df has length %d, head=\n%s" % - (chunk_num+1, Util.get_process_mem_use_str(), len(overlap_path_df), overlap_path_df.head().to_string())) + FastTripsLogger.debug( + "calculate_overlap() chunk_num %d: Added path summed count,time,distance. mem_use=%s overlap_path_df has length %d, head=\n%s" % + (chunk_num + 1, Util.get_process_mem_use_str(), len(overlap_path_df), + overlap_path_df.head().to_string())) # add the path summed variables to the link dataframe overlap_df = pd.merge(overlap_df, overlap_path_df, - how="left", - on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.TRIP_LIST_COLUMN_TRACE, - Passenger.PF_COL_PATH_NUM]) - del overlap_path_df # we're done with this + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.TRIP_LIST_COLUMN_TRACE, + Passenger.PF_COL_PATH_NUM]) + del overlap_path_df # we're done with this # outer join on trip_list_id_num means when they match, we'll get a cartesian product of the links overlap_df = pd.merge(overlap_df, overlap_df.copy(), - on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, - Passenger.TRIP_LIST_COLUMN_TRACE], how="outer") + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, + Passenger.TRIP_LIST_COLUMN_TRACE], how="outer") # count matches -- matching A,B,mode overlap_df["match"] = 0 - overlap_df.loc[ (overlap_df["A_id_num_x"]==overlap_df["A_id_num_y"])& - (overlap_df["B_id_num_x"]==overlap_df["B_id_num_y"])& - (overlap_df["mode_x" ]==overlap_df["mode_y" ]) , "match"] = 1 + overlap_df.loc[(overlap_df["A_id_num_x"] == overlap_df["A_id_num_y"]) & + (overlap_df["B_id_num_x"] == overlap_df["B_id_num_y"]) & + (overlap_df["mode_x"] == overlap_df["mode_y"]), "match"] = 1 if PathSet.OVERLAP_VARIABLE == PathSet.OVERLAP_COUNT: - overlap_df["link_prop_x"] = 1.0/overlap_df["path_count_x"] # l_a/L_i - overlap_df["pathlen_x_y"] = overlap_df["path_count_x"]/overlap_df["path_count_y"] # L_i/L_j + overlap_df["link_prop_x"] = 1.0 / overlap_df["path_count_x"] # l_a/L_i + overlap_df["pathlen_x_y"] = overlap_df["path_count_x"] / overlap_df["path_count_y"] # L_i/L_j elif PathSet.OVERLAP_VARIABLE == PathSet.OVERLAP_TIME: - overlap_df["link_prop_x"] = overlap_df["new_linktime_x"]/overlap_df["path_time_x"] # l_a/L_i - overlap_df["pathlen_x_y"] = overlap_df["path_time_x"]/overlap_df["path_time_y"] # L_i/L_j + overlap_df["link_prop_x"] = overlap_df["new_linktime_x"] / overlap_df["path_time_x"] # l_a/L_i + overlap_df["pathlen_x_y"] = overlap_df["path_time_x"] / overlap_df["path_time_y"] # L_i/L_j elif PathSet.OVERLAP_VARIABLE == PathSet.OVERLAP_DISTANCE: - overlap_df["link_prop_x"] = overlap_df["distance_x"]/overlap_df["path_distance_x"] # l_a/L_i - overlap_df["pathlen_x_y"] = overlap_df["path_distance_x"]/overlap_df["path_distance_y"] # L_i/L_j + overlap_df["link_prop_x"] = overlap_df["distance_x"] / overlap_df["path_distance_x"] # l_a/L_i + overlap_df["pathlen_x_y"] = overlap_df["path_distance_x"] / overlap_df["path_distance_y"] # L_i/L_j - overlap_df["pathlen_x_y_scale"] = overlap_df[["pathlen_x_y"]].pow(PathSet.OVERLAP_SCALE_PARAMETER) # (L_i/L_j)^gamma + overlap_df["pathlen_x_y_scale"] = overlap_df[["pathlen_x_y"]].pow( + PathSet.OVERLAP_SCALE_PARAMETER) # (L_i/L_j)^gamma # zero it out if it's not a match - overlap_df.loc[overlap_df["match"]==0, "pathlen_x_y_scale"] = 0 + overlap_df.loc[overlap_df["match"] == 0, "pathlen_x_y_scale"] = 0 # now pathlen_x_y_scale = (L_i/L_j)^gamma x delta_aj if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_overlap(): overlap_df trace\n%s" % str(overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_overlap(): overlap_df trace\n%s" % str( + overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # debug # overlap_df_temp = overlap_df.groupby([Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, "pathnum_x","linknum_x","link_prop_x","pathnum_y"]).aggregate({"match":"sum", "pathlen_x_y_scale":"sum"}) @@ -1417,43 +1547,52 @@ def calculate_overlap(pathset_links_to_use): Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.TRIP_LIST_COLUMN_TRACE, - "pathnum_x","linknum_x","link_prop_x"]).aggregate({"pathlen_x_y_scale":"sum"}).reset_index() + "pathnum_x", "linknum_x", "link_prop_x"]).aggregate( + {"pathlen_x_y_scale": "sum"}).reset_index() # now pathlen_x_y_scale = SUM_j (L_i/L_j)^gamma x delta_aj - overlap_df["PS"] = overlap_df["link_prop_x"]/overlap_df["pathlen_x_y_scale"] # l_a/L_i * 1/(SUM_j (L_i/L_j)^gamma x delta_aj) + overlap_df["PS"] = overlap_df["link_prop_x"] / overlap_df[ + "pathlen_x_y_scale"] # l_a/L_i * 1/(SUM_j (L_i/L_j)^gamma x delta_aj) if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_overlap() trace overlap_df\n%s" % str(overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_overlap() trace overlap_df\n%s" % str( + overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) - FastTripsLogger.debug("calculate_overlap() chunk_num %d: Cartesian product of links. mem_use=%s overlap_df has length %d, head=\n%s" % - (chunk_num+1, Util.get_process_mem_use_str(), len(overlap_df), overlap_df.head().to_string())) + FastTripsLogger.debug( + "calculate_overlap() chunk_num %d: Cartesian product of links. mem_use=%s overlap_df has length %d, head=\n%s" % + (chunk_num + 1, Util.get_process_mem_use_str(), len(overlap_df), overlap_df.head().to_string())) # sum across link in path - overlap_df = overlap_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # preserve ordering + overlap_df = overlap_df.groupby([Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM, # preserve ordering Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.TRIP_LIST_COLUMN_TRACE,"pathnum_x"]).aggregate({"PS":"sum"}).reset_index(drop=False) + Passenger.TRIP_LIST_COLUMN_TRACE, "pathnum_x"]).aggregate( + {"PS": "sum"}).reset_index(drop=False) # Check all pathsizes are in [0,1] min_PS = overlap_df["PS"].min() max_PS = overlap_df["PS"].max() FastTripsLogger.debug("PathSize min=%f max=%f" % (min_PS, max_PS)) if min_PS < 0: - FastTripsLogger.fatal("Min pathsize = %f < 0:\n%s" % (min_PS, overlap_df.loc[overlap_df["PS"]==min_PS].to_string())) + FastTripsLogger.fatal( + "Min pathsize = %f < 0:\n%s" % (min_PS, overlap_df.loc[overlap_df["PS"] == min_PS].to_string())) if max_PS > 1.0001: - FastTripsLogger.fatal("Max pathsize = %f > 1:\n%s" % (max_PS, overlap_df.loc[overlap_df["PS"]==max_PS].to_string())) + FastTripsLogger.fatal( + "Max pathsize = %f > 1:\n%s" % (max_PS, overlap_df.loc[overlap_df["PS"] == max_PS].to_string())) overlap_df[Assignment.SIM_COL_PAX_LNPS] = np.log(overlap_df["PS"]) if len(Assignment.TRACE_IDS) > 0: - FastTripsLogger.debug("calculate_overlap() overlap_df trace\n%s" % str(overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE]==True])) + FastTripsLogger.debug("calculate_overlap() overlap_df trace\n%s" % str( + overlap_df.loc[overlap_df[Passenger.TRIP_LIST_COLUMN_TRACE] == True])) # rename pathnum_x to pathnum and drop PS. Now overlap_df has columns trip_list_id_num, pathnum, ln_PS - overlap_df.rename(columns={"pathnum_x":Passenger.PF_COL_PATH_NUM}, inplace=True) - overlap_df.drop(["PS"], axis=1, inplace=True) # we have ln_PS + overlap_df.rename(columns={"pathnum_x": Passenger.PF_COL_PATH_NUM}, inplace=True) + overlap_df.drop(["PS"], axis=1, inplace=True) # we have ln_PS if len(full_overlap_df) == 0: full_overlap_df = overlap_df else: full_overlap_df = full_overlap_df.append(overlap_df) - FastTripsLogger.debug("calculate_overlap() chunk_num %d: mem_use=%s full_overlap_df has length %d" % (chunk_num+1, Util.get_process_mem_use_str(), len(full_overlap_df))) + FastTripsLogger.debug("calculate_overlap() chunk_num %d: mem_use=%s full_overlap_df has length %d" % ( + chunk_num + 1, Util.get_process_mem_use_str(), len(full_overlap_df))) FastTripsLogger.debug("calculate_overlap() complete: full_overlap_df head=\n%s" % full_overlap_df.head(30)) diff --git a/fasttrips/Performance.py b/fasttrips/Performance.py index dfb4cbb1..09d898a9 100644 --- a/fasttrips/Performance.py +++ b/fasttrips/Performance.py @@ -2,7 +2,7 @@ from builtins import object __copyright__ = "Copyright 2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -15,13 +15,14 @@ See the License for the specific language governing permissions and limitations under the License. """ + import datetime import os import pandas as pd from .Passenger import Passenger -from .Util import Util +from .Util import Util class Performance(object): @@ -31,103 +32,103 @@ class Performance(object): in Fast-Trips and for the the bigger loops. """ #: Performance column: Iteration - PERFORMANCE_PF_COL_ITERATION = "iteration" + PERFORMANCE_PF_COL_ITERATION = "iteration" #: Performance column: Pathfinding Iteration - PERFORMANCE_PF_COL_PATHFINDING_ITERATION = "pathfinding_iteration" + PERFORMANCE_PF_COL_PATHFINDING_ITERATION = "pathfinding_iteration" #: Performance column: Person ID - PERFORMANCE_PF_COL_PERSON_ID = Passenger.TRIP_LIST_COLUMN_PERSON_ID + PERFORMANCE_PF_COL_PERSON_ID = Passenger.TRIP_LIST_COLUMN_PERSON_ID #: Performance column: Person Trip ID - PERFORMANCE_PF_COL_PERSON_TRIP_ID = Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID + PERFORMANCE_PF_COL_PERSON_TRIP_ID = Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID #: Performance column: Process number - PERFORMANCE_PF_COL_PROCESS_NUM = "process number" + PERFORMANCE_PF_COL_PROCESS_NUM = "process number" #: Performance column: Pathfinding status - PERFORMANCE_PF_COL_PATHFINDING_STATUS = "pathfinding_status" + PERFORMANCE_PF_COL_PATHFINDING_STATUS = "pathfinding_status" #: Performance column: Number of label iterations - PERFORMANCE_PF_COL_LABEL_ITERATIONS = "label iterations" + PERFORMANCE_PF_COL_LABEL_ITERATIONS = "label iterations" #: Performance column: Number of labeled stops - PERFORMANCE_PF_COL_NUM_LABELED_STOPS = "num labeled stops" + PERFORMANCE_PF_COL_NUM_LABELED_STOPS = "num labeled stops" #: Performance column: Maximum number of times a stop was processed PERFORMANCE_PF_COL_MAX_STOP_PROCESS_COUNT = "max stop process count" #: Performance column: Time spent labeling (timedelta) - PERFORMANCE_PF_COL_TIME_LABELING = "time labeling" + PERFORMANCE_PF_COL_TIME_LABELING = "time labeling" #: Performance column: Time spent labeling (milliseconds) - PERFORMANCE_PF_COL_TIME_LABELING_MS = "time labeling milliseconds" + PERFORMANCE_PF_COL_TIME_LABELING_MS = "time labeling milliseconds" #: Performance column: Time spent enumerating (timedelta) - PERFORMANCE_PF_COL_TIME_ENUMERATING = "time enumerating" + PERFORMANCE_PF_COL_TIME_ENUMERATING = "time enumerating" #: Performance column: Time spent enumerating (milliseconds) - PERFORMANCE_PF_COL_TIME_ENUMERATING_MS = "time enumerating milliseconds" + PERFORMANCE_PF_COL_TIME_ENUMERATING_MS = "time enumerating milliseconds" #: Performance column: Traced, since this affects performance - PERFORMANCE_PF_COL_TRACED = "traced" + PERFORMANCE_PF_COL_TRACED = "traced" #: Performance column: Working set in memory, in bytes - PERFORMANCE_PF_COL_WORKING_SET_BYTES = "working set bytes" + PERFORMANCE_PF_COL_WORKING_SET_BYTES = "working set bytes" #: Performance column: Private usage in memory, in bytes - PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES = "private usage bytes" + PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES = "private usage bytes" #: Performance column: Timestamp of memory query, in a datetime.datetime - PERFORMANCE_PF_COL_MEM_TIMESTAMP = "mem_timestamp" + PERFORMANCE_PF_COL_MEM_TIMESTAMP = "mem_timestamp" #: File to write performance results - OUTPUT_PERFORMANCE_PF_FILE = 'ft_output_performance_pathfinding.csv' + OUTPUT_PERFORMANCE_PF_FILE = 'ft_output_performance_pathfinding.csv' #: For general performance (not pathfinding) #: Performance column: Step name (e.g. read inputs). String. - PERFORMANCE_COL_STEP_NAME = "step_name" + PERFORMANCE_COL_STEP_NAME = "step_name" #: Performance column: Iteration. Integer. - PERFORMANCE_COL_ITERATION = PERFORMANCE_PF_COL_ITERATION + PERFORMANCE_COL_ITERATION = PERFORMANCE_PF_COL_ITERATION #: Performance column: Pathfinding Iteration. Integer. - PERFORMANCE_COL_PATHFINDING_ITERATION = PERFORMANCE_PF_COL_PATHFINDING_ITERATION + PERFORMANCE_COL_PATHFINDING_ITERATION = PERFORMANCE_PF_COL_PATHFINDING_ITERATION #: Performance column: Simulation Iteration. Integer. - PERFORMANCE_COL_SIMULATION_ITERATION = "simulation_iteration" + PERFORMANCE_COL_SIMULATION_ITERATION = "simulation_iteration" #: Performance column: step start time - PERFORMANCE_COL_START_TIME = "start_time" - PERFORMANCE_COL_END_TIME = "end_time" - PERFORMANCE_COL_STEP_DURATION = "step_duration" - PERFORMANCE_COL_START_MEM_MB = "start_mem_MB" - PERFORMANCE_COL_END_MEM_MB = "end_mem_MB" + PERFORMANCE_COL_START_TIME = "start_time" + PERFORMANCE_COL_END_TIME = "end_time" + PERFORMANCE_COL_STEP_DURATION = "step_duration" + PERFORMANCE_COL_START_MEM_MB = "start_mem_MB" + PERFORMANCE_COL_END_MEM_MB = "end_mem_MB" #: File to write performance results - OUTPUT_PERFORMANCE_FILE = 'ft_output_performance.csv' + OUTPUT_PERFORMANCE_FILE = 'ft_output_performance.csv' def __init__(self): """ Constructor. Initialize empty dataframe for performance info. """ self.performance_pf_dict = { - Performance.PERFORMANCE_PF_COL_ITERATION :[], - Performance.PERFORMANCE_PF_COL_PATHFINDING_ITERATION :[], - Performance.PERFORMANCE_PF_COL_PERSON_ID :[], - Performance.PERFORMANCE_PF_COL_PERSON_TRIP_ID :[], - Performance.PERFORMANCE_PF_COL_PROCESS_NUM :[], - Performance.PERFORMANCE_PF_COL_PATHFINDING_STATUS :[], - Performance.PERFORMANCE_PF_COL_NUM_LABELED_STOPS :[], - Performance.PERFORMANCE_PF_COL_TRACED :[], - Performance.PERFORMANCE_PF_COL_LABEL_ITERATIONS :[], - Performance.PERFORMANCE_PF_COL_MAX_STOP_PROCESS_COUNT :[], - Performance.PERFORMANCE_PF_COL_TIME_LABELING :[], - Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS :[], - Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING :[], - Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS :[], - Performance.PERFORMANCE_PF_COL_WORKING_SET_BYTES :[], - Performance.PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES :[], - Performance.PERFORMANCE_PF_COL_MEM_TIMESTAMP :[] + Performance.PERFORMANCE_PF_COL_ITERATION: [], + Performance.PERFORMANCE_PF_COL_PATHFINDING_ITERATION: [], + Performance.PERFORMANCE_PF_COL_PERSON_ID: [], + Performance.PERFORMANCE_PF_COL_PERSON_TRIP_ID: [], + Performance.PERFORMANCE_PF_COL_PROCESS_NUM: [], + Performance.PERFORMANCE_PF_COL_PATHFINDING_STATUS: [], + Performance.PERFORMANCE_PF_COL_NUM_LABELED_STOPS: [], + Performance.PERFORMANCE_PF_COL_TRACED: [], + Performance.PERFORMANCE_PF_COL_LABEL_ITERATIONS: [], + Performance.PERFORMANCE_PF_COL_MAX_STOP_PROCESS_COUNT: [], + Performance.PERFORMANCE_PF_COL_TIME_LABELING: [], + Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS: [], + Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING: [], + Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS: [], + Performance.PERFORMANCE_PF_COL_WORKING_SET_BYTES: [], + Performance.PERFORMANCE_PF_COL_PRIVATE_USAGE_BYTES: [], + Performance.PERFORMANCE_PF_COL_MEM_TIMESTAMP: [] } # maps PERFORMANCE_COLUMN* to arrays of values self.step_record_dict = { - Performance.PERFORMANCE_COL_STEP_NAME :[], - Performance.PERFORMANCE_COL_ITERATION :[], - Performance.PERFORMANCE_COL_PATHFINDING_ITERATION :[], - Performance.PERFORMANCE_COL_SIMULATION_ITERATION :[], - Performance.PERFORMANCE_COL_START_TIME :[], - Performance.PERFORMANCE_COL_END_TIME :[], + Performance.PERFORMANCE_COL_STEP_NAME: [], + Performance.PERFORMANCE_COL_ITERATION: [], + Performance.PERFORMANCE_COL_PATHFINDING_ITERATION: [], + Performance.PERFORMANCE_COL_SIMULATION_ITERATION: [], + Performance.PERFORMANCE_COL_START_TIME: [], + Performance.PERFORMANCE_COL_END_TIME: [], # Performance.PERFORMANCE_COL_STEP_DURATION :[], # do this at the end - Performance.PERFORMANCE_COL_START_MEM_MB :[], - Performance.PERFORMANCE_COL_END_MEM_MB :[] + Performance.PERFORMANCE_COL_START_MEM_MB: [], + Performance.PERFORMANCE_COL_END_MEM_MB: [] } # will map (iteration, pathfinding_iteration, simulation_iteration) => (step_name, start time (a datetime.datetime), starting mem usage in bytes) self.steps = {} - def add_info(self, iteration, pathfinding_iteration, person_id, person_trip_id, perf_dict): + def add_info(self, iteration, pathfinding_iteration, person_id, person_trip_id, perf_dict): """ Add this row to the performance dict of arrays. Assumes time values are in milliseconds. @@ -151,17 +152,19 @@ def add_info(self, iteration, pathfinding_iteration, person_id, person_trip_id, self.performance_pf_dict[key].append(perf_dict[key]) # convert milliseconds time to timedeltas - self.performance_pf_dict[Performance.PERFORMANCE_PF_COL_TIME_LABELING ].append(datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS ])) - self.performance_pf_dict[Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING].append(datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS])) + self.performance_pf_dict[Performance.PERFORMANCE_PF_COL_TIME_LABELING].append( + datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_PF_COL_TIME_LABELING_MS])) + self.performance_pf_dict[Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING].append( + datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_PF_COL_TIME_ENUMERATING_MS])) def record_step_start(self, iteration, pathfinding_iteration, simulation_iteration, step_name): """ Records the step start. If there was previously a step at this level, then ends that and saves the duration of that step. """ - key = (iteration, pathfinding_iteration, simulation_iteration) - now = datetime.datetime.now() - mem_use = Util.get_process_mem_use_bytes()/1000000.0 + key = (iteration, pathfinding_iteration, simulation_iteration) + now = datetime.datetime.now() + mem_use = Util.get_process_mem_use_bytes() / 1000000.0 # if we were already doing something, record it if key in self.steps: @@ -174,23 +177,23 @@ def record_step_end(self, iteration, pathfinding_iteration, simulation_iteration """ Explicitly ends whatever step was happening at this level. """ - key = (iteration, pathfinding_iteration, simulation_iteration) - now = datetime.datetime.now() - mem_use = Util.get_process_mem_use_bytes()/1000000.0 + key = (iteration, pathfinding_iteration, simulation_iteration) + now = datetime.datetime.now() + mem_use = Util.get_process_mem_use_bytes() / 1000000.0 if key not in self.steps: return prev_step = self.steps[key] - self.step_record_dict[Performance.PERFORMANCE_COL_STEP_NAME ].append(prev_step[0]) - self.step_record_dict[Performance.PERFORMANCE_COL_ITERATION ].append(iteration) + self.step_record_dict[Performance.PERFORMANCE_COL_STEP_NAME].append(prev_step[0]) + self.step_record_dict[Performance.PERFORMANCE_COL_ITERATION].append(iteration) self.step_record_dict[Performance.PERFORMANCE_COL_PATHFINDING_ITERATION].append(pathfinding_iteration) - self.step_record_dict[Performance.PERFORMANCE_COL_SIMULATION_ITERATION ].append(simulation_iteration) - self.step_record_dict[Performance.PERFORMANCE_COL_START_TIME ].append(prev_step[1]) - self.step_record_dict[Performance.PERFORMANCE_COL_END_TIME ].append(now) - self.step_record_dict[Performance.PERFORMANCE_COL_START_MEM_MB ].append(prev_step[2]) - self.step_record_dict[Performance.PERFORMANCE_COL_END_MEM_MB ].append(mem_use) + self.step_record_dict[Performance.PERFORMANCE_COL_SIMULATION_ITERATION].append(simulation_iteration) + self.step_record_dict[Performance.PERFORMANCE_COL_START_TIME].append(prev_step[1]) + self.step_record_dict[Performance.PERFORMANCE_COL_END_TIME].append(now) + self.step_record_dict[Performance.PERFORMANCE_COL_START_MEM_MB].append(prev_step[2]) + self.step_record_dict[Performance.PERFORMANCE_COL_END_MEM_MB].append(mem_use) del self.steps[key] @@ -200,7 +203,8 @@ def write_pathfinding(self, output_dir, append): """ performance_df = pd.DataFrame.from_dict(self.performance_pf_dict) - Util.write_dataframe(performance_df, "performance_df", os.path.join(output_dir, Performance.OUTPUT_PERFORMANCE_PF_FILE), append=append) + Util.write_dataframe(performance_df, "performance_df", + os.path.join(output_dir, Performance.OUTPUT_PERFORMANCE_PF_FILE), append=append) # reset dict to blank for key in list(self.performance_pf_dict.keys()): @@ -212,6 +216,10 @@ def write(self, output_dir): """ performance_df = pd.DataFrame.from_dict(self.step_record_dict) - performance_df[Performance.PERFORMANCE_COL_STEP_DURATION] = performance_df[Performance.PERFORMANCE_COL_END_TIME] - performance_df[Performance.PERFORMANCE_COL_START_TIME] + performance_df[Performance.PERFORMANCE_COL_STEP_DURATION] = performance_df[ + Performance.PERFORMANCE_COL_END_TIME] - \ + performance_df[ + Performance.PERFORMANCE_COL_START_TIME] - Util.write_dataframe(performance_df, "performance_df", os.path.join(output_dir, Performance.OUTPUT_PERFORMANCE_FILE), append=False) + Util.write_dataframe(performance_df, "performance_df", + os.path.join(output_dir, Performance.OUTPUT_PERFORMANCE_FILE), append=False) diff --git a/fasttrips/Route.py b/fasttrips/Route.py index fe6c79b0..ad1304f9 100644 --- a/fasttrips/Route.py +++ b/fasttrips/Route.py @@ -3,7 +3,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,14 +16,15 @@ See the License for the specific language governing permissions and limitations under the License. """ + import os import numpy as np import pandas as pd -from .Error import NetworkInputError, NotImplementedError, UnexpectedError +from .Error import NetworkInputError, NotImplementedError, UnexpectedError from .Logger import FastTripsLogger -from .Util import Util +from .Util import Util class Route(object): @@ -42,109 +43,109 @@ class Route(object): #: File with fasttrips routes information (this extends the #: `gtfs routes `_ file). #: See `routes_ft specification `_. - INPUT_ROUTES_FILE = "routes_ft.txt" + INPUT_ROUTES_FILE = "routes_ft.txt" #: gtfs Routes column name: Unique identifier - ROUTES_COLUMN_ROUTE_ID = "route_id" + ROUTES_COLUMN_ROUTE_ID = "route_id" #: gtfs Routes column name: Short name - ROUTES_COLUMN_ROUTE_SHORT_NAME = "route_short_name" + ROUTES_COLUMN_ROUTE_SHORT_NAME = "route_short_name" #: gtfs Routes column name: Long name - ROUTES_COLUMN_ROUTE_LONG_NAME = "route_long_name" + ROUTES_COLUMN_ROUTE_LONG_NAME = "route_long_name" #: gtfs Routes column name: Route type - ROUTES_COLUMN_ROUTE_TYPE = "route_type" + ROUTES_COLUMN_ROUTE_TYPE = "route_type" #: gtfs Routes column name: Agency ID - ROUTES_COLUMN_AGENCY_ID = "agency_id" + ROUTES_COLUMN_AGENCY_ID = "agency_id" #: fasttrips Routes column name: Mode - ROUTES_COLUMN_MODE = "mode" + ROUTES_COLUMN_MODE = "mode" #: fasttrips Routes column name: Proof of Payment - ROUTES_COLUMN_PROOF_OF_PAYMENT = "proof_of_payment" + ROUTES_COLUMN_PROOF_OF_PAYMENT = "proof_of_payment" # ========== Added by fasttrips ======================================================= #: fasttrips Routes column name: Mode number - ROUTES_COLUMN_ROUTE_ID_NUM = "route_id_num" + ROUTES_COLUMN_ROUTE_ID_NUM = "route_id_num" #: fasttrips Routes column name: Mode number - ROUTES_COLUMN_MODE_NUM = "mode_num" + ROUTES_COLUMN_MODE_NUM = "mode_num" #: fasttrips Routes column name: Mode type - ROUTES_COLUMN_MODE_TYPE = "mode_type" + ROUTES_COLUMN_MODE_TYPE = "mode_type" #: Value for :py:attr:`Route.ROUTES_COLUMN_MODE_TYPE` column: access - MODE_TYPE_ACCESS = "access" + MODE_TYPE_ACCESS = "access" #: Value for :py:attr:`Route.ROUTES_COLUMN_MODE_TYPE` column: egress - MODE_TYPE_EGRESS = "egress" + MODE_TYPE_EGRESS = "egress" #: Value for :py:attr:`Route.ROUTES_COLUMN_MODE_TYPE` column: transit - MODE_TYPE_TRANSIT = "transit" + MODE_TYPE_TRANSIT = "transit" #: Value for :py:attr:`Route.ROUTES_COLUMN_MODE_TYPE` column: transfer - MODE_TYPE_TRANSFER = "transfer" + MODE_TYPE_TRANSFER = "transfer" #: Access mode numbers start from here - MODE_NUM_START_ACCESS = 101 + MODE_NUM_START_ACCESS = 101 #: Egress mode numbers start from here - MODE_NUM_START_EGRESS = 201 + MODE_NUM_START_EGRESS = 201 #: Route mode numbers start from here - MODE_NUM_START_ROUTE = 301 + MODE_NUM_START_ROUTE = 301 #: File with fasttrips fare attributes information (this *subsitutes rather than extends* the #: `gtfs fare_attributes `_ file). #: See `fare_attributes_ft specification `_. - INPUT_FARE_ATTRIBUTES_FILE = "fare_attributes_ft.txt" + INPUT_FARE_ATTRIBUTES_FILE = "fare_attributes_ft.txt" # fasttrips Fare attributes column name: Fare Period - FARE_ATTR_COLUMN_FARE_PERIOD = "fare_period" + FARE_ATTR_COLUMN_FARE_PERIOD = "fare_period" # fasttrips Fare attributes column name: Price - FARE_ATTR_COLUMN_PRICE = "price" + FARE_ATTR_COLUMN_PRICE = "price" # fasttrips Fare attributes column name: Currency Type - FARE_ATTR_COLUMN_CURRENCY_TYPE = "currency_type" + FARE_ATTR_COLUMN_CURRENCY_TYPE = "currency_type" # fasttrips Fare attributes column name: Payment Method - FARE_ATTR_COLUMN_PAYMENT_METHOD = "payment_method" + FARE_ATTR_COLUMN_PAYMENT_METHOD = "payment_method" # fasttrips Fare attributes column name: Transfers (number permitted on this fare) - FARE_ATTR_COLUMN_TRANSFERS = "transfers" + FARE_ATTR_COLUMN_TRANSFERS = "transfers" # fasttrips Fare attributes column name: Transfer duration (Integer length of time in seconds before transfer expires. Omit or leave empty if they do not.) - FARE_ATTR_COLUMN_TRANSFER_DURATION = "transfer_duration" + FARE_ATTR_COLUMN_TRANSFER_DURATION = "transfer_duration" #: File with fasttrips fare periods information #: See `fare_rules_ft specification `_. - INPUT_FARE_PERIODS_FILE = "fare_periods_ft.txt" + INPUT_FARE_PERIODS_FILE = "fare_periods_ft.txt" #: fasttrips Fare rules column name: Fare ID - FARE_RULES_COLUMN_FARE_ID = "fare_id" + FARE_RULES_COLUMN_FARE_ID = "fare_id" #: GTFS fare rules column name: Route ID - FARE_RULES_COLUMN_ROUTE_ID = ROUTES_COLUMN_ROUTE_ID + FARE_RULES_COLUMN_ROUTE_ID = ROUTES_COLUMN_ROUTE_ID #: GTFS fare rules column name: Origin Zone ID - FARE_RULES_COLUMN_ORIGIN_ID = "origin_id" + FARE_RULES_COLUMN_ORIGIN_ID = "origin_id" #: GTFS fare rules column name: Destination Zone ID - FARE_RULES_COLUMN_DESTINATION_ID = "destination_id" + FARE_RULES_COLUMN_DESTINATION_ID = "destination_id" #: GTFS fare rules column name: Contains ID - FARE_RULES_COLUMN_CONTAINS_ID = "contains_id" + FARE_RULES_COLUMN_CONTAINS_ID = "contains_id" #: fasttrips Fare rules column name: Fare class - FARE_RULES_COLUMN_FARE_PERIOD = FARE_ATTR_COLUMN_FARE_PERIOD + FARE_RULES_COLUMN_FARE_PERIOD = FARE_ATTR_COLUMN_FARE_PERIOD #: fasttrips Fare rules column name: Start time for the fare. A DateTime - FARE_RULES_COLUMN_START_TIME = "start_time" + FARE_RULES_COLUMN_START_TIME = "start_time" #: fasttrips Fare rules column name: End time for the fare rule. A DateTime. - FARE_RULES_COLUMN_END_TIME = "end_time" + FARE_RULES_COLUMN_END_TIME = "end_time" # ========== Added by fasttrips ======================================================= #: fasttrips Fare rules column name: Fare ID num - FARE_RULES_COLUMN_FARE_ID_NUM = "fare_id_num" + FARE_RULES_COLUMN_FARE_ID_NUM = "fare_id_num" #: fasttrips Fare rules column name: Route ID num - FARE_RULES_COLUMN_ROUTE_ID_NUM = ROUTES_COLUMN_ROUTE_ID_NUM + FARE_RULES_COLUMN_ROUTE_ID_NUM = ROUTES_COLUMN_ROUTE_ID_NUM #: fasttrips fare rules column name: Origin Zone ID number - FARE_RULES_COLUMN_ORIGIN_ID_NUM = "origin_id_num" + FARE_RULES_COLUMN_ORIGIN_ID_NUM = "origin_id_num" #: fasttrips fare rules column name: Destination ID number - FARE_RULES_COLUMN_DESTINATION_ID_NUM = "destination_id_num" + FARE_RULES_COLUMN_DESTINATION_ID_NUM = "destination_id_num" #: File with fasttrips fare transfer rules information. #: See `fare_transfer_rules specification `_. - INPUT_FARE_TRANSFER_RULES_FILE = "fare_transfer_rules_ft.txt" + INPUT_FARE_TRANSFER_RULES_FILE = "fare_transfer_rules_ft.txt" #: fasttrips Fare transfer rules column name: From Fare Class FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD = "from_fare_period" #: fasttrips Fare transfer rules column name: To Fare Class - FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD = "to_fare_period" + FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD = "to_fare_period" #: fasttrips Fare transfer rules column name: Transfer type? - FARE_TRANSFER_RULES_COLUMN_TYPE = "transfer_fare_type" + FARE_TRANSFER_RULES_COLUMN_TYPE = "transfer_fare_type" #: fasttrips Fare transfer rules column name: Transfer amount (discount or fare) - FARE_TRANSFER_RULES_COLUMN_AMOUNT = "transfer_fare" + FARE_TRANSFER_RULES_COLUMN_AMOUNT = "transfer_fare" #: Value for :py:attr:`Route.FARE_TRANSFER_RULES_COLUMN_TYPE`: transfer discount TRANSFER_TYPE_TRANSFER_DISCOUNT = "transfer_discount" #: Value for :py:attr:`Route.FARE_TRANSFER_RULES_COLUMN_TYPE`: free transfer - TRANSFER_TYPE_TRANSFER_FREE = "transfer_free" + TRANSFER_TYPE_TRANSFER_FREE = "transfer_free" #: Value for :py:attr:`Route.FARE_TRANSFER_RULES_COLUMN_TYPE`: transfer fare cost - TRANSFER_TYPE_TRANSFER_COST = "transfer_cost" + TRANSFER_TYPE_TRANSFER_COST = "transfer_cost" #: Valid options for :py:attr:`Route.FARE_TRANSFER_RULES_COLUMN_TYPE` TRANSFER_TYPE_OPTIONS = [TRANSFER_TYPE_TRANSFER_DISCOUNT, @@ -152,20 +153,20 @@ class Route(object): TRANSFER_TYPE_TRANSFER_COST] #: File with route ID, route ID number correspondence (and fare id num) - OUTPUT_ROUTE_ID_NUM_FILE = "ft_intermediate_route_id.txt" + OUTPUT_ROUTE_ID_NUM_FILE = "ft_intermediate_route_id.txt" #: File with fare id num, fare id, fare class, price, xfers - OUTPUT_FARE_ID_FILE = "ft_intermediate_fare.txt" + OUTPUT_FARE_ID_FILE = "ft_intermediate_fare.txt" #: File with fare transfer rules - OUTPUT_FARE_TRANSFER_FILE = "ft_intermediate_fare_transfers.txt" + OUTPUT_FARE_TRANSFER_FILE = "ft_intermediate_fare_transfers.txt" #: File with mode, mode number correspondence - OUTPUT_MODE_NUM_FILE = "ft_intermediate_supply_mode_id.txt" + OUTPUT_MODE_NUM_FILE = "ft_intermediate_supply_mode_id.txt" def __init__(self, input_archive, output_dir, gtfs, today, stops): """ Constructor. Reads the gtfs data from the transitfeed schedule, and the additional fast-trips routes data from the input file in *input_archive*. """ - self.output_dir = output_dir + self.output_dir = output_dir self.routes_df = gtfs.routes @@ -177,13 +178,14 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): # verify required columns are present routes_ft_cols = list(routes_ft_df.columns.values) - assert(Route.ROUTES_COLUMN_ROUTE_ID in routes_ft_cols) - assert(Route.ROUTES_COLUMN_MODE in routes_ft_cols) + assert (Route.ROUTES_COLUMN_ROUTE_ID in routes_ft_cols) + assert (Route.ROUTES_COLUMN_MODE in routes_ft_cols) # verify no routes_ids are duplicated - if routes_ft_df.duplicated(subset=[Route.ROUTES_COLUMN_ROUTE_ID]).sum()>0: - error_msg = "Found %d duplicate %s in %s" % (routes_ft_df.duplicated(subset=[Route.ROUTES_COLUMN_ROUTE_ID]).sum(), - Route.ROUTES_COLUMN_ROUTE_ID, Route.INPUT_ROUTES_FILE) + if routes_ft_df.duplicated(subset=[Route.ROUTES_COLUMN_ROUTE_ID]).sum() > 0: + error_msg = "Found %d duplicate %s in %s" % ( + routes_ft_df.duplicated(subset=[Route.ROUTES_COLUMN_ROUTE_ID]).sum(), + Route.ROUTES_COLUMN_ROUTE_ID, Route.INPUT_ROUTES_FILE) FastTripsLogger.fatal(error_msg) FastTripsLogger.fatal("\nDuplicates:\n%s" % \ str(routes_ft_df.loc[routes_ft_df.duplicated(subset=[Route.ROUTES_COLUMN_ROUTE_ID])])) @@ -191,8 +193,8 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): # Join to the routes dataframe self.routes_df = pd.merge(left=self.routes_df, right=routes_ft_df, - how='left', - on=Route.ROUTES_COLUMN_ROUTE_ID) + how='left', + on=Route.ROUTES_COLUMN_ROUTE_ID) # Get the mode list self.modes_df = self.routes_df[[Route.ROUTES_COLUMN_MODE]].drop_duplicates().reset_index(drop=True) self.modes_df[Route.ROUTES_COLUMN_MODE_NUM] = self.modes_df.index + Route.MODE_NUM_START_ROUTE @@ -200,7 +202,7 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): # Join to mode numbering self.routes_df = Util.add_new_id(self.routes_df, Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM, - self.modes_df, Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM) + self.modes_df, Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM) # Route IDs are strings. Create a unique numeric route ID. self.route_id_df = Util.add_numeric_column(self.routes_df[[Route.ROUTES_COLUMN_ROUTE_ID]], @@ -214,21 +216,21 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): numeric_newcolname=Route.ROUTES_COLUMN_ROUTE_ID_NUM) FastTripsLogger.debug("=========== ROUTES ===========\n" + str(self.routes_df.head())) - FastTripsLogger.debug("\n"+str(self.routes_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.routes_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.routes_df), "routes", "routes.txt", Route.INPUT_ROUTES_FILE)) self.agencies_df = gtfs.agency FastTripsLogger.debug("=========== AGENCIES ===========\n" + str(self.agencies_df.head())) - FastTripsLogger.debug("\n"+str(self.agencies_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.agencies_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.agencies_df), "agencies", "agency.txt")) self.fare_attrs_df = gtfs.fare_attributes FastTripsLogger.debug("=========== FARE ATTRIBUTES ===========\n" + str(self.fare_attrs_df.head())) - FastTripsLogger.debug("\n"+str(self.fare_attrs_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.fare_attrs_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.fare_attrs_df), "fare attributes", "fare_attributes.txt")) @@ -237,17 +239,17 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): if not self.fare_attrs_df.empty: # verify required columns are present fare_attrs_cols = list(self.fare_attrs_df.columns.values) - assert(Route.FARE_ATTR_COLUMN_FARE_PERIOD in fare_attrs_cols) - assert(Route.FARE_ATTR_COLUMN_PRICE in fare_attrs_cols) - assert(Route.FARE_ATTR_COLUMN_CURRENCY_TYPE in fare_attrs_cols) - assert(Route.FARE_ATTR_COLUMN_PAYMENT_METHOD in fare_attrs_cols) - assert(Route.FARE_ATTR_COLUMN_TRANSFERS in fare_attrs_cols) + assert (Route.FARE_ATTR_COLUMN_FARE_PERIOD in fare_attrs_cols) + assert (Route.FARE_ATTR_COLUMN_PRICE in fare_attrs_cols) + assert (Route.FARE_ATTR_COLUMN_CURRENCY_TYPE in fare_attrs_cols) + assert (Route.FARE_ATTR_COLUMN_PAYMENT_METHOD in fare_attrs_cols) + assert (Route.FARE_ATTR_COLUMN_TRANSFERS in fare_attrs_cols) if Route.FARE_ATTR_COLUMN_TRANSFER_DURATION not in fare_attrs_cols: self.fare_attrs_df[Route.FARE_ATTR_COLUMN_TRANSFER_DURATION] = np.nan FastTripsLogger.debug("===> REPLACED BY FARE ATTRIBUTES FT\n" + str(self.fare_attrs_df.head())) - FastTripsLogger.debug("\n"+str(self.fare_attrs_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.fare_attrs_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.fare_attrs_df), "fare attributes", Route.INPUT_FARE_ATTRIBUTES_FILE)) @@ -263,46 +265,48 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): self.fare_ids_df = Util.add_numeric_column(self.fare_rules_df[[Route.FARE_RULES_COLUMN_FARE_ID]], id_colname=Route.FARE_RULES_COLUMN_FARE_ID, numeric_newcolname=Route.FARE_RULES_COLUMN_FARE_ID_NUM) - self.fare_rules_df = pd.merge(left =self.fare_rules_df, - right =self.fare_ids_df, - how ="left") + self.fare_rules_df = pd.merge(left=self.fare_rules_df, + right=self.fare_ids_df, + how="left") else: self.fare_ids_df = pd.DataFrame() - # optionally reverse those with origin/destinations if configured from .Assignment import Assignment if Assignment.FARE_ZONE_SYMMETRY: FastTripsLogger.debug("applying FARE_ZONE_SYMMETRY to %d fare rules" % len(self.fare_rules_df)) # select only those with an origin and destination - reverse_fare_rules = self.fare_rules_df.loc[ pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID])& - pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID]) ].copy() + reverse_fare_rules = self.fare_rules_df.loc[ + pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) & + pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])].copy() # FastTripsLogger.debug("reverse_fare_rules 1 head()=\n%s" % str(reverse_fare_rules.head())) # reverse them - reverse_fare_rules.rename(columns={Route.FARE_RULES_COLUMN_ORIGIN_ID : Route.FARE_RULES_COLUMN_DESTINATION_ID, - Route.FARE_RULES_COLUMN_DESTINATION_ID : Route.FARE_RULES_COLUMN_ORIGIN_ID}, - inplace=True) + reverse_fare_rules.rename( + columns={Route.FARE_RULES_COLUMN_ORIGIN_ID: Route.FARE_RULES_COLUMN_DESTINATION_ID, + Route.FARE_RULES_COLUMN_DESTINATION_ID: Route.FARE_RULES_COLUMN_ORIGIN_ID}, + inplace=True) # FastTripsLogger.debug("reverse_fare_rules 2 head()=\n%s" % str(reverse_fare_rules.head())) # join them to eliminate dupes - reverse_fare_rules = pd.merge(left =reverse_fare_rules, - right =self.fare_rules_df, - how ="left", - on =[Route.FARE_RULES_COLUMN_FARE_ID, - Route.FARE_RULES_COLUMN_FARE_ID_NUM, - Route.FARE_RULES_COLUMN_ROUTE_ID, - Route.FARE_RULES_COLUMN_ORIGIN_ID, - Route.FARE_RULES_COLUMN_DESTINATION_ID, - Route.FARE_RULES_COLUMN_CONTAINS_ID], - indicator=True) + reverse_fare_rules = pd.merge(left=reverse_fare_rules, + right=self.fare_rules_df, + how="left", + on=[Route.FARE_RULES_COLUMN_FARE_ID, + Route.FARE_RULES_COLUMN_FARE_ID_NUM, + Route.FARE_RULES_COLUMN_ROUTE_ID, + Route.FARE_RULES_COLUMN_ORIGIN_ID, + Route.FARE_RULES_COLUMN_DESTINATION_ID, + Route.FARE_RULES_COLUMN_CONTAINS_ID], + indicator=True) # dupes exist in both -- drop those - reverse_fare_rules = reverse_fare_rules.loc[ reverse_fare_rules["_merge"]=="left_only"] + reverse_fare_rules = reverse_fare_rules.loc[reverse_fare_rules["_merge"] == "left_only"] reverse_fare_rules.drop(["_merge"], axis=1, inplace=True) # add them to fare rules self.fare_rules_df = pd.concat([self.fare_rules_df, reverse_fare_rules]) - FastTripsLogger.debug("fare rules with symmetry %d head()=\n%s" % (len(self.fare_rules_df), str(self.fare_rules_df.head()))) + FastTripsLogger.debug( + "fare rules with symmetry %d head()=\n%s" % (len(self.fare_rules_df), str(self.fare_rules_df.head()))) # sort by fare ID num so zone-to-zone and their reverse are together if len(self.fare_rules_df) > 0: @@ -312,18 +316,18 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): if not fare_rules_ft_df.empty: # verify required columns are present fare_rules_ft_cols = list(fare_rules_ft_df.columns.values) - assert(Route.FARE_RULES_COLUMN_FARE_ID in fare_rules_ft_cols) - assert(Route.FARE_RULES_COLUMN_FARE_PERIOD in fare_rules_ft_cols) - assert(Route.FARE_RULES_COLUMN_START_TIME in fare_rules_ft_cols) - assert(Route.FARE_RULES_COLUMN_END_TIME in fare_rules_ft_cols) + assert (Route.FARE_RULES_COLUMN_FARE_ID in fare_rules_ft_cols) + assert (Route.FARE_RULES_COLUMN_FARE_PERIOD in fare_rules_ft_cols) + assert (Route.FARE_RULES_COLUMN_START_TIME in fare_rules_ft_cols) + assert (Route.FARE_RULES_COLUMN_END_TIME in fare_rules_ft_cols) # Split fare classes so they don't overlap fare_rules_ft_df = self.remove_fare_period_overlap(fare_rules_ft_df) # join to fare rules dataframe self.fare_rules_df = pd.merge(left=self.fare_rules_df, right=fare_rules_ft_df, - how='left', - on=Route.FARE_RULES_COLUMN_FARE_ID) + how='left', + on=Route.FARE_RULES_COLUMN_FARE_ID) # add route id numbering if applicable if Route.FARE_RULES_COLUMN_ROUTE_ID in list(self.fare_rules_df.columns.values): @@ -332,13 +336,13 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): Route.FARE_RULES_COLUMN_ROUTE_ID_NUM) # add origin zone numbering if applicable if (Route.FARE_RULES_COLUMN_ORIGIN_ID in list(self.fare_rules_df.columns.values)) and \ - (pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]).sum() > 0): + (pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]).sum() > 0): self.fare_rules_df = stops.add_numeric_stop_zone_id(self.fare_rules_df, Route.FARE_RULES_COLUMN_ORIGIN_ID, Route.FARE_RULES_COLUMN_ORIGIN_ID_NUM) # add destination zone numbering if applicable if (Route.FARE_RULES_COLUMN_DESTINATION_ID in list(self.fare_rules_df.columns.values)) and \ - (pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID]).sum() > 0): + (pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID]).sum() > 0): self.fare_rules_df = stops.add_numeric_stop_zone_id(self.fare_rules_df, Route.FARE_RULES_COLUMN_DESTINATION_ID, Route.FARE_RULES_COLUMN_DESTINATION_ID_NUM) @@ -350,16 +354,18 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): raise NotImplementedError(error_str) # check for each row, either both are present or neither -- use xor, or ^ - xor_id = self.fare_rules_df.loc[ pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID])^ - pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID]) ] + xor_id = self.fare_rules_df.loc[pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) ^ + pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])] if len(xor_id) > 0: - error_str = "Fast-trips supports fare rules with both origin id and destination id specified, or neither ONLY.\n%s" % str(xor_id) + error_str = "Fast-trips supports fare rules with both origin id and destination id specified, or neither ONLY.\n%s" % str( + xor_id) FastTripsLogger.fatal(error_str) raise NotImplementedError(error_str) # We don't support contains_id if Route.FARE_RULES_COLUMN_CONTAINS_ID in list(self.fare_rules_df.columns.values): - non_null_contains_id = self.fare_rules_df.loc[pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_CONTAINS_ID])] + non_null_contains_id = self.fare_rules_df.loc[ + pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_CONTAINS_ID])] if len(non_null_contains_id) > 0: error_str = "Fast-trips does not support contains_id in fare rules:\n%s" % str(non_null_contains_id) FastTripsLogger.fatal(error_str) @@ -369,9 +375,10 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): elif len(self.fare_rules_df) > 0: # we have fare rules but no fare periods -- make the fare periods the same - self.fare_rules_df[Route.FARE_RULES_COLUMN_FARE_PERIOD] = self.fare_rules_df[Route.FARE_RULES_COLUMN_FARE_ID] + self.fare_rules_df[Route.FARE_RULES_COLUMN_FARE_PERIOD] = self.fare_rules_df[ + Route.FARE_RULES_COLUMN_FARE_ID] self.fare_rules_df[Route.FARE_RULES_COLUMN_START_TIME] = Util.read_time("00:00:00") - self.fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME ] = Util.read_time("24:00:00") + self.fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME] = Util.read_time("24:00:00") # join to fare_attributes on fare_period if we have it, or fare_id if we don't if len(self.fare_rules_df) > 0: @@ -419,45 +426,51 @@ def __init__(self, input_archive, output_dir, gtfs, today, stops): .. _fare_attributes: https://github.com/osplanning-data-standards/GTFS-PLUS/blob/master/files/fare_attributes.md .. _fare_attributes_ft: https://github.com/osplanning-data-standards/GTFS-PLUS/blob/master/files/fare_attributes_ft.md """ - self.fare_rules_df = pd.merge(left =self.fare_rules_df, - right=self.fare_attrs_df, - how ='left', - on = Route.FARE_RULES_COLUMN_FARE_PERIOD if self.fare_by_class else Route.FARE_RULES_COLUMN_FARE_ID) - - - FastTripsLogger.debug("=========== FARE RULES ===========\n" + str(self.fare_rules_df.head(10).to_string(formatters=\ - {Route.FARE_RULES_COLUMN_START_TIME:Util.datetime64_formatter, - Route.FARE_RULES_COLUMN_END_TIME :Util.datetime64_formatter}))) - FastTripsLogger.debug("\n"+str(self.fare_rules_df.dtypes)) + self.fare_rules_df = pd.merge(left=self.fare_rules_df, + right=self.fare_attrs_df, + how='left', + on=Route.FARE_RULES_COLUMN_FARE_PERIOD if self.fare_by_class else Route.FARE_RULES_COLUMN_FARE_ID) + + FastTripsLogger.debug( + "=========== FARE RULES ===========\n" + str(self.fare_rules_df.head(10).to_string(formatters= \ + { + Route.FARE_RULES_COLUMN_START_TIME: Util.datetime64_formatter, + Route.FARE_RULES_COLUMN_END_TIME: Util.datetime64_formatter}))) + FastTripsLogger.debug("\n" + str(self.fare_rules_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.fare_rules_df), "fare rules", "fare_rules.txt", self.INPUT_FARE_PERIODS_FILE)) self.fare_transfer_rules_df = gtfs.get(Route.INPUT_FARE_TRANSFER_RULES_FILE) if not self.fare_transfer_rules_df.empty: # verify required columns are present fare_transfer_rules_cols = list(self.fare_transfer_rules_df.columns.values) - assert(Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD in fare_transfer_rules_cols) - assert(Route.FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD in fare_transfer_rules_cols) - assert(Route.FARE_TRANSFER_RULES_COLUMN_TYPE in fare_transfer_rules_cols) - assert(Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT in fare_transfer_rules_cols) + assert (Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD in fare_transfer_rules_cols) + assert (Route.FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD in fare_transfer_rules_cols) + assert (Route.FARE_TRANSFER_RULES_COLUMN_TYPE in fare_transfer_rules_cols) + assert (Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT in fare_transfer_rules_cols) # verify valid values for transfer type - invalid_type = self.fare_transfer_rules_df.loc[ self.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE].isin(Route.TRANSFER_TYPE_OPTIONS)==False ] + invalid_type = self.fare_transfer_rules_df.loc[ + self.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE].isin( + Route.TRANSFER_TYPE_OPTIONS) == False] if len(invalid_type) > 0: error_msg = "Invalid value for %s:\n%s" % (Route.FARE_TRANSFER_RULES_COLUMN_TYPE, str(invalid_type)) FastTripsLogger.fatal(error_msg) raise NetworkInputError(Route.INPUT_FARE_TRANSFER_RULES_FILE, error_msg) # verify the amount is positive - negative_amount = self.fare_transfer_rules_df.loc[ self.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] < 0] + negative_amount = self.fare_transfer_rules_df.loc[ + self.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] < 0] if len(negative_amount) > 0: error_msg = "Negative transfer amounts are invalid:\n%s" % str(negative_amount) FastTripsLogger.fatal(error_msg) raise NetworkInputError(Route.INPUT_FARE_TRANSFER_RULES_FILE, error_msg) - FastTripsLogger.debug("=========== FARE TRANSFER RULES ===========\n" + str(self.fare_transfer_rules_df.head())) - FastTripsLogger.debug("\n"+str(self.fare_transfer_rules_df.dtypes)) + FastTripsLogger.debug( + "=========== FARE TRANSFER RULES ===========\n" + str(self.fare_transfer_rules_df.head())) + FastTripsLogger.debug("\n" + str(self.fare_transfer_rules_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % - (len(self.fare_transfer_rules_df), "fare xfer rules", Route.INPUT_FARE_TRANSFER_RULES_FILE)) + (len(self.fare_transfer_rules_df), "fare xfer rules", + Route.INPUT_FARE_TRANSFER_RULES_FILE)) else: self.fare_transfer_rules_df = pd.DataFrame() @@ -481,13 +494,13 @@ def add_access_egress_modes(self, access_modes_df, egress_modes_df): access_modes_df[Route.ROUTES_COLUMN_MODE_TYPE] = Route.MODE_TYPE_ACCESS egress_modes_df[Route.ROUTES_COLUMN_MODE_TYPE] = Route.MODE_TYPE_EGRESS implicit_modes_df = pd.DataFrame({Route.ROUTES_COLUMN_MODE_TYPE: [Route.MODE_TYPE_TRANSFER], - Route.ROUTES_COLUMN_MODE: [Route.MODE_TYPE_TRANSFER], - Route.ROUTES_COLUMN_MODE_NUM: [ 1]}) + Route.ROUTES_COLUMN_MODE: [Route.MODE_TYPE_TRANSFER], + Route.ROUTES_COLUMN_MODE_NUM: [1]}) self.modes_df = pd.concat([implicit_modes_df, - self.modes_df, - access_modes_df, - egress_modes_df], axis=0) + self.modes_df, + access_modes_df, + egress_modes_df], axis=0) self.modes_df.reset_index(inplace=True) # write intermediate files @@ -511,25 +524,25 @@ def remove_fare_period_overlap(self, fare_rules_ft_df): """ Split fare classes so they don't overlap """ - fare_rules_ft_df["fare_period_id"] = fare_rules_ft_df.index+1 + fare_rules_ft_df["fare_period_id"] = fare_rules_ft_df.index + 1 # FastTripsLogger.debug("remove_fare_period_overlap: initial\n%s" % fare_rules_ft_df) max_fare_period_id = fare_rules_ft_df["fare_period_id"].max() loop_iters = 0 while True: # join with itself to see if any are contained - df = pd.merge(left =fare_rules_ft_df, - right=fare_rules_ft_df, - on =Route.FARE_RULES_COLUMN_FARE_ID, - how ="outer") + df = pd.merge(left=fare_rules_ft_df, + right=fare_rules_ft_df, + on=Route.FARE_RULES_COLUMN_FARE_ID, + how="outer") # if there's one fare period per fare id, nothing to do - if len(df)==len(fare_rules_ft_df): + if len(df) == len(fare_rules_ft_df): FastTripsLogger.debug("One fare period per fare id, no need to split") return fare_rules_ft_df # remove dupes - df = df.loc[ df["fare_period_id_x"] != df["fare_period_id_y"] ] + df = df.loc[df["fare_period_id_x"] != df["fare_period_id_y"]] FastTripsLogger.debug("remove_fare_period_overlap:\n%s" % df) @@ -537,9 +550,9 @@ def remove_fare_period_overlap(self, fare_rules_ft_df): # ____y_______ x starts after y starts # ______x______ x starts before y ends # x ends after y ends - intersecting_fare_periods = df.loc[ (df["start_time_x"]>df["start_time_y"])& \ - (df["start_time_x"]df["end_time_y"]) ] + intersecting_fare_periods = df.loc[(df["start_time_x"] > df["start_time_y"]) & \ + (df["start_time_x"] < df["end_time_y"]) & \ + (df["end_time_x"] > df["end_time_y"])] if len(intersecting_fare_periods) > 0: error_msg = "Partially overlapping fare periods are ambiguous. \n%s" % str(intersecting_fare_periods) FastTripsLogger.error(error_msg) @@ -548,8 +561,8 @@ def remove_fare_period_overlap(self, fare_rules_ft_df): # is x a subset of y? # ___x___ x starts after y starts # ______y_______ x ends before y ends - subset_fare_periods = df.loc[ (df["start_time_x"]>=df["start_time_y"])& \ - (df["end_time_x" ]<=df["end_time_y"]) ] + subset_fare_periods = df.loc[(df["start_time_x"] >= df["start_time_y"]) & \ + (df["end_time_x"] <= df["end_time_y"])] # if no subsets, done -- return if len(subset_fare_periods) == 0: FastTripsLogger.debug("remove_fare_period_overlap returning\n%s" % fare_rules_ft_df) @@ -559,29 +572,29 @@ def remove_fare_period_overlap(self, fare_rules_ft_df): FastTripsLogger.debug("splitting\n%s" % str(subset_fare_periods)) row_dict = subset_fare_periods.head(1).to_dict(orient="records")[0] FastTripsLogger.debug(row_dict) - y_1 = {'fare_id' :row_dict['fare_id'], - 'fare_period' :row_dict['fare_period_y'], - 'start_time' :row_dict['start_time_y'], - 'end_time' :row_dict['start_time_x'], - 'fare_period_id' :row_dict['fare_period_id_y']} - x = {'fare_id' :row_dict['fare_id'], - 'fare_period' :row_dict['fare_period_x'], - 'start_time' :row_dict['start_time_x'], - 'end_time' :row_dict['end_time_x'], - 'fare_period_id' :row_dict['fare_period_id_x']} - y_2 = {'fare_id' :row_dict['fare_id'], - 'fare_period' :row_dict['fare_period_y'], - 'start_time' :row_dict['end_time_x'], - 'end_time' :row_dict['end_time_y'], - 'fare_period_id' :max_fare_period_id+1} # new + y_1 = {'fare_id': row_dict['fare_id'], + 'fare_period': row_dict['fare_period_y'], + 'start_time': row_dict['start_time_y'], + 'end_time': row_dict['start_time_x'], + 'fare_period_id': row_dict['fare_period_id_y']} + x = {'fare_id': row_dict['fare_id'], + 'fare_period': row_dict['fare_period_x'], + 'start_time': row_dict['start_time_x'], + 'end_time': row_dict['end_time_x'], + 'fare_period_id': row_dict['fare_period_id_x']} + y_2 = {'fare_id': row_dict['fare_id'], + 'fare_period': row_dict['fare_period_y'], + 'start_time': row_dict['end_time_x'], + 'end_time': row_dict['end_time_y'], + 'fare_period_id': max_fare_period_id + 1} # new max_fare_period_id += 1 - new_df = pd.DataFrame([y_1,x,y_2]) + new_df = pd.DataFrame([y_1, x, y_2]) FastTripsLogger.debug("\n%s" % str(new_df)) # put it together with the unaffected fare_periodes we already had - prev_df = fare_rules_ft_df.loc[ (fare_rules_ft_df["fare_period_id"]!=row_dict["fare_period_id_x"])& - (fare_rules_ft_df["fare_period_id"]!=row_dict["fare_period_id_y"]) ] + prev_df = fare_rules_ft_df.loc[(fare_rules_ft_df["fare_period_id"] != row_dict["fare_period_id_x"]) & + (fare_rules_ft_df["fare_period_id"] != row_dict["fare_period_id_y"])] fare_rules_ft_df = prev_df.append(new_df) # sort by fare_id, start_time @@ -616,45 +629,51 @@ def write_routes_for_extension(self): os.path.join(self.output_dir, Route.OUTPUT_ROUTE_ID_NUM_FILE), sep=" ", index=False) FastTripsLogger.debug("Wrote %s" % os.path.join(self.output_dir, Route.OUTPUT_ROUTE_ID_NUM_FILE)) - # write fare file if len(self.fare_rules_df) > 0: # copy for writing fare_rules_df = self.fare_rules_df.copy() # replace with float versions - fare_rules_df[Route.FARE_RULES_COLUMN_START_TIME] = (fare_rules_df[Route.FARE_RULES_COLUMN_START_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME)/np.timedelta64(1,'m') - fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME ] = (fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME ] - Assignment.NETWORK_BUILD_DATE_START_TIME)/np.timedelta64(1,'m') + fare_rules_df[Route.FARE_RULES_COLUMN_START_TIME] = (fare_rules_df[ + Route.FARE_RULES_COLUMN_START_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME) / np.timedelta64( + 1, 'm') + fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME] = (fare_rules_df[ + Route.FARE_RULES_COLUMN_END_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME) / np.timedelta64( + 1, 'm') # fillna with -1 - for num_col in [Route.FARE_RULES_COLUMN_ROUTE_ID_NUM, Route.FARE_RULES_COLUMN_ORIGIN_ID_NUM, Route.FARE_RULES_COLUMN_DESTINATION_ID_NUM, Route.FARE_ATTR_COLUMN_TRANSFERS]: + for num_col in [Route.FARE_RULES_COLUMN_ROUTE_ID_NUM, Route.FARE_RULES_COLUMN_ORIGIN_ID_NUM, + Route.FARE_RULES_COLUMN_DESTINATION_ID_NUM, Route.FARE_ATTR_COLUMN_TRANSFERS]: if num_col in list(fare_rules_df.columns.values): - fare_rules_df.loc[ pd.isnull(fare_rules_df[num_col]), num_col] = -1 + fare_rules_df.loc[pd.isnull(fare_rules_df[num_col]), num_col] = -1 fare_rules_df[num_col] = fare_rules_df[num_col].astype(int) else: fare_rules_df[num_col] = -1 # temp column: duraton; sort by this so the smallest duration is found first - fare_rules_df["duration"] = fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME ] - fare_rules_df[Route.FARE_RULES_COLUMN_START_TIME] - fare_rules_df.sort_values(by=[Route.FARE_RULES_COLUMN_FARE_ID_NUM,"duration"], ascending=True, inplace=True) + fare_rules_df["duration"] = fare_rules_df[Route.FARE_RULES_COLUMN_END_TIME] - fare_rules_df[ + Route.FARE_RULES_COLUMN_START_TIME] + fare_rules_df.sort_values(by=[Route.FARE_RULES_COLUMN_FARE_ID_NUM, "duration"], ascending=True, + inplace=True) # transfer_duration fillna with -1 - fare_rules_df.fillna({Route.FARE_ATTR_COLUMN_TRANSFER_DURATION:-1}, inplace=True) + fare_rules_df.fillna({Route.FARE_ATTR_COLUMN_TRANSFER_DURATION: -1}, inplace=True) # File with fare id num, fare id, fare class, price, xfers fare_rules_df.to_csv(os.path.join(self.output_dir, Route.OUTPUT_FARE_ID_FILE), - columns=[Route.FARE_RULES_COLUMN_FARE_ID_NUM, - Route.FARE_RULES_COLUMN_FARE_ID, - Route.FARE_ATTR_COLUMN_FARE_PERIOD, - Route.FARE_RULES_COLUMN_ROUTE_ID_NUM, - Route.FARE_RULES_COLUMN_ORIGIN_ID_NUM, - Route.FARE_RULES_COLUMN_DESTINATION_ID_NUM, - Route.FARE_RULES_COLUMN_START_TIME, - Route.FARE_RULES_COLUMN_END_TIME, - Route.FARE_ATTR_COLUMN_PRICE, - Route.FARE_ATTR_COLUMN_TRANSFERS, - Route.FARE_ATTR_COLUMN_TRANSFER_DURATION], - sep=" ", index=False) + columns=[Route.FARE_RULES_COLUMN_FARE_ID_NUM, + Route.FARE_RULES_COLUMN_FARE_ID, + Route.FARE_ATTR_COLUMN_FARE_PERIOD, + Route.FARE_RULES_COLUMN_ROUTE_ID_NUM, + Route.FARE_RULES_COLUMN_ORIGIN_ID_NUM, + Route.FARE_RULES_COLUMN_DESTINATION_ID_NUM, + Route.FARE_RULES_COLUMN_START_TIME, + Route.FARE_RULES_COLUMN_END_TIME, + Route.FARE_ATTR_COLUMN_PRICE, + Route.FARE_ATTR_COLUMN_TRANSFERS, + Route.FARE_ATTR_COLUMN_TRANSFER_DURATION], + sep=" ", index=False) FastTripsLogger.debug("Wrote %s" % os.path.join(self.output_dir, Route.OUTPUT_FARE_ID_FILE)) if len(self.fare_transfer_rules_df) > 0: @@ -663,7 +682,8 @@ def write_routes_for_extension(self): sep=" ", index=False) FastTripsLogger.debug("Wrote %s" % os.path.join(self.output_dir, Route.OUTPUT_FARE_TRANSFER_FILE)) else: - FastTripsLogger.debug("No fare rules so no file %s" % os.path.join(self.output_dir, Route.OUTPUT_FARE_ID_FILE)) + FastTripsLogger.debug( + "No fare rules so no file %s" % os.path.join(self.output_dir, Route.OUTPUT_FARE_ID_FILE)) def add_fares(self, trip_links_df): """ @@ -692,17 +712,17 @@ def add_fares(self, trip_links_df): # no fares configured if len(self.fare_rules_df) == 0: - trip_links_df[Assignment.SIM_COL_PAX_FARE ] = 0 - trip_links_df[Assignment.SIM_COL_PAX_FARE_PERIOD ] = None + trip_links_df[Assignment.SIM_COL_PAX_FARE] = 0 + trip_links_df[Assignment.SIM_COL_PAX_FARE_PERIOD] = None trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD] = None - trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE ] = None - trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT ] = None - trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER ] = None + trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE] = None + trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] = None + trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER] = None return trip_links_df - orig_columns = list(trip_links_df.columns.values) - fare_columns = [Assignment.SIM_COL_PAX_FARE, - Assignment.SIM_COL_PAX_FARE_PERIOD] + orig_columns = list(trip_links_df.columns.values) + fare_columns = [Assignment.SIM_COL_PAX_FARE, + Assignment.SIM_COL_PAX_FARE_PERIOD] transfer_columns = [Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD, Route.FARE_TRANSFER_RULES_COLUMN_TYPE, Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT, @@ -714,52 +734,58 @@ def add_fares(self, trip_links_df): num_trip_links = len(trip_links_df) FastTripsLogger.debug("add_fares initial trips (%d):\n%s" % (num_trip_links, str(trip_links_df.head(20)))) - FastTripsLogger.debug("add_fares initial fare_rules (%d):\n%s" % (len(self.fare_rules_df), str(self.fare_rules_df.head(20)))) + FastTripsLogger.debug( + "add_fares initial fare_rules (%d):\n%s" % (len(self.fare_rules_df), str(self.fare_rules_df.head(20)))) # initialize trip_links_unmatched = trip_links_df - trip_links_matched = pd.DataFrame() + trip_links_matched = pd.DataFrame() del trip_links_df from .Passenger import Passenger # level 0: match on all three - fare_rules0 = self.fare_rules_df.loc[pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID ])& - pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID ])& + fare_rules0 = self.fare_rules_df.loc[pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID]) & + pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) & pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])] if len(fare_rules0) > 0: - trip_links_match0 = pd.merge(left =trip_links_unmatched, - right =fare_rules0, - how ="inner", - left_on =[Route.FARE_RULES_COLUMN_ROUTE_ID,"A_zone_id","B_zone_id"], - right_on =[Route.FARE_RULES_COLUMN_ROUTE_ID,Route.FARE_RULES_COLUMN_ORIGIN_ID,Route.FARE_RULES_COLUMN_DESTINATION_ID], - suffixes =["","_fare_rules"]) + trip_links_match0 = pd.merge(left=trip_links_unmatched, + right=fare_rules0, + how="inner", + left_on=[Route.FARE_RULES_COLUMN_ROUTE_ID, "A_zone_id", "B_zone_id"], + right_on=[Route.FARE_RULES_COLUMN_ROUTE_ID, Route.FARE_RULES_COLUMN_ORIGIN_ID, + Route.FARE_RULES_COLUMN_DESTINATION_ID], + suffixes=["", "_fare_rules"]) # delete rows where the board time is not within the fare period - trip_links_match0 = trip_links_match0.loc[ pd.isnull(trip_links_match0[Route.FARE_ATTR_COLUMN_PRICE])| - ((trip_links_match0[Assignment.SIM_COL_PAX_BOARD_TIME] >= trip_links_match0[Route.FARE_RULES_COLUMN_START_TIME])& - (trip_links_match0[Assignment.SIM_COL_PAX_BOARD_TIME] < trip_links_match0[Route.FARE_RULES_COLUMN_END_TIME])) ] - FastTripsLogger.debug("add_fares level 0 (%d):\n%s" % (len(trip_links_match0), str(trip_links_match0.head(20)))) + trip_links_match0 = trip_links_match0.loc[pd.isnull(trip_links_match0[Route.FARE_ATTR_COLUMN_PRICE]) | + ((trip_links_match0[Assignment.SIM_COL_PAX_BOARD_TIME] >= + trip_links_match0[Route.FARE_RULES_COLUMN_START_TIME]) & + (trip_links_match0[Assignment.SIM_COL_PAX_BOARD_TIME] < + trip_links_match0[Route.FARE_RULES_COLUMN_END_TIME]))] + FastTripsLogger.debug( + "add_fares level 0 (%d):\n%s" % (len(trip_links_match0), str(trip_links_match0.head(20)))) if len(trip_links_match0) > 0: - # update matched and unmatched == they should be disjoint with union = whole - trip_links_unmatched = pd.merge(left =trip_links_unmatched, - right=trip_links_match0[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]], - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - indicator=True) - trip_links_unmatched = trip_links_unmatched.loc[ trip_links_unmatched["_merge"] == "left_only" ] + trip_links_unmatched = pd.merge(left=trip_links_unmatched, + right=trip_links_match0[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]], + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + indicator=True) + trip_links_unmatched = trip_links_unmatched.loc[trip_links_unmatched["_merge"] == "left_only"] trip_links_unmatched.drop(["_merge"], axis=1, inplace=True) trip_links_matched = pd.concat([trip_links_matched, trip_links_match0], axis=0, copy=False) - FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % (len(trip_links_matched), len(trip_links_unmatched), len(trip_links_matched)+len(trip_links_unmatched))) + FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % ( + len(trip_links_matched), len(trip_links_unmatched), + len(trip_links_matched) + len(trip_links_unmatched))) del trip_links_match0 # TODO - Addding stop gap solution - if there are duplicates, drop them @@ -771,84 +797,96 @@ def add_fares(self, trip_links_df): trip_links_matched.drop_duplicates(subset="trip_links_df index", keep="first", inplace=True) # level 1: match on route only - fare_rules1 = self.fare_rules_df.loc[pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID ])& - pd.isnull (self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID ])& - pd.isnull (self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])] + fare_rules1 = self.fare_rules_df.loc[pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID]) & + pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) & + pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])] if len(fare_rules1) > 0: - trip_links_match1 = pd.merge(left =trip_links_unmatched, - right =fare_rules1, - how ="inner", - on =Route.FARE_RULES_COLUMN_ROUTE_ID, - suffixes =["","_fare_rules"]) + trip_links_match1 = pd.merge(left=trip_links_unmatched, + right=fare_rules1, + how="inner", + on=Route.FARE_RULES_COLUMN_ROUTE_ID, + suffixes=["", "_fare_rules"]) # delete rows where the board time is not within the fare period - trip_links_match1 = trip_links_match1.loc[ pd.isnull(trip_links_match1[Route.FARE_ATTR_COLUMN_PRICE])| - ((trip_links_match1[Assignment.SIM_COL_PAX_BOARD_TIME] >= trip_links_match1[Route.FARE_RULES_COLUMN_START_TIME])& - (trip_links_match1[Assignment.SIM_COL_PAX_BOARD_TIME] < trip_links_match1[Route.FARE_RULES_COLUMN_END_TIME])) ] - FastTripsLogger.debug("add_fares level 1 (%d):\n%s" % (len(trip_links_match1), str(trip_links_match1.head()))) + trip_links_match1 = trip_links_match1.loc[pd.isnull(trip_links_match1[Route.FARE_ATTR_COLUMN_PRICE]) | + ((trip_links_match1[Assignment.SIM_COL_PAX_BOARD_TIME] >= + trip_links_match1[Route.FARE_RULES_COLUMN_START_TIME]) & + (trip_links_match1[Assignment.SIM_COL_PAX_BOARD_TIME] < + trip_links_match1[Route.FARE_RULES_COLUMN_END_TIME]))] + FastTripsLogger.debug( + "add_fares level 1 (%d):\n%s" % (len(trip_links_match1), str(trip_links_match1.head()))) if len(trip_links_match1) > 0: # update matched and unmatched == they should be disjoint with union = whole - trip_links_unmatched = pd.merge(left =trip_links_unmatched, - right=trip_links_match1[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]], - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - indicator=True) - trip_links_unmatched = trip_links_unmatched.loc[ trip_links_unmatched["_merge"] == "left_only" ] + trip_links_unmatched = pd.merge(left=trip_links_unmatched, + right=trip_links_match1[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]], + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + indicator=True) + trip_links_unmatched = trip_links_unmatched.loc[trip_links_unmatched["_merge"] == "left_only"] trip_links_unmatched.drop(["_merge"], axis=1, inplace=True) trip_links_matched = pd.concat([trip_links_matched, trip_links_match1], axis=0, copy=False) - FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % (len(trip_links_matched), len(trip_links_unmatched), len(trip_links_matched)+len(trip_links_unmatched))) + FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % ( + len(trip_links_matched), len(trip_links_unmatched), + len(trip_links_matched) + len(trip_links_unmatched))) del trip_links_match1 # level 2: match on origin and destination zones only - fare_rules2 = self.fare_rules_df.loc[pd.isnull (self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID ])& - pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID ])& + fare_rules2 = self.fare_rules_df.loc[pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID]) & + pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) & pd.notnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])] if len(fare_rules2) > 0: - trip_links_match2 = pd.merge(left =trip_links_unmatched, - right =fare_rules2, - how ="inner", - left_on =["A_zone_id","B_zone_id"], - right_on =[Route.FARE_RULES_COLUMN_ORIGIN_ID,Route.FARE_RULES_COLUMN_DESTINATION_ID], - suffixes =["","_fare_rules"]) + trip_links_match2 = pd.merge(left=trip_links_unmatched, + right=fare_rules2, + how="inner", + left_on=["A_zone_id", "B_zone_id"], + right_on=[Route.FARE_RULES_COLUMN_ORIGIN_ID, + Route.FARE_RULES_COLUMN_DESTINATION_ID], + suffixes=["", "_fare_rules"]) # delete rows where the board time is not within the fare period - trip_links_match2 = trip_links_match2.loc[ pd.isnull(trip_links_match2[Route.FARE_ATTR_COLUMN_PRICE])| - ((trip_links_match2[Assignment.SIM_COL_PAX_BOARD_TIME] >= trip_links_match2[Route.FARE_RULES_COLUMN_START_TIME])& - (trip_links_match2[Assignment.SIM_COL_PAX_BOARD_TIME] < trip_links_match2[Route.FARE_RULES_COLUMN_END_TIME])) ] - FastTripsLogger.debug("add_fares level 2 (%d):\n%s" % (len(trip_links_match2), str(trip_links_match2.head()))) + trip_links_match2 = trip_links_match2.loc[pd.isnull(trip_links_match2[Route.FARE_ATTR_COLUMN_PRICE]) | + ((trip_links_match2[Assignment.SIM_COL_PAX_BOARD_TIME] >= + trip_links_match2[Route.FARE_RULES_COLUMN_START_TIME]) & + (trip_links_match2[Assignment.SIM_COL_PAX_BOARD_TIME] < + trip_links_match2[Route.FARE_RULES_COLUMN_END_TIME]))] + FastTripsLogger.debug( + "add_fares level 2 (%d):\n%s" % (len(trip_links_match2), str(trip_links_match2.head()))) if len(trip_links_match2) > 0: # update matched and unmatched == they should be disjoint with union = whole - trip_links_unmatched = pd.merge(left =trip_links_unmatched, - right=trip_links_match2[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]], - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - indicator=True) - trip_links_unmatched = trip_links_unmatched.loc[ trip_links_unmatched["_merge"] == "left_only" ] + trip_links_unmatched = pd.merge(left=trip_links_unmatched, + right=trip_links_match2[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]], + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + indicator=True) + trip_links_unmatched = trip_links_unmatched.loc[trip_links_unmatched["_merge"] == "left_only"] trip_links_unmatched.drop(["_merge"], axis=1, inplace=True) trip_links_matched = pd.concat([trip_links_matched, trip_links_match2], axis=0, copy=False) - FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % (len(trip_links_matched), len(trip_links_unmatched), len(trip_links_matched)+len(trip_links_unmatched))) + FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % ( + len(trip_links_matched), len(trip_links_unmatched), + len(trip_links_matched) + len(trip_links_unmatched))) del trip_links_match2 # level 3: no route, origin or destination specified - fare_rules3 = self.fare_rules_df.loc[pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID ])& - pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID ])& - pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])].copy() + fare_rules3 = self.fare_rules_df.loc[pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ROUTE_ID]) & + pd.isnull(self.fare_rules_df[Route.FARE_RULES_COLUMN_ORIGIN_ID]) & + pd.isnull( + self.fare_rules_df[Route.FARE_RULES_COLUMN_DESTINATION_ID])].copy() if len(fare_rules3) > 0: # need a column to merge on merge_column = "fare level 3 merge col" @@ -857,39 +895,44 @@ def add_fares(self, trip_links_df): FastTripsLogger.debug("fare_rules3 (%d):\n%s" % (len(fare_rules3), str(fare_rules3.head()))) - trip_links_match3 = pd.merge(left =trip_links_unmatched, - right =fare_rules3, - how ="inner", - on =merge_column, - suffixes =["","_fare_rules"]) + trip_links_match3 = pd.merge(left=trip_links_unmatched, + right=fare_rules3, + how="inner", + on=merge_column, + suffixes=["", "_fare_rules"]) trip_links_match3.drop([merge_column], axis=1, inplace=True) trip_links_unmatched.drop([merge_column], axis=1, inplace=True) # delete rows where the board time is not within the fare period - trip_links_match3 = trip_links_match3.loc[ pd.isnull(trip_links_match3[Route.FARE_ATTR_COLUMN_PRICE])| - ((trip_links_match3[Assignment.SIM_COL_PAX_BOARD_TIME] >= trip_links_match3[Route.FARE_RULES_COLUMN_START_TIME])& - (trip_links_match3[Assignment.SIM_COL_PAX_BOARD_TIME] < trip_links_match3[Route.FARE_RULES_COLUMN_END_TIME])) ] - FastTripsLogger.debug("add_fares level 3 (%d):\n%s" % (len(trip_links_match3), str(trip_links_match3.head()))) + trip_links_match3 = trip_links_match3.loc[pd.isnull(trip_links_match3[Route.FARE_ATTR_COLUMN_PRICE]) | + ((trip_links_match3[Assignment.SIM_COL_PAX_BOARD_TIME] >= + trip_links_match3[Route.FARE_RULES_COLUMN_START_TIME]) & + (trip_links_match3[Assignment.SIM_COL_PAX_BOARD_TIME] < + trip_links_match3[Route.FARE_RULES_COLUMN_END_TIME]))] + FastTripsLogger.debug( + "add_fares level 3 (%d):\n%s" % (len(trip_links_match3), str(trip_links_match3.head()))) if len(trip_links_match3) > 0: # update matched and unmatched == they should be disjoint with union = whole - trip_links_unmatched = pd.merge(left =trip_links_unmatched, - right=trip_links_match3[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM]], - how ="left", - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - indicator=True) - trip_links_unmatched = trip_links_unmatched.loc[ trip_links_unmatched["_merge"] == "left_only" ] + trip_links_unmatched = pd.merge(left=trip_links_unmatched, + right=trip_links_match3[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM]], + how="left", + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + indicator=True) + trip_links_unmatched = trip_links_unmatched.loc[trip_links_unmatched["_merge"] == "left_only"] trip_links_unmatched.drop(["_merge"], axis=1, inplace=True) trip_links_matched = pd.concat([trip_links_matched, trip_links_match3], axis=0, copy=False) - FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % (len(trip_links_matched), len(trip_links_unmatched), len(trip_links_matched)+len(trip_links_unmatched))) + FastTripsLogger.debug("matched: %d unmatched: %d total: %d" % ( + len(trip_links_matched), len(trip_links_unmatched), + len(trip_links_matched) + len(trip_links_unmatched))) del trip_links_match3 # put them together @@ -899,18 +942,19 @@ def add_fares(self, trip_links_df): Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM, Passenger.PF_COL_LINK_NUM], - inplace=True) + inplace=True) del trip_links_matched del trip_links_unmatched # rename price to fare - trip_links_df.rename(columns={Route.FARE_ATTR_COLUMN_PRICE:Assignment.SIM_COL_PAX_FARE}, inplace=True) + trip_links_df.rename(columns={Route.FARE_ATTR_COLUMN_PRICE: Assignment.SIM_COL_PAX_FARE}, inplace=True) # join fails mean 0 - trip_links_df.fillna(value={Assignment.SIM_COL_PAX_FARE:0.0}, inplace=True) + trip_links_df.fillna(value={Assignment.SIM_COL_PAX_FARE: 0.0}, inplace=True) # reorder columns - trip_links_df = trip_links_df[orig_columns + fare_columns + [Route.FARE_ATTR_COLUMN_TRANSFERS, Route.FARE_ATTR_COLUMN_TRANSFER_DURATION]] + trip_links_df = trip_links_df[ + orig_columns + fare_columns + [Route.FARE_ATTR_COLUMN_TRANSFERS, Route.FARE_ATTR_COLUMN_TRANSFER_DURATION]] FastTripsLogger.debug("trip_links_df (%d):\n%s" % (len(trip_links_df), str(trip_links_df.head()))) @@ -947,39 +991,39 @@ def apply_fare_transfer_rules(self, trip_links_df): # no transfer rules => nothing to do if len(self.fare_transfer_rules_df) == 0: trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD] = None - trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE] = None - trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] = None + trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE] = None + trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] = None return trip_links_df # FastTripsLogger.debug("apply_fare_transfers (%d):\n%s" % (len(trip_links_df), str(trip_links_df.head(20)))) # previous trip link trip_links_df["%s prev" % Passenger.PF_COL_LINK_NUM] = trip_links_df[Passenger.PF_COL_LINK_NUM] - 2 - trip_links_df = pd.merge(left =trip_links_df, - right =trip_links_df[[Passenger.PERSONS_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - Assignment.SIM_COL_PAX_FARE_PERIOD]], - left_on =[Passenger.PERSONS_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - "%s prev" % Passenger.PF_COL_LINK_NUM], - right_on=[Passenger.PERSONS_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - suffixes=["","_prev"], - how ="left") + trip_links_df = pd.merge(left=trip_links_df, + right=trip_links_df[[Passenger.PERSONS_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + Assignment.SIM_COL_PAX_FARE_PERIOD]], + left_on=[Passenger.PERSONS_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + "%s prev" % Passenger.PF_COL_LINK_NUM], + right_on=[Passenger.PERSONS_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + suffixes=["", "_prev"], + how="left") # extra columns are linknum prev, linknum_prev, fare_prev, fare_period_prev, # join with transfers table - trip_links_df = pd.merge(left =trip_links_df, - right =self.fare_transfer_rules_df, - left_on =["fare_period_prev","fare_period"], - right_on=[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD, - Route.FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD], - how ="left") + trip_links_df = pd.merge(left=trip_links_df, + right=self.fare_transfer_rules_df, + left_on=["fare_period_prev", "fare_period"], + right_on=[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD, + Route.FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD], + how="left") # FastTripsLogger.debug("apply_fare_transfers (%d):\n%s" % (len(trip_links_df), str(trip_links_df.head(20)))) # keep Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD, Route.FARE_TRANSFER_RULES_COLUMN_TYPE, Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT @@ -991,27 +1035,28 @@ def apply_fare_transfer_rules(self, trip_links_df): # FastTripsLogger.debug("apply_fare_transfers (%d):\n%s" % (len(trip_links_df), str(trip_links_df.head(20)))) # apply transfer discount - trip_links_df.loc[ pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE])& - (trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]==Route.TRANSFER_TYPE_TRANSFER_DISCOUNT), - Assignment.SIM_COL_PAX_FARE ] = trip_links_df[Assignment.SIM_COL_PAX_FARE] - trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] + trip_links_df.loc[pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]) & + (trip_links_df[ + Route.FARE_TRANSFER_RULES_COLUMN_TYPE] == Route.TRANSFER_TYPE_TRANSFER_DISCOUNT), + Assignment.SIM_COL_PAX_FARE] = trip_links_df[Assignment.SIM_COL_PAX_FARE] - trip_links_df[ + Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] # apply transfer free - trip_links_df.loc[ pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE])& - (trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]==Route.TRANSFER_TYPE_TRANSFER_FREE), - Assignment.SIM_COL_PAX_FARE ] = 0.0 + trip_links_df.loc[pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]) & + (trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE] == Route.TRANSFER_TYPE_TRANSFER_FREE), + Assignment.SIM_COL_PAX_FARE] = 0.0 # apply transfer fare - trip_links_df.loc[ pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE])& - (trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]==Route.TRANSFER_TYPE_TRANSFER_COST), - Assignment.SIM_COL_PAX_FARE ] = trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] + trip_links_df.loc[pd.notnull(trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE]) & + (trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_TYPE] == Route.TRANSFER_TYPE_TRANSFER_COST), + Assignment.SIM_COL_PAX_FARE] = trip_links_df[Route.FARE_TRANSFER_RULES_COLUMN_AMOUNT] # make sure it's not negative - trip_links_df.loc[ trip_links_df[Assignment.SIM_COL_PAX_FARE] < 0, Assignment.SIM_COL_PAX_FARE] = 0.0 + trip_links_df.loc[trip_links_df[Assignment.SIM_COL_PAX_FARE] < 0, Assignment.SIM_COL_PAX_FARE] = 0.0 FastTripsLogger.debug("apply_fare_transfers (%d):\n%s" % (len(trip_links_df), str(trip_links_df.head(20)))) return trip_links_df - def apply_free_transfers(self, trip_links_df): """ Apply the free transfers allowed in to trip_links_df fare_attributes_ft.txt (configured by columns transfers, transfer_duration). @@ -1019,16 +1064,15 @@ def apply_free_transfers(self, trip_links_df): """ # free transfers within a fare id from .Assignment import Assignment - from .Passenger import Passenger - from .PathSet import PathSet + from .Passenger import Passenger + from .PathSet import PathSet # create a fare_index that counts up for a unique person-trip id, pathnum, and fare_period trip_links_df["fare_index"] = trip_links_df.groupby([Passenger.TRIP_LIST_COLUMN_PERSON_ID, Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, Passenger.PF_COL_PATH_NUM, Assignment.SIM_COL_PAX_FARE_PERIOD]).cumcount() - trip_links_df.loc[ trip_links_df[Passenger.PF_COL_LINK_MODE]!=PathSet.STATE_MODE_TRIP, "fare_index"] = -1 - + trip_links_df.loc[trip_links_df[Passenger.PF_COL_LINK_MODE] != PathSet.STATE_MODE_TRIP, "fare_index"] = -1 # transfer_time in seconds (to compare with transfer_duration) get the first fare board first_fare_board = trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, @@ -1037,64 +1081,75 @@ def apply_free_transfers(self, trip_links_df): Passenger.PF_COL_LINK_NUM, Assignment.SIM_COL_PAX_FARE_PERIOD, "fare_index", - Assignment.SIM_COL_PAX_BOARD_TIME]].loc[trip_links_df["fare_index"]==0] + Assignment.SIM_COL_PAX_BOARD_TIME]].loc[trip_links_df["fare_index"] == 0] FastTripsLogger.debug("apply_free_transfers: first_fare_board=\n%s" % str(first_fare_board.head(10))) - trip_links_df = pd.merge(left =trip_links_df, - right =first_fare_board, - on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Assignment.SIM_COL_PAX_FARE_PERIOD], - how ="left", - suffixes=["","_ffb"]) + trip_links_df = pd.merge(left=trip_links_df, + right=first_fare_board, + on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Assignment.SIM_COL_PAX_FARE_PERIOD], + how="left", + suffixes=["", "_ffb"]) # calculate time from first board (for this fare period id) in seconds - trip_links_df["transfer_time_sec"] = (trip_links_df[Assignment.SIM_COL_PAX_BOARD_TIME]-trip_links_df["%s_ffb" % Assignment.SIM_COL_PAX_BOARD_TIME])/np.timedelta64(1,'s') + trip_links_df["transfer_time_sec"] = (trip_links_df[Assignment.SIM_COL_PAX_BOARD_TIME] - trip_links_df[ + "%s_ffb" % Assignment.SIM_COL_PAX_BOARD_TIME]) / np.timedelta64(1, 's') # FastTripsLogger.debug("apply_free_transfers: trip_links_df=\n%s" % str(trip_links_df.loc[ trip_links_df["transfer_time_sec"] >0 ])) trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER] = 0.0 # free transfer if transfers > 0 and 0 < fare_index <= transfers - trip_links_df.loc[ (trip_links_df[Route.FARE_ATTR_COLUMN_TRANSFERS] > 0)& # transfers > 0 - (trip_links_df["fare_index"]>0)& # is a transfer - (trip_links_df["fare_index"]<=trip_links_df[Route.FARE_ATTR_COLUMN_TRANSFERS]), # is within the number of free transfers allowed - Assignment.SIM_COL_PAX_FREE_TRANSFER] = 1.0 + trip_links_df.loc[(trip_links_df[Route.FARE_ATTR_COLUMN_TRANSFERS] > 0) & # transfers > 0 + (trip_links_df["fare_index"] > 0) & # is a transfer + (trip_links_df["fare_index"] <= trip_links_df[ + Route.FARE_ATTR_COLUMN_TRANSFERS]), # is within the number of free transfers allowed + Assignment.SIM_COL_PAX_FREE_TRANSFER] = 1.0 # only applicable to transit links - trip_links_df.loc[ trip_links_df[Passenger.PF_COL_LINK_MODE]!=PathSet.STATE_MODE_TRIP, - Assignment.SIM_COL_PAX_FREE_TRANSFER ] = None + trip_links_df.loc[trip_links_df[Passenger.PF_COL_LINK_MODE] != PathSet.STATE_MODE_TRIP, + Assignment.SIM_COL_PAX_FREE_TRANSFER] = None # only applicable if transfer is within transfer_duration -- revoke if transfer_time_sec > transfer duration - trip_links_df.loc[ (trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER]==1.0) & - (trip_links_df["transfer_time_sec"] > trip_links_df[Route.FARE_ATTR_COLUMN_TRANSFER_DURATION]), - Assignment.SIM_COL_PAX_FREE_TRANSFER] = 0.0 + trip_links_df.loc[(trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER] == 1.0) & + (trip_links_df["transfer_time_sec"] > trip_links_df[ + Route.FARE_ATTR_COLUMN_TRANSFER_DURATION]), + Assignment.SIM_COL_PAX_FREE_TRANSFER] = 0.0 # make the transfer free - trip_links_df.loc[ trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER]==1.0, Assignment.SIM_COL_PAX_FARE] = 0.0 + trip_links_df.loc[trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER] == 1.0, Assignment.SIM_COL_PAX_FARE] = 0.0 # debug: show transfers within fare period - FastTripsLogger.debug("apply_free_transfers: fare_index>0\n%s" % str(trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - Passenger.PF_COL_LINK_MODE, - Assignment.SIM_COL_PAX_BOARD_TIME, - Assignment.SIM_COL_PAX_FARE_PERIOD, - Route.FARE_ATTR_COLUMN_TRANSFERS, - Route.FARE_ATTR_COLUMN_TRANSFER_DURATION,"transfer_time_sec", - "fare_index",Assignment.SIM_COL_PAX_FREE_TRANSFER]].loc[trip_links_df["fare_index"] > 0].head(10))) + FastTripsLogger.debug( + "apply_free_transfers: fare_index>0\n%s" % str(trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + Passenger.PF_COL_LINK_MODE, + Assignment.SIM_COL_PAX_BOARD_TIME, + Assignment.SIM_COL_PAX_FARE_PERIOD, + Route.FARE_ATTR_COLUMN_TRANSFERS, + Route.FARE_ATTR_COLUMN_TRANSFER_DURATION, + "transfer_time_sec", + "fare_index", + Assignment.SIM_COL_PAX_FREE_TRANSFER]].loc[ + trip_links_df["fare_index"] > 0].head(10))) # debug: show transfers within fare period - FastTripsLogger.debug("apply_free_transfers: free_transfer=1.0\n%s" % str(trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - Passenger.PF_COL_LINK_MODE, - Assignment.SIM_COL_PAX_BOARD_TIME, - Assignment.SIM_COL_PAX_FARE_PERIOD, - Route.FARE_ATTR_COLUMN_TRANSFERS, - Route.FARE_ATTR_COLUMN_TRANSFER_DURATION,"transfer_time_sec", - "fare_index",Assignment.SIM_COL_PAX_FREE_TRANSFER]].loc[trip_links_df[Assignment.SIM_COL_PAX_FREE_TRANSFER] > 0].head(10))) - - + FastTripsLogger.debug( + "apply_free_transfers: free_transfer=1.0\n%s" % str(trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + Passenger.PF_COL_LINK_MODE, + Assignment.SIM_COL_PAX_BOARD_TIME, + Assignment.SIM_COL_PAX_FARE_PERIOD, + Route.FARE_ATTR_COLUMN_TRANSFERS, + Route.FARE_ATTR_COLUMN_TRANSFER_DURATION, + "transfer_time_sec", + "fare_index", + Assignment.SIM_COL_PAX_FREE_TRANSFER]].loc[ + trip_links_df[ + Assignment.SIM_COL_PAX_FREE_TRANSFER] > 0].head( + 10))) # drop new columns trip_links_df.drop(["fare_index", "fare_index_ffb", "transfer_time_sec", diff --git a/fasttrips/Run.py b/fasttrips/Run.py index 8a853a8a..a7e248ee 100644 --- a/fasttrips/Run.py +++ b/fasttrips/Run.py @@ -5,8 +5,9 @@ from __future__ import absolute_import from builtins import str + __copyright__ = "Copyright 2015-2017 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -19,6 +20,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import argparse import datetime import os @@ -33,13 +35,12 @@ def run_setup(input_network_dir, run_config, iters, output_dir, - pathfinding_type = 'stochastic', - input_functions = None, - output_folder = None, - trace_only = False, + pathfinding_type='stochastic', + input_functions=None, + output_folder=None, + trace_only=False, **kwargs): - """ Reads run configuration files from network and demand input directories. If additional parameters are input here, they will override the run configuration files. @@ -98,22 +99,23 @@ def run_setup(input_network_dir, FastTripsLogger.fatal(msg) raise fasttrips.ConfigurationError("external input", msg) - if pathfinding_type not in ['deterministic','stochastic','file']: - msg = "pathfinding.type [%s] not defined. Expected values: %s" % (pathfinding_type,['deterministic','stochastic','file']) + if pathfinding_type not in ['deterministic', 'stochastic', 'file']: + msg = "pathfinding.type [%s] not defined. Expected values: %s" % ( + pathfinding_type, ['deterministic', 'stochastic', 'file']) FastTripsLogger.fatal(msg) raise fasttrips.ConfigurationError("external override", msg) - # Setup Output Directory if not output_folder: - output_folder = "output_%s_iter%d_%s" % (pathfinding_type, iters, "cap" if ("capacity" in kwargs and kwargs["capacity"]==True) else "nocap") + output_folder = "output_%s_iter%d_%s" % ( + pathfinding_type, iters, "cap" if ("capacity" in kwargs and kwargs["capacity"] == True) else "nocap") # don't override full run results if trace_only: output_folder = "%s_trace" % output_folder if not output_dir: - output_dir = os.path.basename(input_demand_dir) + output_dir = os.path.basename(input_demand_dir) # create folder if it doesn't already exist full_output_dir = os.path.join(output_dir, output_folder) @@ -126,16 +128,17 @@ def run_setup(input_network_dir, os.mkdir(full_output_dir) # Create fast-trips instance - ft = fasttrips.FastTrips(input_network_dir, input_demand_dir, input_weights, run_config, full_output_dir, input_functions=input_functions) + ft = fasttrips.FastTrips(input_network_dir, input_demand_dir, input_weights, run_config, full_output_dir, + input_functions=input_functions) # Read the configuration file and overwrite with any options called with the function call ft.read_configuration() if iters > 0: - fasttrips.Assignment.MAX_ITERATIONS = int(iters) + fasttrips.Assignment.MAX_ITERATIONS = int(iters) if "pf_iters" in kwargs: - fasttrips.Assignment.MAX_PF_ITERATIONS = kwargs["pf_iters"] + fasttrips.Assignment.MAX_PF_ITERATIONS = kwargs["pf_iters"] if "number_of_processes" in kwargs: fasttrips.Assignment.NUMBER_OF_PROCESSES = kwargs["number_of_processes"] @@ -147,31 +150,31 @@ def run_setup(input_network_dir, if len(fasttrips.Assignment.TRACE_IDS) == 0: print("Trace only requested but no trace IDs are specified in configuration.") sys.exit(2) - fasttrips.Assignment.DEBUG_TRACE_ONLY = True + fasttrips.Assignment.DEBUG_TRACE_ONLY = True fasttrips.Assignment.NUMBER_OF_PROCESSES = 1 if "pathfinding_type" in list(kwargs.keys()): - fasttrips.Assignment.PATHFINDING_TYPE = kwargs["pathfinding_type"] + fasttrips.Assignment.PATHFINDING_TYPE = kwargs["pathfinding_type"] if "learning_convergence" in list(kwargs.keys()): fasttrips.PathSet.LEARN_ROUTES = kwargs["learning_convergence"] if "max_stop_process_count" in list(kwargs.keys()): - fasttrips.Assignment.STOCH_MAX_STOP_PROCESS_COUNT = kwargs["max_stop_process_count"] if "debug_output_columns" in list(kwargs.keys()): fasttrips.Assignment.DEBUG_OUTPUT_COLUMNS = kwargs["debug_output_columns"] if "overlap_variable" in list(kwargs.keys()): - if kwargs["overlap_variable"] not in ['None','count','distance','time']: - msg = "pathfinding.overlap_variable [%s] not defined. Expected values: %s" % (kwargs["overlap_variable"], str(fasttrips.PathSet.OVERLAP_VARIABLE_OPTIONS)) + if kwargs["overlap_variable"] not in ['None', 'count', 'distance', 'time']: + msg = "pathfinding.overlap_variable [%s] not defined. Expected values: %s" % ( + kwargs["overlap_variable"], str(fasttrips.PathSet.OVERLAP_VARIABLE_OPTIONS)) fasttrips.FastTripsLogger.fatal(msg) raise fasttrips.ConfigurationError("external override", msg) - fasttrips.PathSet.OVERLAP_VARIABLE = kwargs["overlap_variable"] + fasttrips.PathSet.OVERLAP_VARIABLE = kwargs["overlap_variable"] if "overlap_split_transit" in list(kwargs.keys()): - fasttrips.PathSet.OVERLAP_SPLIT_TRANSIT = kwargs["overlap_split_transit"] + fasttrips.PathSet.OVERLAP_SPLIT_TRANSIT = kwargs["overlap_split_transit"] if "transfer_fare_ignore_pathfinding" in list(kwargs.keys()): fasttrips.Assignment.TRANSFER_FARE_IGNORE_PATHFINDING = kwargs["transfer_fare_ignore_pathfinding"] @@ -180,16 +183,16 @@ def run_setup(input_network_dir, fasttrips.Assignment.TRANSFER_FARE_IGNORE_PATHENUM = kwargs["transfer_fare_ignore_pathenum"] if "time_window" in list(kwargs.keys()): - fasttrips.Assignment.TIME_WINDOW = datetime.timedelta(minutes=float(kwargs["time_window"])) + fasttrips.Assignment.TIME_WINDOW = datetime.timedelta(minutes=float(kwargs["time_window"])) if "utils_conversion_factor" in list(kwargs.keys()): - fasttrips.Assignment.UTILS_CONVERSION = kwargs["utils_conversion_factor"] + fasttrips.Assignment.UTILS_CONVERSION = kwargs["utils_conversion_factor"] if "dispersion" in list(kwargs.keys()): - fasttrips.Assignment.STOCH_DISPERSION = kwargs["dispersion"] + fasttrips.Assignment.STOCH_DISPERSION = kwargs["dispersion"] if "num_trips" in list(kwargs.keys()): - fasttrips.Assignment.DEBUG_NUM_TRIPS = kwargs["num_trips"] + fasttrips.Assignment.DEBUG_NUM_TRIPS = kwargs["num_trips"] if "capacity" in list(kwargs.keys()): fasttrips.Assignment.CAPACITY_CONSTRAINT = kwargs["capacity"] @@ -198,7 +201,7 @@ def run_setup(input_network_dir, fasttrips.Assignment.OUTPUT_PATHSET_PER_SIM_ITER = kwargs["output_pathset_per_sim_iter"] if "user_class_function" in list(kwargs.keys()): - fasttrips.PathSet.USER_CLASS_FUNCTION = kwargs["user_class_function"] + fasttrips.PathSet.USER_CLASS_FUNCTION = kwargs["user_class_function"] return ft @@ -228,42 +231,50 @@ def run_fasttrips(**kwargs): """ + def main(): """ Does arg parsing for command line interface. """ def str2bool(v): - #susendberg's function + # susendberg's function return v.lower() in ("yes", "true", "t", "1") parser = argparse.ArgumentParser(usage=USAGE) - parser.register('type','bool',str2bool) - parser.add_argument('-t','--trace_only', action='store_true', help="Run only the trace persons?") - parser.add_argument('-n','--num_trips', type=int, help="Number of person trips to run, to run a subset of the whole demand.") - parser.add_argument('-d','--dispersion', type=float,help="Stochastic dispersion parameter") - parser.add_argument('-m','--max_stop_process_count', type=int, help="Max times to process a stop in stochastic pathfinding") - parser.add_argument('-c','--capacity', action='store_true', help="Enable capacity constraint") - parser.add_argument('-o','--output_folder', type=str, help="Directory within output_loc to write fasttrips outtput. If none specified, will construct one.") - parser.add_argument('--debug_output_columns', action='store_true', help="Include debug columns in output") - parser.add_argument('--overlap_variable', choices=['None','count','distance','time'], help="Variable to use for overlap penalty calculation") - parser.add_argument('--overlap_split_transit', action='store_true', help="Split transit for path overlap penalty calculation") - parser.add_argument('--transfer_fare_ignore_pathfinding', action='store_true', help="In path-finding, suppress trying to adjust fares using transfer rules. For performance.") - parser.add_argument('--transfer_fare_ignore_pathenum', action='store_true', help="In path-enumeration, suppress trying to adjust fares using transfer rules. For performance.") - parser.add_argument("pathfinding_type", choices=['deterministic','stochastic','file'], help="Type of pathfinding") - parser.add_argument("iters", type=int, help="Number of iterations to run") - parser.add_argument("run_config", type=str, help="The run configuration file") - parser.add_argument("input_network_dir", type=str, help="Location of the input network") - parser.add_argument("input_demand_dir", type=str, help="Location of the input demand") - parser.add_argument("input_weights", type=str, help="Location of the pathweights file") - parser.add_argument("output_dir", type=str, help="Location to write fasttrips output") + parser.register('type', 'bool', str2bool) + parser.add_argument('-t', '--trace_only', action='store_true', help="Run only the trace persons?") + parser.add_argument('-n', '--num_trips', type=int, + help="Number of person trips to run, to run a subset of the whole demand.") + parser.add_argument('-d', '--dispersion', type=float, help="Stochastic dispersion parameter") + parser.add_argument('-m', '--max_stop_process_count', type=int, + help="Max times to process a stop in stochastic pathfinding") + parser.add_argument('-c', '--capacity', action='store_true', help="Enable capacity constraint") + parser.add_argument('-o', '--output_folder', type=str, + help="Directory within output_loc to write fasttrips outtput. If none specified, will construct one.") + parser.add_argument('--debug_output_columns', action='store_true', help="Include debug columns in output") + parser.add_argument('--overlap_variable', choices=['None', 'count', 'distance', 'time'], + help="Variable to use for overlap penalty calculation") + parser.add_argument('--overlap_split_transit', action='store_true', + help="Split transit for path overlap penalty calculation") + parser.add_argument('--transfer_fare_ignore_pathfinding', action='store_true', + help="In path-finding, suppress trying to adjust fares using transfer rules. For performance.") + parser.add_argument('--transfer_fare_ignore_pathenum', action='store_true', + help="In path-enumeration, suppress trying to adjust fares using transfer rules. For performance.") + parser.add_argument("pathfinding_type", choices=['deterministic', 'stochastic', 'file'], help="Type of pathfinding") + parser.add_argument("iters", type=int, help="Number of iterations to run") + parser.add_argument("run_config", type=str, help="The run configuration file") + parser.add_argument("input_network_dir", type=str, help="Location of the input network") + parser.add_argument("input_demand_dir", type=str, help="Location of the input demand") + parser.add_argument("input_weights", type=str, help="Location of the pathweights file") + parser.add_argument("output_dir", type=str, help="Location to write fasttrips output") args = parser.parse_args(sys.argv[1:]) # don't pass on items that aren't set args_dict = vars(args) for key in list(args_dict.keys()): - if args_dict[key]==None: del args_dict[key] + if args_dict[key] == None: del args_dict[key] # if config_ft.py exists in demand dir, specify it for input_functions func_file = os.path.join(args.input_demand_dir, "config_ft.py") @@ -273,5 +284,6 @@ def str2bool(v): r = fasttrips.Run.run_fasttrips(**args_dict) + if __name__ == "__main__": main() diff --git a/fasttrips/Stop.py b/fasttrips/Stop.py index 63a2e9bb..5b760790 100644 --- a/fasttrips/Stop.py +++ b/fasttrips/Stop.py @@ -3,7 +3,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,6 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import os import pandas as pd @@ -37,44 +38,43 @@ class Stop(object): #: File with fasttrips stop information (this extends the #: `gtfs stops `_ file). #: See `stops_ft specification `_. - INPUT_STOPS_FILE = "stops_ft.txt" + INPUT_STOPS_FILE = "stops_ft.txt" #: gtfs Stops column name: Unique identifier (object) - STOPS_COLUMN_STOP_ID = 'stop_id' + STOPS_COLUMN_STOP_ID = 'stop_id' #: gtfs Stops column name: Stop name (string) - STOPS_COLUMN_STOP_NAME = 'stop_name' + STOPS_COLUMN_STOP_NAME = 'stop_name' #: gtfs Stops column name: Latitude - STOPS_COLUMN_STOP_LATITUDE = 'stop_lat' + STOPS_COLUMN_STOP_LATITUDE = 'stop_lat' #: gtfs Stops column name: Longitude - STOPS_COLUMN_STOP_LONGITUDE = 'stop_lon' + STOPS_COLUMN_STOP_LONGITUDE = 'stop_lon' #: gtfs Stops column name: Zone ID - STOPS_COLUMN_ZONE_ID = 'zone_id' + STOPS_COLUMN_ZONE_ID = 'zone_id' #: fasttrips Stops column name: Shelter - STOPS_COLUMN_SHELTER = 'shelter' + STOPS_COLUMN_SHELTER = 'shelter' #: fasttrips Stops column name: Lighting - STOPS_COLUMN_LIGHTING = 'lighting' + STOPS_COLUMN_LIGHTING = 'lighting' #: fasttrips Stops column name: Bike Parking - STOPS_COLUMN_BIKE_PARKING = 'bike_parking' + STOPS_COLUMN_BIKE_PARKING = 'bike_parking' #: fasttrips Stops column name: Bike Share Station - STOPS_COLUMN_BIKE_SHARE_STATION = 'bike_share_station' + STOPS_COLUMN_BIKE_SHARE_STATION = 'bike_share_station' #: fasttrips Stops column name: Seating - STOPS_COLUMN_SEATING = 'seating' + STOPS_COLUMN_SEATING = 'seating' #: fasttrips Stops column name: Platform Height - STOPS_COLUMN_PLATFORM_HEIGHT = 'platform_height' + STOPS_COLUMN_PLATFORM_HEIGHT = 'platform_height' #: fasttrips Stops column name: Level - STOPS_COLUMN_LEVEL = 'level' + STOPS_COLUMN_LEVEL = 'level' #: fasttrips Stops column name: Off-Board Payment - STOPS_COLUMN_OFF_BOARD_PAYMENT = 'off_board_payment' + STOPS_COLUMN_OFF_BOARD_PAYMENT = 'off_board_payment' # ========== Added by fasttrips ======================================================= #: fasttrips Stops column name: Stop Numerical Identifier. Int. - STOPS_COLUMN_STOP_ID_NUM = 'stop_id_num' + STOPS_COLUMN_STOP_ID_NUM = 'stop_id_num' #: fasttrips Stops column name: Zone Numerical Identifier. Int. - STOPS_COLUMN_ZONE_ID_NUM = 'zone_id_num' - + STOPS_COLUMN_ZONE_ID_NUM = 'zone_id_num' #: File with stop ID, stop ID number correspondence - OUTPUT_STOP_ID_NUM_FILE = 'ft_intermediate_stop_id.txt' + OUTPUT_STOP_ID_NUM_FILE = 'ft_intermediate_stop_id.txt' def __init__(self, input_archive, output_dir, gtfs, today): """ @@ -82,7 +82,7 @@ def __init__(self, input_archive, output_dir, gtfs, today): fast-trips stops data from the input files in *input_archive*. """ # keep this for later - self.output_dir = output_dir + self.output_dir = output_dir self.stops_df = gtfs.stops @@ -91,16 +91,16 @@ def __init__(self, input_archive, output_dir, gtfs, today): # Read the fast-trips supplemental stops data file. Make sure stop ID is read as a string. stops_ft_df = gtfs.get(Stop.INPUT_STOPS_FILE) - assert(len(stops_ft_df) > 0) + assert (len(stops_ft_df) > 0) # verify required columns are present stops_ft_cols = list(stops_ft_df.columns.values) - assert(Stop.STOPS_COLUMN_STOP_ID in stops_ft_cols) + assert (Stop.STOPS_COLUMN_STOP_ID in stops_ft_cols) # if more than one column, join to the stops dataframe if len(stops_ft_cols) > 1: self.stops_df = pd.merge(left=self.stops_df, right=stops_ft_df, - how='left', on=Stop.STOPS_COLUMN_STOP_ID) + how='left', on=Stop.STOPS_COLUMN_STOP_ID) # Stop IDs are strings. Create a unique numeric stop ID. self.stop_id_df = Util.add_numeric_column(self.stops_df[[Stop.STOPS_COLUMN_STOP_ID]], @@ -121,31 +121,32 @@ def __init__(self, input_archive, output_dir, gtfs, today): self.zone_id_df = pd.DataFrame() if Stop.STOPS_COLUMN_ZONE_ID in list(self.stops_df.columns.values): # Blank zone IDs should be null - self.stops_df.loc[ self.stops_df[Stop.STOPS_COLUMN_ZONE_ID]=="", Stop.STOPS_COLUMN_ZONE_ID] = None - zones_df = self.stops_df.loc[ pd.notnull(self.stops_df[Stop.STOPS_COLUMN_ZONE_ID]) ] + self.stops_df.loc[self.stops_df[Stop.STOPS_COLUMN_ZONE_ID] == "", Stop.STOPS_COLUMN_ZONE_ID] = None + zones_df = self.stops_df.loc[pd.notnull(self.stops_df[Stop.STOPS_COLUMN_ZONE_ID])] if len(zones_df) > 0: self.zone_id_df = Util.add_numeric_column(zones_df[[Stop.STOPS_COLUMN_ZONE_ID]], id_colname=Stop.STOPS_COLUMN_ZONE_ID, numeric_newcolname=Stop.STOPS_COLUMN_ZONE_ID_NUM) # add it to the stops - self.stops_df = pd.merge(left=self.stops_df, right=self.zone_id_df, how="left", on=Stop.STOPS_COLUMN_ZONE_ID) + self.stops_df = pd.merge(left=self.stops_df, right=self.zone_id_df, how="left", + on=Stop.STOPS_COLUMN_ZONE_ID) # and the stop_id_df - self.stop_id_df = pd.merge(left =self.stops_df, - right=self.stops_df[[Stop.STOPS_COLUMN_STOP_ID_NUM, - Stop.STOPS_COLUMN_ZONE_ID, - Stop.STOPS_COLUMN_ZONE_ID_NUM]].drop_duplicates(), - how="left") + self.stop_id_df = pd.merge(left=self.stops_df, + right=self.stops_df[[Stop.STOPS_COLUMN_STOP_ID_NUM, + Stop.STOPS_COLUMN_ZONE_ID, + Stop.STOPS_COLUMN_ZONE_ID_NUM]].drop_duplicates(), + how="left") FastTripsLogger.debug("Zone ID to number correspondence\n" + str(self.zone_id_df.head())) FastTripsLogger.debug("=========== STOPS ===========\n" + str(self.stops_df.head())) - FastTripsLogger.debug("\n"+str(self.stops_df.index.dtype)+"\n"+str(self.stops_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.stops_df.index.dtype) + "\n" + str(self.stops_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.stops_df), "stops", "stops.txt", Stop.INPUT_STOPS_FILE)) #: Trips table. - self.trip_times_df = None + self.trip_times_df = None def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname): """ @@ -158,13 +159,13 @@ def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname) This method will also update the :py:attr:`Stop.transfers_df` with Stop IDs since this is now possible since DAPs needed to be numbered for this to work. """ - assert(len(dap_df.columns) == 1) + assert (len(dap_df.columns) == 1) # make sure the DAP IDs are unique from Stop IDs - daps_unique_df = dap_df.drop_duplicates().reset_index(drop=True) + daps_unique_df = dap_df.drop_duplicates().reset_index(drop=True) join_daps_stops = pd.merge(left=daps_unique_df, right=self.stop_id_df, - how="left", - left_on=dap_id_colname, right_on=Stop.STOPS_COLUMN_STOP_ID) + how="left", + left_on=dap_id_colname, right_on=Stop.STOPS_COLUMN_STOP_ID) # there should be only NaNs since DAP lot IDs need to be unique from Stop IDs # non_unique_lots = join_daps_stops.loc[ pd.notnull(join_daps_stops[Stop.STOPS_COLUMN_STOP_ID]) ] # if len(non_unique_lots) > 0: @@ -177,7 +178,7 @@ def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname) daps_unique_df[Stop.STOPS_COLUMN_STOP_ID_NUM] = daps_unique_df.index + self.max_stop_id_num + 1 # rename DAP lot id to stop id - daps_unique_df.rename(columns={dap_id_colname:Stop.STOPS_COLUMN_STOP_ID}, inplace=True) + daps_unique_df.rename(columns={dap_id_colname: Stop.STOPS_COLUMN_STOP_ID}, inplace=True) # append daps to stop ids self.stop_id_df = pd.concat([self.stop_id_df, daps_unique_df], axis=0) @@ -185,21 +186,21 @@ def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname) self.max_dap_id_num = self.stop_id_df[Stop.STOPS_COLUMN_STOP_ID_NUM].max() ############################################################################################## - assert(len(taz_df.columns) == 1) + assert (len(taz_df.columns) == 1) # make sure the TAZ IDs are unique from Stop IDs - tazs_unique_df = taz_df.drop_duplicates().reset_index(drop=True) - join_tazs_stops = pd.merge(left=tazs_unique_df, right=self.stop_id_df, - how="left", - left_on=taz_id_colname, right_on=Stop.STOPS_COLUMN_STOP_ID) + tazs_unique_df = taz_df.drop_duplicates().reset_index(drop=True) + join_tazs_stops = pd.merge(left=tazs_unique_df, right=self.stop_id_df, + how="left", + left_on=taz_id_colname, right_on=Stop.STOPS_COLUMN_STOP_ID) # there should be only NaNs since TAZ IDs need to be unique from Stop IDs - assert(pd.isnull(join_tazs_stops[Stop.STOPS_COLUMN_STOP_ID]).sum() == len(join_tazs_stops)) + assert (pd.isnull(join_tazs_stops[Stop.STOPS_COLUMN_STOP_ID]).sum() == len(join_tazs_stops)) # number them starting at self.max_stop_id_num tazs_unique_df[Stop.STOPS_COLUMN_STOP_ID_NUM] = tazs_unique_df.index + self.max_dap_id_num + 1 # rename TAZ id to stop id - tazs_unique_df.rename(columns={taz_id_colname:Stop.STOPS_COLUMN_STOP_ID}, inplace=True) + tazs_unique_df.rename(columns={taz_id_colname: Stop.STOPS_COLUMN_STOP_ID}, inplace=True) # append daps to stop ids self.stop_id_df = pd.concat([self.stop_id_df, tazs_unique_df], axis=0) @@ -209,8 +210,8 @@ def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname) stop_id_df = self.stop_id_df # local copy with filled NA if Stop.STOPS_COLUMN_ZONE_ID_NUM not in stop_id_df.columns.values: stop_id_df[Stop.STOPS_COLUMN_ZONE_ID_NUM] = -1 - stop_id_df.fillna(value={Stop.STOPS_COLUMN_ZONE_ID_NUM:-1, - Stop.STOPS_COLUMN_ZONE_ID:"None"}, inplace=True) + stop_id_df.fillna(value={Stop.STOPS_COLUMN_ZONE_ID_NUM: -1, + Stop.STOPS_COLUMN_ZONE_ID: "None"}, inplace=True) stop_id_df[Stop.STOPS_COLUMN_ZONE_ID_NUM] = stop_id_df[Stop.STOPS_COLUMN_ZONE_ID_NUM].astype(int) stop_id_df.to_csv(os.path.join(self.output_dir, Stop.OUTPUT_STOP_ID_NUM_FILE), columns=[Stop.STOPS_COLUMN_STOP_ID_NUM, Stop.STOPS_COLUMN_STOP_ID, @@ -218,17 +219,17 @@ def add_daps_tazs_to_stops(self, dap_df, dap_id_colname, taz_df, taz_id_colname) sep=" ", index=False) FastTripsLogger.debug("Wrote %s" % os.path.join(self.output_dir, Stop.OUTPUT_STOP_ID_NUM_FILE)) - - def add_numeric_stop_id(self, input_df, id_colname, numeric_newcolname, warn=False, warn_msg=None, drop_failures=True): + def add_numeric_stop_id(self, input_df, id_colname, numeric_newcolname, warn=False, warn_msg=None, + drop_failures=True): """ Passing a :py:class:`pandas.DataFrame` with a stop ID column called *id_colname*, adds the numeric stop id as a column named *numeric_newcolname* and returns it. """ return Util.add_new_id(input_df, id_colname, numeric_newcolname, - mapping_df=self.stop_id_df[[Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_STOP_ID_NUM]], - mapping_id_colname=Stop.STOPS_COLUMN_STOP_ID, - mapping_newid_colname=Stop.STOPS_COLUMN_STOP_ID_NUM, - warn=warn, warn_msg=warn_msg, drop_failures=drop_failures) + mapping_df=self.stop_id_df[[Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_STOP_ID_NUM]], + mapping_id_colname=Stop.STOPS_COLUMN_STOP_ID, + mapping_newid_colname=Stop.STOPS_COLUMN_STOP_ID_NUM, + warn=warn, warn_msg=warn_msg, drop_failures=drop_failures) def add_stop_id_for_numeric_id(self, input_df, numeric_id, id_colname): """ @@ -263,17 +264,17 @@ def add_stop_lat_lon(self, input_df, id_colname, new_lat_colname, new_lon_colnam stop_cols = [Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_STOP_LATITUDE, Stop.STOPS_COLUMN_STOP_LONGITUDE] if new_stop_name_colname: stop_cols.append(Stop.STOPS_COLUMN_STOP_NAME) - input_df = pd.merge(left =input_df, - right =self.stops_df[stop_cols], - how ="left", - left_on =id_colname, - right_on=Stop.STOPS_COLUMN_STOP_ID) + input_df = pd.merge(left=input_df, + right=self.stops_df[stop_cols], + how="left", + left_on=id_colname, + right_on=Stop.STOPS_COLUMN_STOP_ID) # don't want to add this column if Stop.STOPS_COLUMN_STOP_ID != id_colname: input_df.drop(Stop.STOPS_COLUMN_STOP_ID, axis=1, inplace=True) - rename_dict = {Stop.STOPS_COLUMN_STOP_LATITUDE :new_lat_colname, - Stop.STOPS_COLUMN_STOP_LONGITUDE:new_lon_colname} + rename_dict = {Stop.STOPS_COLUMN_STOP_LATITUDE: new_lat_colname, + Stop.STOPS_COLUMN_STOP_LONGITUDE: new_lon_colname} if new_stop_name_colname: rename_dict[Stop.STOPS_COLUMN_STOP_NAME] = new_stop_name_colname input_df.rename(columns=rename_dict, inplace=True) return input_df @@ -288,16 +289,16 @@ def add_stop_zone_id(self, input_df, id_colname, zone_colname): if Stop.STOPS_COLUMN_ZONE_ID not in self.stops_df.columns.values: return input_df - input_df = pd.merge(left =input_df, - right = self.stops_df[[Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_ZONE_ID]], - how ="left", - left_on =id_colname, - right_on=Stop.STOPS_COLUMN_STOP_ID) + input_df = pd.merge(left=input_df, + right=self.stops_df[[Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_ZONE_ID]], + how="left", + left_on=id_colname, + right_on=Stop.STOPS_COLUMN_STOP_ID) # don't want to add this column if Stop.STOPS_COLUMN_STOP_ID != id_colname: input_df.drop(Stop.STOPS_COLUMN_STOP_ID, axis=1, inplace=True) - input_df.rename(columns={Stop.STOPS_COLUMN_ZONE_ID:zone_colname}, inplace=True) + input_df.rename(columns={Stop.STOPS_COLUMN_ZONE_ID: zone_colname}, inplace=True) return input_df def add_trips(self, stop_times_df): @@ -310,15 +311,16 @@ def add_trips(self, stop_times_df): """ self.trip_times_df = stop_times_df.copy() self.trip_times_df.reset_index(inplace=True) - self.trip_times_df.set_index([Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_SEQUENCE], inplace=True, verify_integrity=True) + self.trip_times_df.set_index( + [Trip.STOPTIMES_COLUMN_STOP_ID, Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_SEQUENCE], + inplace=True, verify_integrity=True) FastTripsLogger.debug("Stop trip_times_df\n" + str(self.trip_times_df.head())) def get_transfers(self, stop_id, xfer_from): if xfer_from: - return self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_FROM_STOP]==stop_id] + return self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_FROM_STOP] == stop_id] else: - return self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_TO_STOP]==stop_id] - + return self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_TO_STOP] == stop_id] def get_trips_arriving_within_time(self, stop_id, latest_arrival, time_window): """ @@ -330,20 +332,21 @@ def get_trips_arriving_within_time(self, stop_id, latest_arrival, time_window): :type time_window: a :py:class:`datetime.timedelta` instance """ - latest_arrival_min = 60.0*latest_arrival.hour + latest_arrival.minute + (latest_arrival.second/60.0) + latest_arrival_min = 60.0 * latest_arrival.hour + latest_arrival.minute + (latest_arrival.second / 60.0) # filter to stop df = self.trip_times_df.loc[stop_id] # arrive before latest arrival and arrive within time window - df = df.loc[(df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN] < latest_arrival_min)& - (df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN] > (latest_arrival_min - (time_window.total_seconds()/60.0)))] + df = df.loc[(df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN] < latest_arrival_min) & + (df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN] > ( + latest_arrival_min - (time_window.total_seconds() / 60.0)))] to_return = [] df = df[[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME]] for index, row in df.iterrows(): - to_return.append( (index[0], # trip id - index[1], # sequence, - row[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] # arrival time - ) ) + to_return.append((index[0], # trip id + index[1], # sequence, + row[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] # arrival time + )) return to_return def get_trips_departing_within_time(self, stop_id, earliest_departure, time_window): @@ -356,30 +359,32 @@ def get_trips_departing_within_time(self, stop_id, earliest_departure, time_wind :type time_window: a :py:class:`datetime.timedelta` instance """ - earliest_departure_min = 60.0*earliest_departure.hour + earliest_departure.minute + (earliest_departure.second/60.0) + earliest_departure_min = 60.0 * earliest_departure.hour + earliest_departure.minute + ( + earliest_departure.second / 60.0) # filter to stop df = self.trip_times_df.loc[stop_id] # depart after the earliest departure - df = df.loc[(df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN] > earliest_departure_min)& - (df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN] < (earliest_departure_min + (time_window.total_seconds()/60.0)))] + df = df.loc[(df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN] > earliest_departure_min) & + (df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN] < ( + earliest_departure_min + (time_window.total_seconds() / 60.0)))] to_return = [] df = df[[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME]] for index, row in df.iterrows(): - to_return.append( (index[0], # trip id - index[1], # sequence, - row[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] # arrival time - ) ) + to_return.append((index[0], # trip id + index[1], # sequence, + row[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] # arrival time + )) return to_return - def is_transfer(self, stop_id, xfer_from): """ Returns true iff this is a transfer stop; e.g. if it's served by multiple routes or has a transfer link. """ - if xfer_from and len(self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_FROM_STOP]==stop_id]) > 0: + if xfer_from and len(self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_FROM_STOP] == stop_id]) > 0: return True - if not xfer_from and len(self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_TO_STOP]==stop_id]) > 0: + if not xfer_from and len( + self.transfers_df.loc[self.transfers_df[Stop.TRANSFERS_COLUMN_TO_STOP] == stop_id]) > 0: return True return False diff --git a/fasttrips/TAZ.py b/fasttrips/TAZ.py index ba16ed4b..582ee3ae 100644 --- a/fasttrips/TAZ.py +++ b/fasttrips/TAZ.py @@ -4,7 +4,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,17 +17,18 @@ See the License for the specific language governing permissions and limitations under the License. """ + import datetime import os import numpy as np import pandas as pd -from .Error import NetworkInputError -from .Logger import FastTripsLogger -from .Route import Route -from .Stop import Stop -from .Transfer import Transfer +from .Error import NetworkInputError +from .Logger import FastTripsLogger +from .Route import Route +from .Stop import Stop +from .Transfer import Transfer class TAZ(object): @@ -45,175 +46,175 @@ class TAZ(object): #: File with fasttrips walk access information. #: See `walk_access specification `_. - INPUT_WALK_ACCESS_FILE = "walk_access_ft.txt" + INPUT_WALK_ACCESS_FILE = "walk_access_ft.txt" #: Walk access links column name: TAZ Identifier. String. - WALK_ACCESS_COLUMN_TAZ = 'taz' + WALK_ACCESS_COLUMN_TAZ = 'taz' #: Walk access links column name: Stop Identifier. String. - WALK_ACCESS_COLUMN_STOP = 'stop_id' + WALK_ACCESS_COLUMN_STOP = 'stop_id' #: Walk access links column name: Direction (access or egress) - WALK_ACCESS_COLUMN_DIRECTION = "direction" + WALK_ACCESS_COLUMN_DIRECTION = "direction" #: Walk access links column name: Walk Distance - WALK_ACCESS_COLUMN_DIST = 'dist' + WALK_ACCESS_COLUMN_DIST = 'dist' #: fasttrips Walk access links column name: Elevation Gain, feet gained along link. - WALK_ACCESS_COLUMN_ELEVATION_GAIN = 'elevation_gain' + WALK_ACCESS_COLUMN_ELEVATION_GAIN = 'elevation_gain' #: fasttrips Walk access links column name: Population Density, people per square mile. Float. - WALK_ACCESS_COLUMN_POPULATION_DENSITY = 'population_density' + WALK_ACCESS_COLUMN_POPULATION_DENSITY = 'population_density' #: fasttrips Walk access links column name: Employment Density, employees per square mile. Float. - WALK_ACCESS_COLUMN_EMPLOYMENT_DENSITY = 'employment_density' + WALK_ACCESS_COLUMN_EMPLOYMENT_DENSITY = 'employment_density' #: fasttrips Walk access links column name: Retail Density, employees per square mile. Float. - #WALK_ACCESS_COLUMN_RETAIL_DENSITY = 'retail_density' + # WALK_ACCESS_COLUMN_RETAIL_DENSITY = 'retail_density' #: fasttrips Walk access links column name: Employment Density, employees per square mile. Float. - WALK_ACCESS_COLUMN_EMPLOYMENT_DENSITY = 'employment_density' + WALK_ACCESS_COLUMN_EMPLOYMENT_DENSITY = 'employment_density' #: fasttrips Walk access links column name: Auto Capacity, vehicles per hour per mile. Float. - WALK_ACCESS_COLUMN_AUTO_CAPACITY = 'auto_capacity' + WALK_ACCESS_COLUMN_AUTO_CAPACITY = 'auto_capacity' #: fasttrips Walk access links column name: Indirectness, ratio of Manhattan distance to crow-fly distance. Float. - WALK_ACCESS_COLUMN_INDIRECTNESS = 'indirectness' + WALK_ACCESS_COLUMN_INDIRECTNESS = 'indirectness' # ========== Added by fasttrips ======================================================= #: Walk access links column name: TAZ Numerical Identifier. Int. - WALK_ACCESS_COLUMN_TAZ_NUM = 'taz_num' + WALK_ACCESS_COLUMN_TAZ_NUM = 'taz_num' #: Walk access links column name: Stop Numerical Identifier. Int. - WALK_ACCESS_COLUMN_STOP_NUM = 'stop_id_num' + WALK_ACCESS_COLUMN_STOP_NUM = 'stop_id_num' #: Walk access links column name: Link walk time. This is a TimeDelta - WALK_ACCESS_COLUMN_TIME = 'time' + WALK_ACCESS_COLUMN_TIME = 'time' #: Walk access links column name: Link walk time in minutes. This is float. - WALK_ACCESS_COLUMN_TIME_MIN = 'time_min' + WALK_ACCESS_COLUMN_TIME_MIN = 'time_min' #: Walk acess cost column name: Link generic cost for accessing stop from TAZ. Float. - WALK_ACCESS_COLUMN_ACC_COST = 'access_cost' + WALK_ACCESS_COLUMN_ACC_COST = 'access_cost' #: Walk acess cost column name: Link generic cost for egressing to TAZ from stop. Float. - WALK_ACCESS_COLUMN_EGR_COST = 'egress_cost' + WALK_ACCESS_COLUMN_EGR_COST = 'egress_cost' #: Walk access links column name: Supply mode. String. - WALK_ACCESS_COLUMN_SUPPLY_MODE = 'supply_mode' + WALK_ACCESS_COLUMN_SUPPLY_MODE = 'supply_mode' #: Walk access links column name: Supply mode number. Int. - WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM = 'supply_mode_num' + WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM = 'supply_mode_num' #: File with fasttrips drive access information. #: See `drive_access specification `_. - INPUT_DRIVE_ACCESS_FILE = "drive_access_ft.txt" + INPUT_DRIVE_ACCESS_FILE = "drive_access_ft.txt" #: Drive access links column name: TAZ Identifier. String. - DRIVE_ACCESS_COLUMN_TAZ = WALK_ACCESS_COLUMN_TAZ + DRIVE_ACCESS_COLUMN_TAZ = WALK_ACCESS_COLUMN_TAZ #: Drive access links column name: Stop Identifier. String. - DRIVE_ACCESS_COLUMN_LOT_ID = 'lot_id' + DRIVE_ACCESS_COLUMN_LOT_ID = 'lot_id' #: Drive access links column name: Direction ('access' or 'egress') - DRIVE_ACCESS_COLUMN_DIRECTION = 'direction' + DRIVE_ACCESS_COLUMN_DIRECTION = 'direction' #: Drive access links column name: Drive distance - DRIVE_ACCESS_COLUMN_DISTANCE = 'dist' + DRIVE_ACCESS_COLUMN_DISTANCE = 'dist' #: Drive access links column name: Drive cost in cents (integer) - DRIVE_ACCESS_COLUMN_COST = 'cost' + DRIVE_ACCESS_COLUMN_COST = 'cost' #: Drive access links column name: Driving time in minutes between TAZ and lot (TimeDelta) - DRIVE_ACCESS_COLUMN_TRAVEL_TIME = 'travel_time' + DRIVE_ACCESS_COLUMN_TRAVEL_TIME = 'travel_time' #: Drive access links column name: Start time (e.g. time period these attributes apply), minutes after midnight - DRIVE_ACCESS_COLUMN_START_TIME_MIN = 'start_time_min' + DRIVE_ACCESS_COLUMN_START_TIME_MIN = 'start_time_min' #: Drive access links column name: Start time (e.g. time period these attributes apply). A DateTime instance - DRIVE_ACCESS_COLUMN_START_TIME = 'start_time' + DRIVE_ACCESS_COLUMN_START_TIME = 'start_time' #: Drive access links column name: End time (e.g. time period these attributes apply), minutes after midnight - DRIVE_ACCESS_COLUMN_END_TIME_MIN = 'end_time_min' + DRIVE_ACCESS_COLUMN_END_TIME_MIN = 'end_time_min' #: Drive access links column name: End time (e.g. time period these attributes apply). A DateTime instance - DRIVE_ACCESS_COLUMN_END_TIME = 'end_time' + DRIVE_ACCESS_COLUMN_END_TIME = 'end_time' #: fasttrips Drive access links column name: Elevation Gain, feet gained along link. - DRIVE_ACCESS_COLUMN_ELEVATION_GAIN = 'elevation_gain' + DRIVE_ACCESS_COLUMN_ELEVATION_GAIN = 'elevation_gain' #: fasttrips Drive access links column name: Population Density, people per square mile. Float. - DRIVE_ACCESS_COLUMN_POPULATION_DENSITY = 'population_density' + DRIVE_ACCESS_COLUMN_POPULATION_DENSITY = 'population_density' #: fasttrips Drive access links column name: Retail Density, employees per square mile. Float. - DRIVE_ACCESS_COLUMN_RETAIL_DENSITY = 'retail_density' + DRIVE_ACCESS_COLUMN_RETAIL_DENSITY = 'retail_density' #: fasttrips Drive access links column name: Auto Capacity, vehicles per hour per mile. Float. - DRIVE_ACCESS_COLUMN_AUTO_CAPACITY = 'auto_capacity' + DRIVE_ACCESS_COLUMN_AUTO_CAPACITY = 'auto_capacity' #: fasttrips Drive access links column name: Indirectness, ratio of Manhattan distance to crow-fly distance. Float. - DRIVE_ACCESS_COLUMN_INDIRECTNESS = 'indirectness' + DRIVE_ACCESS_COLUMN_INDIRECTNESS = 'indirectness' # ========== Added by fasttrips ======================================================= #: fasttrips These are the original attributes but renamed to be clear they are the drive component (as opposed to the walk) - DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE = 'drive_dist' - DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME = 'drive_travel_time' + DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE = 'drive_dist' + DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME = 'drive_travel_time' #: Drive access links column name: Driving time in minutes between TAZ and lot (float) - DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN= 'drive_time_min' + DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN = 'drive_time_min' #: fasttrips Drive access links column name: TAZ Numerical Identifier. Int. - DRIVE_ACCESS_COLUMN_TAZ_NUM = WALK_ACCESS_COLUMN_TAZ_NUM + DRIVE_ACCESS_COLUMN_TAZ_NUM = WALK_ACCESS_COLUMN_TAZ_NUM #: fasttrips Drive access links column name: Stop Numerical Identifier. Int. - DRIVE_ACCESS_COLUMN_STOP = WALK_ACCESS_COLUMN_STOP + DRIVE_ACCESS_COLUMN_STOP = WALK_ACCESS_COLUMN_STOP #: fasttrips Drive access links column name: Stop Numerical Identifier. Int. - DRIVE_ACCESS_COLUMN_STOP_NUM = WALK_ACCESS_COLUMN_STOP_NUM + DRIVE_ACCESS_COLUMN_STOP_NUM = WALK_ACCESS_COLUMN_STOP_NUM #: fasttrips Drive access links column name: Walk distance from lot to transit. Miles. Float. - DRIVE_ACCESS_COLUMN_WALK_DISTANCE = 'walk_dist' + DRIVE_ACCESS_COLUMN_WALK_DISTANCE = 'walk_dist' #: fasttrips Drive access links column name: Walk time from lot to transit. TimeDelta. - DRIVE_ACCESS_COLUMN_WALK_TIME = 'walk_time' + DRIVE_ACCESS_COLUMN_WALK_TIME = 'walk_time' #: fasttrips Drive access links column name: Walk time from lot to transit. Int. - DRIVE_ACCESS_COLUMN_WALK_TIME_MIN = 'walk_time_min' + DRIVE_ACCESS_COLUMN_WALK_TIME_MIN = 'walk_time_min' #: fasttrips Drive access links column name: Supply mode. String. - DRIVE_ACCESS_COLUMN_SUPPLY_MODE = WALK_ACCESS_COLUMN_SUPPLY_MODE + DRIVE_ACCESS_COLUMN_SUPPLY_MODE = WALK_ACCESS_COLUMN_SUPPLY_MODE #: Drive access links column name: Supply mode number. Int. - DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM = WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM + DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM = WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM #: File with fasttrips drive access points information. #: See `Drive access points specification `_. - INPUT_DAP_FILE = 'drive_access_points_ft.txt' + INPUT_DAP_FILE = 'drive_access_points_ft.txt' #: fasttrips DAP column name: Lot ID. String. - DAP_COLUMN_LOT_ID = DRIVE_ACCESS_COLUMN_LOT_ID + DAP_COLUMN_LOT_ID = DRIVE_ACCESS_COLUMN_LOT_ID #: fasttrips DAP column name: Lot Latitude (WGS 84) - DAP_COLUMN_LOT_LATITUDE = 'lot_lat' + DAP_COLUMN_LOT_LATITUDE = 'lot_lat' #: fasttrips DAP column name: Lot Longitude (WGS 84) - DAP_COLUMN_LOT_LONGITUDE = 'lot_lon' + DAP_COLUMN_LOT_LONGITUDE = 'lot_lon' #: fasttrips DAP column name: Name of the Lot. String. - DAP_COLUMN_NAME = 'name' + DAP_COLUMN_NAME = 'name' #: fasttrips DAP column name: Drop-Off. Boolean. - DAP_COLUMN_DROP_OFF = 'drop_off' + DAP_COLUMN_DROP_OFF = 'drop_off' #: fasttrips DAP column name: Capacity (number of parking spaces) - DAP_COLUMN_CAPACITY = 'capacity' + DAP_COLUMN_CAPACITY = 'capacity' #: fasttrips DAP column name: Hourly Cost in cents. Integer. - DAP_COLUMN_HOURLY_COST = 'hourly_cost' + DAP_COLUMN_HOURLY_COST = 'hourly_cost' #: fasttrips DAP column name: Maximum Daily Cost in cents. Integer. - DAP_COLUMN_MAXIMUM_COST = 'max_cost' + DAP_COLUMN_MAXIMUM_COST = 'max_cost' #: fasttrips DAP column name: Type - DAP_COLUMN_TYPE = 'type' + DAP_COLUMN_TYPE = 'type' #: mode column - MODE_COLUMN_MODE = 'mode' + MODE_COLUMN_MODE = 'mode' #: mode number - MODE_COLUMN_MODE_NUM = 'mode_num' + MODE_COLUMN_MODE_NUM = 'mode_num' #: access and egress modes. First is default. - ACCESS_EGRESS_MODES = ["walk","bike_own","bike_share","PNR","KNR"] + ACCESS_EGRESS_MODES = ["walk", "bike_own", "bike_share", "PNR", "KNR"] #: Access mode: Walk - MODE_ACCESS_WALK = 101 + MODE_ACCESS_WALK = 101 #: Access mode: Bike (own) - MODE_ACCESS_BIKE_OWN = 102 + MODE_ACCESS_BIKE_OWN = 102 #: Access mode: Bike (share) - MODE_ACCESS_BIKE_SHARE = 103 + MODE_ACCESS_BIKE_SHARE = 103 #: Access mode: Drive to PNR - MODE_ACCESS_PNR = 104 + MODE_ACCESS_PNR = 104 #: Access mode: Drive to KNR - MODE_ACCESS_KNR = 105 + MODE_ACCESS_KNR = 105 #: Egress mode: Walk - MODE_EGRESS_WALK = 201 + MODE_EGRESS_WALK = 201 #: Egress mode: Bike (own) - MODE_EGRESS_BIKE_OWN = 202 + MODE_EGRESS_BIKE_OWN = 202 #: Egress mode: Bike (share) - MODE_EGRESS_BIKE_SHARE = 203 + MODE_EGRESS_BIKE_SHARE = 203 #: Egress mode: Drive to PNR - MODE_EGRESS_PNR = 204 + MODE_EGRESS_PNR = 204 #: Egress mode: Drive to KNR - MODE_EGRESS_KNR = 205 + MODE_EGRESS_KNR = 205 #: Access mode number list, in order of ACCESS_EGRESS_MODES ACCESS_MODE_NUMS = [MODE_ACCESS_WALK, MODE_ACCESS_BIKE_OWN, MODE_ACCESS_BIKE_SHARE, - MODE_ACCESS_PNR, MODE_ACCESS_KNR] + MODE_ACCESS_PNR, MODE_ACCESS_KNR] #: Egress mode number list, in order of ACCESS_EGRESS_MODES EGRESS_MODE_NUMS = [MODE_EGRESS_WALK, MODE_EGRESS_BIKE_OWN, MODE_EGRESS_BIKE_SHARE, - MODE_EGRESS_PNR, MODE_EGRESS_KNR] + MODE_EGRESS_PNR, MODE_EGRESS_KNR] #: Walk mode number list - WALK_MODE_NUMS = [MODE_ACCESS_WALK, - MODE_EGRESS_WALK] + WALK_MODE_NUMS = [MODE_ACCESS_WALK, + MODE_EGRESS_WALK] #: Bike mode number list - BIKE_MODE_NUMS = [MODE_ACCESS_BIKE_OWN, MODE_ACCESS_BIKE_SHARE, - MODE_EGRESS_BIKE_OWN, MODE_EGRESS_BIKE_SHARE] + BIKE_MODE_NUMS = [MODE_ACCESS_BIKE_OWN, MODE_ACCESS_BIKE_SHARE, + MODE_EGRESS_BIKE_OWN, MODE_EGRESS_BIKE_SHARE] #: Drive mode number list DRIVE_MODE_NUMS = [MODE_ACCESS_PNR, MODE_ACCESS_KNR, MODE_EGRESS_PNR, MODE_EGRESS_KNR] @@ -221,7 +222,7 @@ class TAZ(object): #: File with access/egress links for C++ extension #: It's easier to pass it via a file rather than through the #: initialize_fasttrips_extension() because of the strings involved, I think. - OUTPUT_ACCESS_EGRESS_FILE = "ft_intermediate_access_egress.txt" + OUTPUT_ACCESS_EGRESS_FILE = "ft_intermediate_access_egress.txt" def __init__(self, output_dir, gtfs, today, stops, transfers, routes): """ @@ -229,15 +230,15 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): """ from .Assignment import Assignment - self.access_modes_df = pd.DataFrame(data={TAZ.MODE_COLUMN_MODE :TAZ.ACCESS_EGRESS_MODES, - TAZ.MODE_COLUMN_MODE_NUM:TAZ.ACCESS_MODE_NUMS }) - self.access_modes_df[TAZ.MODE_COLUMN_MODE] = self.access_modes_df[TAZ.MODE_COLUMN_MODE]\ - .apply(lambda x:'%s_%s' % (x, Route.MODE_TYPE_ACCESS)) + self.access_modes_df = pd.DataFrame(data={TAZ.MODE_COLUMN_MODE: TAZ.ACCESS_EGRESS_MODES, + TAZ.MODE_COLUMN_MODE_NUM: TAZ.ACCESS_MODE_NUMS}) + self.access_modes_df[TAZ.MODE_COLUMN_MODE] = self.access_modes_df[TAZ.MODE_COLUMN_MODE] \ + .apply(lambda x: '%s_%s' % (x, Route.MODE_TYPE_ACCESS)) - self.egress_modes_df = pd.DataFrame(data={TAZ.MODE_COLUMN_MODE :TAZ.ACCESS_EGRESS_MODES, - TAZ.MODE_COLUMN_MODE_NUM:TAZ.EGRESS_MODE_NUMS }) - self.egress_modes_df[TAZ.MODE_COLUMN_MODE] = self.egress_modes_df[TAZ.MODE_COLUMN_MODE]\ - .apply(lambda x:'%s_%s' % (x, Route.MODE_TYPE_EGRESS)) + self.egress_modes_df = pd.DataFrame(data={TAZ.MODE_COLUMN_MODE: TAZ.ACCESS_EGRESS_MODES, + TAZ.MODE_COLUMN_MODE_NUM: TAZ.EGRESS_MODE_NUMS}) + self.egress_modes_df[TAZ.MODE_COLUMN_MODE] = self.egress_modes_df[TAZ.MODE_COLUMN_MODE] \ + .apply(lambda x: '%s_%s' % (x, Route.MODE_TYPE_EGRESS)) routes.add_access_egress_modes(self.access_modes_df, self.egress_modes_df) @@ -246,24 +247,26 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): # verify required columns are present walk_access_cols = list(self.walk_access_df.columns.values) - assert(TAZ.WALK_ACCESS_COLUMN_TAZ in walk_access_cols) - assert(TAZ.WALK_ACCESS_COLUMN_STOP in walk_access_cols) - assert(TAZ.WALK_ACCESS_COLUMN_DIRECTION in walk_access_cols) - assert(TAZ.WALK_ACCESS_COLUMN_DIST in walk_access_cols) + assert (TAZ.WALK_ACCESS_COLUMN_TAZ in walk_access_cols) + assert (TAZ.WALK_ACCESS_COLUMN_STOP in walk_access_cols) + assert (TAZ.WALK_ACCESS_COLUMN_DIRECTION in walk_access_cols) + assert (TAZ.WALK_ACCESS_COLUMN_DIST in walk_access_cols) # printing this before setting index FastTripsLogger.debug("=========== WALK ACCESS ===========\n" + str(self.walk_access_df.head())) - FastTripsLogger.debug("As read\n"+str(self.walk_access_df.dtypes)) + FastTripsLogger.debug("As read\n" + str(self.walk_access_df.dtypes)) # Verify direction is valid - invalid_direction = self.walk_access_df.loc[ self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_DIRECTION].isin(["access","egress"])==False ] + invalid_direction = self.walk_access_df.loc[ + self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_DIRECTION].isin(["access", "egress"]) == False] if len(invalid_direction) > 0: error_msg = "Invalid direction in walk access links: \n%s" % str(invalid_direction) FastTripsLogger.fatal(error_msg) raise NetworkInputError(TAZ.INPUT_WALK_ACCESS_FILE, error_msg) # TODO: remove? Or put walk speed some place? - self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_TIME_MIN] = self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_DIST]*60.0/2.7; + self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_TIME_MIN] = self.walk_access_df[ + TAZ.WALK_ACCESS_COLUMN_DIST] * 60.0 / 2.7; # convert time column from float to timedelta self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_TIME] = \ self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_TIME_MIN].map(lambda x: datetime.timedelta(minutes=x)) @@ -274,23 +277,23 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): TAZ.WALK_ACCESS_COLUMN_DIRECTION], keep=False) if walk_access_dupes.sum() > 0: self.walk_access_df["duplicates"] = walk_access_dupes - error_msg = "Duplicate taz/stop pairs in walk access links: \n%s" % str(self.walk_access_df.loc[ self.walk_access_df["duplicates"]]) + error_msg = "Duplicate taz/stop pairs in walk access links: \n%s" % str( + self.walk_access_df.loc[self.walk_access_df["duplicates"]]) FastTripsLogger.fatal(error_msg) raise NetworkInputError(TAZ.INPUT_WALK_ACCESS_FILE, error_msg) - FastTripsLogger.debug("Final\n"+str(self.walk_access_df.dtypes)) + FastTripsLogger.debug("Final\n" + str(self.walk_access_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.walk_access_df), "walk access", TAZ.INPUT_WALK_ACCESS_FILE)) self.dap_df = gtfs.get(TAZ.INPUT_DAP_FILE) if not self.dap_df.empty: - # verify required columns are present dap_cols = list(self.dap_df.columns.values) - assert(TAZ.DAP_COLUMN_LOT_ID in dap_cols) - assert(TAZ.DAP_COLUMN_LOT_LATITUDE in dap_cols) - assert(TAZ.DAP_COLUMN_LOT_LONGITUDE in dap_cols) + assert (TAZ.DAP_COLUMN_LOT_ID in dap_cols) + assert (TAZ.DAP_COLUMN_LOT_LATITUDE in dap_cols) + assert (TAZ.DAP_COLUMN_LOT_LONGITUDE in dap_cols) # default capacity = 0 if TAZ.DAP_COLUMN_CAPACITY not in dap_cols: @@ -304,7 +307,7 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): self.dap_df = pd.DataFrame() FastTripsLogger.debug("=========== DAPS ===========\n" + str(self.dap_df.head())) - FastTripsLogger.debug("\n"+str(self.dap_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.dap_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.dap_df), "DAPs", TAZ.INPUT_DAP_FILE)) @@ -313,23 +316,23 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): if not self.drive_access_df.empty: # verify required columns are present drive_access_cols = list(self.drive_access_df.columns.values) - assert(TAZ.DRIVE_ACCESS_COLUMN_TAZ in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_LOT_ID in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_DIRECTION in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_DISTANCE in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_COST in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_TRAVEL_TIME in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_START_TIME in drive_access_cols) - assert(TAZ.DRIVE_ACCESS_COLUMN_END_TIME in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_TAZ in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_LOT_ID in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_DIRECTION in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_DISTANCE in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_COST in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_TRAVEL_TIME in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_START_TIME in drive_access_cols) + assert (TAZ.DRIVE_ACCESS_COLUMN_END_TIME in drive_access_cols) # printing this before setting index FastTripsLogger.debug("=========== DRIVE ACCESS ===========\n" + str(self.drive_access_df.head())) - FastTripsLogger.debug("As read\n"+str(self.drive_access_df.dtypes)) # Rename dist to drive_dist + FastTripsLogger.debug("As read\n" + str(self.drive_access_df.dtypes)) # Rename dist to drive_dist # the distance and times here are for DRIVING self.drive_access_df.rename( - columns = {TAZ.DRIVE_ACCESS_COLUMN_DISTANCE : TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE, - TAZ.DRIVE_ACCESS_COLUMN_TRAVEL_TIME : TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME}, + columns={TAZ.DRIVE_ACCESS_COLUMN_DISTANCE: TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE, + TAZ.DRIVE_ACCESS_COLUMN_TRAVEL_TIME: TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME}, inplace=True) self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN] = \ @@ -337,44 +340,53 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): # if there are any that go past midnight, duplicate sim_day_end = Assignment.NETWORK_BUILD_DATE_START_TIME + datetime.timedelta(days=1) - dupes = self.drive_access_df.loc[self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, :].copy() + dupes = self.drive_access_df.loc[self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, + :].copy() if len(dupes) > 0: # e.g. 18:00 - 27:00 # dupe: 00:00 - 3:00 - dupes.loc[ dupes[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_START_TIME] = Assignment.NETWORK_BUILD_DATE_START_TIME - dupes.loc[ dupes[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_END_TIME ] = dupes[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] - datetime.timedelta(days=1) + dupes.loc[dupes[ + TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_START_TIME] = Assignment.NETWORK_BUILD_DATE_START_TIME + dupes.loc[dupes[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_END_TIME] = \ + dupes[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] - datetime.timedelta(days=1) # orig: 18:00 - 24:00 - self.drive_access_df.loc[ self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_END_TIME ] = sim_day_end - FastTripsLogger.debug("Added %d morning hour drive access links. Head:\n%s" % (len(dupes), dupes.head().to_string())) + self.drive_access_df.loc[self.drive_access_df[ + TAZ.DRIVE_ACCESS_COLUMN_END_TIME] > sim_day_end, TAZ.DRIVE_ACCESS_COLUMN_END_TIME] = sim_day_end + FastTripsLogger.debug( + "Added %d morning hour drive access links. Head:\n%s" % (len(dupes), dupes.head().to_string())) # combine self.drive_access_df = self.drive_access_df.append(dupes) # drive access period start/end time: float version self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN] = \ - (self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_START_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME)/np.timedelta64(1,'m') + (self.drive_access_df[ + TAZ.DRIVE_ACCESS_COLUMN_START_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME) / np.timedelta64(1, + 'm') self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN] = \ - (self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME)/np.timedelta64(1,'m') - + (self.drive_access_df[ + TAZ.DRIVE_ACCESS_COLUMN_END_TIME] - Assignment.NETWORK_BUILD_DATE_START_TIME) / np.timedelta64(1, + 'm') # convert time column from number to timedelta self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME] = \ - self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN].map(lambda x: datetime.timedelta(minutes=float(x))) + self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN].map( + lambda x: datetime.timedelta(minutes=float(x))) # need PNRs and KNRs - get them from the dap - knr_dap_df = self.dap_df.loc[self.dap_df[TAZ.DAP_COLUMN_DROP_OFF]==True].copy() - pnr_dap_df = self.dap_df.loc[self.dap_df[TAZ.DAP_COLUMN_CAPACITY] > 0 ].copy() + knr_dap_df = self.dap_df.loc[self.dap_df[TAZ.DAP_COLUMN_DROP_OFF] == True].copy() + pnr_dap_df = self.dap_df.loc[self.dap_df[TAZ.DAP_COLUMN_CAPACITY] > 0].copy() knr_dap_df['dap_type'] = 'KNR' pnr_dap_df['dap_type'] = 'PNR' self.drive_access_df = pd.merge(left=self.drive_access_df, - right=pd.concat([knr_dap_df,pnr_dap_df], axis=0), - on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, - how='left') + right=pd.concat([knr_dap_df, pnr_dap_df], axis=0), + on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, + how='left') # look for required column being null lots_not_found = self.drive_access_df.loc[pd.isnull(self.drive_access_df[TAZ.DAP_COLUMN_LOT_LATITUDE])] if len(lots_not_found) > 0: error_msg = "Found %d drive access links in %s with lots not specified in %s" % \ - (len(lots_not_found), TAZ.INPUT_DRIVE_ACCESS_FILE, TAZ.INPUT_DAP_FILE) + (len(lots_not_found), TAZ.INPUT_DRIVE_ACCESS_FILE, TAZ.INPUT_DAP_FILE) FastTripsLogger.fatal(error_msg) FastTripsLogger.fatal("\nFirst five drive access links with lots not found:\n%s" % \ str(lots_not_found.head().to_string())) @@ -392,17 +404,17 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): # join with transfers to go from taz -> lot -> stop drive_access = pd.merge(left=drive_access, - right=transfers.transfers_df, - left_on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, - right_on=Transfer.TRANSFERS_COLUMN_FROM_STOP, - how='left') + right=transfers.transfers_df, + left_on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, + right_on=Transfer.TRANSFERS_COLUMN_FROM_STOP, + how='left') drive_access[TAZ.DRIVE_ACCESS_COLUMN_STOP] = drive_access[Transfer.TRANSFERS_COLUMN_TO_STOP] # join with transfers to go from stop -> lot -> taz drive_egress = pd.merge(left=drive_egress, - right=transfers.transfers_df, - left_on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, - right_on=Transfer.TRANSFERS_COLUMN_TO_STOP, - how='left') + right=transfers.transfers_df, + left_on=TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, + right_on=Transfer.TRANSFERS_COLUMN_TO_STOP, + how='left') drive_egress[TAZ.DRIVE_ACCESS_COLUMN_STOP] = drive_egress[Transfer.TRANSFERS_COLUMN_FROM_STOP] self.drive_access_df = pd.concat([drive_access, drive_egress], axis=0) @@ -425,66 +437,71 @@ def __init__(self, output_dir, gtfs, today, stops, transfers, routes): self.drive_access_df.drop([Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN], axis=1, inplace=True) # some may have no lot to stop connections -- check for null stop ids - null_stop_ids = self.drive_access_df.loc[pd.isnull( self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP])] + null_stop_ids = self.drive_access_df.loc[pd.isnull(self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP])] if len(null_stop_ids) > 0: - FastTripsLogger.warn("Dropping %d drive links that don't connect to stops:\n%s" % (len(null_stop_ids), str(null_stop_ids))) + FastTripsLogger.warn("Dropping %d drive links that don't connect to stops:\n%s" % ( + len(null_stop_ids), str(null_stop_ids))) # drop them - self.drive_access_df = self.drive_access_df.loc[ pd.notnull(self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP])] + self.drive_access_df = self.drive_access_df.loc[ + pd.notnull(self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP])] # rename walk attributes to be clear self.drive_access_df.rename( columns={ - Transfer.TRANSFERS_COLUMN_DISTANCE:TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE, - Transfer.TRANSFERS_COLUMN_TIME :TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME, - Transfer.TRANSFERS_COLUMN_TIME_MIN:TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME_MIN}, + Transfer.TRANSFERS_COLUMN_DISTANCE: TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE, + Transfer.TRANSFERS_COLUMN_TIME: TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME, + Transfer.TRANSFERS_COLUMN_TIME_MIN: TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME_MIN}, inplace=True) # add generic distance and time - self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DISTANCE] = self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE] + \ - self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE] + self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DISTANCE] = self.drive_access_df[ + TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE] + \ + self.drive_access_df[ + TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE] self.drive_access_df["time_min"] = self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME_MIN] + \ self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME_MIN] - FastTripsLogger.debug("Final (%d) types:\n%s\nhead:\n%s" % (len(self.drive_access_df), str(self.drive_access_df.dtypes), str(self.drive_access_df.head()))) + FastTripsLogger.debug("Final (%d) types:\n%s\nhead:\n%s" % ( + len(self.drive_access_df), str(self.drive_access_df.dtypes), str(self.drive_access_df.head()))) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.drive_access_df), "drive access", TAZ.INPUT_DRIVE_ACCESS_FILE)) self.has_drive_access = True else: self.has_drive_access = False - self.drive_access_df = pd.DataFrame(columns=[TAZ.DRIVE_ACCESS_COLUMN_TAZ, TAZ.DRIVE_ACCESS_COLUMN_LOT_ID]) + self.drive_access_df = pd.DataFrame(columns=[TAZ.DRIVE_ACCESS_COLUMN_TAZ, TAZ.DRIVE_ACCESS_COLUMN_LOT_ID]) FastTripsLogger.debug("=========== NO DRIVE ACCESS ===========\n") # add DAPs IDs and TAZ IDs to stop ID list stops.add_daps_tazs_to_stops(self.drive_access_df[[TAZ.DRIVE_ACCESS_COLUMN_LOT_ID]], TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, pd.concat([self.walk_access_df[[TAZ.WALK_ACCESS_COLUMN_TAZ]], - self.drive_access_df[[TAZ.DRIVE_ACCESS_COLUMN_TAZ]]], axis=0), + self.drive_access_df[[TAZ.DRIVE_ACCESS_COLUMN_TAZ]]], axis=0), TAZ.WALK_ACCESS_COLUMN_TAZ) # transfers can add stop numeric IDs now that DAPs are available transfers.add_numeric_stop_id(stops) - # Add numeric stop ID to walk access links - self.walk_access_df = stops.add_numeric_stop_id(self.walk_access_df, - id_colname=TAZ.WALK_ACCESS_COLUMN_STOP, - numeric_newcolname=TAZ.WALK_ACCESS_COLUMN_STOP_NUM, - warn=True, - warn_msg="Numeric stop id not found for walk access links") + self.walk_access_df = stops.add_numeric_stop_id(self.walk_access_df, + id_colname=TAZ.WALK_ACCESS_COLUMN_STOP, + numeric_newcolname=TAZ.WALK_ACCESS_COLUMN_STOP_NUM, + warn=True, + warn_msg="Numeric stop id not found for walk access links") # Add TAZ stop ID to walk and drive access links - self.walk_access_df = stops.add_numeric_stop_id(self.walk_access_df, - id_colname=TAZ.WALK_ACCESS_COLUMN_TAZ, - numeric_newcolname=TAZ.WALK_ACCESS_COLUMN_TAZ_NUM) + self.walk_access_df = stops.add_numeric_stop_id(self.walk_access_df, + id_colname=TAZ.WALK_ACCESS_COLUMN_TAZ, + numeric_newcolname=TAZ.WALK_ACCESS_COLUMN_TAZ_NUM) # These have direction now. Set supply mode string - self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE] = "walk_" + self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_DIRECTION] + self.walk_access_df[TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE] = "walk_" + self.walk_access_df[ + TAZ.WALK_ACCESS_COLUMN_DIRECTION] self.walk_access_df = routes.add_numeric_mode_id(self.walk_access_df, id_colname=TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE, numeric_newcolname=TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM) if self.has_drive_access: - print(self.drive_access_df.loc[ self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP] == "9065"]) + print(self.drive_access_df.loc[self.drive_access_df[TAZ.DRIVE_ACCESS_COLUMN_STOP] == "9065"]) self.drive_access_df = stops.add_numeric_stop_id(self.drive_access_df, id_colname=TAZ.DRIVE_ACCESS_COLUMN_STOP, @@ -516,27 +533,29 @@ def add_distance(self, links_df, dist_col): TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM, TAZ.WALK_ACCESS_COLUMN_DIST]].copy() - walk_dists.rename(columns={TAZ.WALK_ACCESS_COLUMN_DIST:"walk_dist"}, inplace=True) + walk_dists.rename(columns={TAZ.WALK_ACCESS_COLUMN_DIST: "walk_dist"}, inplace=True) # walk access - links_df = pd.merge(left =links_df, - left_on =["A_id_num","B_id_num","mode_num"], - right =walk_dists, - right_on=[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM], - how ="left") - links_df.loc[ pd.notnull(links_df["walk_dist"]), dist_col ] = links_df["walk_dist"] + links_df = pd.merge(left=links_df, + left_on=["A_id_num", "B_id_num", "mode_num"], + right=walk_dists, + right_on=[TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_STOP_NUM, + TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM], + how="left") + links_df.loc[pd.notnull(links_df["walk_dist"]), dist_col] = links_df["walk_dist"] links_df.drop([TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM, "walk_dist"], axis=1, inplace=True) # walk egress - links_df = pd.merge(left =links_df, - left_on =["A_id_num","B_id_num","mode_num"], - right =walk_dists, - right_on=[TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM], - how ="left") - links_df.loc[ pd.notnull(links_df["walk_dist"]), dist_col ] = links_df["walk_dist"] + links_df = pd.merge(left=links_df, + left_on=["A_id_num", "B_id_num", "mode_num"], + right=walk_dists, + right_on=[TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, + TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM], + how="left") + links_df.loc[pd.notnull(links_df["walk_dist"]), dist_col] = links_df["walk_dist"] links_df.drop([TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, TAZ.WALK_ACCESS_COLUMN_STOP_NUM, TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM, @@ -552,29 +571,33 @@ def add_distance(self, links_df, dist_col): TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE, TAZ.DRIVE_ACCESS_COLUMN_START_TIME, TAZ.DRIVE_ACCESS_COLUMN_END_TIME]].copy() - drive_dists["drive_total_dist"] = drive_dists[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE] + drive_dists[TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE] - drive_dists.drop([TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE, TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE], axis=1, inplace=True) + drive_dists["drive_total_dist"] = drive_dists[TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE] + drive_dists[ + TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE] + drive_dists.drop([TAZ.DRIVE_ACCESS_COLUMN_DRIVE_DISTANCE, TAZ.DRIVE_ACCESS_COLUMN_WALK_DISTANCE], axis=1, + inplace=True) # drive access - links_df = pd.merge(left =links_df, - left_on =["A_id_num","B_id_num","mode_num"], - right =drive_dists, - right_on=[TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM], - how ="left") + links_df = pd.merge(left=links_df, + left_on=["A_id_num", "B_id_num", "mode_num"], + right=drive_dists, + right_on=[TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, + TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM], + how="left") # TODO: drop those with drive access links covering different times - links_df.loc[ pd.notnull(links_df["drive_total_dist"]), dist_col ] = links_df["drive_total_dist"] + links_df.loc[pd.notnull(links_df["drive_total_dist"]), dist_col] = links_df["drive_total_dist"] links_df.drop([TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM, "drive_total_dist"], axis=1, inplace=True) # drive egress - links_df = pd.merge(left =links_df, - left_on =["A_id_num","B_id_num","mode_num"], - right =drive_dists, - right_on=[TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM], - how ="left") - links_df.loc[ pd.notnull(links_df["drive_total_dist"]), dist_col ] = links_df["drive_total_dist"] + links_df = pd.merge(left=links_df, + left_on=["A_id_num", "B_id_num", "mode_num"], + right=drive_dists, + right_on=[TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, + TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM], + how="left") + links_df.loc[pd.notnull(links_df["drive_total_dist"]), dist_col] = links_df["drive_total_dist"] links_df.drop([TAZ.DRIVE_ACCESS_COLUMN_TAZ_NUM, TAZ.DRIVE_ACCESS_COLUMN_STOP_NUM, TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE_NUM, @@ -591,12 +614,13 @@ def warn_on_stops_without_walk_access(self, stops): # FastTripsLogger.debug("warn_on_stops_without_walk_access: \n%s", self.walk_access_df.head() ) # join stops to walk access - no_access_stops = pd.merge(left = stops.stops_df[[Stop.STOPS_COLUMN_STOP_ID]], - right = self.walk_access_df[[TAZ.WALK_ACCESS_COLUMN_STOP, TAZ.WALK_ACCESS_COLUMN_TAZ]], - how = "left") + no_access_stops = pd.merge(left=stops.stops_df[[Stop.STOPS_COLUMN_STOP_ID]], + right=self.walk_access_df[[TAZ.WALK_ACCESS_COLUMN_STOP, TAZ.WALK_ACCESS_COLUMN_TAZ]], + how="left") no_access_stops = no_access_stops.loc[pd.isnull(no_access_stops[TAZ.WALK_ACCESS_COLUMN_TAZ])] if len(no_access_stops) > 0: - FastTripsLogger.warn("The following %d stop ids have no walk access: \n%s" % (len(no_access_stops), no_access_stops.to_string())) + FastTripsLogger.warn("The following %d stop ids have no walk access: \n%s" % ( + len(no_access_stops), no_access_stops.to_string())) def write_access_egress_for_extension(self, output_dir): """ @@ -615,12 +639,12 @@ def write_access_egress_for_extension(self, output_dir): # start with all walk columns self.walk_df = self.walk_access_df.copy() # drop the redundant columns - drop_fields = [TAZ.WALK_ACCESS_COLUMN_TAZ, # use numerical version - TAZ.WALK_ACCESS_COLUMN_STOP, # use numerical version - TAZ.WALK_ACCESS_COLUMN_DIRECTION, # it's in the supply mode num - TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE, # use numerical version - TAZ.WALK_ACCESS_COLUMN_TIME, # use numerical version - ] + drop_fields = [TAZ.WALK_ACCESS_COLUMN_TAZ, # use numerical version + TAZ.WALK_ACCESS_COLUMN_STOP, # use numerical version + TAZ.WALK_ACCESS_COLUMN_DIRECTION, # it's in the supply mode num + TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE, # use numerical version + TAZ.WALK_ACCESS_COLUMN_TIME, # use numerical version + ] # we can only drop fields that are in the dataframe walk_fields = list(self.walk_df.columns.values) valid_drop_fields = [] @@ -630,14 +654,14 @@ def write_access_egress_for_extension(self, output_dir): self.walk_df.drop(valid_drop_fields, axis=1, inplace=True) # make walk access valid all times -- need this for consistency self.walk_df[TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN] = 0.0 - self.walk_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN ] = 60.0*24.0 + self.walk_df[TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN] = 60.0 * 24.0 # the index is TAZ num, supply mode num, and stop num self.walk_df.set_index([TAZ.WALK_ACCESS_COLUMN_TAZ_NUM, - TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM, - TAZ.WALK_ACCESS_COLUMN_STOP_NUM, - TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN, - TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN], inplace=True) + TAZ.WALK_ACCESS_COLUMN_SUPPLY_MODE_NUM, + TAZ.WALK_ACCESS_COLUMN_STOP_NUM, + TAZ.DRIVE_ACCESS_COLUMN_START_TIME_MIN, + TAZ.DRIVE_ACCESS_COLUMN_END_TIME_MIN], inplace=True) # ========== Drive access/egres ================================================= self.drive_df = self.drive_access_df.copy() @@ -648,19 +672,19 @@ def write_access_egress_for_extension(self, output_dir): drive_fields = list(self.drive_df.columns.values) # drop some of the attributes - drop_fields = [TAZ.DRIVE_ACCESS_COLUMN_TAZ, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_STOP, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_START_TIME, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_END_TIME, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME, # use numerical version - TAZ.DRIVE_ACCESS_COLUMN_DIRECTION, # redundant with supply mode - TAZ.DAP_COLUMN_DROP_OFF, # redundant with supply mode - TAZ.DAP_COLUMN_LOT_LATITUDE, # probably not useful - TAZ.DAP_COLUMN_LOT_LONGITUDE, # probably not useful - TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, # probably not useful - ] + drop_fields = [TAZ.DRIVE_ACCESS_COLUMN_TAZ, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_STOP, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_SUPPLY_MODE, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_DRIVE_TRAVEL_TIME, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_START_TIME, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_END_TIME, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_WALK_TIME, # use numerical version + TAZ.DRIVE_ACCESS_COLUMN_DIRECTION, # redundant with supply mode + TAZ.DAP_COLUMN_DROP_OFF, # redundant with supply mode + TAZ.DAP_COLUMN_LOT_LATITUDE, # probably not useful + TAZ.DAP_COLUMN_LOT_LONGITUDE, # probably not useful + TAZ.DRIVE_ACCESS_COLUMN_LOT_ID, # probably not useful + ] valid_drop_fields = [] for field in drop_fields: if field in drive_fields: valid_drop_fields.append(field) @@ -684,7 +708,7 @@ def write_access_egress_for_extension(self, output_dir): access_df.reset_index(inplace=True) # rename from these default column names - access_df.rename(columns={"level_3":"attr_name", 0:"attr_value"}, inplace=True) + access_df.rename(columns={"level_3": "attr_name", 0: "attr_value"}, inplace=True) # make attr_value a float instead of an object access_df["attr_value"] = access_df["attr_value"].astype(float) @@ -697,7 +721,7 @@ def write_access_egress_for_extension(self, output_dir): FastTripsLogger.warn("write_access_egress_for_extension null_stop_ids:\n%s" % str(null_stop_ids)) # for now, drop rows with null stop id nums - access_df = access_df.loc[ pd.notnull(access_df["stop_id_num"]) ] + access_df = access_df.loc[pd.notnull(access_df["stop_id_num"])] access_df["stop_id_num"] = access_df["stop_id_num"].astype(int) diff --git a/fasttrips/Transfer.py b/fasttrips/Transfer.py index 89b0c67d..d4bd6a99 100644 --- a/fasttrips/Transfer.py +++ b/fasttrips/Transfer.py @@ -3,7 +3,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,12 +16,13 @@ See the License for the specific language governing permissions and limitations under the License. """ + import datetime import os import pandas as pd -from .Error import NetworkInputError +from .Error import NetworkInputError from .Logger import FastTripsLogger @@ -37,72 +38,72 @@ class Transfer(object): #: File with fasttrips transfer information (this extends the #: `gtfs transfers `_ file). #: See `transfers_ft specification `_. - INPUT_TRANSFERS_FILE = "transfers_ft.txt" + INPUT_TRANSFERS_FILE = "transfers_ft.txt" #: gtfs Transfers column name: Origin stop identifier - TRANSFERS_COLUMN_FROM_STOP = 'from_stop_id' + TRANSFERS_COLUMN_FROM_STOP = 'from_stop_id' #: gtfs Transfers column name: Destination stop identifier - TRANSFERS_COLUMN_TO_STOP = 'to_stop_id' + TRANSFERS_COLUMN_TO_STOP = 'to_stop_id' #: gtfs Transfers column name: Transfer Type - TRANSFERS_COLUMN_TRANSFER_TYPE = 'transfer_type' + TRANSFERS_COLUMN_TRANSFER_TYPE = 'transfer_type' #: gtfs Transfers column name: Minimum transfer time for transfer_type=2. Float, seconds. - TRANSFERS_COLUMN_MIN_TRANSFER_TIME = 'min_transfer_time' + TRANSFERS_COLUMN_MIN_TRANSFER_TIME = 'min_transfer_time' #: fasttrips Transfers column name: Link walk distance, in miles. This is a float. - TRANSFERS_COLUMN_DISTANCE = 'dist' + TRANSFERS_COLUMN_DISTANCE = 'dist' #: fasttrips Transfers column name: Origin route identifier - TRANSFERS_COLUMN_FROM_ROUTE = 'from_route_id' + TRANSFERS_COLUMN_FROM_ROUTE = 'from_route_id' #: fasttrips Transfers column name: Destination route identifier - TRANSFERS_COLUMN_TO_ROUTE = 'to_route_id' + TRANSFERS_COLUMN_TO_ROUTE = 'to_route_id' #: fasttrips Transfers column name: Schedule precedence - TRANSFERS_COLUMN_SCHEDULE_PRECEDENCE = 'schedule_precedence' - - #: fasttrips Transfers column name: Elevation Gain, feet gained along link. Integer. - TRANSFERS_COLUMN_ELEVATION_GAIN = 'elevation_gain' - #: fasttrips Transfers column name: Population Density, people per square mile. Float. - TRANSFERS_COLUMN_POPULATION_DENSITY = 'population_density' - #: fasttrips Transfers column name: Retail Density, employees per square mile. Float. - TRANSFERS_COLUMN_RETAIL_DENSITY = 'retail_density' - #: fasttrips Transfers column name: Auto Capacity, vehicles per hour per mile. Float. - TRANSFERS_COLUMN_AUTO_CAPACITY = 'auto_capacity' - #: fasttrips Transfers column name: Indirectness, ratio of Manhattan distance to crow-fly distance. Float. - TRANSFERS_COLUMN_INDIRECTNESS = 'indirectness' + TRANSFERS_COLUMN_SCHEDULE_PRECEDENCE = 'schedule_precedence' + + #: fasttrips Transfers column name: Elevation Gain, feet gained along link. Integer. + TRANSFERS_COLUMN_ELEVATION_GAIN = 'elevation_gain' + #: fasttrips Transfers column name: Population Density, people per square mile. Float. + TRANSFERS_COLUMN_POPULATION_DENSITY = 'population_density' + #: fasttrips Transfers column name: Retail Density, employees per square mile. Float. + TRANSFERS_COLUMN_RETAIL_DENSITY = 'retail_density' + #: fasttrips Transfers column name: Auto Capacity, vehicles per hour per mile. Float. + TRANSFERS_COLUMN_AUTO_CAPACITY = 'auto_capacity' + #: fasttrips Transfers column name: Indirectness, ratio of Manhattan distance to crow-fly distance. Float. + TRANSFERS_COLUMN_INDIRECTNESS = 'indirectness' # ========== Added by fasttrips ======================================================= #: fasttrips Transfers column name: Is this a stop-to-stop transfer? (e.g. from transfers.txt, and not involving a lot) - TRANSFERS_COLUMN_STOP_TO_STOP = "stop2stop" + TRANSFERS_COLUMN_STOP_TO_STOP = "stop2stop" #: fasttrips Transfers column name: Origin Stop Numerical Identifier. Int. - TRANSFERS_COLUMN_FROM_STOP_NUM = 'from_stop_id_num' + TRANSFERS_COLUMN_FROM_STOP_NUM = 'from_stop_id_num' #: fasttrips Transfers column name: Destination Stop Numerical Identifier. Int. - TRANSFERS_COLUMN_TO_STOP_NUM = 'to_stop_id_num' + TRANSFERS_COLUMN_TO_STOP_NUM = 'to_stop_id_num' #: gtfs Transfers column name: Minimum transfer time for transfer_type=2. Float, min. - TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN = 'min_transfer_time_min' + TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN = 'min_transfer_time_min' #: Transfer walk speed, in miles per hour #: #: .. todo:: Make this configurable? #: - WALK_SPEED_MILES_PER_HOUR = 2.7 + WALK_SPEED_MILES_PER_HOUR = 2.7 #: Transfers column name: Link walk time. This is a TimeDelta. #: #: .. todo:: Remove these? Maybe weights should be distance based? Walk speed is configured how? #: - TRANSFERS_COLUMN_TIME = 'time' + TRANSFERS_COLUMN_TIME = 'time' #: Transfers column name: Link walk time in minutes. This is a float. - TRANSFERS_COLUMN_TIME_MIN = 'time_min' + TRANSFERS_COLUMN_TIME_MIN = 'time_min' #: Transfers column name: Link generic cost. Float. - TRANSFERS_COLUMN_PENALTY = 'transfer_penalty' + TRANSFERS_COLUMN_PENALTY = 'transfer_penalty' #: File with transfer links for C++ extension #: It's easier to pass it via file rather than through the #: initialize_fasttrips_extension() because of the strings involved - OUTPUT_TRANSFERS_FILE = "ft_intermediate_transfers.txt" + OUTPUT_TRANSFERS_FILE = "ft_intermediate_transfers.txt" def __init__(self, input_archive, output_dir, gtfs_feed): """ Constructor. Reads the gtfs data from the transitfeed schedule, and the additional fast-trips transfers data from the input files in *input_archive*. """ - self.output_dir = output_dir + self.output_dir = output_dir # Combine all gtfs Transfer objects to a single pandas DataFrame self.transfers_df = gtfs_feed.transfers @@ -119,23 +120,23 @@ def __init__(self, input_archive, output_dir, gtfs_feed): # verify required columns are present transfer_ft_cols = list(transfers_ft_df.columns.values) - assert(Transfer.TRANSFERS_COLUMN_FROM_STOP in transfer_ft_cols) - assert(Transfer.TRANSFERS_COLUMN_TO_STOP in transfer_ft_cols) - assert(Transfer.TRANSFERS_COLUMN_DISTANCE in transfer_ft_cols) + assert (Transfer.TRANSFERS_COLUMN_FROM_STOP in transfer_ft_cols) + assert (Transfer.TRANSFERS_COLUMN_TO_STOP in transfer_ft_cols) + assert (Transfer.TRANSFERS_COLUMN_DISTANCE in transfer_ft_cols) # join to the transfers dataframe -- need to use the transfers_ft as the primary because # it may have PNR lot id to/from stop transfers (while gtfs transfers does not), # and we don't want to drop them if len(transfers_ft_df) > 0: self.transfers_df = pd.merge(left=self.transfers_df, right=transfers_ft_df, - how='right', - on=[Transfer.TRANSFERS_COLUMN_FROM_STOP, - Transfer.TRANSFERS_COLUMN_TO_STOP]) + how='right', + on=[Transfer.TRANSFERS_COLUMN_FROM_STOP, + Transfer.TRANSFERS_COLUMN_TO_STOP]) # fill in NAN - self.transfers_df.fillna(value={Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME:0, - Transfer.TRANSFERS_COLUMN_TRANSFER_TYPE:0, - Transfer.TRANSFERS_COLUMN_STOP_TO_STOP:False}, + self.transfers_df.fillna(value={Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME: 0, + Transfer.TRANSFERS_COLUMN_TRANSFER_TYPE: 0, + Transfer.TRANSFERS_COLUMN_STOP_TO_STOP: False}, inplace=True) if Transfer.TRANSFERS_COLUMN_FROM_ROUTE not in self.transfers_df.columns.values: @@ -144,12 +145,13 @@ def __init__(self, input_archive, output_dir, gtfs_feed): self.transfers_df[Transfer.TRANSFERS_COLUMN_TO_ROUTE] = None # support BOTH TRANSFERS_COLUMN_FROM_ROUTE and TRANSFERS_COLUMN_TO_ROUTE but not one - one_route_specified_df = self.transfers_df.loc[ self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].notnull()^ - self.transfers_df[Transfer.TRANSFERS_COLUMN_TO_ROUTE].notnull() ] + one_route_specified_df = self.transfers_df.loc[ + self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].notnull() ^ + self.transfers_df[Transfer.TRANSFERS_COLUMN_TO_ROUTE].notnull()] if len(one_route_specified_df): error_msg = "Only one of %s or %s specified for transfer: need both or neither:\n%s" % \ - (Transfer.TRANSFERS_COLUMN_FROM_ROUTE, Transfer.TRANSFERS_COLUMN_TO_ROUTE, - str(one_route_specified_df)) + (Transfer.TRANSFERS_COLUMN_FROM_ROUTE, Transfer.TRANSFERS_COLUMN_TO_ROUTE, + str(one_route_specified_df)) FastTripsLogger.fatal(error_msg) raise NetworkInputError(Transfer.INPUT_TRANSFERS_FILE, error_msg) @@ -157,24 +159,24 @@ def __init__(self, input_archive, output_dir, gtfs_feed): self.transfers_df[Transfer.TRANSFERS_COLUMN_PENALTY] = 1.0 FastTripsLogger.debug("=========== TRANSFERS ===========\n" + str(self.transfers_df.head())) - FastTripsLogger.debug("\n"+str(self.transfers_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.transfers_df.dtypes)) # TODO: this is to be consistent with original implementation. Remove? if len(self.transfers_df) > 0: self.transfers_df[Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN] = \ - self.transfers_df[Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME]/60.0 + self.transfers_df[Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME] / 60.0 # fill in null dist - null_dist = self.transfers_df.loc[ self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE].isnull() ] + null_dist = self.transfers_df.loc[self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE].isnull()] if len(null_dist) > 0: FastTripsLogger.warn("Filling in %d transfers with null dist" % len(null_dist)) - self.transfers_df.loc[ self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE].isnull(), - Transfer.TRANSFERS_COLUMN_DISTANCE ] = 0.0 + self.transfers_df.loc[self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE].isnull(), + Transfer.TRANSFERS_COLUMN_DISTANCE] = 0.0 # transfer time is based on distance self.transfers_df[Transfer.TRANSFERS_COLUMN_TIME_MIN] = \ - self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE]*60.0/Transfer.WALK_SPEED_MILES_PER_HOUR + self.transfers_df[Transfer.TRANSFERS_COLUMN_DISTANCE] * 60.0 / Transfer.WALK_SPEED_MILES_PER_HOUR # Sanity check transfer times. A 13 hour-long walk transfer is suspicious. # TODO: make this less arbitrary? It's based on the max SFCTA xfer link but it is too high @@ -186,16 +188,17 @@ def __init__(self, input_archive, output_dir, gtfs_feed): FastTripsLogger.fatal("\n%s\n" % str(too_long_transfers.head())) raise NetworkInputError(Transfer.INPUT_TRANSFERS_FILE, error_msg) - self.transfers_df.loc[\ - self.transfers_df[Transfer.TRANSFERS_COLUMN_TIME_MIN] < self.transfers_df[Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN], \ + self.transfers_df.loc[ \ + self.transfers_df[Transfer.TRANSFERS_COLUMN_TIME_MIN] < self.transfers_df[ + Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN], \ Transfer.TRANSFERS_COLUMN_TIME_MIN] = self.transfers_df[Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME_MIN] # convert time column from float to timedelta self.transfers_df[Transfer.TRANSFERS_COLUMN_TIME] = \ self.transfers_df[Transfer.TRANSFERS_COLUMN_TIME_MIN].map(lambda x: datetime.timedelta(minutes=x)) - FastTripsLogger.debug("Final\n"+str(self.transfers_df)) - FastTripsLogger.debug("\n"+str(self.transfers_df.dtypes)) + FastTripsLogger.debug("Final\n" + str(self.transfers_df)) + FastTripsLogger.debug("\n" + str(self.transfers_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.transfers_df), "transfers", "transfers.txt", Transfer.INPUT_TRANSFERS_FILE)) @@ -208,15 +211,15 @@ def add_numeric_stop_id(self, stops): # Add the numeric stop ids to transfers if len(self.transfers_df) > 0: self.transfers_df = stops.add_numeric_stop_id(self.transfers_df, - id_colname=Transfer.TRANSFERS_COLUMN_FROM_STOP, - numeric_newcolname=Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, - warn=True, - warn_msg="Numeric stop id not found for transfer from_stop_id") + id_colname=Transfer.TRANSFERS_COLUMN_FROM_STOP, + numeric_newcolname=Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, + warn=True, + warn_msg="Numeric stop id not found for transfer from_stop_id") self.transfers_df = stops.add_numeric_stop_id(self.transfers_df, - id_colname=Transfer.TRANSFERS_COLUMN_TO_STOP, - numeric_newcolname=Transfer.TRANSFERS_COLUMN_TO_STOP_NUM, - warn=True, - warn_msg="Numeric stop id not found for transfer to_stop_id") + id_colname=Transfer.TRANSFERS_COLUMN_TO_STOP, + numeric_newcolname=Transfer.TRANSFERS_COLUMN_TO_STOP_NUM, + warn=True, + warn_msg="Numeric stop id not found for transfer to_stop_id") # We're ready to write it self.write_transfers_for_extension() @@ -229,9 +232,10 @@ def add_transfer_attributes(self, transfer_links_df, all_links_df): from .Passenger import Passenger len_transfer_links_df = len(transfer_links_df) - transfer_links_cols = list(transfer_links_df.columns.values) - FastTripsLogger.debug("add_transfer_attributes: transfer_links_df(%d) head(20)=\n%s\ntransfers_df head(20)=\n%s" % \ - (len_transfer_links_df, transfer_links_df.head(20).to_string(), self.transfers_df.head(20).to_string())) + transfer_links_cols = list(transfer_links_df.columns.values) + FastTripsLogger.debug( + "add_transfer_attributes: transfer_links_df(%d) head(20)=\n%s\ntransfers_df head(20)=\n%s" % \ + (len_transfer_links_df, transfer_links_df.head(20).to_string(), self.transfers_df.head(20).to_string())) # nothing to do if len_transfer_links_df == 0: @@ -245,109 +249,115 @@ def add_transfer_attributes(self, transfer_links_df, all_links_df): # match on both from route and to route if Transfer.TRANSFERS_COLUMN_FROM_ROUTE not in self.transfers_df.columns.values: transfers_with_routes_df = pd.DataFrame() - transfers_wo_routes_df = self.transfers_df + transfers_wo_routes_df = self.transfers_df else: - transfers_with_routes_df = self.transfers_df.loc[ self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].notnull() ] - transfers_wo_routes_df = self.transfers_df.loc[ self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].isnull() ] + transfers_with_routes_df = self.transfers_df.loc[ + self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].notnull()] + transfers_wo_routes_df = self.transfers_df.loc[ + self.transfers_df[Transfer.TRANSFERS_COLUMN_FROM_ROUTE].isnull()] FastTripsLogger.debug("add_transfer_attributes: have %d transfers with routes and %d transfers without routes" % \ (len(transfers_with_routes_df), len(transfers_wo_routes_df))) if len(transfers_with_routes_df) > 0: # this is what we need of the trips - trip_links_df = all_links_df.loc[ all_links_df[Passenger.PF_COL_ROUTE_ID].notnull(), - [Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM, - Passenger.PF_COL_ROUTE_ID] - ] + trip_links_df = all_links_df.loc[all_links_df[Passenger.PF_COL_ROUTE_ID].notnull(), + [Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM, + Passenger.PF_COL_ROUTE_ID] + ] # FastTripsLogger.debug("trip_links_df head(20)=\n%s" % trip_links_df.head().to_string()) # match transfer with trip's next link to get from route_id trip_links_df["next_link_num"] = trip_links_df[Passenger.PF_COL_LINK_NUM] + 1 - transfer_links_df = pd.merge(left =transfer_links_df, - left_on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - right =trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - "next_link_num", - Passenger.PF_COL_ROUTE_ID]], - right_on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - "next_link_num"], - suffixes=[""," from"], - how="left") - transfer_links_df.rename(columns={"%s from" % Passenger.PF_COL_ROUTE_ID:Transfer.TRANSFERS_COLUMN_FROM_ROUTE}, inplace=True) + transfer_links_df = pd.merge(left=transfer_links_df, + left_on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + right=trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + "next_link_num", + Passenger.PF_COL_ROUTE_ID]], + right_on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + "next_link_num"], + suffixes=["", " from"], + how="left") + transfer_links_df.rename( + columns={"%s from" % Passenger.PF_COL_ROUTE_ID: Transfer.TRANSFERS_COLUMN_FROM_ROUTE}, inplace=True) # match transfer with trip's prev link to get to route_id trip_links_df["prev_link_num"] = trip_links_df[Passenger.PF_COL_LINK_NUM] - 1 - transfer_links_df = pd.merge(left =transfer_links_df, - left_on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - Passenger.PF_COL_LINK_NUM], - right =trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - "prev_link_num", - Passenger.PF_COL_ROUTE_ID]], - right_on =[Passenger.TRIP_LIST_COLUMN_PERSON_ID, - Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, - Passenger.PF_COL_PATH_NUM, - "prev_link_num"], - suffixes=[""," to"], - how="left") - transfer_links_df.rename(columns={"%s to" % Passenger.PF_COL_ROUTE_ID:Transfer.TRANSFERS_COLUMN_TO_ROUTE}, inplace=True) - transfer_links_df.drop(["prev_link_num","next_link_num"], axis=1, inplace=True) + transfer_links_df = pd.merge(left=transfer_links_df, + left_on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + Passenger.PF_COL_LINK_NUM], + right=trip_links_df[[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + "prev_link_num", + Passenger.PF_COL_ROUTE_ID]], + right_on=[Passenger.TRIP_LIST_COLUMN_PERSON_ID, + Passenger.TRIP_LIST_COLUMN_PERSON_TRIP_ID, + Passenger.PF_COL_PATH_NUM, + "prev_link_num"], + suffixes=["", " to"], + how="left") + transfer_links_df.rename(columns={"%s to" % Passenger.PF_COL_ROUTE_ID: Transfer.TRANSFERS_COLUMN_TO_ROUTE}, + inplace=True) + transfer_links_df.drop(["prev_link_num", "next_link_num"], axis=1, inplace=True) # FastTripsLogger.debug("transfer_links_df after adding route info:\n%s" % transfer_links_df.head(20).to_string()) # match on transfer attributes - transfer_links_df = pd.merge(left =transfer_links_df, - left_on =["A_id_num","B_id_num", - Transfer.TRANSFERS_COLUMN_FROM_ROUTE, - Transfer.TRANSFERS_COLUMN_TO_ROUTE], - right =transfers_with_routes_df, - right_on =[Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, - Transfer.TRANSFERS_COLUMN_TO_STOP_NUM, - Transfer.TRANSFERS_COLUMN_FROM_ROUTE, - Transfer.TRANSFERS_COLUMN_TO_ROUTE], - how ="left", - indicator=True) + transfer_links_df = pd.merge(left=transfer_links_df, + left_on=["A_id_num", "B_id_num", + Transfer.TRANSFERS_COLUMN_FROM_ROUTE, + Transfer.TRANSFERS_COLUMN_TO_ROUTE], + right=transfers_with_routes_df, + right_on=[Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, + Transfer.TRANSFERS_COLUMN_TO_STOP_NUM, + Transfer.TRANSFERS_COLUMN_FROM_ROUTE, + Transfer.TRANSFERS_COLUMN_TO_ROUTE], + how="left", + indicator=True) # FastTripsLogger.debug("transfer_links_df _merge: \n%s" % str(transfer_links_df["_merge"].value_counts())) - transfer_links_df.drop([Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM,Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], axis=1, inplace=True) + transfer_links_df.drop([Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], + axis=1, inplace=True) # now some of these have attributes, some still need - transfer_links_done = transfer_links_df.loc[transfer_links_df["_merge"]=="both"].copy() + transfer_links_done = transfer_links_df.loc[transfer_links_df["_merge"] == "both"].copy() transfer_links_done.drop(["_merge"], axis=1, inplace=True) # select remainder and reset to original columns - transfer_links_df = transfer_links_df.loc[ transfer_links_df["_merge"]=="left_only", transfer_links_cols ] + transfer_links_df = transfer_links_df.loc[transfer_links_df["_merge"] == "left_only", transfer_links_cols] # FastTripsLogger.debug("transfer_links_df split into %d done:\n%s" % (len(transfer_links_done), transfer_links_done.head(20).to_string())) # FastTripsLogger.debug("transfer_links_df split into %d not done:\n%s" % (len(transfer_links_df), transfer_links_df.head(20).to_string())) # match on both from stops ONLY if len(transfers_wo_routes_df) > 0: - transfer_links_df = pd.merge(left =transfer_links_df, - left_on =["A_id_num","B_id_num"], - right =transfers_wo_routes_df, - right_on =[Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, - Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], - how ="left") - transfer_links_df.drop([Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM,Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], axis=1, inplace=True) + transfer_links_df = pd.merge(left=transfer_links_df, + left_on=["A_id_num", "B_id_num"], + right=transfers_wo_routes_df, + right_on=[Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, + Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], + how="left") + transfer_links_df.drop([Transfer.TRANSFERS_COLUMN_FROM_STOP_NUM, Transfer.TRANSFERS_COLUMN_TO_STOP_NUM], + axis=1, inplace=True) # put the two parts back together if len(transfer_links_done) > 0: transfer_links_df = pd.concat([transfer_links_df, transfer_links_done], axis=0, ignore_index=True) # make sure we didn't lose anything - assert(len_transfer_links_df == len(transfer_links_df)) + assert (len_transfer_links_df == len(transfer_links_df)) return transfer_links_df @@ -358,22 +368,22 @@ def write_transfers_for_extension(self): Only write the stop/stop transfers since lot/stop transfers are only used for creating drive access links. """ - transfers_df = self.transfers_df.loc[self.transfers_df[Transfer.TRANSFERS_COLUMN_STOP_TO_STOP]==True].copy() + transfers_df = self.transfers_df.loc[self.transfers_df[Transfer.TRANSFERS_COLUMN_STOP_TO_STOP] == True].copy() # drop transfer_type==3 => that means no transfer possible # https://github.com/osplanning-data-standards/GTFS-PLUS/blob/master/files/transfers.md transfers_df = transfers_df.loc[transfers_df[Transfer.TRANSFERS_COLUMN_TRANSFER_TYPE] != 3] # drop some of the attributes - drop_attrs = [Transfer.TRANSFERS_COLUMN_TIME, # use numerical version - Transfer.TRANSFERS_COLUMN_FROM_STOP, # use numerical version - Transfer.TRANSFERS_COLUMN_TO_STOP, # use numerical version - Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME, # minute version is sufficient - Transfer.TRANSFERS_COLUMN_SCHEDULE_PRECEDENCE, # don't know what to do with this - Transfer.TRANSFERS_COLUMN_STOP_TO_STOP, # not needed - Transfer.TRANSFERS_COLUMN_FROM_ROUTE, # TODO? - Transfer.TRANSFERS_COLUMN_TO_ROUTE # TODO? - ] + drop_attrs = [Transfer.TRANSFERS_COLUMN_TIME, # use numerical version + Transfer.TRANSFERS_COLUMN_FROM_STOP, # use numerical version + Transfer.TRANSFERS_COLUMN_TO_STOP, # use numerical version + Transfer.TRANSFERS_COLUMN_MIN_TRANSFER_TIME, # minute version is sufficient + Transfer.TRANSFERS_COLUMN_SCHEDULE_PRECEDENCE, # don't know what to do with this + Transfer.TRANSFERS_COLUMN_STOP_TO_STOP, # not needed + Transfer.TRANSFERS_COLUMN_FROM_ROUTE, # TODO? + Transfer.TRANSFERS_COLUMN_TO_ROUTE # TODO? + ] keep_attrs = set(list(transfers_df.columns.values)) - set(drop_attrs) transfers_df = transfers_df[list(keep_attrs)] @@ -386,7 +396,7 @@ def write_transfers_for_extension(self): # this will make it so beyond from stop num and to stop num, # the remaining columns collapse to variable name, variable value transfers_df = transfers_df.stack().reset_index() - transfers_df.rename(columns={"level_2":"attr_name", 0:"attr_value"}, inplace=True) + transfers_df.rename(columns={"level_2": "attr_name", 0: "attr_value"}, inplace=True) transfers_df.to_csv(os.path.join(self.output_dir, Transfer.OUTPUT_TRANSFERS_FILE), sep=" ", index=False) diff --git a/fasttrips/Trip.py b/fasttrips/Trip.py index 497c754e..b4e0dd8d 100644 --- a/fasttrips/Trip.py +++ b/fasttrips/Trip.py @@ -3,7 +3,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,16 +16,17 @@ See the License for the specific language governing permissions and limitations under the License. """ + import datetime import os import numpy as np import pandas as pd -from .Error import NetworkInputError +from .Error import NetworkInputError from .Logger import FastTripsLogger -from .Route import Route -from .Util import Util +from .Route import Route +from .Util import Util class Trip(object): @@ -45,169 +46,168 @@ class Trip(object): #: File with fasttrips trip information (this extends the #: `gtfs trips `_ file). # See `trips_ft specification `_. - INPUT_TRIPS_FILE = "trips_ft.txt" + INPUT_TRIPS_FILE = "trips_ft.txt" #: gtfs Trips column name: Unique identifier. This will be the index of the trips table. (object) - TRIPS_COLUMN_TRIP_ID = 'trip_id' + TRIPS_COLUMN_TRIP_ID = 'trip_id' #: gtfs Trips column name: Route unique identifier. - TRIPS_COLUMN_ROUTE_ID = 'route_id' + TRIPS_COLUMN_ROUTE_ID = 'route_id' #: gtfs Trips column name: Service unique identifier. - TRIPS_COLUMN_SERVICE_ID = 'service_id' + TRIPS_COLUMN_SERVICE_ID = 'service_id' #: gtfs Trip column name: Direction binary identifier. - TRIPS_COLUMN_DIRECTION_ID = 'direction_id' + TRIPS_COLUMN_DIRECTION_ID = 'direction_id' #: gtfs Trip column name: Shape ID - TRIPS_COLUMN_SHAPE_ID = 'shape_id' + TRIPS_COLUMN_SHAPE_ID = 'shape_id' #: fasttrips Trips column name: Vehicle Name - TRIPS_COLUMN_VEHICLE_NAME = 'vehicle_name' + TRIPS_COLUMN_VEHICLE_NAME = 'vehicle_name' # ========== Added by fasttrips ======================================================= #: fasttrips Trips column name: Trip Numerical Identifier. Int. - TRIPS_COLUMN_TRIP_ID_NUM = 'trip_id_num' + TRIPS_COLUMN_TRIP_ID_NUM = 'trip_id_num' #: fasttrips Trips column name: Route Numerical Identifier. Int. - TRIPS_COLUMN_ROUTE_ID_NUM = Route.ROUTES_COLUMN_ROUTE_ID_NUM + TRIPS_COLUMN_ROUTE_ID_NUM = Route.ROUTES_COLUMN_ROUTE_ID_NUM #: fasttrips Trips column name: Mode Numerical Identifier. Int. - TRIPS_COLUMN_MODE_NUM = Route.ROUTES_COLUMN_MODE_NUM + TRIPS_COLUMN_MODE_NUM = Route.ROUTES_COLUMN_MODE_NUM #: fasttrips Trips column name: Max Stop Sequence number. Int. - TRIPS_COLUMN_MAX_STOP_SEQUENCE = 'max_stop_seq' + TRIPS_COLUMN_MAX_STOP_SEQUENCE = 'max_stop_seq' #: fasttrips Trip column name: Trip departure time (from the first stop). - TRIPS_COLUMN_TRIP_DEPARTURE_TIME = 'trip_departure_time' + TRIPS_COLUMN_TRIP_DEPARTURE_TIME = 'trip_departure_time' #: File with fasttrips vehicles information. #: See `vehicles_ft specification `_. - INPUT_VEHICLES_FILE = 'vehicles_ft.txt' + INPUT_VEHICLES_FILE = 'vehicles_ft.txt' #: fasttrips Vehicles column name: Vehicle name (identifier) - VEHICLES_COLUMN_VEHICLE_NAME = TRIPS_COLUMN_VEHICLE_NAME + VEHICLES_COLUMN_VEHICLE_NAME = TRIPS_COLUMN_VEHICLE_NAME #: fasttrips Vehicles column name: Vehicle Description - VEHICLES_COLUMN_VEHICLE_DESCRIPTION = 'vehicle_description' + VEHICLES_COLUMN_VEHICLE_DESCRIPTION = 'vehicle_description' #: fasttrips Vehicles column name: Seated Capacity - VEHICLES_COLUMN_SEATED_CAPACITY = 'seated_capacity' + VEHICLES_COLUMN_SEATED_CAPACITY = 'seated_capacity' #: fasttrips Vehicles column name: Standing Capacity - VEHICLES_COLUMN_STANDING_CAPACITY = 'standing_capacity' + VEHICLES_COLUMN_STANDING_CAPACITY = 'standing_capacity' #: fasttrips Vehicles column name: Number of Doors - VEHICLES_COLUMN_NUMBER_OF_DOORS = 'number_of_doors' + VEHICLES_COLUMN_NUMBER_OF_DOORS = 'number_of_doors' #: fasttrips Vehicles column name: Maximum Speed (mph) - VEHICLES_COLUMN_MAXIMUM_SPEED = 'max_speed' + VEHICLES_COLUMN_MAXIMUM_SPEED = 'max_speed' #: fasttrips Vehicles column name: Vehicle Length (feet) - VEHICLES_COLUMN_VEHICLE_LENGTH = 'vehicle_length' + VEHICLES_COLUMN_VEHICLE_LENGTH = 'vehicle_length' #: fasttrips Vehicles column name: Platform Height (inches) - VEHICLES_COLUMN_PLATFORM_HEIGHT = 'platform_height' + VEHICLES_COLUMN_PLATFORM_HEIGHT = 'platform_height' #: fasttrips Vehicles column name: Propulsion Type - VEHICLES_COLUMN_PROPULSION_TYPE = 'propulsion_type' + VEHICLES_COLUMN_PROPULSION_TYPE = 'propulsion_type' #: fasttrips Vehicles column name: Wheelchair Capacity (overrides trip) - VEHICLES_COLUMN_WHEELCHAIR_CAPACITY = 'wheelchair_capacity' + VEHICLES_COLUMN_WHEELCHAIR_CAPACITY = 'wheelchair_capacity' #: fasttrips Vehicles column name: Bicycle Capacity - VEHICLES_COLUMN_BICYCLE_CAPACITY = 'bicycle_capacity' + VEHICLES_COLUMN_BICYCLE_CAPACITY = 'bicycle_capacity' #: fasttrips Vehicles column name: Acceleration (feet per (second^2)) float. - VEHICLES_COLUMN_ACCELERATION = 'acceleration' + VEHICLES_COLUMN_ACCELERATION = 'acceleration' #: fasttrips Vehicles column name: Decelration (feet per (second^2)) float. - VEHICLES_COLUMN_DECELERATION = 'deceleration' + VEHICLES_COLUMN_DECELERATION = 'deceleration' #: fasttrips Vehilces column name: Dwell formula (string) - VEHICLES_COLUMN_DWELL_FORMULA = 'dwell_formula' + VEHICLES_COLUMN_DWELL_FORMULA = 'dwell_formula' # ========== Added by fasttrips ======================================================= #: fasttrips Trips column name: Vehicle Total (Seated + Standing) Capacity - VEHICLES_COLUMN_TOTAL_CAPACITY = 'capacity' + VEHICLES_COLUMN_TOTAL_CAPACITY = 'capacity' #: fasttrips Vehicles column name: Maximum Speed (fps) - VEHICLES_COLUMN_MAXIMUM_SPEED_FPS = 'max_speed_fps' + VEHICLES_COLUMN_MAXIMUM_SPEED_FPS = 'max_speed_fps' #: fasttrips Service column name: Start Date string in 'YYYYMMDD' format - SERVICE_COLUMN_START_DATE_STR = 'start_date_str' + SERVICE_COLUMN_START_DATE_STR = 'start_date_str' #: fasttrips Service column name: Start Date as datetime.date - SERVICE_COLUMN_START_DATE = 'start_date' + SERVICE_COLUMN_START_DATE = 'start_date' #: fasttrips Service column name: End Date string in 'YYYYMMDD' format - SERVICE_COLUMN_END_DATE_STR = 'end_date_str' + SERVICE_COLUMN_END_DATE_STR = 'end_date_str' #: fasttrips Service column name: End Date as datetime.date - SERVICE_COLUMN_END_DATE = 'end_date' + SERVICE_COLUMN_END_DATE = 'end_date' #: File with fasttrips stop time information (this extends the #: `gtfs stop times `_ file). # See `stop_times_ft specification `_. - INPUT_STOPTIMES_FILE = "stop_times_ft.txt" + INPUT_STOPTIMES_FILE = "stop_times_ft.txt" #: gtfs Stop times column name: Trip unique identifier. (String) - STOPTIMES_COLUMN_TRIP_ID = 'trip_id' + STOPTIMES_COLUMN_TRIP_ID = 'trip_id' #: gtfs Stop times column name: Stop unique identifier - STOPTIMES_COLUMN_STOP_ID = 'stop_id' + STOPTIMES_COLUMN_STOP_ID = 'stop_id' #: gtfs Stop times column name: Sequence number of stop within a trip. #: Starts at 1 and is sequential - STOPTIMES_COLUMN_STOP_SEQUENCE = 'stop_sequence' + STOPTIMES_COLUMN_STOP_SEQUENCE = 'stop_sequence' #: Stop times column name: Arrival time. This is a float, minutes after midnight. - STOPTIMES_COLUMN_ARRIVAL_TIME_MIN = 'arrival_time_min' + STOPTIMES_COLUMN_ARRIVAL_TIME_MIN = 'arrival_time_min' #: gtfs Stop times column name: Arrival time. This is a DateTime. - STOPTIMES_COLUMN_ARRIVAL_TIME = 'arrival_time' + STOPTIMES_COLUMN_ARRIVAL_TIME = 'arrival_time' #: Stop times column name: Departure time. This is a float, minutes after midnight. - STOPTIMES_COLUMN_DEPARTURE_TIME_MIN = 'departure_time_min' + STOPTIMES_COLUMN_DEPARTURE_TIME_MIN = 'departure_time_min' #: gtfs Stop times column name: Departure time. This is a DateTime. - STOPTIMES_COLUMN_DEPARTURE_TIME = 'departure_time' + STOPTIMES_COLUMN_DEPARTURE_TIME = 'departure_time' #: gtfs Stop times stop times column name: Stop Headsign - STOPTIMES_COLUMN_HEADSIGN = 'stop_headsign' + STOPTIMES_COLUMN_HEADSIGN = 'stop_headsign' #: gtfs Stop times stop times column name: Pickup Type - STOPTIMES_COLUMN_PICKUP_TYPE = 'pickup_type' + STOPTIMES_COLUMN_PICKUP_TYPE = 'pickup_type' #: gtfs Stop times stop times column name: Drop Off Type - STOPTIMES_COLUMN_DROP_OFF_TYPE = 'drop_off_type' + STOPTIMES_COLUMN_DROP_OFF_TYPE = 'drop_off_type' #: gtfs Stop times stop times column name: Shape Distance Traveled - STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED = 'shape_dist_traveled' + STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED = 'shape_dist_traveled' #: gtfs Stop times stop times column name: Time Point - STOPTIMES_COLUMN_TIMEPOINT = 'timepoint' + STOPTIMES_COLUMN_TIMEPOINT = 'timepoint' # ========== Added by fasttrips ======================================================= #: fasttrips Trips column name: Trip Numerical Identifier. Int. - STOPTIMES_COLUMN_TRIP_ID_NUM = TRIPS_COLUMN_TRIP_ID_NUM + STOPTIMES_COLUMN_TRIP_ID_NUM = TRIPS_COLUMN_TRIP_ID_NUM #: fasttrips Trips column name: Stop Numerical Identifier. Int. - STOPTIMES_COLUMN_STOP_ID_NUM = 'stop_id_num' + STOPTIMES_COLUMN_STOP_ID_NUM = 'stop_id_num' #: fasttrips Trips column name: Original Travel Time. This is a timedelta. #: This represents the travel from original input, and is assumed to include #: accel from first stop and decel to last stop, but no dwell times and no other #: accel/decel times. - STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME = "original_travel_time" + STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME = "original_travel_time" #: fasttrips Trips column name: Travel Time. This is a timedelta. - STOPTIMES_COLUMN_TRAVEL_TIME = "travel_time" + STOPTIMES_COLUMN_TRAVEL_TIME = "travel_time" #: fasttrips Trips column name: Travel Time in seconds. Float. - STOPTIMES_COLUMN_TRAVEL_TIME_SEC = "travel_time_sec" + STOPTIMES_COLUMN_TRAVEL_TIME_SEC = "travel_time_sec" #: fasttrips Trips column name: Dwell Time. This is a timedelta. - STOPTIMES_COLUMN_DWELL_TIME = "dwell_time" + STOPTIMES_COLUMN_DWELL_TIME = "dwell_time" #: fasttrips Trips column name: Dwell Time in seconds. Float - STOPTIMES_COLUMN_DWELL_TIME_SEC = "dwell_time_sec" - + STOPTIMES_COLUMN_DWELL_TIME_SEC = "dwell_time_sec" #: File with trip ID, trip ID number correspondence - OUTPUT_TRIP_ID_NUM_FILE = 'ft_intermediate_trip_id.txt' + OUTPUT_TRIP_ID_NUM_FILE = 'ft_intermediate_trip_id.txt' #: File with trip information - OUTPUT_TRIPINFO_FILE = 'ft_intermediate_trip_info.txt' + OUTPUT_TRIPINFO_FILE = 'ft_intermediate_trip_info.txt' #: Default headway if no previous matching route/trip - DEFAULT_HEADWAY = 60 + DEFAULT_HEADWAY = 60 # ========== Simulation column names ======================================================= #: Result column name: Boards. Int. - SIM_COL_VEH_BOARDS = 'boards' + SIM_COL_VEH_BOARDS = 'boards' #: Result column name: Alights. Int. - SIM_COL_VEH_ALIGHTS = 'alights' + SIM_COL_VEH_ALIGHTS = 'alights' #: Result column name: Onboard. Cumulative sum of :py:attr:`SIM_COL_VEH_BOARDS` - :py:attr:`SIM_COL_VEH_ALIGHTS`. Int. - SIM_COL_VEH_ONBOARD = 'onboard' + SIM_COL_VEH_ONBOARD = 'onboard' #: Result column name: Standees. Onboard - sitting capacity, if configured. Int. - SIM_COL_VEH_STANDEES = 'standees' + SIM_COL_VEH_STANDEES = 'standees' #: Result column name: Friction. Where positive, friction = on+off+standees. Int. - SIM_COL_VEH_FRICTION = 'friction' + SIM_COL_VEH_FRICTION = 'friction' #: Result column name: Number of onboard passengers minus capacity. Int. Can be negative. - SIM_COL_VEH_OVERCAP = 'overcap' + SIM_COL_VEH_OVERCAP = 'overcap' #: Result column name: Percentage of boards that are problematic. (overcap/boards) - SIM_COL_VEH_OVERCAP_FRAC = 'overcap_frac' + SIM_COL_VEH_OVERCAP_FRAC = 'overcap_frac' #: Result column name: MSA of column :py:attr:`SIM_COL_VEH_BOARDS`. Float. - SIM_COL_VEH_MSA_BOARDS = 'msa_boards' + SIM_COL_VEH_MSA_BOARDS = 'msa_boards' #: Result column name: MSA of column :py:attr:`SIM_COL_VEH_ALIGHTS`. Float. - SIM_COL_VEH_MSA_ALIGHTS = 'msa_alights' + SIM_COL_VEH_MSA_ALIGHTS = 'msa_alights' #: Result column name: MSA Onboard. Cumulative sum of :py:attr:`SIM_COL_VEH_MSA_BOARDS` - :py:attr:`SIM_COL_VEH_MSA_ALIGHTS`. Float. - SIM_COL_VEH_MSA_ONBOARD = 'msa_onboard' + SIM_COL_VEH_MSA_ONBOARD = 'msa_onboard' #: Result column name: MSA Standeeds. MSA onboard - sitting capacity, if configured. Float. - SIM_COL_VEH_MSA_STANDEES = 'msa_standees' + SIM_COL_VEH_MSA_STANDEES = 'msa_standees' #: Result column name: MSA Friction. Where positive, MSA friction = MSA boards + MSA alights + MSA standees. Float. - SIM_COL_VEH_MSA_FRICTION = 'msa_friction' + SIM_COL_VEH_MSA_FRICTION = 'msa_friction' #: Result column name: Number of MSA onboard passengers minus capacity. Float. - SIM_COL_VEH_MSA_OVERCAP = 'msa_overcap' + SIM_COL_VEH_MSA_OVERCAP = 'msa_overcap' def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, prepend_route_id_to_trip_id): """ @@ -221,7 +221,7 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p trips_ft_df = gtfs_feed.get(Trip.INPUT_TRIPS_FILE) # verify the required columns are present - assert(Trip.VEHICLES_COLUMN_VEHICLE_NAME in self.vehicles_df) + assert (Trip.VEHICLES_COLUMN_VEHICLE_NAME in self.vehicles_df) if ({Trip.VEHICLES_COLUMN_SEATED_CAPACITY, Trip.VEHICLES_COLUMN_STANDING_CAPACITY}.issubset(self.vehicles_df)): self.vehicles_df[Trip.VEHICLES_COLUMN_TOTAL_CAPACITY] = \ @@ -235,7 +235,7 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p from .Assignment import Assignment if Assignment.CAPACITY_CONSTRAINT: error_str = "capacity_constraint is configured to be True but either %s or %s is missing from vehicle information" % \ - (Trip.VEHICLES_COLUMN_SEATED_CAPACITY, Trip.VEHICLES_COLUMN_STANDING_CAPACITY) + (Trip.VEHICLES_COLUMN_SEATED_CAPACITY, Trip.VEHICLES_COLUMN_STANDING_CAPACITY) FastTripsLogger.fatal(error_str) raise NetworkInputError(Trip.INPUT_VEHICLES_FILE, error_str) @@ -243,10 +243,10 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p # convert mph to fps for maximum speed if Trip.VEHICLES_COLUMN_MAXIMUM_SPEED in self.vehicles_df: self.vehicles_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS] = \ - self.vehicles_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED]*5280.0/(60.0*60.0) + self.vehicles_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED] * 5280.0 / (60.0 * 60.0) FastTripsLogger.debug("=========== VEHICLES ===========\n" + str(self.vehicles_df.head())) - FastTripsLogger.debug("\n"+str(self.vehicles_df.index.dtype)+"\n"+str(self.vehicles_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.vehicles_df.index.dtype) + "\n" + str(self.vehicles_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s" % (len(self.vehicles_df), "vehicles", self.INPUT_VEHICLES_FILE)) @@ -258,15 +258,13 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p raise NetworkInputError(Trip.INPUT_TRIPS_FILE, error_str) - # verify required columns are present - assert({Trip.TRIPS_COLUMN_TRIP_ID,Trip.TRIPS_COLUMN_VEHICLE_NAME}.issubset(trips_ft_df)) + assert ({Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_VEHICLE_NAME}.issubset(trips_ft_df)) # Join to the trips dataframe self.trips_df = pd.merge(left=self.trips_df, right=trips_ft_df, - how='left', - on=Trip.TRIPS_COLUMN_TRIP_ID) - + how='left', + on=Trip.TRIPS_COLUMN_TRIP_ID) # Trip IDs are strings. Create a unique numeric trip ID. self.trip_id_df = Util.add_numeric_column(self.trips_df[[Trip.TRIPS_COLUMN_TRIP_ID]], @@ -277,35 +275,37 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p # prepend_route_id_to_trip_id if prepend_route_id_to_trip_id: # get the route id back again - trip_id_df = pd.merge(self.trip_id_df, self.trips_df[[Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_ROUTE_ID]], - how='left', on=Trip.TRIPS_COLUMN_TRIP_ID) + trip_id_df = pd.merge(self.trip_id_df, + self.trips_df[[Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_ROUTE_ID]], + how='left', on=Trip.TRIPS_COLUMN_TRIP_ID) trip_id_df.rename(columns={Trip.TRIPS_COLUMN_TRIP_ID: 'trip_id_orig'}, inplace=True) - trip_id_df[Trip.TRIPS_COLUMN_TRIP_ID] = trip_id_df[Trip.TRIPS_COLUMN_ROUTE_ID].map(str) + str("_") + trip_id_df['trip_id_orig'] + trip_id_df[Trip.TRIPS_COLUMN_TRIP_ID] = trip_id_df[Trip.TRIPS_COLUMN_ROUTE_ID].map(str) + str("_") + \ + trip_id_df['trip_id_orig'] else: trip_id_df = self.trip_id_df trip_id_df.to_csv(os.path.join(output_dir, Trip.OUTPUT_TRIP_ID_NUM_FILE), - columns=[Trip.TRIPS_COLUMN_TRIP_ID_NUM, Trip.TRIPS_COLUMN_TRIP_ID], - sep=" ", index=False) + columns=[Trip.TRIPS_COLUMN_TRIP_ID_NUM, Trip.TRIPS_COLUMN_TRIP_ID], + sep=" ", index=False) FastTripsLogger.debug("Wrote %s" % os.path.join(output_dir, Trip.OUTPUT_TRIP_ID_NUM_FILE)) self.trips_df = pd.merge(left=self.trips_df, right=self.trip_id_df, on=Trip.TRIPS_COLUMN_TRIP_ID, how='left') # Merge vehicles self.trips_df = pd.merge(left=self.trips_df, right=self.vehicles_df, how='left', - left_on=Trip.TRIPS_COLUMN_VEHICLE_NAME, right_on=Trip.VEHICLES_COLUMN_VEHICLE_NAME) + left_on=Trip.TRIPS_COLUMN_VEHICLE_NAME, right_on=Trip.VEHICLES_COLUMN_VEHICLE_NAME) FastTripsLogger.debug("=========== TRIPS ===========\n" + str(self.trips_df.head())) - FastTripsLogger.debug("\n"+str(self.trips_df.index.dtype)+"\n"+str(self.trips_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.trips_df.index.dtype) + "\n" + str(self.trips_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.trips_df), "trips", "trips.txt", self.INPUT_TRIPS_FILE)) # Join with routes self.trips_df = pd.merge(left=self.trips_df, right=routes.routes_df, - how='left', - on=Trip.TRIPS_COLUMN_ROUTE_ID) + how='left', + on=Trip.TRIPS_COLUMN_ROUTE_ID) FastTripsLogger.debug("Final (%d)\n%s" % (len(self.trips_df), str(self.trips_df.head()))) - FastTripsLogger.debug("\n"+str(self.trips_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.trips_df.dtypes)) self.stop_times_df = gtfs_feed.stop_times @@ -314,22 +314,24 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p if not stop_times_ft_df.empty: # verify required columns are present stop_times_ft_cols = list(stop_times_ft_df) - assert({Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_ID}.issubset(stop_times_ft_df)) + assert ({Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_ID}.issubset(stop_times_ft_df)) # Join to the trips dataframe if len(stop_times_ft_df.columns) > 2: self.stop_times_df = pd.merge(left=self.stop_times_df, right=stop_times_ft_df, - how='left', - on=[Trip.STOPTIMES_COLUMN_TRIP_ID, - Trip.STOPTIMES_COLUMN_STOP_ID]) + how='left', + on=[Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_STOP_ID]) FastTripsLogger.debug("=========== STOP TIMES ===========\n" + str(self.stop_times_df.head())) - FastTripsLogger.debug("\n"+str(self.stop_times_df.index.dtype)+"\n"+str(self.stop_times_df.dtypes)) + FastTripsLogger.debug("\n" + str(self.stop_times_df.index.dtype) + "\n" + str(self.stop_times_df.dtypes)) self.add_shape_dist_traveled(stops) - self.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = self.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME]/ 60 - self.stop_times_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] = self.stop_times_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME]/ 60 + self.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = self.stop_times_df[ + Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] / 60 + self.stop_times_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] = self.stop_times_df[ + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] / 60 self.stop_times_df.rename(columns={ Trip.STOPTIMES_COLUMN_ARRIVAL_TIME: Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME: Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN, @@ -354,10 +356,11 @@ def __init__(self, input_archive, output_dir, gtfs_feed, today, stops, routes, p self.add_original_travel_time_and_dwell() self.add_trip_attrs_from_stoptimes() - FastTripsLogger.debug("Final\n" + str(self.stop_times_df.head().to_string(formatters=\ - {Trip.STOPTIMES_COLUMN_DEPARTURE_TIME:Util.datetime64_formatter, - Trip.STOPTIMES_COLUMN_ARRIVAL_TIME :Util.datetime64_formatter})) + \ - "\n" +str(self.stop_times_df.dtypes) ) + FastTripsLogger.debug("Final\n" + str(self.stop_times_df.head().to_string(formatters= \ + { + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME: Util.datetime64_formatter, + Trip.STOPTIMES_COLUMN_ARRIVAL_TIME: Util.datetime64_formatter})) + \ + "\n" + str(self.stop_times_df.dtypes)) FastTripsLogger.info("Read %7d %15s from %25s, %25s" % (len(self.stop_times_df), "stop times", "stop_times.txt", Trip.INPUT_STOPTIMES_FILE)) @@ -375,9 +378,9 @@ def add_numeric_trip_id(self, input_df, id_colname, numeric_newcolname): adds the numeric trip id as a column named *numeric_newcolname* and returns it. """ return Util.add_new_id(input_df, id_colname, numeric_newcolname, - mapping_df=self.trip_id_df, - mapping_id_colname=Trip.TRIPS_COLUMN_TRIP_ID, - mapping_newid_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM) + mapping_df=self.trip_id_df, + mapping_id_colname=Trip.TRIPS_COLUMN_TRIP_ID, + mapping_newid_colname=Trip.TRIPS_COLUMN_TRIP_ID_NUM) def get_stop_times(self, trip_id): """ @@ -385,13 +388,13 @@ def get_stop_times(self, trip_id): """ return self.stop_times_df.loc[ self.stop_times_df[Trip.STOPTIMES_COLUMN_TRIP_ID] == trip_id - ] + ] def number_of_stops(self, trip_id): """ Return the number of stops in this trip. """ - return(len(self.stop_times_df.loc[trip_id])) + return (len(self.stop_times_df.loc[trip_id])) def get_scheduled_departure(self, trip_id, stop_id): """ @@ -421,13 +424,13 @@ def add_original_travel_time_and_dwell(self): next_stop_df = self.stop_times_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.STOPTIMES_COLUMN_ARRIVAL_TIME]].copy() - next_stop_df.loc[:,Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]-1 - next_stop_df.rename(columns={Trip.STOPTIMES_COLUMN_ARRIVAL_TIME:"next_stop_arrival"}, inplace=True) + next_stop_df.loc[:, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] - 1 + next_stop_df.rename(columns={Trip.STOPTIMES_COLUMN_ARRIVAL_TIME: "next_stop_arrival"}, inplace=True) FastTripsLogger.debug("next stop arrival:\n%s\n" % next_stop_df.head().to_string()) self.stop_times_df = pd.merge(left=self.stop_times_df, right=next_stop_df, how='left') - assert(stop_times_len_df == len(self.stop_times_df)) + assert (stop_times_len_df == len(self.stop_times_df)) # this will be NaT for last stops self.stop_times_df[Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME] = \ @@ -436,15 +439,17 @@ def add_original_travel_time_and_dwell(self): self.stop_times_df.drop(["next_stop_arrival"], axis=1, inplace=True) # copy - self.stop_times_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME ] = self.stop_times_df[Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME] + self.stop_times_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME] = self.stop_times_df[ + Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME] self.stop_times_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = \ - (self.stop_times_df[Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME]/np.timedelta64(1, 's')) + (self.stop_times_df[Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME] / np.timedelta64(1, 's')) # dwell time self.stop_times_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] = \ - self.stop_times_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - self.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] + self.stop_times_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - self.stop_times_df[ + Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] self.stop_times_df[Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC] = \ - (self.stop_times_df[Trip.STOPTIMES_COLUMN_DWELL_TIME]/np.timedelta64(1, 's')) + (self.stop_times_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] / np.timedelta64(1, 's')) def add_trip_attrs_from_stoptimes(self): """ @@ -454,16 +459,17 @@ def add_trip_attrs_from_stoptimes(self): trips_len = len(self.trips_df) stops_by_trip = self.stop_times_df.groupby(Trip.STOPTIMES_COLUMN_TRIP_ID_NUM).agg( - {Trip.STOPTIMES_COLUMN_STOP_SEQUENCE :'max', - Trip.STOPTIMES_COLUMN_DEPARTURE_TIME:'min'}).reset_index() + {Trip.STOPTIMES_COLUMN_STOP_SEQUENCE: 'max', + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME: 'min'}).reset_index() # rename it to max_stop_seq - stops_by_trip.rename(columns={Trip.STOPTIMES_COLUMN_STOP_SEQUENCE :Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE, - Trip.STOPTIMES_COLUMN_DEPARTURE_TIME:Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME}, inplace=True) + stops_by_trip.rename(columns={Trip.STOPTIMES_COLUMN_STOP_SEQUENCE: Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE, + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME: Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME}, + inplace=True) # add it to trips self.trips_df = pd.merge(left=self.trips_df, right=stops_by_trip) # make sure we didn't change the length - assert(trips_len == len(self.trips_df)) + assert (trips_len == len(self.trips_df)) def add_shape_dist_traveled(self, stops): """ @@ -477,9 +483,11 @@ def add_shape_dist_traveled(self, stops): if Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED not in self.stop_times_df: self.stop_times_df[Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED] = np.nan - self.stop_times_df["null_shape_dist_traveled"] = pd.isnull(self.stop_times_df[Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED]) - FastTripsLogger.debug("add_shape_dist_traveled: missing %d out of %d shape_dist_traveled values in the stop times dataframe" % - (self.stop_times_df["null_shape_dist_traveled"].sum(), len(self.stop_times_df["null_shape_dist_traveled"]))) + self.stop_times_df["null_shape_dist_traveled"] = pd.isnull( + self.stop_times_df[Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED]) + FastTripsLogger.debug( + "add_shape_dist_traveled: missing %d out of %d shape_dist_traveled values in the stop times dataframe" % + (self.stop_times_df["null_shape_dist_traveled"].sum(), len(self.stop_times_df["null_shape_dist_traveled"]))) # if they're all set, nothing to do if self.stop_times_df["null_shape_dist_traveled"].sum() == 0: @@ -488,9 +496,10 @@ def add_shape_dist_traveled(self, stops): FastTripsLogger.warn("Adding shape_dist_traveled in miles") # aggregate to trip to find which trips are missing this field - stop_times_trips = self.stop_times_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, "null_shape_dist_traveled"]].groupby([Trip.STOPTIMES_COLUMN_TRIP_ID]).sum() + stop_times_trips = self.stop_times_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, "null_shape_dist_traveled"]].groupby( + [Trip.STOPTIMES_COLUMN_TRIP_ID]).sum() stop_times_trips["null_shape_dist_traveled_bool"] = False - stop_times_trips.loc[ stop_times_trips["null_shape_dist_traveled"] > 0, "null_shape_dist_traveled_bool"] = True + stop_times_trips.loc[stop_times_trips["null_shape_dist_traveled"] > 0, "null_shape_dist_traveled_bool"] = True stop_times_trips.drop(["null_shape_dist_traveled"], axis=1, inplace=True) stop_times_trips.reset_index(drop=False, inplace=True) @@ -498,33 +507,39 @@ def add_shape_dist_traveled(self, stops): # join this back to the stop_times dataframe self.stop_times_df.drop(["null_shape_dist_traveled"], axis=1, inplace=True) - self.stop_times_df = pd.merge(left =self.stop_times_df, - right=stop_times_trips, - on =Trip.STOPTIMES_COLUMN_TRIP_ID, - how ="left") + self.stop_times_df = pd.merge(left=self.stop_times_df, + right=stop_times_trips, + on=Trip.STOPTIMES_COLUMN_TRIP_ID, + how="left") # join stop information - self.stop_times_df = stops.add_stop_lat_lon(self.stop_times_df, Trip.STOPTIMES_COLUMN_STOP_ID, "stop_lat", "stop_lon") + self.stop_times_df = stops.add_stop_lat_lon(self.stop_times_df, Trip.STOPTIMES_COLUMN_STOP_ID, "stop_lat", + "stop_lon") # convert to links by adding prev_stop_seq -- this goes from 0 to max-1 self.stop_times_df["stop_seq_prev"] = self.stop_times_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] - 1 - self.stop_times_df = pd.merge(left =self.stop_times_df, - left_on =[Trip.STOPTIMES_COLUMN_TRIP_ID, "stop_seq_prev"], - right =self.stop_times_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - "stop_lat","stop_lon"]], - right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - how ="left", - suffixes=["","_prev"]) - Util.calculate_distance_miles(self.stop_times_df, "stop_lat","stop_lon","stop_lat_prev","stop_lon_prev","calc shape_dist_traveled") + self.stop_times_df = pd.merge(left=self.stop_times_df, + left_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, "stop_seq_prev"], + right=self.stop_times_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, + "stop_lat", "stop_lon"]], + right_on=[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + how="left", + suffixes=["", "_prev"]) + Util.calculate_distance_miles(self.stop_times_df, "stop_lat", "stop_lon", "stop_lat_prev", "stop_lon_prev", + "calc shape_dist_traveled") # make it cumulative - self.stop_times_df.loc[ (self.stop_times_df["null_shape_dist_traveled_bool"])& - (self.stop_times_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]==1), "calc shape_dist_traveled" ] = 0.0 - self.stop_times_df["calc shape_dist_traveled"] = self.stop_times_df.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID])["calc shape_dist_traveled"].apply(lambda x: x.cumsum()) + self.stop_times_df.loc[(self.stop_times_df["null_shape_dist_traveled_bool"]) & + (self.stop_times_df[ + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] == 1), "calc shape_dist_traveled"] = 0.0 + self.stop_times_df["calc shape_dist_traveled"] = self.stop_times_df.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID])[ + "calc shape_dist_traveled"].apply(lambda x: x.cumsum()) # incorporate it - self.stop_times_df.loc[ (self.stop_times_df["null_shape_dist_traveled_bool"]), Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED] = self.stop_times_df["calc shape_dist_traveled"] + self.stop_times_df.loc[ + (self.stop_times_df["null_shape_dist_traveled_bool"]), Trip.STOPTIMES_COLUMN_SHAPE_DIST_TRAVELED] = \ + self.stop_times_df["calc shape_dist_traveled"] FastTripsLogger.debug("add_shape_dist_traveled() stop_times_df\n%s" % str(self.stop_times_df.head())) @@ -541,26 +556,26 @@ def get_full_trips(self): Returns the fullest dataframe of trip + stop information. """ # join with trips to get additional fields - df = pd.merge(left = self.stop_times_df, - right= self.trips_df, - how = 'left', - on =[Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_TRIP_ID_NUM]) - assert(len(self.stop_times_df) == len(df)) + df = pd.merge(left=self.stop_times_df, + right=self.trips_df, + how='left', + on=[Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_TRIP_ID_NUM]) + assert (len(self.stop_times_df) == len(df)) # blank boards, alights and onboard - df[Trip.SIM_COL_VEH_BOARDS ] = 0 - df[Trip.SIM_COL_VEH_ALIGHTS ] = 0 - df[Trip.SIM_COL_VEH_ONBOARD ] = 0 - df[Trip.SIM_COL_VEH_FRICTION ] = 0 - df[Trip.SIM_COL_VEH_STANDEES ] = 0 - df[Trip.SIM_COL_VEH_OVERCAP ] = -1 # assume there's room - - df[Trip.SIM_COL_VEH_MSA_BOARDS ] = 0.0 - df[Trip.SIM_COL_VEH_MSA_ALIGHTS ] = 0.0 - df[Trip.SIM_COL_VEH_MSA_ONBOARD ] = 0.0 + df[Trip.SIM_COL_VEH_BOARDS] = 0 + df[Trip.SIM_COL_VEH_ALIGHTS] = 0 + df[Trip.SIM_COL_VEH_ONBOARD] = 0 + df[Trip.SIM_COL_VEH_FRICTION] = 0 + df[Trip.SIM_COL_VEH_STANDEES] = 0 + df[Trip.SIM_COL_VEH_OVERCAP] = -1 # assume there's room + + df[Trip.SIM_COL_VEH_MSA_BOARDS] = 0.0 + df[Trip.SIM_COL_VEH_MSA_ALIGHTS] = 0.0 + df[Trip.SIM_COL_VEH_MSA_ONBOARD] = 0.0 df[Trip.SIM_COL_VEH_MSA_FRICTION] = 0.0 df[Trip.SIM_COL_VEH_MSA_STANDEES] = 0.0 - df[Trip.SIM_COL_VEH_MSA_OVERCAP ] =-1.0 # assume there's room + df[Trip.SIM_COL_VEH_MSA_OVERCAP] = -1.0 # assume there's room return df def write_trips_for_extension(self): @@ -571,20 +586,20 @@ def write_trips_for_extension(self): trips_df = self.trips_df.copy() # drop some of the attributes - drop_fields = [Trip.TRIPS_COLUMN_TRIP_ID, # use numerical version - Trip.TRIPS_COLUMN_ROUTE_ID, # use numerical version - Trip.TRIPS_COLUMN_SERVICE_ID, # I don't think this is useful - Trip.TRIPS_COLUMN_DIRECTION_ID, # I don't think this is useful - Trip.TRIPS_COLUMN_VEHICLE_NAME, # could pass numerical version - Trip.TRIPS_COLUMN_SHAPE_ID, # I don't think this is useful - Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE, # I don't think this is useful - Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME, # I don't think this is useful - Route.ROUTES_COLUMN_MODE_TYPE, # I don't think this is useful -- should be transit + drop_fields = [Trip.TRIPS_COLUMN_TRIP_ID, # use numerical version + Trip.TRIPS_COLUMN_ROUTE_ID, # use numerical version + Trip.TRIPS_COLUMN_SERVICE_ID, # I don't think this is useful + Trip.TRIPS_COLUMN_DIRECTION_ID, # I don't think this is useful + Trip.TRIPS_COLUMN_VEHICLE_NAME, # could pass numerical version + Trip.TRIPS_COLUMN_SHAPE_ID, # I don't think this is useful + Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE, # I don't think this is useful + Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME, # I don't think this is useful + Route.ROUTES_COLUMN_MODE_TYPE, # I don't think this is useful -- should be transit Route.ROUTES_COLUMN_ROUTE_SHORT_NAME, # I don't think this is useful - Route.ROUTES_COLUMN_ROUTE_LONG_NAME, # I don't think this is useful - Route.ROUTES_COLUMN_ROUTE_TYPE, # I don't think this is useful - Route.ROUTES_COLUMN_MODE, # use numerical version - Route.FARE_ATTR_COLUMN_FARE_PERIOD, # text + Route.ROUTES_COLUMN_ROUTE_LONG_NAME, # I don't think this is useful + Route.ROUTES_COLUMN_ROUTE_TYPE, # I don't think this is useful + Route.ROUTES_COLUMN_MODE, # use numerical version + Route.FARE_ATTR_COLUMN_FARE_PERIOD, # text Route.ROUTES_COLUMN_PROOF_OF_PAYMENT, # text ] # we can only drop fields that are in the dataframe @@ -598,14 +613,14 @@ def write_trips_for_extension(self): # only pass on numeric columns -- for now, drop the rest FastTripsLogger.debug("Dropping non-numeric trip info\n" + str(trips_df.head())) trips_df = trips_df.select_dtypes(exclude=['object']) - FastTripsLogger.debug("\n"+str(trips_df.head())) + FastTripsLogger.debug("\n" + str(trips_df.head())) # the index is the trip_id_num trips_df.set_index(Trip.TRIPS_COLUMN_TRIP_ID_NUM, inplace=True) # this will make it so beyond trip id num # the remaining columns collapse to variable name, variable value trips_df = trips_df.stack().reset_index() - trips_df.rename(columns={"level_1":"attr_name", 0:"attr_value"}, inplace=True) + trips_df.rename(columns={"level_1": "attr_name", 0: "attr_value"}, inplace=True) trips_df.to_csv(os.path.join(self.output_dir, Trip.OUTPUT_TRIPINFO_FILE), sep=" ", index=False) @@ -616,19 +631,19 @@ def reset_onboard(df): """ Resets the onboard fields for the given vehicle trip table. """ - df[Trip.SIM_COL_VEH_BOARDS ] = 0 - df[Trip.SIM_COL_VEH_ALIGHTS ] = 0 - df[Trip.SIM_COL_VEH_ONBOARD ] = 0 - df[Trip.SIM_COL_VEH_FRICTION ] = 0 - df[Trip.SIM_COL_VEH_STANDEES ] = 0 - df[Trip.SIM_COL_VEH_OVERCAP ] = -1 # assume there's room - - df[Trip.SIM_COL_VEH_MSA_BOARDS ] = 0.0 - df[Trip.SIM_COL_VEH_MSA_ALIGHTS ] = 0.0 - df[Trip.SIM_COL_VEH_MSA_ONBOARD ] = 0.0 + df[Trip.SIM_COL_VEH_BOARDS] = 0 + df[Trip.SIM_COL_VEH_ALIGHTS] = 0 + df[Trip.SIM_COL_VEH_ONBOARD] = 0 + df[Trip.SIM_COL_VEH_FRICTION] = 0 + df[Trip.SIM_COL_VEH_STANDEES] = 0 + df[Trip.SIM_COL_VEH_OVERCAP] = -1 # assume there's room + + df[Trip.SIM_COL_VEH_MSA_BOARDS] = 0.0 + df[Trip.SIM_COL_VEH_MSA_ALIGHTS] = 0.0 + df[Trip.SIM_COL_VEH_MSA_ONBOARD] = 0.0 df[Trip.SIM_COL_VEH_MSA_FRICTION] = 0.0 df[Trip.SIM_COL_VEH_MSA_STANDEES] = 0.0 - df[Trip.SIM_COL_VEH_MSA_OVERCAP ] =-1.0 # assume there's room + df[Trip.SIM_COL_VEH_MSA_OVERCAP] = -1.0 # assume there's room @staticmethod def update_trip_times(trips_df, MSA_RESULTS): @@ -658,34 +673,44 @@ def update_trip_times(trips_df, MSA_RESULTS): trip_cols = list(trips_df.columns.values) # Default to 0 - trips_df[Trip.SIM_COL_VEH_FRICTION ] = 0.0 + trips_df[Trip.SIM_COL_VEH_FRICTION] = 0.0 trips_df[Trip.SIM_COL_VEH_MSA_FRICTION] = 0.0 if Trip.VEHICLES_COLUMN_SEATED_CAPACITY in trip_cols: # log null seated capacities if pd.isnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY]).sum() > 0: - FastTripsLogger.warn("Trip.update_trip_times(): some [%s] not configured; assuming zero friction for those vehicles" % Trip.VEHICLES_COLUMN_SEATED_CAPACITY) - FastTripsLogger.warn("\n%s" % trips_df[[Trip.VEHICLES_COLUMN_VEHICLE_NAME, Trip.VEHICLES_COLUMN_SEATED_CAPACITY]].loc[pd.isnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])].drop_duplicates()) + FastTripsLogger.warn( + "Trip.update_trip_times(): some [%s] not configured; assuming zero friction for those vehicles" % Trip.VEHICLES_COLUMN_SEATED_CAPACITY) + FastTripsLogger.warn( + "\n%s" % trips_df[[Trip.VEHICLES_COLUMN_VEHICLE_NAME, Trip.VEHICLES_COLUMN_SEATED_CAPACITY]].loc[ + pd.isnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])].drop_duplicates()) # set standeeds - trips_df[Trip.SIM_COL_VEH_STANDEES ] = trips_df[Trip.SIM_COL_VEH_ONBOARD ] - trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY] - trips_df[Trip.SIM_COL_VEH_MSA_STANDEES] = trips_df[Trip.SIM_COL_VEH_MSA_ONBOARD] - trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY] + trips_df[Trip.SIM_COL_VEH_STANDEES] = trips_df[Trip.SIM_COL_VEH_ONBOARD] - trips_df[ + Trip.VEHICLES_COLUMN_SEATED_CAPACITY] + trips_df[Trip.SIM_COL_VEH_MSA_STANDEES] = trips_df[Trip.SIM_COL_VEH_MSA_ONBOARD] - trips_df[ + Trip.VEHICLES_COLUMN_SEATED_CAPACITY] # it can only be non-negative - trips_df.loc[trips_df[Trip.SIM_COL_VEH_STANDEES ]<0, Trip.SIM_COL_VEH_STANDEES ] = 0 - trips_df.loc[trips_df[Trip.SIM_COL_VEH_MSA_STANDEES]<0, Trip.SIM_COL_VEH_MSA_STANDEES] = 0 + trips_df.loc[trips_df[Trip.SIM_COL_VEH_STANDEES] < 0, Trip.SIM_COL_VEH_STANDEES] = 0 + trips_df.loc[trips_df[Trip.SIM_COL_VEH_MSA_STANDEES] < 0, Trip.SIM_COL_VEH_MSA_STANDEES] = 0 # where it is positive, friction = on+off+standees - trips_df.loc[(trips_df[Trip.SIM_COL_VEH_STANDEES ]>0)&(pd.notnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])), Trip.SIM_COL_VEH_FRICTION ] = \ - trips_df[Trip.SIM_COL_VEH_BOARDS ] + trips_df[Trip.SIM_COL_VEH_ALIGHTS ] + trips_df[Trip.SIM_COL_VEH_STANDEES ] - trips_df.loc[(trips_df[Trip.SIM_COL_VEH_MSA_STANDEES]>0)&(pd.notnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])), Trip.SIM_COL_VEH_MSA_FRICTION] = \ - trips_df[Trip.SIM_COL_VEH_MSA_BOARDS] + trips_df[Trip.SIM_COL_VEH_MSA_ALIGHTS] + trips_df[Trip.SIM_COL_VEH_MSA_STANDEES] + trips_df.loc[(trips_df[Trip.SIM_COL_VEH_STANDEES] > 0) & ( + pd.notnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])), Trip.SIM_COL_VEH_FRICTION] = \ + trips_df[Trip.SIM_COL_VEH_BOARDS] + trips_df[Trip.SIM_COL_VEH_ALIGHTS] + trips_df[ + Trip.SIM_COL_VEH_STANDEES] + trips_df.loc[(trips_df[Trip.SIM_COL_VEH_MSA_STANDEES] > 0) & ( + pd.notnull(trips_df[Trip.VEHICLES_COLUMN_SEATED_CAPACITY])), Trip.SIM_COL_VEH_MSA_FRICTION] = \ + trips_df[Trip.SIM_COL_VEH_MSA_BOARDS] + trips_df[Trip.SIM_COL_VEH_MSA_ALIGHTS] + trips_df[ + Trip.SIM_COL_VEH_MSA_STANDEES] else: # log no seated capacities at all - FastTripsLogger.warn("Trip.update_trip_times(): Cannot calculate friction because [%s] not configured" % Trip.VEHICLES_COLUMN_SEATED_CAPACITY) + FastTripsLogger.warn( + "Trip.update_trip_times(): Cannot calculate friction because [%s] not configured" % Trip.VEHICLES_COLUMN_SEATED_CAPACITY) # Update the dwell time if Trip.VEHICLES_COLUMN_DWELL_FORMULA in trip_cols: - all_dwell_df = None + all_dwell_df = None all_dwell_init = False # grouppy unique dwell time forumulas dwell_groups = trips_df.groupby(Trip.VEHICLES_COLUMN_DWELL_FORMULA) @@ -693,14 +718,14 @@ def update_trip_times(trips_df, MSA_RESULTS): dwell_df = dwell_group.copy() FastTripsLogger.debug("dwell_formula %s has %d rows" % (str(dwell_formula), len(dwell_df))) - if isinstance(dwell_formula,str): + if isinstance(dwell_formula, str): if MSA_RESULTS: # replace [boards], [alights], etc with trip_df['msa_boards'], trip_df['msa_alights'], etc - dwell_formula = dwell_formula.replace("[","dwell_df['msa_") + dwell_formula = dwell_formula.replace("[", "dwell_df['msa_") else: # replace [boards], [alights], etc with trip_df['boards'], trip_df['alights'], etc - dwell_formula = dwell_formula.replace("[","dwell_df['") - dwell_formula = dwell_formula.replace("]","']") + dwell_formula = dwell_formula.replace("[", "dwell_df['") + dwell_formula = dwell_formula.replace("]", "']") # eval it dwell_df[Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC] = eval(dwell_formula) @@ -718,24 +743,26 @@ def update_trip_times(trips_df, MSA_RESULTS): trips_df = all_dwell_df # keep the dwell time - trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] = trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC].map(lambda x: datetime.timedelta(seconds=x)) + trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] = trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC].map( + lambda x: datetime.timedelta(seconds=x)) # the vehicle stops if someone boards or someone alights or both - trips_df["does_stop"] = (trips_df[Trip.SIM_COL_VEH_BOARDS]>0) | (trips_df[Trip.SIM_COL_VEH_ALIGHTS]>0) + trips_df["does_stop"] = (trips_df[Trip.SIM_COL_VEH_BOARDS] > 0) | (trips_df[Trip.SIM_COL_VEH_ALIGHTS] > 0) # we need information about the next stop next_stop_df = trips_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE, "does_stop"]].copy() - next_stop_df.loc[:,"is_last_stop"] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] == next_stop_df[Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE] - next_stop_df.loc[:,Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]-1 - next_stop_df.rename(columns={"does_stop" :"next_does_stop", - "is_last_stop" :"next_is_last_stop"}, inplace=True) + next_stop_df.loc[:, "is_last_stop"] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] == next_stop_df[ + Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE] + next_stop_df.loc[:, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] = next_stop_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] - 1 + next_stop_df.rename(columns={"does_stop": "next_does_stop", + "is_last_stop": "next_is_last_stop"}, inplace=True) FastTripsLogger.debug("next_stop_df:\n%s\n" % next_stop_df.head().to_string()) trips_df = pd.merge(left=trips_df, right=next_stop_df, how='left') - assert(trips_df_len==len(trips_df)) + assert (trips_df_len == len(trips_df)) # Start with original travel time for the link FROM this stop to the NEXT stop trip_cols = list(trips_df.columns.values) @@ -744,34 +771,45 @@ def update_trip_times(trips_df, MSA_RESULTS): # Skip first stop because we assume it's already there, and last since we don't go anywhere trips_df["accel_secs"] = 0.0 if (Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS in trip_cols) and \ - (Trip.VEHICLES_COLUMN_ACCELERATION in trip_cols): + (Trip.VEHICLES_COLUMN_ACCELERATION in trip_cols): trips_df.loc[trips_df["does_stop"] & \ (trips_df[Trip.VEHICLES_COLUMN_ACCELERATION] > 0) & - (trips_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]>1) & \ - (trips_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] 1) & \ + (trips_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] < trips_df[ + Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE]), "accel_secs"] = \ + trips_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS] / trips_df[Trip.VEHICLES_COLUMN_ACCELERATION] # Add deceleration to next stop. # Skip stop with next stop = last stop because we assume it's already there trips_df["decel_secs"] = 0.0 if (Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS in trip_cols) and \ - (Trip.VEHICLES_COLUMN_DECELERATION in trip_cols): + (Trip.VEHICLES_COLUMN_DECELERATION in trip_cols): trips_df.loc[(trips_df["next_does_stop"]) & \ (trips_df[Trip.VEHICLES_COLUMN_DECELERATION] > 0) & - (trips_df["next_is_last_stop"]==False), "decel_secs"] = \ - trips_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS]/trips_df[Trip.VEHICLES_COLUMN_DECELERATION] + (trips_df["next_is_last_stop"] == False), "decel_secs"] = \ + trips_df[Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS] / trips_df[Trip.VEHICLES_COLUMN_DECELERATION] # update the travel time - trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME]/np.timedelta64(1, 's') - trips_df.loc[ pd.notnull(trips_df["accel_secs"]), Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + trips_df["accel_secs"] - trips_df.loc[ pd.notnull(trips_df["decel_secs"]), Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + trips_df["decel_secs"] - - trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME ] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC].map(lambda x: datetime.timedelta(seconds=x)) + trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[ + Trip.STOPTIMES_COLUMN_ORIGINAL_TRAVEL_TIME] / np.timedelta64( + 1, 's') + trips_df.loc[pd.notnull(trips_df["accel_secs"]), Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[ + Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + \ + trips_df["accel_secs"] + trips_df.loc[pd.notnull(trips_df["decel_secs"]), Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] = trips_df[ + Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + \ + trips_df["decel_secs"] + + trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC].map( + lambda x: datetime.timedelta(seconds=x)) # put travel time + dwell together because that's the full time for a link (stop arrival time to next stop arrival time) - trips_df["travel_dwell_sec"] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC] + trips_df["travel_dwell_sec"] = trips_df[Trip.STOPTIMES_COLUMN_TRAVEL_TIME_SEC] + trips_df[ + Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC] # cumulatively sum it to get arrival times times for the trip - trips_df["travel_dwell_sec_cum"] = trips_df.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID_NUM])["travel_dwell_sec"].cumsum() - trips_df["travel_dwell_cum" ] = trips_df["travel_dwell_sec_cum"].map(lambda x: datetime.timedelta(seconds=x) if pd.notnull(x) else None) + trips_df["travel_dwell_sec_cum"] = trips_df.groupby([Trip.STOPTIMES_COLUMN_TRIP_ID_NUM])[ + "travel_dwell_sec"].cumsum() + trips_df["travel_dwell_cum"] = trips_df["travel_dwell_sec_cum"].map( + lambda x: datetime.timedelta(seconds=x) if pd.notnull(x) else None) # verifying cumsum did as expected # FastTripsLogger.debug("\n"+ trips_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, "travel_dwell_sec","travel_dwell_sec_cum"]].to_string()) @@ -781,10 +819,12 @@ def update_trip_times(trips_df, MSA_RESULTS): Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME, "travel_dwell_cum"]].copy() # need to start from trip arrival time. For some reason can't aggregate STOPTIMES_COLUMN_DWELL_TIME, only the seconds version - first_dwell_df = trips_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM,Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC]]. \ - groupby([Trip.STOPTIMES_COLUMN_TRIP_ID_NUM]).agg({Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC:'first'}).reset_index() - first_dwell_df.rename(columns={Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC:"trip_first_dwell_sec"}, inplace=True) - first_dwell_df["trip_first_dwell"] = first_dwell_df["trip_first_dwell_sec"].map(lambda x: datetime.timedelta(seconds=x)) + first_dwell_df = trips_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC]]. \ + groupby([Trip.STOPTIMES_COLUMN_TRIP_ID_NUM]).agg( + {Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC: 'first'}).reset_index() + first_dwell_df.rename(columns={Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC: "trip_first_dwell_sec"}, inplace=True) + first_dwell_df["trip_first_dwell"] = first_dwell_df["trip_first_dwell_sec"].map( + lambda x: datetime.timedelta(seconds=x)) # verify first dwell is correct # FastTripsLogger.debug("first_dwell:\n%s\n" % first_dwell_df.head().to_string()) @@ -792,51 +832,70 @@ def update_trip_times(trips_df, MSA_RESULTS): FastTripsLogger.debug("next_stop_df:\n%s\n" % next_stop_df.head().to_string()) - next_stop_df["trip_arrival_time" ] = next_stop_df[Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME] - next_stop_df["trip_first_dwell"] - next_stop_df["new_arrival_time" ] = next_stop_df["trip_arrival_time"] + trips_df["travel_dwell_cum"] - next_stop_df.loc[:,Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] += 1 + next_stop_df["trip_arrival_time"] = next_stop_df[Trip.TRIPS_COLUMN_TRIP_DEPARTURE_TIME] - next_stop_df[ + "trip_first_dwell"] + next_stop_df["new_arrival_time"] = next_stop_df["trip_arrival_time"] + trips_df["travel_dwell_cum"] + next_stop_df.loc[:, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] += 1 FastTripsLogger.debug("next_stop_df:\n%s\n" % next_stop_df.head().to_string()) trips_df = pd.merge(left=trips_df, - right=next_stop_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - "new_arrival_time"]], - how="left") + right=next_stop_df[[Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, + "new_arrival_time"]], + how="left") # the first ones will be NaT but that's perfect -- we don't want to set those anyway - trips_df.loc[pd.notnull(trips_df["new_arrival_time"]),Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = trips_df["new_arrival_time"] + trips_df.loc[pd.notnull(trips_df["new_arrival_time"]), Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = trips_df[ + "new_arrival_time"] # set the first ones to be departure time minus dwell time - trips_df.loc[pd.isnull( trips_df["new_arrival_time"]),Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = trips_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] + trips_df.loc[pd.isnull(trips_df["new_arrival_time"]), Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] = trips_df[ + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - \ + trips_df[ + Trip.STOPTIMES_COLUMN_DWELL_TIME] # departure time is arrival time + dwell - trips_df.loc[:,Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] = trips_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] + trips_df[Trip.STOPTIMES_COLUMN_DWELL_TIME] + trips_df.loc[:, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] = trips_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME] + trips_df[ + Trip.STOPTIMES_COLUMN_DWELL_TIME] # float version trips_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME_MIN] = \ trips_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME].map(lambda x: \ - 60*x.time().hour + x.time().minute + (x.time().second / 60.0) ) + 60 * x.time().hour + x.time().minute + ( + x.time().second / 60.0)) trips_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME_MIN] = \ trips_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME].map(lambda x: \ - 60*x.time().hour + x.time().minute + (x.time().second / 60.0) ) + 60 * x.time().hour + x.time().minute + ( + x.time().second / 60.0)) FastTripsLogger.debug("Trips:update_trip_times() trips_df:\n%s\n" % \ - trips_df.loc[trips_df[Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE]>1,[Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - Trip.STOPTIMES_COLUMN_ARRIVAL_TIME, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME, - Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS, Trip.VEHICLES_COLUMN_ACCELERATION, Trip.VEHICLES_COLUMN_DECELERATION, - Trip. VEHICLES_COLUMN_SEATED_CAPACITY, - Trip.SIM_COL_VEH_BOARDS, Trip.SIM_COL_VEH_ALIGHTS, Trip.SIM_COL_VEH_ONBOARD, Trip.SIM_COL_VEH_STANDEES, Trip.SIM_COL_VEH_FRICTION, - "does_stop","next_does_stop","next_is_last_stop", - "accel_secs","decel_secs", - Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC, - "travel_dwell_sec","travel_dwell_sec_cum","new_arrival_time" - ]].head(15).to_string()) - - - assert(trips_df_len==len(trips_df)) + trips_df.loc[ + trips_df[Trip.TRIPS_COLUMN_MAX_STOP_SEQUENCE] > 1, [Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_TRIP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, + Trip.STOPTIMES_COLUMN_ARRIVAL_TIME, + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME, + Trip.VEHICLES_COLUMN_MAXIMUM_SPEED_FPS, + Trip.VEHICLES_COLUMN_ACCELERATION, + Trip.VEHICLES_COLUMN_DECELERATION, + Trip.VEHICLES_COLUMN_SEATED_CAPACITY, + Trip.SIM_COL_VEH_BOARDS, + Trip.SIM_COL_VEH_ALIGHTS, + Trip.SIM_COL_VEH_ONBOARD, + Trip.SIM_COL_VEH_STANDEES, + Trip.SIM_COL_VEH_FRICTION, + "does_stop", "next_does_stop", + "next_is_last_stop", + "accel_secs", "decel_secs", + Trip.STOPTIMES_COLUMN_DWELL_TIME_SEC, + "travel_dwell_sec", + "travel_dwell_sec_cum", + "new_arrival_time" + ]].head(15).to_string()) + + assert (trips_df_len == len(trips_df)) # drop all the intermediate columns - trips_df.drop(["does_stop","next_does_stop","next_is_last_stop", - "accel_secs","decel_secs", - "travel_dwell_sec","travel_dwell_sec_cum","travel_dwell_cum", - "new_arrival_time"], axis=1, inplace=True) + trips_df.drop(["does_stop", "next_does_stop", "next_is_last_stop", + "accel_secs", "decel_secs", + "travel_dwell_sec", "travel_dwell_sec_cum", "travel_dwell_cum", + "new_arrival_time"], axis=1, inplace=True) FastTripsLogger.debug("trips_df.dtypes=\n%s\n" % str(trips_df.dtypes)) return trips_df @@ -868,30 +927,32 @@ def linkify_vehicle_trips(veh_trips_df, stops): Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, Trip.STOPTIMES_COLUMN_ARRIVAL_TIME, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME]].copy() - veh_temp_df = stops.add_stop_lat_lon(veh_temp_df, id_colname=Trip.STOPTIMES_COLUMN_STOP_ID, new_lat_colname="lat", new_lon_colname="lon") + veh_temp_df = stops.add_stop_lat_lon(veh_temp_df, id_colname=Trip.STOPTIMES_COLUMN_STOP_ID, + new_lat_colname="lat", new_lon_colname="lon") veh_temp_df["next_stop_seq"] = veh_temp_df[Trip.STOPTIMES_COLUMN_STOP_SEQUENCE] + 1 # merge with next stop - veh_temp_df = pd.merge(left = veh_temp_df, - left_on = [Route.ROUTES_COLUMN_MODE, Trip.TRIPS_COLUMN_MODE_NUM, - Trip.TRIPS_COLUMN_ROUTE_ID, Trip.TRIPS_COLUMN_ROUTE_ID_NUM, - Trip.TRIPS_COLUMN_TRIP_ID,Trip.TRIPS_COLUMN_TRIP_ID_NUM, "next_stop_seq"], - right = veh_temp_df, - right_on = [Route.ROUTES_COLUMN_MODE, Trip.TRIPS_COLUMN_MODE_NUM, - Trip.TRIPS_COLUMN_ROUTE_ID, Trip.TRIPS_COLUMN_ROUTE_ID_NUM, - Trip.TRIPS_COLUMN_TRIP_ID,Trip.TRIPS_COLUMN_TRIP_ID_NUM, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], - how = "left", - suffixes = ["_A","_B"]) + veh_temp_df = pd.merge(left=veh_temp_df, + left_on=[Route.ROUTES_COLUMN_MODE, Trip.TRIPS_COLUMN_MODE_NUM, + Trip.TRIPS_COLUMN_ROUTE_ID, Trip.TRIPS_COLUMN_ROUTE_ID_NUM, + Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_TRIP_ID_NUM, "next_stop_seq"], + right=veh_temp_df, + right_on=[Route.ROUTES_COLUMN_MODE, Trip.TRIPS_COLUMN_MODE_NUM, + Trip.TRIPS_COLUMN_ROUTE_ID, Trip.TRIPS_COLUMN_ROUTE_ID_NUM, + Trip.TRIPS_COLUMN_TRIP_ID, Trip.TRIPS_COLUMN_TRIP_ID_NUM, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE], + how="left", + suffixes=["_A", "_B"]) # drop the ones that have NA - those are last links - veh_temp_df = veh_temp_df.loc[ pd.notnull(veh_temp_df["stop_id_B"])] + veh_temp_df = veh_temp_df.loc[pd.notnull(veh_temp_df["stop_id_B"])] # rename -- maybe our convention should have put A and B as suffixes, eh? - rename_cols = {"%s_A" % Trip.STOPTIMES_COLUMN_STOP_SEQUENCE:"A_seq", - "%s_B" % Trip.STOPTIMES_COLUMN_STOP_SEQUENCE:"B_seq", - "%s_A" % Trip.STOPTIMES_COLUMN_STOP_ID:"A_id", - "%s_B" % Trip.STOPTIMES_COLUMN_STOP_ID:"B_id", - "stop_id_num_A":"A_id_num", - "stop_id_num_B":"B_id_num"} + rename_cols = {"%s_A" % Trip.STOPTIMES_COLUMN_STOP_SEQUENCE: "A_seq", + "%s_B" % Trip.STOPTIMES_COLUMN_STOP_SEQUENCE: "B_seq", + "%s_A" % Trip.STOPTIMES_COLUMN_STOP_ID: "A_id", + "%s_B" % Trip.STOPTIMES_COLUMN_STOP_ID: "B_id", + "stop_id_num_A": "A_id_num", + "stop_id_num_B": "B_id_num"} for colname in ["lat", "lon", Trip.STOPTIMES_COLUMN_ARRIVAL_TIME, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME]: @@ -903,7 +964,7 @@ def linkify_vehicle_trips(veh_trips_df, stops): veh_temp_df.drop(["next_stop_seq_A", "next_stop_seq_B"], axis=1, inplace=True) # B_seq, B_id_num are ints. they shouldn't be null - veh_temp_df["B_seq"] = veh_temp_df["B_seq"].astype(int) + veh_temp_df["B_seq"] = veh_temp_df["B_seq"].astype(int) veh_temp_df["B_id_num"] = veh_temp_df["B_id_num"].astype(int) # FastTripsLogger.debug("linkify_vehicle_trips: veh_temp_df (%d)\n%s\n%s" % (len(veh_temp_df), veh_temp_df.head(30), str(veh_temp_df.dtypes))) @@ -928,38 +989,41 @@ def calculate_headways(trips_df): Trip.STOPTIMES_COLUMN_DEPARTURE_TIME, Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]].groupby([Trip.STOPTIMES_COLUMN_STOP_ID, - Trip.TRIPS_COLUMN_ROUTE_ID, - Trip.TRIPS_COLUMN_DIRECTION_ID]) + Trip.TRIPS_COLUMN_ROUTE_ID, + Trip.TRIPS_COLUMN_DIRECTION_ID]) else: stop_group = trips_df[[Trip.STOPTIMES_COLUMN_STOP_ID, Trip.TRIPS_COLUMN_ROUTE_ID, Trip.STOPTIMES_COLUMN_DEPARTURE_TIME, Trip.STOPTIMES_COLUMN_TRIP_ID, Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]].groupby([Trip.STOPTIMES_COLUMN_STOP_ID, - Trip.TRIPS_COLUMN_ROUTE_ID]) + Trip.TRIPS_COLUMN_ROUTE_ID]) stop_group_df = stop_group.apply(lambda x: x.sort_values(Trip.STOPTIMES_COLUMN_DEPARTURE_TIME)) # set headway, in minutes stop_group_shift_df = stop_group_df.shift() - stop_group_df['headway'] = (stop_group_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - stop_group_shift_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME])/np.timedelta64(1,'m') + stop_group_df['headway'] = (stop_group_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - stop_group_shift_df[ + Trip.STOPTIMES_COLUMN_DEPARTURE_TIME]) / np.timedelta64(1, 'm') # zero out the first in each group if has_direction_id: - stop_group_df.loc[(stop_group_df.stop_id !=stop_group_shift_df.stop_id )| - (stop_group_df.route_id !=stop_group_shift_df.route_id )| - (stop_group_df.direction_id!=stop_group_shift_df.direction_id), 'headway'] = Trip.DEFAULT_HEADWAY + stop_group_df.loc[(stop_group_df.stop_id != stop_group_shift_df.stop_id) | + (stop_group_df.route_id != stop_group_shift_df.route_id) | + ( + stop_group_df.direction_id != stop_group_shift_df.direction_id), 'headway'] = Trip.DEFAULT_HEADWAY else: - stop_group_df.loc[(stop_group_df.stop_id !=stop_group_shift_df.stop_id )| - (stop_group_df.route_id !=stop_group_shift_df.route_id ), 'headway'] = Trip.DEFAULT_HEADWAY + stop_group_df.loc[(stop_group_df.stop_id != stop_group_shift_df.stop_id) | + ( + stop_group_df.route_id != stop_group_shift_df.route_id), 'headway'] = Trip.DEFAULT_HEADWAY # print stop_group_df trips_df_len = len(trips_df) - trips_df = pd.merge(left = trips_df, - right = stop_group_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, - Trip.STOPTIMES_COLUMN_STOP_ID, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, - 'headway']], - on = [Trip.STOPTIMES_COLUMN_TRIP_ID, - Trip.STOPTIMES_COLUMN_STOP_ID, - Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]) - assert(len(trips_df)==trips_df_len) + trips_df = pd.merge(left=trips_df, + right=stop_group_df[[Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_STOP_ID, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE, + 'headway']], + on=[Trip.STOPTIMES_COLUMN_TRIP_ID, + Trip.STOPTIMES_COLUMN_STOP_ID, + Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]) + assert (len(trips_df) == trips_df_len) return trips_df diff --git a/fasttrips/Util.py b/fasttrips/Util.py index 3495fe02..d93e34df 100644 --- a/fasttrips/Util.py +++ b/fasttrips/Util.py @@ -6,7 +6,7 @@ from builtins import object __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -19,6 +19,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import csv import datetime import logging @@ -28,7 +29,7 @@ import pandas as pd import partridge as ptg -from .Error import UnexpectedError +from .Error import UnexpectedError from .Logger import FastTripsLogger @@ -39,34 +40,35 @@ class Util(object): Collect useful stuff here that doesn't belong in any particular existing class. """ #: Use this as the date - #SIMULATION_DAY = datetime.datetime(year=2016,day=1,month=1, hour=0, minute=0, second=0) + # SIMULATION_DAY = datetime.datetime(year=2016,day=1,month=1, hour=0, minute=0, second=0) #: Use this for the start time - the start of :py:attr:`Util.SIMULATION_DAY` - #SIMULATION_DAY_START = datetime.datetime.combine(SIMULATION_DAY, datetime.time()) + # SIMULATION_DAY_START = datetime.datetime.combine(SIMULATION_DAY, datetime.time()) #: Maps timedelta columns to units for :py:meth:`Util.write_dataframe` - TIMEDELTA_COLUMNS_TO_UNITS = { - 'time enumerating' : 'milliseconds', # performance - 'time labeling' : 'milliseconds', # performance - 'step_duration' : 'seconds', # performance - 'pf_linktime' : 'min', - 'pf_linkcost' : 'min', - 'pf_waittime' : 'min', - 'new_linktime' : 'min', - 'new_waittime' : 'min' + TIMEDELTA_COLUMNS_TO_UNITS = { + 'time enumerating': 'milliseconds', # performance + 'time labeling': 'milliseconds', # performance + 'step_duration': 'seconds', # performance + 'pf_linktime': 'min', + 'pf_linkcost': 'min', + 'pf_waittime': 'min', + 'new_linktime': 'min', + 'new_waittime': 'min' } #: Debug columns to drop DROP_DEBUG_COLUMNS = [ # drop these? - "A_lat","A_lon","B_lat","B_lon",# "distance", + "A_lat", "A_lon", "B_lat", "B_lon", # "distance", # numeric versions of other columns - "trip_list_id_num","trip_id_num","A_id_num","B_id_num","mode_num", + "trip_list_id_num", "trip_id_num", "A_id_num", "B_id_num", "mode_num", # simulation debugging - "bump_iter","bumpstop_boarded","alight_delay_min" + "bump_iter", "bumpstop_boarded", "alight_delay_min" ] DROP_PATHFINDING_COLUMNS = [ # pathfinding debugging - "pf_iteration","pf_A_time","pf_B_time","pf_linktime","pf_linkcost","pf_linkdist","pf_waittime","pf_linkfare","pf_cost","pf_fare","pf_initcost","pf_initfare" + "pf_iteration", "pf_A_time", "pf_B_time", "pf_linktime", "pf_linkcost", "pf_linkdist", "pf_waittime", + "pf_linkfare", "pf_cost", "pf_fare", "pf_initcost", "pf_initfare" ] @staticmethod @@ -77,7 +79,7 @@ def add_numeric_column(input_df, id_colname, numeric_newcolname): Returns the dataframe with the new column. """ - assert(len(input_df.columns) == 1) + assert (len(input_df.columns) == 1) # drop duplicates - this is an ID and drop the index since it's not useful return_df = input_df.drop_duplicates().reset_index(drop=True) @@ -91,9 +93,9 @@ def add_numeric_column(input_df, id_colname, numeric_newcolname): return return_df @staticmethod - def add_new_id(input_df, id_colname, newid_colname, + def add_new_id(input_df, id_colname, newid_colname, mapping_df, mapping_id_colname, mapping_newid_colname, - warn=False, warn_msg=None, drop_failures=True): + warn=False, warn_msg=None, drop_failures=True): """ Passing a :py:class:`pandas.DataFrame` *input_df* with an ID column called *id_colname*, adds the numeric id as a column named *newid_colname* and returns it. @@ -106,10 +108,10 @@ def add_new_id(input_df, id_colname, newid_colname, input_cols = list(input_df.columns.values) # add the new id column return_df = pd.merge(left=input_df, right=mapping_df, - how='left', - left_on=id_colname, - right_on=mapping_id_colname, - suffixes=("","_mapping")) + how='left', + left_on=id_colname, + right_on=mapping_id_colname, + suffixes=("", "_mapping")) # print "RETURN_DF==================" # print return_df.head() @@ -124,12 +126,14 @@ def add_new_id(input_df, id_colname, newid_colname, if warn: msg_level = logging.WARN if warn_msg: FastTripsLogger.log(msg_level, warn_msg) - FastTripsLogger.log(msg_level,"Util.add_new_id failed to map all ids to numbers") + FastTripsLogger.log(msg_level, "Util.add_new_id failed to map all ids to numbers") # FastTripsLogger.log(msg_level,"pd.isnull(return_df[%s]).sum() = %d" % (mapping_newid_colname_chk, pd.isnull(return_df[mapping_newid_colname_chk]).sum())) - FastTripsLogger.log(msg_level,"\n%s\n" % str(return_df.loc[pd.isnull(return_df[mapping_newid_colname_chk]),[id_colname,mapping_newid_colname_chk]].drop_duplicates())) + FastTripsLogger.log(msg_level, "\n%s\n" % str(return_df.loc[ + pd.isnull(return_df[mapping_newid_colname_chk]), [ + id_colname, + mapping_newid_colname_chk]].drop_duplicates())) # FastTripsLogger.log(msg_level,"pd.isnull(input_df[%s]).sum() = %d" % (id_colname, pd.isnull(input_df[id_colname]).sum())) - if drop_failures: # remove them return_df = return_df.loc[pd.notnull(return_df[mapping_newid_colname_chk])] @@ -149,9 +153,9 @@ def add_new_id(input_df, id_colname, newid_colname, # rename it as requested (if necessary) if newid_colname != mapping_newid_colname: if mapping_newid_colname in input_cols: - return_df.rename(columns={"%s_mapping" % mapping_newid_colname:newid_colname}, inplace=True) + return_df.rename(columns={"%s_mapping" % mapping_newid_colname: newid_colname}, inplace=True) else: - return_df.rename(columns={mapping_newid_colname:newid_colname}, inplace=True) + return_df.rename(columns={mapping_newid_colname: newid_colname}, inplace=True) # print "FINAL RETURN_DF==================" # print return_df.head() @@ -189,7 +193,7 @@ def pretty(df): df_cp = df.copy() df_cols = list(df.columns.values) for col_idx in range(len(df_cols)): - if str(df.dtypes[col_idx]) == "datetime64[ns]": # print as HH:MM:SS + if str(df.dtypes[col_idx]) == "datetime64[ns]": # print as HH:MM:SS df_cp[df_cols[col_idx]] = df[df_cols[col_idx]].apply(Util.datetime64_formatter) return df_cp @@ -199,25 +203,25 @@ def datetime64_min_formatter(x): Formatter to convert :py:class:`numpy.datetime64` to minutes after midnight (with two decimal places) """ - return '%.2f' % (pd.to_datetime(x).hour*60.0 + \ + return '%.2f' % (pd.to_datetime(x).hour * 60.0 + \ pd.to_datetime(x).minute + \ - (pd.to_datetime(x).second/60.0)) + (pd.to_datetime(x).second / 60.0)) @staticmethod def timedelta_formatter(x): """ Formatter to convert :py:class:`numpy.timedelta64` to string that looks like `4m 35.6s` """ - seconds = (x/np.timedelta64(1,'s')) - minutes = int(seconds/60) - seconds -= minutes*60 - return '%4dm %04.1fs' % (minutes,seconds) + seconds = (x / np.timedelta64(1, 's')) + minutes = int(seconds / 60) + seconds -= minutes * 60 + return '%4dm %04.1fs' % (minutes, seconds) @staticmethod def read_time(x, end_of_day=False): from .Assignment import Assignment try: - if x=='' or x.lower()=='default' or pd.isnull(x): + if x == '' or x.lower() == 'default' or pd.isnull(x): x = '24:00:00' if end_of_day else '00:00:00' except: if pd.isnull(x): @@ -226,7 +230,7 @@ def read_time(x, end_of_day=False): hour = int(time_split[0]) day = Assignment.NETWORK_BUILD_DATE if hour >= 24: - time_split[0] = '%02d' %(hour-24) + time_split[0] = '%02d' % (hour - 24) day += datetime.timedelta(days=1) x = ':'.join(time_split) return datetime.datetime.combine(day, datetime.datetime.strptime(x, '%H:%M:%S').time()) @@ -241,9 +245,9 @@ def parse_minutes_to_time(minutes): elapsed_time = datetime.timedelta(minutes=minutes) return datetime.datetime.combine(Assignment.NETWORK_BUILD_DATE, datetime.time()) + elapsed_time - @staticmethod - def write_dataframe(df, name, output_file, append=False, keep_duration_columns=False, drop_debug_columns=True, drop_pathfinding_columns=True): + def write_dataframe(df, name, output_file, append=False, keep_duration_columns=False, drop_debug_columns=True, + drop_pathfinding_columns=True): """ Convenience method to write a dataframe but make some of the fields more usable. @@ -302,14 +306,14 @@ def write_dataframe(df, name, output_file, append=False, keep_duration_columns=F if str(df_toprint.dtypes[col_idx]) == "timedelta64[ns]": # lookup timedelta units - units_str = Util.TIMEDELTA_COLUMNS_TO_UNITS[old_colname] + units_str = Util.TIMEDELTA_COLUMNS_TO_UNITS[old_colname] new_colname = "%s %s" % (old_colname, units_str) if units_str == "milliseconds": - units = np.timedelta64(1,'ms') + units = np.timedelta64(1, 'ms') elif units_str == "min": - units = np.timedelta64(1,'m') + units = np.timedelta64(1, 'm') elif units_str == "seconds": - units = np.timedelta64(1,'s') + units = np.timedelta64(1, 's') else: raise Exception @@ -317,10 +321,10 @@ def write_dataframe(df, name, output_file, append=False, keep_duration_columns=F if new_colname in df_cols: continue # otherwise make the new one and add or replace it - df_toprint[new_colname] = (df_toprint[old_colname]/units) - if keep_duration_columns: # add + df_toprint[new_colname] = (df_toprint[old_colname] / units) + if keep_duration_columns: # add df_cols.append(new_colname) - else: # replace + else: # replace df_cols[col_idx] = new_colname elif str(df_toprint.dtypes[col_idx]) == "datetime64[ns]": @@ -350,14 +354,17 @@ def calculate_distance_miles(dataframe, origin_lat, origin_lon, destination_lat, Given a dataframe with columns origin_lat, origin_lon, destination_lat, destination_lon, calculates the distance in miles between origin and destination based on Haversine. Results are added to the dataframe in a column called *distance_colname*. """ - radius = 3963.190592 # mi + radius = 3963.190592 # mi # assume these aren't in here - dataframe["dist_lat" ] = np.radians(dataframe[destination_lat]-dataframe[origin_lat]) - dataframe["dist_lon" ] = np.radians(dataframe[destination_lon]-dataframe[origin_lon]) - dataframe["dist_hava"] = (np.sin(dataframe["dist_lat"]/2) * np.sin(dataframe["dist_lat"]/2)) + \ - (np.cos(np.radians(dataframe[origin_lat])) * np.cos(np.radians(dataframe[destination_lat])) * np.sin(dataframe["dist_lon"]/2.0) * np.sin(dataframe["dist_lon"]/2.0)) - dataframe["dist_havc"] = 2.0*np.arctan2(np.sqrt(dataframe["dist_hava"]), np.sqrt(1.0-dataframe["dist_hava"])) + dataframe["dist_lat"] = np.radians(dataframe[destination_lat] - dataframe[origin_lat]) + dataframe["dist_lon"] = np.radians(dataframe[destination_lon] - dataframe[origin_lon]) + dataframe["dist_hava"] = (np.sin(dataframe["dist_lat"] / 2) * np.sin(dataframe["dist_lat"] / 2)) + \ + (np.cos(np.radians(dataframe[origin_lat])) * np.cos( + np.radians(dataframe[destination_lat])) * np.sin( + dataframe["dist_lon"] / 2.0) * np.sin(dataframe["dist_lon"] / 2.0)) + dataframe["dist_havc"] = 2.0 * np.arctan2(np.sqrt(dataframe["dist_hava"]), + np.sqrt(1.0 - dataframe["dist_hava"])) dataframe[distance_colname] = radius * dataframe["dist_havc"] # FastTripsLogger.debug("calculate_distance_miles\n%s", dataframe.to_string()) @@ -366,12 +373,13 @@ def calculate_distance_miles(dataframe, origin_lat, origin_lon, destination_lat, min_dist = dataframe[distance_colname].min() max_dist = dataframe[distance_colname].max() if min_dist < 0: - FastTripsLogger.warn("calculate_distance_miles: min is negative\n%s" % dataframe.loc[dataframe[distance_colname]<0].to_string()) + FastTripsLogger.warn("calculate_distance_miles: min is negative\n%s" % dataframe.loc[ + dataframe[distance_colname] < 0].to_string()) if max_dist > 1000: - FastTripsLogger.warn("calculate_distance_miles: max is greater than 1k\n%s" % dataframe.loc[dataframe[distance_colname]>1000].to_string()) - - dataframe.drop(["dist_lat","dist_lon","dist_hava","dist_havc"], axis=1, inplace=True) + FastTripsLogger.warn("calculate_distance_miles: max is greater than 1k\n%s" % dataframe.loc[ + dataframe[distance_colname] > 1000].to_string()) + dataframe.drop(["dist_lat", "dist_lon", "dist_hava", "dist_havc"], axis=1, inplace=True) @staticmethod def get_process_mem_use_bytes(): @@ -399,11 +407,11 @@ def get_process_mem_use_str(): if bytes < 1000: return "%d bytes" % bytes - if bytes < 1000*1000: - return "%.1f KB" % (bytes/1000.0) - if bytes < 1000*1000*1000: - return "%.1f MB" % (bytes/(1000.0*1000.0)) - return "%.1f GB" % (bytes/(1000.0*1000.0*1000.0)) + if bytes < 1000 * 1000: + return "%.1f KB" % (bytes / 1000.0) + if bytes < 1000 * 1000 * 1000: + return "%.1f MB" % (bytes / (1000.0 * 1000.0)) + return "%.1f GB" % (bytes / (1000.0 * 1000.0 * 1000.0)) @staticmethod def merge_two_dicts(x, y): @@ -450,38 +458,42 @@ def calculate_pathweight_costs(df, result_col): from fasttrips import PathSet # default is constant (constant weight) - df[result_col] = df['var_value']*df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE] + df[result_col] = df['var_value'] * df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE] if PathSet.EXP_GROWTH_MODEL in df[PathSet.WEIGHTS_GROWTH_TYPE].values: - df.loc[df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.EXP_GROWTH_MODEL, result_col] = \ + df.loc[df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.EXP_GROWTH_MODEL, result_col] = \ Util.exponential_integration(df['var_value'], df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE]) if PathSet.LOGARITHMIC_GROWTH_MODEL in df[PathSet.WEIGHTS_GROWTH_TYPE].values: - assert {'var_value', PathSet.WEIGHTS_GROWTH_LOG_BASE}.issubset(df), "Logarithmic pathweight growth_type formula specified. Missing var_value, or growth_log_base." + assert {'var_value', PathSet.WEIGHTS_GROWTH_LOG_BASE}.issubset( + df), "Logarithmic pathweight growth_type formula specified. Missing var_value, or growth_log_base." df.loc[df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGARITHMIC_GROWTH_MODEL, result_col] = \ - Util.logarithmic_integration(df['var_value'], df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE], df[PathSet.WEIGHTS_GROWTH_LOG_BASE]) + Util.logarithmic_integration(df['var_value'], df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE], + df[PathSet.WEIGHTS_GROWTH_LOG_BASE]) if PathSet.LOGISTIC_GROWTH_MODEL in df[PathSet.WEIGHTS_GROWTH_TYPE].values: - assert {'var_value', PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX, PathSet.WEIGHTS_GROWTH_LOGISTIC_MID}.issubset(df), "Logistic pathweight growth_type formula specified. Missing var_value, growth_logistic_max, or growth_logistic_mid." + assert {'var_value', PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX, PathSet.WEIGHTS_GROWTH_LOGISTIC_MID}.issubset( + df), "Logistic pathweight growth_type formula specified. Missing var_value, growth_logistic_max, or growth_logistic_mid." df.loc[df[PathSet.WEIGHTS_GROWTH_TYPE] == PathSet.LOGISTIC_GROWTH_MODEL, result_col] = \ - Util.logistic_integration(df['var_value'], df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE], df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX], df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MID]) + Util.logistic_integration(df['var_value'], df[PathSet.WEIGHTS_COLUMN_WEIGHT_VALUE], + df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MAX], + df[PathSet.WEIGHTS_GROWTH_LOGISTIC_MID]) # TODO: option: make these more subtle? # missed_xfer has huge cost if 'missed_xfer' in df: - df.loc[df['missed_xfer']==1, result_col] = PathSet.HUGE_COST + df.loc[df['missed_xfer'] == 1, result_col] = PathSet.HUGE_COST # bump iter means over capacity if 'bump_iter' in df: - df.loc[df['bump_iter']>=0, result_col] = PathSet.HUGE_COST + df.loc[df['bump_iter'] >= 0, result_col] = PathSet.HUGE_COST # negative cost is invalid if (df[result_col] < 0).any(): FastTripsLogger.warn("---Pathweight costs has negative values. Setting to zero.---\n{}".format( df[df[result_col] < 0].to_string()) ) - df.loc[ df[result_col] < 0, result_col ] = 0.0 - + df.loc[df[result_col] < 0, result_col] = 0.0 @staticmethod def exponential_integration(penalty_min, growth_rate): @@ -493,8 +505,7 @@ def exponential_integration(penalty_min, growth_rate): :param growth_rate: float: Exponetial growth factor :return: float or :py:class:`pandas.Series` of floats depending on inputs """ - return (np.power(1.0 + growth_rate, penalty_min) - 1)/ np.log(1.0 + growth_rate) - + return (np.power(1.0 + growth_rate, penalty_min) - 1) / np.log(1.0 + growth_rate) @staticmethod def logarithmic_integration(penalty_min, growth_rate, log_base=np.exp(1)): @@ -509,7 +520,6 @@ def logarithmic_integration(penalty_min, growth_rate, log_base=np.exp(1)): """ return growth_rate * ((penalty_min + 1) * np.log(penalty_min + 1) - penalty_min) / np.log(log_base) - @staticmethod def logistic_integration(penalty_minute, growth_rate, max_logit, sigmoid_mid): """ @@ -525,12 +535,12 @@ def logistic_integration(penalty_minute, growth_rate, max_logit, sigmoid_mid): :return: float or :py:class:`pandas.Series` of floats depending on inputs """ - max_integral = ((max_logit/ growth_rate)) * np.log(np.exp(growth_rate * penalty_minute) + np.exp(growth_rate * sigmoid_mid)) - min_integral = ((max_logit/ growth_rate)) * np.log(1 + np.exp(growth_rate * sigmoid_mid)) + max_integral = ((max_logit / growth_rate)) * np.log( + np.exp(growth_rate * penalty_minute) + np.exp(growth_rate * sigmoid_mid)) + min_integral = ((max_logit / growth_rate)) * np.log(1 + np.exp(growth_rate * sigmoid_mid)) return max_integral - min_integral - @staticmethod def get_fast_trips_config(): """ diff --git a/fasttrips/__init__.py b/fasttrips/__init__.py index fe3e836c..9b1aca9f 100644 --- a/fasttrips/__init__.py +++ b/fasttrips/__init__.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -13,25 +13,25 @@ limitations under the License. """ -from .Assignment import Assignment -from .Error import Error, ConfigurationError -from .FastTrips import FastTrips -from .Logger import FastTripsLogger, setupLogging -from .Passenger import Passenger -from .PathSet import PathSet +from .Assignment import Assignment +from .Error import Error, ConfigurationError +from .FastTrips import FastTrips +from .Logger import FastTripsLogger, setupLogging +from .Passenger import Passenger +from .PathSet import PathSet from .Performance import Performance -from .Route import Route -from .Run import run_fasttrips, main -from .Stop import Stop -from .TAZ import TAZ -from .Transfer import Transfer -from .Trip import Trip -from .Util import Util +from .Route import Route +from .Run import run_fasttrips, main +from .Stop import Stop +from .TAZ import TAZ +from .Transfer import Transfer +from .Trip import Trip +from .Util import Util __all__ = [ 'Event', 'FastTrips', - 'FastTripsLogger','setupLogging', + 'FastTripsLogger', 'setupLogging', 'Passenger', 'PathSet', 'Route', diff --git a/fasttrips/utils/__init__.py b/fasttrips/utils/__init__.py index 3ec77df8..227ca769 100644 --- a/fasttrips/utils/__init__.py +++ b/fasttrips/utils/__init__.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -17,4 +17,4 @@ 'maps', 'plots', 'widgets', - ] +] diff --git a/fasttrips/utils/map_prep.py b/fasttrips/utils/map_prep.py index 327503c8..445b6046 100644 --- a/fasttrips/utils/map_prep.py +++ b/fasttrips/utils/map_prep.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015-2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -16,41 +16,45 @@ import os import pandas as pd + def convert_stop_coordinates_to_mercator(row): - from pyproj import Proj,transform + from pyproj import Proj, transform """ Expects to get a pandas data frame row from being called from an apply(). Outputs a pandas series in mercator coordinates appropriate for using with things like Bokeh map tiles. """ - inProj = Proj(init='epsg:4326') + inProj = Proj(init='epsg:4326') outProj = Proj(init='epsg:3857') - x2,y2 = transform(inProj,outProj,row['stop_lon'],row['stop_lat']) - return pd.Series({'stop_m-lon':x2,'stop_m-lat':y2}) + x2, y2 = transform(inProj, outProj, row['stop_lon'], row['stop_lat']) + return pd.Series({'stop_m-lon': x2, 'stop_m-lat': y2}) + def convert_stop_coordinates_to_mercator(row): - from pyproj import Proj,transform + from pyproj import Proj, transform """ Expects to get a pandas data frame row from being called from an apply(). Outputs a pandas series in mercator coordinates appropriate for using with things like Bokeh map tiles. """ - inProj = Proj(init='epsg:4326') + inProj = Proj(init='epsg:4326') outProj = Proj(init='epsg:3857') - x2,y2 = transform(inProj,outProj,row['stop_lon'],row['stop_lat']) - return pd.Series({'stop_m-lon':x2,'stop_m-lat':y2}) + x2, y2 = transform(inProj, outProj, row['stop_lon'], row['stop_lat']) + return pd.Series({'stop_m-lon': x2, 'stop_m-lat': y2}) + def convert_taz_coords_to_mercator(row): - from pyproj import Proj,transform + from pyproj import Proj, transform """ Expects to get a pandas data frame row from being called from an apply(). Outputs a pandas series in mercator coordinates appropriate for using with things like Bokeh map tiles. """ - inProj = Proj(init='epsg:4326') + inProj = Proj(init='epsg:4326') outProj = Proj(init='epsg:3857') - x2,y2 = transform(inProj,outProj,row['lon'],row['lat']) - return pd.Series({'taz-lon':x2,'taz-lat':y2}) + x2, y2 = transform(inProj, outProj, row['lon'], row['lat']) + return pd.Series({'taz-lon': x2, 'taz-lat': y2}) + def prepare_demand(network_dir, demand_dir): """ @@ -62,50 +66,54 @@ def prepare_demand(network_dir, demand_dir): - `od_trip_count_df` aggregated trips to origin-destination pairs. """ - triplist_df = pd.read_csv(os.path.join(demand_dir,'trip_list.txt'), usecols=['departure_time', 'arrival_time', 'person_id','time_target','o_taz','d_taz' ]) + triplist_df = pd.read_csv(os.path.join(demand_dir, 'trip_list.txt'), + usecols=['departure_time', 'arrival_time', 'person_id', 'time_target', 'o_taz', 'd_taz']) - taz_df = pd.read_csv(os.path.join(network_dir,'taz_coords.txt')) - taz_xy_df = taz_df.apply(convert_taz_coords_to_mercator, axis=1) - taz_df = pd.concat([taz_df,taz_xy_df], axis=1) - taz_df.drop(columns=['lat','lon'],inplace=True) + taz_df = pd.read_csv(os.path.join(network_dir, 'taz_coords.txt')) + taz_xy_df = taz_df.apply(convert_taz_coords_to_mercator, axis=1) + taz_df = pd.concat([taz_df, taz_xy_df], axis=1) + taz_df.drop(columns=['lat', 'lon'], inplace=True) - triplist_xy_df = pd.merge(triplist_df, taz_df, left_on='o_taz', right_on='taz', how='left') + triplist_xy_df = pd.merge(triplist_df, taz_df, left_on='o_taz', right_on='taz', how='left') triplist_xy_df.rename(index=str, columns={"taz-lon": "o-lon", "taz-lat": "o-lat"}, inplace=True) - triplist_xy_df.drop(columns=['taz'],inplace=True) - triplist_xy_df = pd.merge(triplist_xy_df, taz_df,left_on='d_taz', right_on='taz', how='left') + triplist_xy_df.drop(columns=['taz'], inplace=True) + triplist_xy_df = pd.merge(triplist_xy_df, taz_df, left_on='d_taz', right_on='taz', how='left') triplist_xy_df.rename(index=str, columns={"taz-lon": "d-lon", "taz-lat": "d-lat"}, inplace=True) - triplist_xy_df.drop(columns=['taz'],inplace=True) + triplist_xy_df.drop(columns=['taz'], inplace=True) # GROUP ORIGIN TRIPS o_trip_count_df = triplist_xy_df.groupby(['o_taz']).size().reset_index(name='origins') - o_trip_count_df.rename(columns={'o_taz':'taz'}, inplace=True) + o_trip_count_df.rename(columns={'o_taz': 'taz'}, inplace=True) # GROUP DESTINATION TRIPS d_trip_count_df = triplist_xy_df.groupby(['d_taz']).size().reset_index(name='destinations') - d_trip_count_df.rename(columns={'d_taz':'taz'}, inplace=True) + d_trip_count_df.rename(columns={'d_taz': 'taz'}, inplace=True) # PUT TOGETHER - trip_count_df = pd.merge(taz_df, o_trip_count_df, how='left', on='taz') - trip_count_df = pd.merge(trip_count_df, d_trip_count_df, how='left', on='taz') + trip_count_df = pd.merge(taz_df, o_trip_count_df, how='left', on='taz') + trip_count_df = pd.merge(trip_count_df, d_trip_count_df, how='left', on='taz') # GROUP OD TRIPS - od_trip_count_df = triplist_xy_df.groupby(['o_taz','d_taz']).size().reset_index(name='trips') + od_trip_count_df = triplist_xy_df.groupby(['o_taz', 'd_taz']).size().reset_index(name='trips') - od_trip_count_df = pd.merge(od_trip_count_df, taz_df, left_on='o_taz', right_on='taz', how='left') + od_trip_count_df = pd.merge(od_trip_count_df, taz_df, left_on='o_taz', right_on='taz', how='left') od_trip_count_df.rename(index=str, columns={"taz-lon": "o-lon", "taz-lat": "o-lat"}, inplace=True) - od_trip_count_df.drop(columns=['taz'],inplace=True) - od_trip_count_df = pd.merge(od_trip_count_df, taz_df,left_on='d_taz', right_on='taz', how='left') + od_trip_count_df.drop(columns=['taz'], inplace=True) + od_trip_count_df = pd.merge(od_trip_count_df, taz_df, left_on='d_taz', right_on='taz', how='left') od_trip_count_df.rename(index=str, columns={"taz-lon": "d-lon", "taz-lat": "d-lat"}, inplace=True) return triplist_xy_df, trip_count_df, od_trip_count_df, taz_df + def prepare_network_for_map(network_dir): - stoptimes_df = pd.read_csv(os.path.join(network_dir, 'stop_times.txt'), usecols=['trip_id','stop_id','stop_sequence']) - stops_df = pd.read_csv(os.path.join(network_dir, 'stops.txt'), usecols=['stop_id','stop_name','stop_lat','stop_lon']) - xy_merc = stops_df.apply(convert_stop_coordinates_to_mercator, axis=1) - stops_df = pd.concat([stops_df, xy_merc], axis=1) - trip_stops_df = pd.merge(stops_df, stoptimes_df, on='stop_id', how='left') - trip_count_df = trip_stops_df.groupby(['stop_id']).size().reset_index(name='trips') - - stops_size_df = pd.merge(stops_df, trip_count_df, on='stop_id') + stoptimes_df = pd.read_csv(os.path.join(network_dir, 'stop_times.txt'), + usecols=['trip_id', 'stop_id', 'stop_sequence']) + stops_df = pd.read_csv(os.path.join(network_dir, 'stops.txt'), + usecols=['stop_id', 'stop_name', 'stop_lat', 'stop_lon']) + xy_merc = stops_df.apply(convert_stop_coordinates_to_mercator, axis=1) + stops_df = pd.concat([stops_df, xy_merc], axis=1) + trip_stops_df = pd.merge(stops_df, stoptimes_df, on='stop_id', how='left') + trip_count_df = trip_stops_df.groupby(['stop_id']).size().reset_index(name='trips') + + stops_size_df = pd.merge(stops_df, trip_count_df, on='stop_id') return stops_size_df diff --git a/fasttrips/utils/maps.py b/fasttrips/utils/maps.py index fd419efb..facce03f 100644 --- a/fasttrips/utils/maps.py +++ b/fasttrips/utils/maps.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015-2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -24,38 +24,39 @@ from bokeh.tile_providers import STAMEN_TONER from bokeh.models import Arrow, OpenHead -def map_demand(demand, network): +def map_demand(demand, network): demand_xy_df, agg_demand_df, agg_od_df, taz_df = prepare_demand(network, demand) o_trip_count_df = demand_xy_df.groupby(['o_taz']).size().reset_index(name='o_trips') d_trip_count_df = demand_xy_df.groupby(['d_taz']).size().reset_index(name='d_trips') - p_map = figure(x_axis_type="mercator", y_axis_type="mercator", title="Origin and Destinations for Transit Trips", width=800) + p_map = figure(x_axis_type="mercator", y_axis_type="mercator", title="Origin and Destinations for Transit Trips", + width=800) p_map.grid.visible = False p_map.add_tile(STAMEN_TONER) source = ColumnDataSource(data=dict( - x =list(agg_demand_df['taz-lon']), - y =list(agg_demand_df['taz-lat']), - origins = list(agg_demand_df['origins']), - destinations = list(agg_demand_df['destinations']))) - - p_map.circle(x = 'x', - y = 'y', - source = source, - size = "origins", - legend = "origin", + x=list(agg_demand_df['taz-lon']), + y=list(agg_demand_df['taz-lat']), + origins=list(agg_demand_df['origins']), + destinations=list(agg_demand_df['destinations']))) + + p_map.circle(x='x', + y='y', + source=source, + size="origins", + legend="origin", fill_color="green", line_color="green", fill_alpha=0.15, line_alpha=0.15) - p_map.circle(x = 'x', - y = 'y', - source = source, - size = "destinations", - legend = "destination", + p_map.circle(x='x', + y='y', + source=source, + size="destinations", + legend="destination", fill_color="navy", line_color="blue", line_width=4, @@ -68,42 +69,42 @@ def map_demand(demand, network): y1 = row['o-lat'] y2 = row['d-lat'] - p_map.line([x1,x2], [y1,y2], line_width= row['trips'], line_alpha=0.2) - p_map.add_layout(Arrow(end=OpenHead(size=row['trips']), line_color="navy", line_alpha = 0.2, - x_start=x1, y_start=y1, x_end=x2, y_end=y2)) + p_map.line([x1, x2], [y1, y2], line_width=row['trips'], line_alpha=0.2) + p_map.add_layout(Arrow(end=OpenHead(size=row['trips']), line_color="navy", line_alpha=0.2, + x_start=x1, y_start=y1, x_end=x2, y_end=y2)) return p_map -def map_stops(network_dir, stop_labels): +def map_stops(network_dir, stop_labels): stops_size_df = prepare_network_for_map(network_dir) - size_factor = stops_size_df['trips'].max()/70 + size_factor = stops_size_df['trips'].max() / 70 if stops_size_df['trips'].max() < 10: size_factor = 0.05 source = ColumnDataSource(data=dict( - x =list(stops_size_df['stop_m-lon']), - y =list(stops_size_df['stop_m-lat']), - trips=list(stops_size_df['trips']), - size =list(stops_size_df['trips']/size_factor), - offs =list(stops_size_df['trips']/(size_factor*2)), - name =list(stops_size_df['stop_name']))) + x=list(stops_size_df['stop_m-lon']), + y=list(stops_size_df['stop_m-lat']), + trips=list(stops_size_df['trips']), + size=list(stops_size_df['trips'] / size_factor), + offs=list(stops_size_df['trips'] / (size_factor * 2)), + name=list(stops_size_df['stop_name']))) - hover = HoverTool(tooltips=[("stop", "@name"),("trips","@trips") ]) + hover = HoverTool(tooltips=[("stop", "@name"), ("trips", "@trips")]) - p = figure(x_axis_type="mercator", y_axis_type="mercator", title = "Transit Stop Map", width = 800) + p = figure(x_axis_type="mercator", y_axis_type="mercator", title="Transit Stop Map", width=800) p.add_tile(STAMEN_TONER) - p.circle(x = 'x', - y = 'y', - source = source, - size = 'size', - fill_color="#FF1493", - line_color="#FF69B4", - fill_alpha=0.15, - line_alpha=0.40) - - labels = LabelSet(x='x', y='y', text='name', level='glyph',text_color="#FF1493", - x_offset='offs', y_offset='offs', source=source, render_mode='canvas') + p.circle(x='x', + y='y', + source=source, + size='size', + fill_color="#FF1493", + line_color="#FF69B4", + fill_alpha=0.15, + line_alpha=0.40) + + labels = LabelSet(x='x', y='y', text='name', level='glyph', text_color="#FF1493", + x_offset='offs', y_offset='offs', source=source, render_mode='canvas') if stop_labels: p.add_layout(labels) diff --git a/fasttrips/utils/plots.py b/fasttrips/utils/plots.py index fb152e0a..9bb804c1 100644 --- a/fasttrips/utils/plots.py +++ b/fasttrips/utils/plots.py @@ -1,5 +1,5 @@ __copyright__ = "Copyright 2015-2016 Contributing Entities" -__license__ = """ +__license__ = """ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -23,78 +23,92 @@ from bokeh.plotting import figure, show, output_notebook from bokeh.tile_providers import STAMEN_TONER + def plot_demand_time(demand_dir, aggregation_level=20): - triplist_df = pd.read_csv(os.path.join(demand_dir,'trip_list.txt'), usecols=['departure_time', 'arrival_time', 'person_id','time_target' ]) + triplist_df = pd.read_csv(os.path.join(demand_dir, 'trip_list.txt'), + usecols=['departure_time', 'arrival_time', 'person_id', 'time_target']) triplist_df["freq"] = 1 triplist_df["departure_time_dt"] = pd.to_datetime(triplist_df["departure_time"]) - triplist_df["arrival_time_dt"] = pd.to_datetime(triplist_df["arrival_time"]) - depart_agg = triplist_df[triplist_df['time_target']=='departure'].resample('%dT' % (aggregation_level), on='departure_time_dt').sum().rename(columns={'freq': 'trips_depart'}) - arrive_agg = triplist_df[triplist_df['time_target']=='arrival'].resample('%dT' % (aggregation_level), on='arrival_time_dt').sum().rename(columns={'freq': 'trips_arrive'}) - - p = figure(plot_height=350, x_axis_type="datetime",toolbar_location=None, tools="", title="Time of Day Distribution of Transit Trips in Demand", width=800) + triplist_df["arrival_time_dt"] = pd.to_datetime(triplist_df["arrival_time"]) + depart_agg = triplist_df[triplist_df['time_target'] == 'departure'].resample('%dT' % (aggregation_level), + on='departure_time_dt').sum().rename( + columns={'freq': 'trips_depart'}) + arrive_agg = triplist_df[triplist_df['time_target'] == 'arrival'].resample('%dT' % (aggregation_level), + on='arrival_time_dt').sum().rename( + columns={'freq': 'trips_arrive'}) + + p = figure(plot_height=350, x_axis_type="datetime", toolbar_location=None, tools="", + title="Time of Day Distribution of Transit Trips in Demand", width=800) p.xaxis.axis_label = "Time of Day" p.yaxis.axis_label = "Number of trips, aggregated to %d minutes" % (aggregation_level) - p.line(x=list(depart_agg.index), y=list(depart_agg['trips_depart']), legend="departures", color="green", line_width=2) + p.line(x=list(depart_agg.index), y=list(depart_agg['trips_depart']), legend="departures", color="green", + line_width=2) p.line(x=list(arrive_agg.index), y=list(arrive_agg['trips_arrive']), legend="arrivals", color="blue", line_width=2) - p.vbar(x=[x for x in list(depart_agg.index)], width=aggregation_level/2, bottom=0, top=list(depart_agg['trips_depart']), color="green") - p.vbar(x=[x for x in list(arrive_agg.index)], width=aggregation_level/2, bottom=0, top=list(arrive_agg['trips_arrive']), color="blue") + p.vbar(x=[x for x in list(depart_agg.index)], width=aggregation_level / 2, bottom=0, + top=list(depart_agg['trips_depart']), color="green") + p.vbar(x=[x for x in list(arrive_agg.index)], width=aggregation_level / 2, bottom=0, + top=list(arrive_agg['trips_arrive']), color="blue") return p + def plot_route_timing(network_dir, sel_routes=[]): - stoptimes_df = pd.read_csv(os.path.join(network_dir, 'stop_times.txt'), usecols=['trip_id','stop_sequence','arrival_time']) - firststop_df = stoptimes_df[stoptimes_df['stop_sequence']==1] - trips_df = pd.read_csv(os.path.join(network_dir, 'trips.txt')) - firststop_df = pd.merge(firststop_df, trips_df, on='trip_id', how='left') + stoptimes_df = pd.read_csv(os.path.join(network_dir, 'stop_times.txt'), + usecols=['trip_id', 'stop_sequence', 'arrival_time']) + firststop_df = stoptimes_df[stoptimes_df['stop_sequence'] == 1] + trips_df = pd.read_csv(os.path.join(network_dir, 'trips.txt')) + firststop_df = pd.merge(firststop_df, trips_df, on='trip_id', how='left') if 'direction_id' not in firststop_df: firststop_df['direction_id'] = 0 - firststop_df.loc[firststop_df['direction_id']==0,'route_id-dir'] = firststop_df.loc[firststop_df['direction_id']==0,'route_id']+"-Out" - firststop_df.loc[firststop_df['direction_id']==1,'route_id-dir'] = firststop_df.loc[firststop_df['direction_id']==1,'route_id']+"-In" + firststop_df.loc[firststop_df['direction_id'] == 0, 'route_id-dir'] = firststop_df.loc[firststop_df[ + 'direction_id'] == 0, 'route_id'] + "-Out" + firststop_df.loc[firststop_df['direction_id'] == 1, 'route_id-dir'] = firststop_df.loc[firststop_df[ + 'direction_id'] == 1, 'route_id'] + "-In" routes = sorted(list(set(firststop_df["route_id-dir"]))) - out_df = firststop_df[firststop_df['direction_id']==0] - in_df = firststop_df[firststop_df['direction_id']==1] + out_df = firststop_df[firststop_df['direction_id'] == 0] + in_df = firststop_df[firststop_df['direction_id'] == 1] route_in = ColumnDataSource(data=dict( - x =[ pd.to_datetime(t) for t in list(in_df['arrival_time'])], - y =list(in_df['route_id-dir']), - t =list(in_df['arrival_time']), - name =list(in_df['trip_id']))) + x=[pd.to_datetime(t) for t in list(in_df['arrival_time'])], + y=list(in_df['route_id-dir']), + t=list(in_df['arrival_time']), + name=list(in_df['trip_id']))) route_out = ColumnDataSource(data=dict( - x =[ pd.to_datetime(t) for t in list(out_df['arrival_time'])], - y =list(out_df['route_id-dir']), - t =list(out_df['arrival_time']), - name =list(out_df['trip_id']))) + x=[pd.to_datetime(t) for t in list(out_df['arrival_time'])], + y=list(out_df['route_id-dir']), + t=list(out_df['arrival_time']), + name=list(out_df['trip_id']))) - tips=[ + tips = [ ("trip", "@name"), ("time", "@t"), ] - p_route = figure(x_axis_type="datetime",width=800, y_range=routes, + p_route = figure(x_axis_type="datetime", width=800, y_range=routes, title='Route Timing') - p_route.circle(x = 'x', - y = 'y', - source = route_in, - size = 20, - fill_color="#0892D0", - line_color="#0892D0", - fill_alpha=.5, - line_alpha=0.40) - - p_route.circle(x = 'x', - y = 'y', - source = route_out, - size = 20, - fill_color="#FF1493", - line_color="#FF69B4", - fill_alpha=.5, - line_alpha=0.40) + p_route.circle(x='x', + y='y', + source=route_in, + size=20, + fill_color="#0892D0", + line_color="#0892D0", + fill_alpha=.5, + line_alpha=0.40) + + p_route.circle(x='x', + y='y', + source=route_out, + size=20, + fill_color="#FF1493", + line_color="#FF69B4", + fill_alpha=.5, + line_alpha=0.40) p_route.add_tools(HoverTool(tooltips=tips)) diff --git a/scripts/calculate_cost.py b/scripts/calculate_cost.py index 1bdf31c2..1e0ae9af 100644 --- a/scripts/calculate_cost.py +++ b/scripts/calculate_cost.py @@ -4,23 +4,21 @@ from fasttrips import PathSet, Run - -BASE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples') -TEST_FOLDER = 'calculate_cost' +BASE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples') +TEST_FOLDER = 'calculate_cost' # DIRECTORY LOCATIONS -OUTPUT_DIR = os.path.join(BASE_DIR, 'output') -INPUT_NETWORK = os.path.join(BASE_DIR, 'networks', 'simple') -INPUT_DEMAND = os.path.join(BASE_DIR, 'demand', 'demand_twopaths') -DF_DIR = os.path.join(BASE_DIR, 'misc', TEST_FOLDER) +OUTPUT_DIR = os.path.join(BASE_DIR, 'output') +INPUT_NETWORK = os.path.join(BASE_DIR, 'networks', 'simple') +INPUT_DEMAND = os.path.join(BASE_DIR, 'demand', 'demand_twopaths') +DF_DIR = os.path.join(BASE_DIR, 'misc', TEST_FOLDER) # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_DEMAND, 'config_ft.txt') -INPUT_FUNCTIONS = os.path.join(INPUT_DEMAND, 'config_ft.py') -INPUT_WEIGHTS = os.path.join(INPUT_DEMAND, 'pathweight_ft.txt') -PATHSET_PATHS_OUT = os.path.join(DF_DIR, 'output_pathset_paths_calculate_cost.csv') -PATHSET_LINKS_OUT = os.path.join(DF_DIR, 'output_pathset_links_calculate_cost.csv') - +CONFIG_FILE = os.path.join(INPUT_DEMAND, 'config_ft.txt') +INPUT_FUNCTIONS = os.path.join(INPUT_DEMAND, 'config_ft.py') +INPUT_WEIGHTS = os.path.join(INPUT_DEMAND, 'pathweight_ft.txt') +PATHSET_PATHS_OUT = os.path.join(DF_DIR, 'output_pathset_paths_calculate_cost.csv') +PATHSET_LINKS_OUT = os.path.join(DF_DIR, 'output_pathset_links_calculate_cost.csv') STOCHASTIC_DISPERSION = 0.5 @@ -75,24 +73,26 @@ def run_calculate_cost(ft): ######## LOAD IN PATHSET PATHS ################# pathset_paths_loc = os.path.join(DF_DIR, 'input_pathset_paths.csv') pathset_paths_cols = [ - 'person_id', 'person_trip_id', 'trip_list_id_num','trace','pathdir', 'pathmode', 'pathnum' + 'person_id', 'person_trip_id', 'trip_list_id_num', 'trace', 'pathdir', 'pathmode', 'pathnum' ] pathset_paths_df = pd.read_csv(pathset_paths_loc, usecols=pathset_paths_cols, - dtype={'person_id':str, 'person_trip_id':str}) + dtype={'person_id': str, 'person_trip_id': str}) ######## LOAD IN PATHSET LINKS ################# pathset_links_loc = os.path.join(DF_DIR, 'input_pathset_links.csv') pathset_links_cols = [ 'person_id', 'person_trip_id', 'trip_list_id_num', 'trace', 'pathnum', 'linkmode', - 'trip_id_num', 'A_seq', 'B_seq', 'A_id_num', 'B_id_num','pf_A_time','pf_B_time','pf_linkdist','linknum', 'A_id', - 'B_id', 'trip_id', 'route_id', 'mode_num', 'mode', 'bump_iter', 'board_state','new_A_time', - 'new_B_time', 'new_linktime', 'pf_linktime','missed_xfer','board_time', 'overcap', 'alight_time', + 'trip_id_num', 'A_seq', 'B_seq', 'A_id_num', 'B_id_num', 'pf_A_time', 'pf_B_time', 'pf_linkdist', 'linknum', + 'A_id', + 'B_id', 'trip_id', 'route_id', 'mode_num', 'mode', 'bump_iter', 'board_state', 'new_A_time', + 'new_B_time', 'new_linktime', 'pf_linktime', 'missed_xfer', 'board_time', 'overcap', 'alight_time', ] pathset_links_df = pd.read_csv(pathset_links_loc, usecols=pathset_links_cols, - parse_dates=['new_A_time', 'new_B_time', 'pf_A_time', 'pf_B_time', 'pf_linktime', 'board_time'], + parse_dates=['new_A_time', 'new_B_time', 'pf_A_time', 'pf_B_time', 'pf_linktime', + 'board_time'], infer_datetime_format=True, - dtype={'person_id':str, 'person_trip_id': str}) + dtype={'person_id': str, 'person_trip_id': str}) pathset_links_df['pf_linktime'] = pd.to_timedelta(pathset_links_df['pf_linktime'], 'm') pathset_links_df['new_linktime'] = pd.to_timedelta(pathset_links_df['new_linktime'], 'm') @@ -106,7 +106,7 @@ def run_calculate_cost(ft): ] veh_trips_df = pd.read_csv(veh_trips_loc, usecols=veh_trips_col, - infer_datetime_format=True, parse_dates=['arrival_time','departure_time']) + infer_datetime_format=True, parse_dates=['arrival_time', 'departure_time']) (paths_df, links_df) = PathSet.calculate_cost(ft, STOCHASTIC_DISPERSION, pathset_paths_df, pathset_links_df, veh_trips_df, reset_bump_iter=False) diff --git a/scripts/compare_output.py b/scripts/compare_output.py index 17b27f40..b885d930 100644 --- a/scripts/compare_output.py +++ b/scripts/compare_output.py @@ -28,9 +28,10 @@ # TODO: this should match the input in question STOCH_DISPERSION = 1.0 -SPLIT_COLS = {"ft_output_loadProfile.txt" :[], - "ft_output_passengerTimes.txt":['arrivalTimes','boardingTimes','alightingTimes'], - "ft_output_passengerPaths.txt":['boardingStops','boardingTrips','alightingStops','walkingTimes']} +SPLIT_COLS = {"ft_output_loadProfile.txt": [], + "ft_output_passengerTimes.txt": ['arrivalTimes', 'boardingTimes', 'alightingTimes'], + "ft_output_passengerPaths.txt": ['boardingStops', 'boardingTrips', 'alightingStops', 'walkingTimes']} + def compare_file(dir1, dir2, filename): """ @@ -48,16 +49,16 @@ def compare_file(dir1, dir2, filename): df1 = pd.read_csv(filename1, sep=sep) df2 = pd.read_csv(filename2, sep=sep) if filename == "ft_output_loadProfile.txt": - index_cols = ['rownum','route_id', 'trip_id', 'direction', 'stop_id'] + index_cols = ['rownum', 'route_id', 'trip_id', 'direction', 'stop_id'] if 'direction' not in df1.columns.values: index_cols.remove('direction') # the keys are not unique since some stops come up twice; add rownum columns - df1['rownum'] = list(range(1, len(df1)+1)) + df1['rownum'] = list(range(1, len(df1) + 1)) df1.set_index(keys=index_cols, inplace=True) - df2['rownum'] = list(range(1, len(df2)+1)) + df2['rownum'] = list(range(1, len(df2) + 1)) df2.set_index(keys=index_cols, inplace=True) else: - df1.set_index(keys=['person_id','trip_list_id_num','mode','originTaz','destinationTaz'], inplace=True) - df2.set_index(keys=['person_id','trip_list_id_num','mode','originTaz','destinationTaz'], inplace=True) + df1.set_index(keys=['person_id', 'trip_list_id_num', 'mode', 'originTaz', 'destinationTaz'], inplace=True) + df2.set_index(keys=['person_id', 'trip_list_id_num', 'mode', 'originTaz', 'destinationTaz'], inplace=True) # startTime needs to be read as a time if 'startTime' in df1.columns.values: @@ -77,14 +78,14 @@ def compare_file(dir1, dir2, filename): if col.endswith('Times'): # these are formatted 11:12:13 - if filename=='ft_output_passengerTimes.txt': + if filename == 'ft_output_passengerTimes.txt': split_df1 = split_df1.applymap(lambda x: fasttrips.Util.read_time(x)) split_df2 = split_df2.applymap(lambda x: fasttrips.Util.read_time(x)) else: split_df1 = split_df1.astype('float') split_df2 = split_df2.astype('float') - rename_cols1 = dict( (k,"%s%s" % (col,str(k))) for k in list(split_df1.columns.values)) - rename_cols2 = dict( (k,"%s%s" % (col,str(k))) for k in list(split_df2.columns.values)) + rename_cols1 = dict((k, "%s%s" % (col, str(k))) for k in list(split_df1.columns.values)) + rename_cols2 = dict((k, "%s%s" % (col, str(k))) for k in list(split_df2.columns.values)) split_df1.rename(columns=rename_cols1, inplace=True) split_df2.rename(columns=rename_cols2, inplace=True) split_df1['num_%s' % col] = split_df1.notnull().sum(axis=1) @@ -92,16 +93,16 @@ def compare_file(dir1, dir2, filename): df1 = pd.concat(objs=[df1, split_df1], axis=1) df2 = pd.concat(objs=[df2, split_df2], axis=1) if len(rename_cols1) < len(rename_cols2): - for k,v in rename_cols2.items(): + for k, v in rename_cols2.items(): if k not in rename_cols1: df1[v] = np.NaN if len(rename_cols2) < len(rename_cols1): - for k,v in rename_cols1.items(): + for k, v in rename_cols1.items(): if k not in rename_cols2: df2[v] = np.NaN FastTripsLogger.info("Read %10d rows from %s" % (len(df1), filename1)) FastTripsLogger.info("Read %10d rows from %s" % (len(df2), filename2)) - df_diff = df1.merge(right=df2, how='outer', left_index=True, right_index=True, suffixes=('_1','_2')) + df_diff = df1.merge(right=df2, how='outer', left_index=True, right_index=True, suffixes=('_1', '_2')) FastTripsLogger.info("Merged %10d rows" % len(df_diff)) # print df1.columns.values @@ -122,10 +123,11 @@ def compare_file(dir1, dir2, filename): df_diff[col1] = df_diff[col1] % 1440 df_diff[col2] = df_diff[col2] % 1440 - coldiff = "%s_diff" % colname - colabsdiff = "%s_absdiff" % colname + coldiff = "%s_diff" % colname + colabsdiff = "%s_absdiff" % colname if df_diff[col1].dtype != df_diff[col2].dtype: - FastTripsLogger.debug("mismatching dtypes for %s and %s: %s vs %s" % (col1, col2, str(df_diff[col1].dtype), str(df_diff[col2].dtype))) + FastTripsLogger.debug("mismatching dtypes for %s and %s: %s vs %s" % ( + col1, col2, str(df_diff[col1].dtype), str(df_diff[col2].dtype))) df_diff[coldiff] = 0 elif str(df_diff[col1].dtype) == 'object': df_diff[coldiff] = ((df_diff[col1] != df_diff[col2]) & (df_diff[col1].notnull() | df_diff[col2].notnull())) @@ -134,38 +136,46 @@ def compare_file(dir1, dir2, filename): df_diff[colabsdiff] = abs(df_diff[coldiff]) # Detailed output - FastTripsLogger.debug("============================================ %s ============================================" % colname) + FastTripsLogger.debug( + "============================================ %s ============================================" % colname) FastTripsLogger.debug(" -- dtypes --\n" + str(df_diff[[col1, col2, coldiff]].dtypes)) FastTripsLogger.debug(" -- head --\n" + str(df_diff[[col1, col2, coldiff]].head()) + "\n") FastTripsLogger.debug(" -- describe --\n" + str(df_diff[[col1, col2, coldiff]].describe()) + "\n") if df_diff[colabsdiff].max() == 0: FastTripsLogger.debug("-- no diffs --") - elif filename=="ft_output_passengerPaths.txt": + elif filename == "ft_output_passengerPaths.txt": FastTripsLogger.debug(" -- diffs --\n" + \ - str(df_diff.reset_index().sort(columns=[colabsdiff, 'trip_list_id_num'], ascending=[False,True]).loc[:,['trip_list_id_num','mode','originTaz','destinationTaz',col1, col2, coldiff, colabsdiff]].head()) + "\n") + str(df_diff.reset_index().sort(columns=[colabsdiff, 'trip_list_id_num'], + ascending=[False, True]).loc[:, + ['trip_list_id_num', 'mode', 'originTaz', 'destinationTaz', col1, col2, coldiff, + colabsdiff]].head()) + "\n") else: FastTripsLogger.debug(" -- diffs --\n" + \ - str(df_diff.sort(columns=colabsdiff, ascending=False).loc[:,[col1, col2, coldiff, colabsdiff]].head()) + "\n") + str(df_diff.sort(columns=colabsdiff, ascending=False).loc[:, + [col1, col2, coldiff, colabsdiff]].head()) + "\n") # Quick output status = "" if df_diff[colabsdiff].max() == 0: status = "Match" elif str(df_diff[col1].dtype) == 'object': - status = "%d/%d objects differ" % (len(df_diff.loc[df_diff[coldiff]==True]), len(df_diff)) + status = "%d/%d objects differ" % (len(df_diff.loc[df_diff[coldiff] == True]), len(df_diff)) elif str(df_diff[col1].dtype)[:3] == 'int': - status = "Values differ by [% 8.2f,% 8.2f] with %d values differing" % (df_diff[coldiff].min(), df_diff[coldiff].max(), len(df_diff.loc[df_diff[coldiff]!=0])) + status = "Values differ by [% 8.2f,% 8.2f] with %d values differing" % ( + df_diff[coldiff].min(), df_diff[coldiff].max(), len(df_diff.loc[df_diff[coldiff] != 0])) elif str(df_diff[col1].dtype) == 'datetime64[ns]': - status = "Values differ by [%s,%s] mins with mean %s" % (str(df_diff[coldiff].min()), str(df_diff[coldiff].max()), str(df_diff[coldiff].mean())) + status = "Values differ by [%s,%s] mins with mean %s" % ( + str(df_diff[coldiff].min()), str(df_diff[coldiff].max()), str(df_diff[coldiff].mean())) else: - status = "Values differ by [% 8.2f,% 8.2f] with mean % 8.2f" % (df_diff[coldiff].min(), df_diff[coldiff].max(), df_diff[coldiff].mean()) + status = "Values differ by [% 8.2f,% 8.2f] with mean % 8.2f" % ( + df_diff[coldiff].min(), df_diff[coldiff].max(), df_diff[coldiff].mean()) FastTripsLogger.info(" %-20s %s" % (colname, status)) - new_cols.extend([col1, col2, coldiff]) # df_diff = df_diff[new_cols] + def compare_pathset(dir1, dir2): """ Compare the pathset to see if the pathset is 'better' @@ -178,27 +188,30 @@ def compare_pathset(dir1, dir2): df1 = pd.read_csv(filename1, sep="\s+") df2 = pd.read_csv(filename2, sep="\s+") - merge_cols = ['iteration','passenger_id_num','trip_list_id_num','path_board_stops','path_trips','path_alight_stops'] + merge_cols = ['iteration', 'passenger_id_num', 'trip_list_id_num', 'path_board_stops', 'path_trips', + 'path_alight_stops'] # outer join - df_diff = df1.merge(right=df2, how='outer', on=merge_cols, suffixes=('_1','_2')) + df_diff = df1.merge(right=df2, how='outer', on=merge_cols, suffixes=('_1', '_2')) # make a single path_cost column - df_diff['path_cost'] = df_diff[['path_cost_1','path_cost_2']].min(axis=1) + df_diff['path_cost'] = df_diff[['path_cost_1', 'path_cost_2']].min(axis=1) # create probabilities based on the union of the path sets - df_diff['exp_util'] = np.exp(-1.0*STOCH_DISPERSION*df_diff['path_cost']) + df_diff['exp_util'] = np.exp(-1.0 * STOCH_DISPERSION * df_diff['path_cost']) # aggregate it to each person trip df_diff['num total paths'] = 1 - df_diff_counts = df_diff.groupby(['iteration','passenger_id_num','trip_list_id_num']).agg({'num total paths':'count', 'exp_util':'sum'}) - df_diff_counts.rename(columns={'exp_util':'sum_exp_util'}, inplace=True) + df_diff_counts = df_diff.groupby(['iteration', 'passenger_id_num', 'trip_list_id_num']).agg( + {'num total paths': 'count', 'exp_util': 'sum'}) + df_diff_counts.rename(columns={'exp_util': 'sum_exp_util'}, inplace=True) # join it back to get the probability given a union pathset - df_diff = df_diff.merge(df_diff_counts.reset_index()[['iteration','passenger_id_num','trip_list_id_num','sum_exp_util']], how='left') - df_diff['union pathset probability'] = df_diff['exp_util']/df_diff['sum_exp_util'] + df_diff = df_diff.merge( + df_diff_counts.reset_index()[['iteration', 'passenger_id_num', 'trip_list_id_num', 'sum_exp_util']], how='left') + df_diff['union pathset probability'] = df_diff['exp_util'] / df_diff['sum_exp_util'] # drop these - df_diff.drop(['exp_util','sum_exp_util'], axis=1, inplace=True) + df_diff.drop(['exp_util', 'sum_exp_util'], axis=1, inplace=True) df_diff_counts.drop(['sum_exp_util'], axis=1, inplace=True) # write the joined one @@ -207,21 +220,27 @@ def compare_pathset(dir1, dir2): FastTripsLogger.info("Wrote joined pathset diff info to %s" % join_filename) # look at the nulls - df1_only = df_diff.loc[pd.isnull(df_diff.path_cost_2)].groupby(['iteration','passenger_id_num','trip_list_id_num']).agg({'union pathset probability':'max','path_cost_1':'count'}) - df1_only.rename(columns={'union pathset probability':'max prob missing from file2', - 'path_cost_1':'num paths missing from file2'}, inplace=True) - df2_only = df_diff.loc[pd.isnull(df_diff.path_cost_1)].groupby(['iteration','passenger_id_num','trip_list_id_num']).agg({'union pathset probability':'max','path_cost_2':'count'}) - df2_only.rename(columns={'union pathset probability':'max prob missing from file1', - 'path_cost_2':'num paths missing from file1'}, inplace=True) + df1_only = df_diff.loc[pd.isnull(df_diff.path_cost_2)].groupby( + ['iteration', 'passenger_id_num', 'trip_list_id_num']).agg( + {'union pathset probability': 'max', 'path_cost_1': 'count'}) + df1_only.rename(columns={'union pathset probability': 'max prob missing from file2', + 'path_cost_1': 'num paths missing from file2'}, inplace=True) + df2_only = df_diff.loc[pd.isnull(df_diff.path_cost_1)].groupby( + ['iteration', 'passenger_id_num', 'trip_list_id_num']).agg( + {'union pathset probability': 'max', 'path_cost_2': 'count'}) + df2_only.rename(columns={'union pathset probability': 'max prob missing from file1', + 'path_cost_2': 'num paths missing from file1'}, inplace=True) df_diff_summary = df_diff_counts.merge(df1_only, how='left', left_index=True, right_index=True) df_diff_summary = df_diff_summary.merge(df2_only, how='left', left_index=True, right_index=True) # note paths for which we didn't find ANY in one or the other run df_diff_summary['only in file1'] = 0 - df_diff_summary.loc[df_diff_summary['num paths missing from file2']==df_diff_summary['num total paths'],'only in file1'] = 1 + df_diff_summary.loc[ + df_diff_summary['num paths missing from file2'] == df_diff_summary['num total paths'], 'only in file1'] = 1 df_diff_summary['only in file2'] = 0 - df_diff_summary.loc[df_diff_summary['num paths missing from file1']==df_diff_summary['num total paths'],'only in file2'] = 1 + df_diff_summary.loc[ + df_diff_summary['num paths missing from file1'] == df_diff_summary['num total paths'], 'only in file2'] = 1 # NaN means zero df_diff_summary.loc[pd.isnull(df_diff_summary['num paths missing from file1']), 'num paths missing from file1'] = 0 df_diff_summary.loc[pd.isnull(df_diff_summary['num paths missing from file2']), 'num paths missing from file2'] = 0 @@ -232,30 +251,43 @@ def compare_pathset(dir1, dir2): FastTripsLogger.info("Wrote detailed pathset diff info to %s" % detail_file) # Report - FastTripsLogger.info(" Average pathset size: %.1f" % df_diff_summary['num total paths'].mean()) + FastTripsLogger.info( + " Average pathset size: %.1f" % df_diff_summary['num total paths'].mean()) FastTripsLogger.info(" Trips with paths ONLY in pathset 1: %d" % df_diff_summary['only in file1'].sum()) FastTripsLogger.debug(" -- diffs --\n" + \ - str(df_diff_summary.loc[df_diff_summary['only in file1']==1]) + "\n") + str(df_diff_summary.loc[df_diff_summary['only in file1'] == 1]) + "\n") FastTripsLogger.info(" Trips with paths ONLY in pathset 2: %d" % df_diff_summary['only in file2'].sum()) FastTripsLogger.debug(" -- diffs --\n" + \ - str(df_diff_summary.loc[df_diff_summary['only in file2']==1]) + "\n") - - FastTripsLogger.info(" Average paths missing from pathset 1: %.1f" % df_diff_summary['num paths missing from file1'].mean()) - FastTripsLogger.info(" Max probability missing from pathset 1: %.3f%%" % (100*df_diff_summary['max prob missing from file1'].max())) - FastTripsLogger.info(" # trips w/ paths>10%% missing from pathset 1: %d" % len(df_diff_summary.loc[df_diff_summary['max prob missing from file1']>0.10])) - FastTripsLogger.info(" # trips w/ paths> 1%% missing from pathset 1: %d" % len(df_diff_summary.loc[df_diff_summary['max prob missing from file1']>0.01])) - temp_df = df_diff_summary[['max prob missing from file1','num paths missing from file1']].reset_index().sort_values(by=['max prob missing from file1', 'trip_list_id_num'], ascending=[False,True]) + str(df_diff_summary.loc[df_diff_summary['only in file2'] == 1]) + "\n") + + FastTripsLogger.info( + " Average paths missing from pathset 1: %.1f" % df_diff_summary['num paths missing from file1'].mean()) + FastTripsLogger.info(" Max probability missing from pathset 1: %.3f%%" % ( + 100 * df_diff_summary['max prob missing from file1'].max())) + FastTripsLogger.info(" # trips w/ paths>10%% missing from pathset 1: %d" % len( + df_diff_summary.loc[df_diff_summary['max prob missing from file1'] > 0.10])) + FastTripsLogger.info(" # trips w/ paths> 1%% missing from pathset 1: %d" % len( + df_diff_summary.loc[df_diff_summary['max prob missing from file1'] > 0.01])) + temp_df = df_diff_summary[ + ['max prob missing from file1', 'num paths missing from file1']].reset_index().sort_values( + by=['max prob missing from file1', 'trip_list_id_num'], ascending=[False, True]) FastTripsLogger.debug(" trips w/ paths> 1%% missing from pathset 1\n%s\n" % - str(temp_df.loc[temp_df['max prob missing from file1']>0.01])) - - FastTripsLogger.info(" Average paths missing from pathset 2: %.1f" % df_diff_summary['num paths missing from file2'].mean()) - FastTripsLogger.info(" Max probability missing from pathset 2: %.3f%%" % (100*df_diff_summary['max prob missing from file2'].max())) - FastTripsLogger.info(" # trips w/ paths>10%% missing from pathset 2: %d" % len(df_diff_summary.loc[df_diff_summary['max prob missing from file2']>0.10])) - FastTripsLogger.info(" # trips w/ paths> 1%% missing from pathset 2: %d" % len(df_diff_summary.loc[df_diff_summary['max prob missing from file2']>0.01])) - temp_df = df_diff_summary[['max prob missing from file2','num paths missing from file2']].reset_index().sort_values(by=['max prob missing from file2', 'trip_list_id_num'], ascending=[False,True]) + str(temp_df.loc[temp_df['max prob missing from file1'] > 0.01])) + + FastTripsLogger.info( + " Average paths missing from pathset 2: %.1f" % df_diff_summary['num paths missing from file2'].mean()) + FastTripsLogger.info(" Max probability missing from pathset 2: %.3f%%" % ( + 100 * df_diff_summary['max prob missing from file2'].max())) + FastTripsLogger.info(" # trips w/ paths>10%% missing from pathset 2: %d" % len( + df_diff_summary.loc[df_diff_summary['max prob missing from file2'] > 0.10])) + FastTripsLogger.info(" # trips w/ paths> 1%% missing from pathset 2: %d" % len( + df_diff_summary.loc[df_diff_summary['max prob missing from file2'] > 0.01])) + temp_df = df_diff_summary[ + ['max prob missing from file2', 'num paths missing from file2']].reset_index().sort_values( + by=['max prob missing from file2', 'trip_list_id_num'], ascending=[False, True]) FastTripsLogger.debug(" trips w/ paths> 1%% missing from pathset 2\n%s\n" % - str(temp_df.loc[temp_df['max prob missing from file2']>0.01])) + str(temp_df.loc[temp_df['max prob missing from file2'] > 0.01])) def compare_performance(dir1, dir2): @@ -271,16 +303,19 @@ def compare_performance(dir1, dir2): df2 = pd.read_csv(filename2, sep="\t") # drop the text-y ones - df1.drop([fasttrips.Performance.PERFORMANCE_COLUMN_TIME_LABELING, fasttrips.Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING], axis=1, inplace=True) - df2.drop([fasttrips.Performance.PERFORMANCE_COLUMN_TIME_LABELING, fasttrips.Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING], axis=1, inplace=True) + df1.drop([fasttrips.Performance.PERFORMANCE_COLUMN_TIME_LABELING, + fasttrips.Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING], axis=1, inplace=True) + df2.drop([fasttrips.Performance.PERFORMANCE_COLUMN_TIME_LABELING, + fasttrips.Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING], axis=1, inplace=True) - df_perf_diff = df1.merge(right=df2, how='outer', on=['iteration','trip_list_id_num'], suffixes=('_1','_2')) + df_perf_diff = df1.merge(right=df2, how='outer', on=['iteration', 'trip_list_id_num'], suffixes=('_1', '_2')) # write the joined one join_filename = os.path.join(dir1, "ft_compare_performance.csv") df_perf_diff.to_csv(join_filename, sep=",", index=False) FastTripsLogger.info("Wrote joined performance info to %s" % join_filename) + if __name__ == "__main__": if len(sys.argv) != 3: diff --git a/scripts/create_tableau_path_map.py b/scripts/create_tableau_path_map.py index 591a11ed..e445ddcd 100644 --- a/scripts/create_tableau_path_map.py +++ b/scripts/create_tableau_path_map.py @@ -5,42 +5,43 @@ import fasttrips - USAGE = r""" Creates a tableau file with points for pathset paths. """ + def add_taz_coords(network_dir, pathset_links_df): fasttrips.FastTripsLogger.info("Adding TAZ coordinates") # just need the taz coords taz_coords_file = os.path.join(network_dir, "taz_coords.txt") - taz_coords_df = pd.read_csv(taz_coords_file, dtype={"taz":object}) + taz_coords_df = pd.read_csv(taz_coords_file, dtype={"taz": object}) fasttrips.FastTripsLogger.debug("taz_coords_df=\n%s" % str(taz_coords_df.head())) # join to links - pathset_links_df = pd.merge(left =pathset_links_df, - left_on ="A_id", - right =taz_coords_df, - right_on="taz", - how ="left") + pathset_links_df = pd.merge(left=pathset_links_df, + left_on="A_id", + right=taz_coords_df, + right_on="taz", + how="left") # get lat, lon for access links - pathset_links_df.loc[pathset_links_df["linkmode"]=="access", "A_lat"] = pathset_links_df["lat"] - pathset_links_df.loc[pathset_links_df["linkmode"]=="access", "A_lon"] = pathset_links_df["lon"] - pathset_links_df.drop(["taz","lat","lon"], axis=1, inplace=True) - - pathset_links_df = pd.merge(left =pathset_links_df, - left_on ="B_id", - right =taz_coords_df, - right_on="taz", - how ="left") + pathset_links_df.loc[pathset_links_df["linkmode"] == "access", "A_lat"] = pathset_links_df["lat"] + pathset_links_df.loc[pathset_links_df["linkmode"] == "access", "A_lon"] = pathset_links_df["lon"] + pathset_links_df.drop(["taz", "lat", "lon"], axis=1, inplace=True) + + pathset_links_df = pd.merge(left=pathset_links_df, + left_on="B_id", + right=taz_coords_df, + right_on="taz", + how="left") # get lat, lon for egress links - pathset_links_df.loc[pathset_links_df["linkmode"]=="egress", "B_lat"] = pathset_links_df["lat"] - pathset_links_df.loc[pathset_links_df["linkmode"]=="egress", "B_lon"] = pathset_links_df["lon"] - pathset_links_df.drop(["taz","lat","lon"], axis=1, inplace=True) + pathset_links_df.loc[pathset_links_df["linkmode"] == "egress", "B_lat"] = pathset_links_df["lat"] + pathset_links_df.loc[pathset_links_df["linkmode"] == "egress", "B_lon"] = pathset_links_df["lon"] + pathset_links_df.drop(["taz", "lat", "lon"], axis=1, inplace=True) return pathset_links_df + def add_stop_coords(ft, pathset_links_df): fasttrips.FastTripsLogger.info("Adding stop coordinates") # add stop lats and lons if they're not there @@ -53,59 +54,66 @@ def add_stop_coords(ft, pathset_links_df): new_lat_colname="stop_B_lat", new_lon_colname="stop_B_lon") # not valid for access A_id and egress B_id - pathset_links_df.loc[pathset_links_df["linkmode"]!="access", "A_lat"] = pathset_links_df["stop_A_lat"] - pathset_links_df.loc[pathset_links_df["linkmode"]!="access", "A_lon"] = pathset_links_df["stop_A_lon"] - pathset_links_df.loc[pathset_links_df["linkmode"]!="egress", "B_lat"] = pathset_links_df["stop_B_lat"] - pathset_links_df.loc[pathset_links_df["linkmode"]!="egress", "B_lon"] = pathset_links_df["stop_B_lon"] - pathset_links_df.drop(["stop_A_lat","stop_A_lon", - "stop_B_lat","stop_B_lon"], axis=1, inplace=True) + pathset_links_df.loc[pathset_links_df["linkmode"] != "access", "A_lat"] = pathset_links_df["stop_A_lat"] + pathset_links_df.loc[pathset_links_df["linkmode"] != "access", "A_lon"] = pathset_links_df["stop_A_lon"] + pathset_links_df.loc[pathset_links_df["linkmode"] != "egress", "B_lat"] = pathset_links_df["stop_B_lat"] + pathset_links_df.loc[pathset_links_df["linkmode"] != "egress", "B_lon"] = pathset_links_df["stop_B_lon"] + pathset_links_df.drop(["stop_A_lat", "stop_A_lon", + "stop_B_lat", "stop_B_lon"], axis=1, inplace=True) return pathset_links_df + def impute_nostop_transit_link_stops(ft, pathset_links_df, veh_trips_df, prev_next): """ Tries to fill in A_id, A_seq, A_lat, A_lon, B_id, B_seq, B_lat, B_lon, for transit links that have none of those filled in. """ fasttrips.FastTripsLogger.info("Imputing nostop transit link stops using %s link" % prev_next) - fasttrips.FastTripsLogger.debug("impute_nostop_transit_link_stops: veh_trips_df head=\n%s" % str(veh_trips_df.head())) - - prevnext_links_df = pathset_links_df[["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum","linkmode", - "A_id","A_id_num","A_lat","A_lon", - "B_id","B_id_num","B_lat","B_lon"]].copy() - if prev_next=="prev": - prevnext_links_df["linknum"] = prevnext_links_df["linknum"]+1 - elif prev_next=="next": - prevnext_links_df["linknum"] = prevnext_links_df["linknum"]-1 - else: # this shouldn't happen + fasttrips.FastTripsLogger.debug( + "impute_nostop_transit_link_stops: veh_trips_df head=\n%s" % str(veh_trips_df.head())) + + prevnext_links_df = pathset_links_df[ + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum", "linkmode", + "A_id", "A_id_num", "A_lat", "A_lon", + "B_id", "B_id_num", "B_lat", "B_lon"]].copy() + if prev_next == "prev": + prevnext_links_df["linknum"] = prevnext_links_df["linknum"] + 1 + elif prev_next == "next": + prevnext_links_df["linknum"] = prevnext_links_df["linknum"] - 1 + else: # this shouldn't happen raise # get next or prev link information on the link - pathset_links_df = pd.merge(left =pathset_links_df, - right =prevnext_links_df, - how ="left", - on =["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"], - suffixes=["","_%s" % prev_next]) + pathset_links_df = pd.merge(left=pathset_links_df, + right=prevnext_links_df, + how="left", + on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum", "linknum"], + suffixes=["", "_%s" % prev_next]) # target links to imput nodes find_near_df = None - AB = None - if prev_next=="prev": + AB = None + if prev_next == "prev": AB = "A" # this is what we're setting # A_id: if the previous link is access and the B_id is null, get the A_id (taz) and find a stop_id close to it # A_id: if the previous link is transfer and the B_id is null, get the A_id and find a stop_id close to it - find_near_df = pathset_links_df.loc[ ((pathset_links_df["linkmode_prev"]=="access")|(pathset_links_df["linkmode_prev"]=="transfer"))& - pd.isnull(pathset_links_df["A_id"] )& - pd.isnull(pathset_links_df["B_id_prev"])& - pd.notnull(pathset_links_df["A_id_prev"]) ] + find_near_df = pathset_links_df.loc[ + ((pathset_links_df["linkmode_prev"] == "access") | (pathset_links_df["linkmode_prev"] == "transfer")) & + pd.isnull(pathset_links_df["A_id"]) & + pd.isnull(pathset_links_df["B_id_prev"]) & + pd.notnull(pathset_links_df["A_id_prev"])] fasttrips.FastTripsLogger.info("Imputing A_id from previous link for %d links" % len(find_near_df)) else: AB = "B" # this is what we're setting # B_id: if the next link is transfer and A_id is null, get the B_id and find a stop_id close to it # B_id: if the next link is egress and the A_id is null, get the B_id (taz) and find a stop_id close to it - find_near_df = pathset_links_df.loc[ ((pathset_links_df["linkmode_next"]=="egress")|(pathset_links_df["linkmode_next"]=="transfer"))& - pd.isnull(pathset_links_df["B_id"] )& - pd.isnull(pathset_links_df["A_id_next"])& - pd.notnull(pathset_links_df["B_id_next"]) ] + find_near_df = pathset_links_df.loc[ + ((pathset_links_df["linkmode_next"] == "egress") | (pathset_links_df["linkmode_next"] == "transfer")) & + pd.isnull(pathset_links_df["B_id"]) & + pd.isnull(pathset_links_df["A_id_next"]) & + pd.notnull(pathset_links_df["B_id_next"])] fasttrips.FastTripsLogger.info("Imputing B_id from next link for %d links" % len(find_near_df)) # nothing to do @@ -118,18 +126,22 @@ def impute_nostop_transit_link_stops(ft, pathset_links_df, veh_trips_df, prev_ne impute_count = 0 for service_id in service_id_list: - impute_id = find_near_df.loc[ find_near_df["service_id"]==service_id] - fasttrips.FastTripsLogger.debug("Processing service_id [%s] with %d possible imputes" % (service_id, len(impute_id))) + impute_id = find_near_df.loc[find_near_df["service_id"] == service_id] + fasttrips.FastTripsLogger.debug( + "Processing service_id [%s] with %d possible imputes" % (service_id, len(impute_id))) - service_vehicle_stops = veh_trips_df.loc[ veh_trips_df["service_id"] == service_id, ["service_id","mode","stop_id","stop_id_num","stop_name","stop_lat","stop_lon"]].drop_duplicates() + service_vehicle_stops = veh_trips_df.loc[ + veh_trips_df["service_id"] == service_id, ["service_id", "mode", "stop_id", "stop_id_num", "stop_name", + "stop_lat", "stop_lon"]].drop_duplicates() fasttrips.FastTripsLogger.debug("impute_id len=%d head()=\n%s" % (len(impute_id), str(impute_id.head()))) - fasttrips.FastTripsLogger.debug("service_vehicle_stops len=%d head()=\n%s" % (len(service_vehicle_stops), str(service_vehicle_stops.head()))) + fasttrips.FastTripsLogger.debug("service_vehicle_stops len=%d head()=\n%s" % ( + len(service_vehicle_stops), str(service_vehicle_stops.head()))) - near_stops_df = pd.merge(left =impute_id, - right=service_vehicle_stops, - on =["service_id","mode"], - how ="left") + near_stops_df = pd.merge(left=impute_id, + right=service_vehicle_stops, + on=["service_id", "mode"], + how="left") # no join success -- nothing we can do if pd.notnull(near_stops_df["stop_id"]).sum() == 0: fasttrips.FastTripsLogger.info("Imputing %6d out of %6d stops for %s" % (0, len(impute_id), service_id)) @@ -137,45 +149,59 @@ def impute_nostop_transit_link_stops(ft, pathset_links_df, veh_trips_df, prev_ne else: # calculate the distance from [person_prefix]_lat, [person_prefix]_lon, and the stop fasttrips.Util.calculate_distance_miles(near_stops_df, - origin_lat ="%s_lat_%s" % (AB, prev_next), - origin_lon ="%s_lon_%s" % (AB, prev_next), - destination_lat ="stop_lat", - destination_lon ="stop_lon", + origin_lat="%s_lat_%s" % (AB, prev_next), + origin_lon="%s_lon_%s" % (AB, prev_next), + destination_lat="stop_lat", + destination_lon="stop_lon", distance_colname="stop_dist") - fasttrips.FastTripsLogger.debug("near_stops_df len=%d head()=\n%s" % (len(near_stops_df), str(near_stops_df.head()))) + fasttrips.FastTripsLogger.debug( + "near_stops_df len=%d head()=\n%s" % (len(near_stops_df), str(near_stops_df.head()))) # pick the closest one - near_stops_df = near_stops_df.loc[ near_stops_df.groupby(["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"])["stop_dist"].idxmin(), - ["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum","stop_id","stop_id_num","stop_name","stop_lat","stop_lon","stop_dist"] ] + near_stops_df = near_stops_df.loc[near_stops_df.groupby( + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum"])["stop_dist"].idxmin(), + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum", "linknum", "stop_id", "stop_id_num", + "stop_name", "stop_lat", "stop_lon", "stop_dist"]] - fasttrips.FastTripsLogger.debug("near_stops_df len=%d head()=\n%s" % (len(near_stops_df), str(near_stops_df.head()))) + fasttrips.FastTripsLogger.debug( + "near_stops_df len=%d head()=\n%s" % (len(near_stops_df), str(near_stops_df.head()))) # set it into pathset_links_df pathset_links_df = pd.merge(left=pathset_links_df, - right=near_stops_df, - on=["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"], - how="left", - suffixes=["","_near"]) - fasttrips.FastTripsLogger.debug("pathset_links_df head=\n%s" % str(pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"])].head())) + right=near_stops_df, + on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum", "linknum"], + how="left", + suffixes=["", "_near"]) + fasttrips.FastTripsLogger.debug("pathset_links_df head=\n%s" % str( + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"])].head())) # set it - fasttrips.FastTripsLogger.info("Imputing %6d out of %6d stops for %s" % (pd.notnull(pathset_links_df["stop_id"]).sum(), len(impute_id), service_id)) + fasttrips.FastTripsLogger.info("Imputing %6d out of %6d stops for %s" % ( + pd.notnull(pathset_links_df["stop_id"]).sum(), len(impute_id), service_id)) impute_count += pd.notnull(pathset_links_df["stop_id"]).sum() - pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"]), "%s_id" % AB] = pathset_links_df["stop_id" ] - pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"]), "%s_id_num" % AB] = pathset_links_df["stop_id_num"] - pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"]), "%s_lat" % AB] = pathset_links_df["stop_lat" ] - pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"]), "%s_lon" % AB] = pathset_links_df["stop_lon" ] - pathset_links_df.loc[ pd.notnull(pathset_links_df["stop_id"]), "%s_impute" % AB] = True + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"]), "%s_id" % AB] = pathset_links_df["stop_id"] + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"]), "%s_id_num" % AB] = pathset_links_df[ + "stop_id_num"] + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"]), "%s_lat" % AB] = pathset_links_df["stop_lat"] + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"]), "%s_lon" % AB] = pathset_links_df["stop_lon"] + pathset_links_df.loc[pd.notnull(pathset_links_df["stop_id"]), "%s_impute" % AB] = True # we're done with these fields - pathset_links_df.drop(["stop_id","stop_id_num","stop_name","stop_lat","stop_lon","stop_dist"], axis=1, inplace=True) + pathset_links_df.drop(["stop_id", "stop_id_num", "stop_name", "stop_lat", "stop_lon", "stop_dist"], axis=1, + inplace=True) fasttrips.FastTripsLogger.info("=> Imputed %6d total" % impute_count) # drop the prev columns pathset_links_df.drop(["linkmode_%s" % prev_next, - "A_id_%s" % prev_next,"A_id_num_%s" % prev_next,"A_lat_%s" % prev_next,"A_lon_%s" % prev_next, - "B_id_%s" % prev_next,"B_id_num_%s" % prev_next,"B_lat_%s" % prev_next,"B_lon_%s" % prev_next], axis=1, inplace=True) + "A_id_%s" % prev_next, "A_id_num_%s" % prev_next, "A_lat_%s" % prev_next, + "A_lon_%s" % prev_next, + "B_id_%s" % prev_next, "B_id_num_%s" % prev_next, "B_lat_%s" % prev_next, + "B_lon_%s" % prev_next], axis=1, inplace=True) return pathset_links_df + def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): """ If prev_next is "next", sets B from next A. @@ -183,10 +209,10 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): """ AB_set = None # setting this AB_use = None # using this - if prev_next=="next": + if prev_next == "next": AB_set = "B" AB_use = "A" - elif prev_next=="prev": + elif prev_next == "prev": AB_set = "A" AB_use = "B" else: @@ -197,46 +223,61 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): # fill in unknown Bs from next link fasttrips.FastTripsLogger.info("Trying to impute %8d null %s_id values" % (target_count, AB_set)) - prevnext_links_df = pathset_links_df[["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum", - "%s_id" % AB_use,"%s_id_num" % AB_use, "%s_lat" % AB_use, "%s_lon" % AB_use]].copy() - if prev_next=="next": - prevnext_links_df["linknum"] = prevnext_links_df["linknum"]-1 + prevnext_links_df = pathset_links_df[ + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum", + "%s_id" % AB_use, "%s_id_num" % AB_use, "%s_lat" % AB_use, "%s_lon" % AB_use]].copy() + if prev_next == "next": + prevnext_links_df["linknum"] = prevnext_links_df["linknum"] - 1 else: - prevnext_links_df["linknum"] = prevnext_links_df["linknum"]+1 + prevnext_links_df["linknum"] = prevnext_links_df["linknum"] + 1 - pathset_links_df = pd.merge(left =pathset_links_df, - right =prevnext_links_df, - how ="left", - on =["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"], - suffixes=["","_%s" % prev_next]) + pathset_links_df = pd.merge(left=pathset_links_df, + right=prevnext_links_df, + how="left", + on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum", "linknum"], + suffixes=["", "_%s" % prev_next]) # inpute the AB_set - impute_count = len(pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)])]) - pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_impute" % AB_set] = True - pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_id_num" % AB_set] = pathset_links_df["%s_id_num_%s" % (AB_use, prev_next)] - pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_lat" % AB_set] = pathset_links_df["%s_lat_%s" % (AB_use, prev_next)] - pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_lon" % AB_set] = pathset_links_df["%s_lon_%s" % (AB_use, prev_next)] - pathset_links_df.loc[ pd.isnull(pathset_links_df["%s_id" % AB_set])&pd.notnull(pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_id" % AB_set] = pathset_links_df["%s_id_%s" % (AB_use, prev_next)] + impute_count = len(pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)])]) + pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_impute" % AB_set] = True + pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_id_num" % AB_set] = pathset_links_df[ + "%s_id_num_%s" % (AB_use, prev_next)] + pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_lat" % AB_set] = pathset_links_df[ + "%s_lat_%s" % (AB_use, prev_next)] + pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_lon" % AB_set] = pathset_links_df[ + "%s_lon_%s" % (AB_use, prev_next)] + pathset_links_df.loc[pd.isnull(pathset_links_df["%s_id" % AB_set]) & pd.notnull( + pathset_links_df["%s_id_%s" % (AB_use, prev_next)]), "%s_id" % AB_set] = pathset_links_df[ + "%s_id_%s" % (AB_use, prev_next)] # done - pathset_links_df.drop(["%s_id_%s" % (AB_use, prev_next), + pathset_links_df.drop(["%s_id_%s" % (AB_use, prev_next), "%s_id_num_%s" % (AB_use, prev_next), - "%s_lat_%s" % (AB_use, prev_next), - "%s_lon_%s" % (AB_use, prev_next)], axis=1, inplace=True) + "%s_lat_%s" % (AB_use, prev_next), + "%s_lon_%s" % (AB_use, prev_next)], axis=1, inplace=True) - fasttrips.FastTripsLogger.info("Imputed %8d values for %s_id => Have %8d null %s_id values" % (impute_count, AB_set, pd.isnull(pathset_links_df["%s_id" % AB_set]).sum(), AB_set)) + fasttrips.FastTripsLogger.info("Imputed %8d values for %s_id => Have %8d null %s_id values" % ( + impute_count, AB_set, pd.isnull(pathset_links_df["%s_id" % AB_set]).sum(), AB_set)) return pathset_links_df + if __name__ == "__main__": - pd.set_option('display.width', 1000) - pd.set_option('display.max_rows', 1000) + pd.set_option('display.width', 1000) + pd.set_option('display.max_rows', 1000) pd.set_option('display.max_columns', 100) parser = argparse.ArgumentParser(description=USAGE) parser.add_argument('input_network_dir', type=str, nargs=1, help="Directory with input network") - parser.add_argument('input_path_dir', type=str, nargs=1, help="Directory with pathset_[links,paths].csv files") - parser.add_argument('--description', dest='use_description', action='store_true', - help="Specify this to use path description as ID. Otherwise will use standard pathset fields (person_id, person_trip_id, iteration, pathfinding_iteration, simulation_iteration, pathnum)") + parser.add_argument('input_path_dir', type=str, nargs=1, help="Directory with pathset_[links,paths].csv files") + parser.add_argument('--description', dest='use_description', action='store_true', + help="Specify this to use path description as ID. Otherwise will use standard pathset fields (person_id, person_trip_id, iteration, pathfinding_iteration, simulation_iteration, pathnum)") args = parser.parse_args() LOG_DIR = "create_tableau_path_map" @@ -245,13 +286,14 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): os.mkdir(LOG_DIR) # Use fasttrips to read the network - ft = fasttrips.FastTrips(input_network_dir= args.input_network_dir[0], - input_demand_dir = None, - output_dir = LOG_DIR) + ft = fasttrips.FastTrips(input_network_dir=args.input_network_dir[0], + input_demand_dir=None, + output_dir=LOG_DIR) ft.read_input_files() # and the pathset files - (pathset_paths_df, pathset_links_df) = ft.passengers.read_passenger_pathsets(args.input_path_dir[0], ft.stops, ft.routes.modes_df) + (pathset_paths_df, pathset_links_df) = ft.passengers.read_passenger_pathsets(args.input_path_dir[0], ft.stops, + ft.routes.modes_df) fasttrips.Assignment.TRACE_PERSON_IDS = pathset_paths_df["person_id"].head(5).tolist() fasttrips.FastTripsLogger.debug("Read passenger pathsets. head=\n%s" % str(pathset_links_df.head(100))) @@ -283,7 +325,8 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): # first, try to impute transit links for which nothing is known veh_trips_df = ft.trips.get_full_trips() - veh_trips_df = ft.stops.add_stop_lat_lon(veh_trips_df, id_colname="stop_id", new_lat_colname="stop_lat", new_lon_colname="stop_lon", new_stop_name_colname="stop_name") + veh_trips_df = ft.stops.add_stop_lat_lon(veh_trips_df, id_colname="stop_id", new_lat_colname="stop_lat", + new_lon_colname="stop_lon", new_stop_name_colname="stop_name") pathset_links_df = impute_nostop_transit_link_stops(ft, pathset_links_df, veh_trips_df, "prev") pathset_links_df = impute_nostop_transit_link_stops(ft, pathset_links_df, veh_trips_df, "next") @@ -292,25 +335,26 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): pathset_links_df = impute_stop_from_adjacent_stop(ft, pathset_links_df, "next") # split the pathset links into component bits -- only the ones with trip_ids - pathset_links_trip = pathset_links_df.loc[pd.notnull(pathset_links_df["trip_id"])].copy() + pathset_links_trip = pathset_links_df.loc[pd.notnull(pathset_links_df["trip_id"])].copy() pathset_links_notrip = pathset_links_df.loc[pd.isnull(pathset_links_df["trip_id"])] - fasttrips.FastTripsLogger.info("Splitting pathset_links_df (%d) into links with trip_id (%d) and links without (%d)" % - (len(pathset_links_df), len(pathset_links_trip), len(pathset_links_notrip))) + fasttrips.FastTripsLogger.info( + "Splitting pathset_links_df (%d) into links with trip_id (%d) and links without (%d)" % + (len(pathset_links_df), len(pathset_links_trip), len(pathset_links_notrip))) if len(pathset_links_trip) == 0: pathset_links_df = pathset_links_notrip else: - pathset_links_trip = fasttrips.PathSet.split_transit_links(pathset_links_trip, veh_trips_df, ft.stops) + pathset_links_trip = fasttrips.PathSet.split_transit_links(pathset_links_trip, veh_trips_df, ft.stops) fasttrips.FastTripsLogger.info("Split links with trip_id into parts => %d links" % len(pathset_links_trip)) - pathset_links_df = pathset_links_trip.append(pathset_links_notrip, ignore_index=True) + pathset_links_df = pathset_links_trip.append(pathset_links_notrip, ignore_index=True) fasttrips.FastTripsLogger.info("Back together to make %d pathset_links_df" % len(pathset_links_df)) - pathset_links_df.sort_values(by=["person_id","person_trip_id","pathnum","linknum"], inplace=True) + pathset_links_df.sort_values(by=["person_id", "person_trip_id", "pathnum", "linknum"], inplace=True) pathset_links_df.reset_index(drop=True, inplace=True) fasttrips.FastTripsLogger.debug("pathset_links_df.head(100)=\n%s" % str(pathset_links_df.head(100))) - missing_lat_lon = pathset_links_df.loc[pd.isnull(pathset_links_df["A_lat"])|pd.isnull(pathset_links_df["B_lat"])] + missing_lat_lon = pathset_links_df.loc[pd.isnull(pathset_links_df["A_lat"]) | pd.isnull(pathset_links_df["B_lat"])] if len(missing_lat_lon) > 0: fasttrips.FastTripsLogger.info("Missing %d lat/lons" % len(missing_lat_lon)) @@ -320,17 +364,19 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): fasttrips.FastTripsLogger.debug("Wrote %s" % debug_file) # select out just the fields we want for the map - map_link_fields = ["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum", - "linknum","linkmode","mode","route_id","trip_id", - "A_id","A_seq","A_lat","A_lon", - "B_id","B_seq","B_lat","B_lon"] + map_link_fields = ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", + "pathnum", + "linknum", "linkmode", "mode", "route_id", "trip_id", + "A_id", "A_seq", "A_lat", "A_lon", + "B_id", "B_seq", "B_lat", "B_lon"] pathset_links_df = pathset_links_df[map_link_fields] # drop the B fields map_points_df = pathset_links_df[map_link_fields[:-4]].copy() - pathset_paths_df = pathset_paths_df[["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum", - "description"]] + pathset_paths_df = pathset_paths_df[ + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "description"]] pathset_paths_df[["pathnum"]] = pathset_paths_df[["pathnum"]].astype(int) # Each link will be it's own map line. A split link is a single map line but with a bunch of points. @@ -338,41 +384,49 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): # So the line index is "person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum" # add point id starting with 1 for A; the B point IS is just that plus one - map_points_df["point_id"] = map_points_df.groupby(["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"]).cumcount() + 1 + map_points_df["point_id"] = map_points_df.groupby( + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum"]).cumcount() + 1 # rename A_id, A_seq, A_lat, A_lon - map_points_df.rename(columns={"A_id" :"stop_or_taz_id", - "A_seq":"stop_sequence", - "A_lat":"latitude", - "A_lon":"longitude"}, inplace=True) + map_points_df.rename(columns={"A_id": "stop_or_taz_id", + "A_seq": "stop_sequence", + "A_lat": "latitude", + "A_lon": "longitude"}, inplace=True) - fasttrips.FastTripsLogger.debug("map_points_df head=\n%s\ntail=\n%s" % (str(map_points_df.head(30)), str(map_points_df.tail(30)))) + fasttrips.FastTripsLogger.debug( + "map_points_df head=\n%s\ntail=\n%s" % (str(map_points_df.head(30)), str(map_points_df.tail(30)))) # group for the last one - pathset_links_df["point_id"]=1 # we need this set to *something* agg - last_point_df = pathset_links_df.groupby(["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum"]).agg( - {"point_id":"count", - "B_id":"last", - "B_seq":"last", - "B_lat":"last", - "B_lon":"last", - "route_id":"last", - "trip_id" :"last", - "linkmode":"last", - "mode" :"last"}).reset_index() - last_point_df.rename(columns={"B_id" :"stop_or_taz_id", - "B_seq" :"stop_sequence", - "B_lat" :"latitude", - "B_lon" :"longitude"}, inplace=True) + pathset_links_df["point_id"] = 1 # we need this set to *something* agg + last_point_df = pathset_links_df.groupby( + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum"]).agg( + {"point_id": "count", + "B_id": "last", + "B_seq": "last", + "B_lat": "last", + "B_lon": "last", + "route_id": "last", + "trip_id": "last", + "linkmode": "last", + "mode": "last"}).reset_index() + last_point_df.rename(columns={"B_id": "stop_or_taz_id", + "B_seq": "stop_sequence", + "B_lat": "latitude", + "B_lon": "longitude"}, inplace=True) last_point_df["point_id"] = last_point_df["point_id"] + 1 fasttrips.FastTripsLogger.debug("last_point_df=\n%s" % str(last_point_df.head(30))) # combine map points with the last map point map_points_df = map_points_df.append(last_point_df, ignore_index=True) - map_points_df.sort_values(by=["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","linknum","point_id"], inplace=True) + map_points_df.sort_values( + by=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum", + "linknum", "point_id"], inplace=True) map_points_df.reset_index(drop=True, inplace=True) - map_points_df[["pathnum","linknum"]] = map_points_df[["pathnum","linknum"]].astype(int) - fasttrips.FastTripsLogger.debug("map_points_df head=\n%s\ntail=\n%s" % (str(map_points_df.head(100)), str(map_points_df.tail(100)))) + map_points_df[["pathnum", "linknum"]] = map_points_df[["pathnum", "linknum"]].astype(int) + fasttrips.FastTripsLogger.debug( + "map_points_df head=\n%s\ntail=\n%s" % (str(map_points_df.head(100)), str(map_points_df.tail(100)))) fasttrips.FastTripsLogger.info("Have %d map points" % len(map_points_df)) if args.use_description: @@ -380,25 +434,34 @@ def impute_stop_from_adjacent_stop(ft, pathset_links_df, prev_next): # get unique descriptions num_paths = len(pathset_paths_df) pathset_paths_df.drop_duplicates(subset=["description"], keep="first", inplace=True) - fasttrips.FastTripsLogger.info("Dropping duplicate path descriptions. Went from %d to %d paths" % (num_paths, len(pathset_paths_df))) + fasttrips.FastTripsLogger.info( + "Dropping duplicate path descriptions. Went from %d to %d paths" % (num_paths, len(pathset_paths_df))) # join map points to them - map_points_df = pd.merge(left =pathset_paths_df, - left_on =["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum"], - right =map_points_df, - right_on=["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum"], - how ="inner") + map_points_df = pd.merge(left=pathset_paths_df, + left_on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum"], + right=map_points_df, + right_on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum"], + how="inner") fasttrips.FastTripsLogger.info("Have %d map points" % len(map_points_df)) # drop the non-descript columns - map_points_df.drop(["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum"], axis=1, inplace=True) + map_points_df.drop( + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", "simulation_iteration", "pathnum"], + axis=1, inplace=True) else: # just add descriptions - map_points_df = pd.merge(left =map_points_df, - left_on =["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum"], - right =pathset_paths_df[["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum","description"]], - right_on=["person_id","person_trip_id","iteration","pathfinding_iteration","simulation_iteration","pathnum"], - how ="left") + map_points_df = pd.merge(left=map_points_df, + left_on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum"], + right=pathset_paths_df[ + ["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum", "description"]], + right_on=["person_id", "person_trip_id", "iteration", "pathfinding_iteration", + "simulation_iteration", "pathnum"], + how="left") # write it output_file = os.path.join(args.input_path_dir[0], "pathset_map_points.csv") map_points_df.to_csv(output_file, sep=",", index=False) diff --git a/tests/conftest.py b/tests/conftest.py index 2bd9a242..7497c2cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,10 +10,12 @@ HOME_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples", ) NETWORK_HOME_DIR = os.path.join(HOME_DIR, 'networks') + @pytest.fixture(scope="module", params=["simple", "psrc_1_1"]) def network(request): yield request.param + @pytest.fixture(scope="module") def zip_file(network): network_dir = os.path.join(NETWORK_HOME_DIR, network) @@ -34,6 +36,7 @@ def network_date(network): } yield dates[network] + @pytest.fixture(scope="function") def gtfs_feed(zip_file, network_date): from fasttrips.Assignment import Assignment @@ -41,9 +44,9 @@ def gtfs_feed(zip_file, network_date): service_ids_by_date = ptg.read_service_ids_by_date(zip_file) service_ids = service_ids_by_date[network_date] feed = ptg.feed(os.path.join(zip_file), - config=Util.get_fast_trips_config(), view={ - 'trips.txt': { - 'service_id': service_ids - }, - }) + config=Util.get_fast_trips_config(), view={ + 'trips.txt': { + 'service_id': service_ids + }, + }) yield feed diff --git a/tests/test_assignment_type.py b/tests/test_assignment_type.py index 46919e8b..7fb5bb6c 100644 --- a/tests/test_assignment_type.py +++ b/tests/test_assignment_type.py @@ -2,42 +2,43 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') # LIST OF RUN PARAMETERS -ASSIGNMENT_TYPES = ["stochastic","deterministic"] +ASSIGNMENT_TYPES = ["stochastic", "deterministic"] -@pytest.mark.parametrize("assignment_type", ASSIGNMENT_TYPES) +@pytest.mark.parametrize("assignment_type", ASSIGNMENT_TYPES) def test_assignment_type(assignment_type): - OUTPUT_FOLDER = "assignment_type_%s" % (assignment_type) + OUTPUT_FOLDER = "assignment_type_%s" % (assignment_type) r = Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = OUTPUT_FOLDER, - overlap_variable = "None", - pf_iters = 2, - max_stop_process_count = 2, - pathfinding_type = assignment_type, - iters = 1, - dispersion = 0.50, - num_trips = 5) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder=OUTPUT_FOLDER, + overlap_variable="None", + pf_iters=2, + max_stop_process_count=2, + pathfinding_type=assignment_type, + iters=1, + dispersion=0.50, + num_trips=5) assert r["passengers_arrived"] > 0 + if __name__ == "__main__": - for at in ["stochastic","deterministic"]: + for at in ["stochastic", "deterministic"]: test_assignment_type(at) diff --git a/tests/test_bunny.py b/tests/test_bunny.py index 91e1eb79..8506e156 100644 --- a/tests/test_bunny.py +++ b/tests/test_bunny.py @@ -7,42 +7,44 @@ Run just the tests labeled basic using `pytest -v -m basic` """ -demand_options = ["backward_bunnies","forward_bunnies"] -network_options = ["bunny_hop","many_bunny_hops"] +demand_options = ["backward_bunnies", "forward_bunnies"] +network_options = ["bunny_hop", "many_bunny_hops"] + @pytest.mark.parametrize("demand", demand_options) @pytest.mark.parametrize("network", network_options) - @pytest.mark.basic @pytest.mark.travis def test_bunny(demand, network): """ Test to ensure that the most simple of networks and demand is working. """ - EXAMPLE_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples","Bunny_Hop") + EXAMPLE_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples", "Bunny_Hop") INPUT_NETWORK = os.path.join(EXAMPLE_DIR, "networks", network) - INPUT_DEMAND = os.path.join(EXAMPLE_DIR, "demand" , demand) - INPUT_CONFIG = os.path.join(EXAMPLE_DIR, "configs","base") - OUTPUT_DIR = os.path.join(EXAMPLE_DIR, "output") + INPUT_DEMAND = os.path.join(EXAMPLE_DIR, "demand", demand) + INPUT_CONFIG = os.path.join(EXAMPLE_DIR, "configs", "base") + OUTPUT_DIR = os.path.join(EXAMPLE_DIR, "output") Run.run_fasttrips( - input_network_dir = INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = os.path.join(INPUT_CONFIG, "config_ft.txt"), - input_functions = os.path.join(INPUT_CONFIG, 'config_ft.py'), - input_weights = os.path.join(INPUT_CONFIG, "pathweight_ft.txt"), - output_dir = OUTPUT_DIR, - output_folder = demand+"-"+network, - pathfinding_type = "stochastic", - capacity = False, - iters = 1, - OVERLAP = "None", - dispersion = 0.5 + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=os.path.join(INPUT_CONFIG, "config_ft.txt"), + input_functions=os.path.join(INPUT_CONFIG, 'config_ft.py'), + input_weights=os.path.join(INPUT_CONFIG, "pathweight_ft.txt"), + output_dir=OUTPUT_DIR, + output_folder=demand + "-" + network, + pathfinding_type="stochastic", + capacity=False, + iters=1, + OVERLAP="None", + dispersion=0.5 ) + if __name__ == '__main__': import itertools - for demand,network in list(itertools.product(demand_options, network_options)): - print("running %s %s" % (demand,network)) + + for demand, network in list(itertools.product(demand_options, network_options)): + print("running %s %s" % (demand, network)) test_bunny(demand, network) diff --git a/tests/test_calculate_cost.py b/tests/test_calculate_cost.py index 4b9736a1..b70e2adb 100644 --- a/tests/test_calculate_cost.py +++ b/tests/test_calculate_cost.py @@ -6,27 +6,28 @@ from fasttrips import PathSet, Run, Util -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'B') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -TEST_FOLDER = os.path.join(EXAMPLE_DIR, 'output','calculate_cost') -DF_DIR = os.path.join(EXAMPLE_DIR, 'misc', 'test_controls', 'calculate_cost') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'B') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +TEST_FOLDER = os.path.join(EXAMPLE_DIR, 'output', 'calculate_cost') +DF_DIR = os.path.join(EXAMPLE_DIR, 'misc', 'test_controls', 'calculate_cost') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') -PATHSET_PATHS_OUT = os.path.join(DF_DIR, 'output_pathset_paths_calculate_cost.csv') -PATHSET_LINKS_OUT = os.path.join(DF_DIR, 'output_pathset_links_calculate_cost.csv') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +PATHSET_PATHS_OUT = os.path.join(DF_DIR, 'output_pathset_paths_calculate_cost.csv') +PATHSET_LINKS_OUT = os.path.join(DF_DIR, 'output_pathset_links_calculate_cost.csv') STOCHASTIC_DISPERSION = 0.5 -PATHSET_PATHS_CTL = os.path.join(DF_DIR, 'control_result_pathset_paths.csv') -PATHSET_LINKS_CTL = os.path.join(DF_DIR, 'control_result_pathset_links.csv') +PATHSET_PATHS_CTL = os.path.join(DF_DIR, 'control_result_pathset_paths.csv') +PATHSET_LINKS_CTL = os.path.join(DF_DIR, 'control_result_pathset_links.csv') + @pytest.mark.travis def test_growth_type_cost_calculation(): @@ -40,7 +41,7 @@ def test_growth_type_cost_calculation(): } compare_dtypes = { - 'sim_cost': np.float64, + 'sim_cost': np.float64, } result_set = [94.32, 3.13704, 0.51001, 127.04425] @@ -53,6 +54,7 @@ def test_growth_type_cost_calculation(): if __name__ == '__main__': import traceback + try: test_growth_type_cost_calculation() except Exception as err: diff --git a/tests/test_convergence.py b/tests/test_convergence.py index 91761175..56fef4e2 100644 --- a/tests/test_convergence.py +++ b/tests/test_convergence.py @@ -2,21 +2,20 @@ import pytest from fasttrips import Run - -EXAMPLE_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples","Springfield") +EXAMPLE_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples", "Springfield") # DIRECTORY LOCATIONS -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'C') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -OUTPUT_FOLDER = "test_convergence" +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'C') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +OUTPUT_FOLDER = "test_convergence" # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') def test_convergence(): @@ -25,17 +24,17 @@ def test_convergence(): ''' Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_functions = INPUT_FUNCTIONS, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = OUTPUT_FOLDER, - capacity = True, - iters = 10, - dispersion = 0.50, - num_trips = 10, + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_functions=INPUT_FUNCTIONS, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder=OUTPUT_FOLDER, + capacity=True, + iters=10, + dispersion=0.50, + num_trips=10, ) diff --git a/tests/test_cost_symmetry.py b/tests/test_cost_symmetry.py index a8188b53..887f6b14 100644 --- a/tests/test_cost_symmetry.py +++ b/tests/test_cost_symmetry.py @@ -11,15 +11,15 @@ from fasttrips import PathSet from fasttrips import Trip - EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', "Springfield") -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') + +CONFIGS = ['A', 'A.alt', 'A.pat'] -CONFIGS = ['A', 'A.alt', 'A.pat'] +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') @pytest.fixture(scope='module', params=CONFIGS) def config_scenario(request): @@ -28,6 +28,7 @@ def config_scenario(request): """ return request.param + @pytest.fixture(scope='module') def ft_instance(config_scenario): """ @@ -44,8 +45,8 @@ def ft_instance(config_scenario): ft = FastTrips( INPUT_NETWORK, INPUT_DEMAND, - os.path.join(EXAMPLE_DIR,'configs', config_scenario, 'pathweight_ft.txt'), - os.path.join(EXAMPLE_DIR,'configs', config_scenario, 'config_ft.txt'), + os.path.join(EXAMPLE_DIR, 'configs', config_scenario, 'pathweight_ft.txt'), + os.path.join(EXAMPLE_DIR, 'configs', config_scenario, 'config_ft.txt'), OUTPUT_FOLDER ) @@ -100,9 +101,9 @@ def pathfinder_paths(ft_instance, config_scenario): trip_pathset.pathdict = pathdict yield ft.passengers.setup_passenger_pathsets(1, 1, ft.stops, ft.trips.trip_id_df, - ft.trips.trips_df, ft.routes.modes_df, - ft.transfers, ft.tazs, - Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID) + ft.trips.trips_df, ft.routes.modes_df, + ft.transfers, ft.tazs, + Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID) @pytest.fixture(scope='module') @@ -133,6 +134,7 @@ def simulation_paths(ft_instance, pathfinder_paths): reset_bump_iter=False ) + @pytest.mark.cost def test_cost_symmetry(pathfinder_paths, simulation_paths): paths_join_col = ['trip_list_id_num', 'pathnum'] @@ -151,11 +153,11 @@ def test_cost_symmetry(pathfinder_paths, simulation_paths): paths = pd.merge(pf_pathset_paths[paths_join_col + ['pf_cost']], sim_pathset_paths[paths_join_col + ['sim_cost']], on=paths_join_col, - suffixes=['pf','sim']) + suffixes=['pf', 'sim']) links = pd.merge(pf_pathset_links[links_join_col + ['pf_linkcost']], sim_pathset_links[links_join_col + ['sim_cost']], on=links_join_col, - suffixes=['pf','sim']) + suffixes=['pf', 'sim']) # Assert that the join resulted in the same number of records assert pf_pathset_paths.shape[0] == paths.shape[0] diff --git a/tests/test_dispersion.py b/tests/test_dispersion.py index e7fce2c2..2a1055c9 100644 --- a/tests/test_dispersion.py +++ b/tests/test_dispersion.py @@ -6,56 +6,61 @@ # TEST OPTIONS test_thetas = [1.0, 0.5, 0.2] -test_size = 5 +test_size = 5 disperson_rate_util_multiplier_factor = 10.0 # DIRECTORY LOCATIONS -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') + +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') @pytest.fixture(scope='module', params=test_thetas) def dispersion_rate(request): return request.param + @pytest.fixture(scope='module') def passengers_arrived(dispersion_rate): - arrived = dict(list(zip(test_thetas,[test_size]*len(test_thetas)))) + arrived = dict(list(zip(test_thetas, [test_size] * len(test_thetas)))) return arrived[dispersion_rate] + @pytest.fixture(scope='module') def utils_conversion_factor(dispersion_rate): - factor = dispersion_rate*disperson_rate_util_multiplier_factor + factor = dispersion_rate * disperson_rate_util_multiplier_factor return factor + @pytest.mark.travis def test_dispersion(dispersion_rate, utils_conversion_factor, passengers_arrived): - r = Run.run_fasttrips( - input_network_dir = INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = os.path.join(INPUT_CONFIG,"config_ft.txt"), - input_weights = os.path.join(INPUT_CONFIG,"pathweight_ft.txt"), - output_dir = OUTPUT_DIR, - output_folder = "test_dispers_%4.2f" % dispersion_rate, - max_stop_process_count = 2, - utils_conversion_factor = utils_conversion_factor, - pf_iters = 2, - overlap_variable = "None", - pathfinding_type = "stochastic", - iters = 1, - dispersion = dispersion_rate, - num_trips = test_size ) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=os.path.join(INPUT_CONFIG, "config_ft.txt"), + input_weights=os.path.join(INPUT_CONFIG, "pathweight_ft.txt"), + output_dir=OUTPUT_DIR, + output_folder="test_dispers_%4.2f" % dispersion_rate, + max_stop_process_count=2, + utils_conversion_factor=utils_conversion_factor, + pf_iters=2, + overlap_variable="None", + pathfinding_type="stochastic", + iters=1, + dispersion=dispersion_rate, + num_trips=test_size) assert passengers_arrived == r["passengers_arrived"] + if __name__ == "__main__": for dr in test_thetas: - util_factor = dr*disperson_rate_util_multiplier_factor - print("Running test_dispersion.py with: disperson: %f1.2, util_factor: %f2.2, test_size: %d" % (dr, util_factor, test_size)) + util_factor = dr * disperson_rate_util_multiplier_factor + print("Running test_dispersion.py with: disperson: %f1.2, util_factor: %f2.2, test_size: %d" % ( + dr, util_factor, test_size)) test_dispersion(dr, util_factor, test_size) diff --git a/tests/test_distance.py b/tests/test_distance.py index 08eca21d..b355429c 100644 --- a/tests/test_distance.py +++ b/tests/test_distance.py @@ -10,9 +10,10 @@ from fasttrips import Util TEST_NETWORKS = {"Seattle_Region": "psrc_1_1", - "Springfield" : "vermont"} + "Springfield": "vermont"} + +EXAMPLES_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples") -EXAMPLES_DIR = os.path.join(os.getcwd(), "fasttrips", "Examples") @pytest.fixture(scope="module") def network_results(network): @@ -38,12 +39,13 @@ def network_results(network): } yield results[network] + def test_calculate_distance_miles(): orig_lat, orig_lon = 32.707431, -117.157058 dest_lat, dest_lon = 32.740792, -117.211333 - cols = ['orig_lat','orig_lon','dest_lat','dest_lon','dist'] + cols = ['orig_lat', 'orig_lon', 'dest_lat', 'dest_lon', 'dist'] - df = pd.DataFrame([[orig_lat,orig_lon,dest_lat,dest_lon,np.nan]], + df = pd.DataFrame([[orig_lat, orig_lon, dest_lat, dest_lon, np.nan]], columns=cols) Util.calculate_distance_miles(df, cols[0], cols[1], cols[2], cols[3], cols[4]) diff --git a/tests/test_fares.py b/tests/test_fares.py index f1f240bd..257d8fc5 100644 --- a/tests/test_fares.py +++ b/tests/test_fares.py @@ -3,50 +3,51 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') # LIST OF RUN PARAMETERS ignore_PF_fares_options = [True] ignore_EN_fares_options = [False, True] + @pytest.mark.parametrize("ignore_PF_fares", ignore_PF_fares_options) @pytest.mark.parametrize("ignore_EN_fares", ignore_EN_fares_options) - @pytest.mark.travis -def test_fares(ignore_PF_fares,ignore_EN_fares): - +def test_fares(ignore_PF_fares, ignore_EN_fares): r = Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = "test_ignore_fares_PF-%s_EN-%s" % (ignore_PF_fares,ignore_EN_fares), - pathfinding_type = "stochastic", - max_stop_process_count = 2, - pf_iters = 2, - overlap_variable = "None", - iters = 1, - dispersion = 0.50, - utils_conversion_factor = 10, - num_trips = 5, - transfer_fare_ignore_pathfinding = ignore_PF_fares, - transfer_fare_ignore_pathenum = ignore_EN_fares) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder="test_ignore_fares_PF-%s_EN-%s" % (ignore_PF_fares, ignore_EN_fares), + pathfinding_type="stochastic", + max_stop_process_count=2, + pf_iters=2, + overlap_variable="None", + iters=1, + dispersion=0.50, + utils_conversion_factor=10, + num_trips=5, + transfer_fare_ignore_pathfinding=ignore_PF_fares, + transfer_fare_ignore_pathenum=ignore_EN_fares) assert r["passengers_arrived"] > 0 + if __name__ == '__main__': import itertools - for ignore_PF_fares,ignore_EN_fares in list(itertools.product(ignore_PF_fares_options, ignore_EN_fares_options)): - print("running %s %s" % (ignore_PF_fares,ignore_EN_fares)) - test_fares(ignore_PF_fares,ignore_EN_fares) + + for ignore_PF_fares, ignore_EN_fares in list(itertools.product(ignore_PF_fares_options, ignore_EN_fares_options)): + print("running %s %s" % (ignore_PF_fares, ignore_EN_fares)) + test_fares(ignore_PF_fares, ignore_EN_fares) diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 1245a909..00824c62 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -3,17 +3,17 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') # LIST OF RUN PARAMETERS ITERS = [3] @@ -22,31 +22,31 @@ @pytest.mark.parametrize("feedback_iters", ITERS) @pytest.mark.parametrize("capacity_const", CAPACITY_CONSTRAINT) - @pytest.mark.travis -def test_feedback(feedback_iters,capacity_const): - +def test_feedback(feedback_iters, capacity_const): r = Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = "test_feedback_iters-%d_capConst-%s" % (feedback_iters,capacity_const), - max_stop_process_count = 2, - pf_iters = 2, - overlap_variable = "None", - utils_conversion_factor = 10, - pathfinding_type = "stochastic", - capacity = capacity_const, - iters = feedback_iters, - num_trips = 5, - dispersion = 0.50) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder="test_feedback_iters-%d_capConst-%s" % (feedback_iters, capacity_const), + max_stop_process_count=2, + pf_iters=2, + overlap_variable="None", + utils_conversion_factor=10, + pathfinding_type="stochastic", + capacity=capacity_const, + iters=feedback_iters, + num_trips=5, + dispersion=0.50) assert r["passengers_arrived"] > 0 + if __name__ == '__main__': import itertools - for iter, cap_const in list(itertools.product(ITERS,CAPACITY_CONSTRAINT)): + + for iter, cap_const in list(itertools.product(ITERS, CAPACITY_CONSTRAINT)): print("running %s %s" % (iter, cap_const)) test_feedback(iter, cap_const) diff --git a/tests/test_gtfs_reader.py b/tests/test_gtfs_reader.py index 53293bf5..df4bb4ff 100644 --- a/tests/test_gtfs_reader.py +++ b/tests/test_gtfs_reader.py @@ -17,6 +17,7 @@ [os.path.join(EXAMPLE_DIR, 'Springfield', 'networks', 'vermont'), datetime.date(2015, 2, 3)], ] + def get_gtfs_feed(network, network_date): from fasttrips.Assignment import Assignment from fasttrips.Util import Util @@ -32,6 +33,7 @@ def get_gtfs_feed(network, network_date): }) return feed + @pytest.mark.parametrize('network_dir, network_date', CONFIGS) def test_stops_load(network_dir, network_date): ''' @@ -47,13 +49,13 @@ def test_stops_load(network_dir, network_date): stops = Stop(network_dir, out_dir, gtfs_feed, network_date) - #Test existence, length, and required columns + # Test existence, length, and required columns assert not stops.stop_id_df.empty assert len(stops.stop_id_df) == 8 assert ({Stop.STOPS_COLUMN_STOP_ID, Stop.STOPS_COLUMN_STOP_ID_NUM}.issubset(stops.stop_id_df)) assert stops.max_stop_id_num == 8 - #Test existence, length, dtype, and column names for stops_df + # Test existence, length, dtype, and column names for stops_df assert not stops.stops_df.empty assert len(stops.stops_df == 8) stop_df_dtypes = { @@ -69,20 +71,22 @@ def test_stops_load(network_dir, network_date): assert stops.stops_df.dtypes.to_dict() == stop_df_dtypes pd.testing.assert_frame_equal(stops.stop_id_df.set_index(Stop.STOPS_COLUMN_STOP_ID_NUM), stops.stops_df.set_index(Stop.STOPS_COLUMN_STOP_ID_NUM), - ) + ) - #Test to make sure stop_id and stops dataframe are identical + # Test to make sure stop_id and stops dataframe are identical stops_with_zones = stops.stops_df[stops.stops_df[Stop.STOPS_COLUMN_ZONE_ID_NUM].notnull()] assert len(stops_with_zones) == 2 - assert len(stops_with_zones[stops_with_zones[Stop.STOPS_COLUMN_STOP_ID].isin(['B1','B3'])]) == 2 - + assert len(stops_with_zones[stops_with_zones[Stop.STOPS_COLUMN_STOP_ID].isin(['B1', 'B3'])]) == 2 assert len(stops.zone_id_df) == 2 - pd.testing.assert_frame_equal(stops_with_zones[[Stop.STOPS_COLUMN_ZONE_ID_NUM, Stop.STOPS_COLUMN_ZONE_ID]].drop_duplicates().set_index(Stop.STOPS_COLUMN_ZONE_ID_NUM), + pd.testing.assert_frame_equal( + stops_with_zones[[Stop.STOPS_COLUMN_ZONE_ID_NUM, Stop.STOPS_COLUMN_ZONE_ID]].drop_duplicates().set_index( + Stop.STOPS_COLUMN_ZONE_ID_NUM), stops.zone_id_df.set_index(Stop.STOPS_COLUMN_ZONE_ID_NUM), check_dtype=False, check_index_type=False) assert not stops.trip_times_df + @pytest.mark.parametrize('network_dir, network_date', CONFIGS) def test_routes_load(network_dir, network_date): """ @@ -98,51 +102,54 @@ def test_routes_load(network_dir, network_date): raise stops = Stop(network_dir, out_dir, - gtfs_feed, network_date) + gtfs_feed, network_date) routes = Route(network_dir, out_dir, - gtfs_feed, network_date, stops) + gtfs_feed, network_date, stops) - #routes.routes_df + # routes.routes_df assert not routes.routes_df.empty assert len(routes.routes_df) == 4 routes_df_dtypes = { - 'route_id': object, - 'route_long_name': object, - 'route_short_name': object, - 'route_type': np.int64, - 'mode': object, - 'proof_of_payment': bool, - 'mode_num': np.int64, - 'mode_type': object, - 'route_id_num': np.int64, + 'route_id': object, + 'route_long_name': object, + 'route_short_name': object, + 'route_type': np.int64, + 'mode': object, + 'proof_of_payment': bool, + 'mode_num': np.int64, + 'mode_type': object, + 'route_id_num': np.int64, } assert (set(routes_df_dtypes.keys()).issubset(routes.routes_df)) assert routes.routes_df.dtypes.to_dict() == routes_df_dtypes assert len(routes.routes_df[routes.routes_df[Route.ROUTES_COLUMN_PROOF_OF_PAYMENT]]) == 1 - #routes.modes_df + # routes.modes_df assert not routes.modes_df.empty assert len(routes.routes_df[routes.routes_df['mode'].isin(routes.modes_df['mode'])]) == len(routes.routes_df) - pd.testing.assert_frame_equal(routes.routes_df[[Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE_TYPE]].drop_duplicates().set_index(Route.ROUTES_COLUMN_MODE_NUM), - routes.modes_df.set_index(Route.ROUTES_COLUMN_MODE_NUM)) + pd.testing.assert_frame_equal(routes.routes_df[[Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM, + Route.ROUTES_COLUMN_MODE_TYPE]].drop_duplicates().set_index( + Route.ROUTES_COLUMN_MODE_NUM), + routes.modes_df.set_index(Route.ROUTES_COLUMN_MODE_NUM)) - #routes.route_id_df + # routes.route_id_df assert not routes.route_id_df.empty assert len(routes.route_id_df) == len(routes.routes_df) - pd.testing.assert_frame_equal(routes.routes_df[[Route.ROUTES_COLUMN_ROUTE_ID_NUM, Route.ROUTES_COLUMN_ROUTE_ID]].set_index(Route.ROUTES_COLUMN_ROUTE_ID_NUM), - routes.route_id_df.set_index(Route.ROUTES_COLUMN_ROUTE_ID_NUM)) + pd.testing.assert_frame_equal( + routes.routes_df[[Route.ROUTES_COLUMN_ROUTE_ID_NUM, Route.ROUTES_COLUMN_ROUTE_ID]].set_index( + Route.ROUTES_COLUMN_ROUTE_ID_NUM), + routes.route_id_df.set_index(Route.ROUTES_COLUMN_ROUTE_ID_NUM)) - #routes.agencies_df + # routes.agencies_df assert not routes.agencies_df.empty - #The SIMPLE network doesn't really do much with agency. - #TODO: Consider adding here once these tests includes more networks + # The SIMPLE network doesn't really do much with agency. + # TODO: Consider adding here once these tests includes more networks - #routes.fare_rules_df + # routes.fare_rules_df assert not routes.fare_rules_df.empty - - #routes.fare_attrs_df + # routes.fare_attrs_df assert not routes.fare_attrs_df.empty fare_attrs_df_dtype = { 'fare_period': object, @@ -155,27 +162,31 @@ def test_routes_load(network_dir, network_date): assert (set(fare_attrs_df_dtype.keys()).issubset(routes.fare_attrs_df)) assert routes.fare_attrs_df.dtypes.to_dict() == fare_attrs_df_dtype assert len(routes.fare_attrs_df) == 9 - match_fare_rule_attr_df = routes.fare_rules_df[routes.fare_rules_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD].isin(routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])] + match_fare_rule_attr_df = routes.fare_rules_df[routes.fare_rules_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD].isin( + routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])] assert len(match_fare_rule_attr_df) == len(routes.fare_rules_df) - #pd.testing.assert_frame_equal(routes.fare_rules_df[fare_attrs_df_dtype.keys()].drop_duplicates().set_index(Route.FARE_ATTR_COLUMN_FARE_PERIOD), + # pd.testing.assert_frame_equal(routes.fare_rules_df[fare_attrs_df_dtype.keys()].drop_duplicates().set_index(Route.FARE_ATTR_COLUMN_FARE_PERIOD), # routes.fare_attrs_df[routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD].isin(match_fare_rule_attr_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])].set_index(Route.FARE_ATTR_COLUMN_FARE_PERIOD), # check_dtype=False, check_index_type=False) - #routes.fare_ids_df + # routes.fare_ids_df assert not routes.fare_ids_df.empty - pd.testing.assert_frame_equal(routes.fare_rules_df[[Route.FARE_RULES_COLUMN_FARE_ID, Route.FARE_RULES_COLUMN_FARE_ID_NUM]].drop_duplicates().set_index(Route.FARE_RULES_COLUMN_FARE_ID_NUM), - routes.fare_ids_df.set_index(Route.FARE_RULES_COLUMN_FARE_ID_NUM)) + pd.testing.assert_frame_equal(routes.fare_rules_df[[Route.FARE_RULES_COLUMN_FARE_ID, + Route.FARE_RULES_COLUMN_FARE_ID_NUM]].drop_duplicates().set_index( + Route.FARE_RULES_COLUMN_FARE_ID_NUM), + routes.fare_ids_df.set_index(Route.FARE_RULES_COLUMN_FARE_ID_NUM)) - #routes.fare_by_class + # routes.fare_by_class assert routes.fare_by_class - #routes.fare_transfer_rules_df + # routes.fare_transfer_rules_df assert not routes.fare_transfer_rules_df.empty - assert len(routes.fare_transfer_rules_df[routes.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD] - .isin(routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])]) == len(routes.fare_transfer_rules_df) + assert len( + routes.fare_transfer_rules_df[routes.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_FROM_FARE_PERIOD] + .isin(routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])]) == len(routes.fare_transfer_rules_df) assert len( routes.fare_transfer_rules_df[routes.fare_transfer_rules_df[Route.FARE_TRANSFER_RULES_COLUMN_TO_FARE_PERIOD] - .isin(routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])]) == len(routes.fare_transfer_rules_df) + .isin(routes.fare_attrs_df[Route.FARE_ATTR_COLUMN_FARE_PERIOD])]) == len(routes.fare_transfer_rules_df) @pytest.mark.parametrize('network_dir, network_date', CONFIGS) @@ -255,22 +266,24 @@ def test_trips_load(network_dir, network_date): assert (set(trips_df_dtypes.keys()).issubset(trips.trips_df)) assert trips.trips_df.dtypes.to_dict() == trips_df_dtypes assert len(trips.trips_df[Route.ROUTES_COLUMN_ROUTE_ID].isin(routes.routes_df[Route.ROUTES_COLUMN_ROUTE_ID])) == \ - len(trips.trips_df) + len(trips.trips_df) assert len(trips.trips_df[trips.trips_df.duplicated(subset=Trip.TRIPS_COLUMN_TRIP_ID)]) == 0 pd.testing.assert_frame_equal(trips.trips_df[[ - Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE_TYPE]] - .drop_duplicates() - .sort_values(by=Route.ROUTES_COLUMN_MODE_NUM) - .set_index(Route.ROUTES_COLUMN_MODE_NUM), - routes.modes_df.sort_values(by=Route.ROUTES_COLUMN_MODE_NUM).set_index(Route.ROUTES_COLUMN_MODE_NUM)) + Route.ROUTES_COLUMN_MODE, Route.ROUTES_COLUMN_MODE_NUM, Route.ROUTES_COLUMN_MODE_TYPE]] + .drop_duplicates() + .sort_values(by=Route.ROUTES_COLUMN_MODE_NUM) + .set_index(Route.ROUTES_COLUMN_MODE_NUM), + routes.modes_df.sort_values(by=Route.ROUTES_COLUMN_MODE_NUM).set_index( + Route.ROUTES_COLUMN_MODE_NUM)) pd.testing.assert_frame_equal(trips.trips_df[[ - Route.ROUTES_COLUMN_ROUTE_ID_NUM, Route.ROUTES_COLUMN_ROUTE_ID]] + Route.ROUTES_COLUMN_ROUTE_ID_NUM, Route.ROUTES_COLUMN_ROUTE_ID]] .drop_duplicates() .sort_values(by=Route.ROUTES_COLUMN_ROUTE_ID_NUM) .set_index(Route.ROUTES_COLUMN_ROUTE_ID_NUM), - routes.route_id_df.sort_values(by=Route.ROUTES_COLUMN_ROUTE_ID_NUM).set_index(Route.ROUTES_COLUMN_ROUTE_ID_NUM)) + routes.route_id_df.sort_values(by=Route.ROUTES_COLUMN_ROUTE_ID_NUM).set_index( + Route.ROUTES_COLUMN_ROUTE_ID_NUM)) assert not trips.trip_id_df.empty pd.testing.assert_frame_equal(trips.trips_df[[ @@ -284,7 +297,7 @@ def test_trips_load(network_dir, network_date): assert not trips.stop_times_df.empty assert len( trips.stop_times_df[(trips.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME].dt.month == network_date.month) & - (trips.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME].dt.day == network_date.day) & + (trips.stop_times_df[Trip.STOPTIMES_COLUMN_ARRIVAL_TIME].dt.day == network_date.day) & (trips.stop_times_df[ Trip.STOPTIMES_COLUMN_ARRIVAL_TIME].dt.year == network_date.year)]) == len( trips.stop_times_df) diff --git a/tests/test_maxStopProcessCount.py b/tests/test_maxStopProcessCount.py index 4ce4a721..a17b9f3c 100644 --- a/tests/test_maxStopProcessCount.py +++ b/tests/test_maxStopProcessCount.py @@ -3,21 +3,22 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') # TEST PARAMETERS test_mspc = [10, 50, 100] -test_size = 5 +test_size = 5 + @pytest.fixture(scope='module', params=test_mspc) def stop_process_count(request): @@ -26,26 +27,25 @@ def stop_process_count(request): @pytest.fixture(scope='module') def passengers_arrived(stop_process_count): - arrived = dict(list(zip(test_mspc,[test_size]*len(test_mspc)))) + arrived = dict(list(zip(test_mspc, [test_size] * len(test_mspc)))) return arrived[stop_process_count] def test_max_stop_process_count(stop_process_count, passengers_arrived): - r = Run.run_fasttrips( - input_network_dir = INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = "test_dispers_%4.2d" % stop_process_count, - overlap_variable = "None", - pf_iters = 2, - pathfinding_type = "stochastic", - max_stop_process_count = stop_process_count, - iters = 1, - num_trips = test_size, - dispersion = 0.50 ) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder="test_dispers_%4.2d" % stop_process_count, + overlap_variable="None", + pf_iters=2, + pathfinding_type="stochastic", + max_stop_process_count=stop_process_count, + iters=1, + num_trips=test_size, + dispersion=0.50) assert passengers_arrived == r["passengers_arrived"] diff --git a/tests/test_overlap.py b/tests/test_overlap.py index 197e8e06..0f6f2063 100644 --- a/tests/test_overlap.py +++ b/tests/test_overlap.py @@ -1,52 +1,52 @@ -import os,sys +import os, sys import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') # TEST PARAMETERS OVERLAP_VARIABLES = ["None", "count", "distance", "time"] + @pytest.mark.parametrize("overlap_var", OVERLAP_VARIABLES) @pytest.mark.parametrize("split_links", [False, True]) - @pytest.mark.travis -def test_overlap(overlap_var, split_links ): - - if overlap_var == "time" and sys.version_info > (3,0): +def test_overlap(overlap_var, split_links): + if overlap_var == "time" and sys.version_info > (3, 0): pytest.xfail("python and pandas updates make time division fail for now Issue #172") r = Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = "test_overlap_var-%s_split-%s" % (overlap_var, split_links), - max_stop_process_count = 2, - pf_iters = 2, - pathfinding_type = "stochastic", - overlap_variable = overlap_var, - overlap_split_transit = split_links, - iters = 1, - dispersion = 0.50, - num_trips = 5) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder="test_overlap_var-%s_split-%s" % (overlap_var, split_links), + max_stop_process_count=2, + pf_iters=2, + pathfinding_type="stochastic", + overlap_variable=overlap_var, + overlap_split_transit=split_links, + iters=1, + dispersion=0.50, + num_trips=5) assert r["passengers_arrived"] > 0 + if __name__ == "__main__": for var in OVERLAP_VARIABLES: - print("Running test_overlap.py with variable: %s, NO link split" % (var)) - test_overlap(var,False) - print("Running test_overlap.py with variable: %s, YES link split" % (var)) - test_overlap(var,True) + print("Running test_overlap.py with variable: %s, NO link split" % (var)) + test_overlap(var, False) + print("Running test_overlap.py with variable: %s, YES link split" % (var)) + test_overlap(var, True) diff --git a/tests/test_pat_variation.py b/tests/test_pat_variation.py index dd29e5c3..a3bed449 100644 --- a/tests/test_pat_variation.py +++ b/tests/test_pat_variation.py @@ -11,44 +11,45 @@ from fasttrips import PathSet from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A.pat') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'general') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'A.pat') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') + @pytest.mark.travis @pytest.mark.pat -#@pytest.mark.skip(reason="Not working - need to fix") +# @pytest.mark.skip(reason="Not working - need to fix") def test_pat_before_and_after(): """ Test to ensure that some of the pathfinder trips are returned before preferred departure or after preferred arrival. """ - OUTPUT_FOLDER = 'pat_scenario' + OUTPUT_FOLDER = 'pat_scenario' r = Run.run_fasttrips( - input_network_dir = INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - output_dir = OUTPUT_DIR, - output_folder = OUTPUT_FOLDER, - pathfinding_type = "stochastic", - overlap_variable = "count", - iters = 1, - dispersion = 0.50 + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + output_dir=OUTPUT_DIR, + output_folder=OUTPUT_FOLDER, + pathfinding_type="stochastic", + overlap_variable="count", + iters=1, + dispersion=0.50 ) links = pd.read_csv( os.path.join(OUTPUT_DIR, OUTPUT_FOLDER, 'pathset_links.csv'), - usecols=['person_trip_id','pathnum', 'linkmode', 'linknum','new_A_time','new_B_time'], + usecols=['person_trip_id', 'pathnum', 'linkmode', 'linknum', 'new_A_time', 'new_B_time'], parse_dates=['new_A_time', 'new_B_time'], infer_datetime_format=True ) @@ -59,8 +60,9 @@ def test_pat_before_and_after(): ) departure_link = links.loc[links.groupby(['person_trip_id', 'pathnum'])['linknum'].idxmin()] - #The C++ Pathfinder doesn't seem to respect the last egress leg from a preferred time perspective - arrival_link = links.loc[links[links.linkmode == 'transit'].groupby(['person_trip_id', 'pathnum'])['linknum'].idxmax()] + # The C++ Pathfinder doesn't seem to respect the last egress leg from a preferred time perspective + arrival_link = links.loc[ + links[links.linkmode == 'transit'].groupby(['person_trip_id', 'pathnum'])['linknum'].idxmax()] network_date = links['new_A_time'].dt.date.unique()[0] trips['arrival_time'] = trips['arrival_time'].apply(lambda x: parse_date(network_date, x)) @@ -83,13 +85,16 @@ def test_pat_before_and_after(): early_departure = departures[departures['new_A_time'] < departures['departure_time']] size = early_departure.shape[0] assert size > 0 - confirm_size = early_departure[((early_departure['departure_time'] - early_departure['new_A_time'])/ np.timedelta64(1, 'm')) <= 10].shape[0] + confirm_size = early_departure[ + ((early_departure['departure_time'] - early_departure['new_A_time']) / np.timedelta64(1, 'm')) <= 10].shape[0] assert size == confirm_size late_arrivals = arrivals[arrivals['new_B_time'] > arrivals['arrival_time']] size = late_arrivals.shape[0] assert size > 0 - confirm_size = late_arrivals[((late_arrivals['new_B_time'] - late_arrivals['arrival_time'])/ np.timedelta64(1, 'm')) <= 10].shape[0] + confirm_size = \ + late_arrivals[((late_arrivals['new_B_time'] - late_arrivals['arrival_time']) / np.timedelta64(1, 'm')) <= 10].shape[ + 0] assert size == confirm_size @@ -99,28 +104,28 @@ def test_pat_off(): or after preferred arrival. """ - OUTPUT_FOLDER = 'pat_scenario_reg' + OUTPUT_FOLDER = 'pat_scenario_reg' in_cfg = os.path.join(EXAMPLE_DIR, 'configs', 'A') cfg_file = os.path.join(in_cfg, 'config_ft.txt') in_weights = os.path.join(in_cfg, 'pathweight_ft.txt') r = Run.run_fasttrips( - input_network_dir = INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = cfg_file, - input_weights = in_weights, - output_dir = OUTPUT_DIR, - output_folder = OUTPUT_FOLDER, - pathfinding_type = "stochastic", - overlap_variable = "None", - iters = 1, - dispersion = 0.50 + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=cfg_file, + input_weights=in_weights, + output_dir=OUTPUT_DIR, + output_folder=OUTPUT_FOLDER, + pathfinding_type="stochastic", + overlap_variable="None", + iters=1, + dispersion=0.50 ) links = pd.read_csv( os.path.join(OUTPUT_DIR, OUTPUT_FOLDER, 'pathset_links.csv'), - usecols=['person_trip_id','pathnum', 'linkmode', 'linknum','new_A_time','new_B_time'], + usecols=['person_trip_id', 'pathnum', 'linkmode', 'linknum', 'new_A_time', 'new_B_time'], parse_dates=['new_A_time', 'new_B_time'], infer_datetime_format=True ) @@ -132,7 +137,8 @@ def test_pat_off(): departure_link = links.loc[links.groupby(['person_trip_id', 'pathnum'])['linknum'].idxmin()] # The C++ Pathfinder doesn't seem to respect the last egress leg from a preferred time perspective - arrival_link = links.loc[links[links.linkmode == 'transit'].groupby(['person_trip_id', 'pathnum'])['linknum'].idxmax()] + arrival_link = links.loc[ + links[links.linkmode == 'transit'].groupby(['person_trip_id', 'pathnum'])['linknum'].idxmax()] network_date = links['new_A_time'].dt.date.unique()[0] trips['arrival_time'] = trips['arrival_time'].apply(lambda x: parse_date(network_date, x)) @@ -155,13 +161,16 @@ def test_pat_off(): early_departure = departures[departures['new_A_time'] < departures['departure_time']] size = early_departure.shape[0] assert 0 == size - confirm_size = early_departure[((early_departure['departure_time'] - early_departure['new_A_time'])/ np.timedelta64(1, 'm')) <= 10].shape[0] + confirm_size = early_departure[ + ((early_departure['departure_time'] - early_departure['new_A_time']) / np.timedelta64(1, 'm')) <= 10].shape[0] assert 0 == confirm_size late_arrivals = arrivals[arrivals['new_B_time'] > arrivals['arrival_time']] size = late_arrivals.shape[0] assert 0 == size - confirm_size = late_arrivals[((late_arrivals['new_B_time'] - late_arrivals['arrival_time'])/ np.timedelta64(1, 'm')) <= 10].shape[0] + confirm_size = \ + late_arrivals[((late_arrivals['new_B_time'] - late_arrivals['arrival_time']) / np.timedelta64(1, 'm')) <= 10].shape[ + 0] assert 0 == confirm_size @@ -189,9 +198,9 @@ def test_pat_growth_type_validation(): assert not check expected_error = '\n-------Errors: pathweight_ft.txt---------------\n' \ - 'Logistic qualifier includes log_base modifier\n' \ - 'Logarithmic qualifier missing necessary log_base modifier\n' \ - 'Logistic qualifier missing necessary modifiers\n' + 'Logistic qualifier includes log_base modifier\n' \ + 'Logarithmic qualifier missing necessary log_base modifier\n' \ + 'Logistic qualifier missing necessary modifiers\n' assert expected_error == error_str diff --git a/tests/test_penalty_functions.py b/tests/test_penalty_functions.py index eba9f3ba..77925a15 100644 --- a/tests/test_penalty_functions.py +++ b/tests/test_penalty_functions.py @@ -6,10 +6,11 @@ from fasttrips import Util + @pytest.mark.travis def test_exponential_integral(): test_series = pd.Series( - data=[6., 5./3, 111./11, 45./7, 524./13], + data=[6., 5. / 3, 111. / 11, 45. / 7, 524. / 13], index=[1, 4, 7, 25, 627], name='cost_series' ) @@ -24,10 +25,11 @@ def test_exponential_integral(): pd.testing.assert_series_equal(validation_series, output, check_less_precise=6) + @pytest.mark.travis def test_logarithmic_integral(): test_series = pd.Series( - data=[6., 5./3, 111./11, 45./7, 524./13], + data=[6., 5. / 3, 111. / 11, 45. / 7, 524. / 13], index=[1, 4, 7, 25, 627], name='cost_series' ) @@ -42,10 +44,11 @@ def test_logarithmic_integral(): pd.testing.assert_series_equal(validation_series, output, check_less_precise=6) + @pytest.mark.travis def test_logistic_integral(): test_series = pd.Series( - data=[6., 5./3, 111./11, 45./7, 524./13], + data=[6., 5. / 3, 111. / 11, 45. / 7, 524. / 13], index=[1, 4, 7, 25, 627], name='cost_series' ) diff --git a/tests/test_psrc.py b/tests/test_psrc.py index fc26fabe..7a009c56 100644 --- a/tests/test_psrc.py +++ b/tests/test_psrc.py @@ -2,18 +2,19 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Seattle_Region') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Seattle_Region') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'psrc_1_1') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'psrc_1_1') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'base') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'psrc_1_1') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'psrc_1_1') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'base') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') + @pytest.mark.travis def test_psrc(): @@ -23,22 +24,22 @@ def test_psrc(): """ Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - input_functions = INPUT_FUNCTIONS, - output_dir = OUTPUT_DIR, - output_folder = "test_psrc", - max_stop_process_count = 2, - pf_iters = 2, - overlap_variable = "None", - pathfinding_type = "stochastic", - capacity = True, - iters = 1, - OVERLAP = "None", - dispersion = 1.0, - num_trips = 5, + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + input_functions=INPUT_FUNCTIONS, + output_dir=OUTPUT_DIR, + output_folder="test_psrc", + max_stop_process_count=2, + pf_iters=2, + overlap_variable="None", + pathfinding_type="stochastic", + capacity=True, + iters=1, + OVERLAP="None", + dispersion=1.0, + num_trips=5, ) diff --git a/tests/test_user_classes.py b/tests/test_user_classes.py index 568b5daf..88d2325c 100644 --- a/tests/test_user_classes.py +++ b/tests/test_user_classes.py @@ -2,39 +2,40 @@ import pytest from fasttrips import Run -EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') +EXAMPLE_DIR = os.path.join(os.getcwd(), 'fasttrips', 'Examples', 'Springfield') # DIRECTORY LOCATIONS -INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') -INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') -INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'B') -OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') +INPUT_NETWORK = os.path.join(EXAMPLE_DIR, 'networks', 'vermont') +INPUT_DEMAND = os.path.join(EXAMPLE_DIR, 'demand', 'simpson_zorn') +INPUT_CONFIG = os.path.join(EXAMPLE_DIR, 'configs', 'B') +OUTPUT_DIR = os.path.join(EXAMPLE_DIR, 'output') # INPUT FILE LOCATIONS -CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') -INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') -INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') +CONFIG_FILE = os.path.join(INPUT_CONFIG, 'config_ft.txt') +INPUT_FUNCTIONS = os.path.join(INPUT_CONFIG, 'config_ft.py') +INPUT_WEIGHTS = os.path.join(INPUT_CONFIG, 'pathweight_ft.txt') -def test_user_classes(): +def test_user_classes(): r = Run.run_fasttrips( - input_network_dir= INPUT_NETWORK, - input_demand_dir = INPUT_DEMAND, - run_config = CONFIG_FILE, - input_weights = INPUT_WEIGHTS, - input_functions = INPUT_FUNCTIONS, - output_dir = OUTPUT_DIR, - output_folder = "test_userclasses", - max_stop_process_count = 2, - pf_iters = 2, - overlap_variable = "None", - pathfinding_type = "stochastic", - iters = 1, - dispersion = 0.50, - num_trips = 5, - number_of_processes = 1) + input_network_dir=INPUT_NETWORK, + input_demand_dir=INPUT_DEMAND, + run_config=CONFIG_FILE, + input_weights=INPUT_WEIGHTS, + input_functions=INPUT_FUNCTIONS, + output_dir=OUTPUT_DIR, + output_folder="test_userclasses", + max_stop_process_count=2, + pf_iters=2, + overlap_variable="None", + pathfinding_type="stochastic", + iters=1, + dispersion=0.50, + num_trips=5, + number_of_processes=1) assert r["passengers_arrived"] > 0 + if __name__ == '__main__': test_user_classes() diff --git a/tests/test_weight_qualifiers.py b/tests/test_weight_qualifiers.py index 440fde4a..ed6eef2e 100644 --- a/tests/test_weight_qualifiers.py +++ b/tests/test_weight_qualifiers.py @@ -9,6 +9,7 @@ sort_cols = ['user_class', 'purpose', 'demand_mode_type', 'demand_mode', 'supply_mode', 'weight_name'] + @pytest.fixture def sample_dataframe(): sample_dict = { @@ -68,6 +69,7 @@ def expected_dataframe(): yield pd.DataFrame(data=expected_dict) + @pytest.mark.travis def test_parse_weight_qualifiers(sample_dataframe, expected_dataframe): """ @@ -79,28 +81,32 @@ def test_parse_weight_qualifiers(sample_dataframe, expected_dataframe): expected_dataframe.sort_values(sort_cols).reset_index(drop=True), check_like=True) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_key(sample_dataframe): """ Test to ensure the validation on qualifier types is working. """ - sample_dataframe.loc[sample_dataframe[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.contains('logistic') , 'weight_name'] = \ + sample_dataframe.loc[sample_dataframe[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.contains('logistic'), 'weight_name'] = \ sample_dataframe[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME].str.replace('logistic', 'logging') with raises(KeyError, message="Expecting KeyError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_logrithmic_qualifier(sample_dataframe): """ Test to ensure the validation on qualifier types is working. """ - sample_dataframe.loc[sample_dataframe[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == 'arrive_late_cost_min.logarithmic.log_base' , 'weight_name'] = \ + sample_dataframe.loc[sample_dataframe[ + PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == 'arrive_late_cost_min.logarithmic.log_base', 'weight_name'] = \ 'arrive_late_cost_min.logarithmic.log_base_bad' with raises(AssertionError, message="Expecting AssertionError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_logrithmic_qualifier_value(sample_dataframe): """ @@ -111,17 +117,20 @@ def test_parse_weight_qualifiers_bad_logrithmic_qualifier_value(sample_dataframe with raises(AssertionError, message="Expecting AssertionError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_logistic_qualifier(sample_dataframe): """ Test to ensure the validation on qualifier types is working. """ - sample_dataframe.loc[sample_dataframe[PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == 'depart_early_cost_min.logistic.logistic_max' , 'weight_name'] = \ + sample_dataframe.loc[sample_dataframe[ + PathSet.WEIGHTS_COLUMN_WEIGHT_NAME] == 'depart_early_cost_min.logistic.logistic_max', 'weight_name'] = \ 'depart_early_cost_min.logistic.logistic_max_bad' with raises(AssertionError, message="Expecting AssertionError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_logistic_max_qualifier_value(sample_dataframe): """ @@ -132,6 +141,7 @@ def test_parse_weight_qualifiers_bad_logistic_max_qualifier_value(sample_datafra with raises(AssertionError, message="Expecting AssertionError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_parse_weight_qualifiers_bad_logistic_mid_qualifier_value(sample_dataframe): """ @@ -142,6 +152,7 @@ def test_parse_weight_qualifiers_bad_logistic_mid_qualifier_value(sample_datafra with raises(AssertionError, message="Expecting AssertionError"): Assignment.process_weight_qualifiers(sample_dataframe) + @pytest.mark.travis def test_no_qualifiers(sample_dataframe): """