GTFS-Flex: database, copier, validation support

adapters/direct/reader.go

@@ -11,34 +11,38 @@ var bufferSize = 1000
 
 // Reader is a mocked up Reader used for testing.
 type Reader struct {
-	AgencyList           []gtfs.Agency
-	RouteList            []gtfs.Route
-	TripList             []gtfs.Trip
-	StopList             []gtfs.Stop
-	StopTimeList         []gtfs.StopTime
-	ShapeList            []gtfs.Shape
-	CalendarList         []gtfs.Calendar
-	CalendarDateList     []gtfs.CalendarDate
-	FeedInfoList         []gtfs.FeedInfo
-	FareRuleList         []gtfs.FareRule
-	FareAttributeList    []gtfs.FareAttribute
-	FrequencyList        []gtfs.Frequency
-	TransferList         []gtfs.Transfer
-	LevelList            []gtfs.Level
-	PathwayList          []gtfs.Pathway
-	AttributionList      []gtfs.Attribution
-	TranslationList      []gtfs.Translation
-	AreaList             []gtfs.Area
-	StopAreaList         []gtfs.StopArea
-	FareLegRuleList      []gtfs.FareLegRule
-	FareTransferRuleList []gtfs.FareTransferRule
-	FareMediaList        []gtfs.FareMedia
-	FareProductList      []gtfs.FareProduct
-	RiderCategoryList    []gtfs.RiderCategory
-	TimeframeList        []gtfs.Timeframe
-	NetworkList          []gtfs.Network
-	RouteNetworkList     []gtfs.RouteNetwork
-	OtherList            []tt.Entity
+	AgencyList            []gtfs.Agency
+	RouteList             []gtfs.Route
+	TripList              []gtfs.Trip
+	StopList              []gtfs.Stop
+	StopTimeList          []gtfs.StopTime
+	ShapeList             []gtfs.Shape
+	CalendarList          []gtfs.Calendar
+	CalendarDateList      []gtfs.CalendarDate
+	FeedInfoList          []gtfs.FeedInfo
+	FareRuleList          []gtfs.FareRule
+	FareAttributeList     []gtfs.FareAttribute
+	FrequencyList         []gtfs.Frequency
+	TransferList          []gtfs.Transfer
+	LevelList             []gtfs.Level
+	PathwayList           []gtfs.Pathway
+	AttributionList       []gtfs.Attribution
+	TranslationList       []gtfs.Translation
+	AreaList              []gtfs.Area
+	StopAreaList          []gtfs.StopArea
+	FareLegRuleList       []gtfs.FareLegRule
+	FareTransferRuleList  []gtfs.FareTransferRule
+	FareMediaList         []gtfs.FareMedia
+	FareProductList       []gtfs.FareProduct
+	RiderCategoryList     []gtfs.RiderCategory
+	TimeframeList         []gtfs.Timeframe
+	NetworkList           []gtfs.Network
+	RouteNetworkList      []gtfs.RouteNetwork
+	LocationGroupList     []gtfs.LocationGroup
+	LocationGroupStopList []gtfs.LocationGroupStop
+	BookingRuleList       []gtfs.BookingRule
+	LocationList          []gtfs.Location
+	OtherList             []tt.Entity
 }
 
 // NewReader returns a new Reader.
@@ -432,3 +436,47 @@ func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork {
 	}()
 	return out
 }
+
+func (mr *Reader) LocationGroups() chan gtfs.LocationGroup {
+	out := make(chan gtfs.LocationGroup, bufferSize)
+	go func() {
+		for _, ent := range mr.LocationGroupList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}
+
+func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop {
+	out := make(chan gtfs.LocationGroupStop, bufferSize)
+	go func() {
+		for _, ent := range mr.LocationGroupStopList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}
+
+func (mr *Reader) BookingRules() chan gtfs.BookingRule {
+	out := make(chan gtfs.BookingRule, bufferSize)
+	go func() {
+		for _, ent := range mr.BookingRuleList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}
+
+func (mr *Reader) Locations() chan gtfs.Location {
+	out := make(chan gtfs.Location, bufferSize)
+	go func() {
+		for _, ent := range mr.LocationList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}

adapters/empty/reader.go

@@ -139,6 +139,34 @@ func (mr *Reader) RiderCategories() chan gtfs.RiderCategory {
 	return readNullEntities[gtfs.RiderCategory](mr)
 }
 
+func (mr *Reader) Timeframes() chan gtfs.Timeframe {
+	return readNullEntities[gtfs.Timeframe](mr)
+}
+
+func (mr *Reader) Networks() chan gtfs.Network {
+	return readNullEntities[gtfs.Network](mr)
+}
+
+func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork {
+	return readNullEntities[gtfs.RouteNetwork](mr)
+}
+
+func (mr *Reader) LocationGroups() chan gtfs.LocationGroup {
+	return readNullEntities[gtfs.LocationGroup](mr)
+}
+
+func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop {
+	return readNullEntities[gtfs.LocationGroupStop](mr)
+}
+
+func (mr *Reader) BookingRules() chan gtfs.BookingRule {
+	return readNullEntities[gtfs.BookingRule](mr)
+}
+
+func (mr *Reader) Locations() chan gtfs.Location {
+	return readNullEntities[gtfs.Location](mr)
+}
+
 func readNullEntities[T any](reader *Reader) chan T {
 	out := make(chan T, bufferSize)
 	go func() {

adapters/multireader/reader.go

@@ -176,6 +176,22 @@ func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork {
 	return readEntities(mr, func(r adapters.Reader) chan gtfs.RouteNetwork { return r.RouteNetworks() }, setFv[*gtfs.RouteNetwork])
 }
 
+func (mr *Reader) LocationGroups() chan gtfs.LocationGroup {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.LocationGroup { return r.LocationGroups() }, setFv[*gtfs.LocationGroup])
+}
+
+func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.LocationGroupStop { return r.LocationGroupStops() }, setFv[*gtfs.LocationGroupStop])
+}
+
+func (mr *Reader) BookingRules() chan gtfs.BookingRule {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.BookingRule { return r.BookingRules() }, setFv[*gtfs.BookingRule])
+}
+
+func (mr *Reader) Locations() chan gtfs.Location {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.Location { return r.Locations() }, setFv[*gtfs.Location])
+}
+
 type canSetFV interface {
 	SetFeedVersionID(int)
 }

copier/copier.go

@@ -291,6 +291,11 @@ func NewCopier(ctx context.Context, reader adapters.Reader, writer adapters.Writ
 			&rules.CalendarDuplicateDates{},
 			&rules.FareProductRiderCategoryDefaultCheck{},
 			&rules.TransferStopLocationTypeCheck{},
+			// GTFS-Flex validators
+			&rules.FlexGeographyIDUniqueCheck{},
+			&rules.FlexStopLocationTypeCheck{},
+			&rules.FlexLocationGeometryCheck{},
+			&rules.FlexZoneIDConditionalCheck{},
 		)
 	}
 
@@ -612,6 +617,10 @@ func (copier *Copier) Copy(ctx context.Context) (*Result, error) {
 			)
 		},
 		copier.copyCalendars,
+		func() error { return batchCopy(copier, batchChan(r.BookingRules(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.Locations(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.LocationGroups(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.LocationGroupStops(), bs, nil)) },
 		copier.copyTripsAndStopTimes,
 		func() error { return batchCopy(copier, batchChan(r.Pathways(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.FareAttributes(), bs, nil)) },

copier/stoppattern_test.go

@@ -11,7 +11,7 @@ import (
 func Benchmark_stopPatternKey(b *testing.B) {
 	stoptimes := []gtfs.StopTime{}
 	for i := 0; i < 50; i++ {
-		stoptimes = append(stoptimes, gtfs.StopTime{StopID: tt.NewString(fmt.Sprintf("%d", i*100))})
+		stoptimes = append(stoptimes, gtfs.StopTime{StopID: tt.NewKey(fmt.Sprintf("%d", i*100))})
 	}
 	m := map[string]int{}
 	b.ResetTimer()

dmfr/feed_version_tables.go

@@ -87,6 +87,7 @@ func GetFeedVersionTables() FeedVersionTables {
 			"gtfs_fare_rules",
 			"gtfs_attributions",
 			"gtfs_translations",
+			"gtfs_location_group_stops",
 		},
 		GtfsEntityTables: []string{
 			"gtfs_fare_media",
@@ -102,6 +103,9 @@ func GetFeedVersionTables() FeedVersionTables {
 			"gtfs_fare_attributes",
 			"gtfs_trips",
 			"gtfs_shapes",
+			"gtfs_booking_rules",
+			"gtfs_location_groups",
+			"gtfs_locations",
 			"gtfs_calendars",
 			"gtfs_stops",
 			"gtfs_levels",

gtfs/booking_rule.go

@@ -0,0 +1,104 @@
+package gtfs
+
+import (
+	"github.com/interline-io/transitland-lib/causes"
+	"github.com/interline-io/transitland-lib/tt"
+)
+
+// BookingRule booking_rules.txt
+type BookingRule struct {
+	BookingRuleID          tt.String `csv:",required"`
+	BookingType            tt.Int    `csv:",required" enum:"0,1,2"`
+	PriorNoticeDurationMin tt.Int
+	PriorNoticeDurationMax tt.Int
+	PriorNoticeLastDay     tt.Int
+	PriorNoticeLastTime    tt.Seconds
+	PriorNoticeStartDay    tt.Int
+	PriorNoticeStartTime   tt.Seconds
+	PriorNoticeServiceID   tt.Key `target:"calendar.txt"`
+	Message                tt.String
+	PickupMessage          tt.String
+	DropOffMessage         tt.String
+	PhoneNumber            tt.String
+	InfoURL                tt.Url
+	BookingURL             tt.Url
+	tt.BaseEntity
+}
+
+func (ent *BookingRule) EntityKey() string {
+	return ent.BookingRuleID.Val
+}
+
+func (ent *BookingRule) EntityID() string {
+	return entID(ent.ID, ent.BookingRuleID.Val)
+}
+
+func (ent *BookingRule) Filename() string {
+	return "booking_rules.txt"
+}
+
+func (ent *BookingRule) TableName() string {
+	return "gtfs_booking_rules"
+}
+
+// UpdateKeys updates Entity references.
+func (ent *BookingRule) UpdateKeys(emap *tt.EntityMap) error {
+	return tt.TrySetField(emap.UpdateKey(&ent.PriorNoticeServiceID, "calendar.txt"), "prior_notice_service_id")
+}
+
+// ConditionalErrors for this Entity.
+func (ent *BookingRule) ConditionalErrors() (errs []error) {
+	bookingType := ent.BookingType.Val
+
+	// booking_type=1: prior_notice_duration_min is required
+	if bookingType == 1 {
+		if !ent.PriorNoticeDurationMin.Valid {
+			errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_duration_min"))
+		}
+		// prior_notice_duration_max is optional for booking_type=1
+	}
+
+	// booking_type=2: prior_notice_last_day is required
+	if bookingType == 2 {
+		if !ent.PriorNoticeLastDay.Valid {
+			errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_last_day"))
+		}
+		// prior_notice_duration_max is forbidden for booking_type=2
+		if ent.PriorNoticeDurationMax.Valid {
+			errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_duration_max", ent.PriorNoticeDurationMax.String(), "prior_notice_duration_max is forbidden for booking_type=2"))
+		}
+	}
+
+	// booking_type=0: prior_notice_duration_max is forbidden
+	if bookingType == 0 {
+		if ent.PriorNoticeDurationMax.Valid {
+			errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_duration_max", ent.PriorNoticeDurationMax.String(), "prior_notice_duration_max is forbidden for booking_type=0"))
+		}
+		// prior_notice_start_day is forbidden for booking_type=0
+		if ent.PriorNoticeStartDay.Valid {
+			errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_start_day", ent.PriorNoticeStartDay.String(), "prior_notice_start_day is forbidden for booking_type=0"))
+		}
+	}
+
+	// prior_notice_last_time requires prior_notice_last_day
+	if ent.PriorNoticeLastTime.Valid && !ent.PriorNoticeLastDay.Valid {
+		errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_last_day"))
+	}
+
+	// prior_notice_start_time requires prior_notice_start_day
+	if ent.PriorNoticeStartTime.Valid && !ent.PriorNoticeStartDay.Valid {
+		errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_start_day"))
+	}
+
+	// prior_notice_service_id is forbidden except for booking_type=2
+	if bookingType != 2 && ent.PriorNoticeServiceID.Valid {
+		errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_service_id", ent.PriorNoticeServiceID.Val, "prior_notice_service_id is only allowed for booking_type=2"))
+	}
+
+	// booking_type=1: prior_notice_start_day forbidden if prior_notice_duration_max is defined
+	if bookingType == 1 && ent.PriorNoticeDurationMax.Valid && ent.PriorNoticeStartDay.Valid {
+		errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_start_day", ent.PriorNoticeStartDay.String(), "prior_notice_start_day is forbidden when prior_notice_duration_max is defined for booking_type=1"))
+	}
+
+	return errs
+}