diff --git a/adapters/direct/reader.go b/adapters/direct/reader.go index aacc38fd6..373de0f51 100644 --- a/adapters/direct/reader.go +++ b/adapters/direct/reader.go @@ -11,34 +11,38 @@ var bufferSize = 1000 // Reader is a mocked up Reader used for testing. type Reader struct { - AgencyList []gtfs.Agency - RouteList []gtfs.Route - TripList []gtfs.Trip - StopList []gtfs.Stop - StopTimeList []gtfs.StopTime - ShapeList []gtfs.Shape - CalendarList []gtfs.Calendar - CalendarDateList []gtfs.CalendarDate - FeedInfoList []gtfs.FeedInfo - FareRuleList []gtfs.FareRule - FareAttributeList []gtfs.FareAttribute - FrequencyList []gtfs.Frequency - TransferList []gtfs.Transfer - LevelList []gtfs.Level - PathwayList []gtfs.Pathway - AttributionList []gtfs.Attribution - TranslationList []gtfs.Translation - AreaList []gtfs.Area - StopAreaList []gtfs.StopArea - FareLegRuleList []gtfs.FareLegRule - FareTransferRuleList []gtfs.FareTransferRule - FareMediaList []gtfs.FareMedia - FareProductList []gtfs.FareProduct - RiderCategoryList []gtfs.RiderCategory - TimeframeList []gtfs.Timeframe - NetworkList []gtfs.Network - RouteNetworkList []gtfs.RouteNetwork - OtherList []tt.Entity + AgencyList []gtfs.Agency + RouteList []gtfs.Route + TripList []gtfs.Trip + StopList []gtfs.Stop + StopTimeList []gtfs.StopTime + ShapeList []gtfs.Shape + CalendarList []gtfs.Calendar + CalendarDateList []gtfs.CalendarDate + FeedInfoList []gtfs.FeedInfo + FareRuleList []gtfs.FareRule + FareAttributeList []gtfs.FareAttribute + FrequencyList []gtfs.Frequency + TransferList []gtfs.Transfer + LevelList []gtfs.Level + PathwayList []gtfs.Pathway + AttributionList []gtfs.Attribution + TranslationList []gtfs.Translation + AreaList []gtfs.Area + StopAreaList []gtfs.StopArea + FareLegRuleList []gtfs.FareLegRule + FareTransferRuleList []gtfs.FareTransferRule + FareMediaList []gtfs.FareMedia + FareProductList []gtfs.FareProduct + RiderCategoryList []gtfs.RiderCategory + TimeframeList []gtfs.Timeframe + NetworkList []gtfs.Network + RouteNetworkList []gtfs.RouteNetwork + LocationGroupList []gtfs.LocationGroup + LocationGroupStopList []gtfs.LocationGroupStop + BookingRuleList []gtfs.BookingRule + LocationList []gtfs.Location + OtherList []tt.Entity } // NewReader returns a new Reader. @@ -432,3 +436,47 @@ func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork { }() return out } + +func (mr *Reader) LocationGroups() chan gtfs.LocationGroup { + out := make(chan gtfs.LocationGroup, bufferSize) + go func() { + for _, ent := range mr.LocationGroupList { + out <- ent + } + close(out) + }() + return out +} + +func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop { + out := make(chan gtfs.LocationGroupStop, bufferSize) + go func() { + for _, ent := range mr.LocationGroupStopList { + out <- ent + } + close(out) + }() + return out +} + +func (mr *Reader) BookingRules() chan gtfs.BookingRule { + out := make(chan gtfs.BookingRule, bufferSize) + go func() { + for _, ent := range mr.BookingRuleList { + out <- ent + } + close(out) + }() + return out +} + +func (mr *Reader) Locations() chan gtfs.Location { + out := make(chan gtfs.Location, bufferSize) + go func() { + for _, ent := range mr.LocationList { + out <- ent + } + close(out) + }() + return out +} diff --git a/adapters/empty/reader.go b/adapters/empty/reader.go index 1b3dbc4ab..613d8558f 100644 --- a/adapters/empty/reader.go +++ b/adapters/empty/reader.go @@ -139,6 +139,34 @@ func (mr *Reader) RiderCategories() chan gtfs.RiderCategory { return readNullEntities[gtfs.RiderCategory](mr) } +func (mr *Reader) Timeframes() chan gtfs.Timeframe { + return readNullEntities[gtfs.Timeframe](mr) +} + +func (mr *Reader) Networks() chan gtfs.Network { + return readNullEntities[gtfs.Network](mr) +} + +func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork { + return readNullEntities[gtfs.RouteNetwork](mr) +} + +func (mr *Reader) LocationGroups() chan gtfs.LocationGroup { + return readNullEntities[gtfs.LocationGroup](mr) +} + +func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop { + return readNullEntities[gtfs.LocationGroupStop](mr) +} + +func (mr *Reader) BookingRules() chan gtfs.BookingRule { + return readNullEntities[gtfs.BookingRule](mr) +} + +func (mr *Reader) Locations() chan gtfs.Location { + return readNullEntities[gtfs.Location](mr) +} + func readNullEntities[T any](reader *Reader) chan T { out := make(chan T, bufferSize) go func() { diff --git a/adapters/multireader/reader.go b/adapters/multireader/reader.go index f4704a8b0..1cf256881 100644 --- a/adapters/multireader/reader.go +++ b/adapters/multireader/reader.go @@ -176,6 +176,22 @@ func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork { return readEntities(mr, func(r adapters.Reader) chan gtfs.RouteNetwork { return r.RouteNetworks() }, setFv[*gtfs.RouteNetwork]) } +func (mr *Reader) LocationGroups() chan gtfs.LocationGroup { + return readEntities(mr, func(r adapters.Reader) chan gtfs.LocationGroup { return r.LocationGroups() }, setFv[*gtfs.LocationGroup]) +} + +func (mr *Reader) LocationGroupStops() chan gtfs.LocationGroupStop { + return readEntities(mr, func(r adapters.Reader) chan gtfs.LocationGroupStop { return r.LocationGroupStops() }, setFv[*gtfs.LocationGroupStop]) +} + +func (mr *Reader) BookingRules() chan gtfs.BookingRule { + return readEntities(mr, func(r adapters.Reader) chan gtfs.BookingRule { return r.BookingRules() }, setFv[*gtfs.BookingRule]) +} + +func (mr *Reader) Locations() chan gtfs.Location { + return readEntities(mr, func(r adapters.Reader) chan gtfs.Location { return r.Locations() }, setFv[*gtfs.Location]) +} + type canSetFV interface { SetFeedVersionID(int) } diff --git a/cmds/merge_cmd_test.go b/cmds/merge_cmd_test.go index 6bd7d65ad..8f62608f3 100644 --- a/cmds/merge_cmd_test.go +++ b/cmds/merge_cmd_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/interline-io/transitland-lib/ext" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tt" "github.com/stretchr/testify/assert" ) @@ -13,8 +13,8 @@ import ( func TestMerge(t *testing.T) { ctx := context.TODO() t.Run("merge", func(t *testing.T) { - f1 := testutil.ExampleFeedBART - f2 := testutil.ExampleFeedCaltrain + f1 := testreader.ExampleFeedBART + f2 := testreader.ExampleFeedCaltrain cmd := MergeCommand{} tdir := t.TempDir() if err := cmd.Parse([]string{tdir, f1.URL, f2.URL}); err != nil { @@ -28,7 +28,7 @@ func TestMerge(t *testing.T) { t.Fatal(err) } entCount := map[string]int{} - testutil.AllEntities(outReader, func(ent tt.Entity) { + testreader.AllEntities(outReader, func(ent tt.Entity) { entCount[ent.Filename()] += 1 }) expectCount := map[string]int{} diff --git a/copier/copier.go b/copier/copier.go index 6da9964e9..51da0c802 100644 --- a/copier/copier.go +++ b/copier/copier.go @@ -291,6 +291,11 @@ func NewCopier(ctx context.Context, reader adapters.Reader, writer adapters.Writ &rules.CalendarDuplicateDates{}, &rules.FareProductRiderCategoryDefaultCheck{}, &rules.TransferStopLocationTypeCheck{}, + // GTFS-Flex validators + &rules.FlexGeographyIDUniqueCheck{}, + &rules.FlexStopLocationTypeCheck{}, + &rules.FlexLocationGeometryCheck{}, + &rules.FlexZoneIDConditionalCheck{}, ) } @@ -612,6 +617,10 @@ func (copier *Copier) Copy(ctx context.Context) (*Result, error) { ) }, copier.copyCalendars, + func() error { return batchCopy(copier, batchChan(r.BookingRules(), bs, nil)) }, + func() error { return batchCopy(copier, batchChan(r.Locations(), bs, nil)) }, + func() error { return batchCopy(copier, batchChan(r.LocationGroups(), bs, nil)) }, + func() error { return batchCopy(copier, batchChan(r.LocationGroupStops(), bs, nil)) }, copier.copyTripsAndStopTimes, func() error { return batchCopy(copier, batchChan(r.Pathways(), bs, nil)) }, func() error { return batchCopy(copier, batchChan(r.FareAttributes(), bs, nil)) }, @@ -1016,6 +1025,9 @@ func (copier *Copier) logCount(ent tt.Entity) { func (copier *Copier) createMissingShape(shapeID string, stoptimes []gtfs.StopTime) (string, error) { stopids := []string{} for _, st := range stoptimes { + if !st.StopID.Valid { + continue + } stopids = append(stopids, st.StopID.Val) } line, dists, err := copier.geomCache.MakeShape(stopids...) diff --git a/copier/stoppattern.go b/copier/stoppattern.go index 6ab70476f..cb89c2e0d 100644 --- a/copier/stoppattern.go +++ b/copier/stoppattern.go @@ -9,9 +9,11 @@ import ( ) func stopPatternKey(stoptimes []gtfs.StopTime) string { - key := make([]string, len(stoptimes)) + key := make([]string, 0, len(stoptimes)) for i := 0; i < len(stoptimes); i++ { - key[i] = stoptimes[i].StopID.Val + if stoptimes[i].StopID.Valid { + key = append(key, stoptimes[i].StopID.Val) + } } return strings.Join(key, string(byte(0))) } @@ -34,11 +36,15 @@ func journeyPatternKey(trip *gtfs.Trip) string { ))) for i := 0; i < len(trip.StopTimes); i++ { st := trip.StopTimes[i] + stopID := "" + if st.StopID.Valid { + stopID = st.StopID.Val + } m.Write([]byte(fmt.Sprintf( "%d-%d-%s-%s-%d-%d-%d", st.ArrivalTime.Val-a.Val, st.DepartureTime.Val-b.Val, - st.StopID.Val, + stopID, st.StopHeadsign.Val, st.PickupType.Val, st.DropOffType.Val, diff --git a/copier/stoppattern_test.go b/copier/stoppattern_test.go index baa14ae59..3ddeb7c45 100644 --- a/copier/stoppattern_test.go +++ b/copier/stoppattern_test.go @@ -11,7 +11,7 @@ import ( func Benchmark_stopPatternKey(b *testing.B) { stoptimes := []gtfs.StopTime{} for i := 0; i < 50; i++ { - stoptimes = append(stoptimes, gtfs.StopTime{StopID: tt.NewString(fmt.Sprintf("%d", i*100))}) + stoptimes = append(stoptimes, gtfs.StopTime{StopID: tt.NewKey(fmt.Sprintf("%d", i*100))}) } m := map[string]int{} b.ResetTimer() diff --git a/dmfr/feed_version_tables.go b/dmfr/feed_version_tables.go index 38f280bd7..f1608f0cf 100644 --- a/dmfr/feed_version_tables.go +++ b/dmfr/feed_version_tables.go @@ -87,6 +87,7 @@ func GetFeedVersionTables() FeedVersionTables { "gtfs_fare_rules", "gtfs_attributions", "gtfs_translations", + "gtfs_location_group_stops", }, GtfsEntityTables: []string{ "gtfs_fare_media", @@ -102,6 +103,9 @@ func GetFeedVersionTables() FeedVersionTables { "gtfs_fare_attributes", "gtfs_trips", "gtfs_shapes", + "gtfs_booking_rules", + "gtfs_location_groups", + "gtfs_locations", "gtfs_calendars", "gtfs_stops", "gtfs_levels", diff --git a/ext/builders/agency_place_builder.go b/ext/builders/agency_place_builder.go index a67ac9bec..8dc4f00d4 100644 --- a/ext/builders/agency_place_builder.go +++ b/ext/builders/agency_place_builder.go @@ -62,6 +62,9 @@ func (pp *AgencyPlaceBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.Ent case *gtfs.Trip: pp.tripAgency[eid] = pp.routeAgency[v.RouteID.Val] case *gtfs.StopTime: + if !v.StopID.Valid { + return nil + } aid := pp.tripAgency[v.TripID.Val] if sg, ok := pp.stops[v.StopID.Val]; ok { if v, ok := pp.agencyStops[aid]; ok { diff --git a/ext/builders/convex_hull_builder.go b/ext/builders/convex_hull_builder.go index 525bdc528..f1a94232b 100644 --- a/ext/builders/convex_hull_builder.go +++ b/ext/builders/convex_hull_builder.go @@ -73,6 +73,9 @@ func (pp *ConvexHullBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.Enti case *gtfs.Trip: pp.tripRoutes[eid] = v.RouteID.Val case *gtfs.StopTime: + if !v.StopID.Valid { + return nil + } r, ok := pp.routeStopGeoms[pp.tripRoutes[v.TripID.Val]] if !ok { // log.For(ctx).Debug().Msgf("no route:", v.TripID, pp.tripRoutes[v.TripID]) diff --git a/ext/builders/convex_hull_builder_test.go b/ext/builders/convex_hull_builder_test.go index 638de1634..2be8989b8 100644 --- a/ext/builders/convex_hull_builder_test.go +++ b/ext/builders/convex_hull_builder_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/stretchr/testify/assert" ) @@ -19,7 +19,7 @@ func TestConvexHullBuilder(t *testing.T) { } groups := map[string]testgroup{ "ExampleFeed": { - testutil.ExampleZip.URL, + testreader.ExampleZip.URL, []testcase{ { FeedVersionGeometry: []float64{-117.133162, 36.425288, -116.81797, 36.88108, -116.76821, 36.914893, -116.751677, 36.915682, -116.40094, 36.641496, -117.133162, 36.425288}, @@ -30,7 +30,7 @@ func TestConvexHullBuilder(t *testing.T) { }, }, "Caltrain": { - testutil.ExampleFeedCaltrain.URL, + testreader.ExampleFeedCaltrain.URL, []testcase{{ FeedVersionGeometry: []float64{-121.566225, 37.003485, -122.232, 37.486101, -122.386832, 37.599797, -122.412076, 37.631108, -122.394992, 37.77639, -122.394935, 37.776348, -121.650244, 37.129363, -121.610936, 37.086653, -121.610049, 37.085225, -121.566088, 37.003538, -121.566225, 37.003485}, AgencyGeoms: map[string][]float64{ @@ -39,7 +39,7 @@ func TestConvexHullBuilder(t *testing.T) { }}, }, "BART": { - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, []testcase{ { FeedVersionGeometry: []float64{-121.939313, 37.502171, -122.386702, 37.600271, -122.466233, 37.684638, -122.469081, 37.706121, -122.353099, 37.936853, -122.024653, 38.003193, -121.945154, 38.018914, -121.889457, 38.016941, -121.78042, 37.995388, -121.939313, 37.502171}, diff --git a/ext/builders/onestop_id_builder.go b/ext/builders/onestop_id_builder.go index 1a49a0f0a..136ad60f6 100644 --- a/ext/builders/onestop_id_builder.go +++ b/ext/builders/onestop_id_builder.go @@ -94,6 +94,9 @@ func (pp *OnestopIDBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.Entit case *gtfs.Trip: pp.tripRoutes[eid] = v.RouteID.Val case *gtfs.StopTime: + if !v.StopID.Valid { + return nil + } r, ok := pp.routeStopGeoms[pp.tripRoutes[v.TripID.Val]] if !ok { // log.For(ctx).Debug().Msgf("OnestopIDBuilder no route:", v.TripID, pp.tripRoutes[v.TripID]) diff --git a/ext/builders/onestop_id_builder_test.go b/ext/builders/onestop_id_builder_test.go index 91444c1b5..6cea88299 100644 --- a/ext/builders/onestop_id_builder_test.go +++ b/ext/builders/onestop_id_builder_test.go @@ -3,7 +3,7 @@ package builders import ( "testing" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/stretchr/testify/assert" ) @@ -19,7 +19,7 @@ func TestOnestopIDBuilder(t *testing.T) { } groups := map[string]testgroup{ "ExampleFeed": { - testutil.ExampleZip.URL, + testreader.ExampleZip.URL, []testcase{ {"o-9qs-demotransitauthority", hw{"o-9qs-demotransitauthority": []string{"DTA"}}}, {"r-9qsb-20", hw{"r-9qsb-20": []string{"BFC"}}}, @@ -39,7 +39,7 @@ func TestOnestopIDBuilder(t *testing.T) { }, }, "Caltrain": { - testutil.ExampleFeedCaltrain.URL, + testreader.ExampleFeedCaltrain.URL, []testcase{ {"o-9q9-caltrain", hw{"o-9q9-caltrain": []string{"caltrain-ca-us"}}}, {"r-9q9-limited", hw{"r-9q9-limited": []string{"Li-130"}}}, @@ -57,7 +57,7 @@ func TestOnestopIDBuilder(t *testing.T) { }, }, "BART": { - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, []testcase{ {"o-9q9-bayarearapidtransit", hw{"o-9q9-bayarearapidtransit": []string{"BART"}}}, {"r-9q8y-richmond~dalycity~millbrae", hw{"r-9q8y-richmond~dalycity~millbrae": []string{"07"}}}, diff --git a/ext/builders/route_geometry_builder_test.go b/ext/builders/route_geometry_builder_test.go index e65df9a6d..600e13a76 100644 --- a/ext/builders/route_geometry_builder_test.go +++ b/ext/builders/route_geometry_builder_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlxy" "github.com/stretchr/testify/assert" "github.com/twpayne/go-geom" @@ -34,14 +34,14 @@ func TestRouteGeometryBuilder(t *testing.T) { } groups := map[string]testgroup{ "Caltrain": { - testutil.ExampleFeedCaltrain.URL, + testreader.ExampleFeedCaltrain.URL, []testcase{ {RouteID: "Bu-130", ExpectLength: 75274.982973, ExpectLineStrings: 4}, {RouteID: "Lo-130", ExpectLength: 75274.982973, ExpectLineStrings: 4}, }, }, "BART": { - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, []testcase{ {RouteID: "07", ExpectLength: 58890.123340, ExpectLineStrings: 2}, {RouteID: "03", ExpectLength: 65574.875547, ExpectLineStrings: 2}, diff --git a/ext/builders/route_headway_builder.go b/ext/builders/route_headway_builder.go index 94d26c815..f50292d7f 100644 --- a/ext/builders/route_headway_builder.go +++ b/ext/builders/route_headway_builder.go @@ -81,6 +81,9 @@ func (pp *RouteHeadwayBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.En // Process StopTimes assuming they will all be written // otherwise this breaks on journey pattern deduplication. for _, st := range v.StopTimes { + if !st.StopID.Valid { + continue + } stopId, ok := emap.Get("stops.txt", st.StopID.Val) if !ok { continue diff --git a/ext/builders/route_headway_builder_test.go b/ext/builders/route_headway_builder_test.go index bd3b5bd46..ac22bd57a 100644 --- a/ext/builders/route_headway_builder_test.go +++ b/ext/builders/route_headway_builder_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/stretchr/testify/assert" ) @@ -26,14 +26,14 @@ func TestRouteHeadwayBuilder(t *testing.T) { } groups := map[string]testgroup{ "Caltrain": { - testutil.ExampleFeedCaltrain.URL, + testreader.ExampleFeedCaltrain.URL, []testcase{ {RouteID: "Bu-130", DowCat: 1, DirectionID: 0, StopID: "70011", ServiceDate: "2017-10-02"}, {RouteID: "Lo-130", DowCat: 1, DirectionID: 0, StopID: "70011", ServiceDate: "2017-10-02"}, }, }, "BART": { - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, []testcase{ {RouteID: "07", DowCat: 1, DirectionID: 0, StopID: "12TH", ServiceDate: "2018-05-29", HeadwaySecs: 900}, {RouteID: "07", DowCat: 1, DirectionID: 1, StopID: "12TH", ServiceDate: "2018-05-29", HeadwaySecs: 900}, diff --git a/ext/builders/route_stop_builder.go b/ext/builders/route_stop_builder.go index 1608ee643..ce470c385 100644 --- a/ext/builders/route_stop_builder.go +++ b/ext/builders/route_stop_builder.go @@ -45,6 +45,9 @@ func (pp *RouteStopBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.Entit case *gtfs.Trip: pp.tripRoutes[eid] = v.RouteID.Val case *gtfs.StopTime: + if !v.StopID.Valid { + return nil + } rid := pp.tripRoutes[v.TripID.Val] rs, ok := pp.routeStops[rid] if !ok { diff --git a/ext/builders/route_stop_builder_test.go b/ext/builders/route_stop_builder_test.go index b5e90811a..65cf860e7 100644 --- a/ext/builders/route_stop_builder_test.go +++ b/ext/builders/route_stop_builder_test.go @@ -6,7 +6,7 @@ import ( "github.com/interline-io/transitland-lib/adapters/direct" "github.com/interline-io/transitland-lib/copier" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlcsv" "github.com/stretchr/testify/assert" ) @@ -31,7 +31,7 @@ func newMockCopier(url string, exts ...any) (*copier.Result, *direct.Writer, err func TestRouteStopBuilder(t *testing.T) { e := NewRouteStopBuilder() - _, writer, err := newMockCopier(testutil.ExampleFeedBART.URL, e) + _, writer, err := newMockCopier(testreader.ExampleFeedBART.URL, e) if err != nil { t.Fatal(err) } diff --git a/extract/marker_test.go b/extract/marker_test.go index d07d255ca..6c2747fa3 100644 --- a/extract/marker_test.go +++ b/extract/marker_test.go @@ -5,7 +5,7 @@ import ( "github.com/interline-io/transitland-lib/internal/graph" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlcsv" ) @@ -18,7 +18,7 @@ func nn(filename, eid string) node { func TestExtract_Filter_BART(t *testing.T) { em := NewMarker() - reader, err := tlcsv.NewReader(testutil.ExampleFeedBART.URL) + reader, err := tlcsv.NewReader(testreader.ExampleFeedBART.URL) if err != nil { t.Error(err) } @@ -41,7 +41,7 @@ func TestExtract_Filter_BART(t *testing.T) { func TestExtract_Bbox(t *testing.T) { em := NewMarker() em.bbox = "-122.276929,37.794923,-122.259099,37.834413" - reader, err := tlcsv.NewReader(testutil.ExampleFeedBART.URL) + reader, err := tlcsv.NewReader(testreader.ExampleFeedBART.URL) if err != nil { t.Error(err) } diff --git a/fetch/static_fetch_test.go b/fetch/static_fetch_test.go index dfdf8553d..966b4d31c 100644 --- a/fetch/static_fetch_test.go +++ b/fetch/static_fetch_test.go @@ -11,12 +11,12 @@ import ( "github.com/interline-io/transitland-lib/dmfr" "github.com/interline-io/transitland-lib/internal/testdb" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tldb" "github.com/stretchr/testify/assert" ) -var ExampleZip = testutil.ExampleZip +var ExampleZip = testreader.ExampleZip func TestStaticFetch(t *testing.T) { basedir := "" diff --git a/filters/redate_filter_test.go b/filters/redate_filter_test.go index e5a7a9973..78293aeed 100644 --- a/filters/redate_filter_test.go +++ b/filters/redate_filter_test.go @@ -7,7 +7,7 @@ import ( "github.com/interline-io/transitland-lib/adapters/direct" "github.com/interline-io/transitland-lib/gtfs" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/service" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tt" @@ -46,7 +46,7 @@ func TestRedateFilter(t *testing.T) { {"no target days", "2018-06-04", "2022-01-03", 7, 0, true, service.Service{}}, {"different weekday", "2018-06-04", "2022-01-04", 7, 7, true, service.Service{}}, } - reader, err := tlcsv.NewReader(testutil.ExampleFeedBART.URL) + reader, err := tlcsv.NewReader(testreader.ExampleFeedBART.URL) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestRedateFilter(t *testing.T) { } // rf.AllowInactive = true w := direct.NewWriter() - cp, err := testutil.NewDirectCopier(reader, w, testutil.DirectCopierOptions{}) + cp, err := testreader.NewDirectCopier(reader, w, testreader.DirectCopierOptions{}) if err != nil { t.Fatal(err) } diff --git a/gtfs/booking_rule.go b/gtfs/booking_rule.go new file mode 100644 index 000000000..157db5390 --- /dev/null +++ b/gtfs/booking_rule.go @@ -0,0 +1,104 @@ +package gtfs + +import ( + "github.com/interline-io/transitland-lib/causes" + "github.com/interline-io/transitland-lib/tt" +) + +// BookingRule booking_rules.txt +type BookingRule struct { + BookingRuleID tt.String `csv:",required"` + BookingType tt.Int `csv:",required" enum:"0,1,2"` + PriorNoticeDurationMin tt.Int + PriorNoticeDurationMax tt.Int + PriorNoticeLastDay tt.Int + PriorNoticeLastTime tt.Seconds + PriorNoticeStartDay tt.Int + PriorNoticeStartTime tt.Seconds + PriorNoticeServiceID tt.Key `target:"calendar.txt"` + Message tt.String + PickupMessage tt.String + DropOffMessage tt.String + PhoneNumber tt.String + InfoURL tt.Url + BookingURL tt.Url + tt.BaseEntity +} + +func (ent *BookingRule) EntityKey() string { + return ent.BookingRuleID.Val +} + +func (ent *BookingRule) EntityID() string { + return entID(ent.ID, ent.BookingRuleID.Val) +} + +func (ent *BookingRule) Filename() string { + return "booking_rules.txt" +} + +func (ent *BookingRule) TableName() string { + return "gtfs_booking_rules" +} + +// UpdateKeys updates Entity references. +func (ent *BookingRule) UpdateKeys(emap *tt.EntityMap) error { + return tt.TrySetField(emap.UpdateKey(&ent.PriorNoticeServiceID, "calendar.txt"), "prior_notice_service_id") +} + +// ConditionalErrors for this Entity. +func (ent *BookingRule) ConditionalErrors() (errs []error) { + bookingType := ent.BookingType.Val + + // booking_type=1: prior_notice_duration_min is required + if bookingType == 1 { + if !ent.PriorNoticeDurationMin.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_duration_min")) + } + // prior_notice_duration_max is optional for booking_type=1 + } + + // booking_type=2: prior_notice_last_day is required + if bookingType == 2 { + if !ent.PriorNoticeLastDay.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_last_day")) + } + // prior_notice_duration_max is forbidden for booking_type=2 + if ent.PriorNoticeDurationMax.Valid { + errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_duration_max", ent.PriorNoticeDurationMax.String(), "prior_notice_duration_max is forbidden for booking_type=2")) + } + } + + // booking_type=0: prior_notice_duration_max is forbidden + if bookingType == 0 { + if ent.PriorNoticeDurationMax.Valid { + errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_duration_max", ent.PriorNoticeDurationMax.String(), "prior_notice_duration_max is forbidden for booking_type=0")) + } + // prior_notice_start_day is forbidden for booking_type=0 + if ent.PriorNoticeStartDay.Valid { + errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_start_day", ent.PriorNoticeStartDay.String(), "prior_notice_start_day is forbidden for booking_type=0")) + } + } + + // prior_notice_last_time requires prior_notice_last_day + if ent.PriorNoticeLastTime.Valid && !ent.PriorNoticeLastDay.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_last_day")) + } + + // prior_notice_start_time requires prior_notice_start_day + if ent.PriorNoticeStartTime.Valid && !ent.PriorNoticeStartDay.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("prior_notice_start_day")) + } + + // prior_notice_service_id is forbidden except for booking_type=2 + if bookingType != 2 && ent.PriorNoticeServiceID.Valid { + errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_service_id", ent.PriorNoticeServiceID.Val, "prior_notice_service_id is only allowed for booking_type=2")) + } + + // booking_type=1: prior_notice_start_day forbidden if prior_notice_duration_max is defined + if bookingType == 1 && ent.PriorNoticeDurationMax.Valid && ent.PriorNoticeStartDay.Valid { + errs = append(errs, causes.NewConditionallyForbiddenFieldError("prior_notice_start_day", ent.PriorNoticeStartDay.String(), "prior_notice_start_day is forbidden when prior_notice_duration_max is defined for booking_type=1")) + } + + return errs +} diff --git a/gtfs/booking_rule_test.go b/gtfs/booking_rule_test.go new file mode 100644 index 000000000..c8f176e1f --- /dev/null +++ b/gtfs/booking_rule_test.go @@ -0,0 +1,292 @@ +package gtfs + +import ( + "testing" + + "github.com/interline-io/transitland-lib/tt" +) + +func TestBookingRule_ConditionalErrors(t *testing.T) { + tests := []struct { + name string + bookingRule *BookingRule + expectedErrors []ExpectError + }{ + // ===== BOOKING_TYPE=0 TESTS ===== + { + name: "Valid: booking_type=0 with minimal fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule1"), + BookingType: tt.NewInt(0), + PriorNoticeDurationMin: tt.NewInt(15), + }, + expectedErrors: nil, + }, + { + name: "Invalid: booking_type=0 with prior_notice_duration_max", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule2"), + BookingType: tt.NewInt(0), + PriorNoticeDurationMin: tt.NewInt(15), + PriorNoticeDurationMax: tt.NewInt(60), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_duration_max"), + }, + { + name: "Invalid: booking_type=0 with prior_notice_start_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule3"), + BookingType: tt.NewInt(0), + PriorNoticeDurationMin: tt.NewInt(15), + PriorNoticeStartDay: tt.NewInt(1), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_start_day"), + }, + { + name: "Invalid: booking_type=0 with both forbidden fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule4"), + BookingType: tt.NewInt(0), + PriorNoticeDurationMin: tt.NewInt(15), + PriorNoticeDurationMax: tt.NewInt(60), + PriorNoticeStartDay: tt.NewInt(1), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyForbiddenFieldError:prior_notice_duration_max", + "ConditionallyForbiddenFieldError:prior_notice_start_day", + ), + }, + { + name: "Invalid: booking_type=0 with prior_notice_service_id", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule5"), + BookingType: tt.NewInt(0), + PriorNoticeServiceID: tt.NewKey("service1"), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_service_id"), + }, + + // ===== BOOKING_TYPE=1 TESTS ===== + { + name: "Invalid: booking_type=1 without prior_notice_duration_min", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule6"), + BookingType: tt.NewInt(1), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:prior_notice_duration_min"), + }, + { + name: "Valid: booking_type=1 with required fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule7"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + }, + expectedErrors: nil, + }, + { + name: "Valid: booking_type=1 with optional prior_notice_duration_max", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule8"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + PriorNoticeDurationMax: tt.NewInt(120), + }, + expectedErrors: nil, + }, + { + name: "Invalid: booking_type=1 with prior_notice_duration_max and prior_notice_start_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule9"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + PriorNoticeDurationMax: tt.NewInt(120), + PriorNoticeStartDay: tt.NewInt(1), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_start_day"), + }, + { + name: "Valid: booking_type=1 with prior_notice_start_day without prior_notice_duration_max", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule10"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + PriorNoticeStartDay: tt.NewInt(1), + }, + expectedErrors: nil, + }, + { + name: "Invalid: booking_type=1 with prior_notice_service_id", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule11"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + PriorNoticeServiceID: tt.NewKey("service1"), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_service_id"), + }, + + // ===== BOOKING_TYPE=2 TESTS ===== + { + name: "Invalid: booking_type=2 without prior_notice_last_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule12"), + BookingType: tt.NewInt(2), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:prior_notice_last_day"), + }, + { + name: "Valid: booking_type=2 with required fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule13"), + BookingType: tt.NewInt(2), + PriorNoticeLastDay: tt.NewInt(1), + }, + expectedErrors: nil, + }, + { + name: "Invalid: booking_type=2 with prior_notice_duration_max", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule14"), + BookingType: tt.NewInt(2), + PriorNoticeLastDay: tt.NewInt(1), + PriorNoticeDurationMax: tt.NewInt(120), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:prior_notice_duration_max"), + }, + { + name: "Valid: booking_type=2 with prior_notice_service_id", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule15"), + BookingType: tt.NewInt(2), + PriorNoticeLastDay: tt.NewInt(1), + PriorNoticeServiceID: tt.NewKey("service1"), + }, + expectedErrors: nil, + }, + { + name: "Invalid: booking_type=2 with both prior_notice_last_day missing and prior_notice_duration_max present", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule16"), + BookingType: tt.NewInt(2), + PriorNoticeDurationMax: tt.NewInt(120), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:prior_notice_last_day", + "ConditionallyForbiddenFieldError:prior_notice_duration_max", + ), + }, + + // ===== TIME AND DAY DEPENDENCY TESTS ===== + { + name: "Invalid: prior_notice_last_time without prior_notice_last_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule17"), + BookingType: tt.NewInt(1), + PriorNoticeLastTime: tt.NewSeconds(3600), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:prior_notice_duration_min", + "ConditionallyRequiredFieldError:prior_notice_last_day", + ), + }, + { + name: "Valid: prior_notice_last_time with prior_notice_last_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule18"), + BookingType: tt.NewInt(2), + PriorNoticeLastDay: tt.NewInt(1), + PriorNoticeLastTime: tt.NewSeconds(3600), + }, + expectedErrors: nil, + }, + { + name: "Invalid: prior_notice_start_time without prior_notice_start_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule19"), + BookingType: tt.NewInt(1), + PriorNoticeStartTime: tt.NewSeconds(7200), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:prior_notice_duration_min", + "ConditionallyRequiredFieldError:prior_notice_start_day", + ), + }, + { + name: "Valid: prior_notice_start_time with prior_notice_start_day", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule20"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(30), + PriorNoticeStartDay: tt.NewInt(2), + PriorNoticeStartTime: tt.NewSeconds(7200), + }, + expectedErrors: nil, + }, + + // ===== COMPLEX SCENARIOS ===== + { + name: "Valid: booking_type=2 with all compatible optional fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule21"), + BookingType: tt.NewInt(2), + PriorNoticeLastDay: tt.NewInt(3), + PriorNoticeLastTime: tt.NewSeconds(3600), + PriorNoticeStartDay: tt.NewInt(1), + PriorNoticeStartTime: tt.NewSeconds(7200), + PriorNoticeServiceID: tt.NewKey("service1"), + Message: tt.NewString("Please book in advance"), + PhoneNumber: tt.NewString("555-1234"), + }, + expectedErrors: nil, + }, + { + name: "Valid: booking_type=1 with all compatible optional fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule22"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMin: tt.NewInt(15), + PriorNoticeDurationMax: tt.NewInt(90), + Message: tt.NewString("Advanced booking required"), + PickupMessage: tt.NewString("Call when ready"), + DropOffMessage: tt.NewString("Confirm drop-off"), + }, + expectedErrors: nil, + }, + { + name: "Valid: booking_type=0 with all compatible optional fields", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule23"), + BookingType: tt.NewInt(0), + PriorNoticeDurationMin: tt.NewInt(10), + Message: tt.NewString("Same-day service"), + InfoURL: tt.NewUrl("https://example.com/info"), + BookingURL: tt.NewUrl("https://example.com/book"), + }, + expectedErrors: nil, + }, + { + name: "Invalid: Multiple errors across different rules", + bookingRule: &BookingRule{ + BookingRuleID: tt.NewString("rule24"), + BookingType: tt.NewInt(1), + PriorNoticeDurationMax: tt.NewInt(120), + PriorNoticeStartDay: tt.NewInt(1), + PriorNoticeStartTime: tt.NewSeconds(3600), + PriorNoticeServiceID: tt.NewKey("service1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:prior_notice_duration_min", + "ConditionallyForbiddenFieldError:prior_notice_service_id", + "ConditionallyForbiddenFieldError:prior_notice_start_day", + ), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := tc.bookingRule.ConditionalErrors() + CheckErrors(tc.expectedErrors, errs, t) + }) + } +} diff --git a/gtfs/gtfs.go b/gtfs/gtfs.go index 48c8fa19c..1ea55bc28 100644 --- a/gtfs/gtfs.go +++ b/gtfs/gtfs.go @@ -39,4 +39,8 @@ type Reader interface { Timeframes() chan Timeframe Networks() chan Network RouteNetworks() chan RouteNetwork + LocationGroups() chan LocationGroup + LocationGroupStops() chan LocationGroupStop + BookingRules() chan BookingRule + Locations() chan Location } diff --git a/gtfs/gtfs_flex_test.go b/gtfs/gtfs_flex_test.go new file mode 100644 index 000000000..0d01040ae --- /dev/null +++ b/gtfs/gtfs_flex_test.go @@ -0,0 +1,123 @@ +package gtfs + +import ( + "testing" + + "github.com/interline-io/transitland-lib/tt" + "github.com/stretchr/testify/assert" +) + +func TestLocationGroup(t *testing.T) { + lg := LocationGroup{ + LocationGroupID: tt.NewString("lg1"), + LocationGroupName: tt.NewString("Downtown"), + } + assert.Equal(t, "lg1", lg.EntityKey()) + assert.Equal(t, "location_groups.txt", lg.Filename()) + assert.Equal(t, "gtfs_location_groups", lg.TableName()) +} + +func TestLocationGroupStop(t *testing.T) { + lgs := LocationGroupStop{ + LocationGroupID: tt.NewKey("lg1"), + StopID: tt.NewKey("stop1"), + } + assert.Equal(t, "location_group_stops.txt", lgs.Filename()) + assert.Equal(t, "gtfs_location_group_stops", lgs.TableName()) +} + +func TestBookingRule(t *testing.T) { + t.Run("booking_type=0", func(t *testing.T) { + br := BookingRule{ + BookingRuleID: tt.NewString("rule1"), + BookingType: tt.NewInt(0), + } + assert.Equal(t, "rule1", br.EntityKey()) + assert.Equal(t, "booking_rules.txt", br.Filename()) + assert.Equal(t, "gtfs_booking_rules", br.TableName()) + + // booking_type=0: prior_notice_duration_max forbidden + errs := br.ConditionalErrors() + assert.Len(t, errs, 0) + + br.PriorNoticeDurationMax = tt.NewInt(30) + errs = br.ConditionalErrors() + assert.Greater(t, len(errs), 0) + }) + + t.Run("booking_type=1", func(t *testing.T) { + br := BookingRule{ + BookingRuleID: tt.NewString("rule2"), + BookingType: tt.NewInt(1), + } + + // booking_type=1: prior_notice_duration_min required + errs := br.ConditionalErrors() + assert.Greater(t, len(errs), 0) + + br.PriorNoticeDurationMin = tt.NewInt(15) + errs = br.ConditionalErrors() + assert.Len(t, errs, 0) + }) + + t.Run("booking_type=2", func(t *testing.T) { + br := BookingRule{ + BookingRuleID: tt.NewString("rule3"), + BookingType: tt.NewInt(2), + } + + // booking_type=2: prior_notice_last_day required + errs := br.ConditionalErrors() + assert.Greater(t, len(errs), 0) + + br.PriorNoticeLastDay = tt.NewInt(1) + errs = br.ConditionalErrors() + assert.Len(t, errs, 0) + + // booking_type=2: prior_notice_duration_max forbidden + br.PriorNoticeDurationMax = tt.NewInt(30) + errs = br.ConditionalErrors() + assert.Greater(t, len(errs), 0) + }) +} + +func TestLocation(t *testing.T) { + loc := Location{ + LocationID: tt.NewString("loc1"), + StopName: tt.NewString("Flexible Zone"), + StopDesc: tt.NewString("On-demand service area"), + } + assert.Equal(t, "loc1", loc.EntityKey()) + assert.Equal(t, "locations.geojson", loc.Filename()) + assert.Equal(t, "gtfs_locations", loc.TableName()) + + // Geometry is required + errs := loc.ConditionalErrors() + assert.Greater(t, len(errs), 0) +} + +func TestStopTimeFlexFields(t *testing.T) { + st := StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + StartPickupDropOffWindow: tt.NewSeconds(28800), // 8:00:00 + EndPickupDropOffWindow: tt.NewSeconds(32400), // 9:00:00 + PickupBookingRuleID: tt.NewKey("rule1"), + MeanDurationFactor: tt.NewFloat(1.5), + SafeDurationFactor: tt.NewFloat(2.0), + } + + // Verify fields can be set and retrieved + val, err := st.GetString("start_pickup_drop_off_window") + assert.NoError(t, err) + assert.Equal(t, "08:00:00", val) + + val, err = st.GetString("pickup_booking_rule_id") + assert.NoError(t, err) + assert.Equal(t, "rule1", val) + + val, err = st.GetString("mean_duration_factor") + assert.NoError(t, err) + assert.Contains(t, val, "1.5") +} diff --git a/gtfs/gtfs_test.go b/gtfs/gtfs_test.go index c989d8b26..cca7d721b 100644 --- a/gtfs/gtfs_test.go +++ b/gtfs/gtfs_test.go @@ -4,11 +4,27 @@ import ( "fmt" "testing" + "github.com/interline-io/transitland-lib/internal/testutil" "github.com/interline-io/transitland-lib/tt" ) ///////////////////////// +// ExpectError is an alias to testutil.ExpectError +type ExpectError = testutil.ExpectError + +// ParseExpectErrors wraps testutil.ParseExpectErrors +func ParseExpectErrors(errorStrings ...string) []ExpectError { + return testutil.ParseExpectErrors(errorStrings...) +} + +// CheckErrors wraps testutil.CheckErrors +func CheckErrors(expectedErrors []ExpectError, errs []error, t *testing.T) { + testutil.CheckErrors(expectedErrors, errs, t) +} + +///////////////////////// + // frequencies func TestFrequencyRepeatCount(t *testing.T) { diff --git a/gtfs/location.go b/gtfs/location.go new file mode 100644 index 000000000..2865e0b44 --- /dev/null +++ b/gtfs/location.go @@ -0,0 +1,47 @@ +package gtfs + +import ( + "github.com/interline-io/transitland-lib/causes" + "github.com/interline-io/transitland-lib/tt" +) + +// Location represents a GeoJSON feature from locations.geojson +// Locations define zones using GeoJSON Polygon or MultiPolygon geometries +// where riders can request pickups or drop-offs for flexible services +type Location struct { + // ID is the feature ID from the GeoJSON, shares namespace with stop_id + LocationID tt.String `json:"id" csv:",required"` + // Properties + StopName tt.String `json:"stop_name"` + StopDesc tt.String `json:"stop_desc"` + ZoneID tt.String `json:"zone_id"` + StopURL tt.Url `json:"stop_url"` + Geometry tt.Geometry `json:"geometry"` + tt.BaseEntity +} + +func (ent *Location) EntityKey() string { + return ent.LocationID.Val +} + +func (ent *Location) EntityID() string { + return entID(ent.ID, ent.LocationID.Val) +} + +func (ent *Location) Filename() string { + return "locations.geojson" +} + +func (ent *Location) TableName() string { + return "gtfs_locations" +} + +// ConditionalErrors for this Entity. +func (ent *Location) ConditionalErrors() (errs []error) { + // Geometry must be present + if !ent.Geometry.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("geometry")) + } + + return errs +} diff --git a/gtfs/location_group.go b/gtfs/location_group.go new file mode 100644 index 000000000..30604e3c5 --- /dev/null +++ b/gtfs/location_group.go @@ -0,0 +1,29 @@ +package gtfs + +import ( + "github.com/interline-io/transitland-lib/tt" +) + +// LocationGroup location_groups.txt +type LocationGroup struct { + LocationGroupID tt.String `csv:",required"` + LocationGroupName tt.String + tt.BaseEntity +} + +func (ent *LocationGroup) EntityKey() string { + return ent.LocationGroupID.Val +} + +func (ent *LocationGroup) EntityID() string { + return entID(ent.ID, ent.LocationGroupID.Val) +} + +func (ent *LocationGroup) Filename() string { + return "location_groups.txt" +} + +func (ent *LocationGroup) TableName() string { + return "gtfs_location_groups" +} + diff --git a/gtfs/location_group_stop.go b/gtfs/location_group_stop.go new file mode 100644 index 000000000..cc1c7bd43 --- /dev/null +++ b/gtfs/location_group_stop.go @@ -0,0 +1,28 @@ +package gtfs + +import ( + "github.com/interline-io/transitland-lib/tt" +) + +// LocationGroupStop location_group_stops.txt +type LocationGroupStop struct { + LocationGroupID tt.Key `csv:",required" target:"location_groups.txt"` + StopID tt.Key `csv:",required" target:"stops.txt"` + tt.BaseEntity +} + +func (ent *LocationGroupStop) Filename() string { + return "location_group_stops.txt" +} + +func (ent *LocationGroupStop) TableName() string { + return "gtfs_location_group_stops" +} + +// UpdateKeys updates Entity references. +func (ent *LocationGroupStop) UpdateKeys(emap *tt.EntityMap) error { + return tt.FirstError( + tt.TrySetField(emap.UpdateKey(&ent.LocationGroupID, "location_groups.txt"), "location_group_id"), + tt.TrySetField(emap.UpdateKey(&ent.StopID, "stops.txt"), "stop_id"), + ) +} diff --git a/gtfs/location_group_stop_test.go b/gtfs/location_group_stop_test.go new file mode 100644 index 000000000..87afcb903 --- /dev/null +++ b/gtfs/location_group_stop_test.go @@ -0,0 +1,212 @@ +package gtfs + +import ( + "testing" + + "github.com/interline-io/transitland-lib/tt" +) + +func TestLocationGroupStop_Errors(t *testing.T) { + tests := []struct { + name string + locationGroupStop *LocationGroupStop + expectedErrors []ExpectError + }{ + { + name: "Valid: basic location_group_stop", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg1"), + StopID: tt.NewKey("stop1"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group_stop with alphanumeric IDs", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg_downtown_001"), + StopID: tt.NewKey("stop_main_street_123"), + }, + expectedErrors: nil, + }, + { + name: "Invalid: missing location_group_id", + locationGroupStop: &LocationGroupStop{ + StopID: tt.NewKey("stop1"), + }, + expectedErrors: ParseExpectErrors("RequiredFieldError:location_group_id"), + }, + { + name: "Invalid: missing stop_id", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg1"), + }, + expectedErrors: ParseExpectErrors("RequiredFieldError:stop_id"), + }, + { + name: "Invalid: missing both required fields", + locationGroupStop: &LocationGroupStop{}, + expectedErrors: ParseExpectErrors("RequiredFieldError:location_group_id", "RequiredFieldError:stop_id"), + }, + { + name: "Valid: location_group_stop with mixed case IDs", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("LG-Zone1"), + StopID: tt.NewKey("StopID-ABC"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group_stop with long IDs", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("location_group_for_downtown_service_area_zone_1_extended"), + StopID: tt.NewKey("stop_at_main_street_and_first_avenue_northeast_corner"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group_stop with unicode IDs", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("市中心区域"), + StopID: tt.NewKey("駅001"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group_stop with URL-encoded IDs", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg%20zone%201"), + StopID: tt.NewKey("stop%2001"), + }, + expectedErrors: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := tt.CheckErrors(tc.locationGroupStop) + CheckErrors(tc.expectedErrors, errs, t) + }) + } +} + +func TestLocationGroupStop_EntityMethods(t *testing.T) { + lgs := LocationGroupStop{ + LocationGroupID: tt.NewKey("lg_test"), + StopID: tt.NewKey("stop_test"), + } + + if filename := lgs.Filename(); filename != "location_group_stops.txt" { + t.Errorf("Filename() = %q, want %q", filename, "location_group_stops.txt") + } + + if table := lgs.TableName(); table != "gtfs_location_group_stops" { + t.Errorf("TableName() = %q, want %q", table, "gtfs_location_group_stops") + } + + // Test with ID set + lgs.ID = 456 + if id := lgs.ID; id != 456 { + t.Errorf("ID = %d, want %d", id, 456) + } +} + +func TestLocationGroupStop_UpdateKeys(t *testing.T) { + tests := []struct { + name string + locationGroupStop *LocationGroupStop + setupEntityMap func() *tt.EntityMap + expectError bool + errorField string + }{ + { + name: "Valid: both keys resolve successfully", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg1"), + StopID: tt.NewKey("stop1"), + }, + setupEntityMap: func() *tt.EntityMap { + emap := tt.NewEntityMap() + emap.Set("location_groups.txt", "lg1", "100") + emap.Set("stops.txt", "stop1", "200") + return emap + }, + expectError: false, + }, + { + name: "Invalid: location_group_id not found", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg_unknown"), + StopID: tt.NewKey("stop1"), + }, + setupEntityMap: func() *tt.EntityMap { + emap := tt.NewEntityMap() + emap.Set("stops.txt", "stop1", "200") + return emap + }, + expectError: true, + errorField: "location_group_id", + }, + { + name: "Invalid: stop_id not found", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg1"), + StopID: tt.NewKey("stop_unknown"), + }, + setupEntityMap: func() *tt.EntityMap { + emap := tt.NewEntityMap() + emap.Set("location_groups.txt", "lg1", "100") + return emap + }, + expectError: true, + errorField: "stop_id", + }, + { + name: "Invalid: both keys not found", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg_unknown"), + StopID: tt.NewKey("stop_unknown"), + }, + setupEntityMap: func() *tt.EntityMap { + return tt.NewEntityMap() + }, + expectError: true, + errorField: "location_group_id", // First error is returned + }, + { + name: "Valid: keys with special characters", + locationGroupStop: &LocationGroupStop{ + LocationGroupID: tt.NewKey("lg:zone-1"), + StopID: tt.NewKey("stop_123-456"), + }, + setupEntityMap: func() *tt.EntityMap { + emap := tt.NewEntityMap() + emap.Set("location_groups.txt", "lg:zone-1", "100") + emap.Set("stops.txt", "stop_123-456", "200") + return emap + }, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + emap := tc.setupEntityMap() + err := tc.locationGroupStop.UpdateKeys(emap) + + if tc.expectError && err == nil { + t.Error("Expected error but got nil") + } + + if !tc.expectError && err != nil { + t.Errorf("Expected no error but got: %v", err) + } + + if tc.expectError && err != nil && tc.errorField != "" { + errStr := err.Error() + if len(errStr) == 0 || errStr == "" { + t.Errorf("Expected error message containing %q but got empty error", tc.errorField) + } + } + }) + } +} diff --git a/gtfs/location_group_test.go b/gtfs/location_group_test.go new file mode 100644 index 000000000..be8f01b74 --- /dev/null +++ b/gtfs/location_group_test.go @@ -0,0 +1,106 @@ +package gtfs + +import ( + "testing" + + "github.com/interline-io/transitland-lib/tt" +) + +func TestLocationGroup_Errors(t *testing.T) { + tests := []struct { + name string + locationGroup *LocationGroup + expectedErrors []ExpectError + }{ + { + name: "Valid: location_group with ID and name", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg1"), + LocationGroupName: tt.NewString("Transit Mall"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group with ID only", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg2"), + }, + expectedErrors: nil, + }, + { + name: "Invalid: location_group without required location_group_id", + locationGroup: &LocationGroup{ + LocationGroupName: tt.NewString("Transit Mall"), + }, + expectedErrors: ParseExpectErrors("RequiredFieldError:location_group_id"), + }, + { + name: "Valid: location_group with special characters in ID", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg_zone-1:downtown"), + LocationGroupName: tt.NewString("Zone 1: Downtown"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group with unicode name", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg_unicode"), + LocationGroupName: tt.NewString("市中心區域 (Downtown)"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group with long name", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg_long"), + LocationGroupName: tt.NewString("This is a very long location group name that describes a comprehensive service area covering multiple neighborhoods and districts"), + }, + expectedErrors: nil, + }, + { + name: "Valid: location_group with empty name (optional field)", + locationGroup: &LocationGroup{ + LocationGroupID: tt.NewString("lg_no_name"), + }, + expectedErrors: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := tt.CheckErrors(tc.locationGroup) + CheckErrors(tc.expectedErrors, errs, t) + }) + } +} + +func TestLocationGroup_EntityMethods(t *testing.T) { + lg := LocationGroup{ + LocationGroupID: tt.NewString("test_lg"), + LocationGroupName: tt.NewString("Test Group"), + } + + if key := lg.EntityKey(); key != "test_lg" { + t.Errorf("EntityKey() = %q, want %q", key, "test_lg") + } + + if filename := lg.Filename(); filename != "location_groups.txt" { + t.Errorf("Filename() = %q, want %q", filename, "location_groups.txt") + } + + if table := lg.TableName(); table != "gtfs_location_groups" { + t.Errorf("TableName() = %q, want %q", table, "gtfs_location_groups") + } + + // Test EntityID with no ID set + if entityID := lg.EntityID(); entityID != "test_lg" { + t.Errorf("EntityID() without ID = %q, want %q", entityID, "test_lg") + } + + // Test EntityID with ID set + lg.ID = 123 + if entityID := lg.EntityID(); entityID != "123" { + t.Errorf("EntityID() with ID = %q, want %q", entityID, "123") + } +} diff --git a/gtfs/stop_time.go b/gtfs/stop_time.go index 8f5afd0ee..ab5a53963 100644 --- a/gtfs/stop_time.go +++ b/gtfs/stop_time.go @@ -10,7 +10,9 @@ import ( // StopTime stop_times.txt type StopTime struct { TripID tt.String `csv:",required" target:"trips.txt"` - StopID tt.String `csv:",required" target:"stops.txt"` + StopID tt.Key `target:"stops.txt"` + LocationGroupID tt.Key `target:"location_groups.txt"` + LocationID tt.Key `target:"locations.txt"` StopSequence tt.Int `csv:",required"` StopHeadsign tt.String ArrivalTime tt.Seconds @@ -22,6 +24,17 @@ type StopTime struct { ShapeDistTraveled tt.Float Timepoint tt.Int Interpolated tt.Int `csv:"-"` // interpolated times: 0 for provided, 1 interpolated // TODO: 1 for shape, 2 for straight-line + // GTFS-Flex fields (officially adopted) + StartPickupDropOffWindow tt.Seconds + EndPickupDropOffWindow tt.Seconds + PickupBookingRuleID tt.Key `target:"booking_rules.txt"` + DropOffBookingRuleID tt.Key `target:"booking_rules.txt"` + // GTFS-Flex proposed fields (not yet formally adopted, may change) + // See: https://github.com/MobilityData/gtfs-flex/blob/master/spec/reference.md + MeanDurationFactor tt.Float // proposed gtfs-flex + MeanDurationOffset tt.Float // proposed gtfs-flex + SafeDurationFactor tt.Float // proposed gtfs-flex + SafeDurationOffset tt.Float // proposed gtfs-flex tt.MinEntity tt.ErrorEntity tt.ExtraEntity @@ -43,7 +56,6 @@ func (ent *StopTime) Errors() []error { // Don't use reflection based path errs := []error{} errs = append(errs, tt.CheckPresent("trip_id", ent.TripID.Val)...) - errs = append(errs, tt.CheckPresent("stop_id", ent.StopID.Val)...) errs = append(errs, tt.CheckPositiveInt("stop_sequence", ent.StopSequence.Val)...) errs = append(errs, tt.CheckInsideRangeInt("pickup_type", ent.PickupType.Val, 0, 3)...) errs = append(errs, tt.CheckInsideRangeInt("drop_off_type", ent.DropOffType.Val, 0, 3)...) @@ -61,12 +73,186 @@ func (ent *StopTime) Errors() []error { return errs } +// ConditionalErrors for StopTime - includes GTFS-Flex validation +func (ent *StopTime) ConditionalErrors() (errs []error) { + // Check which location identifier is used + hasStopID := ent.StopID.IsPresent() + hasLocationGroupID := ent.LocationGroupID.IsPresent() + hasLocationID := ent.LocationID.IsPresent() + hasTimeWindow := (ent.StartPickupDropOffWindow.Valid && ent.StartPickupDropOffWindow.Val > 0) || (ent.EndPickupDropOffWindow.Valid && ent.EndPickupDropOffWindow.Val > 0) + + // 1. Mutual exclusion: stop_id, location_id, location_group_id + // Exactly one of these must be defined + if !hasStopID && !hasLocationGroupID && !hasLocationID { + errs = append(errs, causes.NewConditionallyRequiredFieldError("stop_id")) + } + + if hasStopID && hasLocationGroupID { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "location_group_id", + ent.LocationGroupID.Val, + "location_group_id is forbidden when stop_id is defined", + )) + } + if hasStopID && hasLocationID { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "location_id", + ent.LocationID.Val, + "location_id is forbidden when stop_id is defined", + )) + } + if hasLocationGroupID && hasLocationID { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "location_id", + ent.LocationID.Val, + "location_id is forbidden when location_group_id is defined", + )) + } + + // 2. Time windows required if location_group_id or location_id is defined + if hasLocationGroupID || hasLocationID { + if !ent.StartPickupDropOffWindow.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("start_pickup_drop_off_window")) + } + if !ent.EndPickupDropOffWindow.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("end_pickup_drop_off_window")) + } + } + + // 3. Time windows consistency + hasStartWindow := ent.StartPickupDropOffWindow.Valid + hasEndWindow := ent.EndPickupDropOffWindow.Valid + + if hasStartWindow && !hasEndWindow { + errs = append(errs, causes.NewConditionallyRequiredFieldError("end_pickup_drop_off_window")) + } + if hasEndWindow && !hasStartWindow { + errs = append(errs, causes.NewConditionallyRequiredFieldError("start_pickup_drop_off_window")) + } + + // 4. If both windows are present, end must be >= start + if hasStartWindow && hasEndWindow { + if ent.EndPickupDropOffWindow.Val < ent.StartPickupDropOffWindow.Val { + errs = append(errs, causes.NewInvalidFieldError( + "end_pickup_drop_off_window", + fmt.Sprintf("%d", ent.EndPickupDropOffWindow.Val), + fmt.Errorf("must be greater than or equal to start_pickup_drop_off_window (%d)", ent.StartPickupDropOffWindow.Val), + )) + } + } + + // 5. Time windows vs fixed times + // If using time windows, arrival_time and departure_time are forbidden + if hasTimeWindow { + if ent.ArrivalTime.Valid && ent.ArrivalTime.Int() > 0 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "arrival_time", + ent.ArrivalTime.String(), + "arrival_time is forbidden when start_pickup_drop_off_window or end_pickup_drop_off_window are defined", + )) + } + if ent.DepartureTime.Valid && ent.DepartureTime.Int() > 0 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "departure_time", + ent.DepartureTime.String(), + "departure_time is forbidden when start_pickup_drop_off_window or end_pickup_drop_off_window are defined", + )) + } + } + + // 6. pickup_type restrictions with time windows + // pickup_type=0 (regularly scheduled) is forbidden if time windows are defined + // pickup_type=3 (coordinate with driver) is forbidden if time windows are defined + if hasTimeWindow && ent.PickupType.Valid { + if ent.PickupType.Val == 0 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "pickup_type", + fmt.Sprintf("%d", ent.PickupType.Val), + "pickup_type=0 (regularly scheduled) is forbidden when time windows are defined", + )) + } + if ent.PickupType.Val == 3 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "pickup_type", + fmt.Sprintf("%d", ent.PickupType.Val), + "pickup_type=3 (coordinate with driver) is forbidden when time windows are defined", + )) + } + } + + // 7. drop_off_type restrictions with time windows + // drop_off_type=0 (regularly scheduled) is forbidden if time windows are defined + if hasTimeWindow && ent.DropOffType.Valid && ent.DropOffType.Val == 0 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "drop_off_type", + fmt.Sprintf("%d", ent.DropOffType.Val), + "drop_off_type=0 (regularly scheduled) is forbidden when time windows are defined", + )) + } + + // 8. continuous_pickup restrictions with time windows + // Any value other than 1 or empty is forbidden if time windows are defined + if hasTimeWindow && ent.ContinuousPickup.Valid { + if ent.ContinuousPickup.Val != 1 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "continuous_pickup", + fmt.Sprintf("%d", ent.ContinuousPickup.Val), + "continuous_pickup must be 1 (no continuous pickup) or empty when time windows are defined", + )) + } + } + + // 9. continuous_drop_off restrictions with time windows + // Any value other than 1 or empty is forbidden if time windows are defined + if hasTimeWindow && ent.ContinuousDropOff.Valid { + if ent.ContinuousDropOff.Val != 1 { + errs = append(errs, causes.NewConditionallyForbiddenFieldError( + "continuous_drop_off", + fmt.Sprintf("%d", ent.ContinuousDropOff.Val), + "continuous_drop_off must be 1 (no continuous drop off) or empty when time windows are defined", + )) + } + } + + // 10. Validate mean/safe duration factor/offset + // mean_duration_factor and mean_duration_offset work together + if ent.MeanDurationFactor.Valid && !ent.MeanDurationOffset.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("mean_duration_offset")) + } + if ent.MeanDurationOffset.Valid && !ent.MeanDurationFactor.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("mean_duration_factor")) + } + + // safe_duration_factor and safe_duration_offset work together + if ent.SafeDurationFactor.Valid && !ent.SafeDurationOffset.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("safe_duration_offset")) + } + if ent.SafeDurationOffset.Valid && !ent.SafeDurationFactor.Valid { + errs = append(errs, causes.NewConditionallyRequiredFieldError("safe_duration_factor")) + } + + // mean_duration_factor should be positive + if ent.MeanDurationFactor.Valid && ent.MeanDurationFactor.Val <= 0 { + errs = append(errs, causes.NewInvalidFieldError( + "mean_duration_factor", + fmt.Sprintf("%f", ent.MeanDurationFactor.Val), + fmt.Errorf("must be positive"), + )) + } + + return errs +} + // UpdateKeys updates Entity references. func (ent *StopTime) UpdateKeys(emap *tt.EntityMap) error { // Don't use reflection based path return tt.FirstError( tt.TrySetField(emap.UpdateKey(&ent.TripID, "trips.txt"), "trip_id"), tt.TrySetField(emap.UpdateKey(&ent.StopID, "stops.txt"), "stop_id"), + tt.TrySetField(emap.UpdateKey(&ent.LocationGroupID, "location_groups.txt"), "location_group_id"), + tt.TrySetField(emap.UpdateKey(&ent.LocationID, "locations.geojson"), "location_id"), + tt.TrySetField(emap.UpdateKey(&ent.PickupBookingRuleID, "booking_rules.txt"), "pickup_booking_rule_id"), + tt.TrySetField(emap.UpdateKey(&ent.DropOffBookingRuleID, "booking_rules.txt"), "drop_off_booking_rule_id"), ) } @@ -81,6 +267,10 @@ func (ent *StopTime) GetString(key string) (string, error) { v = ent.StopHeadsign.Val case "stop_id": v = ent.StopID.Val + case "location_group_id": + v = ent.LocationGroupID.Val + case "location_id": + v = ent.LocationID.Val case "arrival_time": v = ent.ArrivalTime.String() case "departure_time": @@ -101,6 +291,30 @@ func (ent *StopTime) GetString(key string) (string, error) { v = ent.ContinuousPickup.String() case "continuous_drop_off": v = ent.ContinuousDropOff.String() + case "start_pickup_drop_off_window": + v = ent.StartPickupDropOffWindow.String() + case "end_pickup_drop_off_window": + v = ent.EndPickupDropOffWindow.String() + case "pickup_booking_rule_id": + v = ent.PickupBookingRuleID.Val + case "drop_off_booking_rule_id": + v = ent.DropOffBookingRuleID.Val + case "mean_duration_factor": + if ent.MeanDurationFactor.Valid { + v = fmt.Sprintf("%0.5f", ent.MeanDurationFactor.Val) + } + case "mean_duration_offset": + if ent.MeanDurationOffset.Valid { + v = fmt.Sprintf("%0.5f", ent.MeanDurationOffset.Val) + } + case "safe_duration_factor": + if ent.SafeDurationFactor.Valid { + v = fmt.Sprintf("%0.5f", ent.SafeDurationFactor.Val) + } + case "safe_duration_offset": + if ent.SafeDurationOffset.Valid { + v = fmt.Sprintf("%0.5f", ent.SafeDurationOffset.Val) + } default: return v, fmt.Errorf("unknown key: %s", key) } @@ -119,6 +333,10 @@ func (ent *StopTime) SetString(key, value string) error { ent.StopHeadsign.Set(hi) case "stop_id": ent.StopID.Set(hi) + case "location_group_id": + ent.LocationGroupID.Set(hi) + case "location_id": + ent.LocationID.Set(hi) case "arrival_time": if err := ent.ArrivalTime.Scan(hi); err != nil { perr = causes.NewFieldParseError("arrival_time", hi) @@ -155,6 +373,34 @@ func (ent *StopTime) SetString(key, value string) error { if err := ent.Timepoint.Scan(hi); err != nil { perr = causes.NewFieldParseError("timepoint", hi) } + case "start_pickup_drop_off_window": + if err := ent.StartPickupDropOffWindow.Scan(hi); err != nil { + perr = causes.NewFieldParseError("start_pickup_drop_off_window", hi) + } + case "end_pickup_drop_off_window": + if err := ent.EndPickupDropOffWindow.Scan(hi); err != nil { + perr = causes.NewFieldParseError("end_pickup_drop_off_window", hi) + } + case "pickup_booking_rule_id": + ent.PickupBookingRuleID.Set(hi) + case "drop_off_booking_rule_id": + ent.DropOffBookingRuleID.Set(hi) + case "mean_duration_factor": + if err := ent.MeanDurationFactor.Scan(hi); err != nil { + perr = causes.NewFieldParseError("mean_duration_factor", hi) + } + case "mean_duration_offset": + if err := ent.MeanDurationOffset.Scan(hi); err != nil { + perr = causes.NewFieldParseError("mean_duration_offset", hi) + } + case "safe_duration_factor": + if err := ent.SafeDurationFactor.Scan(hi); err != nil { + perr = causes.NewFieldParseError("safe_duration_factor", hi) + } + case "safe_duration_offset": + if err := ent.SafeDurationOffset.Scan(hi); err != nil { + perr = causes.NewFieldParseError("safe_duration_offset", hi) + } default: ent.SetExtra(key, hi) } diff --git a/gtfs/stop_time_test.go b/gtfs/stop_time_test.go new file mode 100644 index 000000000..0bd3529ed --- /dev/null +++ b/gtfs/stop_time_test.go @@ -0,0 +1,509 @@ +package gtfs + +import ( + "testing" + + "github.com/interline-io/transitland-lib/tt" +) + +func TestStopTime_Errors(t *testing.T) { + tests := []struct { + name string + stopTime *StopTime + expectedErrors []ExpectError + }{ + { + name: "Valid: Basic stop time", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(3630), + }, + expectedErrors: nil, + }, + { + name: "Invalid: stop_sequence negative", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(-1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(3630), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:stop_sequence"), + }, + { + name: "Invalid: pickup_type out of range", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(3630), + PickupType: tt.NewInt(4), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:pickup_type"), + }, + { + name: "Invalid: drop_off_type out of range", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(3630), + DropOffType: tt.NewInt(4), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:drop_off_type"), + }, + { + name: "Invalid: timepoint out of range", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(3630), + Timepoint: tt.NewInt(2), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:timepoint"), + }, + { + name: "Invalid: departure_time before arrival_time", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3600), + DepartureTime: tt.NewSeconds(1800), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:departure_time"), + }, + { + name: "Invalid: continuous_pickup out of range", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3660), + DepartureTime: tt.NewSeconds(3690), + ContinuousPickup: tt.NewInt(100), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:continuous_pickup"), + }, + { + name: "Invalid: continuous_drop_off out of range", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + ArrivalTime: tt.NewSeconds(3660), + DepartureTime: tt.NewSeconds(3690), + ContinuousDropOff: tt.NewInt(100), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:continuous_drop_off"), + }, + // ConditionalErrors - Location field mutual exclusion tests + { + name: "Valid: Only stop_id present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey("stop1"), + StopSequence: tt.NewInt(1), + }, + expectedErrors: nil, + }, + { + name: "Valid: Only location_group_id present with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + LocationGroupID: tt.NewKey("lg1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + }, + expectedErrors: nil, + }, + { + name: "Valid: Only location_id present with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + LocationID: tt.NewKey("loc1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + }, + expectedErrors: nil, + }, + { + name: "Invalid: No location identifier", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:stop_id"), + }, + { + name: "Invalid: stop_id and location_group_id both present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + LocationGroupID: tt.NewKey("lg1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyForbiddenFieldError:location_group_id", + "ConditionallyRequiredFieldError:start_pickup_drop_off_window", + "ConditionallyRequiredFieldError:end_pickup_drop_off_window", + ), + }, + { + name: "Invalid: stop_id and location_id both present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + LocationID: tt.NewKey("loc1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyForbiddenFieldError:location_id", + "ConditionallyRequiredFieldError:start_pickup_drop_off_window", + "ConditionallyRequiredFieldError:end_pickup_drop_off_window", + ), + }, + { + name: "Invalid: location_group_id and location_id both present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + LocationGroupID: tt.NewKey("lg1"), + LocationID: tt.NewKey("loc1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyForbiddenFieldError:location_id", + "ConditionallyRequiredFieldError:start_pickup_drop_off_window", + "ConditionallyRequiredFieldError:end_pickup_drop_off_window", + ), + }, + { + name: "Invalid: location_group_id without time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + LocationGroupID: tt.NewKey("lg1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:start_pickup_drop_off_window", + "ConditionallyRequiredFieldError:end_pickup_drop_off_window", + ), + }, + { + name: "Invalid: location_id without time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + LocationID: tt.NewKey("loc1"), + }, + expectedErrors: ParseExpectErrors( + "ConditionallyRequiredFieldError:start_pickup_drop_off_window", + "ConditionallyRequiredFieldError:end_pickup_drop_off_window", + ), + }, + { + name: "Valid: Both time windows present and consistent", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + }, + expectedErrors: nil, + }, + { + name: "Invalid: Only start window present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:end_pickup_drop_off_window"), + }, + { + name: "Invalid: Only end window present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + EndPickupDropOffWindow: tt.NewSeconds(7200), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:start_pickup_drop_off_window"), + }, + { + name: "Invalid: End window before start window", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(7200), + EndPickupDropOffWindow: tt.NewSeconds(3600), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:end_pickup_drop_off_window"), + }, + { + name: "Invalid: Time window with arrival_time", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ArrivalTime: tt.NewSeconds(10800), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:arrival_time"), + }, + { + name: "Invalid: Time window with departure_time", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + DepartureTime: tt.NewSeconds(10800), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:departure_time"), + }, + // pickup_type and drop_off_type with time windows + { + name: "Invalid: pickup_type=0 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + PickupType: tt.NewInt(0), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:pickup_type"), + }, + { + name: "Invalid: pickup_type=3 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + PickupType: tt.NewInt(3), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:pickup_type"), + }, + { + name: "Valid: pickup_type=2 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + PickupType: tt.NewInt(2), + }, + expectedErrors: nil, + }, + { + name: "Invalid: drop_off_type=0 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + DropOffType: tt.NewInt(0), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:drop_off_type"), + }, + { + name: "Valid: drop_off_type=2 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + DropOffType: tt.NewInt(2), + }, + expectedErrors: nil, + }, + // continuous_pickup and continuous_drop_off with time windows + { + name: "Invalid: continuous_pickup=0 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ContinuousPickup: tt.NewInt(0), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:continuous_pickup"), + }, + { + name: "Invalid: continuous_pickup=2 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ContinuousPickup: tt.NewInt(2), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:continuous_pickup"), + }, + { + name: "Valid: continuous_pickup=1 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ContinuousPickup: tt.NewInt(1), + }, + expectedErrors: nil, + }, + { + name: "Invalid: continuous_drop_off=0 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ContinuousDropOff: tt.NewInt(0), + }, + expectedErrors: ParseExpectErrors("ConditionallyForbiddenFieldError:continuous_drop_off"), + }, + { + name: "Valid: continuous_drop_off=1 with time windows", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + StartPickupDropOffWindow: tt.NewSeconds(3600), + EndPickupDropOffWindow: tt.NewSeconds(7200), + ContinuousDropOff: tt.NewInt(1), + }, + expectedErrors: nil, + }, + { + name: "Valid: pickup_booking_rule_id with pickup_type=2", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + PickupBookingRuleID: tt.NewKey("rule1"), + PickupType: tt.NewInt(2), + }, + expectedErrors: nil, + }, + { + name: "Valid: pickup_booking_rule_id with pickup_type=0", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + PickupBookingRuleID: tt.NewKey("rule1"), + PickupType: tt.NewInt(0), + }, + expectedErrors: nil, + }, + { + name: "Valid: drop_off_booking_rule_id with drop_off_type=2", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + DropOffBookingRuleID: tt.NewKey("rule1"), + DropOffType: tt.NewInt(2), + }, + expectedErrors: nil, + }, + { + name: "Valid: drop_off_booking_rule_id with drop_off_type=0", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + DropOffBookingRuleID: tt.NewKey("rule1"), + DropOffType: tt.NewInt(0), + }, + expectedErrors: nil, + }, + { + name: "Valid: Both mean_duration fields present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + MeanDurationFactor: tt.NewFloat(1.5), + MeanDurationOffset: tt.NewFloat(300), + }, + expectedErrors: nil, + }, + { + name: "Invalid: Only mean_duration_factor present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + MeanDurationFactor: tt.NewFloat(1.5), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:mean_duration_offset"), + }, + { + name: "Invalid: Only mean_duration_offset present", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + MeanDurationOffset: tt.NewFloat(300), + }, + expectedErrors: ParseExpectErrors("ConditionallyRequiredFieldError:mean_duration_factor"), + }, + { + name: "Invalid: mean_duration_factor is negative", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + MeanDurationFactor: tt.NewFloat(-1.5), + MeanDurationOffset: tt.NewFloat(300), + }, + expectedErrors: ParseExpectErrors("InvalidFieldError:mean_duration_factor"), + }, + { + name: "Valid: safe_duration_factor with mean_duration_factor", + stopTime: &StopTime{ + TripID: tt.NewString("trip1"), + StopSequence: tt.NewInt(1), + StopID: tt.NewKey("stop1"), + MeanDurationFactor: tt.NewFloat(1.5), + MeanDurationOffset: tt.NewFloat(300), + SafeDurationFactor: tt.NewFloat(2.0), + SafeDurationOffset: tt.NewFloat(400), + }, + expectedErrors: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := tt.CheckErrors(tc.stopTime) + CheckErrors(tc.expectedErrors, errs, t) + }) + } +} diff --git a/importer/importer_test.go b/importer/importer_test.go index 209dc0c01..2b0ce8681 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -8,7 +8,7 @@ import ( "github.com/interline-io/transitland-lib/dmfr" "github.com/interline-io/transitland-lib/internal/testdb" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tldb" "github.com/interline-io/transitland-lib/tt" ) @@ -25,7 +25,7 @@ func TestImportFeedVersion(t *testing.T) { } t.Run("Success", func(t *testing.T) { testdb.TempSqlite(func(atx tldb.Adapter) error { - fvid := setup(atx, testutil.ExampleZip.URL) + fvid := setup(atx, testreader.ExampleZip.URL) atx2 := testdb.AdapterIgnoreTx{Adapter: atx} _, err := ImportFeedVersion(ctx, &atx2, Options{Activate: true, FeedVersionID: fvid, Storage: "/"}) if err != nil { @@ -44,7 +44,7 @@ func TestImportFeedVersion(t *testing.T) { t.Errorf("expected in_progress = false") } count := 0 - expstops := testutil.ExampleZip.Counts["stops.txt"] + expstops := testreader.ExampleZip.Counts["stops.txt"] testdb.ShouldGet(t, atx, &count, "SELECT count(*) FROM gtfs_stops WHERE feed_version_id = ?", fvid) if count != expstops { t.Errorf("got %d stops, expect %d stops", count, expstops) @@ -88,7 +88,7 @@ func Test_iImportFeedVersionTx(t *testing.T) { ctx := context.TODO() err := testdb.TempSqlite(func(atx tldb.Adapter) error { // Create FV - fv := dmfr.FeedVersion{File: testutil.ExampleZip.URL} + fv := dmfr.FeedVersion{File: testreader.ExampleZip.URL} fv.EarliestCalendarDate = tt.NewDate(time.Now()) fv.LatestCalendarDate = tt.NewDate(time.Now()) fvid := testdb.ShouldInsert(t, atx, &fv) @@ -100,7 +100,7 @@ func Test_iImportFeedVersionTx(t *testing.T) { } // Check count := 0 - expstops := testutil.ExampleZip.Counts["stops.txt"] + expstops := testreader.ExampleZip.Counts["stops.txt"] testdb.ShouldGet(t, atx, &count, "SELECT count(*) FROM gtfs_stops WHERE feed_version_id = ?", fvid) if count != expstops { t.Errorf("expect %d stops, got %d", count, expstops) diff --git a/importer/unimporter_test.go b/importer/unimporter_test.go index 72c8e2f1e..665e2e633 100644 --- a/importer/unimporter_test.go +++ b/importer/unimporter_test.go @@ -10,7 +10,7 @@ import ( "github.com/interline-io/log" "github.com/interline-io/transitland-lib/dmfr" "github.com/interline-io/transitland-lib/internal/testdb" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/stats" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tldb" @@ -33,14 +33,14 @@ func setupImport(ctx context.Context, t *testing.T, atx tldb.Adapter) int { feed := dmfr.Feed{} feed.FeedID = fmt.Sprintf("feed-%d", time.Now().UnixNano()) feedid := testdb.ShouldInsert(t, atx, &feed) - fv := dmfr.FeedVersion{File: testutil.ExampleZip.URL} + fv := dmfr.FeedVersion{File: testreader.ExampleZip.URL} fv.FeedID = feedid fv.EarliestCalendarDate = tt.NewDate(time.Now()) fv.LatestCalendarDate = tt.NewDate(time.Now()) fvid := testdb.ShouldInsert(t, atx, &fv) fv.ID = fvid // Generate stats - tlreader, err := tlcsv.NewReader(testutil.ExampleZip.URL) + tlreader, err := tlcsv.NewReader(testreader.ExampleZip.URL) if err != nil { t.Fatal(err) } diff --git a/internal/feedstate/manager_test.go b/internal/feedstate/manager_test.go index 622840976..6e2cdc4d4 100644 --- a/internal/feedstate/manager_test.go +++ b/internal/feedstate/manager_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/interline-io/transitland-lib/copier" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/server/dbutil" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tldb" @@ -135,7 +135,7 @@ func verifyFeedState(t *testing.T, adapter tldb.Adapter, feedID int, expectedFee } func TestManager_ActivateFeedVersion(t *testing.T) { - adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) @@ -157,7 +157,7 @@ func TestManager_ActivateFeedVersion(t *testing.T) { } func TestManager_DeactivateFeedVersion(t *testing.T) { - adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) @@ -190,7 +190,7 @@ func TestManager_DeactivateFeedVersion(t *testing.T) { } func TestManager_GetActiveFeedVersions(t *testing.T) { - adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) @@ -213,7 +213,7 @@ func TestManager_GetActiveFeedVersions(t *testing.T) { } func TestManager_SetActiveFeedVersions(t *testing.T) { - adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) @@ -249,7 +249,7 @@ func TestManager_SetActiveFeedVersions(t *testing.T) { feedVersionID2 := feedVersionID + 1 // Import data for second feed (copy the same GTFS data with different feed_id) - reader, err := tlcsv.NewReader(testutil.ExampleFeedCaltrain.URL) + reader, err := tlcsv.NewReader(testreader.ExampleFeedCaltrain.URL) require.NoError(t, err, "failed to create GTFS reader for second feed") writer, err := tldb.NewWriter("sqlite3://:memory:") @@ -300,7 +300,7 @@ func TestManager_SetActiveFeedVersions(t *testing.T) { } func TestManager_MaterializedDataIntegrity(t *testing.T) { - adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, _, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) @@ -375,7 +375,7 @@ func TestManager_MaterializedDataIntegrity(t *testing.T) { } func TestManager_GetFeedIDForFeedVersion(t *testing.T) { - adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testutil.ExampleZip.URL) + adapter, feedID, feedVersionID := setupTestDB(t, testFeedOnestopID, testreader.ExampleZip.URL) defer adapter.Close() manager := NewManager(adapter) diff --git a/internal/geomcache/geomcache.go b/internal/geomcache/geomcache.go index bea22d3c1..3f5bfb405 100644 --- a/internal/geomcache/geomcache.go +++ b/internal/geomcache/geomcache.go @@ -140,13 +140,16 @@ func (g *GeomCache) setStopTimeDists(shapeId string, patternId int64, sts []gtfs stopPositionInfo, ok := g.stopPositions[stopPositionsKey] if !ok { // Generate the stop-to-stop geometry - stopLine := make([]tlxy.Point, len(sts)) + stopLine := make([]tlxy.Point, 0, len(sts)) for i := 0; i < len(sts); i++ { + if !sts[i].StopID.Valid { + continue + } point, ok := g.stops[sts[i].StopID.Val] if !ok { return fmt.Errorf("stop '%s' not in cache", sts[i].StopID) } - stopLine[i] = point + stopLine = append(stopLine, point) } // Get the known shape line and known shape distance diff --git a/internal/geomcache/geomcache_test.go b/internal/geomcache/geomcache_test.go index f2d503c7b..bf63ad33a 100644 --- a/internal/geomcache/geomcache_test.go +++ b/internal/geomcache/geomcache_test.go @@ -4,13 +4,13 @@ import ( "testing" "github.com/interline-io/transitland-lib/gtfs" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/service" "github.com/interline-io/transitland-lib/tlcsv" ) func TestGeomCache(t *testing.T) { - r, err := tlcsv.NewReader(testutil.ExampleDir.URL) + r, err := tlcsv.NewReader(testreader.ExampleDir.URL) if err != nil { t.Error(err) } diff --git a/internal/testutil/copy.go b/internal/testreader/copy.go similarity index 98% rename from internal/testutil/copy.go rename to internal/testreader/copy.go index 9a6fb82c3..235c00eee 100644 --- a/internal/testutil/copy.go +++ b/internal/testreader/copy.go @@ -1,4 +1,4 @@ -package testutil +package testreader import ( "fmt" diff --git a/internal/testutil/copy_test.go b/internal/testreader/copy_test.go similarity index 97% rename from internal/testutil/copy_test.go rename to internal/testreader/copy_test.go index 2c40f9a5d..e97249858 100644 --- a/internal/testutil/copy_test.go +++ b/internal/testreader/copy_test.go @@ -1,4 +1,4 @@ -package testutil +package testreader import ( "testing" diff --git a/internal/testutil/entities.go b/internal/testreader/entities.go similarity index 99% rename from internal/testutil/entities.go rename to internal/testreader/entities.go index d7bf50b73..fb6d3ff0f 100644 --- a/internal/testutil/entities.go +++ b/internal/testreader/entities.go @@ -1,4 +1,4 @@ -package testutil +package testreader import ( "github.com/interline-io/transitland-lib/adapters" diff --git a/internal/testutil/minimal.go b/internal/testreader/minimal.go similarity index 90% rename from internal/testutil/minimal.go rename to internal/testreader/minimal.go index 5e363cbeb..4f8ffa7cf 100644 --- a/internal/testutil/minimal.go +++ b/internal/testreader/minimal.go @@ -1,4 +1,4 @@ -package testutil +package testreader import ( "time" @@ -25,8 +25,8 @@ func NewMinimalTestFeed() (*ReaderTester, *direct.Reader) { {StopID: tt.NewString("stop2"), StopName: tt.NewString("Stop 2"), Geometry: tt.NewPoint(3, 4)}, }, StopTimeList: []gtfs.StopTime{ - {StopID: tt.NewString("stop1"), TripID: tt.NewString("trip1"), StopSequence: tt.NewInt(1), ArrivalTime: tt.NewSeconds(0), DepartureTime: tt.NewSeconds(5)}, - {StopID: tt.NewString("stop2"), TripID: tt.NewString("trip1"), StopSequence: tt.NewInt(2), ArrivalTime: tt.NewSeconds(10), DepartureTime: tt.NewSeconds(15)}, + {StopID: tt.NewKey("stop1"), TripID: tt.NewString("trip1"), StopSequence: tt.NewInt(1), ArrivalTime: tt.NewSeconds(0), DepartureTime: tt.NewSeconds(5)}, + {StopID: tt.NewKey("stop2"), TripID: tt.NewString("trip1"), StopSequence: tt.NewInt(2), ArrivalTime: tt.NewSeconds(10), DepartureTime: tt.NewSeconds(15)}, }, ShapeList: []gtfs.Shape{ {ShapeID: tt.NewString("shape1"), ShapePtLon: tt.NewFloat(1), ShapePtLat: tt.NewFloat(2), ShapePtSequence: tt.NewInt(0)}, diff --git a/internal/testutil/readertester.go b/internal/testreader/readertester.go similarity index 99% rename from internal/testutil/readertester.go rename to internal/testreader/readertester.go index a3b46f544..b395944bf 100644 --- a/internal/testutil/readertester.go +++ b/internal/testreader/readertester.go @@ -1,4 +1,4 @@ -package testutil +package testreader import ( "testing" diff --git a/internal/testutil/test_feeds.go b/internal/testreader/test_feeds.go similarity index 99% rename from internal/testutil/test_feeds.go rename to internal/testreader/test_feeds.go index a1781a964..5859fe78d 100644 --- a/internal/testutil/test_feeds.go +++ b/internal/testreader/test_feeds.go @@ -1,4 +1,4 @@ -package testutil +package testreader import "github.com/interline-io/transitland-lib/internal/testpath" diff --git a/internal/testutil/expect.go b/internal/testutil/expect.go index 24882e231..95abffcb6 100644 --- a/internal/testutil/expect.go +++ b/internal/testutil/expect.go @@ -151,3 +151,11 @@ func ParseExpectError(value string) ExpectError { } return NewExpectError(v[2], v[3], v[1], v[0]) } + +func ParseExpectErrors(values ...string) []ExpectError { + var expecterrs []ExpectError + for _, value := range values { + expecterrs = append(expecterrs, ParseExpectError(value)) + } + return expecterrs +} diff --git a/rules/fast_travel.go b/rules/fast_travel.go index 5fc28e78d..4ae7e340e 100644 --- a/rules/fast_travel.go +++ b/rules/fast_travel.go @@ -97,9 +97,15 @@ func (e *StopTimeFastTravelCheck) Validate(ent tt.Entity) []error { } // todo: cache for trip pattern? var errs []error + if len(trip.StopTimes) == 0 || !trip.StopTimes[0].StopID.Valid { + return errs + } s1 := trip.StopTimes[0].StopID.Val t := trip.StopTimes[0].DepartureTime for i := 1; i < len(trip.StopTimes); i++ { + if !trip.StopTimes[i].StopID.Valid { + continue + } s2 := trip.StopTimes[i].StopID.Val key := s1 + ":" + s2 // todo: use a real separator... dx, ok := e.stopDist[key] diff --git a/rules/flex_geography_id_unique.go b/rules/flex_geography_id_unique.go new file mode 100644 index 000000000..ea9f7f276 --- /dev/null +++ b/rules/flex_geography_id_unique.go @@ -0,0 +1,102 @@ +package rules + +import ( + "fmt" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +// FlexGeographyIDDuplicateError reports when a geography ID is duplicated across +// stops.txt, locations.geojson, or location_groups.txt. +type FlexGeographyIDDuplicateError struct { + GeographyID string + FirstFile string + SecondFile string + bc +} + +func (e *FlexGeographyIDDuplicateError) Error() string { + return fmt.Sprintf( + "geography_id '%s' is duplicated across %s and %s (IDs must be unique across stops.stop_id, locations.geojson id, and location_groups.location_group_id)", + e.GeographyID, + e.FirstFile, + e.SecondFile, + ) +} + +// FlexGeographyIDUniqueCheck validates that geography IDs are unique across +// stops.txt, locations.geojson, and location_groups.txt. +// +// Per GTFS-Flex specification, these three ID spaces are merged into a single +// namespace that can be referenced in stop_times.txt via stop_id or location_id. +type FlexGeographyIDUniqueCheck struct { + geographyIDs map[string]string // geography_id -> filename +} + +func (e *FlexGeographyIDUniqueCheck) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error { + if e.geographyIDs == nil { + e.geographyIDs = map[string]string{} + } + + var geographyID string + var filename string + + switch entity := ent.(type) { + case *gtfs.Stop: + geographyID = entity.StopID.Val + filename = "stops.txt" + case *gtfs.Location: + geographyID = entity.LocationID.Val + filename = "locations.geojson" + case *gtfs.LocationGroup: + geographyID = entity.LocationGroupID.Val + filename = "location_groups.txt" + default: + return nil + } + + // Store the first occurrence + if geographyID != "" { + if _, exists := e.geographyIDs[geographyID]; !exists { + e.geographyIDs[geographyID] = filename + } + } + + return nil +} + +func (e *FlexGeographyIDUniqueCheck) Validate(ent tt.Entity) []error { + var geographyID string + var filename string + + switch entity := ent.(type) { + case *gtfs.Stop: + geographyID = entity.StopID.Val + filename = "stops.txt" + case *gtfs.Location: + geographyID = entity.LocationID.Val + filename = "locations.geojson" + case *gtfs.LocationGroup: + geographyID = entity.LocationGroupID.Val + filename = "location_groups.txt" + default: + return nil + } + + if geographyID == "" { + return nil + } + + // Check if this ID was seen in a different file + if firstFile, exists := e.geographyIDs[geographyID]; exists && firstFile != filename { + return []error{&FlexGeographyIDDuplicateError{ + GeographyID: geographyID, + FirstFile: firstFile, + SecondFile: filename, + }} + } + + return nil +} + diff --git a/rules/flex_geography_id_unique_test.go b/rules/flex_geography_id_unique_test.go new file mode 100644 index 000000000..056d46ca0 --- /dev/null +++ b/rules/flex_geography_id_unique_test.go @@ -0,0 +1,111 @@ +package rules + +import ( + "testing" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +func TestFlexGeographyIDUniqueCheck(t *testing.T) { + tests := []struct { + name string + entities []tt.Entity + expectError bool + errorMsg string + }{ + { + name: "no duplicates - all unique", + entities: []tt.Entity{ + >fs.Stop{StopID: tt.NewString("stop_1")}, + >fs.Stop{StopID: tt.NewString("stop_2")}, + >fs.Location{LocationID: tt.NewString("location_1")}, + >fs.LocationGroup{LocationGroupID: tt.NewString("group_1")}, + }, + expectError: false, + }, + { + name: "duplicate stop_id with location_id", + entities: []tt.Entity{ + >fs.Stop{StopID: tt.NewString("area_1")}, + >fs.Location{LocationID: tt.NewString("area_1")}, + }, + expectError: true, + errorMsg: "geography_id 'area_1' is duplicated across stops.txt and locations.geojson (IDs must be unique across stops.stop_id, locations.geojson id, and location_groups.location_group_id)", + }, + { + name: "duplicate stop_id with location_group_id", + entities: []tt.Entity{ + >fs.Stop{StopID: tt.NewString("zone_5")}, + >fs.LocationGroup{LocationGroupID: tt.NewString("zone_5")}, + }, + expectError: true, + errorMsg: "geography_id 'zone_5' is duplicated across stops.txt and location_groups.txt (IDs must be unique across stops.stop_id, locations.geojson id, and location_groups.location_group_id)", + }, + { + name: "duplicate location_id with location_group_id", + entities: []tt.Entity{ + >fs.Location{LocationID: tt.NewString("flex_area")}, + >fs.LocationGroup{LocationGroupID: tt.NewString("flex_area")}, + }, + expectError: true, + errorMsg: "geography_id 'flex_area' is duplicated across locations.geojson and location_groups.txt (IDs must be unique across stops.stop_id, locations.geojson id, and location_groups.location_group_id)", + }, + { + name: "multiple stops with same ID within stops.txt (should pass this check)", + entities: []tt.Entity{ + >fs.Stop{StopID: tt.NewString("stop_1")}, + >fs.Stop{StopID: tt.NewString("stop_1")}, + }, + expectError: false, // This check doesn't validate duplicates within the same file + }, + { + name: "empty IDs (should not trigger error)", + entities: []tt.Entity{ + >fs.Stop{StopID: tt.NewString("")}, + >fs.Location{LocationID: tt.NewString("")}, + }, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + check := &FlexGeographyIDUniqueCheck{} + emap := tt.NewEntityMap() + + // First pass: AfterWrite to collect IDs + for _, ent := range tc.entities { + err := check.AfterWrite("", ent, emap) + if err != nil { + t.Fatalf("AfterWrite failed: %v", err) + } + } + + // Second pass: Validate to detect duplicates + var foundError bool + var errorMessage string + for _, ent := range tc.entities { + errs := check.Validate(ent) + if len(errs) > 0 { + foundError = true + errorMessage = errs[0].Error() + break + } + } + + if tc.expectError && !foundError { + t.Errorf("Expected error but got none") + } + if !tc.expectError && foundError { + t.Errorf("Expected no error but got: %s", errorMessage) + } + if tc.expectError && foundError && tc.errorMsg != "" { + if errorMessage != tc.errorMsg { + t.Errorf("Expected error message:\n %q\nGot:\n %q", tc.errorMsg, errorMessage) + } + } + }) + } +} + diff --git a/rules/flex_location_geometry.go b/rules/flex_location_geometry.go new file mode 100644 index 000000000..248343b94 --- /dev/null +++ b/rules/flex_location_geometry.go @@ -0,0 +1,64 @@ +package rules + +import ( + "fmt" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +// FlexLocationGeometryError reports when a location has invalid geometry. +type FlexLocationGeometryError struct { + LocationID string + ErrorMessage string + bc +} + +func (e *FlexLocationGeometryError) Error() string { + return fmt.Sprintf( + "location '%s' has invalid geometry: %s", + e.LocationID, + e.ErrorMessage, + ) +} + +// FlexLocationGeometryCheck validates that locations have valid Polygon or MultiPolygon geometries. +type FlexLocationGeometryCheck struct{} + +func (e *FlexLocationGeometryCheck) Validate(ent tt.Entity) []error { + loc, ok := ent.(*gtfs.Location) + if !ok { + return nil + } + + var errs []error + + // Geometry must be present (already checked in ConditionalErrors) + if !loc.Geometry.Valid { + return nil + } + + // Validate that geometry is Polygon or MultiPolygon + geom := loc.Geometry.Val + switch geom.(type) { + case nil: + errs = append(errs, &FlexLocationGeometryError{ + LocationID: loc.LocationID.Val, + ErrorMessage: "geometry is nil", + }) + default: + // Additional geometry validation could go here + // For example: check for self-intersections, minimum area, etc. + // Check that polygon has at least 3 points + coords := loc.Geometry.FlatCoords() + if len(coords) < 6 { // At least 3 points (x,y) = 6 coordinates for a closed ring + errs = append(errs, &FlexLocationGeometryError{ + LocationID: loc.LocationID.Val, + ErrorMessage: fmt.Sprintf("polygon has insufficient coordinates: %d (need at least 6 for closed triangle)", len(coords)), + }) + } + } + + return errs +} + diff --git a/rules/flex_location_group_empty.go b/rules/flex_location_group_empty.go new file mode 100644 index 000000000..6b4715071 --- /dev/null +++ b/rules/flex_location_group_empty.go @@ -0,0 +1,57 @@ +package rules + +import ( + "fmt" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +// FlexLocationGroupEmptyError reports when a location_group has no associated stops. +type FlexLocationGroupEmptyError struct { + LocationGroupID string + bc +} + +func (e *FlexLocationGroupEmptyError) Error() string { + return fmt.Sprintf( + "location_group '%s' has no associated stops in location_group_stops.txt", + e.LocationGroupID, + ) +} + +// FlexLocationGroupEmptyCheck validates that all location_groups have at least one stop. +type FlexLocationGroupEmptyCheck struct { + locationGroupStops map[string]int // location_group_id -> count of stops +} + +func (e *FlexLocationGroupEmptyCheck) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error { + if e.locationGroupStops == nil { + e.locationGroupStops = map[string]int{} + } + + // Count stops for each location_group + if lgs, ok := ent.(*gtfs.LocationGroupStop); ok { + e.locationGroupStops[lgs.LocationGroupID.Val]++ + } + + return nil +} + +func (e *FlexLocationGroupEmptyCheck) Validate(ent tt.Entity) []error { + lg, ok := ent.(*gtfs.LocationGroup) + if !ok { + return nil + } + + // Check if this location_group has any stops + count := e.locationGroupStops[lg.LocationGroupID.Val] + if count == 0 { + return []error{&FlexLocationGroupEmptyError{ + LocationGroupID: lg.LocationGroupID.Val, + }} + } + + return nil +} + diff --git a/rules/flex_stop_location_type.go b/rules/flex_stop_location_type.go new file mode 100644 index 000000000..fad822d9d --- /dev/null +++ b/rules/flex_stop_location_type.go @@ -0,0 +1,88 @@ +package rules + +import ( + "fmt" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +// FlexStopLocationTypeError reports when a stop referenced in a flex service has invalid location_type. +type FlexStopLocationTypeError struct { + StopID string + LocationType int + bc +} + +func (e *FlexStopLocationTypeError) Error() string { + return fmt.Sprintf( + "stop '%s' referenced in flex service has location_type %d, but flex services can only use stops with location_type 0 (stop/platform)", + e.StopID, + e.LocationType, + ) +} + +// FlexStopLocationTypeCheck validates that stops referenced in flex stop_times have appropriate location_type. +// According to GTFS-Flex, stops used in continuous stopping or location_id/location_group_id references +// must be location_type=0 (stop/platform). +type FlexStopLocationTypeCheck struct { + locationTypes map[string]int + flexStops map[string]bool // stops used in flex services +} + +func (e *FlexStopLocationTypeCheck) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error { + if e.locationTypes == nil { + e.locationTypes = map[string]int{} + } + if e.flexStops == nil { + e.flexStops = map[string]bool{} + } + + // Track location_type for all stops + if stop, ok := ent.(*gtfs.Stop); ok { + e.locationTypes[eid] = stop.LocationType.Int() + } + + // Track stops used in flex services + if st, ok := ent.(*gtfs.StopTime); ok { + // Check if this stop_time indicates a flex service + isFlex := (st.ContinuousPickup.Valid && st.ContinuousPickup.Val == 2) || + (st.ContinuousDropOff.Valid && st.ContinuousDropOff.Val == 2) || + (st.PickupType.Valid && st.PickupType.Val == 2) || + (st.DropOffType.Valid && st.DropOffType.Val == 2) || + st.StartPickupDropOffWindow.Valid || + st.EndPickupDropOffWindow.Valid || + st.PickupBookingRuleID.Valid || + st.DropOffBookingRuleID.Valid + + if isFlex && st.StopID.Valid { + e.flexStops[st.StopID.Val] = true + } + } + + return nil +} + +func (e *FlexStopLocationTypeCheck) Validate(ent tt.Entity) []error { + // Run validation after all entities are processed + stop, ok := ent.(*gtfs.Stop) + if !ok { + return nil + } + + // Check if this stop is used in flex services + if !e.flexStops[stop.StopID.Val] { + return nil + } + + // Flex stops must be location_type = 0 + locationType := e.locationTypes[stop.StopID.Val] + if locationType != 0 { + return []error{&FlexStopLocationTypeError{ + StopID: stop.StopID.Val, + LocationType: locationType, + }} + } + + return nil +} diff --git a/rules/flex_validation_test.go b/rules/flex_validation_test.go new file mode 100644 index 000000000..ffd631f77 --- /dev/null +++ b/rules/flex_validation_test.go @@ -0,0 +1,318 @@ +package rules + +import ( + "fmt" + "testing" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" + "github.com/stretchr/testify/assert" +) + +func TestFlexStopLocationTypeCheck(t *testing.T) { + emap := tt.NewEntityMap() + + tests := []struct { + name string + stop *gtfs.Stop + stopTime *gtfs.StopTime + expectError bool + }{ + { + name: "Valid: location_type=0 with flex service", + stop: >fs.Stop{ + StopID: tt.NewString("stop1"), + LocationType: tt.NewInt(0), + }, + stopTime: >fs.StopTime{ + StopID: tt.NewKey("stop1"), + PickupType: tt.NewInt(2), // flex service + }, + expectError: false, + }, + { + name: "Invalid: location_type=1 (station) with flex service", + stop: >fs.Stop{ + StopID: tt.NewString("stop2"), + LocationType: tt.NewInt(1), + }, + stopTime: >fs.StopTime{ + StopID: tt.NewKey("stop2"), + PickupType: tt.NewInt(2), + }, + expectError: true, + }, + { + name: "Valid: location_type=1 without flex service", + stop: >fs.Stop{ + StopID: tt.NewString("stop3"), + LocationType: tt.NewInt(1), + }, + stopTime: >fs.StopTime{ + StopID: tt.NewKey("stop3"), + PickupType: tt.NewInt(0), // regular service + }, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + check := &FlexStopLocationTypeCheck{} + + // Important: Process entities in order - stops first, then stop_times + // This mimics how the copier processes entities + + // 1. Add the stop + tc.stop.ID = 1 + check.AfterWrite(tc.stop.StopID.Val, tc.stop, emap) + + // 2. Add the stop_time (this marks the stop as flex if applicable) + check.AfterWrite(tc.stopTime.StopID.Val, tc.stopTime, emap) + + // 3. Validate the stop (this checks if flex stops have correct location_type) + errs := check.Validate(tc.stop) + + if tc.expectError { + assert.NotEmpty(t, errs, "Expected validation error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexStopLocationTypeError) + assert.True(t, ok, "Expected FlexStopLocationTypeError") + } + } else { + assert.Empty(t, errs, "Expected no validation errors") + } + }) + } +} + +func TestFlexLocationGroupEmptyCheck(t *testing.T) { + emap := tt.NewEntityMap() + + tests := []struct { + name string + group *gtfs.LocationGroup + stops []*gtfs.LocationGroupStop + expectError bool + }{ + { + name: "Valid: location_group with stops", + group: >fs.LocationGroup{ + LocationGroupID: tt.NewString("group1"), + }, + stops: []*gtfs.LocationGroupStop{ + {LocationGroupID: tt.NewKey("group1"), StopID: tt.NewKey("stop1")}, + {LocationGroupID: tt.NewKey("group1"), StopID: tt.NewKey("stop2")}, + }, + expectError: false, + }, + { + name: "Invalid: location_group with no stops", + group: >fs.LocationGroup{ + LocationGroupID: tt.NewString("group2"), + }, + stops: []*gtfs.LocationGroupStop{}, + expectError: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + check := &FlexLocationGroupEmptyCheck{} + + // Add the location_group + tc.group.ID = 1 + check.AfterWrite("1", tc.group, emap) + + // Add the location_group_stops + for i, lgs := range tc.stops { + lgs.ID = i + 2 + check.AfterWrite(fmt.Sprintf("%d", i+2), lgs, emap) + } + + // Validate + errs := check.Validate(tc.group) + + if tc.expectError { + assert.NotEmpty(t, errs, "Expected validation error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexLocationGroupEmptyError) + assert.True(t, ok, "Expected FlexLocationGroupEmptyError") + } + } else { + assert.Empty(t, errs, "Expected no validation errors") + } + }) + } +} + +func TestFlexZoneIDConditionalCheck(t *testing.T) { + emap := tt.NewEntityMap() + + tests := []struct { + name string + location *gtfs.Location + fareRule *gtfs.FareRule + expectError bool + }{ + { + name: "Valid: zone_id present with fare_rules", + location: >fs.Location{ + LocationID: tt.NewString("loc1"), + ZoneID: tt.NewString("zone1"), + }, + fareRule: >fs.FareRule{FareID: tt.NewString("fare1")}, + expectError: false, + }, + { + name: "Invalid: zone_id missing with fare_rules", + location: >fs.Location{ + LocationID: tt.NewString("loc2"), + ZoneID: tt.String{}, + }, + fareRule: >fs.FareRule{FareID: tt.NewString("fare1")}, + expectError: true, + }, + { + name: "Valid: zone_id missing without fare_rules", + location: >fs.Location{ + LocationID: tt.NewString("loc3"), + ZoneID: tt.String{}, + }, + fareRule: nil, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + check := &FlexZoneIDConditionalCheck{} + + // Add fare_rule if present + if tc.fareRule != nil { + tc.fareRule.ID = 1 + check.AfterWrite("1", tc.fareRule, emap) + } + + // Add the location + tc.location.ID = 2 + check.AfterWrite("2", tc.location, emap) + + // Validate + errs := check.Validate(tc.location) + + if tc.expectError { + assert.NotEmpty(t, errs, "Expected validation error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexZoneIDRequiredError) + assert.True(t, ok, "Expected FlexZoneIDRequiredError") + } + } else { + assert.Empty(t, errs, "Expected no validation errors") + } + }) + } +} + +func TestFlexGeographyIDUniqueCheckIntegration(t *testing.T) { + emap := tt.NewEntityMap() + + t.Run("duplicate stop_id with location_group_id", func(t *testing.T) { + check := &FlexGeographyIDUniqueCheck{} + + // Create a stop + stop := >fs.Stop{ + StopID: tt.NewString("stop1"), + StopName: tt.NewString("Test Stop"), + } + stop.ID = 1 + + // Create a location_group with the same ID + locationGroup := >fs.LocationGroup{ + LocationGroupID: tt.NewString("stop1"), + LocationGroupName: tt.NewString("Duplicate ID"), + } + locationGroup.ID = 2 + + // Process entities + check.AfterWrite("1", stop, emap) + check.AfterWrite("2", locationGroup, emap) + + // Validate - should detect duplicate + errs := check.Validate(locationGroup) + assert.NotEmpty(t, errs, "Expected duplicate geography ID error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexGeographyIDDuplicateError) + assert.True(t, ok, "Expected FlexGeographyIDDuplicateError") + assert.Contains(t, errs[0].Error(), "stop1") + assert.Contains(t, errs[0].Error(), "stops.txt") + assert.Contains(t, errs[0].Error(), "location_groups.txt") + } + }) + + t.Run("duplicate stop_id with location_id", func(t *testing.T) { + check := &FlexGeographyIDUniqueCheck{} + + // Create a stop + stop := >fs.Stop{ + StopID: tt.NewString("area_1"), + StopName: tt.NewString("Test Stop"), + } + stop.ID = 1 + + // Create a location with the same ID + location := >fs.Location{ + LocationID: tt.NewString("area_1"), + StopName: tt.NewString("Duplicate ID"), + } + location.ID = 2 + + // Process entities + check.AfterWrite("1", stop, emap) + check.AfterWrite("2", location, emap) + + // Validate - should detect duplicate + errs := check.Validate(location) + assert.NotEmpty(t, errs, "Expected duplicate geography ID error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexGeographyIDDuplicateError) + assert.True(t, ok, "Expected FlexGeographyIDDuplicateError") + assert.Contains(t, errs[0].Error(), "area_1") + assert.Contains(t, errs[0].Error(), "stops.txt") + assert.Contains(t, errs[0].Error(), "locations.geojson") + } + }) + + t.Run("duplicate location_group_id with location_id", func(t *testing.T) { + check := &FlexGeographyIDUniqueCheck{} + + // Create a location_group + locationGroup := >fs.LocationGroup{ + LocationGroupID: tt.NewString("zone_1"), + LocationGroupName: tt.NewString("Zone 1"), + } + locationGroup.ID = 1 + + // Create a location with the same ID + location := >fs.Location{ + LocationID: tt.NewString("zone_1"), + StopName: tt.NewString("Duplicate ID"), + } + location.ID = 2 + + // Process entities + check.AfterWrite("1", locationGroup, emap) + check.AfterWrite("2", location, emap) + + // Validate - should detect duplicate + errs := check.Validate(location) + assert.NotEmpty(t, errs, "Expected duplicate geography ID error") + if len(errs) > 0 { + _, ok := errs[0].(*FlexGeographyIDDuplicateError) + assert.True(t, ok, "Expected FlexGeographyIDDuplicateError") + assert.Contains(t, errs[0].Error(), "zone_1") + assert.Contains(t, errs[0].Error(), "location_groups.txt") + assert.Contains(t, errs[0].Error(), "locations.geojson") + } + }) +} diff --git a/rules/flex_zone_id_conditional.go b/rules/flex_zone_id_conditional.go new file mode 100644 index 000000000..5c615f398 --- /dev/null +++ b/rules/flex_zone_id_conditional.go @@ -0,0 +1,57 @@ +package rules + +import ( + "fmt" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" +) + +// FlexZoneIDRequiredError reports when zone_id is missing in locations.geojson but fare_rules.txt exists. +type FlexZoneIDRequiredError struct { + LocationID string + bc +} + +func (e *FlexZoneIDRequiredError) Error() string { + return fmt.Sprintf( + "location '%s' requires zone_id when fare_rules.txt is defined", + e.LocationID, + ) +} + +// FlexZoneIDConditionalCheck validates that locations have zone_id when fare_rules.txt exists. +type FlexZoneIDConditionalCheck struct { + hasFareRules bool + checkedFiles bool +} + +func (e *FlexZoneIDConditionalCheck) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error { + // Check if fare_rules.txt has any entries + if _, ok := ent.(*gtfs.FareRule); ok { + e.hasFareRules = true + } + e.checkedFiles = true + return nil +} + +func (e *FlexZoneIDConditionalCheck) Validate(ent tt.Entity) []error { + if !e.checkedFiles { + return nil + } + + loc, ok := ent.(*gtfs.Location) + if !ok { + return nil + } + + // If fare_rules.txt exists, zone_id is required in locations + if e.hasFareRules && !loc.ZoneID.Valid { + return []error{&FlexZoneIDRequiredError{ + LocationID: loc.LocationID.Val, + }} + } + + return nil +} + diff --git a/rules/stop_time_sequence.go b/rules/stop_time_sequence.go index cd047aca4..15c96856a 100644 --- a/rules/stop_time_sequence.go +++ b/rules/stop_time_sequence.go @@ -21,9 +21,18 @@ func (e *StopTimeSequenceCheck) Validate(ent tt.Entity) []error { return errs } -// ValidateStopTimes checks if the trip follows GTFS rules. +// hasTimeWindow returns true if the stop_time has GTFS-Flex time windows defined. +// Time windows are mutually exclusive with arrival_time/departure_time. +func hasTimeWindow(st gtfs.StopTime) bool { + return (st.StartPickupDropOffWindow.Valid && st.StartPickupDropOffWindow.Val > 0) || + (st.EndPickupDropOffWindow.Valid && st.EndPickupDropOffWindow.Val > 0) +} + +// ValidateStopTimes checks if the trip follows GTFS rules, including GTFS-Flex extensions. func ValidateStopTimes(stoptimes []gtfs.StopTime) []error { errs := []error{} + + // 1. Check has >= 2 stop_times if len(stoptimes) == 0 { errs = append(errs, causes.NewEmptyTripError(len(stoptimes))) return errs // assumes >= 1 below @@ -31,32 +40,57 @@ func ValidateStopTimes(stoptimes []gtfs.StopTime) []error { if len(stoptimes) < 2 { errs = append(errs, causes.NewEmptyTripError(len(stoptimes))) } - if lastSt := stoptimes[len(stoptimes)-1]; lastSt.ArrivalTime.Int() <= 0 { - errs = append(errs, causes.NewSequenceError("arrival_time", lastSt.ArrivalTime.String())) + + // 2. Last stop validation: Must have arrival_time OR time window + // Note: First stop departure_time is not required by GTFS spec + // (arrival time is meaningless at start of trip, departure time is meaningless at end of trip) + lastSt := stoptimes[len(stoptimes)-1] + if lastSt.ArrivalTime.Int() <= 0 && !hasTimeWindow(lastSt) { + errs = append(errs, causes.NewSequenceError("arrival_time", "missing on last stop (required unless time window present)")) } + + // Initialize tracking variables lastDist := stoptimes[0].ShapeDistTraveled - lastTime := stoptimes[0].DepartureTime + lastScheduledTime := stoptimes[0].DepartureTime // Track time only for scheduled stops lastSequence := stoptimes[0].StopSequence + + // 3-5. Validate stop sequences, time progression, and shape distances for _, st := range stoptimes[1:] { - // Ensure we do not have duplicate StopSequennce + // 3. Stop sequence validation: No duplicates, must increase if st.StopSequence == lastSequence { errs = append(errs, causes.NewSequenceError("stop_sequence", st.StopSequence.String())) } else { lastSequence = st.StopSequence } - // Ensure the arrows of time are pointing towards the future. - if st.ArrivalTime.Int() > 0 && st.ArrivalTime.Int() < lastTime.Int() { - errs = append(errs, causes.NewSequenceError("arrival_time", st.ArrivalTime.String())) - } else if st.DepartureTime.Int() > 0 && st.DepartureTime.Int() < st.ArrivalTime.Int() { - errs = append(errs, causes.NewSequenceError("departure_time", st.DepartureTime.String())) - } else if st.DepartureTime.Int() > 0 { - lastTime = st.DepartureTime + + // 4. Time progression validation (only for scheduled stops, skip flex stops) + if !hasTimeWindow(st) { + // This is a scheduled stop with arrival/departure times + if st.ArrivalTime.Int() > 0 && lastScheduledTime.Int() > 0 && st.ArrivalTime.Int() < lastScheduledTime.Int() { + errs = append(errs, causes.NewSequenceError("arrival_time", st.ArrivalTime.String())) + } + if st.DepartureTime.Int() > 0 && st.ArrivalTime.Int() > 0 && st.DepartureTime.Int() < st.ArrivalTime.Int() { + errs = append(errs, causes.NewSequenceError("departure_time", st.DepartureTime.String())) + } + // Update last scheduled time for next comparison + // Only update if this stop has explicit times (not interpolated/missing) + if st.DepartureTime.Int() > 0 { + lastScheduledTime = st.DepartureTime + } else if st.ArrivalTime.Int() > 0 { + lastScheduledTime = st.ArrivalTime + } + // If both times are 0/missing, keep previous lastScheduledTime for next scheduled stop comparison } - if st.ShapeDistTraveled.Valid && st.ShapeDistTraveled.Val < lastDist.Val { + // else: Flex stop with time window - skip time progression validation + + // 5. Shape distance validation: Must increase when present + if st.ShapeDistTraveled.Valid && lastDist.Valid && st.ShapeDistTraveled.Val < lastDist.Val { errs = append(errs, causes.NewSequenceError("shape_dist_traveled", st.ShapeDistTraveled.String())) - } else if st.ShapeDistTraveled.Valid { + } + if st.ShapeDistTraveled.Valid { lastDist = st.ShapeDistTraveled } } + return errs } diff --git a/rules/stop_time_sequence_test.go b/rules/stop_time_sequence_test.go index 80e95f364..89645a290 100644 --- a/rules/stop_time_sequence_test.go +++ b/rules/stop_time_sequence_test.go @@ -5,80 +5,367 @@ import ( "testing" "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/internal/testutil" "github.com/interline-io/transitland-lib/tt" ) -type expectTrip struct { - ExpectError string - ArrivalTime []int - DepartureTime []int - ShapeDistTraveled []float64 -} +// Test helpers for building stop_time sequences with various configurations. +// These helpers make it easy to construct test cases for both scheduled GTFS +// and GTFS-Flex trips with time windows. -func expectTripToStopTime(e expectTrip) []gtfs.StopTime { - ret := []gtfs.StopTime{} - for i := range e.ArrivalTime { - ret = append(ret, gtfs.StopTime{ - TripID: tt.NewString("1"), - StopID: tt.NewString(strconv.Itoa(i)), - StopSequence: tt.NewInt(i), - ArrivalTime: tt.NewSeconds(e.ArrivalTime[i]), - DepartureTime: tt.NewSeconds(e.DepartureTime[i]), - ShapeDistTraveled: tt.NewFloat(e.ShapeDistTraveled[i]), - }) +// Helper to create a basic stop_time with common defaults +func makeStopTime(stopSeq int, arrivalTime, departureTime int) gtfs.StopTime { + return gtfs.StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey(strconv.Itoa(stopSeq)), + StopSequence: tt.NewInt(stopSeq), + ArrivalTime: tt.NewSeconds(arrivalTime), + DepartureTime: tt.NewSeconds(departureTime), } - return ret } -func TestValidateStopTimes(t *testing.T) { - // base cases - trips := []expectTrip{ - {"1", []int{10, 20, 30}, []int{10, 20, 30}, []float64{0, 1, 2}}, // all specified - {"2", []int{10, 0, 30}, []int{10, 0, 30}, []float64{0, 1, 2}}, // ends specified - {"3", []int{10, 20, 30}, []int{10, 20, 30}, []float64{0, 0, 0}}, // no dist - {"4", []int{0, 20, 30}, []int{10, 20, 30}, []float64{0, 1, 2}}, // missing first arrival_time - {"5", []int{10, 20, 30}, []int{10, 20, 0}, []float64{0, 1, 2}}, // missing last departure_time - {"6", []int{10, 20, 30}, []int{10, 20, 30}, []float64{0, 1, 2}}, // two is OK +// Helper to create a flex stop_time with time windows +func makeFlexStopTime(stopSeq int, startWindow, endWindow int) gtfs.StopTime { + st := gtfs.StopTime{ + TripID: tt.NewString("trip1"), + StopID: tt.NewKey(strconv.Itoa(stopSeq)), + StopSequence: tt.NewInt(stopSeq), } - for _, et := range trips { - t.Run(et.ExpectError, func(t *testing.T) { - stoptimes := expectTripToStopTime(et) - if errs := ValidateStopTimes(stoptimes); len(errs) > 0 { - t.Errorf("got %d errors, expected %d: %s", len(errs), 0, errs) - } - }) + if startWindow > 0 { + st.StartPickupDropOffWindow = tt.NewSeconds(startWindow) } - // error cases - errortrips := []expectTrip{ - {"Error:OneStopTime", []int{10}, []int{10}, []float64{0}}, - {"Error:NoFinalArrivalTime", []int{10, 0}, []int{10, 0}, []float64{0, 0}}, - {"SequenceError:departure_time", []int{10, 20, 5}, []int{10, 20, 5}, []float64{0, 1, 2}}, - {"SequenceError:shape_pt_traveled", []int{10, 20, 30}, []int{10, 20, 30}, []float64{1, 2, 1}}, + if endWindow > 0 { + st.EndPickupDropOffWindow = tt.NewSeconds(endWindow) } - for _, et := range errortrips { - t.Run(et.ExpectError, func(t *testing.T) { - stoptimes := expectTripToStopTime(et) - if errs := ValidateStopTimes(stoptimes); len(errs) != 1 { - t.Errorf("expected 1 error, got 0") - } + return st +} + +// Helper to add shape distance to a stop_time +func withShapeDist(st gtfs.StopTime, dist float64) gtfs.StopTime { + st.ShapeDistTraveled = tt.NewFloat(dist) + return st +} + +// Helper to override stop sequence +func withStopSequence(st gtfs.StopTime, seq int) gtfs.StopTime { + st.StopSequence = tt.NewInt(seq) + return st +} + +func TestValidateStopTimes(t *testing.T) { + tests := []struct { + name string + stopTimes []gtfs.StopTime + expectedErrors []testutil.ExpectError + }{ + // ===== VALID SCHEDULED TRIPS ===== + { + name: "valid_basic_scheduled_trip", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 2000, 2000), + makeStopTime(2, 3000, 3000), + }, + expectedErrors: nil, + }, + { + name: "valid_missing_intermediate_times", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 0, 0), // intermediate can be missing + makeStopTime(2, 3000, 3000), + }, + expectedErrors: nil, + }, + { + name: "valid_first_missing_arrival", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 0, 1000), // first arrival_time can be missing + makeStopTime(1, 2000, 2000), + }, + expectedErrors: nil, + }, + { + name: "valid_last_missing_departure", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 2000, 0), // last departure_time can be missing + }, + expectedErrors: nil, + }, + { + name: "valid_with_shape_distances", + stopTimes: []gtfs.StopTime{ + withShapeDist(makeStopTime(0, 1000, 1000), 0.0), + withShapeDist(makeStopTime(1, 2000, 2000), 5.5), + withShapeDist(makeStopTime(2, 3000, 3000), 12.3), + }, + expectedErrors: nil, + }, + { + name: "valid_no_shape_distances", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 2000, 2000), + makeStopTime(2, 3000, 3000), + }, + expectedErrors: nil, + }, + { + name: "valid_partial_shape_distances", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + withShapeDist(makeStopTime(1, 2000, 2000), 5.5), + withShapeDist(makeStopTime(2, 3000, 3000), 12.3), + }, + expectedErrors: nil, + }, + + // ===== VALID FLEX TRIPS ===== + { + name: "valid_flex_trip_start_window", + stopTimes: []gtfs.StopTime{ + makeFlexStopTime(0, 1000, 0), // start_window only + makeFlexStopTime(1, 2000, 0), + makeFlexStopTime(2, 3000, 0), + }, + expectedErrors: nil, + }, + { + name: "valid_flex_trip_end_window", + stopTimes: []gtfs.StopTime{ + makeFlexStopTime(0, 0, 1000), // end_window only + makeFlexStopTime(1, 0, 2000), + makeFlexStopTime(2, 0, 3000), + }, + expectedErrors: nil, + }, + { + name: "valid_flex_trip_both_windows", + stopTimes: []gtfs.StopTime{ + makeFlexStopTime(0, 1000, 2000), // both windows + makeFlexStopTime(1, 2000, 3000), + makeFlexStopTime(2, 3000, 4000), + }, + expectedErrors: nil, + }, + + // ===== VALID MIXED TRIPS ===== + { + name: "valid_mixed_scheduled_then_flex", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), // scheduled + makeStopTime(1, 2000, 2000), // scheduled + makeFlexStopTime(2, 3000, 4000), // flex + }, + expectedErrors: nil, + }, + { + name: "valid_mixed_flex_then_scheduled", + stopTimes: []gtfs.StopTime{ + makeFlexStopTime(0, 1000, 2000), // flex + makeStopTime(1, 3000, 3000), // scheduled + makeStopTime(2, 4000, 4000), // scheduled + }, + expectedErrors: nil, + }, + { + name: "valid_mixed_interleaved", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), // scheduled + makeFlexStopTime(1, 2000, 3000), // flex + makeStopTime(2, 4000, 4000), // scheduled + }, + expectedErrors: nil, + }, + + // ===== ERRORS: EMPTY/TOO FEW STOPS ===== + { + name: "error_empty_trip", + stopTimes: []gtfs.StopTime{}, + expectedErrors: testutil.ParseExpectErrors("EmptyTripError"), + }, + { + name: "error_single_stop", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + }, + expectedErrors: testutil.ParseExpectErrors("EmptyTripError"), + }, + + // ===== ERRORS: LAST STOP ===== + { + name: "error_last_stop_no_arrival_or_window", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 0, 2000), // missing arrival_time and no time window + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:arrival_time"), + }, + + // ===== ERRORS: STOP SEQUENCE ===== + { + name: "error_duplicate_stop_sequence", + stopTimes: []gtfs.StopTime{ + makeStopTime(1, 1000, 1000), + makeStopTime(2, 2000, 2000), + withStopSequence(makeStopTime(2, 3000, 3000), 2), // duplicate sequence + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:stop_sequence"), + }, + { + name: "error_multiple_duplicate_sequences", + stopTimes: []gtfs.StopTime{ + makeStopTime(5, 1000, 1000), + withStopSequence(makeStopTime(1, 2000, 2000), 5), // duplicate + makeStopTime(10, 3000, 3000), + withStopSequence(makeStopTime(3, 4000, 4000), 10), // duplicate + }, + expectedErrors: testutil.ParseExpectErrors( + "SequenceError:stop_sequence", + "SequenceError:stop_sequence", + ), + }, + + // ===== ERRORS: TIME PROGRESSION (SCHEDULED STOPS ONLY) ===== + { + name: "error_arrival_before_previous_departure", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 500, 2000), // arrival < previous departure + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:arrival_time"), + }, + { + name: "error_departure_before_arrival_same_stop", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 2000, 1500), // departure < arrival at same stop + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:departure_time"), + }, + { + name: "error_multiple_time_violations", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 500, 2000), // arrival < previous departure + makeStopTime(2, 3000, 2500), // departure < arrival + }, + expectedErrors: testutil.ParseExpectErrors( + "SequenceError:arrival_time", + "SequenceError:departure_time", + ), + }, + { + name: "error_time_goes_backward", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), + makeStopTime(1, 2000, 2000), + makeStopTime(2, 1500, 1500), // times go backward + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:arrival_time"), + }, + + // ===== NO ERRORS: TIME PROGRESSION WITH FLEX STOPS ===== + { + name: "no_error_flex_stop_skips_time_validation", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), // scheduled at 1000 + makeFlexStopTime(1, 100, 200), // flex with "earlier" window - should be ignored + makeStopTime(2, 3000, 3000), // scheduled at 3000 (compared to stop 0) + }, + expectedErrors: nil, + }, + { + name: "no_error_mixed_trip_time_validation_skips_flex", + stopTimes: []gtfs.StopTime{ + makeStopTime(0, 1000, 1000), // scheduled + makeFlexStopTime(1, 500, 600), // flex in between + makeStopTime(2, 2000, 2000), // scheduled (compared to stop 0, not stop 1) + }, + expectedErrors: nil, + }, + + // ===== ERRORS: SHAPE DISTANCE ===== + { + name: "error_shape_distance_decreases", + stopTimes: []gtfs.StopTime{ + withShapeDist(makeStopTime(0, 1000, 1000), 10.0), + withShapeDist(makeStopTime(1, 2000, 2000), 5.0), // distance decreases + }, + expectedErrors: testutil.ParseExpectErrors("SequenceError:shape_dist_traveled"), + }, + { + name: "error_multiple_shape_distance_violations", + stopTimes: []gtfs.StopTime{ + withShapeDist(makeStopTime(0, 1000, 1000), 10.0), + withShapeDist(makeStopTime(1, 2000, 2000), 5.0), // decreases + withShapeDist(makeStopTime(2, 3000, 3000), 15.0), + withShapeDist(makeStopTime(3, 4000, 4000), 12.0), // decreases + }, + expectedErrors: testutil.ParseExpectErrors( + "SequenceError:shape_dist_traveled", + "SequenceError:shape_dist_traveled", + ), + }, + { + name: "no_error_shape_distance_stays_same", + stopTimes: []gtfs.StopTime{ + withShapeDist(makeStopTime(0, 1000, 1000), 10.0), + withShapeDist(makeStopTime(1, 2000, 2000), 10.0), // same is OK + withShapeDist(makeStopTime(2, 3000, 3000), 15.0), + }, + expectedErrors: nil, + }, + + // ===== MULTIPLE ERROR TYPES ===== + { + name: "error_combined_sequence_and_time", + stopTimes: []gtfs.StopTime{ + makeStopTime(1, 1000, 1000), + makeStopTime(1, 500, 2000), // duplicate sequence + time violation + withStopSequence(makeStopTime(2, 3000, 3000), 1), // another duplicate sequence + }, + expectedErrors: testutil.ParseExpectErrors( + "SequenceError:stop_sequence", + "SequenceError:arrival_time", + "SequenceError:stop_sequence", + ), + }, + { + name: "error_all_validation_types", + stopTimes: []gtfs.StopTime{ + withShapeDist(makeStopTime(1, 1000, 1000), 10.0), // first stop is valid now + withShapeDist(makeStopTime(1, 2000, 2000), 5.0), // duplicate sequence + shape decreases + withShapeDist(withStopSequence(makeStopTime(2, 0, 3000), 1), 3.0), // another duplicate sequence + missing arrival (last stop) + shape decreases + }, + expectedErrors: testutil.ParseExpectErrors( + "SequenceError:arrival_time", + "SequenceError:stop_sequence", + "SequenceError:shape_dist_traveled", + "SequenceError:stop_sequence", + "SequenceError:shape_dist_traveled", + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errs := ValidateStopTimes(tt.stopTimes) + testutil.CheckErrors(tt.expectedErrors, errs, t) }) } - // Check for duplicate IDs - errorStopSequence := expectTrip{"", []int{10, 20, 30}, []int{10, 20, 30}, []float64{0, 1, 2}} - t.Run("SequenceError:stop_sequence", func(t *testing.T) { - stoptimes := expectTripToStopTime(errorStopSequence) - stoptimes[0].StopSequence.Set(1) - stoptimes[1].StopSequence.Set(2) - stoptimes[2].StopSequence.Set(2) - if errs := ValidateStopTimes(stoptimes); len(errs) != 1 { - t.Errorf("expected 1 error, got 0") - } - }) } func BenchmarkValidateStopTime(b *testing.B) { - trip := expectTrip{"1", []int{10, 20, 30, 40, 50, 60}, []int{10, 20, 30, 40, 50, 60}, []float64{0, 1, 2, 3, 4, 5, 6}} - stoptimes := expectTripToStopTime(trip) + stoptimes := []gtfs.StopTime{ + makeStopTime(0, 10, 10), + makeStopTime(1, 20, 20), + makeStopTime(2, 30, 30), + makeStopTime(3, 40, 40), + makeStopTime(4, 50, 50), + makeStopTime(5, 60, 60), + } b.ResetTimer() for n := 0; n < b.N; n++ { ValidateStopTimes(stoptimes) diff --git a/rules/too_far_from_shape.go b/rules/too_far_from_shape.go index 1ae034f16..f8aba3598 100644 --- a/rules/too_far_from_shape.go +++ b/rules/too_far_from_shape.go @@ -56,6 +56,9 @@ func (e *StopTooFarFromShapeCheck) Validate(ent tt.Entity) []error { } var errs []error for _, st := range v.StopTimes { + if !st.StopID.Valid { + continue + } // Check the cache if e.checked[shapeid][st.StopID.Val] { continue diff --git a/schema/postgres/migrations/20251121000001_gtfs_flex.up.pgsql b/schema/postgres/migrations/20251121000001_gtfs_flex.up.pgsql new file mode 100644 index 000000000..65cd7f381 --- /dev/null +++ b/schema/postgres/migrations/20251121000001_gtfs_flex.up.pgsql @@ -0,0 +1,105 @@ +BEGIN; + +-- GTFS-Flex: location_groups.txt +CREATE TABLE public.gtfs_location_groups ( + id bigserial primary key not null, + feed_version_id bigint REFERENCES feed_versions(id) not null, + created_at timestamp without time zone DEFAULT now() NOT NULL, + updated_at timestamp without time zone DEFAULT now() NOT NULL, + + location_group_id text NOT NULL, + location_group_name text +); +CREATE INDEX ON gtfs_location_groups(feed_version_id); +CREATE UNIQUE INDEX ON gtfs_location_groups(feed_version_id, location_group_id); + + +-- GTFS-Flex: location_group_stops.txt +CREATE TABLE public.gtfs_location_group_stops ( + id bigserial primary key not null, + feed_version_id bigint REFERENCES feed_versions(id) not null, + created_at timestamp without time zone DEFAULT now() NOT NULL, + updated_at timestamp without time zone DEFAULT now() NOT NULL, + + location_group_id bigint not null REFERENCES gtfs_location_groups(id), + stop_id bigint not null REFERENCES gtfs_stops(id) +); +CREATE INDEX ON gtfs_location_group_stops(feed_version_id); +CREATE INDEX ON gtfs_location_group_stops(location_group_id); +CREATE INDEX ON gtfs_location_group_stops(stop_id); +CREATE UNIQUE INDEX ON gtfs_location_group_stops(location_group_id, stop_id); + + +-- GTFS-Flex: booking_rules.txt +CREATE TABLE public.gtfs_booking_rules ( + id bigserial primary key not null, + feed_version_id bigint REFERENCES feed_versions(id) not null, + created_at timestamp without time zone DEFAULT now() NOT NULL, + updated_at timestamp without time zone DEFAULT now() NOT NULL, + + booking_rule_id text NOT NULL, + booking_type integer NOT NULL, + prior_notice_duration_min integer, + prior_notice_duration_max integer, + prior_notice_last_day integer, + prior_notice_last_time integer, + prior_notice_start_day integer, + prior_notice_start_time integer, + prior_notice_service_id bigint REFERENCES gtfs_calendars(id), + message text, + pickup_message text, + drop_off_message text, + phone_number text, + info_url text, + booking_url text +); +CREATE INDEX ON gtfs_booking_rules(feed_version_id); +CREATE UNIQUE INDEX ON gtfs_booking_rules(feed_version_id, booking_rule_id); + + +-- GTFS-Flex: locations.geojson +CREATE TABLE public.gtfs_locations ( + id bigserial primary key not null, + feed_version_id bigint REFERENCES feed_versions(id) not null, + created_at timestamp without time zone DEFAULT now() NOT NULL, + updated_at timestamp without time zone DEFAULT now() NOT NULL, + + location_id text NOT NULL, + stop_name text, + stop_desc text, + zone_id text, + stop_url text, + -- Geometry can be Polygon or MultiPolygon + geometry public.geography(Geometry,4326) +); +CREATE INDEX ON gtfs_locations(feed_version_id); +CREATE UNIQUE INDEX ON gtfs_locations(feed_version_id, location_id); +CREATE INDEX ON gtfs_locations USING GIST(geometry); + + +-- GTFS-Flex: Add new fields to stop_times +ALTER TABLE public.gtfs_stop_times + ADD COLUMN location_group_id bigint REFERENCES gtfs_location_groups(id), + ADD COLUMN location_id bigint REFERENCES gtfs_locations(id), + ADD COLUMN start_pickup_drop_off_window integer, + ADD COLUMN end_pickup_drop_off_window integer, + ADD COLUMN pickup_booking_rule_id bigint REFERENCES gtfs_booking_rules(id), + ADD COLUMN drop_off_booking_rule_id bigint REFERENCES gtfs_booking_rules(id), + ADD COLUMN mean_duration_factor double precision, + ADD COLUMN mean_duration_offset double precision, + ADD COLUMN safe_duration_factor double precision, + ADD COLUMN safe_duration_offset double precision; + +-- GTFS-Flex: Drop not-null constraint on stop_id (now conditionally required) +ALTER TABLE public.gtfs_stop_times + ALTER COLUMN stop_id DROP NOT NULL; + +-- GTFS-Flex: Add indexes on new foreign keys in stop_times for efficient queries and cascading deletes +CREATE INDEX ON gtfs_stop_times(location_group_id) WHERE location_group_id IS NOT NULL; +CREATE INDEX ON gtfs_stop_times(location_id) WHERE location_id IS NOT NULL; +CREATE INDEX ON gtfs_stop_times(pickup_booking_rule_id) WHERE pickup_booking_rule_id IS NOT NULL; +CREATE INDEX ON gtfs_stop_times(drop_off_booking_rule_id) WHERE drop_off_booking_rule_id IS NOT NULL; + +COMMIT; + + diff --git a/schema/sqlite/sqlite.sql b/schema/sqlite/sqlite.sql index 8411d4c64..42d530088 100644 --- a/schema/sqlite/sqlite.sql +++ b/schema/sqlite/sqlite.sql @@ -399,13 +399,32 @@ CREATE TABLE IF NOT EXISTS "gtfs_stop_times" ( "continuous_drop_off" integer, "interpolated" integer, "feed_version_id" integer NOT NULL, + -- GTFS-Flex fields + "location_group_id" integer, + "location_id" integer, + "start_pickup_drop_off_window" integer, + "end_pickup_drop_off_window" integer, + "pickup_booking_rule_id" integer, + "drop_off_booking_rule_id" integer, + "mean_duration_factor" real, + "mean_duration_offset" real, + "safe_duration_factor" real, + "safe_duration_offset" real, foreign key(feed_version_id) REFERENCES feed_versions(id), foreign key(trip_id) references gtfs_trips(id), - foreign key(stop_id) references gtfs_stops(id) + foreign key(stop_id) references gtfs_stops(id), + foreign key(location_group_id) references gtfs_location_groups(id), + foreign key(location_id) references gtfs_locations(id), + foreign key(pickup_booking_rule_id) references gtfs_booking_rules(id), + foreign key(drop_off_booking_rule_id) references gtfs_booking_rules(id) ); CREATE INDEX idx_stop_times_trip_id ON "gtfs_stop_times"(trip_id); CREATE INDEX idx_gtfs_stop_times_stop_id ON "gtfs_stop_times"(stop_id); CREATE INDEX idx_gtfs_stop_times_feed_version_id ON "gtfs_stop_times"(feed_version_id); +CREATE INDEX idx_gtfs_stop_times_location_group_id ON "gtfs_stop_times"(location_group_id) WHERE location_group_id IS NOT NULL; +CREATE INDEX idx_gtfs_stop_times_location_id ON "gtfs_stop_times"(location_id) WHERE location_id IS NOT NULL; +CREATE INDEX idx_gtfs_stop_times_pickup_booking_rule_id ON "gtfs_stop_times"(pickup_booking_rule_id) WHERE pickup_booking_rule_id IS NOT NULL; +CREATE INDEX idx_gtfs_stop_times_drop_off_booking_rule_id ON "gtfs_stop_times"(drop_off_booking_rule_id) WHERE drop_off_booking_rule_id IS NOT NULL; CREATE TABLE IF NOT EXISTS "gtfs_fare_rules" ( "fare_id" integer NOT NULL, "route_id" int, @@ -946,4 +965,78 @@ CREATE TABLE tl_materialized_active_agencies ( CREATE UNIQUE INDEX tl_materialized_active_agencies_id_idx ON tl_materialized_active_agencies(id); CREATE INDEX tl_materialized_active_agencies_agency_id_idx ON tl_materialized_active_agencies(agency_id); CREATE INDEX tl_materialized_active_agencies_feed_version_id_idx ON tl_materialized_active_agencies(feed_version_id); -CREATE INDEX tl_materialized_active_agencies_onestop_id_idx ON tl_materialized_active_agencies(onestop_id); \ No newline at end of file +CREATE INDEX tl_materialized_active_agencies_onestop_id_idx ON tl_materialized_active_agencies(onestop_id); + +-- GTFS-Flex: location_groups.txt +CREATE TABLE IF NOT EXISTS "gtfs_location_groups" ( + "id" integer primary key autoincrement, + "feed_version_id" integer NOT NULL, + "created_at" datetime DEFAULT CURRENT_TIMESTAMP, + "updated_at" datetime DEFAULT CURRENT_TIMESTAMP, + "location_group_id" varchar(255) NOT NULL, + "location_group_name" varchar(255), + foreign key(feed_version_id) REFERENCES feed_versions(id) +); +CREATE INDEX idx_gtfs_location_groups_feed_version_id ON "gtfs_location_groups"(feed_version_id); +CREATE UNIQUE INDEX idx_gtfs_location_groups_fv_id ON "gtfs_location_groups"(feed_version_id, location_group_id); + +-- GTFS-Flex: location_group_stops.txt +CREATE TABLE IF NOT EXISTS "gtfs_location_group_stops" ( + "id" integer primary key autoincrement, + "feed_version_id" integer NOT NULL, + "created_at" datetime DEFAULT CURRENT_TIMESTAMP, + "updated_at" datetime DEFAULT CURRENT_TIMESTAMP, + "location_group_id" integer NOT NULL, + "stop_id" integer NOT NULL, + foreign key(feed_version_id) REFERENCES feed_versions(id), + foreign key(location_group_id) references gtfs_location_groups(id), + foreign key(stop_id) references gtfs_stops(id) +); +CREATE INDEX idx_gtfs_location_group_stops_feed_version_id ON "gtfs_location_group_stops"(feed_version_id); +CREATE INDEX idx_gtfs_location_group_stops_location_group_id ON "gtfs_location_group_stops"(location_group_id); +CREATE INDEX idx_gtfs_location_group_stops_stop_id ON "gtfs_location_group_stops"(stop_id); +CREATE UNIQUE INDEX idx_gtfs_location_group_stops_lg_stop ON "gtfs_location_group_stops"(location_group_id, stop_id); + +-- GTFS-Flex: booking_rules.txt +CREATE TABLE IF NOT EXISTS "gtfs_booking_rules" ( + "id" integer primary key autoincrement, + "feed_version_id" integer NOT NULL, + "created_at" datetime DEFAULT CURRENT_TIMESTAMP, + "updated_at" datetime DEFAULT CURRENT_TIMESTAMP, + "booking_rule_id" varchar(255) NOT NULL, + "booking_type" integer NOT NULL, + "prior_notice_duration_min" integer, + "prior_notice_duration_max" integer, + "prior_notice_last_day" integer, + "prior_notice_last_time" integer, + "prior_notice_start_day" integer, + "prior_notice_start_time" integer, + "prior_notice_service_id" integer, + "message" text, + "pickup_message" text, + "drop_off_message" text, + "phone_number" varchar(255), + "info_url" text, + "booking_url" text, + foreign key(feed_version_id) REFERENCES feed_versions(id), + foreign key(prior_notice_service_id) references gtfs_calendars(id) +); +CREATE INDEX idx_gtfs_booking_rules_feed_version_id ON "gtfs_booking_rules"(feed_version_id); +CREATE UNIQUE INDEX idx_gtfs_booking_rules_fv_id ON "gtfs_booking_rules"(feed_version_id, booking_rule_id); + +-- GTFS-Flex: locations.geojson +CREATE TABLE IF NOT EXISTS "gtfs_locations" ( + "id" integer primary key autoincrement, + "feed_version_id" integer NOT NULL, + "created_at" datetime DEFAULT CURRENT_TIMESTAMP, + "updated_at" datetime DEFAULT CURRENT_TIMESTAMP, + "location_id" varchar(255) NOT NULL, + "stop_name" varchar(255), + "stop_desc" text, + "zone_id" varchar(255), + "stop_url" text, + "geometry" BLOB, + foreign key(feed_version_id) REFERENCES feed_versions(id) +); +CREATE INDEX idx_gtfs_locations_feed_version_id ON "gtfs_locations"(feed_version_id); +CREATE UNIQUE INDEX idx_gtfs_locations_fv_id ON "gtfs_locations"(feed_version_id, location_id); \ No newline at end of file diff --git a/service/tests/service_test.go b/service/tests/service_test.go index 9248610b4..5fe254971 100644 --- a/service/tests/service_test.go +++ b/service/tests/service_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/interline-io/transitland-lib/gtfs" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/service" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tt" @@ -78,7 +78,7 @@ func TestService_Simplify(t *testing.T) { } // get more examples from feeds feedchecks := []string{} - for _, v := range testutil.ExternalTestFeeds { + for _, v := range testreader.ExternalTestFeeds { feedchecks = append(feedchecks, v.URL) } for _, path := range feedchecks { diff --git a/stats/fvfi_test.go b/stats/fvfi_test.go index 9c433fee7..18b58987d 100644 --- a/stats/fvfi_test.go +++ b/stats/fvfi_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/interline-io/transitland-lib/dmfr" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tt" ) @@ -17,7 +17,7 @@ func TestNewFeedVersionFileInfosFromReader(t *testing.T) { }{ { "example", - testutil.ExampleZip.URL, + testreader.ExampleZip.URL, []dmfr.FeedVersionFileInfo{ { Name: "agency.txt", @@ -266,7 +266,7 @@ func TestNewFeedVersionFileInfosFromReader(t *testing.T) { }, { "bart", - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, []dmfr.FeedVersionFileInfo{ { Name: "agency.txt", diff --git a/stats/fvsl_test.go b/stats/fvsl_test.go index ee53781ef..8953a30cb 100644 --- a/stats/fvsl_test.go +++ b/stats/fvsl_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/interline-io/transitland-lib/dmfr" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tt" "github.com/stretchr/testify/assert" @@ -39,13 +39,13 @@ func TestNewFeedVersionServiceLevelsFromReader(t *testing.T) { }{ { "example", - testutil.ExampleZip.URL, + testreader.ExampleZip.URL, msi{"CITY": 4, "AB": 4, "STBA": 4, "": 4}, []string{}, }, { "bart", - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, msi{"01": 12, "11": 12, "03": 12}, []string{ // feed diff --git a/stats/fvsw_test.go b/stats/fvsw_test.go index 108a9e78c..cc5b78984 100644 --- a/stats/fvsw_test.go +++ b/stats/fvsw_test.go @@ -3,7 +3,7 @@ package stats import ( "testing" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tt" "github.com/stretchr/testify/assert" @@ -20,7 +20,7 @@ func TestNewFeedVersionServiceWindowsFromReader(t *testing.T) { }{ { "example", - testutil.ExampleZip.URL, + testreader.ExampleZip.URL, pd(""), pd(""), pd("2007-01-01"), @@ -28,7 +28,7 @@ func TestNewFeedVersionServiceWindowsFromReader(t *testing.T) { }, { "bart", - testutil.ExampleFeedBART.URL, + testreader.ExampleFeedBART.URL, pd("2018-05-26"), pd("2019-07-01"), pd("2018-06-04"), @@ -36,7 +36,7 @@ func TestNewFeedVersionServiceWindowsFromReader(t *testing.T) { }, { "caltrain", - testutil.ExampleFeedCaltrain.URL, + testreader.ExampleFeedCaltrain.URL, pd(""), pd(""), pd("2018-06-18"), diff --git a/testdata/gtfs-examples/bad-entities/booking_rules.txt b/testdata/gtfs-examples/bad-entities/booking_rules.txt new file mode 100644 index 000000000..8f555e532 --- /dev/null +++ b/testdata/gtfs-examples/bad-entities/booking_rules.txt @@ -0,0 +1,24 @@ +booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url,expect_error,expect_error,comment +ok_type0,0,,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,,Valid booking rule type 0 (real-time) +ok_type1,1,,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,,Valid booking rule type 1 (up to same day) +ok_type2,2,30,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,,Valid booking rule type 2 (advance notice) +missing_id,,0,,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,RequiredFieldError:booking_rule_id,booking_rule_id is required +parse_booking_type,ok1,xyz,,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,FieldParseError:booking_type,Non-integer booking_type value +invalid_booking_type,ok1,3,,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,InvalidFieldError:booking_type,booking_type must be 0-2 +parse_prior_notice_min,ok1,2,xyz,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,FieldParseError:prior_notice_duration_min,Non-integer prior_notice_duration_min value +parse_prior_notice_max,ok1,2,30,xyz,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,FieldParseError:prior_notice_duration_max,Non-integer prior_notice_duration_max value +parse_prior_notice_last_day,ok1,2,30,,xyz,,,,,,,,555-1234,https://example.com/info,https://example.com/book,FieldParseError:prior_notice_last_day,Non-integer prior_notice_last_day value +parse_prior_notice_start_day,ok1,2,30,,,,,xyz,,,,,555-1234,https://example.com/info,https://example.com/book,FieldParseError:prior_notice_start_day,Non-integer prior_notice_start_day value +type2_missing_min,ok2,2,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyRequiredFieldError:prior_notice_duration_min,booking_type=2 requires prior_notice_duration_min +type2_forbidden_max,ok2,2,30,60,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_duration_max,booking_type=2 forbids prior_notice_duration_max +type2_forbidden_last_day,ok2,2,30,,1,,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_last_day,booking_type=2 forbids prior_notice_last_day +type2_forbidden_last_time,ok2,2,30,,,12:00:00,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_last_time,booking_type=2 forbids prior_notice_last_time +type2_forbidden_start_day,ok2,2,30,,,,,1,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_start_day,booking_type=2 forbids prior_notice_start_day +type2_forbidden_start_time,ok2,2,30,,,,,,08:00:00,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_start_time,booking_type=2 forbids prior_notice_start_time +type2_forbidden_service_id,ok2,2,30,,,,,,,svc1,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_service_id,booking_type=2 forbids prior_notice_service_id +type01_forbidden_min,ok1,1,30,,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyForbiddenFieldError:prior_notice_duration_min,booking_type=0/1 forbids prior_notice_duration_min +last_time_without_day,ok1,0,,,,12:00:00,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyRequiredFieldError:prior_notice_last_day,prior_notice_last_time requires prior_notice_last_day +last_day_without_time,ok1,0,,,1,,,,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyRequiredFieldError:prior_notice_last_time,prior_notice_last_day requires prior_notice_last_time +start_time_without_day,ok1,0,,,,,,,08:00:00,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyRequiredFieldError:prior_notice_start_day,prior_notice_start_time requires prior_notice_start_day +start_day_without_time,ok1,0,,,,,,1,,,,,555-1234,https://example.com/info,https://example.com/book,ConditionallyRequiredFieldError:prior_notice_start_time,prior_notice_start_day requires prior_notice_start_time +max_less_than_min,ok1,0,30,15,,,,,,,,,555-1234,https://example.com/info,https://example.com/book,InvalidFieldError:prior_notice_duration_max,prior_notice_duration_max must be >= prior_notice_duration_min diff --git a/testdata/gtfs-examples/bad-entities/location_group_stops.txt b/testdata/gtfs-examples/bad-entities/location_group_stops.txt new file mode 100644 index 000000000..2ada82d29 --- /dev/null +++ b/testdata/gtfs-examples/bad-entities/location_group_stops.txt @@ -0,0 +1,4 @@ +location_group_id,stop_id,expect_error,expect_error,comment +lg1,stop1,,Valid location group stop association +missing_location_group,,stop1,RequiredFieldError:location_group_id,location_group_id is required +missing_stop,lg1,,RequiredFieldError:stop_id,stop_id is required diff --git a/testdata/gtfs-examples/bad-entities/location_groups.txt b/testdata/gtfs-examples/bad-entities/location_groups.txt new file mode 100644 index 000000000..3c5cb02bc --- /dev/null +++ b/testdata/gtfs-examples/bad-entities/location_groups.txt @@ -0,0 +1,3 @@ +location_group_id,location_group_name,expect_error,expect_error,comment +ok,Downtown Zone,,Valid location group +missing_id,,Valid Zone Name,RequiredFieldError:location_group_id,location_group_id is required diff --git a/testdata/gtfs-examples/bad-entities/locations.txt b/testdata/gtfs-examples/bad-entities/locations.txt new file mode 100644 index 000000000..5ff9ead38 --- /dev/null +++ b/testdata/gtfs-examples/bad-entities/locations.txt @@ -0,0 +1,5 @@ +location_id,location_name,location_lat,location_lon,expect_error,expect_error,comment +loc1,Flex Zone A,37.7749,-122.4194,,Valid location with coordinates +missing_id,,Valid Zone Name,37.7749,-122.4194,RequiredFieldError:location_id,location_id is required +parse_lat,loc2,Valid Zone,xyz,-122.4194,FieldParseError:location_lat,Non-numeric location_lat value +parse_lon,loc3,Valid Zone,37.7749,xyz,FieldParseError:location_lon,Non-numeric location_lon value diff --git a/testdata/gtfs-external/ctran-flex.zip b/testdata/gtfs-external/ctran-flex.zip new file mode 100644 index 000000000..53fb08329 Binary files /dev/null and b/testdata/gtfs-external/ctran-flex.zip differ diff --git a/tlcsv/adapter_test.go b/tlcsv/adapter_test.go index 3c38147d5..f75c3777a 100644 --- a/tlcsv/adapter_test.go +++ b/tlcsv/adapter_test.go @@ -9,17 +9,17 @@ import ( "github.com/interline-io/transitland-lib/gtfs" "github.com/interline-io/transitland-lib/internal/testpath" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" ) func getTestAdapters() map[string]func() Adapter { adapters := map[string]func() Adapter{ - "DirAdapter": func() Adapter { return NewDirAdapter(testutil.ExampleDir.URL) }, - "ZipAdapter": func() Adapter { return NewZipAdapter(testutil.ExampleZip.URL) }, - "ZipAdapterNestedDir": func() Adapter { return NewZipAdapter(testutil.ExampleZipNestedDir.URL) }, - "ZipAdapterNestedTwoFeeds": func() Adapter { return NewZipAdapter(testutil.ExampleZipNestedTwoFeeds2.URL) }, - "ZipAdapterNestedZip": func() Adapter { return NewZipAdapter(testutil.ExampleZipNestedZip.URL) }, - "OverlayAdapter": func() Adapter { return NewOverlayAdapter(testutil.ExampleDir.URL) }, + "DirAdapter": func() Adapter { return NewDirAdapter(testreader.ExampleDir.URL) }, + "ZipAdapter": func() Adapter { return NewZipAdapter(testreader.ExampleZip.URL) }, + "ZipAdapterNestedDir": func() Adapter { return NewZipAdapter(testreader.ExampleZipNestedDir.URL) }, + "ZipAdapterNestedTwoFeeds": func() Adapter { return NewZipAdapter(testreader.ExampleZipNestedTwoFeeds2.URL) }, + "ZipAdapterNestedZip": func() Adapter { return NewZipAdapter(testreader.ExampleZipNestedZip.URL) }, + "OverlayAdapter": func() Adapter { return NewOverlayAdapter(testreader.ExampleDir.URL) }, } return adapters } @@ -40,8 +40,8 @@ func TestDirAdapter(t *testing.T) { if err != nil { t.Error(err) } - if s != testutil.ExampleDir.DirSHA1 { - t.Errorf("got %s expect %s", s, testutil.ExampleDir.DirSHA1) + if s != testreader.ExampleDir.DirSHA1 { + t.Errorf("got %s expect %s", s, testreader.ExampleDir.DirSHA1) } }) } @@ -70,8 +70,8 @@ func TestZipAdapter(t *testing.T) { if err != nil { t.Error(err) } - if s != testutil.ExampleZip.SHA1 { - t.Errorf("got %s expect %s", s, testutil.ExampleZip.SHA1) + if s != testreader.ExampleZip.SHA1 { + t.Errorf("got %s expect %s", s, testreader.ExampleZip.SHA1) } }) t.Run("DirSHA1", func(t *testing.T) { @@ -84,8 +84,8 @@ func TestZipAdapter(t *testing.T) { if err != nil { t.Error(err) } - if s != testutil.ExampleZip.DirSHA1 { - t.Errorf("got %s expect %s", s, testutil.ExampleZip.DirSHA1) + if s != testreader.ExampleZip.DirSHA1 { + t.Errorf("got %s expect %s", s, testreader.ExampleZip.DirSHA1) } }) } @@ -169,7 +169,7 @@ func TestZipAdapterNestedZip(t *testing.T) { func TestURLAdapter(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - buf, err := os.ReadFile(testutil.ExampleZip.URL) + buf, err := os.ReadFile(testreader.ExampleZip.URL) if err != nil { t.Error(err) } diff --git a/tlcsv/geojson.go b/tlcsv/geojson.go new file mode 100644 index 000000000..2d43d6755 --- /dev/null +++ b/tlcsv/geojson.go @@ -0,0 +1,146 @@ +package tlcsv + +import ( + "encoding/json" + "io" + + "github.com/interline-io/transitland-lib/gtfs" + "github.com/interline-io/transitland-lib/tt" + geom "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/geojson" +) + +// GeoJSONFeatureParser is a callback function for parsing a GeoJSON feature +// into a GTFS entity. It receives the feature and should return the parsed +// entity and whether it was successfully parsed. +type GeoJSONFeatureParser[T any] func(*geojson.Feature) (T, bool) + +// readGeoJSON reads and parses a GeoJSON FeatureCollection file using a +// provided parser function. This provides a generic way to read any GeoJSON +// file format into GTFS entities. +func readGeoJSON[T any](reader *Reader, filename string, parser GeoJSONFeatureParser[T]) ([]T, error) { + var entities []T + var parseErr error + + err := reader.Adapter.OpenFile(filename, func(in io.Reader) { + var fc geojson.FeatureCollection + if err := json.NewDecoder(in).Decode(&fc); err != nil { + parseErr = err + return + } + + for _, feature := range fc.Features { + if entity, ok := parser(feature); ok { + entities = append(entities, entity) + } + } + }) + + if err != nil { + return nil, err + } + if parseErr != nil { + return nil, parseErr + } + + return entities, nil +} + +// parseLocationFeature parses a GeoJSON feature into a gtfs.Location. +// This is used for locations.geojson (GTFS-Flex extension). +func parseLocationFeature(feature *geojson.Feature) (gtfs.Location, bool) { + loc := gtfs.Location{} + + // The ID is at the feature level, not in properties + if feature.ID != "" { + loc.LocationID = tt.NewString(feature.ID) + } + + // Parse properties + if feature.Properties != nil { + if v, ok := feature.Properties["stop_name"].(string); ok { + loc.StopName = tt.NewString(v) + } + if v, ok := feature.Properties["stop_desc"].(string); ok { + loc.StopDesc = tt.NewString(v) + } + if v, ok := feature.Properties["zone_id"].(string); ok { + loc.ZoneID = tt.NewString(v) + } + if v, ok := feature.Properties["stop_url"].(string); ok { + loc.StopURL = tt.NewUrl(v) + } + } + + // Parse geometry - must be Polygon or MultiPolygon for locations + if feature.Geometry != nil { + switch g := feature.Geometry.(type) { + case *geom.Polygon: + g.SetSRID(4326) + loc.Geometry = tt.NewGeometry(g) + case *geom.MultiPolygon: + g.SetSRID(4326) + loc.Geometry = tt.NewGeometry(g) + default: + // Invalid geometry type for location - skip + return loc, false + } + } + + return loc, true +} + +// readLocationsGeoJSON reads and parses locations.geojson from the adapter. +// This is a GTFS-Flex extension that defines zones using GeoJSON Polygon +// or MultiPolygon geometries where riders can request pickups or drop-offs. +func (reader *Reader) readLocationsGeoJSON(filename string) ([]gtfs.Location, error) { + return readGeoJSON(reader, filename, parseLocationFeature) +} + +// Example: To add support for level.geojson in the future, create a parser function: +// +// func parseLevelFeature(feature *geojson.Feature) (gtfs.Level, bool) { +// level := gtfs.Level{} +// if feature.ID != "" { +// level.LevelID = tt.NewString(feature.ID) +// } +// if feature.Properties != nil { +// if v, ok := feature.Properties["level_name"].(string); ok { +// level.LevelName = tt.NewString(v) +// } +// if v, ok := feature.Properties["level_index"].(float64); ok { +// level.LevelIndex = tt.NewFloat(v) +// } +// } +// // Parse geometry (Polygon or MultiPolygon) +// if feature.Geometry != nil { +// switch g := feature.Geometry.(type) { +// case *geom.Polygon, *geom.MultiPolygon: +// g.SetSRID(4326) +// level.Geometry = tt.NewGeometry(g) +// default: +// return level, false +// } +// } +// return level, true +// } +// +// Then in reader.go, add: +// +// func (reader *Reader) Levels() chan gtfs.Level { +// out := make(chan gtfs.Level, bufferSize) +// go func() { +// defer close(out) +// // Try GeoJSON first +// levels, err := readGeoJSON(reader, "levels.geojson", parseLevelFeature) +// if err == nil { +// for _, level := range levels { +// out <- level +// } +// return +// } +// // Fall back to CSV +// ReadEntities[gtfs.Level](reader, "levels.txt") +// }() +// return out +// } diff --git a/tlcsv/geojson_test.go b/tlcsv/geojson_test.go new file mode 100644 index 000000000..fbc0212d6 --- /dev/null +++ b/tlcsv/geojson_test.go @@ -0,0 +1,208 @@ +package tlcsv + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReader_Locations_GeoJSON(t *testing.T) { + // Create a temporary GTFS feed with locations.geojson + tmpDir := t.TempDir() + + // Create minimal required GTFS files + agencyCSV := `agency_id,agency_name,agency_url,agency_timezone +1,Demo Transit,http://example.com,America/Los_Angeles` + + stopsCSV := `stop_id,stop_name,stop_lat,stop_lon +stop1,Stop 1,37.7749,-122.4194` + + routesCSV := `route_id,route_short_name,route_long_name,route_type,agency_id +route1,1,Main Street,3,1` + + // Create locations.geojson with GTFS-Flex zones + geojsonData := `{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "zone_downtown", + "properties": { + "stop_name": "Downtown Flex Zone", + "stop_desc": "On-demand service area in downtown", + "zone_id": "zone1", + "stop_url": "https://example.com/flex/downtown" + }, + "geometry": { + "type": "Polygon", + "coordinates": [[ + [-122.4194, 37.7749], + [-122.4094, 37.7749], + [-122.4094, 37.7649], + [-122.4194, 37.7649], + [-122.4194, 37.7749] + ]] + } + }, + { + "type": "Feature", + "id": "zone_midtown", + "properties": { + "stop_name": "Midtown Flex Zone" + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[ + [-122.5, 37.8], + [-122.4, 37.8], + [-122.4, 37.7], + [-122.5, 37.7], + [-122.5, 37.8] + ]], + [[ + [-122.3, 37.9], + [-122.2, 37.9], + [-122.2, 37.8], + [-122.3, 37.8], + [-122.3, 37.9] + ]] + ] + } + } + ] +}` + + // Write files + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "agency.txt"), []byte(agencyCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "stops.txt"), []byte(stopsCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "routes.txt"), []byte(routesCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "locations.geojson"), []byte(geojsonData), 0644)) + + // Create reader + reader, err := NewReader(tmpDir) + require.NoError(t, err) + require.NoError(t, reader.Open()) + defer reader.Close() + + // Read locations from GeoJSON + locations := reader.Locations() + + locMap := make(map[string]bool) + count := 0 + for loc := range locations { + count++ + locMap[loc.LocationID.Val] = true + + if loc.LocationID.Val == "zone_downtown" { + assert.Equal(t, "Downtown Flex Zone", loc.StopName.Val) + assert.Equal(t, "On-demand service area in downtown", loc.StopDesc.Val) + assert.Equal(t, "zone1", loc.ZoneID.Val) + assert.Equal(t, "https://example.com/flex/downtown", loc.StopURL.Val) + assert.True(t, loc.Geometry.Valid, "Geometry should be valid") + } + + if loc.LocationID.Val == "zone_midtown" { + assert.Equal(t, "Midtown Flex Zone", loc.StopName.Val) + assert.True(t, loc.Geometry.Valid, "MultiPolygon geometry should be valid") + } + } + + assert.Equal(t, 2, count) + assert.True(t, locMap["zone_downtown"]) + assert.True(t, locMap["zone_midtown"]) +} + +func TestReader_Locations_NoGeoJSON(t *testing.T) { + // Create a feed without locations.geojson + tmpDir := t.TempDir() + + agencyCSV := `agency_id,agency_name,agency_url,agency_timezone +1,Demo Transit,http://example.com,America/Los_Angeles` + + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "agency.txt"), []byte(agencyCSV), 0644)) + + reader, err := NewReader(tmpDir) + require.NoError(t, err) + require.NoError(t, reader.Open()) + defer reader.Close() + + // Should return empty channel without error + locations := reader.Locations() + + count := 0 + for range locations { + count++ + } + + assert.Equal(t, 0, count, "Should return no locations when file doesn't exist") +} + +func TestReader_Locations_AllFormats(t *testing.T) { + // Verify reader can handle regular stops AND GeoJSON locations together + tmpDir := t.TempDir() + + agencyCSV := `agency_id,agency_name,agency_url,agency_timezone +1,Demo Transit,http://example.com,America/Los_Angeles` + + stopsCSV := `stop_id,stop_name,stop_lat,stop_lon +stop1,Regular Stop,37.7749,-122.4194` + + routesCSV := `route_id,route_short_name,route_long_name,route_type,agency_id +route1,1,Route 1,3,1` + + geojsonData := `{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "flex1", + "properties": { + "stop_name": "Flex Zone" + }, + "geometry": { + "type": "Polygon", + "coordinates": [[ + [-122.4, 37.7], + [-122.3, 37.7], + [-122.3, 37.6], + [-122.4, 37.6], + [-122.4, 37.7] + ]] + } + } + ] +}` + + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "agency.txt"), []byte(agencyCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "stops.txt"), []byte(stopsCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "routes.txt"), []byte(routesCSV), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "locations.geojson"), []byte(geojsonData), 0644)) + + reader, err := NewReader(tmpDir) + require.NoError(t, err) + require.NoError(t, reader.Open()) + defer reader.Close() + + // Can read regular stops + stops := reader.Stops() + stopCount := 0 + for range stops { + stopCount++ + } + assert.Equal(t, 1, stopCount) + + // Can read flex locations + locations := reader.Locations() + locCount := 0 + for loc := range locations { + locCount++ + assert.Equal(t, "flex1", loc.LocationID.Val) + } + assert.Equal(t, 1, locCount) +} + + diff --git a/tlcsv/reader.go b/tlcsv/reader.go index 9078b5c93..ba26d3a09 100644 --- a/tlcsv/reader.go +++ b/tlcsv/reader.go @@ -426,6 +426,38 @@ func (reader *Reader) RouteNetworks() (out chan gtfs.RouteNetwork) { return ReadEntities[gtfs.RouteNetwork](reader, getFilename(>fs.RouteNetwork{})) } +func (reader *Reader) LocationGroups() (out chan gtfs.LocationGroup) { + return ReadEntities[gtfs.LocationGroup](reader, getFilename(>fs.LocationGroup{})) +} + +func (reader *Reader) LocationGroupStops() (out chan gtfs.LocationGroupStop) { + return ReadEntities[gtfs.LocationGroupStop](reader, getFilename(>fs.LocationGroupStop{})) +} + +func (reader *Reader) BookingRules() (out chan gtfs.BookingRule) { + return ReadEntities[gtfs.BookingRule](reader, getFilename(>fs.BookingRule{})) +} + +func (reader *Reader) Locations() (out chan gtfs.Location) { + // GTFS-Flex: locations.geojson uses GeoJSON format, not CSV + // Try to read locations.geojson first + out = make(chan gtfs.Location, bufferSize) + go func() { + defer close(out) + + locs, err := reader.readLocationsGeoJSON("locations.geojson") + if err != nil { + // File doesn't exist or error reading - just return empty + return + } + + for _, loc := range locs { + out <- loc + } + }() + return out +} + func ReadEntities[T any](reader *Reader, efn string) chan T { eout := make(chan T, bufferSize) go func(fn string, c chan T) { diff --git a/tlcsv/reader_benchmark_test.go b/tlcsv/reader_benchmark_test.go index 1b235d8bd..c961a8ddd 100644 --- a/tlcsv/reader_benchmark_test.go +++ b/tlcsv/reader_benchmark_test.go @@ -3,12 +3,12 @@ package tlcsv import ( "testing" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" ) func BenchmarkReader(b *testing.B) { b.SetParallelism(1) - for k, fe := range testutil.ExternalTestFeeds { + for k, fe := range testreader.ExternalTestFeeds { b.Run(k, func(b *testing.B) { for n := 0; n < b.N; n++ { reader, err := NewReader(fe.URL) @@ -18,7 +18,7 @@ func BenchmarkReader(b *testing.B) { if err := reader.Open(); err != nil { b.Error(err) } - testutil.CheckReader(b, fe, reader) + testreader.CheckReader(b, fe, reader) if err := reader.Close(); err != nil { b.Error(err) } diff --git a/tlcsv/reader_test.go b/tlcsv/reader_test.go index 146bfd2eb..c533e50d6 100644 --- a/tlcsv/reader_test.go +++ b/tlcsv/reader_test.go @@ -7,13 +7,13 @@ import ( "testing" "github.com/interline-io/transitland-lib/adapters" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" ) func TestReader(t *testing.T) { // Start local HTTP server ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - buf, err := os.ReadFile(testutil.ExampleZip.URL) + buf, err := os.ReadFile(testreader.ExampleZip.URL) if err != nil { t.Error(err) } @@ -25,7 +25,7 @@ func TestReader(t *testing.T) { tsa["URL"] = func() Adapter { return &URLAdapter{url: ts.URL} } for k, v := range tsa { t.Run(k, func(t *testing.T) { - testutil.TestReader(t, testutil.ExampleDir, func() adapters.Reader { + testreader.TestReader(t, testreader.ExampleDir, func() adapters.Reader { return &Reader{Adapter: v()} }) }) diff --git a/tlcsv/reflect_benchmark_test.go b/tlcsv/reflect_benchmark_test.go index cd735dc44..d3cba0ad4 100644 --- a/tlcsv/reflect_benchmark_test.go +++ b/tlcsv/reflect_benchmark_test.go @@ -110,7 +110,7 @@ func Benchmark_loadRow_Trip(b *testing.B) { func Benchmark_dumpRow_StopTime(b *testing.B) { ent := gtfs.StopTime{ TripID: tt.NewString("xyz"), - StopID: tt.NewString("abc"), + StopID: tt.NewKey("abc"), StopHeadsign: tt.NewString("hello"), StopSequence: tt.NewInt(123), ArrivalTime: tt.NewSeconds(3600), diff --git a/tlcsv/reflect_test.go b/tlcsv/reflect_test.go index 87b992c17..b310e11ea 100644 --- a/tlcsv/reflect_test.go +++ b/tlcsv/reflect_test.go @@ -11,7 +11,7 @@ import ( func TestGetString(t *testing.T) { ent := gtfs.StopTime{ TripID: tt.NewString("123"), - StopID: tt.NewString("456"), + StopID: tt.NewKey("456"), ArrivalTime: tt.NewSeconds(3600), DepartureTime: tt.NewSeconds(7200), ShapeDistTraveled: tt.NewFloat(123.456), diff --git a/tlcsv/writer_benchmark_test.go b/tlcsv/writer_benchmark_test.go index 248a75b49..fcee6cd41 100644 --- a/tlcsv/writer_benchmark_test.go +++ b/tlcsv/writer_benchmark_test.go @@ -5,12 +5,12 @@ import ( "testing" "github.com/interline-io/transitland-lib/adapters" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" ) func BenchmarkWriter(b *testing.B) { b.SetParallelism(1) - for k, fe := range testutil.ExternalTestFeeds { + for k, fe := range testreader.ExternalTestFeeds { b.Run(k, func(b *testing.B) { for i := 0; i < b.N; i++ { tmpdir, err := os.MkdirTemp("", "gtfs") @@ -22,7 +22,7 @@ func BenchmarkWriter(b *testing.B) { if err != nil { b.Error(err) } - testutil.TestWriter(b, fe, func() adapters.Reader { + testreader.TestWriter(b, fe, func() adapters.Reader { a, err := NewReader(fe.URL) if err != nil { b.Error(err) diff --git a/tlcsv/writer_test.go b/tlcsv/writer_test.go index 0f64dcc0f..3886cd1bb 100644 --- a/tlcsv/writer_test.go +++ b/tlcsv/writer_test.go @@ -6,13 +6,13 @@ import ( "github.com/interline-io/transitland-lib/adapters" "github.com/interline-io/transitland-lib/gtfs" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/stretchr/testify/assert" ) // Round trip Writer test. func TestWriter(t *testing.T) { - fe, reader := testutil.NewMinimalTestFeed() + fe, reader := testreader.NewMinimalTestFeed() tmpdir, err := os.MkdirTemp("", "gtfs") if err != nil { t.Error(err) @@ -21,7 +21,7 @@ func TestWriter(t *testing.T) { if err != nil { t.Error(err) } - testutil.TestWriter(t, *fe, func() adapters.Reader { return reader }, func() adapters.Writer { return writer }) + testreader.TestWriter(t, *fe, func() adapters.Reader { return reader }, func() adapters.Writer { return writer }) // Clean up and double check if err := os.RemoveAll(tmpdir); err != nil { t.Error(err) diff --git a/tldb/reader.go b/tldb/reader.go index bd99ebcec..3a2228621 100644 --- a/tldb/reader.go +++ b/tldb/reader.go @@ -313,6 +313,22 @@ func (reader *Reader) RouteNetworks() (out chan gtfs.RouteNetwork) { return ReadEntities[gtfs.RouteNetwork](reader, GetTableName(>fs.RouteNetwork{})) } +func (reader *Reader) LocationGroups() (out chan gtfs.LocationGroup) { + return ReadEntities[gtfs.LocationGroup](reader, GetTableName(>fs.LocationGroup{})) +} + +func (reader *Reader) LocationGroupStops() (out chan gtfs.LocationGroupStop) { + return ReadEntities[gtfs.LocationGroupStop](reader, GetTableName(>fs.LocationGroupStop{})) +} + +func (reader *Reader) BookingRules() (out chan gtfs.BookingRule) { + return ReadEntities[gtfs.BookingRule](reader, GetTableName(>fs.BookingRule{})) +} + +func (reader *Reader) Locations() (out chan gtfs.Location) { + return ReadEntities[gtfs.Location](reader, GetTableName(>fs.Location{})) +} + func ReadEntities[T tt.EntityWithID](reader *Reader, table string) chan T { ctx := context.TODO() out := make(chan T, bufferSize) diff --git a/tldb/tldbtest/adapter_benchmark.go b/tldb/tldbtest/adapter_benchmark.go index 729bff756..c0e8c2fb9 100644 --- a/tldb/tldbtest/adapter_benchmark.go +++ b/tldb/tldbtest/adapter_benchmark.go @@ -9,7 +9,7 @@ import ( "github.com/interline-io/transitland-lib/dmfr" "github.com/interline-io/transitland-lib/gtfs" - "github.com/interline-io/transitland-lib/internal/testutil" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/tldb" ) @@ -98,11 +98,11 @@ func Benchmark_Adapter_MultiInsert(b *testing.B) { } // Load the minimal test feed... writer := tldb.Writer{Adapter: adapter} - _, reader := testutil.NewMinimalTestFeed() + _, reader := testreader.NewMinimalTestFeed() if err := reader.Open(); err != nil { b.Error(err) } - if err := testutil.DirectCopy(reader, &writer); err != nil { + if err := testreader.DirectCopy(reader, &writer); err != nil { b.Error(err) } // get ids diff --git a/tt/key.go b/tt/key.go index b0d0e74e2..1d0fdbb2f 100644 --- a/tt/key.go +++ b/tt/key.go @@ -7,6 +7,11 @@ type Key struct { Option[string] } +func (r *Key) Set(v string) { + r.Val = v + r.Valid = r.Val != "" +} + func (r *Key) SetInt(v int) { r.Val = strconv.Itoa(v) r.Valid = true diff --git a/tt/string.go b/tt/string.go index 92b62d339..29628a6c4 100644 --- a/tt/string.go +++ b/tt/string.go @@ -10,6 +10,12 @@ func NewString(v string) String { return String{Option: NewOption(v)} } +// TODO: Consider restricting valid to non-empty strings +// func (r *String) Set(v string) { +// r.Val = v +// r.Valid = r.Val != "" +// } + func (r String) String() string { return r.Val } diff --git a/validator/validator.go b/validator/validator.go index c0f193649..ff5a7e5c6 100644 --- a/validator/validator.go +++ b/validator/validator.go @@ -379,6 +379,8 @@ func (v *Validator) copierOptions() copier.Options { cpOpts.AddExtensionWithLevel(&rules.ShapeMaxSegmentLengthCheck{ MaxAllowedDistance: 1_000_000, // 1000 km }, 1) + // GTFS-Flex best practice: location groups should have stops + cpOpts.AddExtensionWithLevel(&rules.FlexLocationGroupEmptyCheck{}, 1) } return cpOpts } diff --git a/validator/validator_test.go b/validator/validator_test.go index 5fb734254..99bf84ed9 100644 --- a/validator/validator_test.go +++ b/validator/validator_test.go @@ -12,6 +12,7 @@ import ( "github.com/interline-io/transitland-lib/internal/testdb" "github.com/interline-io/transitland-lib/internal/testpath" + "github.com/interline-io/transitland-lib/internal/testreader" "github.com/interline-io/transitland-lib/internal/testutil" "github.com/interline-io/transitland-lib/tlcsv" "github.com/interline-io/transitland-lib/tldb" @@ -67,7 +68,7 @@ func TestEntityErrors(t *testing.T) { if err := reader.Open(); err != nil { t.Error(err) } - testutil.AllEntities(reader, func(ent tt.Entity) { + testreader.AllEntities(reader, func(ent tt.Entity) { t.Run(fmt.Sprintf("%s:%s", ent.Filename(), ent.EntityID()), func(t *testing.T) { errs := tt.CheckErrors(ent) expecterrs := testutil.GetExpectErrors(ent)