transitland inspect command for quick overview of GTFS and GTFS Realtime feeds

cmd/transitland/main.go

@@ -13,6 +13,7 @@ import (
 	_ "github.com/interline-io/transitland-lib/ext/plus"
 	"github.com/interline-io/transitland-lib/extract"
 	_ "github.com/interline-io/transitland-lib/filters"
+	"github.com/interline-io/transitland-lib/inspect"
 	"github.com/interline-io/transitland-lib/log"
 	"github.com/interline-io/transitland-lib/merge"
 	"github.com/interline-io/transitland-lib/tl"
@@ -48,6 +49,7 @@ func main() {
 		log.Print("  unimport")
 		log.Print("  sync")
 		log.Print("  dmfr")
+		log.Print("  inspect")
 	}
 	flag.Parse()
 	if versionFlag {
@@ -92,6 +94,8 @@ func main() {
 		r = &sync.Command{}
 	case "merge":
 		r = &merge.Command{}
+	case "inspect":
+		r = &inspect.Command{}
 	case "dmfr": // backwards compat
 		r = &dmfrCommand{}
 	default:

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/iancoleman/orderedmap v0.2.0
+	github.com/jedib0t/go-pretty/v6 v6.4.6
 	github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.6
@@ -21,7 +22,7 @@ require (
 	github.com/rs/zerolog v1.26.1
 	github.com/sergi/go-diff v1.2.0
 	github.com/snabb/isoweek v1.0.1
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.7.4
 	github.com/twpayne/go-geom v1.4.1
 	google.golang.org/protobuf v1.28.0
 )
@@ -50,8 +51,10 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect

go.sum

@@ -86,6 +86,8 @@ github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
 github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA=
 github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw=
+github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
 github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af h1:sh8vAWJ+vr9izhkDAMS3JRGDIjj0tNVwxfwd+2U2xMo=
 github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -110,6 +112,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
 github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
 github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -126,8 +130,11 @@ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXc
 github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
 github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
@@ -140,11 +147,13 @@ github.com/snabb/isoweek v1.0.1/go.mod h1:CAijAxH7NMgjqGc9baHMDE4sTHMt4B/f6X/XLi
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.4 h1:wZRexSlwd7ZXfKINDLsO4r7WBt3gTKONc6K/VesHvHM=
+github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/twpayne/go-geom v1.4.1 h1:LeivFqaGBRfyg0XJJ9pkudcptwhSSrYN9KZUW6HcgdA=
 github.com/twpayne/go-geom v1.4.1/go.mod h1:k/zktXdL+qnA6OgKsdEGUTA17jbQ2ZPTUa3CCySuGpE=
 github.com/twpayne/go-kml v1.5.2/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4=
@@ -178,6 +187,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

inspect/inspect_cmd.go

@@ -0,0 +1,143 @@
+package inspect
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/jedib0t/go-pretty/v6/text"
+
+	"github.com/interline-io/transitland-lib/ext"
+	"github.com/interline-io/transitland-lib/internal/cli"
+	"github.com/interline-io/transitland-lib/log"
+	"github.com/interline-io/transitland-lib/rt/pb"
+	"github.com/interline-io/transitland-lib/validator"
+	"google.golang.org/protobuf/proto"
+)
+
+// Command
+type Command struct {
+	extensions cli.ArrayFlags
+	spec       string
+	readerPath string
+}
+
+func (cmd *Command) Parse(args []string) error {
+	fl := flag.NewFlagSet("inspect", flag.ExitOnError)
+	fl.Usage = func() {
+		log.Print("Usage: inspect <spec> <file path or URL>")
+		fl.PrintDefaults()
+	}
+	fl.Var(&cmd.extensions, "ext", "Include GTFS Extension")
+
+	fl.Parse(args)
+	if fl.NArg() < 2 {
+		fl.Usage()
+		return errors.New("requires spec and a file path or URL")
+	}
+	cmd.spec = fl.Arg(0)
+	cmd.readerPath = fl.Arg(1)
+	return nil
+}
+
+func (cmd *Command) Run() error {
+	if cmd.spec == "gtfs" || cmd.spec == "GTFS" {
+		reader, err := ext.OpenReader(cmd.readerPath)
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+
+		var options validator.Options
+		options.BestPractices = true
+		options.IncludeEntities = true
+		options.IncludeRouteGeometries = true
+
+		v, _ := validator.NewValidator(reader, options)
+
+		result, _ := v.Validate()
+
+		entityCountTable := table.NewWriter()
+		entityCountTable.SetOutputMirror(os.Stdout)
+		entityCountTable.AppendHeader(table.Row{"GTFS File", "Entity Count"})
+		for k, v := range result.EntityCount {
+			entityCountTable.AppendRow(table.Row{k, v})
+		}
+		entityCountTable.SortBy([]table.SortBy{
+			{Name: "GTFS File", Mode: table.Asc},
+		})
+		entityCountTable.Render()
+
+		errorCountTable := table.NewWriter()
+		errorCountTable.SetOutputMirror(os.Stdout)
+		errorCountTable.AppendHeader(table.Row{"Entity Issue", "Severity", "Count"})
+		for k, v := range result.Errors {
+			errorCountTable.AppendRow(table.Row{k, "Error", v.Count})
+		}
+		for k, v := range result.Warnings {
+			errorCountTable.AppendRow(table.Row{k, "Warning", v.Count})
+		}
+		errorCountTable.Render()
+
+		feedInfoTable := table.NewWriter()
+		feedInfoTable.SetOutputMirror(os.Stdout)
+		feedInfoTable.AppendHeader(table.Row{
+			"FeedPublisherName",
+			"FeedPublisherURL",
+			"FeedLang",
+			"FeedVersion",
+			"FeedStartDate",
+			"FeedEndDate",
+			"DefaultLang",
+			"FeedContactEmail",
+			"FeedContactURL"})
+		for _, v := range result.FeedInfos {
+			feedInfoTable.AppendRow(table.Row{
+				v.FeedPublisherName,
+				v.FeedPublisherURL,
+				v.FeedLang,
+				v.FeedVersion,
+				v.FeedStartDate,
+				v.FeedEndDate,
+				v.DefaultLang,
+				v.FeedContactEmail,
+				v.FeedContactURL,
+			})
+		}
+		feedInfoTable.Render()
+
+		agencyTable := table.NewWriter()
+		agencyTable.SetOutputMirror(os.Stdout)
+		agencyTable.AppendHeader(table.Row{"AgencyID", "AgencyName"})
+		for _, v := range result.Agencies {
+			agencyTable.AppendRow(table.Row{v.AgencyID, v.AgencyName})
+		}
+		agencyTable.Render()
+	} else if cmd.spec == "gtfs-rt" || cmd.spec == "gtfs-realtime" || cmd.spec == "rt" {
+    client := &http.Client{}
+    req, err := http.NewRequest("GET", cmd.readerPath, nil)
+    resp, err := client.Do(req)
+    defer resp.Body.Close()
+    if err != nil {
+        log.Errorf(err.Error())
+    }
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+			log.Errorf(err.Error())
+    }
+		feedMessage := pb.FeedMessage{}
+    err = proto.Unmarshal(body, &feedMessage)
+    if err != nil {
+			log.Errorf(err.Error())
+    }
+
+		var jsonTransformer = text.NewJSONTransformer("", "  ")
+		fmt.Print(jsonTransformer(feedMessage))
+	}
+
+	return nil
+}

rules/route_names_prefix.go

@@ -1,9 +1,9 @@
 package rules
 
 import (
-	"fmt"
 	"strings"
 
+	"github.com/interline-io/transitland-lib/log"
 	"github.com/interline-io/transitland-lib/tl"
 )
 
@@ -16,9 +16,9 @@ type RouteNamesPrefixCheck struct {
 
 func (e *RouteNamesPrefixCheck) Validate(ent tl.Entity) []error {
 	if v, ok := ent.(*tl.Route); ok {
-		fmt.Println("checking:", v.RouteShortName, ":", v.RouteLongName)
+		log.Infof("checking:", v.RouteShortName, ":", v.RouteLongName)
 		if v.RouteShortName != "" && v.RouteLongName != "" && strings.HasPrefix(v.RouteLongName, v.RouteShortName) {
-			fmt.Println("prefixed")
+			log.Infof("prefixed")
 			return []error{&RouteNamesPrefixError{}}
 		}
 	}