diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f446ee2e9..0baf70c55c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - /api/1.2/current_stats - /api/1.1/osversions +- Traffic Ops API Routing Blacklist: via the `routing_blacklist` field in `cdn.conf`, enable certain whitelisted Go routes to be handled by Perl instead (via the `perl_routes` list) in case a regression is found in the Go handler, and explicitly disable any routes via the `disabled_routes` list. Requests to disabled routes are immediately given a 503 response. Both fields are lists of Route IDs, and route information (ID, version, method, path, and whether or not it can bypass to Perl) can be found by running `./traffic_ops_golang --api-routes`. To disable a route or have it bypassed to Perl, find its Route ID using the previous command and put it in the `disabled_routes` or `perl_routes` list, respectively. - To support reusing a single riak cluster connection, an optional parameter is added to riak.conf: "HealthCheckInterval". This options takes a 'Duration' value (ie: 10s, 5m) which affects how often the riak cluster is health checked. Default is currently set to: "HealthCheckInterval": "5s". - Added a new Go db/admin binary to replace the Perl db/admin.pl script which is now deprecated and will be removed in a future release. The new db/admin binary is essentially a drop-in replacement for db/admin.pl since it supports all of the same commands and options; therefore, it should be used in place of db/admin.pl for all the same tasks. - Added an API 1.4 endpoint, /api/1.4/cdns/dnsseckeys/refresh, to perform necessary behavior previously served outside the API under `/internal`. diff --git a/docs/source/admin/traffic_ops.rst b/docs/source/admin/traffic_ops.rst index 895522b6a7..608e1f3098 100644 --- a/docs/source/admin/traffic_ops.rst +++ b/docs/source/admin/traffic_ops.rst @@ -262,7 +262,7 @@ The script takes no options other than the ones accepted by `Hypnotoad 0 { + msg := "unknown route IDs in routing_blacklist: " + strings.Join(unknownRouteIDs, ", ") + if d.IgnoreUnknownRoutes { + log.Warnln(msg) + } else { + return nil, nil, nil, errors.New(msg) + } } // rawRoutes are served at the root path. These should be almost exclusively old Perl pre-API routes, which have yet to be converted in all clients. New routes should be in the versioned API path. diff --git a/traffic_ops/traffic_ops_golang/routing/routing.go b/traffic_ops/traffic_ops_golang/routing/routing.go index c2efd22c8a..6cccf0fe41 100644 --- a/traffic_ops/traffic_ops_golang/routing/routing.go +++ b/traffic_ops/traffic_ops_golang/routing/routing.go @@ -21,6 +21,7 @@ package routing import ( "context" + "fmt" "net/http" "regexp" "sort" @@ -53,6 +54,12 @@ type Route struct { RequiredPrivLevel int Authenticated bool Middlewares []Middleware + ID int // unique ID for referencing this Route + CanBypassToPerl bool // if true, this Route can be passed through to Perl +} + +func (r Route) String() string { + return fmt.Sprintf("id=%d method=%s version=%.1f path=%s can_bypass_to_perl=%t", r.ID, r.Method, r.Version, r.Path, r.CanBypassToPerl) } // RawRoute is an HTTP route to be served at the root, rather than under /api/version. Raw Routes should be rare, and almost exclusively converted old Perl routes which have yet to be moved to an API path. @@ -106,17 +113,21 @@ type PathHandler struct { // CreateRouteMap returns a map of methods to a slice of paths and handlers; wrapping the handlers in the appropriate middleware. Uses Semantic Versioning: routes are added to every subsequent minor version, but not subsequent major versions. For example, a 1.2 route is added to 1.3 but not 2.1. Also truncates '2.0' to '2', creating succinct major versions. // Returns the map of routes, and a map of API versions served. -func CreateRouteMap(rs []Route, rawRoutes []RawRoute, authBase AuthBase, reqTimeOutSeconds int) (map[string][]PathHandler, map[float64]struct{}) { +func CreateRouteMap(rs []Route, rawRoutes []RawRoute, perlRouteIDs, disabledRouteIDs []int, perlHandler http.HandlerFunc, authBase AuthBase, reqTimeOutSeconds int) (map[string][]PathHandler, map[float64]struct{}) { // TODO strong types for method, path versions := getSortedRouteVersions(rs) requestTimeout := time.Second * time.Duration(60) if reqTimeOutSeconds > 0 { requestTimeout = time.Second * time.Duration(reqTimeOutSeconds) } + perlRoutes := GetRouteIDMap(perlRouteIDs) + disabledRoutes := GetRouteIDMap(disabledRouteIDs) m := map[string][]PathHandler{} for _, r := range rs { versionI := sort.SearchFloat64s(versions, r.Version) nextMajorVer := float64(int(r.Version) + 1) + _, isPerlRoute := perlRoutes[r.ID] + _, isDisabledRoute := disabledRoutes[r.ID] for _, version := range versions[versionI:] { if version >= nextMajorVer { break @@ -124,7 +135,14 @@ func CreateRouteMap(rs []Route, rawRoutes []RawRoute, authBase AuthBase, reqTime vstr := strconv.FormatFloat(version, 'f', -1, 64) path := RoutePrefix + "/" + vstr + "/" + r.Path middlewares := getRouteMiddleware(r.Middlewares, authBase, r.Authenticated, r.RequiredPrivLevel, requestTimeout) - m[r.Method] = append(m[r.Method], PathHandler{Path: path, Handler: use(r.Handler, middlewares)}) + + if isPerlRoute { + m[r.Method] = append(m[r.Method], PathHandler{Path: path, Handler: perlHandler}) + } else if isDisabledRoute { + m[r.Method] = append(m[r.Method], PathHandler{Path: path, Handler: wrapAccessLog(authBase.secret, DisabledRouteHandler())}) + } else { + m[r.Method] = append(m[r.Method], PathHandler{Path: path, Handler: use(r.Handler, middlewares)}) + } log.Infof("adding route %v %v\n", r.Method, path) } } @@ -276,7 +294,7 @@ func RegisterRoutes(d ServerData) error { } authBase := AuthBase{secret: d.Config.Secrets[0], override: nil} //we know d.Config.Secrets is a slice of at least one or start up would fail. - routes, versions := CreateRouteMap(routeSlice, rawRoutes, authBase, d.RequestTimeout) + routes, versions := CreateRouteMap(routeSlice, rawRoutes, d.PerlRoutes, d.DisabledRoutes, handlerToFunc(catchall), authBase, d.RequestTimeout) compiledRoutes := CompileRoutes(routes) getReqID := nextReqIDGetter() http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { diff --git a/traffic_ops/traffic_ops_golang/routing/routing_test.go b/traffic_ops/traffic_ops_golang/routing/routing_test.go index e5f1298607..c854d9c01b 100644 --- a/traffic_ops/traffic_ops_golang/routing/routing_test.go +++ b/traffic_ops/traffic_ops_golang/routing/routing_test.go @@ -135,7 +135,7 @@ func TestCompileRoutes(t *testing.T) { } authBase := AuthBase{secret: d.Secrets[0], override: nil} - routes, versions := CreateRouteMap(routeSlice, nil, authBase, 1) + routes, versions := CreateRouteMap(routeSlice, nil, nil, nil, nil, authBase, 1) if len(routes) == 0 { t.Error("no routes handler defined") } @@ -172,6 +172,15 @@ func TestCompileRoutes(t *testing.T) { } } +func TestRoutes(t *testing.T) { + fake := ServerData{Config: config.NewFakeConfig()} + _, _, _, err := Routes(fake) + if err != nil { + t.Fatalf("expected: no error getting Routes, actual: %v", err) + } + // TODO: verify that all returned Routes are unique +} + func TestCreateRouteMap(t *testing.T) { authBase := AuthBase{"secret", func(handlerFunc http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -180,6 +189,10 @@ func TestCreateRouteMap(t *testing.T) { } }} + CatchallHandler := func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "catchall") + } + PathOneHandler := func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() authWasCalled := getAuthWasCalled(ctx) @@ -197,15 +210,27 @@ func TestCreateRouteMap(t *testing.T) { authWasCalled := getAuthWasCalled(ctx) fmt.Fprintf(w, "%s %s", "path3", authWasCalled) } + PathFourHandler := func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "path4") + } + + PathFiveHandler := func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "path5") + } routes := []Route{ - {1.2, http.MethodGet, `path1`, PathOneHandler, auth.PrivLevelReadOnly, true, nil}, - {1.2, http.MethodGet, `path2`, PathTwoHandler, 0, false, nil}, - {1.2, http.MethodGet, `path3`, PathThreeHandler, 0, false, []Middleware{}}, + {1.2, http.MethodGet, `path1`, PathOneHandler, auth.PrivLevelReadOnly, true, nil, 0, false}, + {1.2, http.MethodGet, `path2`, PathTwoHandler, 0, false, nil, 1, false}, + {1.2, http.MethodGet, `path3`, PathThreeHandler, 0, false, []Middleware{}, 2, false}, + {1.2, http.MethodGet, `path4`, PathFourHandler, 0, false, []Middleware{}, 3, true}, + {1.2, http.MethodGet, `path5`, PathFiveHandler, 0, false, []Middleware{}, 4, false}, } + perlRoutesIDs := []int{3} + disabledRoutesIDs := []int{4} + rawRoutes := []RawRoute{} - routeMap, _ := CreateRouteMap(routes, rawRoutes, authBase, 60) + routeMap, _ := CreateRouteMap(routes, rawRoutes, perlRoutesIDs, disabledRoutesIDs, CatchallHandler, authBase, 60) route1Handler := routeMap["GET"][0].Handler @@ -244,6 +269,25 @@ func TestCreateRouteMap(t *testing.T) { if v, ok := w.HeaderMap["Access-Control-Allow-Credentials"]; ok { t.Errorf("Unexpected Access-Control-Allow-Credentials: %s", v) } + + // request should be handled by Catchall + route4Handler := routeMap["GET"][3].Handler + w = httptest.NewRecorder() + route4Handler(w, r) + if bytes.Compare(w.Body.Bytes(), []byte("catchall")) != 0 { + t.Errorf("Expected: 'catchall', actual: %s", w.Body.Bytes()) + } + + // request should be handled by DisabledRouteHandler + route5Handler := routeMap["GET"][4].Handler + w = httptest.NewRecorder() + route5Handler(w, r) + if bytes.Compare(w.Body.Bytes(), []byte("path5")) == 0 { + t.Errorf("Expected: not 'path5', actual: '%s'", w.Body.Bytes()) + } + if w.Result().StatusCode != http.StatusServiceUnavailable { + t.Errorf("Expected status: %d, actual: %d", http.StatusServiceUnavailable, w.Result().StatusCode) + } } func getAuthWasCalled(ctx context.Context) string { diff --git a/traffic_ops/traffic_ops_golang/routing/wrappers.go b/traffic_ops/traffic_ops_golang/routing/wrappers.go index 93e8f861de..d66237b389 100644 --- a/traffic_ops/traffic_ops_golang/routing/wrappers.go +++ b/traffic_ops/traffic_ops_golang/routing/wrappers.go @@ -285,3 +285,11 @@ func NotImplementedHandler() http.Handler { w.Write([]byte(`{"alerts":[{"level":"error","text":"The requested api version is not implemented by this server. If you are using a newer client with an older server, you will need to use an older client version or upgrade your server."}]}`)) }) } + +func DisabledRouteHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(tc.ContentType, tc.ApplicationJson) + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"alerts":[{"level":"error","text":"The requested route is currently disabled."}]}` + "\n")) + }) +} diff --git a/traffic_ops/traffic_ops_golang/traffic_ops_golang.go b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go index dff13569c4..635bad9c19 100644 --- a/traffic_ops/traffic_ops_golang/traffic_ops_golang.go +++ b/traffic_ops/traffic_ops_golang/traffic_ops_golang.go @@ -55,6 +55,7 @@ func init() { func main() { showVersion := flag.Bool("version", false, "Show version and exit") showPlugins := flag.Bool("plugins", false, "Show the list of plugins and exit") + showRoutes := flag.Bool("api-routes", false, "Show the list of API routes and exit") configFileName := flag.String("cfg", "", "The config file path") dbConfigFileName := flag.String("dbcfg", "", "The db config file path") riakConfigFileName := flag.String("riakcfg", "", "The riak config file path") @@ -68,6 +69,29 @@ func main() { fmt.Println(strings.Join(plugin.List(), "\n")) os.Exit(0) } + if *showRoutes { + fake := routing.ServerData{Config: config.NewFakeConfig()} + routes, _, _, _ := routing.Routes(fake) + if len(*configFileName) != 0 { + cfg, err := config.LoadCdnConfig(*configFileName) + if err != nil { + fmt.Printf("Loading cdn config from '%s': %v", *configFileName, err) + os.Exit(1) + } + perlRoutes := routing.GetRouteIDMap(cfg.PerlRoutes) + disabledRoutes := routing.GetRouteIDMap(cfg.DisabledRoutes) + for _, r := range routes { + _, isBypassedToPerl := perlRoutes[r.ID] + _, isDisabled := disabledRoutes[r.ID] + fmt.Printf("id=%d\tmethod=%s\tversion=%.1f\tpath=%s\tcan_bypass_to_perl=%t\tis_bypassed_to_perl=%t\tis_disabled=%t\n", r.ID, r.Method, r.Version, r.Path, r.CanBypassToPerl, isBypassedToPerl, isDisabled) + } + } else { + for _, r := range routes { + fmt.Printf("id=%d\tmethod=%s\tversion=%.1f\tpath=%s\tcan_bypass_to_perl=%t\n", r.ID, r.Method, r.Version, r.Path, r.CanBypassToPerl) + } + } + os.Exit(0) + } if len(os.Args) < 2 { flag.Usage() os.Exit(1)