Backport 4.0.x: Fix atscfg meta to omit config files for nonexistent delivery services ( #4256)

CHANGELOG.md

@@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Traffic Ops now supports a "sortOrder" query parameter on some endpoints to return API responses in descending order
 - Traffic Ops now uses a consistent format for audit logs across all Go endpoints
 - Added cache-side config generator, atstccfg, installed with ORT. Includes all configs. Includes a plugin system.
+- Fixed ATS config generation to omit regex remap, header rewrite, URL Sig, and URI Signing files for delivery services not assigned to that server.
 - In Traffic Portal, all tables now include a 'CSV' link to enable the export of table data in CSV format.
 - Pylint configuration now enforced (present in [a file in the Python client directory](./traffic_control/clients/python/pylint.rc))
 - Added an optional SMTP server configuration to the TO configuration file, api now has unused abilitiy to send emails

lib/go-atscfg/meta.go

@@ -48,6 +48,7 @@ func MakeMetaConfig(
 	locationParams map[string]ConfigProfileParams, // map[configFile]params; 'location' and 'URL' Parameters on serverHostName's Profile
 	uriSignedDSes []tc.DeliveryServiceName,
 	scopeParams map[string]string, // map[configFileName]scopeParam
+	dsNames map[tc.DeliveryServiceName]struct{},
 ) string {
 	if tmURL == "" {
 		log.Errorln("ats.GetConfigMetadata: global tm.url parameter missing or empty! Setting empty in meta config!")
@@ -83,7 +84,26 @@ func MakeMetaConfig(
 		}
 	}
 
+locationParamsFor:
 	for cfgFile, cfgParams := range locationParams {
+		if strings.HasSuffix(cfgFile, ".config") {
+			dsConfigFilePrefixes := []string{
+				"hdr_rw_",
+				"regex_remap_",
+				"url_sig_",
+				"uri_signing_",
+			}
+			for _, prefix := range dsConfigFilePrefixes {
+				if strings.HasPrefix(cfgFile, prefix) {
+					dsName := strings.TrimSuffix(strings.TrimPrefix(cfgFile, prefix), ".config")
+					if _, ok := dsNames[tc.DeliveryServiceName(dsName)]; !ok {
+						log.Warnln("Server Profile had 'location' Parameter '" + cfgFile + "', but delivery Service '" + dsName + "' is not assigned to this Server! Not including in meta config!")
+						continue locationParamsFor
+					}
+				}
+			}
+		}
+
 		atsCfg := tc.ATSConfigMetaDataConfigFile{
 			FileNameOnDisk: cfgParams.FileNameOnDisk,
 			Location:       cfgParams.Location,

lib/go-atscfg/meta_test.go

@@ -21,6 +21,7 @@ package atscfg
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/apache/trafficcontrol/lib/go-tc"
@@ -80,6 +81,26 @@ func TestMakeMetaConfig(t *testing.T) {
 			FileNameOnDisk: "uri_signing_mydsname.config",
 			Location:       "/my/location/",
 		},
+		"uri_signing_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "uri_signing_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"regex_remap_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "regex_remap_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"url_sig_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "url_sig_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"hdr_rw_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "hdr_rw_nonexistentds.config",
+			Location:       "/my/location/",
+		},
+		"hdr_rw_mid_nonexistentds.config": ConfigProfileParams{
+			FileNameOnDisk: "hdr_rw_mid_nonexistentds.config",
+			Location:       "/my/location/",
+		},
 		"unknown.config": ConfigProfileParams{
 			FileNameOnDisk: "unknown.config",
 			Location:       "/my/location/",
@@ -90,10 +111,11 @@ func TestMakeMetaConfig(t *testing.T) {
 		},
 	}
 	uriSignedDSes := []tc.DeliveryServiceName{"mydsname"}
+	dses := map[tc.DeliveryServiceName]struct{}{"mydsname": {}}
 
 	scopeParams := map[string]string{"custom.config": string(tc.ATSConfigMetaDataConfigFileScopeProfiles)}
 
-	txt := MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams)
+	txt := MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses)
 
 	cfg := tc.ATSConfigMetaData{}
 	if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
@@ -223,7 +245,7 @@ func TestMakeMetaConfig(t *testing.T) {
 	}
 
 	server.Type = "MID"
-	txt = MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams)
+	txt = MakeMetaConfig(serverHostName, server, tmURL, tmReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dses)
 	cfg = tc.ATSConfigMetaData{}
 	if err := json.Unmarshal([]byte(txt), &cfg); err != nil {
 		t.Fatalf("MakeMetaConfig returned invalid JSON: " + err.Error())
@@ -237,4 +259,7 @@ func TestMakeMetaConfig(t *testing.T) {
 		}
 		break
 	}
+	if strings.Contains(txt, "nonexistentds") {
+		t.Errorf("expected location parameters for nonexistent delivery services to not be added to config, actual '%v'", txt)
+	}
 }

traffic_ops/ort/atstccfg/cfgfile/meta.go

@@ -185,9 +185,6 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string, error)
 
 	dsIDs := []int{}
 	for _, ds := range deliveryServices {
-		if ds.SigningAlgorithm == nil || *ds.SigningAlgorithm != tc.SigningAlgorithmURISigning {
-			continue
-		}
 		if ds.ID == nil {
 			// TODO log error?
 			continue
@@ -210,6 +207,7 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string, error)
 		dssMap[*dss.DeliveryService] = struct{}{}
 	}
 
+	dsNames := map[tc.DeliveryServiceName]struct{}{}
 	uriSignedDSes := []tc.DeliveryServiceName{}
 	for _, ds := range deliveryServices {
 		if ds.ID == nil {
@@ -218,14 +216,14 @@ func GetConfigFileMeta(cfg config.TCCfg, serverNameOrID string) (string, error)
 		if ds.XMLID == nil {
 			continue // TODO log?
 		}
-		if ds.SigningAlgorithm == nil || *ds.SigningAlgorithm != tc.SigningAlgorithmURISigning {
-			continue
-		}
 		if _, ok := dssMap[*ds.ID]; !ok {
 			continue
 		}
-		uriSignedDSes = append(uriSignedDSes, tc.DeliveryServiceName(*ds.XMLID))
+		dsNames[tc.DeliveryServiceName(*ds.XMLID)] = struct{}{}
+		if ds.SigningAlgorithm != nil && *ds.SigningAlgorithm == tc.SigningAlgorithmURISigning {
+			uriSignedDSes = append(uriSignedDSes, tc.DeliveryServiceName(*ds.XMLID))
+		}
 	}
 
-	return atscfg.MakeMetaConfig(tc.CacheName(serverHostName), &serverInfo, toURL, toReverseProxyURL, locationParams, uriSignedDSes, scopeParams), nil
+	return atscfg.MakeMetaConfig(tc.CacheName(serverHostName), &serverInfo, toURL, toReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dsNames), nil
 }

traffic_ops/testing/api/v14/atsconfig_meta_test.go

@@ -24,6 +24,8 @@ import (
 
 func TestATSConfigMeta(t *testing.T) {
 	WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, DeliveryServices}, func() {
+		defer DeleteTestDeliveryServiceServersCreated(t)
+		CreateTestDeliveryServiceServers(t)
 		GetTestATSConfigMeta(t)
 	})
 }
@@ -49,9 +51,9 @@ func GetTestATSConfigMeta(t *testing.T) {
 	}
 
 	expected := tc.ATSConfigMetaDataConfigFile{
-		FileNameOnDisk: "hdr_rw_mid_anymap-ds.config",
+		FileNameOnDisk: "hdr_rw_ds1.config",
 		Location:       "/remap/config/location/parameter",
-		APIURI:         "cdns/cdn1/configfiles/ats/hdr_rw_mid_anymap-ds.config", // expected suffix; config gen doesn't care about API version
+		APIURI:         "cdns/cdn1/configfiles/ats/hdr_rw_ds1.config", // expected suffix; config gen doesn't care about API version
 		URL:            "",
 		Scope:          "cdns",
 	}

traffic_ops/traffic_ops_golang/ats/atsserver/meta.go

@@ -76,7 +76,13 @@ func GetConfigMetaData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	txt := atscfg.MakeMetaConfig(serverName, server, tmParams.URL, tmParams.ReverseProxyURL, locationParams, uriSignedDSes, scopeParams)
+	dsNames, err := ats.GetDSNames(inf.Tx.Tx, server.HostName, server.Port)
+	if err != nil {
+		api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("GetConfigMetaData getting server ds names: "+err.Error()))
+		return
+	}
+
+	txt := atscfg.MakeMetaConfig(serverName, server, tmParams.URL, tmParams.ReverseProxyURL, locationParams, uriSignedDSes, scopeParams, dsNames)
 	w.Header().Set(rfc.ContentType, rfc.ApplicationJSON)
 	w.Write([]byte(txt))
 }

traffic_ops/traffic_ops_golang/ats/db.go

@@ -749,6 +749,36 @@ WHERE
 	return dses, nil
 }
 
+// GetDSNames returns a set of delivery service names which have the given server assigned, and any error.
+func GetDSNames(tx *sql.Tx, serverHostName string, serverPort int) (map[tc.DeliveryServiceName]struct{}, error) {
+	qry := `
+SELECT
+  ds.xml_id
+FROM
+  deliveryservice ds
+  JOIN deliveryservice_server dss ON ds.id = dss.deliveryservice
+  JOIN server s ON s.id = dss.server
+WHERE
+  s.host_name = $1
+  AND s.tcp_port = $2
+`
+	rows, err := tx.Query(qry, serverHostName, serverPort)
+	if err != nil {
+		return nil, errors.New("querying: " + err.Error())
+	}
+	defer rows.Close()
+
+	dses := map[tc.DeliveryServiceName]struct{}{}
+	for rows.Next() {
+		ds := tc.DeliveryServiceName("")
+		if err := rows.Scan(&ds); err != nil {
+			return nil, errors.New("scanning: " + err.Error())
+		}
+		dses[ds] = struct{}{}
+	}
+	return dses, nil
+}
+
 type TMParams struct {
 	URL             string
 	ReverseProxyURL string