From eec68aa5a41bab0a541c0719c6f24f377afa6276 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Sat, 23 Jun 2018 18:37:04 -0600 Subject: [PATCH] Add TO Go deliveryservices/id/capacity --- traffic_monitor/cache/data.go | 17 ++ .../dbhelpers/db_helpers.go | 12 + .../deliveryservice/capacity.go | 241 ++++++++++++++++++ .../traffic_ops_golang/routing/routes.go | 2 + .../util/monitorhlp/monitorhlp.go | 20 ++ 5 files changed, 292 insertions(+) create mode 100644 traffic_ops/traffic_ops_golang/deliveryservice/capacity.go diff --git a/traffic_monitor/cache/data.go b/traffic_monitor/cache/data.go index 83854361a1..32c70580ef 100644 --- a/traffic_monitor/cache/data.go +++ b/traffic_monitor/cache/data.go @@ -96,6 +96,23 @@ func (t *ResultStatVal) MarshalJSON() ([]byte, error) { return json.Marshal(&v) } +func (t *ResultStatVal) UnmarshalJSON(data []byte) error { + v := struct { + Val string `json:"value"` + Time int64 `json:"time"` + Span uint64 `json:"span"` + }{} + json := jsoniter.ConfigFastest // TODO make configurable + err := json.Unmarshal(data, &v) + if err != nil { + return err + } + t.Time = time.Unix(0, v.Time*1000000) + t.Val = v.Val + t.Span = v.Span + return nil +} + func pruneStats(history []ResultStatVal, limit uint64) []ResultStatVal { if uint64(len(history)) > limit { history = history[:limit-1] diff --git a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go index 82ec8c1a1a..a0253cb3ae 100644 --- a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go +++ b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go @@ -278,6 +278,18 @@ func GetDSTenantIDFromXMLID(tx *sql.Tx, xmlid string) (int, bool, error) { return id, true, nil } +func GetDSNameAndCDNFromID(tx *sql.Tx, id int) (tc.DeliveryServiceName, tc.CDNName, bool, error) { + dsName := tc.DeliveryServiceName("") + cdnName := tc.CDNName("") + if err := tx.QueryRow(`SELECT ds.xml_id, cdn.name FROM deliveryservice ds JOIN cdn ON cdn.id = ds.cdn_id WHERE ds.id = $1`, id).Scan(&dsName, &cdnName); err != nil { + if err == sql.ErrNoRows { + return tc.DeliveryServiceName(""), tc.CDNName(""), false, nil + } + return tc.DeliveryServiceName(""), tc.CDNName(""), false, errors.New("querying: " + err.Error()) + } + return dsName, cdnName, true, nil +} + // GetProfileNameFromID returns the profile's name, whether a profile with ID exists, or any error. func GetProfileNameFromID(id int, tx *sql.Tx) (string, bool, error) { name := "" diff --git a/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go b/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go new file mode 100644 index 0000000000..b653c379c4 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/deliveryservice/capacity.go @@ -0,0 +1,241 @@ +package deliveryservice + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + tmcache "github.com/apache/trafficcontrol/traffic_monitor/cache" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/monitorhlp" +) + +func GetCapacity(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + dsID := inf.IntParams["id"] + + userErr, sysErr, errCode = tenant.CheckID(inf.Tx.Tx, inf.User, dsID) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + ds, cdn, ok, err := dbhelpers.GetDSNameAndCDNFromID(inf.Tx.Tx, dsID) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting delivery service name from ID: "+err.Error())) + return + } + if !ok { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusNotFound, nil, nil) + } + + capacity, err := getCapacity(inf.Tx.Tx, ds, cdn) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting delivery service capacity: "+err.Error())) + return + } + + api.WriteResp(w, r, capacity) +} + +type CapacityResp struct { + AvailablePercent float64 `json:"availablePercent"` + UnavailablePercent float64 `json:"unavailablePercent"` + UtilizedPercent float64 `json:"utilizedPercent"` + MaintenancePercent float64 `json:"maintenancePercent"` +} + +type CapData struct { + Available float64 + Unavailable float64 + Maintenance float64 + Capacity float64 +} + +func getCapacity(tx *sql.Tx, ds tc.DeliveryServiceName, cdn tc.CDNName) (CapacityResp, error) { + monitors, err := monitorhlp.GetURLs(tx) + if err != nil { + return CapacityResp{}, errors.New("getting monitor URLs: " + err.Error()) + } + client, err := monitorhlp.GetClient(tx) + if err != nil { + return CapacityResp{}, errors.New("getting monitor client: " + err.Error()) + } + + thresholds, err := getEdgeProfileHealthThresholdBandwidth(tx) + if err != nil { + return CapacityResp{}, errors.New("getting profile thresholds: " + err.Error()) + } + + monitorFQDN, ok := monitors[cdn] + if !ok { + return CapacityResp{}, nil // TODO emulates perl; change to error? + } + + crStates, err := monitorhlp.GetCRStates(monitorFQDN, client) + // TODO on err, try another online monitor + if err != nil { + return CapacityResp{}, errors.New("getting CRStates for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + crConfig, err := monitorhlp.GetCRConfig(monitorFQDN, client) + // TODO on err, try another online monitor + if err != nil { + return CapacityResp{}, errors.New("getting CRConfig for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + cacheStats, err := monitorhlp.GetCacheStats(monitorFQDN, client, []string{"maxKbps", "kbps"}) + if err != nil { + return CapacityResp{}, errors.New("getting CacheStats for delivery service '" + string(ds) + "' monitor '" + monitorFQDN + "': " + err.Error()) + } + cap := addCapacity(CapData{}, ds, cacheStats, crStates, crConfig, thresholds) + if cap.Capacity == 0 { + return CapacityResp{}, errors.New("capacity was zero!") // avoid divide-by-zero below. + } + + return CapacityResp{ + UtilizedPercent: (cap.Available * 100) / cap.Capacity, + UnavailablePercent: (cap.Unavailable * 100) / cap.Capacity, + MaintenancePercent: (cap.Maintenance * 100) / cap.Capacity, + AvailablePercent: ((cap.Capacity - cap.Unavailable - cap.Maintenance - cap.Available) * 100) / cap.Capacity, + }, nil +} + +const StatNameKBPS = "kbps" +const StatNameMaxKBPS = "maxKbps" + +func addCapacity(cap CapData, ds tc.DeliveryServiceName, cacheStats tmcache.Stats, crStates tc.CRStates, crConfig tc.CRConfig, thresholds map[string]float64) CapData { + for cacheName, stats := range cacheStats.Caches { + cache, ok := crConfig.ContentServers[string(cacheName)] + if !ok { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' in CacheStats but not CRConfig, skipping") + continue + } + if _, ok := cache.DeliveryServices[string(ds)]; !ok { + continue + } + if cache.ServerType == nil || !strings.HasPrefix(string(*cache.ServerType), string(tc.CacheTypeEdge)) { + continue + } + if len(stats[StatNameKBPS]) < 1 || len(stats[StatNameMaxKBPS]) < 1 { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats has no kbps or maxKbps, skipping") + continue + } + + kbps, err := statToFloat(stats[StatNameKBPS][0].Val) + if err != nil { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats kbps is not a number, skipping") + continue + } + maxKBPS, err := statToFloat(stats[StatNameMaxKBPS][0].Val) + if err != nil { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CacheStats maxKps is not a number, skipping") + continue + } + if cache.ServerStatus == nil { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CRConfig Status is nil, skipping") + continue + } + if cache.Profile == nil { + log.Warnln("Getting delivery service capacity: delivery service '" + string(ds) + "' cache '" + string(cacheName) + "' CRConfig Profile is nil, skipping") + continue + } + if tc.CacheStatus(*cache.ServerStatus) == tc.CacheStatusReported || tc.CacheStatus(*cache.ServerStatus) == tc.CacheStatusOnline { + if crStates.Caches[cacheName].IsAvailable { + cap.Available += kbps + } else { + cap.Unavailable += kbps + } + } else if tc.CacheStatus(*cache.ServerStatus) == tc.CacheStatusAdminDown { + cap.Maintenance += kbps + } else { + continue // don't add capacity for OFFLINE or other statuses + } + cap.Capacity += maxKBPS - thresholds[*cache.Profile] + } + return cap +} + +// statToFloat converts a CacheStats stat interface{} to a float64 +func statToFloat(s interface{}) (float64, error) { + switch v := s.(type) { + case int: + return float64(v), nil + case int64: + return float64(v), nil + case float64: + return v, nil + case string: + iv, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0.0, errors.New("stat is a string which is not a number: " + err.Error()) + } + return iv, nil + default: + return 0.0, fmt.Errorf("unknown stat type: %T", s) + } +} + +func getEdgeProfileHealthThresholdBandwidth(tx *sql.Tx) (map[string]float64, error) { + rows, err := tx.Query(` +SELECT pr.name as profile, pa.name, pa.config_file, pa.value +FROM parameter as pa +JOIN profile_parameter as pp ON pp.parameter = pa.id +JOIN profile as pr ON pp.profile = pr.id +JOIN server as s ON s.profile = pr.id +JOIN cdn as c ON c.id = s.cdn_id +JOIN type as t ON s.type = t.id +WHERE t.name LIKE 'EDGE%' +AND pa.config_file = 'rascal-config.txt' +AND pa.name = 'health.threshold.availableBandwidthInKbps' +`) + if err != nil { + return nil, errors.New("querying thresholds: " + err.Error()) + } + defer rows.Close() + profileThresholds := map[string]float64{} + for rows.Next() { + profile := "" + threshStr := "" + if err := rows.Scan(&profile, &threshStr); err != nil { + return nil, errors.New("scanning thresholds: " + err.Error()) + } + threshStr = strings.TrimPrefix(threshStr, ">") + thresh, err := strconv.ParseFloat(threshStr, 64) + if err != nil { + return nil, errors.New("profile '" + profile + "' health.threshold.availableBandwidthInKbps is not a number") + } + profileThresholds[profile] = thresh + } + return profileThresholds, nil +} diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 6b37c55788..345404b176 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -302,6 +302,8 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) { {1.1, http.MethodPost, `deliveryservices/request`, deliveryservicerequests.Request, auth.PrivLevelPortal, Authenticated, nil, 740875299, perlBypass}, {1.1, http.MethodGet, `deliveryservice_matches/?(\.json)?$`, deliveryservice.GetMatches, auth.PrivLevelReadOnly, Authenticated, nil, 1191301170, noPerlBypass}, + {1.1, http.MethodGet, `deliveryservices/{id}/capacity/?(\.json)?$`, deliveryservice.GetCapacity, auth.PrivLevelReadOnly, Authenticated, nil, 1231409110, perlBypass}, + //Server {1.1, http.MethodGet, `servers/status$`, server.GetServersStatusCountsHandler, auth.PrivLevelReadOnly, Authenticated, nil, 2052786293, perlBypass}, {1.1, http.MethodGet, `servers/totals$`, handlerToFunc(proxyHandler), 0, NoAuth, []middleware.Middleware{}, 2037840835, noPerlBypass}, diff --git a/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go b/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go index 2679e67a2e..8b0edad6ae 100644 --- a/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go +++ b/traffic_ops/traffic_ops_golang/util/monitorhlp/monitorhlp.go @@ -27,9 +27,11 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/apache/trafficcontrol/lib/go-tc" + tmcache "github.com/apache/trafficcontrol/traffic_monitor/cache" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" ) @@ -120,3 +122,21 @@ func GetCRConfig(monitorFQDN string, client *http.Client) (tc.CRConfig, error) { } return crs, nil } + +// GetCacheStats gets the cache stats from the given monitor. The stats parameters is which stats to get; if stats is empty or nil, all stats are fetched. +func GetCacheStats(monitorFQDN string, client *http.Client, stats []string) (tmcache.Stats, error) { + path := `/publish/CacheStats?hc=1` + if len(stats) > 0 { + path += `&stats=` + strings.Join(stats, `,`) + } + resp, err := client.Get("http://" + monitorFQDN + path) + if err != nil { + return tmcache.Stats{}, errors.New("getting CacheStats from Monitor '" + monitorFQDN + "': " + err.Error()) + } + defer resp.Body.Close() + cacheStats := tmcache.Stats{} + if err := json.NewDecoder(resp.Body).Decode(&cacheStats); err != nil { + return tmcache.Stats{}, errors.New("decoding CacheStats from monitor '" + monitorFQDN + "': " + err.Error()) + } + return cacheStats, nil +}