Reduce non-intersecting map archive downloads

cli/cli.go

@@ -10,12 +10,26 @@ import (
 	"pfeifer.dev/mapd/maps"
 	m "pfeifer.dev/mapd/math"
 	"pfeifer.dev/mapd/params"
+	"pfeifer.dev/mapd/settings"
 )
 
 func Handle() {
 	shouldExit := true
 	cmd := &cli.Command{
 		Commands: []*cli.Command{
+			{
+				Name:  "generate-download-menu",
+				Usage: "Add archive selections to a download menu using local Natural Earth GeoJSON",
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "menu", Value: "settings/download_menu.json"},
+					&cli.StringFlag{Name: "countries", Required: true},
+					&cli.StringFlag{Name: "states", Required: true},
+					&cli.StringFlag{Name: "output", Required: true},
+				},
+				Action: func(ctx context.Context, cmd *cli.Command) error {
+					return settings.GenerateDownloadMenu(cmd.String("menu"), cmd.String("countries"), cmd.String("states"), cmd.String("output"))
+				},
+			},
 			{
 				Name:    "interactive",
 				Aliases: []string{"i"},

docs/overriding-internal-defaults.md

@@ -83,6 +83,15 @@ any desired areas to the file. The structure is as follows:
             "max_lat"
           ]
         },
+        "download_rows": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {"type": "integer"},
+            "minItems": 3,
+            "maxItems": 3
+          }
+        },
         "submenu": {
           "type": "string"
         }
@@ -101,3 +110,47 @@ chaining of the menus when requesting a download, so the submenu value should
 exactly match a top level key in the main object. This value is not used by the
 cli tui however, so selecting an entry with a submenu in the tui will just
 result in that entry being downloaded.
+
+An optional `download_rows` selects archives within the bounding box. Each row
+is `[latitude, first_longitude, exclusive_last_longitude]`, with coordinates on
+the 2-degree archive grid. For example, `[48, -124, -120]` downloads
+`offline/48/-124.tar.gz` and `offline/48/-122.tar.gz`. Rows must be sorted by
+latitude then longitude, must not overlap, and must stay inside the bounding
+box rounded outward to the archive grid. Missing, empty, or invalid rows use
+the full bounding box. Downloads and progress totals use the same selection.
+Remove the rows when changing an area's bounds unless you also update its
+selection. Existing menus with only bounding boxes continue to work.
+
+### Generating archive selections
+
+`mapd generate-download-menu` adds rows to an existing menu from local Natural
+Earth 10m country and state/province GeoJSON files. This is an optional data
+generation step; it does not run during map generation or require downloading
+the OSM planet. Custom entries without matching geometry keep their bounding
+boxes. Names, menu keys, submenus, and bounding boxes are retained.
+
+The included rows use the [Natural Earth source](https://github.com/nvkelso/natural-earth-vector/tree/ca96624a56bd078437bca8184e78163e5039ad19)
+at `ca96624a56bd078437bca8184e78163e5039ad19`. To regenerate from the repository root:
+
+```sh
+source_url=https://raw.githubusercontent.com/nvkelso/natural-earth-vector/ca96624a56bd078437bca8184e78163e5039ad19/geojson
+curl -fL "$source_url/ne_10m_admin_0_countries.geojson" -o /tmp/countries.geojson
+curl -fL "$source_url/ne_10m_admin_1_states_provinces.geojson" -o /tmp/states.geojson
+./build/mapd generate-download-menu --countries /tmp/countries.geojson --states /tmp/states.geojson \
+  --menu settings/download_menu.json --output /tmp/download_menu.json
+```
+
+Review the output before replacing a default or custom menu. The input SHA-256
+hashes are `239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255`
+(countries) and `22d0e3ad85eb3e27f17cabf8ba2d50e554fbc27a87796ff891d958185da62fb5`
+(states). The source data is [public domain](https://www.naturalearthdata.com/about/terms-of-use/).
+
+Generation keeps archives intersecting outer polygon rings, including islands,
+within each existing bounding box's archive grid. Intersection checks allow a
+0.05-degree margin for coarse coastlines near archive edges. This retained two
+coastal cells found by comparison with an independent Canada boundary; it is
+not a universal boundary-accuracy guarantee. Holes and degenerate clipping
+results can conservatively retain extra archives. Morocco, Somalia, and Ukraine retain their full bounding boxes
+because the source's territory assignments would narrow the existing entries.
+These are coarse download selections, not authoritative administrative borders;
+maintain custom regions through the existing menu override.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/gofrs/flock v0.13.0
+	github.com/paulmach/orb v0.1.3
 	github.com/paulmach/osm v0.8.0
 	github.com/pfeiferj/gomsgq v0.1.11
 	github.com/pkg/errors v0.9.1
@@ -33,7 +34,6 @@ require (
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
-	github.com/paulmach/orb v0.1.3 // indirect
 	github.com/paulmach/protoscan v0.2.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sahilm/fuzzy v0.1.1 // indirect

settings/download.go

@@ -18,9 +18,29 @@ import (
 )
 
 type LocationData struct {
-	BoundingBox Bounds `json:"bounding_box"`
-	FullName    string `json:"full_name"`
-	Submenu     string `json:"submenu"`
+	BoundingBox  Bounds       `json:"bounding_box"`
+	FullName     string       `json:"full_name"`
+	Submenu      string       `json:"submenu,omitempty"`
+	DownloadRows DownloadRows `json:"download_rows,omitempty"`
+}
+
+type DownloadRows [][3]int
+
+func (rows *DownloadRows) UnmarshalJSON(data []byte) error {
+	// Pointers distinguish a missing/null coordinate from the valid coordinate zero.
+	var decoded [][]*int
+	*rows = nil
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		return nil // Invalid optional selections retain the bounding-box fallback.
+	}
+	for _, row := range decoded {
+		if len(row) != 3 || row[0] == nil || row[1] == nil || row[2] == nil {
+			*rows = nil
+			return nil
+		}
+		*rows = append(*rows, [3]int{*row[0], *row[1], *row[2]})
+	}
+	return nil
 }
 
 type DownloadMenu map[string]map[string]LocationData
@@ -107,31 +127,33 @@ type download struct {
 	cancelChan   chan bool
 }
 
-func (p *DownloadProgress) addLocationDetails(path string) {
-	p.LocationDetails[path] = &DownloadLocationDetail{
-		TotalFiles: countFilesForBounds(getBoundsForPath(path)),
-	}
-}
-
 func Download(paths string, progressChan chan DownloadProgress, cancelChan chan bool) {
 	slog.Info("download", "paths", paths)
 	pathsSplit := strings.Split(paths, ",")
+	menu := GetDownloadMenu()
+	locations := make([]LocationData, len(pathsSplit))
 	d := download{
 		progress: DownloadProgress{
 			LocationsToDownload: pathsSplit,
-			TotalFiles:          countTotalFiles(pathsSplit),
 			LocationDetails:     make(map[string]*DownloadLocationDetail),
 			Active:              true,
 		},
 		progressChan: progressChan,
 		cancelChan:   cancelChan,
 	}
 
-	for _, p := range pathsSplit {
-		d.progress.addLocationDetails(p)
-		location := getDataForPath(p)
+	for i, path := range pathsSplit {
+		locations[i] = menu.getDataForPath(path)
+		total := locations[i].countFiles()
+		d.progress.TotalFiles += total
+		d.progress.LocationDetails[path] = &DownloadLocationDetail{TotalFiles: total}
+	}
+
+	for i, p := range pathsSplit {
+		location := locations[i]
+		d.progress.LocationDetails[p].DownloadedFiles = 0
 		slog.Info("downloading nation", "nation", location.FullName)
-		err, canceled := d.downloadBounds(location.BoundingBox, p)
+		err, canceled := d.downloadLocation(location, p)
 		if err != nil {
 			slog.Warn("failed to download nation", "error", err, "nation", location.FullName)
 		}
@@ -178,14 +200,13 @@ func adjustedBounds(bounds Bounds) (int, int, int, int) {
 	return minLat, minLon, maxLat, maxLon
 }
 
-func (d *download) downloadBounds(bounds Bounds, locationName string) (err error, cancel bool) {
+func (d *download) downloadLocation(location LocationData, locationName string) (err error, cancel bool) {
+	bounds := location.BoundingBox
 	slog.Info("Downloading Bounds", "min_lat", bounds.MinLat, "min_lon", bounds.MinLon, "max_lat", bounds.MaxLat, "max_lon", bounds.MaxLon)
 
-	// clip given bounds to file areas
-	minLat, minLon, maxLat, maxLon := adjustedBounds(bounds)
-	d.progress.LocationDetails[locationName].TotalFiles = countFilesForBounds(bounds)
-	for i := minLat; i < maxLat; i += GROUP_AREA_BOX_DEGREES {
-		for j := minLon; j < maxLon; j += GROUP_AREA_BOX_DEGREES {
+	for _, row := range location.downloadRows() {
+		i := row[0]
+		for j := row[1]; j < row[2]; j += GROUP_AREA_BOX_DEGREES {
 			d.publishProgress()
 			select { // cancel if sent message
 			case cancel := <-d.cancelChan:
@@ -286,18 +307,48 @@ func (d *download) downloadBounds(bounds Bounds, locationName string) (err error
 	return nil, false
 }
 
-func countFilesForBounds(bounds Bounds) int {
-	minLat, minLon, maxLat, maxLon := adjustedBounds(bounds)
-	return ((maxLat - minLat) / GROUP_AREA_BOX_DEGREES) * ((maxLon - minLon) / GROUP_AREA_BOX_DEGREES)
+// Each row is [latitude, first longitude, exclusive last longitude] on the archive grid.
+func (location LocationData) downloadRows() [][3]int {
+	minLat, minLon, maxLat, maxLon := adjustedBounds(location.BoundingBox)
+	valid := len(location.DownloadRows) > 0
+	for index, row := range location.DownloadRows {
+		if row[0] < minLat || row[0] >= maxLat || row[1] < minLon || row[2] > maxLon || row[1] >= row[2] ||
+			row[0]%GROUP_AREA_BOX_DEGREES != 0 || row[1]%GROUP_AREA_BOX_DEGREES != 0 || row[2]%GROUP_AREA_BOX_DEGREES != 0 {
+			valid = false
+			break
+		}
+		if index > 0 {
+			previous := location.DownloadRows[index-1]
+			if row[0] < previous[0] || (row[0] == previous[0] && row[1] < previous[2]) {
+				valid = false
+				break
+			}
+		}
+	}
+	if valid {
+		return location.DownloadRows
+	}
+	var rows [][3]int
+	for latitude := minLat; latitude < maxLat; latitude += GROUP_AREA_BOX_DEGREES {
+		rows = append(rows, [3]int{latitude, minLon, maxLon})
+	}
+	return rows
 }
 
-func getDataForPath(path string) LocationData {
+func (location LocationData) countFiles() int {
+	total := 0
+	for _, row := range location.downloadRows() {
+		total += (row[2] - row[1]) / GROUP_AREA_BOX_DEGREES
+	}
+	return total
+}
+
+func (menu DownloadMenu) getDataForPath(path string) LocationData {
 	parts := strings.Split(path, ".")
 	if len(parts) < 2 {
 		slog.Warn("ignoring invalid download path", "path", path)
 		return LocationData{}
 	}
-	menu := GetDownloadMenu()
 	box := menu[parts[0]][parts[1]]
 	if len(parts) > 2 {
 		for i := range len(parts) - 2 {
@@ -306,17 +357,3 @@ func getDataForPath(path string) LocationData {
 	}
 	return box
 }
-
-func getBoundsForPath(path string) Bounds {
-	return getDataForPath(path).BoundingBox
-}
-
-func countTotalFiles(paths []string) int {
-	totalFiles := 0
-
-	for _, p := range paths {
-		totalFiles += countFilesForBounds(getBoundsForPath(p))
-	}
-
-	return totalFiles
-}