chore(migrations): Consolidate migrations

Makefile

@@ -12,7 +12,6 @@ PROTOC           := $(LOCAL_BIN)/protoc
 PROTOC_INCLUDE   := $(LOCAL_BIN)/include
 PROTOC_GEN_GO    := $(LOCAL_BIN)/protoc-gen-go
 PROTOC_GEN_GRPC  := $(LOCAL_BIN)/protoc-gen-go-grpc
-PROTOC_GEN_DOC   := $(LOCAL_BIN)/protoc-gen-doc
 
 # --- Sources & Stamps ---
 GO_SOURCES          := $(shell find . -name '*.go' -not -path "./internal/gen/*" -not -path "./vendor/*")
@@ -78,7 +77,7 @@ doctor: $(PROTOC)
 # --- Code Generation Targets ------------------------------------------------------------- #
 
 .PHONY: gen
-gen: gen.proto.go gen.db gen.proto.docs
+gen: gen.proto.go gen.db
 
 .PHONY: gen.db
 gen.db: ${SQLC_STAMP_FILE}
@@ -181,116 +180,6 @@ gen.proto.python: ${PROTOC}
 	@echo "Building wheel..."
 	@cd gen/python && echo $$(uv run python -m setuptools_git_versioning) && uv build
 
-define GEN_DOCS
-## GRPC API Documentation
-
-{{- range .Files -}}
-{{- $$file_name := .Name -}}
-
-{{/* --- SERVICES & METHODS --- */}}
-{{- if .HasServices}}
-{{range .Services -}}
-<a name="{{.FullName | anchor}}"></a>
-
-### {{.Name}} ({{$$file_name}})
-{{.Description}}
-
-{{range .Methods -}}
-<a name="{{.Name | anchor}}"></a>
-
-#### {{.Name}}
-
-{{.Description}}
-
-_[{{.RequestLongType}}](#{{.RequestFullType | anchor}}){{if .RequestStreaming}} stream{{end}} / [{{.ResponseLongType}}](#{{.ResponseFullType | anchor}}){{if .ResponseStreaming}} stream{{end}}_
-
-{{end}}{{/* end methods */}}
-{{- end}}{{/* end services */}}
-{{- end}}{{/* end has_services */}}
-
-{{- if .HasMessages}}
-<details><summary>Messages ({{$$file_name}})</summary>
-
-{{range .Messages -}}
-<a name="{{.FullName | anchor}}"></a>
-<details><summary>{{.LongName}}</summary>
-{{.Description}}
-
-{{if .HasFields -}}
-| Field | Type | Label | Description |
-| ----- | ---- | ----- | ----------- |
-{{range .Fields -}}
-  | {{.Name}} | [{{.LongType}}](#{{.FullType | anchor}}) | {{.Label}} | {{if (index .Options "deprecated"|default false)}}**Deprecated.** {{end}}{{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} |
-{{- end -}}
-{{- end -}}
-
-{{if .HasExtensions -}}
-| Extension | Type | Base | Number | Description |
-| --------- | ---- | ---- | ------ | ----------- |
-{{range .Extensions -}}
-  | {{.Name}} | {{.LongType}} | {{.ContainingLongType}} | {{.Number}} | {{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} |
-{{- end }}
-{{- end }}
-</details>
-
-{{- end }}{{/* end messages */}}
-</details>
-{{ end -}}{{/* end has_messages */}}
-
-{{- if .HasEnums}}
-<details><summary>Enums ({{$$file_name}})</summary>
-
-{{range .Enums -}}
-<a name="{{.FullName | anchor}}"></a>
-
-<details><summary>{{.LongName}}</summary>
-{{.Description}}
-
-| Name | Number | Description |
-| ---- | ------ | ----------- |
-{{range .Values -}}
-  | {{.Name}} | {{.Number}} | {{nobr .Description}} |
-{{- end }}
-</details>
-
-{{- end}}{{/* end enums */}}
-</details>
-{{ end -}}{{/* end has_enums */}}
-
-{{/* --- FILE-LEVEL EXTENSIONS --- */}}
-{{- if .HasExtensions}}
-<a name="{{$$file_name | anchor}}-extensions"></a>
-
-### File-level Extensions ({{$$file_name}})
-| Extension | Type | Base | Number | Description |
-| --------- | ---- | ---- | ------ | ----------- |
-{{range .Extensions -}}
-  | {{.Name}} | {{.LongType}} | {{.ContainingLongType}} | {{.Number}} | {{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} |
-{{end}}
-{{- end}}{{/* end HasExtensions */}}
-
-{{- end}}{{/* end files */}}
-
-endef
-export GEN_DOCS
-
-.PHONY: gen.proto.docs
-gen.proto.docs: ${PROTOC} ${PROTOC_GEN_DOC}
-	@rm -rf gen/docs && mkdir -p gen/docs
-	@echo "$$GEN_DOCS" > gen/docs/markdown.tmpl
-	@${PROTOC} \
-		${PROTO_SOURCES} \
-		-I=proto \
-		-I=$(PROTOC_INCLUDE) \
-		--doc_out=gen/docs \
-		--doc_opt=gen/docs/markdown.tmpl,docs.md:=buf/*,google/*,ocf/dp/dp.rules.proto
-	@sed -n '1,/<!-- DOCS START -->/p' README.md > README.tmp
-	@echo "" >> README.tmp
-	@cat gen/docs/docs.md >> README.tmp
-	@echo "" >> README.tmp
-	@sed -n '/<!-- DOCS END -->/,$$p' README.md >> README.tmp
-	@mv README.tmp README.md
-
 # --- LOCAL RUNNING TARGETS --------------------------------------------------------------------- #
 
 .PHONY: run.db

internal/server/postgres/dataserverimpl.go

@@ -7,10 +7,13 @@
 package postgres
 
 import (
+	"bytes"
+	"cmp"
 	"context"
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"time"
 
 	"github.com/google/uuid"
@@ -1484,6 +1487,23 @@ func (s *DataPlatformDataServiceServerImpl) StreamCreateForecasts(
 			return nil
 		}
 
+		// Sort the batch to match the index to improve clustering in the database.
+		slices.SortFunc(forecastParams, func(a, b db.CreateForecastsParams) int {
+			if c := bytes.Compare(a.GeometryUuid[:], b.GeometryUuid[:]); c != 0 {
+				return c
+			}
+
+			if c := cmp.Compare(a.SourceTypeID, b.SourceTypeID); c != 0 {
+				return c
+			}
+
+			if c := cmp.Compare(a.ForecasterID, b.ForecasterID); c != 0 {
+				return c
+			}
+
+			return bytes.Compare(b.ForecastUuid[:], a.ForecastUuid[:])
+		})
+
 		countF, err := querier.CreateForecasts(ctx, forecastParams)
 		if err != nil || countF < int64(len(forecastParams)) {
 			if err == nil {

internal/server/postgres/mappers.go

@@ -29,7 +29,9 @@ func MapSlice[T, U any](input []T, mapper func(T) U) []U {
 }
 
 // timeWindowToPgWindow converts a TimeWindow protobuf message to a pair of pgtype.Timestamp values.
-// If the TimeWindow is nil or its StartTimestampUtc is nil, it defaults to a window from 48 hours ago to 36 hours in the future. Protovalidate ensures at the boundary that the start is always before the end, so we don't need to check that here.
+// If the TimeWindow is nil or its StartTimestampUtc is nil, it defaults to a window from 48 hours
+// ago to 36 hours in the future. Protovalidate ensures at the boundary that the start is always
+// before the end, so we don't need to check that here.
 func timeWindowToPgWindow(
 	window *pb.TimeWindow,
 ) (start pgtype.Timestamp, end pgtype.Timestamp) {
@@ -60,9 +62,9 @@ func timeptrToPgTimestamp(t *timestamppb.Timestamp) pgtype.Timestamp {
 
 // extractSIPStatSlice builds the array for a single p-level from a forecast's values.
 // Returns nil if no value in the series carries this statistic, so the column is stored as a
-// NULL array rather than a materialised array of nulls (~3 bytes per forecast against ~130).
-// Callers must have run validateForecastValues first: sqlc maps SMALLINT[] to []int16, which
-// cannot express element-level nulls, so partial coverage would silently be written as zeros.
+// NULL array rather than a materialised array of NULLs.
+// NOTE: Callers must have run validateForecastValues first, otherwise partial forecasts would
+// silently be infilled with zeros.
 func extractSIPStatSlice(values []*pb.CreateForecastRequest_ForecastValue, key string) []int16 {
 	out := make([]int16, len(values))
 	present := false
@@ -81,8 +83,9 @@ func extractSIPStatSlice(values []*pb.CreateForecastRequest_ForecastValue, key s
 	return out
 }
 
-// extractP50Slice builds the p50 array. P50 is a top-level field on ForecastValue rather than a
-// key in OtherStatisticsFractions, and is always present.
+// extractP50Slice builds the p50 array.
+// P50 is a top-level field on ForecastValue rather than a key in OtherStatisticsFractions,
+// and is always present.
 func extractP50Slice(values []*pb.CreateForecastRequest_ForecastValue) []int16 {
 	out := make([]int16, len(values))
 	for i, v := range values {
@@ -97,9 +100,7 @@ func sipToFraction(sip int16) float32 {
 	return float32(sip) / 30000.0
 }
 
-// validateForecastValues checks the invariants the array storage layout depends on:
-// at least two values, strictly increasing horizons, evenly spaced, and each optional statistic
-// either present on every value or on none.
+// validateForecastValues checks the forecast valiues against a set of rules.
 func validateForecastValues(values []*pb.CreateForecastRequest_ForecastValue) error {
 	if len(values) < 2 {
 		return errors.New("a forecast must contain at least two values")