From 330fafeeeb07d7b8bfdc5bd0c645c575b128ad91 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:24:30 +0100 Subject: [PATCH 1/4] chore(migrations): Consolidate migrations --- .../postgres/sql/migrations/00001_uuidv7.sql | 62 --- .../sql/migrations/00002_locations.sql | 177 ------- .../sql/migrations/00003_observations.sql | 101 ---- .../sql/migrations/00004_predictions.sql | 236 --------- .../postgres/sql/migrations/00005_iam.sql | 175 ------- .../sql/migrations/00006_sources_mv_index.sql | 9 - .../00007_partman_yearly_retention.sql | 110 ---- .../sql/migrations/00008_simple_iam.sql | 93 ---- .../sql/migrations/00009_optimize_storage.sql | 214 -------- .../sql/migrations/00010_7_plevels.sql | 22 - .../00011_forecast_value_arrays.sql | 75 --- .../00012_rebuild_forecast_partitions.sql | 211 -------- .../migrations/00013_split_rebuild_swap.sql | 336 ------------ .../migrations/00014_consolidated_schema.sql | 484 ++++++++++++++++++ .../sql/migrations/00014_drop_row_storage.sql | 201 -------- 15 files changed, 484 insertions(+), 2022 deletions(-) delete mode 100644 internal/server/postgres/sql/migrations/00001_uuidv7.sql delete mode 100644 internal/server/postgres/sql/migrations/00002_locations.sql delete mode 100644 internal/server/postgres/sql/migrations/00003_observations.sql delete mode 100644 internal/server/postgres/sql/migrations/00004_predictions.sql delete mode 100644 internal/server/postgres/sql/migrations/00005_iam.sql delete mode 100644 internal/server/postgres/sql/migrations/00006_sources_mv_index.sql delete mode 100644 internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql delete mode 100644 internal/server/postgres/sql/migrations/00008_simple_iam.sql delete mode 100644 internal/server/postgres/sql/migrations/00009_optimize_storage.sql delete mode 100644 internal/server/postgres/sql/migrations/00010_7_plevels.sql delete mode 100644 internal/server/postgres/sql/migrations/00011_forecast_value_arrays.sql delete mode 100644 internal/server/postgres/sql/migrations/00012_rebuild_forecast_partitions.sql delete mode 100644 internal/server/postgres/sql/migrations/00013_split_rebuild_swap.sql create mode 100644 internal/server/postgres/sql/migrations/00014_consolidated_schema.sql delete mode 100644 internal/server/postgres/sql/migrations/00014_drop_row_storage.sql diff --git a/internal/server/postgres/sql/migrations/00001_uuidv7.sql b/internal/server/postgres/sql/migrations/00001_uuidv7.sql deleted file mode 100644 index 65545a8..0000000 --- a/internal/server/postgres/sql/migrations/00001_uuidv7.sql +++ /dev/null @@ -1,62 +0,0 @@ --- +goose Up - --- From https://github.com/dverite/postgres-uuidv7-sql/tree/main -/* See the UUID Version 7 specification at - https://www.rfc-editor.org/rfc/rfc9562#name-uuid-version-7 */ - --- +goose StatementBegin -/* Main function to generate a uuidv7 value with millisecond precision */ -CREATE FUNCTION uuidv7(timestamptz DEFAULT clock_timestamp()) RETURNS uuid -AS $$ - -- Replace the first 48 bits of a uuidv4 with the current - -- number of milliseconds since 1970-01-01 UTC - -- and set the "ver" field to 7 by setting additional bits - select encode( - set_bit( - set_bit( - overlay(uuid_send(gen_random_uuid()) placing - substring(int8send((extract(epoch from $1)*1000)::bigint) from 3) - from 1 for 6), - 52, 1), - 53, 1), 'hex')::uuid; -$$ LANGUAGE sql volatile parallel safe; --- +goose StatementEnd - -COMMENT ON FUNCTION uuidv7(timestamptz) IS -'Generate a uuid-v7 value with a 48-bit timestamp (millisecond precision) and 74 bits of randomness'; - --- +goose StatementBegin -/* Extract the timestamp in the first 6 bytes of the uuidv7 value. - Use the fact that 'xHHHHH' (where HHHHH are hexadecimal numbers) - can be cast to bit(N) and then to int8. - */ -CREATE FUNCTION uuidv7_extract_timestamp(uuid) RETURNS timestamptz -AS $$ - select to_timestamp( - right(substring(uuid_send($1) from 1 for 6)::text, -1)::bit(48)::int8 -- milliseconds - /1000.0); -$$ LANGUAGE sql immutable strict parallel safe; --- +goose StatementEnd - -COMMENT ON FUNCTION uuidv7_extract_timestamp(uuid) IS -'Return the timestamp stored in the first 48 bits of the UUID v7 value'; - --- +goose StatementBegin -CREATE FUNCTION uuidv7_boundary(timestamptz) RETURNS uuid -AS $$ - /* uuid fields: version=0b0111, variant=0b10 */ - select encode( - overlay('\x00000000000070008000000000000000'::bytea - placing substring(int8send(floor(extract(epoch from $1) * 1000)::bigint) from 3) - from 1 for 6), - 'hex')::uuid; -$$ LANGUAGE sql stable strict parallel safe; --- +goose StatementEnd - -COMMENT ON FUNCTION uuidv7_boundary(timestamptz) IS -'Generate a non-random uuidv7 with the given timestamp (first 48 bits) and all random bits to 0. As the smallest possible uuidv7 for that timestamp, it may be used as a boundary for partitions.'; - --- +goose Down -DROP FUNCTION uuidv7; -DROP FUNCTION uuidv7_extract_timestamp; -DROP FUNCTION uuidv7_boundary; diff --git a/internal/server/postgres/sql/migrations/00002_locations.sql b/internal/server/postgres/sql/migrations/00002_locations.sql deleted file mode 100644 index ba6593c..0000000 --- a/internal/server/postgres/sql/migrations/00002_locations.sql +++ /dev/null @@ -1,177 +0,0 @@ --- +goose Up - -/* - * Schema and tables to handle location data. - * - * The generation data we store, be it predicted or otherwise, is always tied to a certain - * geometry. These geometries vary in size and scope, from a single site to an entire country, - * and the metadata we may want to store about them will also vary accordingly. - - * From an application standpoint, the geometry is pertinent in the case where we care about the - * generated power as a fraction of the capacity of the geometry, as well as allowing us to - * represent the data on a map. - - * To this degree, what the external application may consider a "location", is represented here as - * a combination of a geometry (the spatial data), and a source (the energy generation capability). - * One geometry can have multiple sources, e.g. the UK nation geometry can have solar, wind, etc. - */ - -CREATE EXTENSION IF NOT EXISTS btree_gist; -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; -CREATE SCHEMA IF NOT EXISTS topology; -CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; - -CREATE SCHEMA loc; - -/*- Lookups -----------------------------------------------------------------------------------*/ - --- Lookup table to store different source types -CREATE TABLE loc.source_types ( - source_type_id SMALLINT GENERATED ALWAYS AS IDENTITY NOT NULL, - source_type_name TEXT NOT NULL, - CONSTRAINT source_type_name_format_check CHECK ( - LENGTH(source_type_name) > 0 - AND LENGTH(source_type_name) <= 48 - AND source_type_name = LOWER(source_type_name) - ), - PRIMARY KEY (source_type_id), - UNIQUE (source_type_name) -); --- The ordering of insertion here matches the .proto enum definitions. Change with caution! -INSERT INTO loc.source_types (source_type_name) VALUES ('solar'), ('wind'), ('hydro'), ('battery'); - --- Lookup table to store different geometry types -CREATE TABLE loc.geometry_types ( - geometry_type_id SMALLINT GENERATED ALWAYS AS IDENTITY NOT NULL, - geometry_type_name TEXT NOT NULL, - CONSTRAINT geometry_type_name_format_check CHECK ( - LENGTH(geometry_type_name) > 0 - AND LENGTH(geometry_type_name) <= 24 - AND geometry_type_name = LOWER(geometry_type_name) - ), - PRIMARY KEY (geometry_type_id), - UNIQUE (geometry_type_name) -); --- The ordering of insertion here matches the .proto enum definitions. Change with caution! -INSERT INTO loc.geometry_types (geometry_type_name) VALUES ('site'), ('gsp'), ('dno'), ('nation'), ('state'), ('county'), ('city'), ('primary_substation'); - - -/*- Tables ----------------------------------------------------------------------------------*/ - --- Table to store spatial data for geometries -CREATE TABLE loc.geometries ( - geometry_uuid UUID DEFAULT UUIDV7() NOT NULL, - geometry_name TEXT NOT NULL, - CONSTRAINT geometry_name_check CHECK ( - LENGTH(geometry_name) > 0 - AND geometry_name = LOWER(geometry_name) - ), - geom GEOMETRY (GEOMETRY, 4326) NOT NULL, - CONSTRAINT geom_validity_check CHECK ( - ST_GEOMETRYTYPE(geom) IN ('ST_Point', 'ST_Polygon', 'ST_MultiPolygon') - AND ST_SRID(geom) = 4326 - AND ST_NDIMS(geom) = 2 - AND ST_ISVALID(geom) - AND ST_XMIN(geom) >= -180 AND ST_XMAX(geom) <= 180 - AND ST_YMIN(geom) >= -90 AND ST_YMAX(geom) <= 90 - ), - geometry_type_id SMALLINT NOT NULL - REFERENCES loc.geometry_types (geometry_type_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - associated_point GEOMETRY (POINT, 4326) NOT NULL, - CONSTRAINT associated_point_validity_check CHECK ( - ST_SRID(associated_point) = 4326 - AND ST_NDIMS(associated_point) = 2 - AND ST_ISVALID(associated_point) - AND ST_X(associated_point) >= -180 AND ST_X(associated_point) <= 180 - AND ST_Y(associated_point) >= -90 AND ST_Y(associated_point) <= 90 - ), - geom_hash TEXT GENERATED ALWAYS AS (MD5(ST_ASBINARY(geom))) STORED, - metadata JSONB DEFAULT NULL, - PRIMARY KEY (geometry_uuid), - UNIQUE (geometry_name, geom_hash) -); --- Required index for efficient spatial-based queries -CREATE INDEX ON loc.geometries USING gist (geom); --- Index for efficiently fetching e.g. all POINT geometry geometries -CREATE INDEX ON loc.geometries (ST_GEOMETRYTYPE(geom)); --- Index for finding all geometries of a certain type -CREATE INDEX ON loc.geometries (geometry_type_id); --- Legacy index for finding gsp geometries by gsp_id -CREATE INDEX idx_geometries_gsp_id_partial -ON loc.geometries (((metadata ->> 'gsp_id')::INTEGER)) -WHERE geometry_type_id = 2 - AND (((metadata ->> 'gsp_id') IS NOT NULL)); - -/* - * Table to store the temporal generation capability of geometries. - * Each geometry can have multiple sources of generation (solar, wind, etc), - * and each source can change over time. For speed of writing, this is handled - * via a simple valid-from timestamp field. - */ -CREATE TABLE loc.sources_history ( - source_type_id SMALLINT NOT NULL - REFERENCES loc.source_types (source_type_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - -- Capacity cap, (for instance during curtailment or repair work), - -- encoded as a smallint percentage (sip) of the capacity; with 0 representing 0% - -- AND 30000 representing 100% of the capacity. However, since things are mostly - -- not limited, NULL indicates no limit, so 30000 is an invalid value. - -- NOTE: This is currently not used. - capacity_limit_sip SMALLINT DEFAULT NULL, - CONSTRAINT capacity_limit_sip_vaildity_check CHECK ( - capacity_limit_sip IS NULL - OR (capacity_limit_sip >= 0 AND capacity_limit_sip < 30000) - ), - -- Capacity in watts. This maxes out at ~9.22 petawatts, which should be sufficient - capacity_watts BIGINT NOT NULL, - CONSTRAINT capacity_nonnegative_check CHECK (capacity_watts >= 0), - valid_from_utc TIMESTAMP DEFAULT NOW() NOT NULL, - geometry_uuid UUID NOT NULL - REFERENCES loc.geometries (geometry_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - -- Metadata about the source, e.g. tilt, orientation, etc. - metadata JSONB DEFAULT NULL, - CONSTRAINT metadata_nonempty_check CHECK ( - metadata IS NULL OR metadata <> '{}'::JSONB -- Null is cheaper - ), - PRIMARY KEY (geometry_uuid, source_type_id, valid_from_utc) -); - -/* - * Materialized view to store the state of sources over time with a system period. - * This allows for quicker reads of the state of sources at a given time. - */ -CREATE MATERIALIZED VIEW loc.sources_mv AS -SELECT - sh.geometry_uuid, - sh.source_type_id, - sh.capacity_watts, - sh.capacity_limit_sip, - sh.metadata, - g.geometry_name, - g.geometry_type_id, - ST_X(g.associated_point)::REAL AS longitude, - ST_Y(g.associated_point)::REAL AS latitude, - TSRANGE( - sh.valid_from_utc, - LEAD(sh.valid_from_utc, 1) OVER ( - PARTITION BY sh.geometry_uuid, sh.source_type_id - ORDER BY sh.valid_from_utc - ) - ) AS sys_period -FROM loc.sources_history AS sh -INNER JOIN loc.geometries AS g USING (geometry_uuid); --- Prevent overlapping records. Required for concurrent refreshes. -CREATE UNIQUE INDEX ON loc.sources_mv (geometry_uuid, source_type_id, sys_period); -CREATE INDEX ON loc.sources_mv USING gist (sys_period); - - --- +goose Down -DROP SCHEMA loc CASCADE; -DROP EXTENSION IF EXISTS postgis_topology; -DROP EXTENSION IF EXISTS postgis; -DROP EXTENSION IF EXISTS btree_gist; diff --git a/internal/server/postgres/sql/migrations/00003_observations.sql b/internal/server/postgres/sql/migrations/00003_observations.sql deleted file mode 100644 index 0a2bc8d..0000000 --- a/internal/server/postgres/sql/migrations/00003_observations.sql +++ /dev/null @@ -1,101 +0,0 @@ --- +goose Up - -/* - * Schema and tables to handle observed generation data. - * - * Observations of generation data is usually measured by providers of inverters, which are - * required in many sources of renewable energy to convert power from DC to AC. Partnerships - * with these providers provide access to the data in order to test the accuracy of predictions. -*/ - -CREATE SCHEMA IF NOT EXISTS partman; -CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman; -CREATE EXTENSION IF NOT EXISTS pg_cron; - -CREATE SCHEMA obs; - -/*- Tables ----------------------------------------------------------------------------------*/ - -/* - * Table to store observers. - * These are providers of actual recorded generation values from inverters - * (mostly - looking at you, pvlive...) -*/ -CREATE TABLE obs.observers ( - observer_uuid UUID NOT NULL DEFAULT UUIDV7() NOT NULL, - observer_name TEXT NOT NULL, - CONSTRAINT observer_name_format_check CHECK ( - LENGTH(observer_name) > 0 AND LENGTH(observer_name) < 128 - AND observer_name = LOWER(observer_name) - ), - PRIMARY KEY (observer_uuid), - UNIQUE (observer_name) -); - -/* - * Table to store observed generation values. - * The generation value is stored as a percentage of the source capacity represented by a - * smallint percent (sip). Since it isn't impossible to measure a little over capacity, 30000 - * represents 100% of capacity instead of the max smallint value (32767). This allows for some - * measurement leeway. - * The table has native partitioning that can then be managed by pg_partman. Note that unique - * indexes will only work if they include the partition key. - */ -CREATE TABLE obs.observed_generation_values ( - value_sip SMALLINT NOT NULL, - CONSTRAINT value_sip_nonnegative_check CHECK (value_sip >= 0), - source_type_id SMALLINT NOT NULL - REFERENCES loc.source_types (source_type_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - observation_timestamp_utc TIMESTAMP NOT NULL, - CONSTRAINT observation_timestamp_utc_recency_check CHECK ( - observation_timestamp_utc <= CURRENT_TIMESTAMP + MAKE_INTERVAL(days => 31) - ), - observer_uuid UUID NOT NULL - REFERENCES obs.observers (observer_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - geometry_uuid UUID NOT NULL - REFERENCES loc.geometries (geometry_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY (geometry_uuid, source_type_id, observer_uuid, observation_timestamp_utc) -) -PARTITION BY RANGE (observation_timestamp_utc); - -/* - * Manage partitions with pg_partman. - * Highlights: - * - `retention_keep_table = true`: detach old partitions instead of dropping them - * - `infinite_time_partitions = true`: retain detached partitions indefinitely for processing - */ -SELECT partman.create_parent( - p_parent_table => 'obs.observed_generation_values', - p_control => 'observation_timestamp_utc', - p_type => 'range', - p_interval => '1 week', - p_automatic_maintenance => 'on', - p_jobmon => FALSE, - p_premake => 7 -); -UPDATE partman.part_config -SET - retention = '1 month', - retention_keep_table = TRUE, - retention_keep_index = FALSE, - infinite_time_partitions = TRUE -WHERE parent_table = 'obs.observed_generation_values'; -SELECT partman.run_maintenance('obs.observed_generation_values'); --- Schedule regular maintenance for the partitioned observed generation values table. -SELECT cron.schedule('partman-maintenance', '@hourly', $$CALL partman.run_maintenance_proc()$$); -SELECT cron.schedule('cron-details-cleanup', '0 12 * * *', $$DELETE FROM cron.job_run_details WHERE end_time < now() - interval '7 days'$$); - - --- +goose Down -SELECT cron.unschedule('partman-maintenance'); -DROP SCHEMA obs CASCADE; - -DROP EXTENSION IF EXISTS pg_cron CASCADE; -DROP EXTENSION IF EXISTS pg_partman CASCADE; -DROP SCHEMA IF EXISTS partman CASCADE; diff --git a/internal/server/postgres/sql/migrations/00004_predictions.sql b/internal/server/postgres/sql/migrations/00004_predictions.sql deleted file mode 100644 index 327c91b..0000000 --- a/internal/server/postgres/sql/migrations/00004_predictions.sql +++ /dev/null @@ -1,236 +0,0 @@ --- +goose Up - -/* - * Schema and tables to handle predicted generation data. - * - * Predicted of generation values are produced by various forecast models for a specific location. - * A forecast is a set of predicted generation values, beginning at the initialisation time. Each - * subsequent generation's target time is equivalent to the initialisation time plus the horizon. - * - * The forecast produced most recently will likely be the most accurate. - */ - -CREATE SCHEMA pred; - -/*- Functions -------------------------------------------------------------------------------*/ - -/* - * Check that all present values in a JSONB blob are valid forecaster statistic fractions. - * Valid statistic fractions are defined as numeric values between 0 and 1.1 (inclusive), - * with up to 4 digits for precision (this final constraint ensures that only 2 bytes are - * used to store each value). - */ --- +goose StatementBegin -CREATE FUNCTION pred.check_all_jsonb_values_are_valid_stat_fractions(stats_blob jsonb) -RETURNS boolean AS $$ -DECLARE - rec record; - val_num numeric; -BEGIN - IF stats_blob IS NULL THEN - RETURN true; - END IF; - - FOR rec IN SELECT key, value FROM jsonb_each(stats_blob) LOOP - IF LENGTH(rec.key) = 0 OR LENGTH(rec.key) > 64 THEN - RETURN false; - END IF; - IF rec.key <> LOWER(rec.key) THEN - RETURN false; - END IF; - - IF jsonb_typeof(rec.value) <> 'number' THEN - RETURN false; - END IF; - val_num := rec.value::numeric; - IF (val_num < 0 OR val_num > 1.1) - OR (val_num >= 0 AND val_num < 1 AND scale(val_num) > 4) - OR (val_num >= 1 AND val_num <= 1.1 AND scale(val_num) > 3) - THEN - RETURN false; - END IF; - - END LOOP; - RETURN true; - -EXCEPTION - WHEN others THEN - RETURN false; -END; -$$ LANGUAGE plpgsql IMMUTABLE; --- +goose StatementEnd - -/*- Tables ----------------------------------------------------------------------------------*/ - -/* - * A forecaster is a source that generates forecast values. This is usually an ML model, - * but could also be an analytical process. Each forecaster's name and version number uniquely - * identifies it. - */ -CREATE TABLE pred.forecasters ( - forecaster_id INTEGER GENERATED ALWAYS AS IDENTITY NOT NULL, - forecaster_name TEXT NOT NULL, - CONSTRAINT forecaster_name_format_check CHECK ( - LENGTH(forecaster_name) > 0 AND LENGTH(forecaster_name) < 64 - AND forecaster_name = LOWER(forecaster_name) - ), - forecaster_version TEXT NOT NULL, - CONSTRAINT forecaster_version_format_check CHECK ( - LENGTH(forecaster_version) > 0 AND LENGTH(forecaster_version) < 64 - AND forecaster_version = LOWER(forecaster_version) - ), - created_at_utc TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT created_at_utc_nonfuture_check CHECK (created_at_utc <= CURRENT_TIMESTAMP), - PRIMARY KEY (forecaster_id), - UNIQUE (forecaster_name, forecaster_version) -); - -/* - * Forecasts refer to the set of forecast values, created by a specific version of a forecaster, - * for a specific location, with some initialization time. Each forecast contains a timeseries of - * forecast values. There can only be one forecast per location per initialization time per - * forecaster; reruns should replace old values. - */ -CREATE TABLE pred.forecasts ( - source_type_id SMALLINT NOT NULL - REFERENCES loc.source_types (source_type_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - value_resolution_mins SMALLINT NOT NULL, - CONSTRAINT value_resolution_mins_size_check CHECK ( - value_resolution_mins > 0 AND value_resolution_mins <= 60 - ), - forecaster_id INTEGER NOT NULL - REFERENCES pred.forecasters (forecaster_id) - ON UPDATE CASCADE - ON DELETE CASCADE, - created_at_utc TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT created_at_utc_valid_check CHECK (created_at_utc <= CURRENT_TIMESTAMP), - init_time_utc TIMESTAMP NOT NULL, - CONSTRAINT init_time_utc_recency_check CHECK ( - init_time_utc >= '2000-01-01 00:00:00'::TIMESTAMP - AND init_time_utc < CURRENT_TIMESTAMP + MAKE_INTERVAL(days => 30) - ), - geometry_uuid UUID NOT NULL - REFERENCES loc.geometries (geometry_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - /* The forecast uuid should be generated using the init time as the time component */ - forecast_uuid UUID NOT NULL, - target_period TSRANGE NOT NULL, - CONSTRAINT target_period_valid_check CHECK ( - UPPER(target_period) > LOWER(target_period) - ), - CONSTRAINT target_period_recency_check CHECK ( - LOWER(target_period) >= '2000-01-01 00:00:00'::TIMESTAMP - AND UPPER(target_period) < CURRENT_TIMESTAMP + MAKE_INTERVAL(days => 30) - ), - metadata JSONB DEFAULT NULL, - PRIMARY KEY (forecast_uuid) -) -PARTITION BY RANGE (forecast_uuid); - -CREATE INDEX idx_forecasts_filter ON pred.forecasts ( - geometry_uuid, - source_type_id, - forecaster_id, - forecast_uuid DESC -) INCLUDE (target_period); - -/* - * Manage partitions with pg_partman. - * Highlights: - * - `retention_keep_table = true`: detach old partitions instead of dropping them - * - `infinite_time_partitions = true`: retain detached partitions indefinitely for processing - */ -SELECT partman.create_parent( - p_parent_table => 'pred.forecasts', - p_control => 'forecast_uuid', - p_type => 'range', - p_interval => '1 week', - p_automatic_maintenance => 'on', - p_jobmon => FALSE, - p_premake => 7, - p_time_encoder => 'partman.uuid7_time_encoder', - p_time_decoder => 'partman.uuid7_time_decoder' -); -SELECT partman.run_maintenance('pred.forecasts'); -SELECT cron.schedule('forecasts-vacuum', '30 4 * * *', $$VACUUM ANALYZE pred.forecasts$$); - -/* - * Table to store predicted generation values. - * Predicted generation values are the output of a forecast model. There can only be one predicted - * generation per forecast per horizon. This table gets very large very quickly, so to save space, - * data is stored as smallints where possible, and the columns are ordered to allow for efficient - * bit-packing. - * - * The p50 column is for the characteristic predicted generation confidence level value, recorded - * as a percentage of capacity (represented by a smallint percentage, "sip". this allows for - * efficient database calculation on the value for aggregation etc. Also, since it isn't impossible - * to predict a little over capacity, 30000 represents 100% of capacity intead of the max smallint - * value (32767). - * - * Any other forecaster outputs (such as other quantiles or averages) should be stored in the - * "other_stats_fractions" JSONB column, as fractions of capacity. This allows for flexibility in - * what other statistics are stored, without needing to modify the table structure. - * - * The horizon_mins column stores the number of minutes difference between the target_time_utc and - * the initialization time of the forecast. It is a more useful index for the kinds of query we - * care about, and enables determination of the init_time anyway. - * The table has native partitioning that can then be managed by pg_partman. Note that unique - * indexes will only work if they include the partition key. - */ -CREATE TABLE pred.predicted_generation_values ( - horizon_mins SMALLINT NOT NULL, - CONSTRAINT horizon_mins_nonnegative_check CHECK (horizon_mins >= 0), - CONSTRAINT horizon_mins_fiveminutely_check CHECK (horizon_mins % 5 = 0), - p50_sip SMALLINT NOT NULL, - CONSTRAINT p50_sip_nonnegative_check CHECK (p50_sip >= 0), - target_time_utc TIMESTAMP NOT NULL, - forecast_uuid UUID NOT NULL - REFERENCES pred.forecasts (forecast_uuid) - ON DELETE CASCADE - ON UPDATE CASCADE, - metadata JSONB DEFAULT NULL - CONSTRAINT metadata_nullifempty CHECK ( - metadata IS NULL OR metadata != '{}' - ), - other_stats_fractions JSONB DEFAULT NULL, - CONSTRAINT other_stats_nullifempty CHECK ( - other_stats_fractions IS NULL OR other_stats_fractions != '{}' - ), - CONSTRAINT other_stats_valid_fractions_check - CHECK (pred.check_all_jsonb_values_are_valid_stat_fractions(other_stats_fractions)), - PRIMARY KEY (forecast_uuid, target_time_utc) -) -PARTITION BY RANGE (forecast_uuid); - -/* - * Manage partitions with pg_partman. - * Highlights: - * - `retention_keep_table = true`: detach old partitions instead of dropping them - * - `infinite_time_partitions = true`: retain detached partitions indefinitely for processing - */ -SELECT partman.create_parent( - p_parent_table => 'pred.predicted_generation_values', - p_control => 'forecast_uuid', - p_type => 'range', - p_interval => '1 week', - p_automatic_maintenance => 'on', - p_jobmon => FALSE, - p_time_encoder => 'partman.uuid7_time_encoder', - p_time_decoder => 'partman.uuid7_time_decoder', - p_premake => 7 -); -UPDATE partman.part_config -SET - retention = '1 month', - retention_keep_table = TRUE, - retention_keep_index = TRUE, - infinite_time_partitions = TRUE -WHERE parent_table = 'pred.predicted_generation_values'; -SELECT partman.run_maintenance('pred.predicted_generation_values'); - --- +goose Down -DROP SCHEMA pred CASCADE; - diff --git a/internal/server/postgres/sql/migrations/00005_iam.sql b/internal/server/postgres/sql/migrations/00005_iam.sql deleted file mode 100644 index adea033..0000000 --- a/internal/server/postgres/sql/migrations/00005_iam.sql +++ /dev/null @@ -1,175 +0,0 @@ --- +goose Up - -/* - * Schema and tables to handle access management data. - * - * This schema isn't for storing any personally identifiable information; rather for detailing - * permissions and policies for user tokens and resources in the database. - * - * Permissions are stored in a lookup table, and are used to determine the allowable - * actions a user can take on a resource. These permissions are then applied to users and - * resources via policies. These policies are simply matchings between service accounts, - * resource ids, and permissions. - */ - -CREATE SCHEMA iam; - -/*- Lookups --------------------------------------------------------------------------------------*/ - --- Lookup table to store the possible permissions -CREATE TABLE iam.permissions ( - permission_id SMALLINT GENERATED ALWAYS AS IDENTITY NOT NULL, - permission_name TEXT NOT NULL, - CONSTRAINT permission_name_format_check CHECK ( - LENGTH(permission_name) > 0 - AND LENGTH(permission_name) <= 64 - AND permission_name = LOWER(permission_name) - ), - PRIMARY KEY (permission_id), - UNIQUE (permission_name) -); --- The ordering of these permissions matches the .proto enum definitions. Change with caution! -INSERT INTO iam.permissions (permission_name) VALUES ('read'), ('write'); - -/*- Tables --------------------------------------------------------------------------------------*/ - -/* - * Table to store organizations. - * An organization is a logical grouping of users. A user can only belong to one organisation. - */ -CREATE TABLE iam.orgs ( - org_uuid UUID DEFAULT UUIDV7() NOT NULL, - org_name TEXT NOT NULL, - CONSTRAINT org_name_format_check CHECK ( - LENGTH(org_name) > 0 - AND LENGTH(org_name) <= 128 - AND org_name = LOWER(org_name) - ), - metadata JSONB DEFAULT NULL, - PRIMARY KEY (org_uuid), - UNIQUE (org_name) -); - -/* - * Table to store users. - * A user is identified by their oauth_id, which is a unique identifier served by the OAuth provider. - * The oauth_id is not personally identifiable information, nor is it the primary key. - * A user belongs to one organization, which defines their access policies. - */ -CREATE TABLE iam.users ( - user_uuid UUID DEFAULT UUIDV7() NOT NULL, - org_uuid UUID NOT NULL - REFERENCES iam.orgs (org_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - oauth_id TEXT NOT NULL, - CONSTRAINT oauth_id_format_check CHECK ( - LENGTH(oauth_id) > 0 - AND LENGTH(oauth_id) <= 128 - ), - metadata JSONB DEFAULT NULL, - PRIMARY KEY (user_uuid), - UNIQUE (oauth_id) -); -CREATE INDEX ON iam.users (org_uuid); - -/* - * Table to store logical groups of location policies. - * This allows for easier assignment of the same set of policies to multiple orgs. - */ -CREATE TABLE iam.location_policy_groups ( - location_policy_group_uuid UUID DEFAULT UUIDV7() NOT NULL, - location_policy_group_name TEXT NOT NULL, - CONSTRAINT location_policy_group_name_format_check CHECK ( - LENGTH(location_policy_group_name) > 0 - AND LENGTH(location_policy_group_name) <= 128 - AND location_policy_group_name = LOWER(location_policy_group_name) - ), - PRIMARY KEY (location_policy_group_uuid), - UNIQUE (location_policy_group_name) -); - -/* - * Pivot table to link orgs to location policy groups. - * An org can belong to multiple location policy groups. - */ -CREATE TABLE iam.org_location_policy_groups ( - org_uuid UUID NOT NULL - REFERENCES iam.orgs (org_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - location_policy_group_uuid UUID NOT NULL - REFERENCES iam.location_policy_groups (location_policy_group_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY (org_uuid, location_policy_group_uuid) -); - -/* - * Pivot table to define location policies. - * These policies match locations to permissions, and each policy is linked to a location group. - * A location group can only have one permission per location and source (can't be an OWNER *and* a - * VIEWER for UK solar, for instance). - * A location is a combination of a geometry and a source type. - */ -CREATE TABLE iam.location_policies ( - permission_id SMALLINT NOT NULL - REFERENCES iam.permissions (permission_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - source_type_id SMALLINT NOT NULL - REFERENCES loc.source_types (source_type_id) - ON UPDATE CASCADE - ON DELETE RESTRICT, - geometry_uuid UUID NOT NULL - REFERENCES loc.geometries (geometry_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - location_policy_group_uuid UUID NOT NULL - REFERENCES iam.location_policy_groups (location_policy_group_uuid) - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY (location_policy_group_uuid, geometry_uuid, source_type_id, permission_id), - UNIQUE (location_policy_group_uuid, geometry_uuid, source_type_id) -); - -/*- Views ---------------------------------------------------------------------------------------*/ - -/* - * View that presents org details in an aggregated format. - */ -CREATE OR REPLACE VIEW iam.org_details_v AS -WITH aggregated_policies AS ( - SELECT - olpg.org_uuid, - ARRAY_AGG(olpg.location_policy_group_uuid)::UUID[] AS location_policy_group_uuids, - ARRAY_AGG(lpg.location_policy_group_name)::TEXT[] AS location_policy_group_names - FROM iam.org_location_policy_groups AS olpg - INNER JOIN iam.location_policy_groups AS lpg USING (location_policy_group_uuid) - GROUP BY olpg.org_uuid -), -aggregated_users AS ( - SELECT - u.org_uuid, - ARRAY_AGG(u.user_uuid)::UUID[] AS user_uuids, - ARRAY_AGG(u.oauth_id)::TEXT[] AS oauth_ids - FROM iam.users AS u - GROUP BY u.org_uuid -) -SELECT - o.org_uuid, - o.org_name, - UUIDV7_EXTRACT_TIMESTAMP(o.org_uuid)::TIMESTAMP AS created_at_utc, - o.metadata, - ap.location_policy_group_uuids, - ap.location_policy_group_names, - au.user_uuids, - au.oauth_ids -FROM iam.orgs AS o - LEFT JOIN aggregated_policies AS ap USING (org_uuid) - LEFT JOIN aggregated_users AS au USING (org_uuid) -ORDER BY o.org_name; - - --- +goose Down -DROP SCHEMA iam CASCADE; diff --git a/internal/server/postgres/sql/migrations/00006_sources_mv_index.sql b/internal/server/postgres/sql/migrations/00006_sources_mv_index.sql deleted file mode 100644 index e2ca59b..0000000 --- a/internal/server/postgres/sql/migrations/00006_sources_mv_index.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up - --- Replace the old materialized view index for one suited to location-specific lookups -DROP INDEX IF EXISTS sources_mv_sys_period_idx; -CREATE INDEX idx_sources_mv_composite_gist ON loc.sources_mv USING gist (geometry_uuid, source_type_id, sys_period); - --- +goose Down -DROP INDEX IF EXISTS idx_sources_mv_composite_gist; -CREATE INDEX sources_mv_sys_period_idx ON loc.sources_mv USING gist (sys_period); diff --git a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql deleted file mode 100644 index 0f708ab..0000000 --- a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql +++ /dev/null @@ -1,110 +0,0 @@ --- +goose Up - -/* - * Removes the month-long retention policy on partman-managed partitions. - * - * This allows for querying of all historical data without needing custom queries. However, some - * partitions may have already been detached by the previous retention policy, so they must also - * be re-attached to the parent table. This is done by iterating through all existing partitions, - * extracting the date from their name, and attaching them with the appropriate range values. - */ - --- +goose StatementBegin -DO $$ -DECLARE - partition_record RECORD; - start_time TIMESTAMP; - end_time TIMESTAMP; - attach_sql TEXT; -BEGIN - UPDATE partman.part_config - SET retention = NULL - WHERE parent_table = 'obs.observed_generation_values'; - - FOR partition_record IN - SELECT - table_schema || '.' || table_name AS full_table_name, - SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str - FROM information_schema.tables - WHERE table_schema = 'obs' - AND table_name LIKE 'observed_generation_values_p________' - AND NOT EXISTS ( - SELECT 1 FROM pg_inherits - WHERE inhrelid = (table_schema || '.' || table_name)::regclass - AND inhparent = 'obs.observed_generation_values'::regclass - ) - LOOP - start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD'); - end_time := start_time + INTERVAL '7 days'; - attach_sql := format( - 'ALTER TABLE obs.observed_generation_values ATTACH PARTITION %s FOR VALUES FROM (%L::TIMESTAMP) TO (%L::TIMESTAMP);', - partition_record.full_table_name, - start_time, - end_time - ); - RAISE NOTICE 'Executing: %', attach_sql; - EXECUTE attach_sql; - END LOOP; -END $$; --- +goose StatementEnd - --- +goose StatementBegin -DO $$ -DECLARE - target_table TEXT; - parent_table_name TEXT; - partition_pattern TEXT; - partition_record RECORD; - start_time TIMESTAMPTZ; - end_time TIMESTAMPTZ; - attach_sql TEXT; -BEGIN - FOREACH target_table IN ARRAY ARRAY['forecasts', 'predicted_generation_values'] - LOOP - parent_table_name := 'pred.' || target_table; - partition_pattern := target_table || '_p________'; - - UPDATE partman.part_config - SET retention = NULL - WHERE parent_table = parent_table_name; - - FOR partition_record IN - SELECT - table_schema || '.' || table_name AS full_table_name, - SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str - FROM information_schema.tables - WHERE table_schema = 'pred' - AND table_name LIKE partition_pattern - AND NOT EXISTS ( - SELECT 1 FROM pg_inherits - WHERE inhrelid = (table_schema || '.' || table_name)::regclass - AND inhparent = parent_table_name::regclass - ) - LOOP - start_time := to_date(partition_record.date_str, 'YYYYMMDD')::TIMESTAMP AT TIME ZONE 'UTC'; - end_time := start_time + INTERVAL '7 days'; - - attach_sql := format( - 'ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (partman.uuid7_time_encoder(%L::TIMESTAMPTZ)) TO (partman.uuid7_time_encoder(%L::TIMESTAMPTZ));', - parent_table_name, - partition_record.full_table_name, - start_time, - end_time - ); - - RAISE NOTICE 'Executing: %', attach_sql; - EXECUTE attach_sql; - END LOOP; - END LOOP; -END $$; --- +goose StatementEnd - --- +goose Down - -UPDATE partman.part_config -SET retention = '1 month' -WHERE parent_table = 'obs.observed_generation_values'; - -UPDATE partman.part_config -SET retention = '1 month' -WHERE parent_table IN ('pred.forecasts', 'pred.predicted_generation_values'); diff --git a/internal/server/postgres/sql/migrations/00008_simple_iam.sql b/internal/server/postgres/sql/migrations/00008_simple_iam.sql deleted file mode 100644 index 65da778..0000000 --- a/internal/server/postgres/sql/migrations/00008_simple_iam.sql +++ /dev/null @@ -1,93 +0,0 @@ --- +goose Up - -/* - * Removes most of the functionality of the IAM setup. - * - * This is due to a new relience on an external system for user management. The only remaining - * element relevant to the data platform is the organistion object. - * - * Note that the dropping of the IAM schema is permanent and not restored by the migration down. - * It is empty in all deployed version of this application and so can and should be safely removed. - */ - -DROP SCHEMA iam CASCADE; - -CREATE TABLE IF NOT EXISTS loc.entities ( - entity_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - external_id TEXT NOT NULL - CONSTRAINT external_id_format_check CHECK ( - external_id IS NOT NULL - AND LENGTH(external_id) > 0 - AND LENGTH(external_id) <= 128 - ), - UNIQUE (external_id) -); - -ALTER TABLE loc.geometries - ADD COLUMN owning_entity_id INTEGER DEFAULT NULL - REFERENCES loc.entities(entity_id) - ON UPDATE CASCADE - ON DELETE SET NULL; - -CREATE INDEX idx_owning_entity_id ON loc.geometries (owning_entity_id); - -DROP MATERIALIZED VIEW IF EXISTS loc.sources_mv; -CREATE MATERIALIZED VIEW loc.sources_mv AS -SELECT - sh.geometry_uuid, - sh.source_type_id, - sh.capacity_watts, - sh.capacity_limit_sip, - sh.metadata, - COALESCE(sh.metadata || g.metadata, sh.metadata, g.metadata)::JSONB AS metadata_jsonb, - g.geometry_name, - g.geometry_type_id, - g.owning_entity_id, - ST_X(g.associated_point)::REAL AS longitude, - ST_Y(g.associated_point)::REAL AS latitude, - TSRANGE( - sh.valid_from_utc, - LEAD(sh.valid_from_utc, 1) OVER ( - PARTITION BY sh.geometry_uuid, sh.source_type_id - ORDER BY sh.valid_from_utc - ) - ) AS sys_period -FROM loc.sources_history AS sh -INNER JOIN loc.geometries AS g USING (geometry_uuid); --- Prevent overlapping records. Required for concurrent refreshes. -CREATE UNIQUE INDEX ON loc.sources_mv (geometry_uuid, source_type_id, sys_period); -CREATE INDEX idx_sources_mv_gist_sys_period ON loc.sources_mv USING gist (sys_period); -CREATE INDEX idx_sources_mv_owning_entity_id ON loc.sources_mv (owning_entity_id); - - --- +goose Down -DROP MATERIALIZED VIEW IF EXISTS loc.sources_mv; - -ALTER TABLE loc.geometries - DROP COLUMN owning_entity_id; - -DROP TABLE IF EXISTS loc.entities; - -CREATE MATERIALIZED VIEW loc.sources_mv AS -SELECT - sh.geometry_uuid, - sh.source_type_id, - sh.capacity_watts, - sh.capacity_limit_sip, - sh.metadata, - g.geometry_name, - g.geometry_type_id, - ST_X(g.associated_point)::REAL AS longitude, - ST_Y(g.associated_point)::REAL AS latitude, - TSRANGE( - sh.valid_from_utc, - LEAD(sh.valid_from_utc, 1) OVER ( - PARTITION BY sh.geometry_uuid, sh.source_type_id - ORDER BY sh.valid_from_utc - ) - ) AS sys_period -FROM loc.sources_history AS sh -INNER JOIN loc.geometries AS g USING (geometry_uuid); -CREATE UNIQUE INDEX ON loc.sources_mv (geometry_uuid, source_type_id, sys_period); -CREATE INDEX ON loc.sources_mv USING gist (sys_period); - diff --git a/internal/server/postgres/sql/migrations/00009_optimize_storage.sql b/internal/server/postgres/sql/migrations/00009_optimize_storage.sql deleted file mode 100644 index 57042db..0000000 --- a/internal/server/postgres/sql/migrations/00009_optimize_storage.sql +++ /dev/null @@ -1,214 +0,0 @@ --- +goose NO TRANSACTION --- +goose Up - -/* - * Reduces the database size by approximately 40%. - * - * Modifies the predicted_generation_values table to optimize it's use of storage. - * This is done through changing the index, removing redundant columns, and replacing - * dynamic columns with small static ones. - * - * The schema modifications and the corresponding data changes are seperated out for - * faster migration. Since the predicted_generation_values table is very large, simple - * DELETES and UPDATES would take a long time, and not actually gain us any storage - * savings (at least until an autovacuum process ran). By instead moving the data - * partition-wise and then replacing the partitions, we keep the process light on CPU. - */ - -DROP INDEX IF EXISTS loc.idx_sources_mv_gist_sys_period; - -CREATE INDEX IF NOT EXISTS idx_sources_mv_composite_lookup -ON loc.sources_mv USING gist (geometry_uuid, source_type_id, sys_period); - --- +goose StatementBegin -DO $$ -DECLARE - pk_name TEXT; -BEGIN - SELECT conname INTO pk_name - FROM pg_constraint - WHERE conrelid = 'pred.predicted_generation_values'::regclass - AND contype = 'p'; - - IF pk_name IS NOT NULL THEN - EXECUTE format('ALTER TABLE pred.predicted_generation_values DROP CONSTRAINT %I CASCADE;', pk_name); - END IF; -END $$; --- +goose StatementEnd - -ALTER TABLE pred.predicted_generation_values - ADD COLUMN IF NOT EXISTS p10_sip SMALLINT, - DROP CONSTRAINT IF EXISTS p10_sip_nonnegative_check, - ADD CONSTRAINT p10_sip_nonnegative_check CHECK (p10_sip >= 0) NOT VALID, - ADD COLUMN IF NOT EXISTS p90_sip SMALLINT, - DROP CONSTRAINT IF EXISTS p90_sip_nonnegative_check, - ADD CONSTRAINT p90_sip_nonnegative_check CHECK (p90_sip >= 0) NOT VALID, - ALTER COLUMN target_time_utc DROP NOT NULL; - --- +goose StatementBegin -CREATE OR REPLACE PROCEDURE pred.swap_predicted_generation_partitions() -LANGUAGE plpgsql -AS $$ -DECLARE - partition_record RECORD; - new_part_name TEXT; - part_bound TEXT; - is_migrated BOOLEAN; -BEGIN - FOR partition_record IN - SELECT child.relname AS table_name, - pg_get_expr(child.relpartbound, child.oid) AS bounds - FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid - JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace - WHERE parent.relname = 'predicted_generation_values' - AND nmsp_parent.nspname = 'pred' - LOOP - EXECUTE format(' - SELECT EXISTS ( - SELECT 1 - FROM pg_index - JOIN pg_class ON pg_index.indrelid = pg_class.oid - JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - WHERE pg_namespace.nspname = ''pred'' - AND pg_class.relname = %L - AND pg_index.indisprimary - )', partition_record.table_name) - INTO is_migrated; - - IF is_migrated THEN - RAISE NOTICE 'Skipping partition (already migrated): %', partition_record.table_name; - CONTINUE; - END IF; - - RAISE NOTICE 'Migrating partition: %', partition_record.table_name; - - new_part_name := partition_record.table_name || '_v2'; - part_bound := partition_record.bounds; - - EXECUTE format('CREATE TABLE pred.%I (LIKE pred.predicted_generation_values INCLUDING ALL);', new_part_name); - - EXECUTE format(' - INSERT INTO pred.%I (horizon_mins, p50_sip, p10_sip, p90_sip, forecast_uuid, target_time_utc, metadata, other_stats_fractions) - SELECT - horizon_mins, - p50_sip, - CASE - WHEN other_stats_fractions IS NULL THEN p10_sip - ELSE LEAST(((other_stats_fractions->>''p10'')::REAL * 30000), 32767)::SMALLINT - END, - CASE - WHEN other_stats_fractions IS NULL THEN p90_sip - ELSE LEAST(((other_stats_fractions->>''p90'')::REAL * 30000), 32767)::SMALLINT - END, - forecast_uuid, - NULL, - NULL, - NULL - FROM pred.%I; - ', new_part_name, partition_record.table_name); - - EXECUTE format('ALTER TABLE pred.%I ADD PRIMARY KEY (forecast_uuid, horizon_mins);', new_part_name); - EXECUTE format('ALTER TABLE pred.predicted_generation_values DETACH PARTITION pred.%I;', partition_record.table_name); - EXECUTE format('ALTER TABLE pred.predicted_generation_values ATTACH PARTITION pred.%I %s;', new_part_name, part_bound); - EXECUTE format('DROP TABLE pred.%I;', partition_record.table_name); - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I;', new_part_name, partition_record.table_name); - - COMMIT; - END LOOP; -END; -$$; --- +goose StatementEnd - -CALL pred.swap_predicted_generation_partitions(); -DROP PROCEDURE pred.swap_predicted_generation_partitions; - -ALTER TABLE pred.predicted_generation_values - DROP COLUMN target_time_utc, - DROP COLUMN other_stats_fractions, - DROP COLUMN metadata, - ADD PRIMARY KEY (forecast_uuid, horizon_mins); - -DROP TABLE IF EXISTS pred.predicted_generation_values_template; -CREATE TABLE pred.predicted_generation_values_template ( - horizon_mins SMALLINT NOT NULL, - CONSTRAINT horizon_mins_nonnegative_check CHECK (horizon_mins >= 0), - CONSTRAINT horizon_mins_fiveminutely_check CHECK (horizon_mins % 5 = 0), - p50_sip SMALLINT NOT NULL, - CONSTRAINT p50_sip_nonnegative_check CHECK (p50_sip >= 0), - p10_sip SMALLINT, - p90_sip SMALLINT, - forecast_uuid UUID NOT NULL REFERENCES pred.forecasts (forecast_uuid) ON DELETE CASCADE ON UPDATE CASCADE, - PRIMARY KEY (forecast_uuid, horizon_mins) -); - -ANALYZE pred.predicted_generation_values; - - --- +goose Down -ALTER TABLE pred.predicted_generation_values - DROP CONSTRAINT predicted_generation_values_pkey CASCADE, - ADD COLUMN target_time_utc TIMESTAMP, - ADD COLUMN other_stats_fractions JSONB DEFAULT NULL, - ADD CONSTRAINT other_stats_nullifempty CHECK (other_stats_fractions IS NULL OR other_stats_fractions != '{}'), - ADD CONSTRAINT other_stats_valid_fractions_check CHECK (pred.check_all_jsonb_values_are_valid_stat_fractions(other_stats_fractions)), - ADD COLUMN metadata JSONB DEFAULT NULL; - - --- +goose StatementBegin -CREATE OR REPLACE PROCEDURE pred.rollback_predicted_generation_partitions() -LANGUAGE plpgsql -AS $$ -DECLARE - partition_record RECORD; -BEGIN - FOR partition_record IN - SELECT child.relname AS table_name - FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid - JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace - WHERE parent.relname = 'predicted_generation_values' - AND nmsp_parent.nspname = 'pred' - LOOP - EXECUTE format(' - UPDATE pred.%I - SET target_time_utc = UUIDV7_EXTRACT_TIMESTAMP(forecast_uuid)::TIMESTAMP + MAKE_INTERVAL(mins => horizon_mins::INTEGER), - other_stats_fractions = CASE WHEN p10_sip IS NOT NULL OR p90_sip IS NOT NULL THEN jsonb_strip_nulls(jsonb_build_object(''p10'', p10_sip::REAL / 30000, ''p90'', p90_sip::REAL / 30000)) - ELSE NULL END; - ', partition_record.table_name); - - EXECUTE format('ALTER TABLE pred.%I ALTER COLUMN target_time_utc SET NOT NULL;', partition_record.table_name); - EXECUTE format('ALTER TABLE pred.%I ADD PRIMARY KEY (forecast_uuid, target_time_utc);', partition_record.table_name); - - COMMIT; - END LOOP; -END; -$$; --- +goose StatementEnd - -CALL pred.rollback_predicted_generation_partitions(); -DROP PROCEDURE pred.rollback_predicted_generation_partitions; - -ALTER TABLE pred.predicted_generation_values - ALTER COLUMN target_time_utc SET NOT NULL, - ADD PRIMARY KEY (forecast_uuid, target_time_utc), - DROP COLUMN p10_sip, DROP COLUMN p90_sip; - -DROP TABLE IF EXISTS pred.predicted_generation_values_template; -CREATE TABLE pred.predicted_generation_values_template ( - horizon_mins SMALLINT NOT NULL, - CONSTRAINT horizon_mins_nonnegative_check CHECK (horizon_mins >= 0), - CONSTRAINT horizon_mins_fiveminutely_check CHECK (horizon_mins % 5 = 0), - p50_sip SMALLINT NOT NULL, - CONSTRAINT p50_sip_nonnegative_check CHECK (p50_sip >= 0), - target_time_utc TIMESTAMP NOT NULL, - forecast_uuid UUID NOT NULL REFERENCES pred.forecasts (forecast_uuid) ON DELETE CASCADE ON UPDATE CASCADE, - metadata JSONB DEFAULT NULL CONSTRAINT metadata_nullifempty CHECK (metadata IS NULL OR metadata != '{}'), - other_stats_fractions JSONB DEFAULT NULL CONSTRAINT other_stats_nullifempty CHECK (other_stats_fractions IS NULL OR other_stats_fractions != '{}'), - CONSTRAINT other_stats_valid_fractions_check CHECK (pred.check_all_jsonb_values_are_valid_stat_fractions(other_stats_fractions)), - PRIMARY KEY (forecast_uuid, target_time_utc) -); - -ANALYZE pred.predicted_generation_values; diff --git a/internal/server/postgres/sql/migrations/00010_7_plevels.sql b/internal/server/postgres/sql/migrations/00010_7_plevels.sql deleted file mode 100644 index fce4033..0000000 --- a/internal/server/postgres/sql/migrations/00010_7_plevels.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose Up - -/* - * Adds 4 new plevels to the predicted_generation_values table. - */ - -ALTER TABLE pred.predicted_generation_values - ADD COLUMN p02_sip SMALLINT, - ADD CONSTRAINT p02_sip_nonnegative_check CHECK (p02_sip >= 0), - ADD COLUMN p98_sip SMALLINT, - ADD CONSTRAINT p98_sip_nonnegative_check CHECK (p98_sip >= 0), - ADD COLUMN p25_sip SMALLINT, - ADD CONSTRAINT p25_sip_nonnegative_check CHECK (p25_sip >= 0), - ADD COLUMN p75_sip SMALLINT, - ADD CONSTRAINT p75_sip_nonnegative_check CHECK (p75_sip >= 0); - --- +goose Down -ALTER TABLE pred.predicted_generation_values - DROP COLUMN p02_sip, - DROP COLUMN p25_sip, - DROP COLUMN p75_sip, - DROP COLUMN p98_sip; diff --git a/internal/server/postgres/sql/migrations/00011_forecast_value_arrays.sql b/internal/server/postgres/sql/migrations/00011_forecast_value_arrays.sql deleted file mode 100644 index 2aad77c..0000000 --- a/internal/server/postgres/sql/migrations/00011_forecast_value_arrays.sql +++ /dev/null @@ -1,75 +0,0 @@ --- +goose Up -DROP FUNCTION IF EXISTS uuidv7_extract_timestamp(UUID); - --- +goose StatementBegin -CREATE FUNCTION uuidv7_extract_timestamp(u UUID) RETURNS TIMESTAMP - LANGUAGE sql - IMMUTABLE STRICT PARALLEL SAFE - RETURN uuid_extract_timestamp(u) AT TIME ZONE 'UTC'; --- +goose StatementEnd - -/* - * Moves predicted values from separate table into arrays. - * - * Array index i (1-based) corresponds to target time: - * target_time = LOWER(target_period) + (i - 1) * value_resolution_mins - * Only works if a forecast has evenly spaced target times. - */ - -ALTER TABLE pred.forecasts - ADD COLUMN p02_sips SMALLINT [], - ADD COLUMN p10_sips SMALLINT [], - ADD COLUMN p25_sips SMALLINT [], - ADD COLUMN p50_sips SMALLINT [], - ADD COLUMN p75_sips SMALLINT [], - ADD COLUMN p90_sips SMALLINT [], - ADD COLUMN p98_sips SMALLINT []; - -ALTER TABLE pred.forecasts - ADD CONSTRAINT plevel_lengths_match_check CHECK ( - p50_sips IS NULL OR ( - ARRAY_LENGTH(p50_sips, 1) > 0 - AND COALESCE(ARRAY_LENGTH(p02_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p10_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p25_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p75_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p90_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p98_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - ) - ) NOT VALID; - -/* - * I want init time to be VIRTUAL, but sqlc doesn't support it yet. - * See https://github.com/sqlc-dev/sqlc/issues/4322. Until then it stays a plain NOT NULL - * column written by the application, so a rebuild that forgets to carry it over fails loudly - * rather than silently nulling the column every read query derives horizons from. - * - * The recency check is dropped: it referenced CURRENT_TIMESTAMP, which is not - * immutable, so it could not be revalidated and breaks ATTACH PARTITION. - */ -ALTER TABLE pred.forecasts - DROP CONSTRAINT IF EXISTS init_time_utc_recency_check; - --- +goose Down -ALTER TABLE pred.forecasts - ADD CONSTRAINT init_time_utc_recency_check CHECK ( - init_time_utc >= '2000-01-01 00:00:00'::TIMESTAMP - AND init_time_utc < CURRENT_TIMESTAMP + MAKE_INTERVAL(days => 30) - ) NOT VALID; - -ALTER TABLE pred.forecasts - DROP CONSTRAINT IF EXISTS plevel_lengths_match_check, - DROP COLUMN p02_sips, DROP COLUMN p10_sips, DROP COLUMN p25_sips, - DROP COLUMN p50_sips, DROP COLUMN p75_sips, DROP COLUMN p90_sips, - DROP COLUMN p98_sips; - -DROP FUNCTION IF EXISTS uuidv7_extract_timestamp(UUID); - --- +goose StatementBegin -CREATE FUNCTION uuidv7_extract_timestamp(UUID) RETURNS TIMESTAMPTZ -AS $$ - SELECT to_timestamp( - right(substring(uuid_send($1) from 1 for 6)::text, -1)::bit(48)::int8 - /1000.0); -$$ LANGUAGE sql immutable strict parallel safe; --- +goose StatementEnd diff --git a/internal/server/postgres/sql/migrations/00012_rebuild_forecast_partitions.sql b/internal/server/postgres/sql/migrations/00012_rebuild_forecast_partitions.sql deleted file mode 100644 index 7c7fed1..0000000 --- a/internal/server/postgres/sql/migrations/00012_rebuild_forecast_partitions.sql +++ /dev/null @@ -1,211 +0,0 @@ --- +goose Up - --- +goose StatementBegin -/* - * Rebuilds one pred.forecasts partition with its values folded into arrays, and its rows - * physically ordered to match idx_forecasts_filter. - * - * A rebuild rather than an UPDATE, because rows grow ~3.6x: an in-place update cannot keep the - * new tuple on its page, so every row would be a non-HOT update leaving a dead tuple and a new - * entry in every index. Rebuilding also lets us choose physical order for free, and packs - * indexes at full density. - * - * Ordering by (geometry_uuid, source_type_id, forecaster_id, forecast_uuid DESC) makes one - * location's forecasts contiguous and matches idx_forecasts_filter, so an index scan walks the - * heap in physical order. Every hot-path query filters on geometry_uuid first, so nothing loses. - * StreamForecastData is the only broad scan and is explicitly rare. - * - * The aggregation is chunked into an unlogged staging table and committed per chunk, so a week's - * worth of values is never sorted in one go. - * - * Forecasts with no rows in the values partition are dropped: the INNER JOIN against staging - * excludes them, and the row count check below is written to expect that. - * - * pred.predicted_generation_values carries a foreign key to pred.forecasts, and PostgreSQL - * refuses to detach a partition that is still referenced. The matching values partition is - * therefore detached and retired in the same transaction as the swap - which is the correct - * coupling anyway, since once a week's forecasts hold arrays its value rows are dead. It is - * renamed rather than dropped so the rebuild stays verifiable and reversible. The parent's - * foreign key is left intact for every partition that has not yet been rebuilt. - * - * Retired tables are left on disk as pred.predicted_generation_values_pXXXXXXXX_retired. They - * are no longer partitions, so the cleanup deployment's DROP TABLE will not remove them - drop - * them explicitly once the rebuild has been verified. - * - * This must be driven one partition at a time rather than looped unattended: DETACH and ATTACH - * each take a brief ACCESS EXCLUSIVE lock on pred.forecasts, and ATTACH validates the partition - * bound and the foreign keys. - * - * CALL pred.rebuild_forecast_partition('forecasts_p20260803') - */ -CREATE OR REPLACE PROCEDURE pred.rebuild_forecast_partition( - p_partition TEXT, - p_chunk INTERVAL DEFAULT INTERVAL '1 hour', - p_work_mem TEXT DEFAULT '256MB' -) -LANGUAGE plpgsql AS $$ -DECLARE - v_values TEXT; - v_fk TEXT; - v_new TEXT := p_partition || '_v2'; - v_bounds TEXT; - v_lo TIMESTAMP; - v_hi TIMESTAMP; - v_t TIMESTAMP; - v_n BIGINT; - v_total BIGINT := 0; - v_all BIGINT; - v_src BIGINT; - v_dst BIGINT; - v_started TIMESTAMPTZ := clock_timestamp(); -BEGIN - SELECT pg_get_expr(c.relpartbound, c.oid) INTO v_bounds - FROM pg_class AS c INNER JOIN pg_namespace AS n ON n.oid = c.relnamespace - WHERE n.nspname = 'pred' AND c.relname = p_partition; - - IF v_bounds IS NULL THEN - RAISE EXCEPTION 'not an attached partition of pred.forecasts: pred.%', p_partition; - END IF; - - /* pg_partman names siblings _p, so the values partition covering the same - * uuid range differs only in its prefix. */ - v_values := 'predicted_generation_values_' || substring(p_partition FROM '^forecasts_(p.+)$'); - - IF v_values IS NULL OR to_regclass('pred.' || quote_ident(v_values)) IS NULL THEN - RAISE EXCEPTION 'no values partition matching pred.%: expected pred.%', - p_partition, v_values; - END IF; - - v_lo := partman.uuid7_time_decoder( - (regexp_match(v_bounds, $re$FROM \('([^']+)'\)$re$))[1]::UUID::TEXT) AT TIME ZONE 'UTC'; - v_hi := partman.uuid7_time_decoder( - (regexp_match(v_bounds, $re$TO \('([^']+)'\)$re$))[1]::UUID::TEXT) AT TIME ZONE 'UTC'; - - RAISE NOTICE 'pred.% covers % .. % (% chunks of %)', - p_partition, v_lo, v_hi, - CEIL(EXTRACT(EPOCH FROM (v_hi - v_lo)) / EXTRACT(EPOCH FROM p_chunk)), p_chunk; - - CREATE UNLOGGED TABLE IF NOT EXISTS pred.fc_staging ( - forecast_uuid UUID PRIMARY KEY, - p02_sips SMALLINT [], p10_sips SMALLINT [], p25_sips SMALLINT [], - p50_sips SMALLINT [], p75_sips SMALLINT [], p90_sips SMALLINT [], - p98_sips SMALLINT [] - ); - - /* An earlier run that failed after filling would otherwise leave rows behind that the - * ON CONFLICT DO NOTHING below would silently keep. */ - TRUNCATE pred.fc_staging; - - v_t := v_lo; - WHILE v_t < v_hi LOOP - /* The CASE WHEN bool_or(...) guard keeps an unused p-level as a NULL array rather than a - * materialised array of nulls: ~3 bytes per forecast against ~130. */ - EXECUTE format($q$ - INSERT INTO pred.fc_staging ( - forecast_uuid, p02_sips, p10_sips, p25_sips, - p50_sips, p75_sips, p90_sips, p98_sips) - SELECT - forecast_uuid, - CASE WHEN bool_or(p02_sip IS NOT NULL) THEN array_agg(p02_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p10_sip IS NOT NULL) THEN array_agg(p10_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p25_sip IS NOT NULL) THEN array_agg(p25_sip ORDER BY horizon_mins) END, - array_agg(p50_sip ORDER BY horizon_mins), - CASE WHEN bool_or(p75_sip IS NOT NULL) THEN array_agg(p75_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p90_sip IS NOT NULL) THEN array_agg(p90_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p98_sip IS NOT NULL) THEN array_agg(p98_sip ORDER BY horizon_mins) END - FROM pred.%I - WHERE forecast_uuid >= uuidv7_boundary(%L::TIMESTAMP AT TIME ZONE 'UTC') - AND forecast_uuid < uuidv7_boundary(%L::TIMESTAMP AT TIME ZONE 'UTC') - GROUP BY forecast_uuid - ON CONFLICT (forecast_uuid) DO NOTHING - $q$, v_values, v_t, v_t + p_chunk); - - GET DIAGNOSTICS v_n = ROW_COUNT; - v_total := v_total + v_n; - - COMMIT; - - RAISE NOTICE '% .. % +% (total %, elapsed %)', - v_t, v_t + p_chunk, v_n, v_total, clock_timestamp() - v_started; - - v_t := v_t + p_chunk; - END LOOP; - - RAISE NOTICE 'staged % forecasts in %', v_total, clock_timestamp() - v_started; - - /* Reverts on commit of the transaction the swap runs in. */ - EXECUTE format('SET LOCAL work_mem = %L', p_work_mem); - - EXECUTE format('CREATE TABLE pred.%I (LIKE pred.forecasts INCLUDING ALL)', v_new); - - EXECUTE format($q$ - INSERT INTO pred.%I ( - forecast_uuid, geometry_uuid, source_type_id, forecaster_id, init_time_utc, - value_resolution_mins, target_period, metadata, created_at_utc, - p02_sips, p10_sips, p25_sips, p50_sips, p75_sips, p90_sips, p98_sips - ) - SELECT f.forecast_uuid, f.geometry_uuid, f.source_type_id, f.forecaster_id, f.init_time_utc, - f.value_resolution_mins, f.target_period, f.metadata, f.created_at_utc, - s.p02_sips, s.p10_sips, s.p25_sips, s.p50_sips, s.p75_sips, s.p90_sips, s.p98_sips - FROM pred.%I AS f - INNER JOIN pred.fc_staging AS s USING (forecast_uuid) - ORDER BY f.geometry_uuid, f.source_type_id, f.forecaster_id, f.forecast_uuid DESC - $q$, v_new, p_partition); - - EXECUTE format('SELECT count(*) FROM pred.%I', p_partition) INTO v_all; - EXECUTE format( - 'SELECT count(*) FROM pred.%I AS f - WHERE EXISTS (SELECT 1 FROM pred.%I AS v WHERE v.forecast_uuid = f.forecast_uuid)', - p_partition, v_values) INTO v_src; - EXECUTE format('SELECT count(*) FROM pred.%I', v_new) INTO v_dst; - - IF v_src <> v_dst THEN - RAISE EXCEPTION 'row count mismatch for %: % source forecasts with values -> % rebuilt rows', - p_partition, v_src, v_dst; - END IF; - - RAISE NOTICE 'rebuilt %: % rows (% forecasts had no values and were dropped)', - p_partition, v_dst, v_all - v_dst; - - /* Marks the partition as migrated. Note this does not buy constraint exclusion on the read - * queries' legacy branch: they filter p50_sips inside a CTE rather than on a direct scan of - * pred.forecasts, so the planner cannot use it to prune. It is an integrity check and an - * operational marker for which partitions are done. Added before ATTACH so the fresh, - * exclusively-locked table is scanned rather than a live partition. */ - EXECUTE format( - 'ALTER TABLE pred.%I ADD CONSTRAINT migrated_check CHECK (p50_sips IS NOT NULL)', v_new); - - EXECUTE format('ANALYZE pred.%I', v_new); - - /* Detaching leaves a standalone copy of the foreign key behind on the values partition, which - * would still block the forecasts detach below, so it has to go too. */ - EXECUTE format( - 'ALTER TABLE pred.predicted_generation_values DETACH PARTITION pred.%I', v_values); - - SELECT conname INTO v_fk - FROM pg_constraint - WHERE conrelid = ('pred.' || quote_ident(v_values))::regclass - AND contype = 'f' - AND confrelid = 'pred.forecasts'::regclass; - - IF v_fk IS NOT NULL THEN - EXECUTE format('ALTER TABLE pred.%I DROP CONSTRAINT %I', v_values, v_fk); - END IF; - - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I', v_values, v_values || '_retired'); - - EXECUTE format('ALTER TABLE pred.forecasts DETACH PARTITION pred.%I', p_partition); - EXECUTE format('ALTER TABLE pred.forecasts ATTACH PARTITION pred.%I %s', v_new, v_bounds); - EXECUTE format('DROP TABLE pred.%I', p_partition); - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I', v_new, p_partition); - - DROP TABLE pred.fc_staging; - - RAISE NOTICE 'done: % in % (old values retained as pred.%_retired, drop once verified)', - p_partition, clock_timestamp() - v_started, v_values; -END; -$$; --- +goose StatementEnd - --- +goose Down -DROP PROCEDURE IF EXISTS pred.rebuild_forecast_partition(TEXT, INTERVAL, TEXT); diff --git a/internal/server/postgres/sql/migrations/00013_split_rebuild_swap.sql b/internal/server/postgres/sql/migrations/00013_split_rebuild_swap.sql deleted file mode 100644 index ba314eb..0000000 --- a/internal/server/postgres/sql/migrations/00013_split_rebuild_swap.sql +++ /dev/null @@ -1,336 +0,0 @@ --- +goose Up - --- +goose StatementBegin -/* - * Splits pred.rebuild_forecast_partition into a build phase and a swap phase. - */ - -CREATE OR REPLACE PROCEDURE pred.build_forecast_partition( - p_partition TEXT, - p_chunk INTERVAL DEFAULT INTERVAL '1 hour', - p_work_mem TEXT DEFAULT '256MB', - p_batch_rows BIGINT DEFAULT 50000 - ) - LANGUAGE plpgsql AS $$ - DECLARE - v_values TEXT; - v_new TEXT := p_partition || '_v2'; - v_bounds TEXT; - v_default BOOLEAN; - v_lo TIMESTAMP; - v_hi TIMESTAMP; - v_t TIMESTAMP; - v_last_uuid UUID; - v_next_uuid UUID; - v_n BIGINT; - v_total BIGINT := 0; - v_all BIGINT; - v_src BIGINT; - v_dst BIGINT; - v_started TIMESTAMPTZ := clock_timestamp(); - BEGIN - SELECT pg_get_expr(c.relpartbound, c.oid) INTO v_bounds - FROM pg_class AS c INNER JOIN pg_namespace AS n ON n.oid = c.relnamespace - WHERE n.nspname = 'pred' AND c.relname = p_partition; - - IF v_bounds IS NULL THEN - RAISE EXCEPTION 'not an attached partition of pred.forecasts: pred.%', p_partition; - END IF; - - v_default := (v_bounds = 'DEFAULT'); - - /* Sibling naming is the same prefix swap for every partition, including the default one: - * forecasts_pXXXXXXXX -> predicted_generation_values_pXXXXXXXX, - * forecasts_default -> predicted_generation_values_default. */ - v_values := 'predicted_generation_values_' || substring(p_partition FROM '^forecasts_(.+)$'); - - IF v_values IS NULL OR to_regclass('pred.' || quote_ident(v_values)) IS NULL THEN - RAISE EXCEPTION 'no values partition matching pred.%: expected pred.%', - p_partition, v_values; - END IF; - - IF to_regclass('pred.' || quote_ident(v_new)) IS NOT NULL THEN - RAISE EXCEPTION 'pred.% already exists', - v_new - USING HINT = format( - 'CALL pred.swap_forecast_partition(%L) to finish that run, or DROP TABLE pred.%I to start over', - p_partition, v_new); - END IF; - - CREATE UNLOGGED TABLE IF NOT EXISTS pred.fc_staging ( - forecast_uuid UUID PRIMARY KEY, - p02_sips SMALLINT [], p10_sips SMALLINT [], p25_sips SMALLINT [], - p50_sips SMALLINT [], p75_sips SMALLINT [], p90_sips SMALLINT [], - p98_sips SMALLINT [] - ); - TRUNCATE pred.fc_staging; - - IF v_default THEN - /* No time bounds to chunk over - forecast_uuid in this partition can be anything that - * fell outside every managed weekly range. Chunk by distinct forecast_uuid instead: - * find the batch boundary first, then aggregate up to it, so a forecast's horizon rows - * never straddle two batches. */ - v_last_uuid := '00000000-0000-0000-0000-000000000000'::UUID; - - LOOP - EXECUTE format($q$ - SELECT forecast_uuid FROM ( - SELECT DISTINCT forecast_uuid FROM pred.%I - WHERE forecast_uuid > %L - ORDER BY forecast_uuid - LIMIT %L - ) AS batch ORDER BY forecast_uuid DESC LIMIT 1 - $q$, v_values, v_last_uuid, p_batch_rows) INTO v_next_uuid; - - EXIT WHEN v_next_uuid IS NULL; - - EXECUTE format($q$ - INSERT INTO pred.fc_staging ( - forecast_uuid, p02_sips, p10_sips, p25_sips, - p50_sips, p75_sips, p90_sips, p98_sips) - SELECT - forecast_uuid, - CASE WHEN bool_or(p02_sip IS NOT NULL) THEN array_agg(p02_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p10_sip IS NOT NULL) THEN array_agg(p10_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p25_sip IS NOT NULL) THEN array_agg(p25_sip ORDER BY horizon_mins) END, - array_agg(p50_sip ORDER BY horizon_mins), - CASE WHEN bool_or(p75_sip IS NOT NULL) THEN array_agg(p75_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p90_sip IS NOT NULL) THEN array_agg(p90_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p98_sip IS NOT NULL) THEN array_agg(p98_sip ORDER BY horizon_mins) END - FROM pred.%I - WHERE forecast_uuid > %L AND forecast_uuid <= %L - GROUP BY forecast_uuid - ON CONFLICT (forecast_uuid) DO NOTHING - $q$, v_values, v_last_uuid, v_next_uuid); - - GET DIAGNOSTICS v_n = ROW_COUNT; - v_total := v_total + v_n; - v_last_uuid := v_next_uuid; - - COMMIT; - - RAISE NOTICE '... % +% (total %, elapsed %)', - v_last_uuid, v_n, v_total, clock_timestamp() - v_started; - END LOOP; - ELSE - v_lo := partman.uuid7_time_decoder( - (regexp_match(v_bounds, $re$FROM \('([^']+)'\)$re$))[1]::UUID::TEXT) AT TIME ZONE 'UTC'; - v_hi := partman.uuid7_time_decoder( - (regexp_match(v_bounds, $re$TO \('([^']+)'\)$re$))[1]::UUID::TEXT) AT TIME ZONE 'UTC'; - - RAISE NOTICE 'pred.% covers % .. % (% chunks of %)', - p_partition, v_lo, v_hi, - CEIL(EXTRACT(EPOCH FROM (v_hi - v_lo)) / EXTRACT(EPOCH FROM p_chunk)), p_chunk; - - v_t := v_lo; - WHILE v_t < v_hi LOOP - EXECUTE format($q$ - INSERT INTO pred.fc_staging ( - forecast_uuid, p02_sips, p10_sips, p25_sips, - p50_sips, p75_sips, p90_sips, p98_sips) - SELECT - forecast_uuid, - CASE WHEN bool_or(p02_sip IS NOT NULL) THEN array_agg(p02_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p10_sip IS NOT NULL) THEN array_agg(p10_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p25_sip IS NOT NULL) THEN array_agg(p25_sip ORDER BY horizon_mins) END, - array_agg(p50_sip ORDER BY horizon_mins), - CASE WHEN bool_or(p75_sip IS NOT NULL) THEN array_agg(p75_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p90_sip IS NOT NULL) THEN array_agg(p90_sip ORDER BY horizon_mins) END, - CASE WHEN bool_or(p98_sip IS NOT NULL) THEN array_agg(p98_sip ORDER BY horizon_mins) END - FROM pred.%I - WHERE forecast_uuid >= uuidv7_boundary(%L::TIMESTAMP AT TIME ZONE 'UTC') - AND forecast_uuid < uuidv7_boundary(%L::TIMESTAMP AT TIME ZONE 'UTC') - GROUP BY forecast_uuid - ON CONFLICT (forecast_uuid) DO NOTHING - $q$, v_values, v_t, v_t + p_chunk); - - GET DIAGNOSTICS v_n = ROW_COUNT; - v_total := v_total + v_n; - - COMMIT; - - RAISE NOTICE '% .. % +% (total %, elapsed %)', - v_t, v_t + p_chunk, v_n, v_total, clock_timestamp() - v_started; - - v_t := v_t + p_chunk; - END LOOP; - END IF; - - RAISE NOTICE 'staged % forecasts in %', v_total, clock_timestamp() - v_started; - - EXECUTE format('SET LOCAL work_mem = %L', p_work_mem); - - EXECUTE format('CREATE TABLE pred.%I (LIKE pred.forecasts INCLUDING ALL)', v_new); - - /* LEFT JOIN rather than INNER, with COALESCE onto the source row's own arrays: a forecast - * that already has arrays (written after the array-write deploy landed) has no row in - * fc_staging and must not be dropped. Only a forecast with neither a staging match nor its - * own arrays is truly value-less and gets excluded by the WHERE below. */ - EXECUTE format($q$ - INSERT INTO pred.%I ( - forecast_uuid, geometry_uuid, source_type_id, forecaster_id, init_time_utc, - value_resolution_mins, target_period, metadata, created_at_utc, - p02_sips, p10_sips, p25_sips, p50_sips, p75_sips, p90_sips, p98_sips - ) - SELECT f.forecast_uuid, f.geometry_uuid, f.source_type_id, f.forecaster_id, f.init_time_utc, - f.value_resolution_mins, f.target_period, f.metadata, f.created_at_utc, - COALESCE(s.p02_sips, f.p02_sips), COALESCE(s.p10_sips, f.p10_sips), - COALESCE(s.p25_sips, f.p25_sips), COALESCE(s.p50_sips, f.p50_sips), - COALESCE(s.p75_sips, f.p75_sips), COALESCE(s.p90_sips, f.p90_sips), - COALESCE(s.p98_sips, f.p98_sips) - FROM pred.%I AS f - LEFT JOIN pred.fc_staging AS s USING (forecast_uuid) - WHERE s.forecast_uuid IS NOT NULL OR f.p50_sips IS NOT NULL - ORDER BY f.geometry_uuid, f.source_type_id, f.forecaster_id, f.forecast_uuid DESC - $q$, v_new, p_partition); - - EXECUTE format('SELECT count(*) FROM pred.%I', p_partition) INTO v_all; - EXECUTE format( - 'SELECT count(*) FROM pred.%I AS f - WHERE f.p50_sips IS NOT NULL - OR EXISTS (SELECT 1 FROM pred.%I AS v WHERE v.forecast_uuid = f.forecast_uuid)', - p_partition, v_values) INTO v_src; - EXECUTE format('SELECT count(*) FROM pred.%I', v_new) INTO v_dst; - - IF v_src <> v_dst THEN - RAISE EXCEPTION 'row count mismatch for %: % source forecasts with values -> % rebuilt rows', - p_partition, v_src, v_dst; - END IF; - - EXECUTE format( - 'ALTER TABLE pred.%I ADD CONSTRAINT p50_sips_not_null CHECK (p50_sips IS NOT NULL)', v_new); - - EXECUTE format('ANALYZE pred.%I', v_new); - - DROP TABLE pred.fc_staging; - - RAISE NOTICE 'built pred.%: % rows (% forecasts had no values and were dropped) in %', - v_new, v_dst, v_all - v_dst, clock_timestamp() - v_started; - RAISE NOTICE 'nothing is swapped yet: CALL pred.swap_forecast_partition(''%'')', p_partition; - END; - $$; --- +goose StatementEnd - --- +goose StatementBegin -/* - * Swaps a table built by pred.build_forecast_partition in for its partition. - */ -CREATE OR REPLACE PROCEDURE pred.swap_forecast_partition( - p_partition TEXT, - p_lock_timeout TEXT DEFAULT '5s' - ) - LANGUAGE plpgsql AS $$ - DECLARE - v_values TEXT; - v_new TEXT := p_partition || '_v2'; - v_old TEXT := p_partition || '_retired'; - v_bounds TEXT; - v_fk TEXT; - v_names JSONB; - v_target TEXT; - r RECORD; - v_started TIMESTAMPTZ := clock_timestamp(); - BEGIN - SELECT pg_get_expr(c.relpartbound, c.oid) INTO v_bounds - FROM pg_class AS c INNER JOIN pg_namespace AS n ON n.oid = c.relnamespace - WHERE n.nspname = 'pred' AND c.relname = p_partition; - - IF v_bounds IS NULL THEN - RAISE EXCEPTION 'not an attached partition of pred.forecasts: pred.%', p_partition; - END IF; - - IF to_regclass('pred.' || quote_ident(v_new)) IS NULL THEN - RAISE EXCEPTION 'no rebuilt table pred.%', v_new - USING HINT = format('CALL pred.build_forecast_partition(%L) first', p_partition); - END IF; - - v_values := 'predicted_generation_values_' || substring(p_partition FROM '^forecasts_(.+)$'); - - IF v_values IS NULL OR to_regclass('pred.' || quote_ident(v_values)) IS NULL THEN - RAISE EXCEPTION 'no values partition matching pred.%: expected pred.%', - p_partition, v_values; - END IF; - - EXECUTE format('SET LOCAL lock_timeout = %L', p_lock_timeout); - LOCK TABLE pred.forecasts IN ACCESS EXCLUSIVE MODE; - LOCK TABLE pred.predicted_generation_values IN ACCESS EXCLUSIVE MODE; - - EXECUTE format( - 'ALTER TABLE pred.predicted_generation_values DETACH PARTITION pred.%I', v_values); - - SELECT conname INTO v_fk - FROM pg_constraint - WHERE conrelid = ('pred.' || quote_ident(v_values))::regclass - AND contype = 'f' - AND confrelid = 'pred.forecasts'::regclass; - - IF v_fk IS NOT NULL THEN - EXECUTE format('ALTER TABLE pred.%I DROP CONSTRAINT %I', v_values, v_fk); - END IF; - - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I', v_values, v_values || '_retired'); - - EXECUTE format('ALTER TABLE pred.forecasts DETACH PARTITION pred.%I', p_partition); - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I', p_partition, v_old); - EXECUTE format('ALTER TABLE pred.forecasts ATTACH PARTITION pred.%I %s', v_new, v_bounds); - EXECUTE format('ALTER TABLE pred.%I RENAME TO %I', v_new, p_partition); - - SELECT jsonb_object_agg(s.definition, s.index_name) INTO v_names - FROM ( - SELECT regexp_replace(pg_get_indexdef(i.indexrelid), - '^CREATE (UNIQUE )?INDEX \S+ ON \S+ ', '') AS definition, - c.relname AS index_name - FROM pg_index AS i INNER JOIN pg_class AS c ON c.oid = i.indexrelid - WHERE i.indrelid = ('pred.' || quote_ident(v_old))::REGCLASS - ) AS s; - - FOR r IN - SELECT c.oid, c.relname - FROM pg_index AS i INNER JOIN pg_class AS c ON c.oid = i.indexrelid - WHERE i.indrelid = ('pred.' || quote_ident(v_old))::REGCLASS - LOOP - EXECUTE format('ALTER INDEX pred.%I RENAME TO %I', r.relname, 'retired_' || r.oid); - END LOOP; - - FOR r IN - SELECT c.relname, - regexp_replace(pg_get_indexdef(i.indexrelid), - '^CREATE (UNIQUE )?INDEX \S+ ON \S+ ', '') AS definition - FROM pg_index AS i INNER JOIN pg_class AS c ON c.oid = i.indexrelid - WHERE i.indrelid = ('pred.' || quote_ident(p_partition))::REGCLASS - LOOP - v_target := COALESCE(v_names ->> r.definition, replace(r.relname, v_new, p_partition)); - - IF v_target <> r.relname THEN - EXECUTE format('ALTER INDEX pred.%I RENAME TO %I', r.relname, v_target); - END IF; - END LOOP; - - RAISE NOTICE 'swapped % in % (old rows retained as pred.%_retired and pred.%_retired)', - p_partition, clock_timestamp() - v_started, p_partition, v_values; - END; - $$; --- +goose StatementEnd - --- +goose StatementBegin -/* Both phases in one call, for a partition being rebuilt start to finish. If the swap loses the - * lock race, retry it alone - the build is committed and does not need repeating. */ -CREATE OR REPLACE PROCEDURE pred.rebuild_forecast_partition( - p_partition TEXT, - p_chunk INTERVAL DEFAULT INTERVAL '1 hour', - p_work_mem TEXT DEFAULT '256MB' -) -LANGUAGE plpgsql AS $$ -BEGIN - CALL pred.build_forecast_partition(p_partition, p_chunk, p_work_mem); - CALL pred.swap_forecast_partition(p_partition); -END; -$$; --- +goose StatementEnd - --- +goose Down -/* 00012's Down drops rebuild_forecast_partition too, so rolling back past this leaves no - * rebuild procedure rather than restoring the version that deadlocks. */ -DROP PROCEDURE IF EXISTS pred.rebuild_forecast_partition(TEXT, INTERVAL, TEXT); -DROP PROCEDURE IF EXISTS pred.swap_forecast_partition(TEXT, TEXT); -DROP PROCEDURE IF EXISTS pred.build_forecast_partition(TEXT, INTERVAL, TEXT); diff --git a/internal/server/postgres/sql/migrations/00014_consolidated_schema.sql b/internal/server/postgres/sql/migrations/00014_consolidated_schema.sql new file mode 100644 index 0000000..32acfe4 --- /dev/null +++ b/internal/server/postgres/sql/migrations/00014_consolidated_schema.sql @@ -0,0 +1,484 @@ +-- +goose Up + +CREATE EXTENSION IF NOT EXISTS btree_gist; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; +CREATE SCHEMA IF NOT EXISTS partman; +CREATE EXTENSION IF NOT EXISTS pg_partman WITH SCHEMA partman; +CREATE EXTENSION IF NOT EXISTS pg_cron; + +SELECT cron.schedule('cron-details-cleanup', '0 12 * * *', $$DELETE FROM cron.job_run_details WHERE end_time < now() - interval '7 days'$$); + + +/* Overwrites the default uuidv7_extract_timestamp function. + * The default function uses the system local timezone to return a TIMESTAMPTZ. + * In order to make the function SAFE, this dependency is removed. + * NOTE: Requires the local timezone to be UTC. + */ +-- +goose StatementBegin +CREATE FUNCTION uuidv7_extract_timestamp(u UUID) RETURNS TIMESTAMP + LANGUAGE sql + IMMUTABLE STRICT PARALLEL SAFE + RETURN uuid_extract_timestamp(u) AT TIME ZONE 'UTC'; +-- +goose StatementEnd + +/* Generate a non-random uuidv7 with the given timestamp (first 48 bits) and all random bits to 0. + * As the smallest possible uuidv7 for that timestamp, it may be used as a boundary for partitions. + */ +-- +goose StatementBegin +CREATE FUNCTION uuidv7_boundary(timestamptz) RETURNS uuid +AS $$ + /* uuid fields: version=0b0111, variant=0b10 */ + select encode( + overlay('\x00000000000070008000000000000000'::bytea + placing substring(int8send(floor(extract(epoch from $1) * 1000)::bigint) from 3) + from 1 for 6), + 'hex')::uuid; +$$ LANGUAGE sql stable strict parallel safe; +-- +goose StatementEnd + + +/* == LOCATIONS =================================================================================== + * + * Schema and tables to handle location data. + * + * The generation data we store, be it predicted or otherwise, is always tied to a certain + * geometry. These geometries vary in size and scope, from a single site to an entire country, + * and the metadata we may want to store about them will also vary accordingly. + + * From an application standpoint, the geometry is pertinent in the case where we care about the + * generated power as a fraction of the capacity of the geometry, as well as allowing us to + * represent the data on a map. + + * To this degree, what the external application may consider a "location", is represented here as + * a combination of a geometry (the spatial data), and a source (the energy generation capability). + * One geometry can have multiple sources, e.g. the UK nation geometry can have solar, wind, etc. + */ + +CREATE SCHEMA loc; + +/*- Lookups -----------------------------------------------------------------------------------*/ + +-- Lookup table to store different source types +CREATE TABLE loc.source_types ( + source_type_id SMALLINT GENERATED ALWAYS AS IDENTITY NOT NULL, + source_type_name TEXT NOT NULL, + CONSTRAINT source_type_name_format_check CHECK ( + LENGTH(source_type_name) > 0 + AND LENGTH(source_type_name) <= 48 + AND source_type_name = LOWER(source_type_name) + ), + PRIMARY KEY (source_type_id), + UNIQUE (source_type_name) +); +-- The ordering of insertion here matches the .proto enum definitions. Change with caution! +INSERT INTO loc.source_types (source_type_name) VALUES ('solar'), ('wind'), ('hydro'), ('battery'); + +-- Lookup table to store different geometry types +CREATE TABLE loc.geometry_types ( + geometry_type_id SMALLINT GENERATED ALWAYS AS IDENTITY NOT NULL, + geometry_type_name TEXT NOT NULL, + CONSTRAINT geometry_type_name_format_check CHECK ( + LENGTH(geometry_type_name) > 0 + AND LENGTH(geometry_type_name) <= 24 + AND geometry_type_name = LOWER(geometry_type_name) + ), + PRIMARY KEY (geometry_type_id), + UNIQUE (geometry_type_name) +); +-- The ordering of insertion here matches the .proto enum definitions. Change with caution! +INSERT INTO loc.geometry_types (geometry_type_name) VALUES ('site'), ('gsp'), ('dno'), ('nation'), ('state'), ('county'), ('city'), ('primary_substation'); + + +/*- Tables ----------------------------------------------------------------------------------*/ + +CREATE TABLE loc.entities ( + entity_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + external_id TEXT NOT NULL + CONSTRAINT external_id_format_check CHECK ( + external_id IS NOT NULL + AND LENGTH(external_id) > 0 + AND LENGTH(external_id) <= 128 + ), + UNIQUE (external_id) +); + + +-- Table to store spatial data for geometries +CREATE TABLE loc.geometries ( + geometry_uuid UUID DEFAULT UUIDV7() NOT NULL, + geometry_name TEXT NOT NULL, + CONSTRAINT geometry_name_check CHECK ( + LENGTH(geometry_name) > 0 + AND geometry_name = LOWER(geometry_name) + ), + geom GEOMETRY (GEOMETRY, 4326) NOT NULL, + CONSTRAINT geom_validity_check CHECK ( + ST_GEOMETRYTYPE(geom) IN ('ST_Point', 'ST_Polygon', 'ST_MultiPolygon') + AND ST_SRID(geom) = 4326 + AND ST_NDIMS(geom) = 2 + AND ST_ISVALID(geom) + AND ST_XMIN(geom) >= -180 AND ST_XMAX(geom) <= 180 + AND ST_YMIN(geom) >= -90 AND ST_YMAX(geom) <= 90 + ), + geometry_type_id SMALLINT NOT NULL + REFERENCES loc.geometry_types (geometry_type_id) + ON UPDATE CASCADE + ON DELETE RESTRICT, + associated_point GEOMETRY (POINT, 4326) NOT NULL, + CONSTRAINT associated_point_validity_check CHECK ( + ST_SRID(associated_point) = 4326 + AND ST_NDIMS(associated_point) = 2 + AND ST_ISVALID(associated_point) + AND ST_X(associated_point) >= -180 AND ST_X(associated_point) <= 180 + AND ST_Y(associated_point) >= -90 AND ST_Y(associated_point) <= 90 + ), + geom_hash TEXT GENERATED ALWAYS AS (MD5(ST_ASBINARY(geom))) STORED, + metadata JSONB DEFAULT NULL, + owning_entity_id INTEGER DEFAULT NULL + REFERENCES loc.entities(entity_id) + ON UPDATE CASCADE + ON DELETE SET NULL, + PRIMARY KEY (geometry_uuid), + UNIQUE (geometry_name, geom_hash) +); +-- Required index for efficient spatial-based queries +CREATE INDEX ON loc.geometries USING gist (geom); +-- Index for efficiently fetching e.g. all POINT geometry geometries +CREATE INDEX ON loc.geometries (ST_GEOMETRYTYPE(geom)); +-- Index for finding all geometries of a certain type +CREATE INDEX ON loc.geometries (geometry_type_id); +-- Legacy index for finding gsp geometries by gsp_id +CREATE INDEX idx_geometries_gsp_id_partial +ON loc.geometries (((metadata ->> 'gsp_id')::INTEGER)) +WHERE geometry_type_id = 2 + AND (((metadata ->> 'gsp_id') IS NOT NULL)); +-- Index for finding all geometries owned by a certain entity +CREATE INDEX idx_owning_entity_id ON loc.geometries (owning_entity_id); + +/* + * Table to store the temporal generation capability of geometries. + * Each geometry can have multiple sources of generation (solar, wind, etc), + * and each source can change over time. For speed of writing, this is handled + * via a simple valid-from timestamp field. + */ +CREATE TABLE loc.sources_history ( + source_type_id SMALLINT NOT NULL + REFERENCES loc.source_types (source_type_id) + ON UPDATE CASCADE + ON DELETE RESTRICT, + -- Capacity cap, (for instance during curtailment or repair work), + -- encoded as a smallint percentage (sip) of the capacity; with 0 representing 0% + -- AND 30000 representing 100% of the capacity. However, since things are mostly + -- not limited, NULL indicates no limit, so 30000 is an invalid value. + -- NOTE: This is currently not used. + capacity_limit_sip SMALLINT DEFAULT NULL, + CONSTRAINT capacity_limit_sip_vaildity_check CHECK ( + capacity_limit_sip IS NULL + OR (capacity_limit_sip >= 0 AND capacity_limit_sip < 30000) + ), + -- Capacity in watts. This maxes out at ~9.22 petawatts, which should be sufficient + capacity_watts BIGINT NOT NULL, + CONSTRAINT capacity_nonnegative_check CHECK (capacity_watts >= 0), + valid_from_utc TIMESTAMP DEFAULT NOW() NOT NULL, + geometry_uuid UUID NOT NULL + REFERENCES loc.geometries (geometry_uuid) + ON UPDATE CASCADE + ON DELETE CASCADE, + -- Metadata about the source, e.g. tilt, orientation, etc. + metadata JSONB DEFAULT NULL, + CONSTRAINT metadata_nonempty_check CHECK ( + metadata IS NULL OR metadata <> '{}'::JSONB -- Null is cheaper + ), + PRIMARY KEY (geometry_uuid, source_type_id, valid_from_utc) +); + +/* + * Materialized view to store the state of sources over time with a system period. + * This allows for quicker reads of the state of sources at a given time. + */ +CREATE MATERIALIZED VIEW loc.sources_mv AS +SELECT + sh.geometry_uuid, + sh.source_type_id, + sh.capacity_watts, + sh.capacity_limit_sip, + sh.metadata, + COALESCE(sh.metadata || g.metadata, sh.metadata, g.metadata)::JSONB AS metadata_jsonb, + g.geometry_name, + g.geometry_type_id, + g.owning_entity_id, + ST_X(g.associated_point)::REAL AS longitude, + ST_Y(g.associated_point)::REAL AS latitude, + TSRANGE( + sh.valid_from_utc, + LEAD(sh.valid_from_utc, 1) OVER ( + PARTITION BY sh.geometry_uuid, sh.source_type_id + ORDER BY sh.valid_from_utc + ) + ) AS sys_period +FROM loc.sources_history AS sh +INNER JOIN loc.geometries AS g USING (geometry_uuid); +-- Prevent overlapping records. Required for concurrent refreshes. +CREATE UNIQUE INDEX ON loc.sources_mv (geometry_uuid, source_type_id, sys_period); +CREATE INDEX idx_sources_mv_owning_entity_id ON loc.sources_mv (owning_entity_id); +CREATE INDEX idx_sources_mv_composite_lookup ON loc.sources_mv USING gist (geometry_uuid, source_type_id, sys_period); + + +/* == OBSERVATIONS ================================================================================ + * + * Schema and tables to handle observed generation data. + * + * Observations of generation data is usually measured by providers of inverters, which are + * required in many sources of renewable energy to convert power from DC to AC. Partnerships + * with these providers provide access to the data in order to test the accuracy of predictions. +*/ + + +CREATE SCHEMA obs; + +/*- Tables ----------------------------------------------------------------------------------*/ + +/* + * Table to store observers. + * These are providers of actual recorded generation values from inverters + * (mostly - looking at you, pvlive...) +*/ +CREATE TABLE obs.observers ( + observer_uuid UUID NOT NULL DEFAULT UUIDV7(), + observer_name TEXT NOT NULL, + CONSTRAINT observer_name_format_check CHECK ( + LENGTH(observer_name) > 0 AND LENGTH(observer_name) < 128 + AND observer_name = LOWER(observer_name) + ), + PRIMARY KEY (observer_uuid), + UNIQUE (observer_name) +); + +/* + * Table to store observed generation values. + * The generation value is stored as a percentage of the source capacity represented by a + * smallint percent (sip). Since it isn't impossible to measure a little over capacity, 30000 + * represents 100% of capacity instead of the max smallint value (32767). This allows for some + * measurement leeway. + * The table has native partitioning that can then be managed by pg_partman. Note that unique + * indexes will only work if they include the partition key. + */ +CREATE TABLE obs.observed_generation_values ( + value_sip SMALLINT NOT NULL, + CONSTRAINT value_sip_nonnegative_check CHECK (value_sip >= 0), + source_type_id SMALLINT NOT NULL + REFERENCES loc.source_types (source_type_id) + ON UPDATE CASCADE + ON DELETE RESTRICT, + observation_timestamp_utc TIMESTAMP NOT NULL, + CONSTRAINT observation_timestamp_utc_recency_check CHECK ( + observation_timestamp_utc <= CURRENT_TIMESTAMP + MAKE_INTERVAL(days => 31) + ), + observer_uuid UUID NOT NULL + REFERENCES obs.observers (observer_uuid) + ON UPDATE CASCADE + ON DELETE CASCADE, + geometry_uuid UUID NOT NULL + REFERENCES loc.geometries (geometry_uuid) + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY (geometry_uuid, source_type_id, observer_uuid, observation_timestamp_utc) +) +PARTITION BY RANGE (observation_timestamp_utc); + +/* + * Manage partitions with pg_partman. + * Highlights: + * - `retention_keep_table = true`: detach old partitions instead of dropping them + * - `infinite_time_partitions = true`: retain detached partitions indefinitely for processing + */ +SELECT partman.create_parent( + p_parent_table => 'obs.observed_generation_values', + p_control => 'observation_timestamp_utc', + p_type => 'range', + p_interval => '1 week', + p_automatic_maintenance => 'on', + p_jobmon => FALSE, + p_premake => 7 +); +UPDATE partman.part_config +SET + retention = NULL, + retention_keep_table = TRUE, + retention_keep_index = TRUE, + infinite_time_partitions = TRUE +WHERE parent_table = 'obs.observed_generation_values'; +SELECT partman.run_maintenance('obs.observed_generation_values'); +-- Schedule regular maintenance for the partitioned observed generation values table. +SELECT cron.schedule('partman-maintenance', '@hourly', $$CALL partman.run_maintenance_proc()$$); + + +/* == PREDICTIONS ================================================================================= + * + * Schema and tables to handle predicted generation data. + * + * Predicted of generation values are produced by various forecast models for a specific location. + * A forecast is a set of predicted generation values, beginning at the initialisation time. Each + * subsequent generation's target time is equivalent to the initialisation time plus the horizon. + * + * The forecast produced most recently will likely be the most accurate. + */ + +CREATE SCHEMA pred; + +/*- Tables ----------------------------------------------------------------------------------*/ + +/* + * A forecaster is a source that generates forecast values. This is usually an ML model, + * but could also be an analytical process. Each forecaster's name and version number uniquely + * identifies it. + */ +CREATE TABLE pred.forecasters ( + forecaster_id INTEGER GENERATED ALWAYS AS IDENTITY NOT NULL, + forecaster_name TEXT NOT NULL, + CONSTRAINT forecaster_name_format_check CHECK ( + LENGTH(forecaster_name) > 0 AND LENGTH(forecaster_name) < 64 + AND forecaster_name = LOWER(forecaster_name) + ), + forecaster_version TEXT NOT NULL, + CONSTRAINT forecaster_version_format_check CHECK ( + LENGTH(forecaster_version) > 0 AND LENGTH(forecaster_version) < 64 + AND forecaster_version = LOWER(forecaster_version) + ), + created_at_utc TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (forecaster_id), + UNIQUE (forecaster_name, forecaster_version) +); + +/* + * Forecasts refer to the set of forecast values, created by a specific version of a forecaster, + * for a specific location, with some initialization time. Each forecast contains a timeseries of + * forecast values. There can only be one forecast per location per initialization time per + * forecaster; reruns should replace old values. + */ +CREATE TABLE pred.forecasts ( + source_type_id SMALLINT NOT NULL + REFERENCES loc.source_types (source_type_id) + ON UPDATE CASCADE + ON DELETE RESTRICT, + value_resolution_mins SMALLINT NOT NULL, + CONSTRAINT value_resolution_mins_size_check CHECK ( + value_resolution_mins > 0 AND value_resolution_mins <= 60 + ), + forecaster_id INTEGER NOT NULL + REFERENCES pred.forecasters (forecaster_id) + ON UPDATE CASCADE + ON DELETE CASCADE, + created_at_utc TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + init_time_utc TIMESTAMP NOT NULL, + geometry_uuid UUID NOT NULL + REFERENCES loc.geometries (geometry_uuid) + ON UPDATE CASCADE + ON DELETE CASCADE, + /* The forecast uuid should be generated using the init time as the time component */ + forecast_uuid UUID NOT NULL, + target_period TSRANGE NOT NULL, + CONSTRAINT target_period_valid_check CHECK ( + UPPER(target_period) > LOWER(target_period) + ), + CONSTRAINT target_period_recency_check CHECK ( + LOWER(target_period) >= '2000-01-01 00:00:00'::TIMESTAMP + ), + metadata JSONB DEFAULT NULL, + p02_sips SMALLINT [], + p10_sips SMALLINT [], + p25_sips SMALLINT [], + p50_sips SMALLINT [] NOT NULL, + p75_sips SMALLINT [], + p90_sips SMALLINT [], + p98_sips SMALLINT [], + CONSTRAINT plevel_lengths_match_check CHECK ( + ARRAY_LENGTH(p50_sips, 1) > 0 + AND COALESCE(ARRAY_LENGTH(p02_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + AND COALESCE(ARRAY_LENGTH(p10_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + AND COALESCE(ARRAY_LENGTH(p25_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + AND COALESCE(ARRAY_LENGTH(p75_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + AND COALESCE(ARRAY_LENGTH(p90_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + AND COALESCE(ARRAY_LENGTH(p98_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) + ), + PRIMARY KEY (forecast_uuid) +) +PARTITION BY RANGE (forecast_uuid); + +CREATE INDEX idx_forecasts_filter ON pred.forecasts ( + geometry_uuid, + source_type_id, + forecaster_id, + forecast_uuid DESC +) INCLUDE (target_period); + +/* + * Manage partitions with pg_partman. + * Highlights: + * - `retention_keep_table = true`: detach old partitions instead of dropping them + * - `infinite_time_partitions = true`: retain detached partitions indefinitely for processing + */ +SELECT partman.create_parent( + p_parent_table => 'pred.forecasts', + p_control => 'forecast_uuid', + p_type => 'range', + p_interval => '1 week', + p_automatic_maintenance => 'on', + p_jobmon => FALSE, + p_premake => 7, + p_time_encoder => 'partman.uuid7_time_encoder', + p_time_decoder => 'partman.uuid7_time_decoder' +); +UPDATE partman.part_config +SET + retention = NULL, + retention_keep_table = TRUE, + retention_keep_index = TRUE, + infinite_time_partitions = TRUE +WHERE parent_table = 'pred.forecasts'; +SELECT partman.run_maintenance('pred.forecasts'); +SELECT cron.schedule('forecasts-vacuum', '30 4 * * *', $$VACUUM ANALYZE pred.forecasts$$); + +/* + * Procedure to cluster closed partitions of the forecasts table. + * At write time, forecasts are naturally ordered on disk by forecast_uuid alone (which + * corresponds to the init time of the forecast). However, the standard query route for + * forecsats is to come through a geometry, source type, and forecaster first. There is + * an index to speed this up, but it can be made even faster by physically co-locating + * the data on disk to match the index. CLUSTER does this. + */ +-- +goose StatementBegin +CREATE PROCEDURE pred.cluster_closed_partitions(p_age INTERVAL DEFAULT '2 weeks') +LANGUAGE plpgsql AS $$ +DECLARE +r RECORD; +BEGIN +PERFORM set_config('lock_timeout', '30s', FALSE); + +FOR r IN + SELECT c.relname, ci.relname AS index_name + FROM pg_class AS c + INNER JOIN pg_inherits AS inh ON inh.inhrelid = c.oid + INNER JOIN pg_class AS p ON p.oid = inh.inhparent AND p.relname = 'forecasts' + INNER JOIN pg_index AS i ON i.indrelid = c.oid + INNER JOIN pg_class AS ci ON ci.oid = i.indexrelid + INNER JOIN pg_inherits AS iinh ON iinh.inhrelid = i.indexrelid + INNER JOIN pg_class AS pi ON pi.oid = iinh.inhparent AND pi.relname = 'idx_forecasts_filter' + WHERE SUBSTRING(c.relname FROM 'p(\d{8})$')::DATE < CURRENT_DATE - p_age + AND NOT i.indisclustered +LOOP + BEGIN + RAISE NOTICE 'clustering pred.%', r.relname; + EXECUTE format('ALTER TABLE pred.%I CLUSTER ON %I', r.relname, r.index_name); + EXECUTE format('CLUSTER pred.%I', r.relname); + EXECUTE format('ANALYZE pred.%I', r.relname); + EXCEPTION WHEN lock_not_available THEN + RAISE WARNING 'skipping pred.%: could not acquire lock', r.relname; + END; + + COMMIT; +END LOOP; +END $$; +-- +goose StatementEnd +SELECT cron.schedule('cluster-forecasts', '0 3 * * 0', $$CALL pred.cluster_closed_partitions()$$); + diff --git a/internal/server/postgres/sql/migrations/00014_drop_row_storage.sql b/internal/server/postgres/sql/migrations/00014_drop_row_storage.sql deleted file mode 100644 index 58d381a..0000000 --- a/internal/server/postgres/sql/migrations/00014_drop_row_storage.sql +++ /dev/null @@ -1,201 +0,0 @@ --- +goose Up - -/* - * Consolidates the array storage migration (00011-00013). - * - * Makes p50_sips NOT NULL, drops pred.predicted_generation_values and the machinery that - * rebuilt partitions into it, and leaves pred.forecasts as the single source of predicted - * values. - * - * This is not reversible. The Down below restores the schema so that a code revert works, - * but the row data is gone. - */ - --- +goose StatementBegin -DO $$ -DECLARE - v_unmigrated BIGINT; -BEGIN - SELECT count(*) INTO v_unmigrated FROM pred.forecasts WHERE p50_sips IS NULL; - - IF v_unmigrated > 0 THEN - RAISE EXCEPTION 'refusing to drop row storage: % forecasts still have no arrays', - v_unmigrated - USING HINT = 'find them with: SELECT tableoid::REGCLASS, count(*) FROM pred.forecasts ' - 'WHERE p50_sips IS NULL GROUP BY 1'; - END IF; -END $$; --- +goose StatementEnd - -/* Every partition rebuilt by 00013 already carries a validated - * CHECK (p50_sips IS NOT NULL) named p50_sips_not_null, which lets SET NOT NULL skip its - * scan. Partitions pg_partman created after the array deploy carry no such constraint - - * they are array-native by construction but unproven, so prove them here. Adding NOT VALID - * and validating separately keeps the scan under SHARE UPDATE EXCLUSIVE instead of - * ACCESS EXCLUSIVE. */ --- +goose StatementBegin -DO $$ -DECLARE - r RECORD; -BEGIN - FOR r IN - SELECT c.relname - FROM pg_class AS c - INNER JOIN pg_inherits AS i ON i.inhrelid = c.oid - INNER JOIN pg_class AS p ON p.oid = i.inhparent - WHERE p.relname = 'forecasts' - AND NOT EXISTS ( - SELECT 1 FROM pg_constraint AS k - WHERE k.conrelid = c.oid - AND k.conname = 'p50_sips_not_null' - AND k.convalidated - ) - LOOP - RAISE NOTICE 'proving p50_sips on pred.%', r.relname; - EXECUTE format( - 'ALTER TABLE pred.%I ADD CONSTRAINT p50_sips_not_null ' - 'CHECK (p50_sips IS NOT NULL) NOT VALID', r.relname); - EXECUTE format( - 'ALTER TABLE pred.%I VALIDATE CONSTRAINT p50_sips_not_null', r.relname); - END LOOP; -END $$; --- +goose StatementEnd - -ALTER TABLE pred.forecasts ALTER COLUMN p50_sips SET NOT NULL; - -/* The per-partition CHECKs were the marker for which partitions had been rebuilt, and the - * proof that let SET NOT NULL skip its scan. The column constraint now subsumes both. */ --- +goose StatementBegin -DO $$ -DECLARE - r RECORD; -BEGIN - FOR r IN - SELECT c.relname - FROM pg_class AS c - INNER JOIN pg_inherits AS i ON i.inhrelid = c.oid - INNER JOIN pg_class AS p ON p.oid = i.inhparent - INNER JOIN pg_constraint AS k - ON k.conrelid = c.oid AND k.conname = 'p50_sips_not_null' - WHERE p.relname = 'forecasts' - LOOP - EXECUTE format( - 'ALTER TABLE pred.%I DROP CONSTRAINT p50_sips_not_null', r.relname); - END LOOP; -END $$; --- +goose StatementEnd - -/* p50_sips can no longer be NULL, so the guard is dead. Left NOT VALID, as it has been - * since 00011 - validating it is a full scan and buys nothing the write path does not - * already enforce. */ -ALTER TABLE pred.forecasts DROP CONSTRAINT plevel_lengths_match_check; - -ALTER TABLE pred.forecasts - ADD CONSTRAINT plevel_lengths_match_check CHECK ( - ARRAY_LENGTH(p50_sips, 1) > 0 - AND COALESCE(ARRAY_LENGTH(p02_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p10_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p25_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p75_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p90_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p98_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - ) NOT VALID; - -/* pg_partman config goes before the table, or the next run_maintenance errors on a parent - * that no longer exists. */ -DELETE FROM partman.part_config_sub -WHERE sub_parent = 'pred.predicted_generation_values'; - -DELETE FROM partman.part_config -WHERE parent_table = 'pred.predicted_generation_values'; - -/* Takes ACCESS EXCLUSIVE on pred.forecasters, loc.geometries and loc.source_types to remove - * this table's foreign key triggers from the referenced side - a lock set that can deadlock - * against the read path. Safe only because goose runs with the API down. */ -SET lock_timeout = '30s'; - -DROP TABLE pred.predicted_generation_values; -DROP TABLE IF EXISTS pred.predicted_generation_values_template; -DROP TABLE IF EXISTS partman.template_pred_predicted_generation_values; - -RESET lock_timeout; - -/* Only reference was the other_stats_fractions constraint on the table just dropped. */ -DROP FUNCTION IF EXISTS pred.check_all_jsonb_values_are_valid_stat_fractions(JSONB); - -DROP PROCEDURE IF EXISTS pred.swap_forecast_partition(TEXT, TEXT); -DROP PROCEDURE IF EXISTS pred.build_forecast_partition(TEXT, INTERVAL, TEXT); -DROP TABLE IF EXISTS pred.fc_staging; - --- +goose Down - -/* - * Restores the schema, not the data. The row storage this migration dropped is gone; the - * arrays on pred.forecasts remain authoritative. This exists so that reverting the binary - * leaves a database whose shape the old code recognises - the legacy read branches will - * simply find no rows, which is the correct answer now that every forecast has arrays. - */ - -ALTER TABLE pred.forecasts ALTER COLUMN p50_sips DROP NOT NULL; - -ALTER TABLE pred.forecasts DROP CONSTRAINT plevel_lengths_match_check; - -ALTER TABLE pred.forecasts - ADD CONSTRAINT plevel_lengths_match_check CHECK ( - p50_sips IS NULL OR ( - ARRAY_LENGTH(p50_sips, 1) > 0 - AND COALESCE(ARRAY_LENGTH(p02_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p10_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p25_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p75_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p90_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - AND COALESCE(ARRAY_LENGTH(p98_sips, 1), ARRAY_LENGTH(p50_sips, 1)) = ARRAY_LENGTH(p50_sips, 1) - ) - ) NOT VALID; - -CREATE TABLE pred.predicted_generation_values ( - horizon_mins SMALLINT NOT NULL, - CONSTRAINT horizon_mins_nonnegative_check CHECK (horizon_mins >= 0), - CONSTRAINT horizon_mins_fiveminutely_check CHECK (horizon_mins % 5 = 0), - p50_sip SMALLINT NOT NULL, - CONSTRAINT p50_sip_nonnegative_check CHECK (p50_sip >= 0), - p10_sip SMALLINT, - CONSTRAINT p10_sip_nonnegative_check CHECK (p10_sip >= 0), - p90_sip SMALLINT, - CONSTRAINT p90_sip_nonnegative_check CHECK (p90_sip >= 0), - p02_sip SMALLINT, - CONSTRAINT p02_sip_nonnegative_check CHECK (p02_sip >= 0), - p25_sip SMALLINT, - CONSTRAINT p25_sip_nonnegative_check CHECK (p25_sip >= 0), - p75_sip SMALLINT, - CONSTRAINT p75_sip_nonnegative_check CHECK (p75_sip >= 0), - p98_sip SMALLINT, - CONSTRAINT p98_sip_nonnegative_check CHECK (p98_sip >= 0), - forecast_uuid UUID NOT NULL - REFERENCES pred.forecasts (forecast_uuid) - ON DELETE CASCADE - ON UPDATE CASCADE, - PRIMARY KEY (forecast_uuid, horizon_mins) -) -PARTITION BY RANGE (forecast_uuid); - -SELECT partman.create_parent( - p_parent_table => 'pred.predicted_generation_values', - p_control => 'forecast_uuid', - p_type => 'range', - p_interval => '1 week', - p_automatic_maintenance => 'on', - p_jobmon => FALSE, - p_time_encoder => 'partman.uuid7_time_encoder', - p_time_decoder => 'partman.uuid7_time_decoder', - p_premake => 7 -); - -UPDATE partman.part_config -SET - retention_keep_table = TRUE, - retention_keep_index = TRUE, - infinite_time_partitions = TRUE -WHERE parent_table = 'pred.predicted_generation_values'; - -SELECT partman.run_maintenance('pred.predicted_generation_values'); From eac1d8a5fd59925fcad83af055e5a154a075ce33 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:46:55 +0100 Subject: [PATCH 2/4] chore(repo): Linting --- Makefile | 113 +--- README.md | 743 --------------------- internal/server/postgres/dataserverimpl.go | 20 + proto/ocf/dp/dp-data.messages.proto | 2 +- 4 files changed, 22 insertions(+), 856 deletions(-) diff --git a/Makefile b/Makefile index a3548a9..32d60df 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,6 @@ PROTOC := $(LOCAL_BIN)/protoc PROTOC_INCLUDE := $(LOCAL_BIN)/include PROTOC_GEN_GO := $(LOCAL_BIN)/protoc-gen-go PROTOC_GEN_GRPC := $(LOCAL_BIN)/protoc-gen-go-grpc -PROTOC_GEN_DOC := $(LOCAL_BIN)/protoc-gen-doc # --- Sources & Stamps --- GO_SOURCES := $(shell find . -name '*.go' -not -path "./internal/gen/*" -not -path "./vendor/*") @@ -78,7 +77,7 @@ doctor: $(PROTOC) # --- Code Generation Targets ------------------------------------------------------------- # .PHONY: gen -gen: gen.proto.go gen.db gen.proto.docs +gen: gen.proto.go gen.db .PHONY: gen.db gen.db: ${SQLC_STAMP_FILE} @@ -181,116 +180,6 @@ gen.proto.python: ${PROTOC} @echo "Building wheel..." @cd gen/python && echo $$(uv run python -m setuptools_git_versioning) && uv build -define GEN_DOCS -## GRPC API Documentation - -{{- range .Files -}} -{{- $$file_name := .Name -}} - -{{/* --- SERVICES & METHODS --- */}} -{{- if .HasServices}} -{{range .Services -}} - - -### {{.Name}} ({{$$file_name}}) -{{.Description}} - -{{range .Methods -}} - - -#### {{.Name}} - -{{.Description}} - -_[{{.RequestLongType}}](#{{.RequestFullType | anchor}}){{if .RequestStreaming}} stream{{end}} / [{{.ResponseLongType}}](#{{.ResponseFullType | anchor}}){{if .ResponseStreaming}} stream{{end}}_ - -{{end}}{{/* end methods */}} -{{- end}}{{/* end services */}} -{{- end}}{{/* end has_services */}} - -{{- if .HasMessages}} -
Messages ({{$$file_name}}) - -{{range .Messages -}} - -
{{.LongName}} -{{.Description}} - -{{if .HasFields -}} -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -{{range .Fields -}} - | {{.Name}} | [{{.LongType}}](#{{.FullType | anchor}}) | {{.Label}} | {{if (index .Options "deprecated"|default false)}}**Deprecated.** {{end}}{{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} | -{{- end -}} -{{- end -}} - -{{if .HasExtensions -}} -| Extension | Type | Base | Number | Description | -| --------- | ---- | ---- | ------ | ----------- | -{{range .Extensions -}} - | {{.Name}} | {{.LongType}} | {{.ContainingLongType}} | {{.Number}} | {{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} | -{{- end }} -{{- end }} -
- -{{- end }}{{/* end messages */}} -
-{{ end -}}{{/* end has_messages */}} - -{{- if .HasEnums}} -
Enums ({{$$file_name}}) - -{{range .Enums -}} - - -
{{.LongName}} -{{.Description}} - -| Name | Number | Description | -| ---- | ------ | ----------- | -{{range .Values -}} - | {{.Name}} | {{.Number}} | {{nobr .Description}} | -{{- end }} -
- -{{- end}}{{/* end enums */}} -
-{{ end -}}{{/* end has_enums */}} - -{{/* --- FILE-LEVEL EXTENSIONS --- */}} -{{- if .HasExtensions}} - - -### File-level Extensions ({{$$file_name}}) -| Extension | Type | Base | Number | Description | -| --------- | ---- | ---- | ------ | ----------- | -{{range .Extensions -}} - | {{.Name}} | {{.LongType}} | {{.ContainingLongType}} | {{.Number}} | {{nobr .Description}}{{if .DefaultValue}} Default: `{{.DefaultValue}}`{{end}} | -{{end}} -{{- end}}{{/* end HasExtensions */}} - -{{- end}}{{/* end files */}} - -endef -export GEN_DOCS - -.PHONY: gen.proto.docs -gen.proto.docs: ${PROTOC} ${PROTOC_GEN_DOC} - @rm -rf gen/docs && mkdir -p gen/docs - @echo "$$GEN_DOCS" > gen/docs/markdown.tmpl - @${PROTOC} \ - ${PROTO_SOURCES} \ - -I=proto \ - -I=$(PROTOC_INCLUDE) \ - --doc_out=gen/docs \ - --doc_opt=gen/docs/markdown.tmpl,docs.md:=buf/*,google/*,ocf/dp/dp.rules.proto - @sed -n '1,//p' README.md > README.tmp - @echo "" >> README.tmp - @cat gen/docs/docs.md >> README.tmp - @echo "" >> README.tmp - @sed -n '//,$$p' README.md >> README.tmp - @mv README.tmp README.md - # --- LOCAL RUNNING TARGETS --------------------------------------------------------------------- # .PHONY: run.db diff --git a/README.md b/README.md index f393586..f352549 100644 --- a/README.md +++ b/README.md @@ -217,746 +217,3 @@ $ make gen.proto.python This places the generated code in `gen/python`. See the `Makefile` for more external targets. - - -## GRPC API Documentation -
Messages (ocf/dp/dp-data.messages.proto) - - -
CreateForecastRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecaster | [Forecaster](#ocf-dp-Forecaster) | | || location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || init_time_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || values | [CreateForecastRequest.ForecastValue](#ocf-dp-CreateForecastRequest-ForecastValue) | repeated | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | optional | || created_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The UTC time to set as the created_timestamp for the forecast. Leave empty to use current time. This is useful for backfilling historical forecasts with accurate created timestamps, but should generally be left empty for new forecasts. | -
-
CreateForecastRequest.ForecastValue - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| horizon_mins | [uint32](#uint32) | | || p50_fraction | [float](#float) | | || other_statistics_fractions | [CreateForecastRequest.ForecastValue.OtherStatisticsFractionsEntry](#ocf-dp-CreateForecastRequest-ForecastValue-OtherStatisticsFractionsEntry) | repeated | Struct for storing additional statistics like p10, p90, mean etc. | -
-
CreateForecastRequest.ForecastValue.OtherStatisticsFractionsEntry - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | || value | [float](#float) | | | -
-
CreateForecastResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecast_uuid | [string](#string) | | | -
-
CreateForecasterRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| name | [string](#string) | | || version | [string](#string) | | | -
-
CreateForecasterResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecaster | [Forecaster](#ocf-dp-Forecaster) | | | -
-
CreateLocationRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_name | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || geometry_wkt | [string](#string) | | A geometry string in Well-Known-Text (WKT) format. Geometry type must be POINT, POLYGON, or MULTIPOLYGON, must have 2 dimensions, and must be in the EPSG:4326 coordinate system (longitude/latitude). || effective_capacity_watts | [uint64](#uint64) | | The effective capacity of the location in watts. This refers to the useable capacity for generation, not the installed capacity. If tracking of installed capacity is required, this should be stored in metadata. || location_type | [LocationType](#ocf-dp-LocationType) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | optional | || valid_from_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The UTC time from which this location is considered valid. Leave empty to use current time. || associated_latlng | [LatLng](#ocf-dp-LatLng) | optional | Optional latitude/longitude to associate with the location. Defaults to the centroid of the geometry if not provided. Not required for Point geometries. | -
-
CreateLocationResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || effective_capacity_watts | [uint64](#uint64) | | | -
-
CreateObservationsRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || observer_name | [string](#string) | | || values | [CreateObservationsRequest.Value](#ocf-dp-CreateObservationsRequest-Value) | repeated | | -
-
CreateObservationsRequest.Value - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || value_watts | [uint64](#uint64) | | | -
-
CreateObservationsResponse - - - -
-
CreateObserverRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| name | [string](#string) | | | -
-
CreateObserverResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| observer_uuid | [string](#string) | | || observer_name | [string](#string) | | | -
-
DeleteForecastRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || forecaster | [Forecaster](#ocf-dp-Forecaster) | | || init_time_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | | -
-
DeleteForecastResponse - - - -
-
ForecastDatum - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| init_timestamp | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || location_uuid | [string](#string) | | || forecaster_fullname | [string](#string) | | || horizon_mins | [uint32](#uint32) | | || p50_fraction | [float](#float) | | || other_statistics_fractions | [ForecastDatum.OtherStatisticsFractionsEntry](#ocf-dp-ForecastDatum-OtherStatisticsFractionsEntry) | repeated | || created_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || effective_capacity_watts | [uint64](#uint64) | | || metadata | [ForecastDatum.MetadataEntry](#ocf-dp-ForecastDatum-MetadataEntry) | repeated | || target_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | | -
-
ForecastDatum.MetadataEntry - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | || value | [string](#string) | | | -
-
ForecastDatum.OtherStatisticsFractionsEntry - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | || value | [float](#float) | | | -
-
Forecaster -Forecaster represents a generative source of predicted values. - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecaster_name | [string](#string) | | || forecaster_version | [string](#string) | | The version of the forecaster to use. If not specified, the latest version will be used. | -
-
GetForecastAsTimeseriesRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || horizon_mins | [uint32](#uint32) | | || time_window | [TimeWindow](#ocf-dp-TimeWindow) | | || forecaster | [Forecaster](#ocf-dp-Forecaster) | | || pivot_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The time to search backwards from to find forecasts. If not specified, the current time will be used. Forecasts created after this time are not included. || initialization_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | An individual init time to filter forecasts by. If specified, only forecasts with this init time will be returned. This enables fetching data from a single, specific forecast run. | -
-
GetForecastAsTimeseriesResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || values | [GetForecastAsTimeseriesResponse.Value](#ocf-dp-GetForecastAsTimeseriesResponse-Value) | repeated | | -
-
GetForecastAsTimeseriesResponse.Value - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| target_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || p50_value_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | || initialization_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || created_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || other_statistics_fractions | [GetForecastAsTimeseriesResponse.Value.OtherStatisticsFractionsEntry](#ocf-dp-GetForecastAsTimeseriesResponse-Value-OtherStatisticsFractionsEntry) | repeated | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | | -
-
GetForecastAsTimeseriesResponse.Value.OtherStatisticsFractionsEntry - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | || value | [float](#float) | | | -
-
GetForecastAtTimestampRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuids | [string](#string) | repeated | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The time to fetch predicted yields for. If not specified, the current time will be used. || forecaster | [Forecaster](#ocf-dp-Forecaster) | | | -
-
GetForecastAtTimestampResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || values | [GetForecastAtTimestampResponse.Value](#ocf-dp-GetForecastAtTimestampResponse-Value) | repeated | | -
-
GetForecastAtTimestampResponse.Value - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || value_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | || latlng | [LatLng](#ocf-dp-LatLng) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | || initialization_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || created_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || other_statistics_fractions | [GetForecastAtTimestampResponse.Value.OtherStatisticsFractionsEntry](#ocf-dp-GetForecastAtTimestampResponse-Value-OtherStatisticsFractionsEntry) | repeated | | -
-
GetForecastAtTimestampResponse.Value.OtherStatisticsFractionsEntry - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | || value | [float](#float) | | | -
-
GetLatestForecastsRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || pivot_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The time to search backwards from to find the 'latest' forecast. If not specified, the current time will be used. | -
-
GetLatestForecastsResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecasts | [GetLatestForecastsResponse.Forecast](#ocf-dp-GetLatestForecastsResponse-Forecast) | repeated | | -
-
GetLatestForecastsResponse.Forecast - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| initialization_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || created_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || forecaster | [Forecaster](#ocf-dp-Forecaster) | | || location_uuid | [string](#string) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | | -
-
GetLatestObservationsRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuids | [string](#string) | repeated | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || observer_name | [string](#string) | | || pivot_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The time to search backwards from to find the 'latest' observation. If not specified, the current time will be used. | -
-
GetLatestObservationsResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| observations | [GetLatestObservationsResponse.Observation](#ocf-dp-GetLatestObservationsResponse-Observation) | repeated | | -
-
GetLatestObservationsResponse.Observation - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || value_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | | -
-
GetLocationAsTimeseriesRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || time_window | [TimeWindow](#ocf-dp-TimeWindow) | | | -
-
GetLocationAsTimeseriesResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| values | [GetLocationAsTimeseriesResponse.LocationSnapshot](#ocf-dp-GetLocationAsTimeseriesResponse-LocationSnapshot) | repeated | | -
-
GetLocationAsTimeseriesResponse.LocationSnapshot - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || effective_capacity_watts | [uint64](#uint64) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | | -
-
GetLocationRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || include_geometry | [bool](#bool) | | If true, the geometry_wkb field will be included in the response. This may be very big, so only include if necessary. || pivot_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The UTC time the data should be valid for. Leave empty to use current time. | -
-
GetLocationResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || latlng | [LatLng](#ocf-dp-LatLng) | | || effective_capacity_watts | [uint64](#uint64) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | || geometry_wkb | [bytes](#bytes) | optional | | -
-
GetLocationsAsGeoJSONRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuids | [string](#string) | repeated | || unsimplified | [bool](#bool) | | If true, the GeoJSON will not be simplified. Defaults to false if not set to reduce response size. | -
-
GetLocationsAsGeoJSONResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| geojson | [string](#string) | | | -
-
GetObservationsAsTimeseriesRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || observer_name | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || time_window | [TimeWindow](#ocf-dp-TimeWindow) | | | -
-
GetObservationsAsTimeseriesResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || values | [GetObservationsAsTimeseriesResponse.Value](#ocf-dp-GetObservationsAsTimeseriesResponse-Value) | repeated | | -
-
GetObservationsAsTimeseriesResponse.Value - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || value_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | | -
-
GetObservationsAtTimestampRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuids | [string](#string) | repeated | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || observer_name | [string](#string) | | || timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The time to fetch observations for. If not specified, the current time will be used. | -
-
GetObservationsAtTimestampResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | || values | [GetObservationsAtTimestampResponse.Value](#ocf-dp-GetObservationsAtTimestampResponse-Value) | repeated | | -
-
GetObservationsAtTimestampResponse.Value - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || value_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | || latlng | [LatLng](#ocf-dp-LatLng) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | | -
-
GetWeekAverageDeltasRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || pivot_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | The characteristic time to detrmine averages for. The time component specifies the initialization time, and the date component is to define the end of the seven-day period over which to average. || forecaster | [Forecaster](#ocf-dp-Forecaster) | | || observer_name | [string](#string) | | | -
-
GetWeekAverageDeltasResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| deltas | [GetWeekAverageDeltasResponse.AverageDelta](#ocf-dp-GetWeekAverageDeltasResponse-AverageDelta) | repeated | || init_time_of_day | [string](#string) | | The initialisation time that was compared across the week. Formatted as HH:MM, e.g. "12:00" | -
-
GetWeekAverageDeltasResponse.AverageDelta - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| horizon_mins | [uint32](#uint32) | | || delta_fraction | [float](#float) | | || effective_capacity_watts | [uint64](#uint64) | | | -
-
ListForecastersRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecaster_names_filter | [string](#string) | repeated | Optional filter to only return forecasters from a given set. If empty, all forecasters will be returned. || latest_versions_only | [bool](#bool) | | If true, only the latest version of each forecaster will be returned. | -
-
ListForecastersResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecasters | [Forecaster](#ocf-dp-Forecaster) | repeated | | -
-
ListLocationsRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| energy_source_filter | [EnergySource](#ocf-dp-EnergySource) | optional | Optional filter to only return locations of a specific energy source. || location_type_filter | [LocationType](#ocf-dp-LocationType) | optional | Optional filter to only return locations of a specific location type. || location_uuids_filter | [string](#string) | repeated | Optional filter to only return locations from a given set. || organisation_id_filter | [string](#string) | optional | Optional filter to only return locations belonging to a specific organisation. || enclosing_location_uuid_filter | [string](#string) | optional | Optional filter to only return locations enclosed within a specific location. || enclosed_location_uuid_filter | [string](#string) | optional | Optional filter to only return locations that enclose a specific location. || location_names_filter | [string](#string) | repeated | Optional filter to only return locations with a specific name. | -
-
ListLocationsResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| locations | [ListLocationsResponse.LocationSummary](#ocf-dp-ListLocationsResponse-LocationSummary) | repeated | | -
-
ListLocationsResponse.LocationSummary - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || location_type | [LocationType](#ocf-dp-LocationType) | | || effective_capacity_watts | [uint64](#uint64) | | || latlng | [LatLng](#ocf-dp-LatLng) | | || metadata | [google.protobuf.Struct](#google-protobuf-Struct) | | | -
-
ListObserversRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| observer_names_filter | [string](#string) | repeated | Optional filter to only return observers from a given set. If empty, all observers will be returned. | -
-
ListObserversResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| observers | [ListObserversResponse.ObserverSummary](#ocf-dp-ListObserversResponse-ObserverSummary) | repeated | | -
-
ListObserversResponse.ObserverSummary - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| observer_uuid | [string](#string) | | || observer_name | [string](#string) | | | -
-
StreamCreateForecastsResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecast_uuids | [string](#string) | repeated | A list of the UUIDs generated for the successfully created forecasts. | -
-
StreamForecastDataRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuids | [string](#string) | repeated | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || time_window | [TimeWindow](#ocf-dp-TimeWindow) | | || forecasters | [Forecaster](#ocf-dp-Forecaster) | repeated | || include_metadata | [bool](#bool) | | | -
-
StreamForecastDataResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| values | [ForecastDatum](#ocf-dp-ForecastDatum) | repeated | | -
-
TimeWindow - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| start_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | The start of the time window, inclusive. Cannot be more than 7 days before end_timestamp_utc, nor more than 1 month in the future. || end_timestamp_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | | The end of the time window, inclusive. Cannot be more than 7 days after start_timestamp_utc. | -
-
UpdateForecasterRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| name | [string](#string) | | || new_version | [string](#string) | | | -
-
UpdateForecasterResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| forecaster | [Forecaster](#ocf-dp-Forecaster) | | | -
-
UpdateLocationOwnerRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || new_organisation_id | [string](#string) | | | -
-
UpdateLocationOwnerResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || organisation_id | [string](#string) | | | -
-
UpdateLocationRequest - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || energy_source | [EnergySource](#ocf-dp-EnergySource) | | || new_location_name | [string](#string) | optional | The new name for the location. || new_effective_capacity_watts | [uint64](#uint64) | optional | || new_metadata | [google.protobuf.Struct](#google-protobuf-Struct) | optional | The new metadata object to set for the location. Note that this will replace any existing metadata, so be sure to include existing fields where needed. || valid_from_utc | [google.protobuf.Timestamp](#google-protobuf-Timestamp) | optional | The UTC time from which this name is considered valid. Leave empty to use current time. | -
-
UpdateLocationResponse - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| location_uuid | [string](#string) | | || location_name | [string](#string) | | || effective_capacity_watts | [uint64](#uint64) | | | -
-
- - - - - -### DataPlatformDataService (ocf/dp/dp-data.service.proto) - - - - -#### GetForecastAsTimeseries - -GetForecastTimeseries fetches a 1-D horizontal slice of predicted data. -These values can either come from a sample of many forecasts; or from one specific forecast. -In the case of the sample, values whose timestamps are shared across overlapping forecasts -are cherry-picked based on the lowest allowable lead time (horizon). - -_[GetForecastAsTimeseriesRequest](#ocf-dp-GetForecastAsTimeseriesRequest) / [GetForecastAsTimeseriesResponse](#ocf-dp-GetForecastAsTimeseriesResponse)_ - - - -#### GetForecastAtTimestamp - -GetForecastAtTimestamp fetches a 1-D vertical slice of predicted data. -Useful for spatial snapshots at a given time, for instance to display on a map. - -_[GetForecastAtTimestampRequest](#ocf-dp-GetForecastAtTimestampRequest) / [GetForecastAtTimestampResponse](#ocf-dp-GetForecastAtTimestampResponse)_ - - - -#### GetObservationsAsTimeseries - -GetObservationsAsTimeseries fetches a 1-D horizontal slice of observed data. -It is the observations analogue of GetForecastAsTimeseries. - -_[GetObservationsAsTimeseriesRequest](#ocf-dp-GetObservationsAsTimeseriesRequest) / [GetObservationsAsTimeseriesResponse](#ocf-dp-GetObservationsAsTimeseriesResponse)_ - - - -#### GetObservationsAtTimestamp - -GetObservationAtTimestamp fetches a 1-D vertical slice of observation data. -It is the observations analogue of GetForecastsAtTimestamp. - -_[GetObservationsAtTimestampRequest](#ocf-dp-GetObservationsAtTimestampRequest) / [GetObservationsAtTimestampResponse](#ocf-dp-GetObservationsAtTimestampResponse)_ - - - -#### GetLocation - -GetLocation fetches a snapshot of information about a specific location at a point in time. -It can also optionally return the geometry of the location. - -_[GetLocationRequest](#ocf-dp-GetLocationRequest) / [GetLocationResponse](#ocf-dp-GetLocationResponse)_ - - - -#### GetLocationAsTimeseries - -GetLocationAsTimeseries fetches the history of a location across a given time window. - -_[GetLocationAsTimeseriesRequest](#ocf-dp-GetLocationAsTimeseriesRequest) / [GetLocationAsTimeseriesResponse](#ocf-dp-GetLocationAsTimeseriesResponse)_ - - - -#### CreateLocation - -CreateLocation registers a new location in which to log or forecast generation. - -_[CreateLocationRequest](#ocf-dp-CreateLocationRequest) / [CreateLocationResponse](#ocf-dp-CreateLocationResponse)_ - - - -#### UpdateLocation - -UpdateLocation modifies various attributes associated with a given location. - -_[UpdateLocationRequest](#ocf-dp-UpdateLocationRequest) / [UpdateLocationResponse](#ocf-dp-UpdateLocationResponse)_ - - - -#### UpdateLocationOwner - -UpdateLocationOwner changes the ownership of a location. - -_[UpdateLocationOwnerRequest](#ocf-dp-UpdateLocationOwnerRequest) / [UpdateLocationOwnerResponse](#ocf-dp-UpdateLocationOwnerResponse)_ - - - -#### ListLocations - -ListLocations fetches a list of registered locations that match the supplied filters. - -_[ListLocationsRequest](#ocf-dp-ListLocationsRequest) / [ListLocationsResponse](#ocf-dp-ListLocationsResponse)_ - - - -#### CreateForecaster - -CreateForecaster registers a new forecaster. -A forecaster is a producer of predicted values. Forecasters are differentiated by their name and version. - -_[CreateForecasterRequest](#ocf-dp-CreateForecasterRequest) / [CreateForecasterResponse](#ocf-dp-CreateForecasterResponse)_ - - - -#### UpdateForecaster - -UpdateForecaster modifies the version of an existing forecaster. - -_[UpdateForecasterRequest](#ocf-dp-UpdateForecasterRequest) / [UpdateForecasterResponse](#ocf-dp-UpdateForecasterResponse)_ - - - -#### ListForecasters - -ListForecasters fetches a list of registered forecasters that match the supplied filters. - -_[ListForecastersRequest](#ocf-dp-ListForecastersRequest) / [ListForecastersResponse](#ocf-dp-ListForecastersResponse)_ - - - -#### CreateForecast - -CreateForecast saves a timeseries of predicted values from a given forecaster. - -_[CreateForecastRequest](#ocf-dp-CreateForecastRequest) / [CreateForecastResponse](#ocf-dp-CreateForecastResponse)_ - - - -#### GetLatestForecasts - -GetLatestForecasts fetches metadata for the most recently produced forecasts. - -_[GetLatestForecastsRequest](#ocf-dp-GetLatestForecastsRequest) / [GetLatestForecastsResponse](#ocf-dp-GetLatestForecastsResponse)_ - - - -#### DeleteForecast - -DeleteForecast removes a series of forecast values from the database. - -_[DeleteForecastRequest](#ocf-dp-DeleteForecastRequest) / [DeleteForecastResponse](#ocf-dp-DeleteForecastResponse)_ - - - -#### CreateObserver - -CreateObserver registers a new observer. -An observer is a producer of observed, or measured, values. - -_[CreateObserverRequest](#ocf-dp-CreateObserverRequest) / [CreateObserverResponse](#ocf-dp-CreateObserverResponse)_ - - - -#### ListObservers - -ListObservers fetches a list of registered observers that match the supplied filters. - -_[ListObserversRequest](#ocf-dp-ListObserversRequest) / [ListObserversResponse](#ocf-dp-ListObserversResponse)_ - - - -#### CreateObservations - -CreateObservations saves a timeseries of observed values from a given observer. - -_[CreateObservationsRequest](#ocf-dp-CreateObservationsRequest) / [CreateObservationsResponse](#ocf-dp-CreateObservationsResponse)_ - - - -#### GetLatestObservations - -GetLatestObservation fetches the most recent observation for a given location and observer. - -_[GetLatestObservationsRequest](#ocf-dp-GetLatestObservationsRequest) / [GetLatestObservationsResponse](#ocf-dp-GetLatestObservationsResponse)_ - - - -#### GetLocationsAsGeoJSON - -GetLocationsAsGeoJSON fetches a given set of locations as GeoJSON, suitable for display on a -map or for integration with GIS software. - -_[GetLocationsAsGeoJSONRequest](#ocf-dp-GetLocationsAsGeoJSONRequest) / [GetLocationsAsGeoJSONResponse](#ocf-dp-GetLocationsAsGeoJSONResponse)_ - - - -#### GetWeekAverageDeltas - -GetWeekAverageDeltas fetches the average delta at the given init time over the past week. -This is useful for making adjustments based on recent performance. - -_[GetWeekAverageDeltasRequest](#ocf-dp-GetWeekAverageDeltasRequest) / [GetWeekAverageDeltasResponse](#ocf-dp-GetWeekAverageDeltasResponse)_ - - - -#### StreamForecastData - -StreamForecastData streams forecast data for a given location, forecasters, and time range. -Useful for analytics and performance monitoring. - -_[StreamForecastDataRequest](#ocf-dp-StreamForecastDataRequest) / [StreamForecastDataResponse](#ocf-dp-StreamForecastDataResponse) stream_ - - - -#### StreamCreateForecasts - -StreamCreateForecasts allows for efficient batch creation of multiple forecasts and their values. -Note: This method is executed in a single transaction. To prevent resource exhaustion, a maximum of 5000 forecasts can be sent per stream. Exceeding this limit will abort the stream and roll back all inserts. - -_[CreateForecastRequest](#ocf-dp-CreateForecastRequest) stream / [StreamCreateForecastsResponse](#ocf-dp-StreamCreateForecastsResponse)_ - - - - -
Messages (ocf/dp/dp.common.proto) - - -
LatLng -LatLng represents a WSG84 coordinate pair. -Float precision enables a resolution of about 1cm, -which is more precise than we'll ever have data for. - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| latitude | [float](#float) | | || longitude | [float](#float) | | | -
-
- -
Enums (ocf/dp/dp.common.proto) - - - -
EnergySource -EnergySource indicates the type of energy generation. -NOTE: These enum numbers are used to find the corresponding entry in the postgres database. -Do not change without considering this first! - -| Name | Number | Description | -| ---- | ------ | ----------- | -| ENERGY_SOURCE_UNSPECIFIED | 0 | || ENERGY_SOURCE_SOLAR | 1 | || ENERGY_SOURCE_WIND | 2 | | -
- -
LocationType -LocationType indicates the type of location. -NOTE: These enum numbers are used to find the corresponding entry in the postgres database. -Do not change without considering this first! -The values are spaced apart in order to allow for future expansion. - -| Name | Number | Description | -| ---- | ------ | ----------- | -| LOCATION_TYPE_UNSPECIFIED | 0 | || LOCATION_TYPE_SITE | 1 | || LOCATION_TYPE_GSP | 2 | || LOCATION_TYPE_DNO | 3 | || LOCATION_TYPE_NATION | 4 | || LOCATION_TYPE_STATE | 5 | || LOCATION_TYPE_COUNTY | 6 | || LOCATION_TYPE_CITY | 7 | || LOCATION_TYPE_PRIMARY_SUBSTATION | 8 | | -
- -
Permission -Permission indicates the level of access a user has to a resource. -NOTE: These enum numbers are used to find the corresponding entry in the postgres database. -Do not change without considering this first! - -| Name | Number | Description | -| ---- | ------ | ----------- | -| PERMISSION_UNSPECIFIED | 0 | || PERMISSION_READ | 1 | || PERMISSION_WRITE | 2 | | -
-
- - - - - - diff --git a/internal/server/postgres/dataserverimpl.go b/internal/server/postgres/dataserverimpl.go index 6b64d31..1aeb7f6 100644 --- a/internal/server/postgres/dataserverimpl.go +++ b/internal/server/postgres/dataserverimpl.go @@ -7,10 +7,13 @@ package postgres import ( + "bytes" + "cmp" "context" "errors" "fmt" "io" + "slices" "time" "github.com/google/uuid" @@ -1484,6 +1487,23 @@ func (s *DataPlatformDataServiceServerImpl) StreamCreateForecasts( return nil } + // Sort the batch to match the index to improve clustering in the database. + slices.SortFunc(forecastParams, func(a, b db.CreateForecastsParams) int { + if c := bytes.Compare(a.GeometryUuid[:], b.GeometryUuid[:]); c != 0 { + return c + } + + if c := cmp.Compare(a.SourceTypeID, b.SourceTypeID); c != 0 { + return c + } + + if c := cmp.Compare(a.ForecasterID, b.ForecasterID); c != 0 { + return c + } + + return bytes.Compare(b.ForecastUuid[:], a.ForecastUuid[:]) + }) + countF, err := querier.CreateForecasts(ctx, forecastParams) if err != nil || countF < int64(len(forecastParams)) { if err == nil { diff --git a/proto/ocf/dp/dp-data.messages.proto b/proto/ocf/dp/dp-data.messages.proto index 66671b6..6311958 100644 --- a/proto/ocf/dp/dp-data.messages.proto +++ b/proto/ocf/dp/dp-data.messages.proto @@ -332,7 +332,7 @@ message CreateForecastRequest { (buf.validate.field).float.lte = 1.1, (buf.validate.field).float.finite = true ]; - // Struct for storing additional statistics like p10, p90, mean etc. + // Struct for storing additional statistics/quartiles map other_statistics_fractions = 3 [ (buf.validate.field).map.min_pairs = 0, (buf.validate.field).map.max_pairs = 20, From 26a27f2739f8d5455c5d56c810a2048d343c2a9b Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:52:06 +0100 Subject: [PATCH 3/4] chore: docs --- internal/server/postgres/mappers.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/server/postgres/mappers.go b/internal/server/postgres/mappers.go index 9ab13b1..c668a87 100644 --- a/internal/server/postgres/mappers.go +++ b/internal/server/postgres/mappers.go @@ -29,7 +29,9 @@ func MapSlice[T, U any](input []T, mapper func(T) U) []U { } // timeWindowToPgWindow converts a TimeWindow protobuf message to a pair of pgtype.Timestamp values. -// If the TimeWindow is nil or its StartTimestampUtc is nil, it defaults to a window from 48 hours ago to 36 hours in the future. Protovalidate ensures at the boundary that the start is always before the end, so we don't need to check that here. +// If the TimeWindow is nil or its StartTimestampUtc is nil, it defaults to a window from 48 hours +// ago to 36 hours in the future. Protovalidate ensures at the boundary that the start is always +// before the end, so we don't need to check that here. func timeWindowToPgWindow( window *pb.TimeWindow, ) (start pgtype.Timestamp, end pgtype.Timestamp) { @@ -60,9 +62,9 @@ func timeptrToPgTimestamp(t *timestamppb.Timestamp) pgtype.Timestamp { // extractSIPStatSlice builds the array for a single p-level from a forecast's values. // Returns nil if no value in the series carries this statistic, so the column is stored as a -// NULL array rather than a materialised array of nulls (~3 bytes per forecast against ~130). -// Callers must have run validateForecastValues first: sqlc maps SMALLINT[] to []int16, which -// cannot express element-level nulls, so partial coverage would silently be written as zeros. +// NULL array rather than a materialised array of NULLs. +// NOTE: Callers must have run validateForecastValues first, otherwise partial forecasts would +// silently be infilled with zeros. func extractSIPStatSlice(values []*pb.CreateForecastRequest_ForecastValue, key string) []int16 { out := make([]int16, len(values)) present := false @@ -81,8 +83,9 @@ func extractSIPStatSlice(values []*pb.CreateForecastRequest_ForecastValue, key s return out } -// extractP50Slice builds the p50 array. P50 is a top-level field on ForecastValue rather than a -// key in OtherStatisticsFractions, and is always present. +// extractP50Slice builds the p50 array. +// P50 is a top-level field on ForecastValue rather than a key in OtherStatisticsFractions, +// and is always present. func extractP50Slice(values []*pb.CreateForecastRequest_ForecastValue) []int16 { out := make([]int16, len(values)) for i, v := range values { @@ -97,9 +100,7 @@ func sipToFraction(sip int16) float32 { return float32(sip) / 30000.0 } -// validateForecastValues checks the invariants the array storage layout depends on: -// at least two values, strictly increasing horizons, evenly spaced, and each optional statistic -// either present on every value or on none. +// validateForecastValues checks the forecast valiues against a set of rules. func validateForecastValues(values []*pb.CreateForecastRequest_ForecastValue) error { if len(values) < 2 { return errors.New("a forecast must contain at least two values") From d686ab288a9100b60cc99e959822376cd3600853 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:22:39 +0100 Subject: [PATCH 4/4] chore(bench): fix seeding --- internal/server/postgres/testdata/seeding.sql | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/server/postgres/testdata/seeding.sql b/internal/server/postgres/testdata/seeding.sql index ac8a5a6..7dd818f 100644 --- a/internal/server/postgres/testdata/seeding.sql +++ b/internal/server/postgres/testdata/seeding.sql @@ -1,3 +1,15 @@ +CREATE OR REPLACE FUNCTION spoof_uuidv7(ts timestamptz) RETURNS uuid AS $$ +SELECT encode( + set_bit( + set_bit( + overlay(uuid_send(gen_random_uuid()) placing + substring(int8send(floor(extract(epoch from ts) * 1000)::bigint) from 3) + from 1 for 6), + 52, 1), + 53, 1), +'hex')::uuid; +$$ LANGUAGE sql volatile; + CREATE OR REPLACE FUNCTION seed_db( name_prefix TEXT DEFAULT '', target_locations INTEGER DEFAULT 100, @@ -59,7 +71,7 @@ BEGIN (forecast_uuid, source_type_id, geometry_uuid, forecaster_id, init_time_utc, value_resolution_mins, target_period, p50_sips, p10_sips, p90_sips) SELECT - UUIDV7(gd.init_time_utc), 1, gd.geo_id, + SPOOF_UUIDV7(gd.init_time_utc AT TIME ZONE 'UTC'), 1, gd.geo_id, (SELECT forecaster_id FROM pred.forecasters WHERE forecaster_name = name_prefix || '_forecaster_1'), gd.init_time_utc, pgv_res_mins::SMALLINT, TSRANGE(gd.init_time_utc, gd.init_time_utc + (forecast_len_mins * INTERVAL '1 minute')),