Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
0ae6d87
feat: resolve field type conflicts across ingestors in distributed mode
parmesant Nov 11, 2026
043ee9f
fix: harden distributed schema resolution
parmesant Nov 11, 2026
d192fc7
refactor: make schema resolution on/off, scope tenants to cluster secret
parmesant Nov 11, 2026
5c1279a
fix: keep cached field renames stable across resolutions
parmesant Nov 12, 2026
abc3c85
refactor: simplify authority discovery, drop suffix depth limit
parmesant Nov 12, 2026
5c23c6f
fix: name conflict-renamed columns after their stored type
parmesant Nov 12, 2026
8d014cf
fix: keep both values when a conflict rename hits an existing key
parmesant Nov 14, 2026
ceb94b4
fix: address review findings in schema resolution
parmesant Nov 14, 2026
d87b944
fix(kafka): pause only the partition blocked by schema resolution
parmesant Nov 14, 2026
d866b0c
fix: accept stale-typed events after any resolved type change; order …
parmesant Nov 14, 2026
3099eab
fix(kafka): share partition pause across workers
parmesant Nov 15, 2026
7fe2b1a
fix: let cluster nodes resolve schemas for any tenant
parmesant Nov 17, 2026
2a7b4a4
fix: keep rename targets distinct and never reseed a saved registry
parmesant Nov 17, 2026
f665b07
fix: let nodes restart after the registry moves on; accept wide resol…
parmesant Nov 18, 2026
07172db
fix: require P_QUERIER_ENDPOINT on the querier
parmesant Nov 18, 2026
7057b45
fix text and timestamp conflict
nikhilsinhaparseable Nov 20, 2026
fa7879f
fix review comments
nikhilsinhaparseable Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docker-compose-distributed-test-with-kafka.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ services:
- P_SEND_ANONYMOUS_USAGE_DATA=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=query
- P_QUERIER_ENDPOINT=parseable-query:8000
- RUST_LOG=warn
networks:
- parseable-internal
Expand Down
1 change: 1 addition & 0 deletions docker-compose-distributed-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ services:
- P_SEND_ANONYMOUS_USAGE_DATA=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=query
- P_QUERIER_ENDPOINT=parseable-query:8000
- RUST_LOG=warn
networks:
- parseable-internal
Expand Down
1 change: 1 addition & 0 deletions docker-compose-gcs-distributed-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ services:
- P_SEND_ANONYMOUS_USAGE_DATA=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=query
- P_QUERIER_ENDPOINT=parseable-query:8000
- RUST_LOG=warn
- GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/${GCS_CREDENTIALS_FILE:-key.json}
networks:
Expand Down
29 changes: 27 additions & 2 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -554,15 +554,40 @@ pub struct Options {
long,
env = "P_QUERIER_ENDPOINT",
default_value = "",
help = "URL to connect to this specific querier. Default is the address of the server"
help = "Address (`<ip address / DNS>:<port>`) other nodes use to reach this querier. Required in query mode"
)]
pub querier_endpoint: String,

#[arg(
long,
env = "P_SCHEMA_RESOLUTION",
default_value = "true",
help = "Distributed ingestors: reconcile new field types with the schema authority (querier or Prism) before writing parquet. Upgrade the authority before the ingestors."
)]
pub schema_resolution: bool,

#[arg(
long,
env = "P_SCHEMA_RESOLVE_MAX_DEFERRED_CYCLES",
default_value = "3",
help = "Failed resolution cycles after which a stream rejects ingestion with 503 (0 = never)"
)]
pub schema_resolve_max_deferred_cycles: u32,

#[arg(
long,
env = "P_SCHEMA_RESOLVE_CONCURRENCY",
default_value = "8",
value_parser = value_parser!(u16).range(1..),
help = "Maximum concurrent schema resolution requests from one ingestor"
)]
pub schema_resolve_concurrency: u16,

#[arg(
long,
env = "P_PRISM_ENDPOINT",
default_value = "",
help = "URL to connect to the prism node. Default is the address of the server"
help = "Address (`<ip address / DNS>:<port>`) other nodes use to reach this prism node. Required in prism mode"
)]
pub prism_endpoint: String,

Expand Down
Loading
Loading