Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions src/handlers/airplane.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use crate::handlers::http::modal::{NodeMetadata, NodeType};
use crate::handlers::http::query::into_query;
use crate::handlers::livetail::cross_origin_config;
use crate::metrics::QUERY_EXECUTE_TIME;
use crate::option::Mode;
use crate::parseable::PARSEABLE;
use crate::query::{QUERY_SESSION, execute, resolve_stream_names};
use crate::utils::arrow::flight::{
Expand Down Expand Up @@ -152,10 +153,11 @@ impl FlightService for AirServiceImpl {
.await
.map_err(|_| Status::internal("Failed to parse query"))?;

let event = if send_to_ingester(
query.time_range.start.timestamp_millis(),
query.time_range.end.timestamp_millis(),
) {
let event = if PARSEABLE.options.mode != Mode::All
&& send_to_ingester(
query.time_range.start.timestamp_millis(),
query.time_range.end.timestamp_millis(),
) {
let sql = format!("select * from \"{}\"", stream_name);
let start_time = ticket.start_time.clone();
let end_time = ticket.end_time.clone();
Expand Down
10 changes: 10 additions & 0 deletions src/handlers/http/cluster/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,10 @@ where
Fut: Future<Output = Result<(), E>> + Send,
E: From<anyhow::Error> + Send + Sync + 'static,
{
if PARSEABLE.options.mode == Mode::All {
return Ok(());
}

let mut nodes = Vec::new();

let ingestor_infos: Vec<NodeMetadata> = get_node_info(NodeType::Ingestor, tenant_id)
Expand Down Expand Up @@ -1125,6 +1129,12 @@ pub async fn get_node_info<T: Metadata + DeserializeOwned>(
node_type: NodeType,
tenant_id: &Option<String>,
) -> anyhow::Result<Vec<T>> {
// Standalone mode owns every service locally. Ignore any stale distributed
// node metadata left in a reused object store so no API can contact it.
if PARSEABLE.options.mode == Mode::All {
return Ok(Vec::new());
}

let metadata = PARSEABLE
.metastore
.get_node_metadata(node_type, tenant_id)
Expand Down
2 changes: 1 addition & 1 deletion src/handlers/http/query_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ pub async fn query_context(

/// Fetches log context after the caller has authorized the dataset and made it
/// available on the local query node.
pub(crate) async fn query_context_for_authorized_dataset(
pub async fn query_context_for_authorized_dataset(
context_request: LogContextRequest,
tenant_id: &Option<String>,
) -> Result<LogContextResponse, QueryError> {
Expand Down
44 changes: 18 additions & 26 deletions src/interactive.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,11 +245,7 @@ const MODE_ENV: &str = "P_MODE";
/// Detects the server mode from the `P_MODE` env var or the `--mode` CLI arg.
fn detect_mode() -> Option<String> {
if let Ok(mode) = std::env::var(MODE_ENV) {
let mode = mode.to_lowercase();
if mode != "all" {
return Some(mode);
}
return None;
return Some(mode.to_lowercase());
}

let args: Vec<String> = std::env::args().collect();
Expand All @@ -260,13 +256,18 @@ fn detect_mode() -> Option<String> {
arg.strip_prefix("--mode=").map(|v| v.to_lowercase())
};
if let Some(mode) = value {
return (mode != "all").then_some(mode);
return Some(mode);
}
}

None
}

/// Whether startup will use standalone mode, including the CLI default.
pub fn enterprise_all_mode_selected() -> bool {
detect_mode().as_deref().unwrap_or("all") == "all"
}

/// Returns storage-specific env var prompts for the given subcommand.
fn get_storage_prompts(subcommand: &str) -> Vec<EnvPrompt> {
match subcommand {
Expand Down Expand Up @@ -596,7 +597,7 @@ pub fn prompt_enterprise_envs() -> Vec<(String, String)> {
let is_interactive = io::stdin().is_terminal();
let mut collected: Vec<(String, String)> = Vec::new();

// Phase 1: base enterprise vars (cluster secret and mode)
// Phase 1: distributed-mode cluster secret.
let base_prompts = get_enterprise_base_prompts();
collect_prompts(
&base_prompts,
Expand All @@ -605,8 +606,7 @@ pub fn prompt_enterprise_envs() -> Vec<(String, String)> {
&mut collected,
);

// Phase 2: now P_MODE is in the environment (from env, CLI, .parseable.env,
// or the interactive prompt above), so mode-specific prompts resolve correctly.
// Phase 2: mode-specific settings.
let mode_prompts = get_enterprise_mode_prompts();
if !mode_prompts.is_empty() {
let mode = detect_mode().unwrap_or_else(|| "unknown".to_string());
Expand Down Expand Up @@ -695,26 +695,18 @@ fn collect_prompts(
println!();
}

/// Returns base enterprise env var prompts (cluster secret and mode).
/// Returns base enterprise prompts. Standalone mode needs no cluster secret.
fn get_enterprise_base_prompts() -> Vec<EnvPrompt> {
let mut prompts = vec![EnvPrompt {
env_var: "P_CLUSTER_SECRET",
display_name: "Cluster Secret",
required: true,
is_secret: true,
}];

// Enterprise rejects "all" mode, so prompt if not explicitly set
if detect_mode().is_none() {
prompts.push(EnvPrompt {
env_var: "P_MODE",
display_name: "Server Mode (query, ingest, index, prism)",
if enterprise_all_mode_selected() {
Vec::new()
} else {
vec![EnvPrompt {
env_var: "P_CLUSTER_SECRET",
display_name: "Cluster Secret",
required: true,
is_secret: false,
});
is_secret: true,
}]
}

prompts
}

/// Returns mode-specific enterprise env var prompts.
Expand Down
8 changes: 5 additions & 3 deletions src/prism/logstream/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ use crate::{
query::{QueryError, update_schema_when_distributed},
},
hottier::HotTierError,
option::Mode,
parseable::{PARSEABLE, StreamNotFound},
query::{CountsRequest, CountsResponse, error::ExecuteError},
rbac::{Users, map::SessionKey, role::Action},
Expand Down Expand Up @@ -107,9 +108,10 @@ async fn get_stats(
let stats = stats::get_current_stats(stream_name, "json", tenant_id)
.ok_or_else(|| StreamNotFound(stream_name.to_owned()))?;

let ingestor_stats = if PARSEABLE
.get_stream(stream_name, tenant_id)
.is_ok_and(|stream| stream.get_stream_type() == StreamType::UserDefined)
let ingestor_stats = if PARSEABLE.options.mode != Mode::All
&& PARSEABLE
.get_stream(stream_name, tenant_id)
.is_ok_and(|stream| stream.get_stream_type() == StreamType::UserDefined)
{
Some(fetch_stats_from_ingestors(stream_name, tenant_id).await?)
} else {
Expand Down
26 changes: 22 additions & 4 deletions src/storage/localfs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ use crate::{
};

use super::{
ALERTS_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError, ObjectStorageProvider,
PARSEABLE_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY,
TOMBSTONE_ROOT_DIRECTORY,
ALERTS_ROOT_DIRECTORY, KEYSTONE_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError,
ObjectStorageProvider, PARSEABLE_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME,
STREAM_ROOT_DIRECTORY, TARGETS_ROOT_DIRECTORY, TOMBSTONE_ROOT_DIRECTORY,
};

#[derive(Debug, Clone, clap::Args)]
Expand Down Expand Up @@ -532,6 +532,8 @@ impl ObjectStorage for LocalFS {
USERS_ROOT_DIR,
ALERTS_ROOT_DIRECTORY,
SETTINGS_ROOT_DIRECTORY,
TARGETS_ROOT_DIRECTORY,
KEYSTONE_ROOT_DIRECTORY,
TOMBSTONE_ROOT_DIRECTORY,
];

Expand All @@ -546,6 +548,8 @@ impl ObjectStorage for LocalFS {
PARSEABLE_ROOT_DIRECTORY,
ALERTS_ROOT_DIRECTORY,
SETTINGS_ROOT_DIRECTORY,
TARGETS_ROOT_DIRECTORY,
KEYSTONE_ROOT_DIRECTORY,
TOMBSTONE_ROOT_DIRECTORY,
];

Expand Down Expand Up @@ -909,7 +913,10 @@ mod list_streams_tombstone_tests {

use super::{LocalFS, ObjectStorage};
use crate::storage::object_storage::{to_bytes, tombstone_path};
use crate::storage::{STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY};
use crate::storage::{
KEYSTONE_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY,
TARGETS_ROOT_DIRECTORY,
};
use relative_path::RelativePathBuf;

// Deliberately not using `object_storage::stream_json_path` here: it
Expand Down Expand Up @@ -939,6 +946,17 @@ mod list_streams_tombstone_tests {
assert!(listed.contains("mystream"));
}

#[tokio::test]
async fn metadata_directories_are_not_treated_as_streams() {
let dir = TempDir::new().unwrap();
let storage = LocalFS::new(dir.path().to_path_buf());
std::fs::create_dir_all(dir.path().join(KEYSTONE_ROOT_DIRECTORY)).unwrap();
std::fs::create_dir_all(dir.path().join(TARGETS_ROOT_DIRECTORY)).unwrap();

let listed = storage.list_streams().await.unwrap();
assert!(listed.is_empty());
}

#[tokio::test]
async fn stream_mid_deletion_is_skipped_not_a_listing_error() {
let dir = TempDir::new().unwrap();
Expand Down
1 change: 1 addition & 0 deletions src/storage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ pub const SCHEMA_FILE_NAME: &str = ".schema";
pub const ALERTS_ROOT_DIRECTORY: &str = ".alerts";
pub const SETTINGS_ROOT_DIRECTORY: &str = ".settings";
pub const TARGETS_ROOT_DIRECTORY: &str = ".targets";
pub const KEYSTONE_ROOT_DIRECTORY: &str = ".keystone";
pub const MANIFEST_FILE: &str = "manifest.json";
// top-level registry of streams currently being deleted; kept outside every
// stream's own prefix so a bulk prefix-delete can never sweep up a marker
Expand Down
32 changes: 30 additions & 2 deletions src/storage/object_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicU8, Ordering};
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Semaphore;
use tokio::task::JoinSet;
use tracing::{Instrument, error, info, info_span, warn};
use ulid::Ulid;
Expand All @@ -73,6 +74,29 @@ use ulid::Ulid;
/// later cleanup attempt removes the local file.
static UPLOADED_STAGED_FILES_PENDING_CLEANUP: Lazy<DashSet<PathBuf>> = Lazy::new(DashSet::new);

/// Global limits keep standalone ingest + query workloads inside process FD budget.
/// A per-stream upload limit multiplied concurrency by number of streams.
static OBJECT_STORE_UPLOAD_SLOTS: Lazy<Arc<Semaphore>> = Lazy::new(|| {
Arc::new(Semaphore::new(concurrency_limit(
"P_MAX_CONCURRENT_OBJECT_STORE_UPLOADS",
100,
)))
});
static OBJECT_STORE_STREAM_SYNC_SLOTS: Lazy<Arc<Semaphore>> = Lazy::new(|| {
Arc::new(Semaphore::new(concurrency_limit(
"P_MAX_CONCURRENT_STREAM_SYNCS",
16,
)))
});

fn concurrency_limit(variable: &str, default: usize) -> usize {
std::env::var(variable)
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(default)
}

/// Claims a staged Parquet for upload unless its remote upload has already
/// succeeded and only local deletion remains pending.
fn claim_staged_file_for_upload(path: PathBuf) -> Option<PathBuf> {
Expand Down Expand Up @@ -1121,7 +1145,6 @@ async fn process_parquet_files(
) -> Result<Vec<UploadedParquetFile>, ObjectStorageError> {
retry_pending_staged_file_cleanup();

let semaphore = Arc::new(tokio::sync::Semaphore::new(100));
let mut join_set = JoinSet::new();
let object_store = PARSEABLE.storage().get_object_store();

Expand Down Expand Up @@ -1158,7 +1181,7 @@ async fn process_parquet_files(
for path in parquet_paths {
spawn_parquet_upload_task(
&mut join_set,
semaphore.clone(),
OBJECT_STORE_UPLOAD_SLOTS.clone(),
object_store.clone(),
upload_context,
stream_name,
Expand Down Expand Up @@ -1774,6 +1797,11 @@ pub fn sync_all_streams(joinset: &mut JoinSet<Result<(), ObjectStorageError>>) {
let span = info_span!("stream_upload", stream_name = %stream_name);
joinset.spawn_on(
async move {
let _stream_sync_permit = OBJECT_STORE_STREAM_SYNC_SLOTS
.clone()
.acquire_owned()
.await
.expect("object store stream sync semaphore should remain open");
let start = Instant::now();
let result = object_store
.upload_files_from_staging(&stream_name, id)
Expand Down
Loading