diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 8aa5e8184689..c7cc3bf48d50 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1585,7 +1585,9 @@ These Historical configurations can be defined in the `historical/runtime.proper |`druid.segmentCache.announceIntervalMillis`|How frequently to announce segments while segments are loading from cache. Set this value to zero to wait for all segments to be loaded before announcing.|5000 (5 seconds)| |`druid.segmentCache.numLoadingThreads`|How many segments to drop or load concurrently from deep storage. Note that the work of loading segments involves downloading segments from deep storage, decompressing them and loading them to a memory mapped location. So the work is not all I/O Bound. Depending on CPU and network load, one could possibly increase this config to a higher value.|max(1,Number of cores / 6)| |`druid.segmentCache.numBootstrapThreads`|How many segments to load concurrently during historical startup.|`druid.segmentCache.numLoadingThreads`| -|`druid.segmentCache.lazyLoadOnStart`|Whether or not to load segment columns metadata lazily during historical startup. When set to true, Historical startup time will be dramatically improved by deferring segment loading until the first time that segment takes part in a query, which will incur this cost instead.|false| +|`druid.segmentCache.lazyLoadOnStart`|_DEPRECATED_ Use `druid.segmentCache.startupLoadStrategy` instead. Whether or not to load segment columns metadata lazily during historical startup. When set to true, Historical startup time will be dramatically improved by deferring segment loading until the first time that segment takes part in a query, which will incur this cost instead.|false| +|`druid.segmentCache.startupLoadStrategy.type`|Selects the segment column metadata loading strategy during historical startup. Possible values are `loadAllEagerly`, `loadAllLazily`, and `loadEagerlyBeforePeriod`. More details on each strategy below.|`loadAllEagerly`| +|`druid.segmentCache.startupLoadStrategy.period`|This property is used only when the startup load strategy is set to `loadEagerlyBeforePeriod`. Suppose `t` is the timestamp when the Historical started up. Any segment metadata with an interval that falls within the range `[t - startupLoadPeriod, t]` will be loaded. For example, setting this property to `P7D` causes Historical services to eagerly load data segments covering intervals spanning 7 days or less.|`null`| |`druid.segmentCache.numThreadsToLoadSegmentsIntoPageCacheOnDownload`|Number of threads to asynchronously read segment index files into null output stream on each new segment download after the Historical service finishes bootstrapping. Recommended to set to 1 or 2 or leave unspecified to disable. See also `druid.segmentCache.numThreadsToLoadSegmentsIntoPageCacheOnBootstrap`|0| |`druid.segmentCache.numThreadsToLoadSegmentsIntoPageCacheOnBootstrap`|Number of threads to asynchronously read segment index files into null output stream during Historical service bootstrap. This thread pool is terminated after Historical service finishes bootstrapping. Recommended to set to half of available cores. If left unspecified, `druid.segmentCache.numThreadsToLoadSegmentsIntoPageCacheOnDownload` will be used. If both configs are unspecified, this feature is disabled. Preemptively loading segments into page cache helps in the sense that later when a segment is queried, it's already in page cache and only a minor page fault needs to be triggered instead of a more costly major page fault to make the query latency more consistent. Note that loading segment into page cache just does a blind loading of segment index files and will evict any existing segments from page cache at the discretion of operating system when the total segment size on local disk is larger than the page cache usable in the RAM, which roughly equals to total available RAM in the host - druid process memory including both heap and direct memory allocated - memory used by other non druid processes on the host, so it is the user's responsibility to ensure the host has enough RAM to host all the segments to avoid random evictions to fully leverage this feature.|`druid.segmentCache.numThreadsToLoadSegmentsIntoPageCacheOnDownload`| @@ -1602,6 +1604,14 @@ In `druid.segmentCache.locationSelector.strategy`, one of `leastBytesUsed`, `rou Note that if `druid.segmentCache.numLoadingThreads` > 1, multiple threads can download different segments at the same time. In this case, with the `leastBytesUsed` strategy or `mostAvailableSize` strategy, Historicals may select a sub-optimal storage location because each decision is based on a snapshot of the storage location status of when a segment is requested to download. +In `druid.segmentCache.startupLoadStrategy`, one of `loadAllEagerly`, `loadAllLazily`, or `loadEagerlyBeforePeriod` could be specified to represent the strategy to load segments when starting the Historical service. + +|Strategy|Description| +|--------|-----------| +|`loadAllEagerly`|The default startup strategy. The Historical service will load all segment column metadata immediately during the initial startup process.| +|`loadAllLazily`|To significantly improve historical system startup time, segments are not loaded during the initial startup sequence. Instead, the loading cost is deferred, and will be incurred the first time a segment is referenced by a query.| +|`loadEagerlyBeforePeriod`|Provides a balance between fast startup and query performance. The Historical service will eagerly load column metadata only for segments that fall within the most recent period defined by `druid.segmentCache.startupLoadPeriod`. Segments outside this recent period will be loaded on-demand when first queried.| + #### Historical query configs ##### Concurrent requests diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderConfig.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderConfig.java index e4822f0661b7..bce04cdc760b 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderConfig.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderConfig.java @@ -22,6 +22,9 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Lists; +import org.apache.druid.server.coordination.startup.HistoricalStartupCacheLoadStrategy; +import org.apache.druid.server.coordination.startup.LoadAllEagerlyStrategy; +import org.apache.druid.server.coordination.startup.LoadAllLazilyStrategy; import org.apache.druid.utils.RuntimeInfo; import java.io.File; @@ -41,9 +44,16 @@ public class SegmentLoaderConfig @JsonProperty private List locations = Collections.emptyList(); + /** + * @deprecated Use {@link #startupLoadStrategy} instead. + */ + @Deprecated @JsonProperty("lazyLoadOnStart") private boolean lazyLoadOnStart = false; + @JsonProperty("startupLoadStrategy") + private HistoricalStartupCacheLoadStrategy startupLoadStrategy = null; + @JsonProperty("deleteOnRemove") private boolean deleteOnRemove = true; @@ -84,11 +94,24 @@ public List getLocations() return locations; } + /** + * @deprecated Use {@link #getStartupCacheLoadStrategy()} instead. + * Removal of this method in the future will requires a change in {@link #getStartupCacheLoadStrategy()} + * to default to {@link LoadAllEagerlyStrategy#STRATEGY_NAME} when {@link #startupLoadStrategy} is null. + */ + @Deprecated public boolean isLazyLoadOnStart() { return lazyLoadOnStart; } + public HistoricalStartupCacheLoadStrategy getStartupCacheLoadStrategy() + { + return startupLoadStrategy == null + ? isLazyLoadOnStart() ? new LoadAllLazilyStrategy() : new LoadAllEagerlyStrategy() + : startupLoadStrategy; + } + public boolean isDeleteOnRemove() { return deleteOnRemove; @@ -184,6 +207,7 @@ public String toString() return "SegmentLoaderConfig{" + "locations=" + locations + ", lazyLoadOnStart=" + lazyLoadOnStart + + ", startupLoadStrategy=" + startupLoadStrategy + ", deleteOnRemove=" + deleteOnRemove + ", dropSegmentDelayMillis=" + dropSegmentDelayMillis + ", announceIntervalMillis=" + announceIntervalMillis + diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java index 104304dce6a4..5ed0176e1397 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLocalCacheManager.java @@ -42,6 +42,7 @@ import org.apache.druid.segment.ReferenceCountedSegmentProvider; import org.apache.druid.segment.Segment; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.server.coordination.startup.HistoricalStartupCacheLoadStrategy; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.utils.CloseableUtils; @@ -77,6 +78,7 @@ public class SegmentLocalCacheManager implements SegmentCacheManager private final SegmentLoaderConfig config; private final ObjectMapper jsonMapper; + private final HistoricalStartupCacheLoadStrategy loadStrategy; private final List locations; @@ -127,6 +129,7 @@ public SegmentLocalCacheManager( this.locations = locations; this.strategy = strategy; this.indexIO = indexIO; + this.loadStrategy = config.getStartupCacheLoadStrategy(); log.info("Using storage location strategy[%s].", this.strategy.getClass().getSimpleName()); @@ -853,7 +856,8 @@ public void mount(StorageLocation mountLocation) throws SegmentLoadingException final SegmentizerFactory factory = getSegmentFactory(storageDir); @SuppressWarnings("ObjectEquality") - final boolean lazy = config.isLazyLoadOnStart() && lazyLoadCallback != SegmentLazyLoadFailCallback.NOOP; + final boolean lazy = lazyLoadCallback != SegmentLazyLoadFailCallback.NOOP + && loadStrategy.shouldLoadLazily(dataSegment); final Segment segment = factory.factorize(dataSegment, storageDir, lazy, lazyLoadCallback); // wipe load callback after calling lazyLoadCallback = SegmentLazyLoadFailCallback.NOOP; diff --git a/server/src/main/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategy.java b/server/src/main/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategy.java new file mode 100644 index 000000000000..dfa804ede691 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategy.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.timeline.DataSegment; + +/** + * Strategy for determining whether segments should be loaded lazily or eagerly during + * Historical process startup. Lazy loading can help to lower Historical startup time at the + * expense of query latency, by deferring the loading process to the first access of that segment. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = LoadAllEagerlyStrategy.STRATEGY_NAME, value = LoadAllEagerlyStrategy.class), + @JsonSubTypes.Type(name = LoadAllLazilyStrategy.STRATEGY_NAME, value = LoadAllLazilyStrategy.class), + @JsonSubTypes.Type(name = LoadEagerlyBeforePeriod.STRATEGY_NAME, value = LoadEagerlyBeforePeriod.class) +}) +public interface HistoricalStartupCacheLoadStrategy +{ + /** + * Indicates whether the provided segment should be loaded lazily during Historical startup. + * + * @param segment the segment being evaluated + * @return {@code true} if the segment should be loaded lazily, {@code false} if it should be loaded eagerly. + */ + boolean shouldLoadLazily(DataSegment segment); +} diff --git a/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllEagerlyStrategy.java b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllEagerlyStrategy.java new file mode 100644 index 000000000000..c06973eb7a34 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllEagerlyStrategy.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.timeline.DataSegment; + +import java.util.Locale; + +/** + * Eagerly loads column metadata for all segments at Historical startup. + *

+ * Optimizes for predictable first-query latency at the cost of longer startup time and higher I/O during bootstrap. + * {@link #shouldLoadLazily(DataSegment)} always returns {@code false}. + */ +public class LoadAllEagerlyStrategy implements HistoricalStartupCacheLoadStrategy +{ + private static final Logger log = new Logger(LoadAllEagerlyStrategy.class); + + public static final String STRATEGY_NAME = "loadAllEagerly"; + + public LoadAllEagerlyStrategy() + { + log.info("Using [%s] strategy", STRATEGY_NAME); + } + + @Override + public boolean shouldLoadLazily(DataSegment segment) + { + return false; + } + + @Override + public String toString() + { + return String.format(Locale.ROOT, "{type=%s}", STRATEGY_NAME); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllLazilyStrategy.java b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllLazilyStrategy.java new file mode 100644 index 000000000000..60f4d7af4aa2 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadAllLazilyStrategy.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.timeline.DataSegment; + +import java.util.Locale; + +/** + * Defers column metadata loading for all segments until first access. + *

+ * Minimizes Historical startup time and I/O during bootstrap at the expense of increased latency on the first + * query that touches a segment. {@link #shouldLoadLazily(DataSegment)} always returns {@code true}. + */ +public class LoadAllLazilyStrategy implements HistoricalStartupCacheLoadStrategy +{ + private static final Logger log = new Logger(LoadAllLazilyStrategy.class); + + public static final String STRATEGY_NAME = "loadAllLazily"; + + public LoadAllLazilyStrategy() + { + log.info("Using [%s] strategy", STRATEGY_NAME); + } + + @Override + public boolean shouldLoadLazily(DataSegment segment) + { + return true; + } + + @Override + public String toString() + { + return String.format(Locale.ROOT, "{type=%s}", STRATEGY_NAME); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriod.java b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriod.java new file mode 100644 index 000000000000..61c1c0fe56f5 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriod.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; + +import java.util.Locale; + +/** + * Eagerly loads column metadata for segments whose intervals overlap a recent sliding window; all others load lazily. + *

+ * Balances bootstrap time and first-query performance by eagerly loading only "hot" segments. The window is + * computed as {@code [now - period, now]} at Historical startup. + */ +public class LoadEagerlyBeforePeriod implements HistoricalStartupCacheLoadStrategy +{ + private static final Logger log = new Logger(LoadEagerlyBeforePeriod.class); + public static final String STRATEGY_NAME = "loadEagerlyBeforePeriod"; + + private final Interval eagerLoadingInterval; + + @VisibleForTesting + @JsonCreator + public LoadEagerlyBeforePeriod( + @JsonProperty("period") Period eagerLoadingPeriod + ) + { + if (eagerLoadingPeriod == null) { + throw DruidException + .forPersona(DruidException.Persona.OPERATOR) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "druid.segmentCache.startupLoadStrategy.period must be configured for Historical startup strategy[%s].", + STRATEGY_NAME + ); + } + + DateTime now = DateTimes.nowUtc(); + this.eagerLoadingInterval = new Interval(now.minus(eagerLoadingPeriod), now); + + log.info("Using [%s] strategy with Interval[%s]", STRATEGY_NAME, eagerLoadingInterval); + } + + @VisibleForTesting + public Interval getEagerLoadingInterval() + { + return this.eagerLoadingInterval; + } + + @Override + public boolean shouldLoadLazily(DataSegment segment) + { + return !segment.getInterval().overlaps(eagerLoadingInterval); + } + + @Override + public String toString() + { + return String.format(Locale.ROOT, "{type=%s,interval=%s}", STRATEGY_NAME, getEagerLoadingInterval()); + } +} diff --git a/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java b/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java index 4afc745ae4fe..5be0a499b0f0 100644 --- a/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java +++ b/server/src/main/java/org/apache/druid/server/metrics/SegmentStatsMonitor.java @@ -33,6 +33,7 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.segment.loading.SegmentLoaderConfig; import org.apache.druid.server.coordination.SegmentLoadDropHandler; +import org.apache.druid.server.coordination.startup.LoadAllLazilyStrategy; import java.util.Map; @@ -63,7 +64,7 @@ public SegmentStatsMonitor( SegmentLoaderConfig segmentLoaderConfig ) { - if (segmentLoaderConfig.isLazyLoadOnStart()) { + if (segmentLoaderConfig.getStartupCacheLoadStrategy() instanceof LoadAllLazilyStrategy) { // log message ensures there is an error displayed at startup if this fails as the exception isn't logged. log.error("Monitor doesn't support working with lazy loading on start"); // throw this exception it kill the process at startup @@ -71,7 +72,6 @@ public SegmentStatsMonitor( } this.serverConfig = serverConfig; this.segmentLoadDropHandler = segmentLoadDropHandler; - } @Override diff --git a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java index 04eafa41ed71..0a92d102577a 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/loading/SegmentLocalCacheManagerTest.java @@ -44,6 +44,8 @@ import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.TestSegmentUtils; import org.apache.druid.segment.column.ColumnConfig; +import org.apache.druid.server.coordination.startup.HistoricalStartupCacheLoadStrategy; +import org.apache.druid.server.coordination.startup.LoadAllLazilyStrategy; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; @@ -847,9 +849,9 @@ public void testGetBootstrapSegmentLazy() throws SegmentLoadingException final SegmentLoaderConfig loaderConfig = new SegmentLoaderConfig() { @Override - public boolean isLazyLoadOnStart() + public HistoricalStartupCacheLoadStrategy getStartupCacheLoadStrategy() { - return true; + return new LoadAllLazilyStrategy(); } @Override diff --git a/server/src/test/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategyTest.java b/server/src/test/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategyTest.java new file mode 100644 index 000000000000..dc2dd02ebd93 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordination/startup/HistoricalStartupCacheLoadStrategyTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.segment.TestHelper; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +public class HistoricalStartupCacheLoadStrategyTest +{ + private final ObjectMapper mapper = TestHelper.JSON_MAPPER; + + @Test + public void testDeserializeLoadAllEagerly() throws Exception + { + final String json = "{\"type\":\"" + LoadAllEagerlyStrategy.STRATEGY_NAME + "\"}"; + final HistoricalStartupCacheLoadStrategy historicalStartupLoadingStrategy = + mapper.readValue(json, HistoricalStartupCacheLoadStrategy.class); + Assert.assertTrue(historicalStartupLoadingStrategy instanceof LoadAllEagerlyStrategy); + + LoadAllEagerlyStrategy strategy = (LoadAllEagerlyStrategy) historicalStartupLoadingStrategy; + Assert.assertEquals("{type=" + LoadAllEagerlyStrategy.STRATEGY_NAME + "}", strategy.toString()); + } + + @Test + public void testDeserializeLoadAllLazily() throws Exception + { + final String json = "{\"type\":\"" + LoadAllLazilyStrategy.STRATEGY_NAME + "\"}"; + final HistoricalStartupCacheLoadStrategy historicalStartupLoadingStrategy = + mapper.readValue(json, HistoricalStartupCacheLoadStrategy.class); + Assert.assertTrue(historicalStartupLoadingStrategy instanceof LoadAllLazilyStrategy); + LoadAllLazilyStrategy strategy = (LoadAllLazilyStrategy) historicalStartupLoadingStrategy; + Assert.assertEquals("{type=" + LoadAllLazilyStrategy.STRATEGY_NAME + "}", strategy.toString()); + } + + @Test + public void testDeserializeLoadEagerlyBeforePeriod() throws Exception + { + final String json = "{\"type\":\"" + LoadEagerlyBeforePeriod.STRATEGY_NAME + "\",\"period\":\"P3D\"}"; + final HistoricalStartupCacheLoadStrategy historicalStartupLoadingStrategy = + mapper.readValue(json, HistoricalStartupCacheLoadStrategy.class); + + Assert.assertTrue(historicalStartupLoadingStrategy instanceof LoadEagerlyBeforePeriod); + LoadEagerlyBeforePeriod strategy = (LoadEagerlyBeforePeriod) historicalStartupLoadingStrategy; + + Assert.assertEquals(Period.days(3).toStandardDuration(), strategy.getEagerLoadingInterval().toDuration()); + Assert.assertEquals("{type=" + LoadEagerlyBeforePeriod.STRATEGY_NAME + ",interval=" + strategy.getEagerLoadingInterval() + "}", strategy.toString()); + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriodTest.java b/server/src/test/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriodTest.java new file mode 100644 index 000000000000..0000bd2e62b8 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordination/startup/LoadEagerlyBeforePeriodTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordination.startup; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.segment.TestSegmentUtils; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +public class LoadEagerlyBeforePeriodTest +{ + @Test + public void testLoadEagerlyForSegments() + { + DateTime now = DateTimes.nowUtc(); + LoadEagerlyBeforePeriod strategy = new LoadEagerlyBeforePeriod(Period.days(7)); + + final DataSegment withinRange = TestSegmentUtils.makeSegment("foo2", "v1", new Interval(now.minusDays(2), now.minusDays(1))); + final DataSegment infiniteRange = TestSegmentUtils.makeSegment("foo1", "v1", Intervals.ETERNITY); + final DataSegment overlappingRange = TestSegmentUtils.makeSegment("foo3", "v1", new Interval(now.minusDays(8), now.minusDays(6))); + + Assert.assertFalse(strategy.shouldLoadLazily(withinRange)); + Assert.assertFalse(strategy.shouldLoadLazily(infiniteRange)); + Assert.assertFalse(strategy.shouldLoadLazily(overlappingRange)); + } + + @Test + public void testLoadLazilyForSegments() + { + LoadEagerlyBeforePeriod strategy = new LoadEagerlyBeforePeriod(Period.days(1)); + DateTime now = DateTimes.nowUtc(); + final DataSegment segmentInFuture = TestSegmentUtils.makeSegment("foo2", "v1", new Interval(now.plusDays(1), now.plusDays(2))); + final DataSegment segmentTooLate = TestSegmentUtils.makeSegment("foo3", "v1", new Interval(now.minusDays(8), now.minusDays(7))); + + Assert.assertTrue(strategy.shouldLoadLazily(segmentInFuture)); + Assert.assertTrue(strategy.shouldLoadLazily(segmentTooLate)); + } +} diff --git a/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java b/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java index 373b1c1dd2d6..5e05c84675a7 100644 --- a/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java +++ b/server/src/test/java/org/apache/druid/server/metrics/SegmentStatsMonitorTest.java @@ -28,6 +28,7 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.segment.loading.SegmentLoaderConfig; import org.apache.druid.server.coordination.SegmentLoadDropHandler; +import org.apache.druid.server.coordination.startup.LoadAllLazilyStrategy; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -73,9 +74,7 @@ public void setUp() public void testLazyLoadOnStartThrowsException() { SegmentLoaderConfig segmentLoaderConfig = Mockito.mock(SegmentLoaderConfig.class); - Mockito.when(segmentLoaderConfig.isLazyLoadOnStart()).thenReturn(true); - - //should throw an exception here + Mockito.when(segmentLoaderConfig.getStartupCacheLoadStrategy()).thenReturn(new LoadAllLazilyStrategy()); new SegmentStatsMonitor(druidServerConfig, segmentLoadDropMgr, segmentLoaderConfig); }