Skip to content

Commit b1ccf63

Browse files
author
amvanbaren
committed
Redis Caching
- Remove ehcache.xml - Use Caffeine for webresource caching - Configure Redis and Caffeine in code - Redis cluster config - Don't include baseUrl in unpkg browse data - Add SearchResult to fix search caching - Use constants for search sortBy values
1 parent d02ca60 commit b1ccf63

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+984
-419
lines changed

‎docker-compose.yml‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,157 @@ services:
4747
profiles:
4848
- kibana
4949

50+
redis-node-1:
51+
image: redis:7.2
52+
container_name: redis-node-1
53+
ports:
54+
- "7001:7000"
55+
command: redis-server /etc/redis.conf
56+
volumes:
57+
- ./redis.conf:/etc/redis.conf
58+
- /redis/node-1:/data
59+
environment:
60+
- REDISCLI_AUTH=openvsx
61+
healthcheck:
62+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
63+
interval: 5s
64+
retries: 5
65+
profiles:
66+
- redis
67+
68+
redis-node-2:
69+
image: redis:7.2
70+
container_name: redis-node-2
71+
depends_on:
72+
redis-node-1:
73+
condition: service_healthy
74+
ports:
75+
- "7002:7000"
76+
command: redis-server /etc/redis.conf
77+
volumes:
78+
- ./redis.conf:/etc/redis.conf
79+
- /redis/node-2:/data
80+
environment:
81+
- REDISCLI_AUTH=openvsx
82+
healthcheck:
83+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
84+
interval: 5s
85+
retries: 5
86+
profiles:
87+
- redis
88+
89+
redis-node-3:
90+
image: redis:7.2
91+
container_name: redis-node-3
92+
depends_on:
93+
redis-node-2:
94+
condition: service_healthy
95+
ports:
96+
- "7003:7000"
97+
command: redis-server /etc/redis.conf
98+
volumes:
99+
- ./redis.conf:/etc/redis.conf
100+
- /redis/node-3:/data
101+
environment:
102+
- REDISCLI_AUTH=openvsx
103+
healthcheck:
104+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
105+
interval: 5s
106+
retries: 5
107+
profiles:
108+
- redis
109+
110+
redis-node-4:
111+
image: redis:7.2
112+
container_name: redis-node-4
113+
depends_on:
114+
redis-node-3:
115+
condition: service_healthy
116+
ports:
117+
- "7004:7000"
118+
command: redis-server /etc/redis.conf
119+
volumes:
120+
- ./redis.conf:/etc/redis.conf
121+
- /redis/node-4:/data
122+
environment:
123+
- REDISCLI_AUTH=openvsx
124+
healthcheck:
125+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
126+
interval: 5s
127+
retries: 5
128+
profiles:
129+
- redis
130+
131+
redis-node-5:
132+
image: redis:7.2
133+
container_name: redis-node-5
134+
depends_on:
135+
redis-node-4:
136+
condition: service_healthy
137+
ports:
138+
- "7005:7000"
139+
command: redis-server /etc/redis.conf
140+
volumes:
141+
- ./redis.conf:/etc/redis.conf
142+
- /redis/node-5:/data
143+
environment:
144+
- REDISCLI_AUTH=openvsx
145+
healthcheck:
146+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
147+
interval: 5s
148+
retries: 5
149+
profiles:
150+
- redis
151+
152+
redis-node-6:
153+
image: redis:7.2
154+
container_name: redis-node-6
155+
depends_on:
156+
redis-node-5:
157+
condition: service_healthy
158+
ports:
159+
- "7006:7000"
160+
command: redis-server /etc/redis.conf
161+
volumes:
162+
- ./redis.conf:/etc/redis.conf
163+
- /redis/node-6:/data
164+
environment:
165+
- REDISCLI_AUTH=openvsx
166+
healthcheck:
167+
test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
168+
interval: 5s
169+
retries: 5
170+
profiles:
171+
- redis
172+
173+
redis-cluster-init:
174+
image: redis:7.2
175+
depends_on:
176+
- redis-node-1
177+
- redis-node-2
178+
- redis-node-3
179+
- redis-node-4
180+
- redis-node-5
181+
- redis-node-6
182+
environment:
183+
- REDISCLI_AUTH=openvsx
184+
entrypoint: >
185+
bash -c "
186+
sleep 10;
187+
echo yes | redis-cli --user openvsx --cluster create
188+
redis-node-1:7000 redis-node-2:7000 redis-node-3:7000
189+
redis-node-4:7000 redis-node-5:7000 redis-node-6:7000
190+
--cluster-replicas 1"
191+
profiles:
192+
- redis
193+
194+
redisinsight:
195+
image: redis/redisinsight
196+
ports:
197+
- '5540:5540'
198+
profiles:
199+
- redisinsight
200+
50201
server:
51202
image: openjdk:17
52203
working_dir: /app

‎redis.conf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Configuration for Redis nodes in docker-compose.yml
2+
port 7000
3+
cluster-enabled yes
4+
cluster-config-file nodes.conf
5+
cluster-node-timeout 5000
6+
appendonly yes
7+
maxmemory 64mb
8+
maxmemory-policy allkeys-lru
9+
user default off
10+
user openvsx on >openvsx ~* +@all
11+
masteruser openvsx
12+
masterauth openvsx

‎server/build.gradle‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,15 @@ def versions = [
3434
woodstox: '6.4.0',
3535
jobrunr: '7.5.0',
3636
bucket4j: '0.12.7',
37-
ehcache: '3.10.8',
37+
bucket4j_redis: '8.10.1',
3838
tika: '3.1.0',
3939
bouncycastle: '1.80',
4040
commons_lang3: '3.12.0',
4141
jaxb_api: '2.3.1',
4242
jaxb_impl: '2.3.8',
43-
gatling: '3.13.5',
44-
loki4j: '1.4.2'
43+
gatling: '3.9.5',
44+
loki4j: '1.4.2',
45+
embedded_redis: '1.4.3'
4546
]
4647
ext['junit-jupiter.version'] = versions.junit
4748
sourceCompatibility = versions.java
@@ -80,6 +81,7 @@ dependencies {
8081
implementation "org.springframework.boot:spring-boot-starter-jooq"
8182
implementation "org.springframework.boot:spring-boot-starter-data-jpa"
8283
implementation "org.springframework.boot:spring-boot-starter-data-elasticsearch"
84+
implementation "org.springframework.boot:spring-boot-starter-data-redis"
8385
implementation "org.springframework.boot:spring-boot-starter-security"
8486
implementation "org.springframework.boot:spring-boot-starter-actuator"
8587
implementation "org.springframework.boot:spring-boot-starter-cache"
@@ -89,8 +91,9 @@ dependencies {
8991
implementation "org.springframework.session:spring-session-jdbc"
9092
implementation "org.springframework.retry:spring-retry"
9193
implementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
92-
implementation "org.ehcache:ehcache:${versions.ehcache}"
94+
implementation "com.github.ben-manes.caffeine:caffeine"
9395
implementation "com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:${versions.bucket4j}"
96+
implementation "com.bucket4j:bucket4j-redis:${versions.bucket4j_redis}"
9497
implementation "org.jobrunr:jobrunr-spring-boot-3-starter:${versions.jobrunr}"
9598
implementation "org.flywaydb:flyway-core:${versions.flyway}"
9699
implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}"
@@ -112,6 +115,7 @@ dependencies {
112115
implementation "io.micrometer:micrometer-tracing"
113116
implementation "io.micrometer:micrometer-tracing-bridge-otel"
114117
implementation "io.opentelemetry:opentelemetry-exporter-zipkin"
118+
implementation "com.github.codemonstur:embedded-redis:${versions.embedded_redis}"
115119
runtimeOnly "io.micrometer:micrometer-registry-prometheus"
116120
runtimeOnly "org.postgresql:postgresql"
117121
jooqGenerator "org.postgresql:postgresql"

‎server/src/dev/resources/application.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,16 @@ spring:
1616
exclude: org.springframework.boot.actuate.autoconfigure.tracing.zipkin.ZipkinAutoConfiguration
1717
profiles:
1818
include: ovsx
19-
cache:
20-
jcache:
21-
config: classpath:ehcache.xml
19+
data:
20+
redis:
21+
# redis standalone configuration
22+
host: localhost
23+
port: 6379
24+
# connect to redis cluster configured in docker-compose.yml
25+
# cluster:
26+
# nodes: '127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006'
27+
# username: openvsx
28+
# password: openvsx
2229
datasource:
2330
url: jdbc:postgresql://localhost:5432/postgres
2431
username: gitpod
@@ -101,6 +108,7 @@ org:
101108

102109
bucket4j:
103110
enabled: true
111+
cache-to-use: redis-jedis # use redis-cluster-jedis when running redis cluster
104112
filters:
105113
- cache-name: buckets
106114
url: '/api/-/(namespace/create|publish)'
@@ -140,7 +148,10 @@ ovsx:
140148
databasesearch:
141149
enabled: false
142150
elasticsearch:
151+
enabled: true
143152
clear-on-start: true
153+
redis:
154+
embedded: true
144155
eclipse:
145156
base-url: https://api.eclipse.org
146157
publisher-agreement:

‎server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ object Scenarios {
318318
val categories = Array("", "Programming Languages", "Themes", "Snippets", "Debuggers", "Linters", "Other")
319319
val sizes = Array("", "5", "500")
320320
val offsets = Array("", "1", "100", "25000")
321-
val sortBys = Array("", "relevance", "timestamp", "averageRating", "downloadCount")
321+
val sortBys = Array("", "relevance", "timestamp", "rating", "downloadCount")
322322
val sortOrders = Array("", "asc", "desc")
323323
val includeAllVersions = Array("", "true", "false")
324324

‎server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.eclipse.openvsx.repositories.RepositoryService;
2222
import org.eclipse.openvsx.search.ExtensionSearch;
2323
import org.eclipse.openvsx.search.ISearchService;
24+
import org.eclipse.openvsx.search.SearchResult;
2425
import org.eclipse.openvsx.search.SearchUtilService;
2526
import org.eclipse.openvsx.storage.StorageUtilService;
2627
import org.eclipse.openvsx.util.*;
@@ -29,7 +30,6 @@
2930
import org.springframework.beans.factory.annotation.Value;
3031
import org.springframework.cache.annotation.Cacheable;
3132
import org.springframework.data.domain.PageRequest;
32-
import org.springframework.data.elasticsearch.core.SearchHits;
3333
import org.springframework.http.HttpStatus;
3434
import org.springframework.http.ResponseEntity;
3535
import org.springframework.stereotype.Component;
@@ -257,11 +257,11 @@ public SearchResultJson search(ISearchService.Options options) {
257257
return json;
258258
}
259259

260-
var searchHits = search.search(options);
261-
if(searchHits.hasSearchHits()) {
262-
json.setExtensions(toSearchEntries(searchHits, options));
260+
var result = search.search(options);
261+
if(result.hasSearchHits()) {
262+
json.setExtensions(toSearchEntries(result, options));
263263
json.setOffset(options.requestedOffset());
264-
json.setTotalSize((int) searchHits.getTotalHits());
264+
json.setTotalSize((int) result.getTotalHits());
265265
} else {
266266
json.setExtensions(Collections.emptyList());
267267
}
@@ -724,9 +724,9 @@ public ResultJson deleteReview(String namespace, String extensionName) {
724724
return ResultJson.success("Deleted review for " + NamingUtil.toExtensionId(extension));
725725
}
726726

727-
private LinkedHashMap<Long, ExtensionVersion> getLatestVersions(SearchHits<ExtensionSearch> searchHits) {
728-
var ids = searchHits.stream()
729-
.map(searchHit -> searchHit.getContent().getId())
727+
private LinkedHashMap<Long, ExtensionVersion> getLatestVersions(SearchResult result) {
728+
var ids = result.getHits().stream()
729+
.map(ExtensionSearch::getId)
730730
.distinct()
731731
.collect(Collectors.toList());
732732

@@ -756,9 +756,9 @@ private LinkedHashMap<Long, ExtensionVersion> findLatestVersions(Collection<Long
756756
}
757757

758758

759-
private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
759+
private List<SearchEntryJson> toSearchEntries(SearchResult result, ISearchService.Options options) {
760760
var serverUrl = UrlUtil.getBaseUrl();
761-
var latestVersions = getLatestVersions(searchHits);
761+
var latestVersions = getLatestVersions(result);
762762
var membershipsByNamespaceId = getMemberships(latestVersions.values());
763763
var searchEntries = latestVersions.entrySet().stream()
764764
.map(e -> {

‎server/src/main/java/org/eclipse/openvsx/RegistryAPI.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.eclipse.openvsx.entities.SemanticVersion;
2323
import org.eclipse.openvsx.json.*;
2424
import org.eclipse.openvsx.search.ISearchService;
25+
import org.eclipse.openvsx.search.SortBy;
2526
import org.eclipse.openvsx.util.*;
2627
import org.slf4j.Logger;
2728
import org.slf4j.LoggerFactory;
@@ -757,8 +758,8 @@ public ResponseEntity<SearchResultJson> search(
757758
@RequestParam(defaultValue = "desc")
758759
@Parameter(description = "Descending or ascending sort order", schema = @Schema(type = "string", allowableValues = {"asc", "desc"}))
759760
String sortOrder,
760-
@RequestParam(defaultValue = "relevance")
761-
@Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {"relevance", "timestamp", "averageRating", "downloadCount"}))
761+
@RequestParam(defaultValue = SortBy.RELEVANCE)
762+
@Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {SortBy.RELEVANCE, SortBy.TIMESTAMP, SortBy.RATING, SortBy.DOWNLOADS}))
762763
String sortBy,
763764
@RequestParam(defaultValue = "false")
764765
@Parameter(description = "Whether to include information on all available versions for each returned entry")

‎server/src/main/java/org/eclipse/openvsx/RegistryApplication.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
@EnableScheduling
3636
@EnableRetry
3737
@EnableAsync
38-
@EnableCaching(proxyTargetClass = true)
3938
@EnableConfigurationProperties(OAuth2AttributesConfig.class)
4039
public class RegistryApplication {
4140

0 commit comments

Comments
 (0)