diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b8653b469a..d6323fd8df 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1311,6 +1311,85 @@ jobs: - uses: ./.github/actions/teardown + CITestsCachingChaoticHA: + needs: PublishJibContainers + container: ubuntu:latest + runs-on: ubuntu-latest + timeout-minutes: 15 + + services: + caching-service: + image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} + env: + MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true + MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown + CACHING_STORAGE_MODE: infinispan + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]" + JGROUPS_BIND_PORT: 7098 + JGROUPS_BIND_ADDRESS: caching-service + JGROUPS_KEYEXCHANGE_PORT: 7118 + APIML_SERVICE_HOSTNAME: caching-service + caching-service-2: + image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} + env: + MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true + MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown + CACHING_STORAGE_MODE: infinispan + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]" + JGROUPS_BIND_PORT: 7098 + JGROUPS_BIND_ADDRESS: caching-service-2 + JGROUPS_KEYEXCHANGE_PORT: 7118 + APIML_SERVICE_HOSTNAME: caching-service-2 + caching-service-3: + image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} + env: + MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true + MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown + CACHING_STORAGE_MODE: infinispan + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]" + JGROUPS_BIND_PORT: 7098 + JGROUPS_BIND_ADDRESS: caching-service-3 + JGROUPS_KEYEXCHANGE_PORT: 7118 + APIML_SERVICE_HOSTNAME: caching-service-3 + mock-services: + image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} + discovery-service: + image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} + volumes: + - /api-defs:/api-defs + gateway-service: + image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - uses: ./.github/actions/setup + + - name: Run Caching Chaotic HA Tests + run: > + ./gradlew :integration-tests:runChaoticHATests --tests org.zowe.apiml.integration.ha.CachingService + --info -Denvironment.config=-ha -Denvironment.offPlatform=true -DcloudGateway.enabled=false + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + + - name: Correct Permisions + run: | + chmod 755 -R .gradle + # Coverage results are not stored in this job as it would not provide much additional data + - name: Store results + uses: actions/upload-artifact@v4 + if: always() + with: + name: CITestsCachingChaoticHA-${{ env.JOB_ID }} + path: | + integration-tests/build/reports/** + results/** + + - uses: ./.github/actions/teardown + CITestsWithInfinispan: needs: PublishJibContainers runs-on: ubuntu-latest diff --git a/caching-service-package/src/main/resources/bin/start.sh b/caching-service-package/src/main/resources/bin/start.sh index 8b462f3055..d9d4ce3fd1 100755 --- a/caching-service-package/src/main/resources/bin/start.sh +++ b/caching-service-package/src/main/resources/bin/start.sh @@ -243,6 +243,16 @@ if [ "${ATTLS_SERVER_ENABLED}" = "true" -a "${APIML_ATTLS_LOAD_KEYRING:-false}" keystore_location= fi +# migration step of Infinispan since version 2.18.4 (see #https://github.com/zowe/api-layer/pull/3960) +original_infinispan_data_location="${ZWE_configs_storage_infinispan_persistence_dataLocation:-${ZWE_zowe_workspaceDirectory:-$(pwd)}}/caching-service/data" +if [ -d "${original_infinispan_data_location}" ]; then + mv -f "${original_infinispan_data_location}" "${ZWE_zowe_workspaceDirectory:-$(pwd)}/caching-service/${ZWE_haInstance_id:-localhost}/${ZWE_configs_storage_infinispan_persistence_dataLocation:-data}" +fi +original_infinispan_index_location="${ZWE_configs_storage_infinispan_persistence_indexLocation:-${ZWE_zowe_workspaceDirectory:-$(pwd)}}/caching-service/index" +if [ -d "${original_infinispan_index_location}" ]; then + mv -f "${original_infinispan_index_location}" "${ZWE_zowe_workspaceDirectory:-$(pwd)}/caching-service/${ZWE_haInstance_id:-localhost}/${ZWE_configs_storage_infinispan_persistence_indexLocation:-index}" +fi + CACHING_CODE=CS _BPXK_AUTOCVT=OFF _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} java \ @@ -273,8 +283,6 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} java \ -Djgroups.bind.port=${ZWE_configs_storage_infinispan_jgroups_port:-7098} \ -Djgroups.keyExchange.port=${ZWE_configs_storage_infinispan_jgroups_keyExchange_port:-7118} \ -Djgroups.tcp.diag.enabled=${ZWE_configs_storage_infinispan_jgroups_tcp_diag_enabled:-false} \ - -Dcaching.storage.infinispan.persistence.dataLocation=${ZWE_configs_storage_infinispan_persistence_dataLocation:-data} \ - -Dcaching.storage.infinispan.persistence.indexLocation=${ZWE_configs_storage_infinispan_persistence_indexLocation:-index} \ -Dcaching.storage.infinispan.initialHosts=${ZWE_configs_storage_infinispan_initialHosts:-localhost[7098]} \ -Dserver.address=0.0.0.0 \ -Dserver.ssl.enabled=${ZWE_configs_server_ssl_enabled:-true} \ diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingController.java b/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingController.java index b73250ca97..81c954a80f 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingController.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingController.java @@ -13,6 +13,7 @@ import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand; import io.swagger.v3.oas.annotations.Operation; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -28,6 +29,7 @@ import javax.servlet.http.HttpServletRequest; import java.util.Optional; +@Slf4j @RestController @RequiredArgsConstructor @RequestMapping("/api/v1") @@ -128,7 +130,13 @@ public ResponseEntity getAllMapItems(@PathVariable String mapKey, HttpSe try { return new ResponseEntity<>(storage.getAllMapItems(s, mapKey), HttpStatus.OK); } catch (Exception exception) { - return handleIncompatibleStorageMethod(exception, request.getRequestURL()); + if ( + (exception instanceof StorageException) && + Messages.INCOMPATIBLE_STORAGE_METHOD.getKey().equals(((StorageException) exception).getKey()) + ) { + return handleIncompatibleStorageMethod(exception, request.getRequestURL()); + } + return handleInternalError(exception, request.getRequestURL()); } } ).orElseGet(this::getUnauthorizedResponse); @@ -145,7 +153,13 @@ public ResponseEntity getAllMaps(HttpServletRequest request) { try { return new ResponseEntity<>(storage.getAllMaps(s), HttpStatus.OK); } catch (Exception exception) { - return handleIncompatibleStorageMethod(exception, request.getRequestURL()); + if ( + (exception instanceof StorageException) && + Messages.INCOMPATIBLE_STORAGE_METHOD.getKey().equals(((StorageException) exception).getKey()) + ) { + return handleIncompatibleStorageMethod(exception, request.getRequestURL()); + } + return handleInternalError(exception, request.getRequestURL()); } } ).orElseGet(this::getUnauthorizedResponse); @@ -199,6 +213,7 @@ public ResponseEntity update(@RequestBody KeyValue keyValue, HttpServlet private ResponseEntity exceptionToResponse(StorageException exception) { + log.debug("Storage exception", exception); Message message = messageService.createMessage(exception.getKey(), (Object[]) exception.getParameters()); return new ResponseEntity<>(message.mapToView(), exception.getStatus()); } @@ -298,12 +313,14 @@ private Optional getHeader(HttpServletRequest request, String headerName } private ResponseEntity handleInternalError(Exception exception, StringBuffer requestURL) { + log.debug("Internal error occurred", exception); Messages internalServerError = Messages.INTERNAL_SERVER_ERROR; Message message = messageService.createMessage(internalServerError.getKey(), requestURL, exception.getMessage(), exception.toString()); return new ResponseEntity<>(message.mapToView(), internalServerError.getStatus()); } private ResponseEntity handleIncompatibleStorageMethod(Exception exception, StringBuffer requestURL) { + log.debug("Incompatible storage method", exception); Messages internalServerError = Messages.INCOMPATIBLE_STORAGE_METHOD; Message message = messageService.createMessage(internalServerError.getKey(), requestURL, exception.getMessage(), exception.toString()); return new ResponseEntity<>(message.mapToView(), internalServerError.getStatus()); diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/Messages.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/Messages.java index b2446d0c81..38e7d9859c 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/Messages.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/Messages.java @@ -26,7 +26,8 @@ public enum Messages { PAYLOAD_TOO_LARGE("org.zowe.apiml.cache.payloadTooLarge", HttpStatus.BAD_REQUEST), INTERNAL_SERVER_ERROR("org.zowe.apiml.common.internalRequestError", HttpStatus.INTERNAL_SERVER_ERROR), MISSING_CERTIFICATE("org.zowe.apiml.cache.missingCertificate", HttpStatus.UNAUTHORIZED), - INCOMPATIBLE_STORAGE_METHOD("org.zowe.apiml.cache.incompatibleStorageMethod", HttpStatus.BAD_REQUEST); + INCOMPATIBLE_STORAGE_METHOD("org.zowe.apiml.cache.incompatibleStorageMethod", HttpStatus.BAD_REQUEST), + CACHE_NOT_AVAILABLE("org.zowe.apiml.cache.notAvailable", HttpStatus.SERVICE_UNAVAILABLE); private final String key; private final HttpStatus status; } diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java index e4b18215b7..3983038b19 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java @@ -10,6 +10,7 @@ package org.zowe.apiml.caching.service.infinispan.config; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.infinispan.commons.api.CacheContainerAdmin; import org.infinispan.commons.dataconversion.MediaType; @@ -21,6 +22,7 @@ import org.infinispan.lock.api.ClusteredLock; import org.infinispan.lock.api.ClusteredLockManager; import org.infinispan.manager.DefaultCacheManager; +import org.infinispan.partitionhandling.AvailabilityException; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -28,20 +30,25 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ResourceLoader; +import org.zowe.apiml.caching.service.Messages; import org.zowe.apiml.caching.service.Storage; +import org.zowe.apiml.caching.service.StorageException; import org.zowe.apiml.caching.service.infinispan.exception.InfinispanConfigException; import org.zowe.apiml.caching.service.infinispan.storage.InfinispanStorage; import javax.annotation.PostConstruct; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl; import static org.zowe.apiml.security.SecurityUtils.isKeyring; +@Slf4j @Configuration @ConfigurationProperties(value = "caching.storage.infinispan") @ConditionalOnProperty(name = "caching.storage.mode", havingValue = "infinispan") @@ -55,12 +62,6 @@ public class InfinispanConfig implements InitializingBean { @Value("${caching.storage.infinispan.initialHosts}") private String initialHosts; - @Value("${caching.storage.infinispan.persistence.dataLocation}") - private String dataLocation; - - @Value("${caching.storage.infinispan.persistence.indexLocation:index}") - private String indexLocation; - @Value("${server.ssl.keyStoreType}") private String keyStoreType; @@ -85,6 +86,8 @@ public class InfinispanConfig implements InitializingBean { @Value("${server.attlsServer.enabled:false}") private boolean isServerAttlsEnabled; + private AtomicReference zoweInvalidatedTokenLock = new AtomicReference<>(); + @Override public void afterPropertiesSet() { updateKeyring(); @@ -98,7 +101,22 @@ void updateKeyring() { } } - @Bean + static String getRootFolder() { + // using getenv().get is because of system compatibility (see non-case sensitive on Windows) + String instanceId = System.getenv().get("ZWE_haInstance_id"); + if (StringUtils.isBlank(instanceId)) { + instanceId = "localhost"; + } + + String workspaceFolder = System.getenv().get("ZWE_zowe_workspaceDirectory"); + if (StringUtils.isBlank(workspaceFolder)) { + return Paths.get("caching-service", instanceId).toString(); + } else { + return Paths.get(workspaceFolder, "caching-service", instanceId).toString(); + } + } + + @Bean(destroyMethod = "stop") synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) { System.setProperty("jgroups.tcpping.initial_hosts", initialHosts); System.setProperty("jgroups.bind.port", port); @@ -124,20 +142,18 @@ synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) { } catch (IOException e) { throw new InfinispanConfigException("Can't read configuration file", e); } + holder.getGlobalConfigurationBuilder().globalState().persistentLocation(getRootFolder()).enable(); + holder.newConfigurationBuilder("default").persistence() + .addSoftIndexFileStore() + .clustering().cacheMode(CacheMode.DIST_SYNC); DefaultCacheManager cacheManager = new DefaultCacheManager(holder, true); ConfigurationBuilder builder = new ConfigurationBuilder(); - builder.clustering() - .cacheMode(CacheMode.REPL_SYNC) - .encoding() - .mediaType("application/x-jboss-marshalling"); - - builder.persistence() - .passivation(true) - .addSoftIndexFileStore() - .shared(false) - .dataLocation(dataLocation).indexLocation(indexLocation); + builder + .encoding().mediaType("application/x-jboss-marshalling") + .persistence().addSoftIndexFileStore().clustering() + .clustering().cacheMode(CacheMode.DIST_SYNC); List caches = Arrays.asList("zoweCache", "zoweInvalidatedTokenCache"); caches.forEach(cacheName -> cacheManager.administration() @@ -151,17 +167,37 @@ synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) { return cacheManager; } - @Bean - public ClusteredLock lock(DefaultCacheManager cacheManager) { - ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(cacheManager); - clm.defineLock("zoweInvalidatedTokenLock"); - return clm.get("zoweInvalidatedTokenLock"); - } + private ClusteredLock lock(DefaultCacheManager cacheManager) { + ClusteredLock lock = zoweInvalidatedTokenLock.get(); + if (lock != null) { + return lock; + } + try { + synchronized (zoweInvalidatedTokenLock) { + lock = zoweInvalidatedTokenLock.get(); + if (lock == null) { + ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(cacheManager); + // it can throw AvailabilityException + clm.defineLock("zoweInvalidatedTokenLock"); + lock = clm.get("zoweInvalidatedTokenLock"); + } + zoweInvalidatedTokenLock.set(lock); + } + return lock; + } catch (AvailabilityException ae) { + log.debug("Cannot obtain lock", ae); + throw new StorageException(Messages.CACHE_NOT_AVAILABLE.getKey(), Messages.CACHE_NOT_AVAILABLE.getStatus(), ae.getMessage()); + } + } @Bean - public Storage storage(DefaultCacheManager cacheManager, ClusteredLock clusteredLock) { - return new InfinispanStorage(cacheManager.getCache("zoweCache"), cacheManager.getCache("zoweInvalidatedTokenCache"), clusteredLock); + public Storage storage(DefaultCacheManager cacheManager) { + return new InfinispanStorage( + cacheManager.getCache("zoweCache"), + cacheManager.getCache("zoweInvalidatedTokenCache"), + () -> lock(cacheManager) + ); } } diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java index 28dfe31414..907b7c09e9 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java @@ -24,7 +24,11 @@ import java.time.LocalDateTime; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; @Slf4j @@ -33,13 +37,17 @@ public class InfinispanStorage implements Storage { private final ConcurrentMap cache; private final ConcurrentMap> tokenCache; - private final ClusteredLock lock; + private final Supplier lockSupplier; private static final ObjectMapper objectMapper = new ObjectMapper(); - public InfinispanStorage(ConcurrentMap cache, ConcurrentMap> tokenCache, ClusteredLock lock) { + public InfinispanStorage( + ConcurrentMap cache, + ConcurrentMap> tokenCache, + Supplier lockSupplier + ) { this.cache = cache; this.tokenCache = tokenCache; - this.lock = lock; + this.lockSupplier = lockSupplier; } static { @@ -61,6 +69,7 @@ public KeyValue create(String serviceId, KeyValue toCreate) { @Override public KeyValue storeMapItem(String serviceId, String mapKey, KeyValue toCreate) { + ClusteredLock lock = lockSupplier.get(); CompletableFuture complete = lock.tryLock(4, TimeUnit.SECONDS).whenComplete((r, ex) -> { if (Boolean.TRUE.equals(r)) { try { @@ -90,10 +99,21 @@ public Map getAllMapItems(String serviceId, String mapKey) { @Override public Map> getAllMaps(String serviceId) { log.info("Reading all records from token cache for service {} ", serviceId); - // filter all maps which belong given service and remove the service name from key names. - return tokenCache.entrySet().stream().filter( - entry -> entry.getKey().startsWith(serviceId)) - .collect(Collectors.toMap(e -> e.getKey().substring(serviceId.length()), Map.Entry::getValue)); + + /** + * Original implementation with stream, collect and lambdas to read keys leads to serializing of lambdas, + * see org.infinispan.marshall.core.LambdaMarshaller#write(java.io.ObjectOutput, java.lang.Object). + * It is difficult to support and also slower (see exchanging lambdas between nodes). + */ + Map> result = new HashMap<>(); + for (String key : tokenCache.keySet()) { + if (!key.startsWith(serviceId)) continue; + + String newKey = key.substring(serviceId.length()); + result.put(newKey, tokenCache.get(key)); + } + + return result; } @Override @@ -154,6 +174,7 @@ public void deleteForService(String serviceId) { @Override public void removeNonRelevantTokens(String serviceId, String mapKey) { + ClusteredLock lock = lockSupplier.get(); CompletableFuture complete = lock.tryLock(4, TimeUnit.SECONDS).whenComplete((r, ex) -> { if (Boolean.TRUE.equals(r)) { try { @@ -184,6 +205,7 @@ private void removeToken(String serviceId, String mapKey) { @Override public void removeNonRelevantRules(String serviceId, String mapKey) { + ClusteredLock lock = lockSupplier.get(); CompletableFuture complete = lock.tryLock(4, TimeUnit.SECONDS).whenComplete((r, ex) -> { if (Boolean.TRUE.equals(r)) { try { @@ -217,4 +239,5 @@ private void completeJoin(CompletableFuture complete) { } } } + } diff --git a/caching-service/src/main/resources/application.yml b/caching-service/src/main/resources/application.yml index 3d048c7fa1..c6a8e7ccad 100644 --- a/caching-service/src/main/resources/application.yml +++ b/caching-service/src/main/resources/application.yml @@ -177,6 +177,10 @@ logging: org.ehcache: INFO org.springframework.cloud.netflix.zuul.filters.post.SendErrorFilter: INFO com.netflix.discovery.shared.transport.decorator: DEBUG + org.apache.tomcat: DEBUG + org.apache.catalina: DEBUG + org.apache.coyote.http11.Http11InputBuffer: TRACE + org.infinispan: DEBUG --- spring: diff --git a/caching-service/src/main/resources/caching-log-messages.yml b/caching-service/src/main/resources/caching-log-messages.yml index 5565289d03..de8daf9fb3 100644 --- a/caching-service/src/main/resources/caching-log-messages.yml +++ b/caching-service/src/main/resources/caching-log-messages.yml @@ -114,3 +114,11 @@ messages: text: "Missing header with certificate." reason: "Call executed without valid client certificate." action: "Verify ssl context in client application." + + - key: org.zowe.apiml.cache.notAvailable + number: ZWECS703 + type: WARNING + text: "Cache is not available: %s" + reason: "Cache is not ready to write at the moment." + action: "Wait till initialization of cache is done. In case it is done verify the instance configuration or connectivity between multiple instances." + diff --git a/caching-service/src/main/resources/infinispan.xml b/caching-service/src/main/resources/infinispan.xml index cd297228b3..d80c07d6b6 100644 --- a/caching-service/src/main/resources/infinispan.xml +++ b/caching-service/src/main/resources/infinispan.xml @@ -5,7 +5,7 @@ response = underTest.getAllMapItems(any(), mockRequest); assertThat(response.getStatusCode(), is(HttpStatus.BAD_REQUEST)); } + + @Test + void givenGenericErrorReadingStorage_thenResponseInternalError() throws StorageException { + when(mockStorage.getAllMapItems(any(), any())).thenThrow(new RuntimeException("error")); + + ResponseEntity response = underTest.getAllMapItems(any(), mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); + } + } @Nested @@ -417,4 +427,66 @@ void givenInCorrectRequest_thenReturn500() throws StorageException { assertThat(responseScopesEviction.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); } } + + @Nested + class WhenGetAll { + + @Nested + class MapItems { + + @Test + void givenWrongStorage_whenGetAllMapItems_thenReturn400() { + Exception storageException = new StorageException(Messages.INCOMPATIBLE_STORAGE_METHOD.getKey(), Messages.INCOMPATIBLE_STORAGE_METHOD.getStatus()); + doThrow(storageException).when(mockStorage).getAllMapItems(any(), any()); + ResponseEntity response = underTest.getAllMapItems(MAP_KEY, mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.BAD_REQUEST)); + } + + @Test + void givenUnexpectedError_whenGetAllMapItems_thenReturn500() { + doThrow(new RuntimeException("unexpected")).when(mockStorage).getAllMapItems(any(), any()); + ResponseEntity response = underTest.getAllMapItems(MAP_KEY, mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); + } + + @Test + void givenUOtherStorageException_whenGetAllMapItems_thenReturn500() { + Exception storageException = new StorageException(Messages.DUPLICATE_KEY.getKey(), Messages.DUPLICATE_KEY.getStatus()); + doThrow(storageException).when(mockStorage).getAllMapItems(any(), any()); + ResponseEntity response = underTest.getAllMapItems(MAP_KEY, mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); + } + + } + + @Nested + class Maps { + + @Test + void givenWrongStorage_whenGetAllMapItems_thenReturn400() { + Exception storageException = new StorageException(Messages.INCOMPATIBLE_STORAGE_METHOD.getKey(), Messages.INCOMPATIBLE_STORAGE_METHOD.getStatus()); + doThrow(storageException).when(mockStorage).getAllMaps(any()); + ResponseEntity response = underTest.getAllMaps(mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.BAD_REQUEST)); + } + + @Test + void givenUnexpectedError_whenGetAllMapItems_thenReturn500() { + doThrow(new RuntimeException("unexpected")).when(mockStorage).getAllMaps(any()); + ResponseEntity response = underTest.getAllMaps(mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); + } + + @Test + void givenUOtherStorageException_whenGetAllMapItems_thenReturn500() { + Exception storageException = new StorageException(Messages.DUPLICATE_KEY.getKey(), Messages.DUPLICATE_KEY.getStatus()); + doThrow(storageException).when(mockStorage).getAllMaps(any()); + ResponseEntity response = underTest.getAllMaps(mockRequest); + assertThat(response.getStatusCode(), is(HttpStatus.INTERNAL_SERVER_ERROR)); + } + + } + + } + } diff --git a/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java b/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java index 09c4a5c10b..00fd9a924b 100644 --- a/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java +++ b/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java @@ -46,8 +46,8 @@ class InfinispanStorageTest { void setup() { cache = mock(Cache.class); tokenCache = mock(AdvancedCache.class); - storage = new InfinispanStorage(cache, tokenCache, lock); lock = mock(ClusteredLock.class); + storage = new InfinispanStorage(cache, tokenCache, () -> lock); } @Nested @@ -124,7 +124,7 @@ void cacheIsUpdated() { @Test void itemIsDeleted() { ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, lock); + InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); assertNull(storage.create(serviceId1, TO_CREATE)); assertEquals(TO_CREATE, storage.delete(serviceId1, TO_CREATE.getKey())); } @@ -132,7 +132,7 @@ void itemIsDeleted() { @Test void returnAll() { ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, lock); + InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); storage.create(serviceId1, new KeyValue("key", "value")); storage.create(serviceId1, new KeyValue("key2", "value2")); assertEquals(2, storage.readForService(serviceId1).size()); @@ -141,7 +141,7 @@ void returnAll() { @Test void removeAll() { ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, lock); + InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); storage.create(serviceId1, new KeyValue("key", "value")); storage.create(serviceId1, new KeyValue("key2", "value2")); assertEquals(2, storage.readForService(serviceId1).size()); @@ -173,7 +173,7 @@ void createStoreWithEntry() { void addToken() { HashMap hashMap = new HashMap<>(); hashMap.put("key", "token"); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, lock); + InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); when(tokenCache.get(anyString())).thenAnswer(invocation -> hashMap); assertNull(storage.storeMapItem(serviceId1, "invalidTokens", new KeyValue("newkey", "newvalue"))); verify(tokenCache, times(1)).put(serviceId1 + "invalidTokens", hashMap); @@ -183,7 +183,7 @@ void addToken() { void updateToken() { HashMap hashMap = new HashMap(); hashMap.put("key", "token"); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, lock); + InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); when(tokenCache.get(serviceId1 + "invalidTokens")).thenReturn(hashMap); KeyValue keyValue = new KeyValue("key", "token2"); assertNull(storage.storeMapItem(serviceId1, "invalidTokens", keyValue)); @@ -223,7 +223,7 @@ void createStorage() { tokenCache.put(serviceId1 + "invalidTokens", tokensService1); tokenCache.put(serviceId1 + "invalidTokenRules", rulesService1); tokenCache.put(serviceId2 + "invalidTokens", tokensService2); - underTest = new InfinispanStorage(cache, tokenCache, lock); + underTest = new InfinispanStorage(cache, tokenCache, () -> lock); } @@ -275,7 +275,7 @@ void createStorage() { tokenCache.put(serviceId1 + "invalidTokens", tokensService); tokenCache.put(serviceId1 + "invalidScopes", rulesService); tokenCache.put(serviceId1 + "invalidUsers", rulesUsers); - underTest = new InfinispanStorage(cache, tokenCache, lock); + underTest = new InfinispanStorage(cache, tokenCache, () -> lock); } @Test void thenEvictItems() { diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/cache/CachingServiceClient.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/cache/CachingServiceClient.java index a609c95534..e4240cbed5 100644 --- a/gateway-service/src/main/java/org/zowe/apiml/gateway/cache/CachingServiceClient.java +++ b/gateway-service/src/main/java/org/zowe/apiml/gateway/cache/CachingServiceClient.java @@ -16,10 +16,8 @@ import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; +import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -149,12 +147,18 @@ public KeyValue read(String key) throws CachingServiceClientException { ResponseEntity response = restTemplate.exchange(gatewayProtocolHostPort + CACHING_API_PATH + "/" + key, HttpMethod.GET, new HttpEntity(null, defaultHeaders), KeyValue.class); if (response != null && response.hasBody()) { //NOSONAR tests return null return response.getBody(); - } else { - throw new CachingServiceClientException("Unable to read key: " + key + ", caused by response from caching service is null or has no body"); } } catch (RestClientException e) { - throw new CachingServiceClientException("Unable to read key: " + key + ", caused by: " + e.getMessage(), e); + if (!( + (e instanceof HttpStatusCodeException) && + (((HttpStatusCodeException) e).getStatusCode() == HttpStatus.NOT_FOUND) + )) { + throw new CachingServiceClientException("Unable to read key: " + key + ", caused by: " + e.getMessage(), e); + } } + + // record not found + throw new CachingServiceClientException("Unable to read key: " + key + ", caused by response from caching service is null or has no body"); } /** diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProvider.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProvider.java index a10561d3f6..0361f172e8 100644 --- a/gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProvider.java +++ b/gateway-service/src/main/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProvider.java @@ -49,8 +49,6 @@ public class ApimlAccessTokenProvider implements AccessTokenProvider { @Qualifier("oidcJwkMapper") private final ObjectMapper objectMapper; - private byte[] salt; - public void invalidateToken(String token) throws CachingServiceClientException, JsonProcessingException { String hashedValue = getHash(token); QueryResponse queryResponse = authenticationService.parseJwtWithSignature(token); @@ -80,10 +78,11 @@ public void invalidateAllTokensForService(String serviceId, long timestamp) thro } public boolean isInvalidated(String token) throws CachingServiceClientException { + byte[] salt = getSalt(); QueryResponse parsedToken = authenticationService.parseJwtWithSignature(token); - String hashedToken = getHash(token); - String hashedUserId = getHash(parsedToken.getUserId().trim().toUpperCase()); - List hashedServiceIds = parsedToken.getScopes().stream().map(this::getHash).collect(Collectors.toList()); + String hashedToken = getHash(token, salt); + String hashedUserId = getHash(parsedToken.getUserId().trim().toUpperCase(), salt); + List hashedServiceIds = parsedToken.getScopes().stream().map(scope -> getHash(scope, salt)).collect(Collectors.toList()); Map> cacheMap = cachingServiceClient.readAllMaps(); if (cacheMap != null && !cacheMap.isEmpty()) { @@ -143,16 +142,26 @@ public void evictNonRelevantTokensAndRules() { cachingServiceClient.evictRules(INVALID_SCOPES_KEY); } + private String getHash(String token, byte[] salt) throws CachingServiceClientException { + return getSecurePassword(token, salt); + } + public String getHash(String token) throws CachingServiceClientException { return getSecurePassword(token, getSalt()); } - private String initializeSalt() throws CachingServiceClientException { + String initializeSalt() throws CachingServiceClientException { String localSalt; try { CachingServiceClient.KeyValue keyValue = cachingServiceClient.read("salt"); localSalt = keyValue.getValue(); } catch (CachingServiceClientException e) { + log.debug("Cannot read salt.", e); + if (e.getCause() != null) { + // it could be because of timeout for example + throw e; + } + // a null value was returned byte[] newSalt = generateSalt(); storeSalt(newSalt); localSalt = new String(newSalt); @@ -180,11 +189,7 @@ public boolean isValidForScopes(String jwtToken, String serviceId) { } public byte[] getSalt() throws CachingServiceClientException { - if (this.salt != null) { - return this.salt; - } - this.salt = initializeSalt().getBytes(); - return this.salt; + return initializeSalt().getBytes(); } private void storeSalt(byte[] salt) throws CachingServiceClientException { diff --git a/gateway-service/src/test/java/org/zowe/apiml/gateway/cache/CachingServiceClientTest.java b/gateway-service/src/test/java/org/zowe/apiml/gateway/cache/CachingServiceClientTest.java index 71f80e21be..66865e4c22 100644 --- a/gateway-service/src/test/java/org/zowe/apiml/gateway/cache/CachingServiceClientTest.java +++ b/gateway-service/src/test/java/org/zowe/apiml/gateway/cache/CachingServiceClientTest.java @@ -16,14 +16,13 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import org.zowe.apiml.models.AccessTokenContainer; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -100,10 +99,35 @@ void readWithoutProblem() throws CachingServiceClientException { } @Test - void readWithExceptonFromRestTemplateThrowsDefined() { + void readWithExceptionFromRestTemplateThrowsDefined() { doThrow(new RestClientException("oops")).when(restTemplate).exchange(anyString(), any(HttpMethod.class), any(HttpEntity.class), eq(String.class)); assertThrows(CachingServiceClientException.class, () -> underTest.read(keyToRead)); } + + @Test + void ioException() { + RestClientException ioException = new RestClientException("io"); + doThrow(ioException).when(restTemplate).exchange(anyString(), any(HttpMethod.class), any(HttpEntity.class), eq(CachingServiceClient.KeyValue.class)); + CachingServiceClientException e = assertThrows(CachingServiceClientException.class, () -> underTest.read(keyToRead)); + assertSame(e.getCause(), ioException); + } + + @Test + void notFound() { + doThrow(HttpClientErrorException.create("record not found", HttpStatus.NOT_FOUND, "notFound", new HttpHeaders(), new byte[0], StandardCharsets.UTF_8)) + .when(restTemplate).exchange(anyString(), any(HttpMethod.class), any(HttpEntity.class), eq(CachingServiceClient.KeyValue.class)); + CachingServiceClientException e = assertThrows(CachingServiceClientException.class, () -> underTest.read(keyToRead)); + assertNull(e.getCause()); + } + + @Test + void noAvailable() { + Exception responseException = HttpClientErrorException.create("service not available", HttpStatus.SERVICE_UNAVAILABLE, "503", new HttpHeaders(), new byte[0], StandardCharsets.UTF_8); + doThrow(responseException).when(restTemplate).exchange(anyString(), any(HttpMethod.class), any(HttpEntity.class), eq(CachingServiceClient.KeyValue.class)); + CachingServiceClientException e = assertThrows(CachingServiceClientException.class, () -> underTest.read(keyToRead)); + assertSame(e.getCause(), responseException); + } + } @Nested diff --git a/gateway-service/src/test/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProviderTest.java b/gateway-service/src/test/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProviderTest.java index 1a0659433d..a303463e8c 100644 --- a/gateway-service/src/test/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProviderTest.java +++ b/gateway-service/src/test/java/org/zowe/apiml/gateway/security/service/token/ApimlAccessTokenProviderTest.java @@ -25,13 +25,12 @@ import org.zowe.apiml.models.AccessTokenContainer; import org.zowe.apiml.security.common.token.QueryResponse; +import java.io.IOException; import java.util.*; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; class ApimlAccessTokenProviderTest { @@ -223,4 +222,26 @@ static String createTestToken(String username, Map claims) { .addClaims(claims).compact(); } + @Nested + class SaltInitialization { + + @Test + void givenUnexpectedError_whenReadSalt_thenThrowIt() { + Exception unexpectedError = new CachingServiceClientException("unexpected error", new IOException("e.g. timeout")); + doThrow(unexpectedError).when(cachingServiceClient).read("salt"); + Exception thrownException = assertThrows(CachingServiceClientException.class, accessTokenProvider::initializeSalt); + assertSame(unexpectedError, thrownException); + } + + @Test + void givenNoSaltInCache_whenInitializing_thenCreateNewOne() { + Exception noRecordException = new CachingServiceClientException("no record"); + doThrow(noRecordException).when(cachingServiceClient).read("salt"); + String salt = accessTokenProvider.initializeSalt(); + assertEquals(16, salt.length()); + verify(cachingServiceClient, times(1)).create(any()); + } + + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/integration/ha/CachingService.java b/integration-tests/src/test/java/org/zowe/apiml/integration/ha/CachingService.java new file mode 100644 index 0000000000..bbb068fb89 --- /dev/null +++ b/integration-tests/src/test/java/org/zowe/apiml/integration/ha/CachingService.java @@ -0,0 +1,226 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.integration.ha; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import io.restassured.RestAssured; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.zowe.apiml.util.categories.ChaoticHATest; +import org.zowe.apiml.util.config.*; +import org.zowe.apiml.util.requests.Endpoints; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import static io.restassured.RestAssured.given; +import static io.restassured.http.ContentType.JSON; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.http.HttpStatus.SC_OK; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.core.Is.is; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.zowe.apiml.util.config.ConfigReader.environmentConfiguration; + +@Slf4j +@ChaoticHATest +@TestInstance(TestInstance.Lifecycle. PER_CLASS) +public class CachingService { + + private static final String SERVICE = "service"; + private static final String KEY = "aCacheKey" + new Random().nextInt(); + private static final String VALUE = "aCacheValue"; + private static final String MAP = "aMap"; + private static final String MAP_KEY = "aMapCacheKey" + new Random().nextInt(); + private static final String MAP_VALUE = "aMapCacheValue"; + private static final String DN = "CN=Zowe Service, OU=API Mediation Layer, O=Zowe Sample, L=Prague, ST=Prague, C=CZ"; + + private static final KeyValue KEY_VALUE = new KeyValue(KEY, VALUE); + private static final KeyValue MAP_KEY_VALUE = new KeyValue(MAP_KEY, MAP_VALUE); + + private Credentials credentials; + private List baseUrls; + + private boolean isUp(int index) { + try { + String url = String.format("%s/cachingservice%s", baseUrls.get(index), Endpoints.HEALTH); + log.info("Check if {}. Caching Service is up: {}", index + 1, url); + + given() + .contentType(JSON) + .auth() + .basic(credentials.getUser(), credentials.getPassword()) + .when() + .get(url) + .then() + .statusCode(200) + .body("status", Matchers.is("UP")); + return true; + } catch (Throwable t) { + log.info("Caching service is down", t); + return false; + } + } + + private boolean isUp() { + for (int i = 0; i < baseUrls.size(); i++) { + if (!isUp(i)) { + return false; + } + } + return true; + } + + @BeforeAll + void setUp() throws Exception { + RestAssured.useRelaxedHTTPSValidation(); + SslContext.prepareSslAuthentication(ItSslConfigFactory.integrationTests()); + + EnvironmentConfiguration environmentConfiguration = environmentConfiguration(); + CachingServiceConfiguration cachingServiceConfiguration = environmentConfiguration.getCachingServiceConfiguration(); + + KEY_VALUE.setServiceId(SERVICE); + + assumeTrue(cachingServiceConfiguration.getHost() != null); + baseUrls = Arrays.stream(cachingServiceConfiguration.getHost().split("[,;]")) + .map(host -> String.format("%s://%s:%d", cachingServiceConfiguration.getScheme(), host, cachingServiceConfiguration.getPort())) + .collect(Collectors.toList()); + credentials = ConfigReader.environmentConfiguration().getCredentials(); + + await() + .atMost(10, MINUTES) + .pollDelay(0, SECONDS) + .pollInterval(10, SECONDS) + .until(this::isUp); + } + + private void assertContent(int index) { + // check the all records (tokenCache) + given() + .config(SslContext.clientCertApiml) + .header("X-Certificate-DistinguishedName", DN) + .when() + .get(baseUrls.get(index) + "/cachingservice/api/v1/cache-list") + .then() + .statusCode(200) + .body(MAP + "." + MAP_KEY, equalTo(MAP_VALUE)); + + // check the all records (tokenCache) + given() + .config(SslContext.clientCertApiml) + .header("X-Certificate-DistinguishedName", DN) + .when() + .get(baseUrls.get(index) + "/cachingservice/api/v1/cache-list/" + MAP) + .then() + .statusCode(200) + .body(MAP_KEY, equalTo(MAP_VALUE)); + + // check the concrete record (cache) + given() + .config(SslContext.clientCertApiml) + .header("X-Certificate-DistinguishedName", DN) + .when() + .get(baseUrls.get(index) + "/cachingservice/api/v1/cache/" + KEY) + .then() + .statusCode(200) + .body("value", equalTo(VALUE)); + } + + @Test + void givenMultipleInstances_whenShareAValue_thenShutdownDoesntChangeTheState() { + log.info("Set value on the first instance to cache storage"); + given() + .config(SslContext.clientCertApiml) + .header("X-Certificate-DistinguishedName", DN) + .contentType(JSON) + .body(KEY_VALUE) + .when() + .post(baseUrls.get(0) + "/cachingservice/api/v1/cache") + .then() + .statusCode(201); + + log.info("Set value on the first instance to tokenCache storage"); + given() + .config(SslContext.clientCertApiml) + .header("X-Certificate-DistinguishedName", DN) + .contentType(JSON) + .body(MAP_KEY_VALUE) + .when() + .post(baseUrls.get(0) + "/cachingservice/api/v1/cache-list/" + MAP) + .then() + .statusCode(201); + + int instances = baseUrls.size(); + for (int i = -1; i < instances - 1; i++) { + if (i >= 0) { + log.info("Kill {}. instance of caching service", i + 1); + given() + .config(SslContext.clientCertApiml) + .contentType(JSON) + .auth().basic(credentials.getUser(), credentials.getPassword()) + .when() + .post(baseUrls.get(i) + "/cachingservice/application/shutdown") + .then() + .statusCode(is(SC_OK)); + } + + for (int j = i + 1; j < instances; j++) { + log.info("Check if the value is accessible {}. instance", j + 1); + assertContent(j); + } + } + + } + + @RequiredArgsConstructor + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @Data + static class KeyValue implements Serializable { + + private final String key; + private final String value; + private String serviceId; + private final String created; + + public KeyValue(String key, String value) { + this.key = key; + this.value = value; + this.serviceId = ""; + this.created = currentTime(); + } + + private static String currentTime() { + return String.valueOf(new Date().getTime()); + } + + @JsonCreator + public KeyValue() { + key = ""; + value = ""; + serviceId = ""; + created = currentTime(); + } + + } + +} diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java index d16af05c28..7f8c155a6e 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java @@ -21,4 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class CachingServiceConfiguration { private String url; + private String scheme; + private String host; + private int port; } diff --git a/integration-tests/src/test/resources/environment-configuration-ha.yml b/integration-tests/src/test/resources/environment-configuration-ha.yml index a7f8aec5db..1de97b0ace 100644 --- a/integration-tests/src/test/resources/environment-configuration-ha.yml +++ b/integration-tests/src/test/resources/environment-configuration-ha.yml @@ -34,6 +34,9 @@ apiCatalogServiceConfiguration: instances: 2 cachingServiceConfiguration: url: + scheme: https + host: caching-service,caching-service-2,caching-service-3 + port: 10016 cloudGatewayConfiguration: scheme: https host: cloud-gateway-service diff --git a/keystore/docker/all-services.keystore.cer b/keystore/docker/all-services.keystore.cer index f57a69a825..bdb1270f5d 100644 --- a/keystore/docker/all-services.keystore.cer +++ b/keystore/docker/all-services.keystore.cer @@ -1,32 +1,33 @@ -----BEGIN CERTIFICATE----- -MIIFlzCCBH+gAwIBAgIUJ6g/GJD3j4lB+bETn26Eeon+8A4wDQYJKoZIhvcNAQEL +MIIFyjCCBLKgAwIBAgIUIOG3WV7VbkbX0KIy4NTraWqn6p0wDQYJKoZIhvcNAQEL BQAwgZ4xCzAJBgNVBAYTAkNaMQ8wDQYDVQQIEwZQcmFndWUxDzANBgNVBAcTBlBy YWd1ZTEUMBIGA1UEChMLWm93ZSBTYW1wbGUxHDAaBgNVBAsTE0FQSSBNZWRpYXRp b24gTGF5ZXIxOTA3BgNVBAMTMFpvd2UgRGV2ZWxvcG1lbnQgSW5zdGFuY2VzIENl -cnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMjEyMTUxMjE1MzJaFw0yNzEyMTQxMjE1 -MzJaMHwxCzAJBgNVBAYTAkNaMQ8wDQYDVQQIDAZQcmFndWUxDzANBgNVBAcMBlBy -YWd1ZTEUMBIGA1UECgwLWm93ZSBTYW1wbGUxHDAaBgNVBAsME0FQSSBNZWRpYXRp -b24gTGF5ZXIxFzAVBgNVBAMMDlpvd2UgQ29tcG9uZW50MIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAsrNR9CgPcNrLDrE0B5hAjA+pgEPJ73olU4Chr5uY -9w6yuEDy08I+W4Bitu/ScI8qbkJB7hITwgM43ZQU23QEqP1luW8msRvYaUzZcuyw -gsGKTjtUmPi1BEQmmo6MFGVwjpdGBNJqbCKuMHVi8G8Xc9CrevCUQp9Pw4SOu4bY -6FZG8nbzYoTgHBXHwvccv8Ov07lLv1ohk3BdHkDjtoEgSfFtJrCW2Jkk/4MYpYFR -t2WKjTdOQLSpGptSS2EC8M6UmpZTYyLhDprP0EAdT/IcKg7/QR4/TbSo7soY0Dx1 -5jFZqg/2mvqCywAtx8vPvhBNzPVOVfL+V46iw13pKm6MnQIDAQABo4IB7DCCAegw -HQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMIIBhQYDVR0RBIIBfDCCAXiC -CWxvY2FsaG9zdIIJMTI3LjAuMC4xgg9nYXRld2F5LXNlcnZpY2WCEWdhdGV3YXkt -c2VydmljZS0yghRhcGktY2F0YWxvZy1zZXJ2aWNlc4IWYXBpLWNhdGFsb2ctc2Vy -dmljZXMtMoIPY2FjaGluZy1zZXJ2aWNlghFjYWNoaW5nLXNlcnZpY2UtMoIRZGlz -Y292ZXJ5LXNlcnZpY2WCE2Rpc2NvdmVyeS1zZXJ2aWNlLTKCE2Rpc2NvdmVyYWJs -ZS1jbGllbnSCFWRpc2NvdmVyYWJsZS1jbGllbnQtMYIVZGlzY292ZXJhYmxlLWNs -aWVudC0yghVkaXNjb3ZlcmFibGUtY2xpZW50LTOCFWRpc2NvdmVyYWJsZS1jbGll -bnQtNIINbW9jay1zZXJ2aWNlc4IPbW9jay1zZXJ2aWNlcy0ygg9tZXRyaWNzLXNl -cnZpY2WCEW1ldHJpY3Mtc2VydmljZS0ygg1yZXZlcnNlLXByb3h5MB0GA1UdDgQW -BBQiM604apFGByEW0V0q+C9KnvbfpzAfBgNVHSMEGDAWgBQkaS422MefPS4tRH1i -sBpkMYiwNjANBgkqhkiG9w0BAQsFAAOCAQEAZApmzT8KLZO6UeP93quH6/JNNYxw -MHWcww4ulCDiRkC6Lg7Td+QYn6ttg60GbEZbw1NszfRNCYh6yz5MSjC314sw+AVO -PaVjNQDdxAyU5xZ57yvmJs5pPub2Z8WD42SXvBoWB1wH3ErUs7XfznhjpTVokeZK -DMUUAfA0a/A1ZuvLWcK5EOHbi5SDqZ5ieI5qnfePVhDQHsS1El90GLXq0B1X+UXf -G4RKG6FV7+zMFig/w36qyCcWm7NMvqEb6LfeIdZEgTWYGPZeYYHbHjd8LCU0TSRg -+R1wYQmT3Kpi/63V7VfHYywt+j5Zofpk17jslRfv18KoW9Yo0KMK3fuixA== +cnRpZmljYXRlIEF1dGhvcml0eTAeFw0yNTEyMTAwOTI1MzdaFw0zMDEyMDkwOTI1 +MzdaMGwxCzAJBgNVBAYTAkNaMRAwDgYDVQQIDAdDemVjaGlhMQ8wDQYDVQQHDAZQ +cmFndWUxETAPBgNVBAoMCEJyb2FkY29tMQwwCgYDVQQLDANNU0QxGTAXBgNVBAMM +ECdab3dlIENvbXBvbmVudCcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQC24tPOjwNlLZV0gRay1YhNe4X7rs0n4Oo2HqvbGkV3mrIK98ADXTuLzr9JFrsL +KKf64aykt0jHJtZdUXsgoQQ4odtGemtWlSaJjtAAHtls+6dxyZWDltuyVH9Nrwr3 +kwr3ESOyXC3LEOhwkpUGMMdK5wMpbXBeaH9zttMTOizKQpv9wntMB9wJQKjxL7oC +Xez+1nCnH+zkQPwxNBw5bA/PAEF2eVXi9Bm4vaRdalsJLczTcXc43SC88IgfpCLd +pMkAoWRVnr06MvgxyewVgjHewhmL7ZLBimbDBeWz2i7fcOhkPc3E6O5W0JSJ1J+a +D5kkHE72h3koaMq/TevPeAafAgMBAAGjggIvMIICKzAdBgNVHSUEFjAUBggrBgEF +BQcDAgYIKwYBBQUHAwEwggHIBgNVHREEggG/MIIBu4IJbG9jYWxob3N0ggkxMjcu +MC4wLjGCD2dhdGV3YXktc2VydmljZYIRZ2F0ZXdheS1zZXJ2aWNlLTKCFGFwaS1j +YXRhbG9nLXNlcnZpY2VzghZhcGktY2F0YWxvZy1zZXJ2aWNlcy0ygg9jYWNoaW5n +LXNlcnZpY2WCEWNhY2hpbmctc2VydmljZS0yghFjYWNoaW5nLXNlcnZpY2UtM4IR +ZGlzY292ZXJ5LXNlcnZpY2WCE2Rpc2NvdmVyeS1zZXJ2aWNlLTKCE2Rpc2NvdmVy +YWJsZS1jbGllbnSCFWRpc2NvdmVyYWJsZS1jbGllbnQtMYIVZGlzY292ZXJhYmxl +LWNsaWVudC0yghVkaXNjb3ZlcmFibGUtY2xpZW50LTOCFWRpc2NvdmVyYWJsZS1j +bGllbnQtNIINbW9jay1zZXJ2aWNlc4IPbW9jay1zZXJ2aWNlcy0ygg9tZXRyaWNz +LXNlcnZpY2WCEW1ldHJpY3Mtc2VydmljZS0ygg1yZXZlcnNlLXByb3h5ghVjbG91 +ZC1nYXRld2F5LXNlcnZpY2WCF2Nsb3VkLWdhdGV3YXktc2VydmljZS0yMB0GA1Ud +DgQWBBRomE05vqsfKS5pYQ71YoVSixeU2TAfBgNVHSMEGDAWgBQkaS422MefPS4t +RH1isBpkMYiwNjANBgkqhkiG9w0BAQsFAAOCAQEAO5wB8COl45U7Cq6saaW3/xqm +od1Dx068ziaDQTj1CL1A5jJ/JTUMNWEHeBBfPFGvKBq1IqpnNMR96+C2mIt9LWck +49khLzWvcWVKLg+9IJixceQrn43ZziD/p11EHVDz1KLn8Hn1ElnDavPL2hBxO7uE +RdZCVY4SFbB3bc2OsK7qmal+SQFRrEpawaF4GXWCOGp2uZhnytBQEaqxQE9v24pj +qoYeSwA6/IrF+C5mW8HbdYIBn9Te8joGJ5xMW1XTtumxxOJytpLy/4zRwxKoxJLA +eXhWn/1S7l/6jWT9+df+8wfiFywJOEGqyawFB5ysWAaQDpl6nkTyVjDLwJq42w== -----END CERTIFICATE----- diff --git a/keystore/docker/all-services.keystore.p12 b/keystore/docker/all-services.keystore.p12 index 96e547fa4f..6424a833f6 100644 Binary files a/keystore/docker/all-services.keystore.p12 and b/keystore/docker/all-services.keystore.p12 differ