fix: Fix infinispan configuration, exception handling, and the whole cache operation

.github/workflows/integration-tests.yml

@@ -1311,6 +1311,85 @@ jobs:
 
             -   uses: ./.github/actions/teardown
 
+    CITestsCachingChaoticHA:
+        needs: PublishJibContainers
+        container: ubuntu:latest
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+
+        services:
+            caching-service:
+                image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true
+                    MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown
+                    CACHING_STORAGE_MODE: infinispan
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]"
+                    JGROUPS_BIND_PORT: 7098
+                    JGROUPS_BIND_ADDRESS: caching-service
+                    JGROUPS_KEYEXCHANGE_PORT: 7118
+                    APIML_SERVICE_HOSTNAME: caching-service
+            caching-service-2:
+                image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true
+                    MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown
+                    CACHING_STORAGE_MODE: infinispan
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]"
+                    JGROUPS_BIND_PORT: 7098
+                    JGROUPS_BIND_ADDRESS: caching-service-2
+                    JGROUPS_KEYEXCHANGE_PORT: 7118
+                    APIML_SERVICE_HOSTNAME: caching-service-2
+            caching-service-3:
+                image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    MANAGEMENT_ENDPOINT_SHUTDOWN_ENABLED: true
+                    MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,hystrixstream,shutdown
+                    CACHING_STORAGE_MODE: infinispan
+                    CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7098],caching-service-2[7098],caching-service-3[7098]"
+                    JGROUPS_BIND_PORT: 7098
+                    JGROUPS_BIND_ADDRESS: caching-service-3
+                    JGROUPS_KEYEXCHANGE_PORT: 7118
+                    APIML_SERVICE_HOSTNAME: caching-service-3
+            mock-services:
+                image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }}
+            discovery-service:
+                image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }}
+                volumes:
+                    - /api-defs:/api-defs
+            gateway-service:
+                image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }}
+        steps:
+            -   uses: actions/checkout@v4
+                with:
+                    ref: ${{ github.head_ref }}
+
+            -   uses: ./.github/actions/setup
+
+            -   name: Run Caching Chaotic HA Tests
+                run: >
+                    ./gradlew :integration-tests:runChaoticHATests --tests org.zowe.apiml.integration.ha.CachingService
+                    --info -Denvironment.config=-ha -Denvironment.offPlatform=true -DcloudGateway.enabled=false
+                    -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD
+                env:
+                    ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
+                    ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
+
+            -   name: Correct Permisions
+                run: |
+                    chmod 755 -R .gradle
+            # Coverage results are not stored in this job as it would not provide much additional data
+            -   name: Store results
+                uses: actions/upload-artifact@v4
+                if: always()
+                with:
+                    name: CITestsCachingChaoticHA-${{ env.JOB_ID }}
+                    path: |
+                        integration-tests/build/reports/**
+                        results/**
+
+            -   uses: ./.github/actions/teardown
+
     CITestsWithInfinispan:
         needs: PublishJibContainers
         runs-on: ubuntu-latest

caching-service-package/src/main/resources/bin/start.sh

@@ -243,6 +243,16 @@ if [ "${ATTLS_SERVER_ENABLED}" = "true" -a "${APIML_ATTLS_LOAD_KEYRING:-false}"
   keystore_location=
 fi
 
+# migration step of Infinispan since version 2.18.4 (see #https://github.com/zowe/api-layer/pull/3960)
+original_infinispan_data_location="${ZWE_configs_storage_infinispan_persistence_dataLocation:-${ZWE_zowe_workspaceDirectory:-$(pwd)}}/caching-service/data"
+if [ -d "${original_infinispan_data_location}" ]; then
+    mv -f "${original_infinispan_data_location}" "${ZWE_zowe_workspaceDirectory:-$(pwd)}/caching-service/${ZWE_haInstance_id:-localhost}/${ZWE_configs_storage_infinispan_persistence_dataLocation:-data}"
+fi
+original_infinispan_index_location="${ZWE_configs_storage_infinispan_persistence_indexLocation:-${ZWE_zowe_workspaceDirectory:-$(pwd)}}/caching-service/index"
+if [ -d "${original_infinispan_index_location}" ]; then
+    mv -f "${original_infinispan_index_location}" "${ZWE_zowe_workspaceDirectory:-$(pwd)}/caching-service/${ZWE_haInstance_id:-localhost}/${ZWE_configs_storage_infinispan_persistence_indexLocation:-index}"
+fi
+
 CACHING_CODE=CS
 _BPXK_AUTOCVT=OFF
 _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} java \
@@ -273,8 +283,6 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} java \
   -Djgroups.bind.port=${ZWE_configs_storage_infinispan_jgroups_port:-7098} \
   -Djgroups.keyExchange.port=${ZWE_configs_storage_infinispan_jgroups_keyExchange_port:-7118} \
   -Djgroups.tcp.diag.enabled=${ZWE_configs_storage_infinispan_jgroups_tcp_diag_enabled:-false} \
-  -Dcaching.storage.infinispan.persistence.dataLocation=${ZWE_configs_storage_infinispan_persistence_dataLocation:-data} \
-  -Dcaching.storage.infinispan.persistence.indexLocation=${ZWE_configs_storage_infinispan_persistence_indexLocation:-index} \
   -Dcaching.storage.infinispan.initialHosts=${ZWE_configs_storage_infinispan_initialHosts:-localhost[7098]} \
   -Dserver.address=0.0.0.0 \
   -Dserver.ssl.enabled=${ZWE_configs_server_ssl_enabled:-true}  \

caching-service/src/main/java/org/zowe/apiml/caching/api/CachingController.java

@@ -13,6 +13,7 @@
 import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -28,6 +29,7 @@
 import javax.servlet.http.HttpServletRequest;
 import java.util.Optional;
 
+@Slf4j
 @RestController
 @RequiredArgsConstructor
 @RequestMapping("/api/v1")
@@ -128,7 +130,13 @@ public ResponseEntity<Object> getAllMapItems(@PathVariable String mapKey, HttpSe
                 try {
                     return new ResponseEntity<>(storage.getAllMapItems(s, mapKey), HttpStatus.OK);
                 } catch (Exception exception) {
-                    return handleIncompatibleStorageMethod(exception, request.getRequestURL());
+                    if (
+                        (exception instanceof StorageException) &&
+                            Messages.INCOMPATIBLE_STORAGE_METHOD.getKey().equals(((StorageException) exception).getKey())
+                    ) {
+                        return handleIncompatibleStorageMethod(exception, request.getRequestURL());
+                    }
+                    return handleInternalError(exception, request.getRequestURL());
                 }
             }
         ).orElseGet(this::getUnauthorizedResponse);
@@ -145,7 +153,13 @@ public ResponseEntity<Object> getAllMaps(HttpServletRequest request) {
                 try {
                     return new ResponseEntity<>(storage.getAllMaps(s), HttpStatus.OK);
                 } catch (Exception exception) {
-                    return handleIncompatibleStorageMethod(exception, request.getRequestURL());
+                    if (
+                        (exception instanceof StorageException) &&
+                        Messages.INCOMPATIBLE_STORAGE_METHOD.getKey().equals(((StorageException) exception).getKey())
+                    ) {
+                        return handleIncompatibleStorageMethod(exception, request.getRequestURL());
+                    }
+                    return handleInternalError(exception, request.getRequestURL());
                 }
             }
         ).orElseGet(this::getUnauthorizedResponse);
@@ -199,6 +213,7 @@ public ResponseEntity<Object> update(@RequestBody KeyValue keyValue, HttpServlet
 
 
     private ResponseEntity<Object> exceptionToResponse(StorageException exception) {
+        log.debug("Storage exception", exception);
         Message message = messageService.createMessage(exception.getKey(), (Object[]) exception.getParameters());
         return new ResponseEntity<>(message.mapToView(), exception.getStatus());
     }
@@ -298,12 +313,14 @@ private Optional<String> getHeader(HttpServletRequest request, String headerName
     }
 
     private ResponseEntity<Object> handleInternalError(Exception exception, StringBuffer requestURL) {
+        log.debug("Internal error occurred", exception);
         Messages internalServerError = Messages.INTERNAL_SERVER_ERROR;
         Message message = messageService.createMessage(internalServerError.getKey(), requestURL, exception.getMessage(), exception.toString());
         return new ResponseEntity<>(message.mapToView(), internalServerError.getStatus());
     }
 
     private ResponseEntity<Object> handleIncompatibleStorageMethod(Exception exception, StringBuffer requestURL) {
+        log.debug("Incompatible storage method", exception);
         Messages internalServerError = Messages.INCOMPATIBLE_STORAGE_METHOD;
         Message message = messageService.createMessage(internalServerError.getKey(), requestURL, exception.getMessage(), exception.toString());
         return new ResponseEntity<>(message.mapToView(), internalServerError.getStatus());

caching-service/src/main/java/org/zowe/apiml/caching/service/Messages.java

@@ -26,7 +26,8 @@ public enum Messages {
     PAYLOAD_TOO_LARGE("org.zowe.apiml.cache.payloadTooLarge", HttpStatus.BAD_REQUEST),
     INTERNAL_SERVER_ERROR("org.zowe.apiml.common.internalRequestError", HttpStatus.INTERNAL_SERVER_ERROR),
     MISSING_CERTIFICATE("org.zowe.apiml.cache.missingCertificate", HttpStatus.UNAUTHORIZED),
-    INCOMPATIBLE_STORAGE_METHOD("org.zowe.apiml.cache.incompatibleStorageMethod", HttpStatus.BAD_REQUEST);
+    INCOMPATIBLE_STORAGE_METHOD("org.zowe.apiml.cache.incompatibleStorageMethod", HttpStatus.BAD_REQUEST),
+    CACHE_NOT_AVAILABLE("org.zowe.apiml.cache.notAvailable", HttpStatus.SERVICE_UNAVAILABLE);
     private final String key;
     private final HttpStatus status;
 }

caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java

@@ -10,6 +10,7 @@
 
 package org.zowe.apiml.caching.service.infinispan.config;
 
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.infinispan.commons.api.CacheContainerAdmin;
 import org.infinispan.commons.dataconversion.MediaType;
@@ -21,27 +22,33 @@
 import org.infinispan.lock.api.ClusteredLock;
 import org.infinispan.lock.api.ClusteredLockManager;
 import org.infinispan.manager.DefaultCacheManager;
+import org.infinispan.partitionhandling.AvailabilityException;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.io.ResourceLoader;
+import org.zowe.apiml.caching.service.Messages;
 import org.zowe.apiml.caching.service.Storage;
+import org.zowe.apiml.caching.service.StorageException;
 import org.zowe.apiml.caching.service.infinispan.exception.InfinispanConfigException;
 import org.zowe.apiml.caching.service.infinispan.storage.InfinispanStorage;
 
 import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl;
 import static org.zowe.apiml.security.SecurityUtils.isKeyring;
 
+@Slf4j
 @Configuration
 @ConfigurationProperties(value = "caching.storage.infinispan")
 @ConditionalOnProperty(name = "caching.storage.mode", havingValue = "infinispan")
@@ -55,12 +62,6 @@ public class InfinispanConfig implements InitializingBean {
     @Value("${caching.storage.infinispan.initialHosts}")
     private String initialHosts;
 
-    @Value("${caching.storage.infinispan.persistence.dataLocation}")
-    private String dataLocation;
-
-    @Value("${caching.storage.infinispan.persistence.indexLocation:index}")
-    private String indexLocation;
-
     @Value("${server.ssl.keyStoreType}")
     private String keyStoreType;
 
@@ -85,6 +86,8 @@ public class InfinispanConfig implements InitializingBean {
     @Value("${server.attlsServer.enabled:false}")
     private boolean isServerAttlsEnabled;
 
+    private AtomicReference<ClusteredLock> zoweInvalidatedTokenLock = new AtomicReference<>();
+
     @Override
     public void afterPropertiesSet() {
         updateKeyring();
@@ -98,7 +101,22 @@ void updateKeyring() {
         }
     }
 
-    @Bean
+    static String getRootFolder() {
+        // using getenv().get is because of system compatibility (see non-case sensitive on Windows)
+        String instanceId = System.getenv().get("ZWE_haInstance_id");
+        if (StringUtils.isBlank(instanceId)) {
+            instanceId = "localhost";
+        }
+
+        String workspaceFolder = System.getenv().get("ZWE_zowe_workspaceDirectory");
+        if (StringUtils.isBlank(workspaceFolder)) {
+            return Paths.get("caching-service", instanceId).toString();
+        } else {
+            return Paths.get(workspaceFolder, "caching-service", instanceId).toString();
+        }
+    }
+
+    @Bean(destroyMethod = "stop")
     synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) {
         System.setProperty("jgroups.tcpping.initial_hosts", initialHosts);
         System.setProperty("jgroups.bind.port", port);
@@ -124,20 +142,18 @@ synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) {
         } catch (IOException e) {
             throw new InfinispanConfigException("Can't read configuration file", e);
         }
+        holder.getGlobalConfigurationBuilder().globalState().persistentLocation(getRootFolder()).enable();
+        holder.newConfigurationBuilder("default").persistence()
+            .addSoftIndexFileStore()
+            .clustering().cacheMode(CacheMode.DIST_SYNC);
 
         DefaultCacheManager cacheManager = new DefaultCacheManager(holder, true);
 
         ConfigurationBuilder builder = new ConfigurationBuilder();
-        builder.clustering()
-            .cacheMode(CacheMode.REPL_SYNC)
-            .encoding()
-            .mediaType("application/x-jboss-marshalling");
-
-        builder.persistence()
-            .passivation(true)
-            .addSoftIndexFileStore()
-            .shared(false)
-            .dataLocation(dataLocation).indexLocation(indexLocation);
+        builder
+            .encoding().mediaType("application/x-jboss-marshalling")
+            .persistence().addSoftIndexFileStore().clustering()
+            .clustering().cacheMode(CacheMode.DIST_SYNC);
 
         List<String> caches = Arrays.asList("zoweCache", "zoweInvalidatedTokenCache");
         caches.forEach(cacheName -> cacheManager.administration()
@@ -151,17 +167,37 @@ synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) {
         return cacheManager;
     }
 
-    @Bean
-    public ClusteredLock lock(DefaultCacheManager cacheManager) {
-        ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(cacheManager);
-        clm.defineLock("zoweInvalidatedTokenLock");
-        return clm.get("zoweInvalidatedTokenLock");
-    }
+    private ClusteredLock lock(DefaultCacheManager cacheManager) {
+        ClusteredLock lock = zoweInvalidatedTokenLock.get();
+        if (lock != null) {
+            return lock;
+        }
 
+        try {
+            synchronized (zoweInvalidatedTokenLock) {
+                lock = zoweInvalidatedTokenLock.get();
+                if (lock == null) {
+                    ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(cacheManager);
+                    // it can throw AvailabilityException
+                    clm.defineLock("zoweInvalidatedTokenLock");
+                    lock = clm.get("zoweInvalidatedTokenLock");
+                }
+                zoweInvalidatedTokenLock.set(lock);
+            }
+            return lock;
+        } catch (AvailabilityException ae) {
+            log.debug("Cannot obtain lock", ae);
+            throw new StorageException(Messages.CACHE_NOT_AVAILABLE.getKey(), Messages.CACHE_NOT_AVAILABLE.getStatus(), ae.getMessage());
+        }
+    }
 
     @Bean
-    public Storage storage(DefaultCacheManager cacheManager, ClusteredLock clusteredLock) {
-        return new InfinispanStorage(cacheManager.getCache("zoweCache"), cacheManager.getCache("zoweInvalidatedTokenCache"), clusteredLock);
+    public Storage storage(DefaultCacheManager cacheManager) {
+        return new InfinispanStorage(
+            cacheManager.getCache("zoweCache"),
+            cacheManager.getCache("zoweInvalidatedTokenCache"),
+            () -> lock(cacheManager)
+        );
     }
 
 }