diff --git a/src/main/java/io/openliberty/tools/common/plugins/util/BinaryScannerUtil.java b/src/main/java/io/openliberty/tools/common/plugins/util/BinaryScannerUtil.java index 67994682..2fbbacd3 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/util/BinaryScannerUtil.java +++ b/src/main/java/io/openliberty/tools/common/plugins/util/BinaryScannerUtil.java @@ -1,5 +1,5 @@ /** - * (C) Copyright IBM Corporation 2021, 2025. + * (C) Copyright IBM Corporation 2021, 2026 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import java.net.URLClassLoader; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public abstract class BinaryScannerUtil { @@ -30,10 +31,42 @@ public abstract class BinaryScannerUtil { public static final String BINARY_SCANNER_MAVEN_GROUP_ID = "com.ibm.websphere.appmod.tools"; public static final String BINARY_SCANNER_MAVEN_ARTIFACT_ID = "binary-app-scanner"; public static final String BINARY_SCANNER_MAVEN_TYPE = "jar"; - public static final String BINARY_SCANNER_MAVEN_VERSION = "[25.0.0.2,)"; + public static final String BINARY_SCANNER_MAVEN_VERSION = "[25.0.0.2.2]"; + + // The coordinates to use for Open Liberty versions 25.0.0.7 and up + public static final String OL_FEATURELIST_GROUP_ID = "io.openliberty.features"; + public static final String OL_FEATURELIST_ARTIFACT_ID = "open_liberty_featurelist"; + public static final String OL_FEATURELIST_TYPE = "xml"; + // the version number is generated at the point of use + // The following key is used when passing the feature list file to the scanner + public static final String OL_FEATURELIST_KEY = "openLiberty"; + + // The coordinates to use for WebSphere Liberty versions 25.0.0.7 to 25.0.0.9 + // We will use them for releases prior to 25.0.0.7 + // io.openliberty.features:websphere_liberty_base__featurelist:xml:25.0.0.7 + // io.openliberty.features:websphere_liberty_core__featurelist:xml:25.0.0.7 + + // The coordinates to use for WebSphere Liberty versions 25.0.0.10 and up + // Publishing stopped with 25.0.0.12 so this value is used for 26.0.0.1 and up. + // com.ibm.websphere.appserver.features:websphere_liberty_base__featurelist:xml:25.0.0.xx + // com.ibm.websphere.appserver.features:websphere_liberty_core__featurelist:xml:25.0.0.xx + + // These coordinates are used in different combinations to access WebSphere Liberty feature lists 25.0.0.7 and up + public static final String WS1_FEATURELIST_GROUP_ID = "io.openliberty.features"; + public static final String WS2_FEATURELIST_GROUP_ID = "com.ibm.websphere.appserver.features"; + public static final String WSBASE_FEATURELIST_ARTIFACT_ID = "websphere_liberty_base__featurelist"; + public static final String WSCORE_FEATURELIST_ARTIFACT_ID = "websphere_liberty_core__featurelist"; + public static final String WS_FEATURELIST_TYPE = "xml"; + // The following keys are used when passing the feature list files to the scanner + public static final String WSBASE_FEATURELIST_KEY = "liberty"; + public static final String WSCORE_FEATURELIST_KEY = "libertyCore"; + public static final String GENERATED_FEATURES_FILE_NAME = "generated-features.xml"; - public static final String GENERATED_FEATURES_FILE_PATH = "configDropins/overrides/" + GENERATED_FEATURES_FILE_NAME; + public static final String GENERATED_FEATURES_DIR_PATH = "configDropins/overrides/"; + public static final String GENERATED_FEATURES_FILE_PATH = GENERATED_FEATURES_DIR_PATH + GENERATED_FEATURES_FILE_NAME; + public static final String GENERATED_FEATURES_TEMP_DIR = ".libertyFeatures"; + public static final String GENERATED_FEATURES_TEMP_PATH = GENERATED_FEATURES_TEMP_DIR + File.separator + GENERATED_FEATURES_FILE_PATH; private static final String FEATURE_MODIFIED_EXCEPTION = "com.ibm.websphere.binary.cmdline.exceptions.RequiredFeatureModifiedException"; private static final String FEATURE_CONFLICT_EXCEPTION = "com.ibm.websphere.binary.cmdline.exceptions.FeatureConflictException"; private static final String PROVIDED_FEATURE_EXCEPTION = "com.ibm.websphere.binary.cmdline.exceptions.ProvidedFeatureConflictException"; @@ -79,13 +112,13 @@ public abstract class BinaryScannerUtil { public abstract boolean isDebugEnabled(); // The jar file containing the binary scanner code - private File binaryScanner; + private File binaryScannerJar; private URLClassLoader binaryScannerClassLoader = null; private Class binaryScannerClass = null; private Method binaryScannerMethod = null; public BinaryScannerUtil(File binaryScanner) { - this.binaryScanner = binaryScanner; + this.binaryScannerJar = binaryScanner; } /** @@ -118,11 +151,11 @@ public BinaryScannerUtil(File binaryScanner) { * scanner when used in combination with each other. E.g. EE 7 and MP 2.1 */ public Set runBinaryScanner(Set currentFeatureSet, List classFiles, Set allClassesDirectories, - String logLocation, String targetJavaEE, String targetMicroProfile, boolean optimize) + String logLocation, String targetJavaEE, String targetMicroProfile, Map featureListFileMap, boolean optimize) throws PluginExecutionException, NoRecommendationException, RecommendationSetException, FeatureModifiedException, FeatureUnavailableException, IllegalTargetException, IllegalTargetComboException { - Set featureList = null; - if (binaryScanner != null && binaryScanner.exists()) { + Set generatedFeatureList = null; + if (binaryScannerJar != null && binaryScannerJar.exists()) { // if we are already generating features for all class files (optimize=true) and // we are not passing any user specified features (currentFeatureSet is empty) // we do not need to rerun the binary scanner if it fails @@ -138,17 +171,18 @@ public Set runBinaryScanner(Set currentFeatureSet, List logLevel = null; logLocation = null; } - debug("Calling " + binaryScanner.getName() + " with the following inputs...\n" + + debug("Calling " + binaryScannerJar.getName() + " with the following inputs...\n" + " binaryInputs: " + binaryInputs + "\n" + " targetJavaEE: " + targetJavaEE + "\n" + " targetMicroP: " + targetMicroProfile + "\n" + " currentFeatures: " + currentFeatureSet + "\n" + + " featureListFileMap: " + featureListFileMap + "\n" + " logLocation: " + logLocation + "\n" + " logLevel: " + logLevel + "\n" + " locale: " + java.util.Locale.getDefault()); - featureList = (Set) generateFeatureSetMethod.invoke(null, binaryInputs, targetJavaEE, targetMicroProfile, - currentFeatureSet, logLocation, logLevel, java.util.Locale.getDefault()); - for (String s : featureList) {debug(s);}; + generatedFeatureList = (Set) generateFeatureSetMethod.invoke(null, binaryInputs, targetJavaEE, targetMicroProfile, + currentFeatureSet, featureListFileMap, logLocation, logLevel, java.util.Locale.getDefault()); + for (String s : generatedFeatureList) {debug(s);}; } catch (InvocationTargetException ite) { // This is the exception from the JVM that indicates there was an exception in the method we // called through reflection. We must extract the actual exception from the 'cause' field. @@ -168,7 +202,7 @@ public Set runBinaryScanner(Set currentFeatureSet, List // The list of features from the app is passed in but it contains conflicts Set conflicts = getFeatures(scannerException); // always rerun binary scanner in this scenario, this exception only occurs if a current feature list is passed to binary scanner - Set sampleFeatureList = reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile); + Set sampleFeatureList = reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile, featureListFileMap); if (sampleFeatureList == null) { throw new NoRecommendationException(conflicts); } else { @@ -178,7 +212,7 @@ public Set runBinaryScanner(Set currentFeatureSet, List // The scanned files conflict with each other or with current features Set conflicts = getFeatures(scannerException); // rerun binary scanner with all class files and without the current feature set to get feature recommendations - Set sampleFeatureList = reRunIfFailed ? reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile): null; + Set sampleFeatureList = reRunIfFailed ? reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile, featureListFileMap): null; if (sampleFeatureList == null) { throw new NoRecommendationException(conflicts); } else { @@ -188,7 +222,7 @@ public Set runBinaryScanner(Set currentFeatureSet, List // The scanned files conflict and the scanner suggests modifying some features Set modifications = getFeatures(scannerException); // rerun binary scanner with all class files and without the current feature set - Set sampleFeatureList = reRunIfFailed ? reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile) : null; + Set sampleFeatureList = reRunIfFailed ? reRunBinaryScanner(allClassesDirectories, logLocation, targetJavaEE, targetMicroProfile, featureListFileMap) : null; throw new FeatureModifiedException(modifications, (sampleFeatureList == null) ? getNoSampleFeatureList() : sampleFeatureList, scannerException.getLocalizedMessage()); } else if (scannerException.getClass().getName().equals(FEATURE_NOT_AVAILABLE_EXCEPTION)) { @@ -229,13 +263,13 @@ public Set runBinaryScanner(Set currentFeatureSet, List throw new PluginExecutionException("An error occurred when trying to call the binary scanner jar: " + loadingException.toString()); } } else { - if (binaryScanner == null) { + if (binaryScannerJar == null) { throw new PluginExecutionException("The binary scanner jar location is not defined."); } else { - throw new PluginExecutionException("Could not find the binary scanner jar at " + binaryScanner.getAbsolutePath()); + throw new PluginExecutionException("Could not find the binary scanner jar at " + binaryScannerJar.getAbsolutePath()); } } - return featureList; + return generatedFeatureList; } /** @@ -252,9 +286,9 @@ public Set runBinaryScanner(Set currentFeatureSet, List * @return - a set of features that will allow the application to run in a Liberty server * @throws PluginExecutionException - any exception that prevents the scanner from running */ - public Set reRunBinaryScanner(Set allClassesDirectories, String logLocation, String targetJavaEE, String targetMicroProfile) - throws PluginExecutionException { - Set featureList = null; + public Set reRunBinaryScanner(Set allClassesDirectories, String logLocation, String targetJavaEE, String targetMicroProfile, + Map featureListFileMap) throws PluginExecutionException { + Set generatedFeatureList = null; try { Method generateFeatureSetMethod = getScannerMethod(); Set binaryInputs = allClassesDirectories; @@ -274,21 +308,21 @@ public Set reRunBinaryScanner(Set allClassesDirectories, String " logLocation: " + logLocation + "\n" + " logLevel: " + logLevel + "\n" + " locale: " + java.util.Locale.getDefault()); - featureList = (Set) generateFeatureSetMethod.invoke(null, binaryInputs, targetJavaEE, targetMicroProfile, - currentFeaturesSet, logLocation, logLevel, java.util.Locale.getDefault()); - for (String s : featureList) {debug(s);}; + generatedFeatureList = (Set) generateFeatureSetMethod.invoke(null, binaryInputs, targetJavaEE, targetMicroProfile, + currentFeaturesSet, featureListFileMap, logLocation, logLevel, java.util.Locale.getDefault()); + for (String s : generatedFeatureList) {debug(s);}; } catch (InvocationTargetException ite) { Throwable scannerException = ite.getCause(); if (scannerException.getClass().getName().equals(PROVIDED_FEATURE_EXCEPTION)) { // this happens when the list of features passed in contains conflicts so now no recommendation possible debug("RuntimeException from re-run of binary scanner", scannerException); // shouldn't happen - featureList = null; + generatedFeatureList = null; } else if (scannerException.getClass().getName().equals(FEATURE_CONFLICT_EXCEPTION)) { // The features in the scanned files conflict with each other, no recommendation possible - featureList = getNoSampleFeatureList(); + generatedFeatureList = getNoSampleFeatureList(); } else if (scannerException.getClass().getName().equals(FEATURE_MODIFIED_EXCEPTION)) { // The features in the scanned files conflict with each other, no recommendation possible - featureList = getNoSampleFeatureList(); + generatedFeatureList = getNoSampleFeatureList(); } else { debug("Exception from rerunning binary scanner.", scannerException); throw new PluginExecutionException("Error scanning the application for Liberty feature recommendations: " + scannerException.toString()); @@ -299,9 +333,9 @@ public Set reRunBinaryScanner(Set allClassesDirectories, String debug("Caused by exception2:"+loadingException.getCause().getClass().getName()); debug("Caused by exception message2:"+loadingException.getCause().getMessage()); } - throw new PluginExecutionException("An error occurred when trying to call the binary scanner jar for recommendations: " + loadingException.toString()); + throw new PluginExecutionException("An error occurred when trying to call the binary scanner jar for Liberty feature recommendations: " + loadingException.toString()); } - return featureList; + return generatedFeatureList; } private Set getNoSampleFeatureList() { @@ -314,7 +348,7 @@ private Set getNoSampleFeatureList() { private ClassLoader getScannerClassLoader() throws MalformedURLException { if (binaryScannerClassLoader == null) { ClassLoader cl = this.getClass().getClassLoader(); - binaryScannerClassLoader = new URLClassLoader(new URL[] { binaryScanner.toURI().toURL() }, cl); + binaryScannerClassLoader = new URLClassLoader(new URL[] { binaryScannerJar.toURI().toURL() }, cl); } return binaryScannerClassLoader; } @@ -330,10 +364,12 @@ private Class getScannerClass() throws MalformedURLException, ClassNotFoundExcep private Method getScannerMethod() throws MalformedURLException, ClassNotFoundException, NoSuchMethodException, PluginExecutionException, SecurityException { if (binaryScannerMethod == null) { Class driveScan = getScannerClass(); - // args: Set, String, String, Set, String, String, Locale - // names: binaryInputs, targetJavaEE, targetMicroProfile, currentFeatures, logLocation, logLevel, locale + // Method name and return type: Set generateFeatureList() + // arg types: Set, String, String, Set, Map, String, String, Locale + // arg names: binaryInputs, targetJavaEE, targetMicroProfile, currentFeatures, featureListFileMap, logLocation, logLevel, locale + binaryScannerMethod = driveScan.getMethod("generateFeatureList", Set.class, String.class, String.class, - Set.class, String.class, String.class, java.util.Locale.class); + Set.class, Map.class, String.class, String.class, java.util.Locale.class); if (binaryScannerMethod == null) { throw new PluginExecutionException("Error finding binary scanner method using reflection"); } diff --git a/src/main/java/io/openliberty/tools/common/plugins/util/DevUtil.java b/src/main/java/io/openliberty/tools/common/plugins/util/DevUtil.java index da23c1cd..47f89290 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/util/DevUtil.java +++ b/src/main/java/io/openliberty/tools/common/plugins/util/DevUtil.java @@ -1,5 +1,5 @@ /** - * (C) Copyright IBM Corporation 2019, 2025. + * (C) Copyright IBM Corporation 2019, 2026 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -435,11 +435,15 @@ private enum FileTrackMode { /** Map of parent build files (parent build file, list of children build files) */ protected Map> parentBuildFiles; private boolean generateFeatures; + private boolean generateToSrc; private Set generatedFeaturesSet; // set of features in generated-features.xml file private boolean generatedFeaturesModified; + private boolean generatedFeaturesCopied; private Set compileArtifactPaths; private Set testArtifactPaths; - protected final File generatedFeaturesFile; + protected File generateFeaturesFile; // the file that is created from the generate-features goal/task + protected File generateFeaturesOutputDir; // output directory for the generate-features goal/task (i.e. where the file is generated) + protected File generateFeaturesTmpDir; // the location where the generated features file is written during dev mode loop when generateToSrc is false private File modifiedSrcBuildFile; protected boolean skipInstallFeature; @@ -455,7 +459,7 @@ public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, File containerfile, File containerBuildContext, String containerRunOpts, int containerBuildTimeout, boolean skipDefaultPorts, JavaCompilerOptions compilerOptions, boolean keepTempContainerfile, String mavenCacheLocation, List upstreamProjects, boolean recompileDependencies, - String packagingType, File buildFile, Map> parentBuildFiles, boolean generateFeatures, + String packagingType, File buildFile, Map> parentBuildFiles, boolean generateFeatures, boolean generateToSrc, Set compileArtifactPaths, Set testArtifactPaths, List monitoredWebResourceDirs, Map projectRecompileMap) { this(buildDirectory, serverDirectory, sourceDirectory, testSourceDirectory, configDirectory, projectDirectory, multiModuleProjectDirectory, resourceDirs, changeOnDemandTestsAction, @@ -465,7 +469,7 @@ public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, containerfile, containerBuildContext, containerRunOpts, containerBuildTimeout, skipDefaultPorts, compilerOptions, keepTempContainerfile, mavenCacheLocation, upstreamProjects, recompileDependencies, - packagingType, buildFile, parentBuildFiles, generateFeatures, + packagingType, buildFile, parentBuildFiles, generateFeatures, generateToSrc, compileArtifactPaths, testArtifactPaths, monitoredWebResourceDirs); // setting projectRecompileMap as empty if input is null from ci.maven this.projectRecompileMap = projectRecompileMap != null ? projectRecompileMap : new HashMap<>(); @@ -480,7 +484,7 @@ public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, File containerfile, File containerBuildContext, String containerRunOpts, int containerBuildTimeout, boolean skipDefaultPorts, JavaCompilerOptions compilerOptions, boolean keepTempContainerfile, String mavenCacheLocation, List upstreamProjects, boolean recompileDependencies, - String packagingType, File buildFile, Map> parentBuildFiles, boolean generateFeatures, + String packagingType, File buildFile, Map> parentBuildFiles, boolean generateFeatures, boolean generateToSrc, Set compileArtifactPaths, Set testArtifactPaths, List monitoredWebResourceDirs) { this.buildDirectory = buildDirectory; this.serverDirectory = serverDirectory; @@ -555,11 +559,14 @@ public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, this.parentBuildFiles = parentBuildFiles; } this.generateFeatures = generateFeatures; + this.generateToSrc = generateToSrc; + this.generateFeaturesTmpDir = new File(buildDirectory, BinaryScannerUtil.GENERATED_FEATURES_TEMP_DIR); + initGenerationContext(); this.compileArtifactPaths = compileArtifactPaths; this.testArtifactPaths = testArtifactPaths; this.monitoredWebResourceDirs = monitoredWebResourceDirs; - this.generatedFeaturesFile = new File(configDirectory, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH); this.generatedFeaturesModified = false; + this.generatedFeaturesCopied = false; if (this.generateFeatures) { this.generatedFeaturesSet = getGeneratedFeatures(); } else { @@ -568,6 +575,17 @@ public DevUtil(File buildDirectory, File serverDirectory, File sourceDirectory, this.modifiedSrcBuildFile = null; } + private void initGenerationContext() { + this.generateFeaturesOutputDir = generateToSrc ? configDirectory : generateFeaturesTmpDir; + this.generateFeaturesFile = new File(generateFeaturesOutputDir, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH); + } + + public void copyGeneratedFeaturesFile(File destinationDir) throws IOException { + copyFile(generateFeaturesFile, generateFeaturesOutputDir, destinationDir, null); + if (destinationDir.equals(serverDirectory)) { + generatedFeaturesCopied = true; // features copied into server dir and not some temp dir + } + } /** * Run unit and/or integration tests * @@ -1910,8 +1928,20 @@ private void addUserId(List commandElements) { } } + /** + * Create a Liberty server using the default name "defaultServer" or the server + * specified by the common parameter "serverName" in pom.xml. + * + * @throws PluginExecutionException + */ public abstract void libertyCreate() throws PluginExecutionException; + /** + * Deploy the default apps on the Liberty server or deploy the apps + * specified in the configuration of the deploy goal in pom.xml. + * + * @throws PluginExecutionException + */ public abstract void libertyDeploy() throws PluginExecutionException; /** @@ -1919,9 +1949,13 @@ private void addUserId(List commandElements) { * * @param classes class file paths features should be generated for (can be null if no modified classes) * @param optimize if true, generate optimized feature list + * @param generateToSrc if true, generate feature list into file in src/main/liberty + * @param useTmpDirOut if true, generate feature file in a hidden directory named in BinaryScannerUtil + * @param useTmpDirIn if true, the hidden directory named in BinaryScannerUtil will be used as the + * context or input values to generate features * @return true if feature generation was successful */ - public abstract boolean libertyGenerateFeatures(Collection classes, boolean optimize); + public abstract boolean libertyGenerateFeatures(Collection classes, boolean optimize, boolean generateToSrc, boolean useTmpDirOut, boolean useTmpDirIn); /** * Install features in regular dev mode. This method should not be used in container mode. @@ -2176,13 +2210,13 @@ public void cleanUpServerEnv() { } } - public void cleanUpTempConfig() { - if (this.tempConfigPath != null) { - File tempConfig = this.tempConfigPath.toFile(); + public void cleanUpTempConfig(Path myTempConfigPath) { + if (myTempConfigPath != null) { + File tempConfig = myTempConfigPath.toFile(); if (tempConfig.exists()) { try { FileUtils.deleteDirectory(tempConfig); - debug("Successfully deleted liberty:dev temporary configuration folder"); + debug("Successfully deleted liberty:dev temporary configuration folder: " + myTempConfigPath); } catch (IOException e) { warn("Could not delete liberty:dev temporary configuration folder: " + e.getMessage()); } @@ -2233,7 +2267,7 @@ private void runShutdownHook(final ThreadPoolExecutor executor) { } setDevStop(true); - cleanUpTempConfig(); + cleanUpTempConfig(this.tempConfigPath); cleanUpServerEnv(); if (hotkeyReader != null) { @@ -2628,18 +2662,16 @@ private void printHelpMessages() { private void printFeatureGenerationStatus() { info(formatAttentionMessage("Automatic generation of features: " + getFormattedBooleanString(generateFeatures))); + info(formatAttentionMessage("Generation of features to src directory: " + getFormattedBooleanString(generateToSrc))); } private void printFeatureGenerationHotkeys() { info(formatAttentionMessage( "g - toggle the automatic generation of features, type 'g' and press Enter.")); - info(formatAttentionMessage( - " A new server configuration file will be generated in the SOURCE configDropins/overrides configuration directory.")); + info(formatAttentionMessage("s - toggle the option to generate features to the src directory, type 's' and press Enter.")); if (generateFeatures) { // If generateFeatures is enabled, then also describe the optimize hotkey info(formatAttentionMessage("o - optimize the list of generated features, type 'o' and press Enter.")); - info(formatAttentionMessage( - " A new server configuration file will be generated in the SOURCE configDropins/overrides configuration directory.")); } } @@ -2663,15 +2695,11 @@ private void toggleFeatureGeneration() { generateFeatures = !generateFeatures; logFeatureGenerationStatus(); if (generateFeatures) { - String generatedFileCanonicalPath; - try { - generatedFileCanonicalPath = new File(configDirectory, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH).getCanonicalPath(); - } catch (IOException e) { - generatedFileCanonicalPath = new File(configDirectory, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH).toString(); + if (generateToSrc) { + infoSrcDirModified(); } - warn("The source configuration directory will be modified. Features will automatically be generated in a new file: " + generatedFileCanonicalPath); // If hotkey is toggled to “true”, generate features right away. - optimizeGenerateFeatures(); + optimizeGenerateFeatures(!generateToSrc); } } @@ -2679,18 +2707,67 @@ private void logFeatureGenerationStatus() { info("Setting automatic generation of features to: " + getFormattedBooleanString(generateFeatures)); } + private void toggleGenerateToSrc() { + generateToSrc = !generateToSrc; + logGenerateToSrcStatus(); + initGenerationContext(); + // When you toggle generateToSrc delete the old file you no longer need + if (generateToSrc) { + deleteGenFeaturesFile(generateFeaturesTmpDir); // delete the old gen file in tmpdir + } else { + deleteGenFeaturesFile(configDirectory); // delete the old gen file in src/main/liberty/config + } + if (generateFeatures) { + if (generateToSrc) { + infoSrcDirModified(); + } + // If hotkey is toggled, generate features right away. + optimizeGenerateFeatures(!generateToSrc); + } + } + + private void deleteGenFeaturesFile(File dir) { + // N.B. processConfigFileChange() will be called upon deletion of generated features file, it should be ignored + File oldGenFeaturesFile = new File(dir, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH); + if (oldGenFeaturesFile.exists()) { + if (!oldGenFeaturesFile.delete()) { + debug("Error trying to delete the generated features file:" + oldGenFeaturesFile.getAbsolutePath()); + } + } + } + + private void logGenerateToSrcStatus() { + info("Setting generation of features in src directory to: " + getFormattedBooleanString(generateToSrc)); + } + + private void infoSrcDirModified() { + String generatedFileCanonicalPath; + try { + generatedFileCanonicalPath = new File(configDirectory, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH).getCanonicalPath(); + } catch (IOException e) { + generatedFileCanonicalPath = new File(configDirectory, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH).toString(); + } + info("The source configuration directory will be modified. Features will automatically be generated in a new file: " + generatedFileCanonicalPath); + } + + // called by Liberty plugins protected void setFeatureGeneration(boolean generateFeatures) { this.generateFeatures = generateFeatures; logFeatureGenerationStatus(); } + protected boolean optimizeGenerateFeatures(boolean useTmpDir) { + debug("Entering optimizeGenerateFeatures(boolean)"); + return optimizeGenerateFeatures(useTmpDir, false); + } + /** * Generate features using all classes and only user specified features. */ - private boolean optimizeGenerateFeatures() { - debug("Generating optimized features list..."); + protected boolean optimizeGenerateFeatures(boolean useTmpDirOut, boolean useTmpDirIn) { + debug("Generating optimized features list...use temp directory for output=" + useTmpDirOut + " use temp directory for input=" + useTmpDirIn); // scan all class files and provide only user specified features - boolean generatedFeatures = libertyGenerateFeatures(null, true); + boolean generatedFeatures = libertyGenerateFeatures(null, true, generateToSrc, useTmpDirOut, useTmpDirIn); if (generatedFeatures) { modifiedClasses.clear(); failedToGenerateClasses.clear(); @@ -2704,12 +2781,12 @@ private boolean optimizeGenerateFeatures() { * Generate features using updated classes and all existing features. * Returns true if successful */ - private boolean incrementGenerateFeatures() { - debug("Generating feature list from incremental changes..."); + private boolean incrementGenerateFeatures(boolean useTmpDir) { + debug("Generating feature list from incremental changes...use temp directory=" + useTmpDir); boolean generatedFeatures = false; try { Collection classPaths = getClassPaths(modifiedClasses); - generatedFeatures = libertyGenerateFeatures(classPaths, false); + generatedFeatures = libertyGenerateFeatures(classPaths, false, generateToSrc, useTmpDir, false); if (generatedFeatures) { modifiedClasses.clear(); failedToGenerateClasses.clear(); @@ -2756,6 +2833,7 @@ private void readInput() { HotKey h = new HotKey("h", "help"); HotKey r = new HotKey("r"); HotKey g = new HotKey("g"); + HotKey s = new HotKey("s"); HotKey o = new HotKey("o"); HotKey t = new HotKey("t"); HotKey p = new HotKey("p"); @@ -2789,9 +2867,11 @@ private void readInput() { info(formatAttentionBarrier()); } else if (g.isPressed(line)) { toggleFeatureGeneration(); + } else if (s.isPressed(line)) { + toggleGenerateToSrc(); } else if (o.isPressed(line)) { if (generateFeatures) { - optimizeGenerateFeatures(); + optimizeGenerateFeatures(!generateToSrc); } else { warn("Cannot optimize features because automatic generation of features is off."); warn("To toggle the automatic generation of features, type 'g' and press Enter."); @@ -3009,6 +3089,11 @@ public void watchFiles(File outputDirectory, File testOutputDirectory, final Thr registerSingleFile(containerfileUsed, executor); } + // Always register the generated file in the temp dir. because generateToSrc can be toggled on and off in dev mode + File hiddenTempGenerateFeaturesFile = new File(generateFeaturesTmpDir, BinaryScannerUtil.GENERATED_FEATURES_FILE_PATH); + hiddenTempGenerateFeaturesFile.getParentFile().mkdirs(); // must only mkdir on the directories + registerSingleFile(hiddenTempGenerateFeaturesFile, executor); + HashMap resourceMap = new HashMap(); for (File resourceDir : resourceDirs) { resourceMap.put(resourceDir, false); @@ -3112,10 +3197,10 @@ public void watchFiles(File outputDirectory, File testOutputDirectory, final Thr // reset lastChangeCompiled and modifiedSrcBuildFile lastChangeCompiled = false; // only needed when recompileDependencies is true modifiedSrcBuildFile = null; // only needed when recompileDependencies is true - long generatedTime = generatedFeaturesFile.lastModified(); + long generatedTime = generateFeaturesFile.lastModified(); int numApplicationUpdatedMessages = countApplicationUpdatedMessages(); - incrementGenerateFeatures(); - if (!generatedFeaturesFile.exists()) { + incrementGenerateFeatures(!generateToSrc); + if (!generateFeaturesFile.exists()) { // run tests if generated-features.xml does not exist as there are no new features to install // (typically tests run after generate features & install when hotTests=true) if (isMultiModuleProject()) { @@ -3123,7 +3208,7 @@ public void watchFiles(File outputDirectory, File testOutputDirectory, final Thr } else { runTestThread(false, executor, -1, false, false, buildFile); } - } else if (generatedFeaturesFile.lastModified() == generatedTime) { + } else if (generateFeaturesFile.lastModified() == generatedTime) { // The generated-features.xml file was not modified by adding or removing features as a // result of the compilation so call tests now. If it had been changed tests would be called // after processing the config file change. @@ -3361,15 +3446,6 @@ public void watchFiles(File outputDirectory, File testOutputDirectory, final Thr } } - /** - * - * @return {@code Collection} of class paths - * @throws IOException - */ - public Collection getJavaSourceClassPaths() throws IOException { - return getClassPaths(modifiedClasses); - } - /** * * @param classFiles javaSourceClassFiles that have been modified @@ -4159,6 +4235,7 @@ private void processFileChanges( Path testSrcPath = this.testSourceDirectory.getCanonicalFile().toPath(); Path configPath = this.configDirectory.getCanonicalFile().toPath(); Path outputPath = this.outputDirectory.getCanonicalFile().toPath(); + Path gfTmpDirPath = this.generateFeaturesTmpDir.getCanonicalFile().toPath(); Path directory = fileChanged.getParentFile().getCanonicalFile().toPath(); @@ -4193,7 +4270,7 @@ private void processFileChanges( if (it.hasNext()) { File newlyRegisteredFile = it.next(); // confirm that the newly registered file is generated features file in configDropins/overrides - if (newlyRegisteredFile.equals(generatedFeaturesFile)) { + if (newlyRegisteredFile.equals(generateFeaturesFile)) { // process file changes for the generated features file so that newly generated features are installed debug("Registered configDropins/overrides directory, processing file changes for generated features file: " + newlyRegisteredFile); @@ -4434,9 +4511,10 @@ private void processFileChanges( // This is for server.xml specified by the configuration parameter // server will load new properties processConfigFileChange(fileChanged, changeType, executor, numApplicationUpdatedMessages, true); - } else if (directory.startsWith(configPath) + } else if ((directory.startsWith(configPath) + || directory.startsWith(gfTmpDirPath)) && !isGeneratedConfigFile(fileChanged, configDirectory, serverDirectory)) { - // configuration file + // configuration file or generate-features.xml in temp directory processConfigFileChange(fileChanged, changeType, executor, numApplicationUpdatedMessages, false); } else if (bootstrapPropertiesFileParent != null && directory.equals(bootstrapPropertiesFileParent.getCanonicalFile().toPath()) @@ -4559,9 +4637,11 @@ private void processFileChanges( private void processConfigFileChange(File fileChanged, ChangeType changeType, ThreadPoolExecutor executor, int numApplicationUpdatedMessages, boolean configuredServerXml) throws IOException, PluginExecutionException { - boolean isGeneratedFeaturesFile = configuredServerXml ? false : fileChanged.equals(generatedFeaturesFile); + boolean isGeneratedFeaturesFile = configuredServerXml ? false : fileChanged.equals(generateFeaturesFile); String targetFileName = configuredServerXml ? "server.xml" : null; // if null file will retain the same name when copied - File fileChangedParentDir = configuredServerXml ? serverXmlFileParent : configDirectory; + // three possible values for the parent directory + File fileChangedParentDir = configuredServerXml ? serverXmlFileParent : + isGeneratedFeaturesFile ? generateFeaturesOutputDir : configDirectory; if (fileChanged.exists() && (changeType == ChangeType.MODIFY || changeType == ChangeType.CREATE)) { debug("Config file modified: " + fileChanged); @@ -4570,26 +4650,51 @@ private void processConfigFileChange(File fileChanged, ChangeType changeType, Th // generate features whenever features have changed and an XML file is modified, // excluding the generated features file + // if generateToSrc is false then we must copy generated file to serverDir after install to temp if (generateFeatures && (fileChanged.getName().endsWith(".xml") && !isGeneratedFeaturesFile) && serverFeaturesModified) { - generateFeaturesSuccess = optimizeGenerateFeatures(); + // If generating to server dir we use the server dir config files and also the + // modified xml file in src dir. Copy them all to the gen. feat. temp dir to + // combine them for feature generation. + if (!generateToSrc) { + // Deleting generateFeaturesTmpDir also deletes generateFeaturesFile which we "watch" in + // dev mode. This causes a deletion event and we are counting on the handler (this method, + // below) not to call generate features and recreate the file. + cleanUpTempConfig(generateFeaturesTmpDir.toPath()); + // copy config files to temp dir + copyToTempDir(serverDirectory, generateFeaturesTmpDir); + // copy changed file to temp dir + copyFile(fileChanged, fileChangedParentDir, generateFeaturesTmpDir, targetFileName); + } + generateFeaturesSuccess = optimizeGenerateFeatures(!generateToSrc, !generateToSrc); } if (serverFeaturesModified) { - // suppress install feature warning - property must be set before calling - // installFeaturesToTempDir + // suppress install feature warning - property must be set before installing using temp dir System.setProperty(SKIP_BETA_INSTALL_WARNING, Boolean.TRUE.toString()); installFeaturesToTempDir(fileChanged, fileChangedParentDir, targetFileName, generateFeaturesSuccess); } - copyFile(fileChanged, fileChangedParentDir, serverDirectory, targetFileName); - - // if the generated features file was modified as a result of another config - // file modification, copy it over to target so the server picks up the changes - // together + // Copy the config file which was changed to the server directory unless it is + // the generated features file. The generated features file may have been copied + // to the server directory already as a result of a change to the build file (pom.xml) + // or the server.xml. + if (!isGeneratedFeaturesFile) { // all other config files + copyFile(fileChanged, fileChangedParentDir, serverDirectory, targetFileName); + } else { + if (!generatedFeaturesCopied) { + copyGeneratedFeaturesFile(serverDirectory); + } + generatedFeaturesCopied = false; + } + // If the generated features file was modified as a result of another config file modification + // (usually server.xml), copy it over to target so the server picks up the two changes together if (generateFeaturesSuccess && generatedFeaturesModified && !isGeneratedFeaturesFile) { - // this logic is not entered if the fileChanged is the generated features file - // copy generated features file to server dir - copyFile(generatedFeaturesFile, configDirectory, serverDirectory, null); + // This logic is not entered if the fileChanged is the generated features file. + // Copy generated features file to server dir and set generatedFeaturesCopied true + // (not referring to generatedFeaturesModified). + // Leave generatedFeaturesCopied true because the call to optimize/incrementalGenerateFeatures that was + // required to get into this IF block will also generate a file change event for generatedFeaturesFile + copyGeneratedFeaturesFile(serverDirectory); generatedFeaturesModified = false; } if (serverFeaturesModified) { @@ -4628,11 +4733,17 @@ private void processConfigFileChange(File fileChanged, ChangeType changeType, Th info("Config file deleted: " + fileChanged.getName()); deleteFile(fileChanged, fileChangedParentDir, serverDirectory, targetFileName); // generate features whenever features have changed and an XML file is deleted, - // excluding the generated-features.xml file + // excluding the generated-features.xml file. This is important also when we delete the + // generateFeaturesTmpDir in the process of handling an xml config modicifcation. + // Deleting that directory could cause generated-features.xml to be deleted and we + // need to be careful how to handle that event e.g. don't call optimizeGenerateFeatures(). + // Another scenario, when we toggle generateToSrc option we delete the old file and change the + // value of generateFeaturesFile. Therefore we must only use the base name of generateFeaturesFile + // in this file name check. if (generateFeatures && (fileChanged.getName().endsWith(".xml") - && !fileChanged.equals(generatedFeaturesFile)) + && !fileChanged.getName().equals(generateFeaturesFile.getName())) && serverFeaturesModified()) { - optimizeGenerateFeatures(); + optimizeGenerateFeatures(!generateToSrc); } // Let this restart if needed for container mode. Otherwise, nothing else needs to be done for config file delete. if (isContainerfileDirectoryChanged(serverDirectory, fileChanged)) { @@ -4750,8 +4861,8 @@ private boolean isContainerfileDirectoryChanged(File... files) throws IOExceptio } /** - * Determines if the corresponding target config file was generated by a Liberty - * plugin + * Determines if the specified config file in target dir was generated by a Liberty + * plugin: bootstrap.properties or jvm.options * * @param fileChanged the file that was changed * @param srcDir the directory of the file changed @@ -4810,27 +4921,40 @@ public void installFeaturesToTempDir(File fileChanged, File srcDir, String targe File tempConfig = tempConfigPath.toFile(); debug("Temporary configuration folder created: " + tempConfig); - FileUtils.copyDirectory(serverDirectory, tempConfig, new FileFilter() { + copyToTempDir(serverDirectory, tempConfig); + copyFile(fileChanged, srcDir, tempConfig, targetFileName); + if (generateFeatures && generateFeaturesSuccess && !fileChanged.equals(generateFeaturesFile)) { + copyGeneratedFeaturesFile(tempConfig); + } + installFeatures(fileChanged, tempConfig, generateFeatures); + cleanUpTempConfig(this.tempConfigPath); + } + + /** + * Copy the liberty config in the sourceDir directory into the supplied temp directory. + * Filter out certain directories used in Liberty configuration: workarea, logs, messaging + * and also the files dev mode usually ignores e.g. .dir, .file, xxx.dmp etc + * + * @param sourceDir copy files from this directory + * @param tempDir target directory to which files are copied + */ + public File copyToTempDir(File sourceDir, File tempConfig) throws IOException { + FileUtils.copyDirectory(sourceDir, tempConfig, new FileFilter() { public boolean accept(File pathname) { String name = pathname.getName(); String parent = pathname.getParentFile().getName(); - String serverDirName = serverDirectory.getName(); + String sourceDirName = sourceDir.getName(); // skip: // - ignore list // - workarea, messaging, and logs dirs from the server directory, since those can be // changing boolean skip = ignoreFileOrDir(pathname) || (pathname.isDirectory() && - (name.equals("workarea") || name.equals("logs") || (name.equals("messaging") && parent.equals(serverDirName)))); + (name.equals("workarea") || name.equals("logs") || (name.equals("messaging") && parent.equals(sourceDirName)))); return !skip; } }, true); - copyFile(fileChanged, srcDir, tempConfig, targetFileName); - if (generateFeatures && generateFeaturesSuccess && !fileChanged.equals(generatedFeaturesFile)) { - // copy generated-features.xml file - copyFile(generatedFeaturesFile, srcDir, tempConfig, generatedFeaturesFile.getName()); - } - installFeatures(fileChanged, tempConfig, generateFeatures); - cleanUpTempConfig(); + + return tempConfig; } /** @@ -5868,12 +5992,17 @@ private boolean serverFeaturesModified() { if (generateFeatures) { // generateFeatures scenario: check if a generated feature has been manually added to other config files + // Here we pass generated-features.xml instead of server.xml to calculate the generated ones + // The second parameter will only be used if generated-features.xml contains an element and + // the current design does not allow this. FeaturesPlatforms fp = servUtil.getServerXmlFeatures(new FeaturesPlatforms(), serverDirectory, - generatedFeaturesFile, null, null); + generateFeaturesFile, null, null); if (fp != null) generatedFeatureSet = fp.getFeatures(); + + // Calculate the features specified in the config excluding those in generated-features Set generatedFiles = new HashSet(); - generatedFiles.add(generatedFeaturesFile.getName()); + generatedFiles.add(generateFeaturesFile.getName()); // if serverXmlFile is null, getServerFeatures will use the default server.xml // in the configDirectory fp = servUtil.getServerFeatures(configDirectory, serverXmlFile, @@ -5913,10 +6042,12 @@ private boolean generatedFeaturesModified() { } // returns the features specified in the generated-features.xml file + // generated-features.xml has a element so it is also a "serverFile" + // The second parameter will not be used for generated-features.xml. private Set getGeneratedFeatures() { ServerFeatureUtil servUtil = getServerFeatureUtilObj(); FeaturesPlatforms fp = servUtil.getServerXmlFeatures(new FeaturesPlatforms(), configDirectory, - generatedFeaturesFile, null, null); + generateFeaturesFile, null, null); return fp!=null ? fp.getFeatures() : new HashSet(); } diff --git a/src/main/java/io/openliberty/tools/common/plugins/util/ServerFeatureUtil.java b/src/main/java/io/openliberty/tools/common/plugins/util/ServerFeatureUtil.java index 5f5b90f6..340f4d22 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/util/ServerFeatureUtil.java +++ b/src/main/java/io/openliberty/tools/common/plugins/util/ServerFeatureUtil.java @@ -358,8 +358,8 @@ public FeaturesPlatforms getServerXmlFeatures(FeaturesPlatforms origResult, File debug("Exception received: "+e.getMessage(), e); return result; } - - info("Parsing the server file for features and includes: " + getRelativeServerFilePath(serverDirectory, serverFile)); + + info("Parsing the server file for features and includes: " + getRelativeServerFilePathForDisplay(serverDirectory, serverFile)); updatedParsedXmls.add(canonicalServerFile); if (!canonicalServerFile.exists()) { warn("The server file " + canonicalServerFile + " does not exist. Skipping its features."); @@ -669,12 +669,16 @@ private Properties getPropertiesFromFile(File propertiesFile) { // return the server file path relative to the server directory - private String getRelativeServerFilePath(File serverDirectory, File serverFile) { + private String getRelativeServerFilePathForDisplay(File serverDirectory, File serverFile) { try { File canonicalServerDirectory = serverDirectory.getCanonicalFile(); URI serverDirectoryUri = canonicalServerDirectory.toURI(); URI serverFileUri = serverFile.toURI(); - return serverDirectory.getName() + File.separator + serverDirectoryUri.relativize(serverFileUri).getPath(); + if (serverDirectoryUri.relativize(serverFileUri).equals(serverFileUri)) { + return serverFileUri.getPath().toString(); + } else { + return serverDirectory.getName() + File.separator + serverDirectoryUri.relativize(serverFileUri).getPath(); + } } catch (IOException e1) { debug("Unable to determine the file path of " + serverFile + " relative to the server directory " + serverDirectory); diff --git a/src/test/java/io/openliberty/tools/common/plugins/util/BaseDevUtilTest.java b/src/test/java/io/openliberty/tools/common/plugins/util/BaseDevUtilTest.java index 508feddf..3a546b9c 100644 --- a/src/test/java/io/openliberty/tools/common/plugins/util/BaseDevUtilTest.java +++ b/src/test/java/io/openliberty/tools/common/plugins/util/BaseDevUtilTest.java @@ -41,13 +41,13 @@ public DevTestUtil(File serverDirectory, File sourceDirectory, File testSourceDi List resourceDirs, List webResourceDirs, boolean hotTests, boolean skipTests) throws IOException { super(temp.newFolder(), serverDirectory, sourceDirectory, testSourceDirectory, configDirectory, null, null, resourceDirs, false, hotTests, skipTests, false, false, false, null, 30, 30, 5, 500, true, false, false, false, - false, null, null, null, 0, false, null, false, null, null, false, null, null, null, false, null, null, webResourceDirs, Collections.emptyMap()); + false, null, null, null, 0, false, null, false, null, null, false, null, null, null, false, false, null, null, webResourceDirs, Collections.emptyMap()); } public DevTestUtil(File serverDirectory, File buildDir) { super(buildDir, serverDirectory, null, null, null, null, null, null, false, false, false, false, false, false, null, 30, 30, 5, 500, true, false, false, false, - false, null, null, null, 0, false, null, false, null, null, false, null, null, null, false, null, null, null, Collections.emptyMap()); + false, null, null, null, 0, false, null, false, null, null, false, null, null, null, false, false, null, null, null, Collections.emptyMap()); } @Override @@ -197,7 +197,7 @@ public void libertyInstallFeature() { } @Override - public boolean libertyGenerateFeatures(Collection classes, boolean optimize) { + public boolean libertyGenerateFeatures(Collection classes, boolean optimize, boolean generateToSrc, boolean useTempOut, boolean useTempIn) { // not needed for tests return true; }