Add 2.0.0 files

Signed-off-by: Brian Presley <[email protected]>

Author: sumobrian

AWSSolutionsArchitecture.png

@@ -4,9 +4,25 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.0.0] - 2024-09-30
+
+### Added
+ - Support for workloads up to 100TB with the new Reindex-from-Snapshot capability
+ - Added Migration Console CLI to facilitate migrations
+ - GovCloud support
+ - Backfill Pause-and-Resume, with the ability to roll back a migration
+ - Increased vertical performance on Replay
+ - Metadata Migration Tool to migrate cluster settings and configurations
+ - Load balancer configuration to facilitate zero-downtime migrations
+ - Support for new migration paths, including 6.x, 7.x, and 1.x as source versions
+ - Support for multi-hop migrations
+
+### Removed
+ - Fetch Backfill support
 
 ## [1.0.0] - 2023-11-17
 
 ### Added
 
 - All files, initial release
+

CHANGELOG.md

@@ -11,7 +11,7 @@ information to effectively respond to your bug report or contribution.
 
 We welcome you to use the GitHub issue tracker to report bugs or suggest features.
 
-When filing an issue, please check [existing open](https://github.com/opensearch-project/opensearch-migrations), or [recently closed](https://github.com/opensearch-project/opensearch-migrations/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already
+When filing an issue, please check [existing open](https://github.com/opensearch-project/opensearch-migrations/issues), or [recently closed](https://github.com/opensearch-project/opensearch-migrations/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already
 reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
 
 * A reproducible test case or series of steps

CONTRIBUTING.md

@@ -50,3 +50,6 @@ requests-aws4auth is under MIT License (MIT License)
 rsa under the Apache Software License (Apache-2.0)
 s3transfer under the Apache Software License (Apache License 2.0)
 xmltodict under the MIT License (MIT)
+coloredlogs under the MIT License (MIT)
+humanfriendly under the MIT License (MIT)
+prometheus_client under Apache Software License Apache 2.0

NOTICE.txt

@@ -50,8 +50,7 @@ The solution facilitates user comparisons of source and target traffic in terms
 
 Deploying this solution with the default parameters builds the following environment in the AWS Cloud.
 
-<img width="818" alt="image" src="https://github.com/user-attachments/assets/1caf8f48-33eb-4b6a-b449-d2f8e3f539bb">
-
+<img width="818" alt="image" src="AWSSolutionsArchitecture.png">
 
 The high-level process flow for the solution components deployed with the AWS CloudFormation template is as follows:
 
@@ -62,7 +61,6 @@ The high-level process flow for the solution components deployed with the AWS Cl
 5. Performance and behavior of traffic sent to source and target clusters are compared by reviewing logs and metrics.
 6. After confirming the target cluster’s functionality meets expectations the use redirects clients to new target.
 
-
 ## Deployment
 
 Please follow the [Implementation Guide](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch/) to deploy the solution in your AWS account.

README.md

@@ -4,6 +4,47 @@
 [ "$DEBUG" == 'true' ] && set -x
 set -e
 
+ECR_ACCOUNT="740797759474.dkr.ecr.us-west-2.amazonaws.com"
+ECR_REPO_URL="$ECR_ACCOUNT/opensearch-migrations/base-images"
+
+# Check load-images-to-ecr.sh for loading needed images into ECR
+pull_from_ecr() {
+  local docker_image_tag=$1
+  local ecr_image_tag=$2
+
+  echo "Checking for local Docker image: $docker_image_tag"
+
+  # Check if the image exists locally
+  if docker image inspect $docker_image_tag >/dev/null 2>&1; then
+    echo "Docker image already exists locally."
+  else
+    echo "Attempting to pull Docker image from ECR: $docker_image_tag"
+
+    docker pull "$ECR_REPO_URL:$ecr_image_tag"
+    docker tag "$ECR_REPO_URL:$ecr_image_tag" "$docker_image_tag"
+  fi
+}
+
+pull_docker_image() {
+  local image_name=$1
+
+  echo "Checking for Docker image: $image_name"
+
+  # Check if the image exists locally
+  if docker image inspect $image_name >/dev/null 2>&1; then
+    echo "Docker image already exists locally."
+  else
+    echo "Attempting to pull Docker image: $image_name"
+
+    # Try to pull the Docker image
+    if docker pull $image_name; then
+      echo "Docker image pulled successfully."
+    else
+      echo "Failed to pull Docker image. Aborting unit tests."
+      exit 1
+    fi
+  fi
+}
 
 prepare_jest_coverage_report() {
 	local component_name=$1
@@ -31,33 +72,36 @@ run_python_tests() {
   echo "cd $component_path"
   cd "$component_path"
 
-  python3 -m venv .venv
-  source .venv/bin/activate
-  pip install -r requirements.txt
-  pip install -r dev-requirements.txt
-  python3 -m coverage run -m unittest
-  python3 -m coverage xml --omit "*/tests/*"
+  echo "python3 -m pip install --upgrade pipenv"
+  python3 -m pip install --upgrade pipenv
+  echo "pipenv install --deploy --dev"
+  pipenv install --deploy --dev
+  echo "pipenv run python -m coverage run -m pytest"
+  pipenv run python -m coverage run -m pytest
+
+  echo "pipenv run python -m coverage xml --omit '*/tests/*'"
+  pipenv run python -m coverage xml --omit "*/tests/*"
+
   # The coverage module uses absolutes paths in its coverage output. To avoid dependencies of tools (such as SonarQube)
   # on different absolute paths for source directories, this substitution is used to convert each absolute source
   # directory path to the corresponding project relative path. The $source_dir holds the absolute path for source
   # directory.
   sed -i -e "s,<source>$source_dir,<source>source,g" coverage.xml
-  deactivate
-  rm -rf .venv
 }
 
 
 run_gradle_tests() {
   local component_path=$1
   local component_name=$2
+  local gradle_arguments=$3
 
   echo "------------------------------------------------------------------------------"
   echo "[Test] Run unit test with coverage for $component_name"
   echo "------------------------------------------------------------------------------"
   echo "cd $component_path"
   cd "$component_path"
 
-  ./gradlew build copyDependencies jacocoTestReport
+  ./gradlew $gradle_arguments
 }
 
 run_npm_tests() {
@@ -96,18 +140,50 @@ check_test_failure() {
   fi
 }
 
+if aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin "$ECR_ACCOUNT" && pull_from_ecr "amazonlinux:2023" "amazonlinux-2023"; then
+  echo "Successful accessed ECR account $ECR_ACCOUNT and pulled amazonlinux-2023, pulling remaining base images from here..."
+  pull_from_ecr "opensearchproject/opensearch:1.3.16" "opensearch-1.3.16"
+  pull_from_ecr "opensearchproject/opensearch:2.14.0" "opensearch-2.14.0"
+  pull_from_ecr "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" "elasticsearch-oss-7.10.2"
+  pull_from_ecr "docker.elastic.co/elasticsearch/elasticsearch:7.17.22" "elasticsearch-7.17.22"
+  pull_from_ecr "docker.elastic.co/elasticsearch/elasticsearch:6.8.23" "elasticsearch-6.8.23"
+  pull_from_ecr "httpd:alpine" "httpd-alpine"
+  pull_from_ecr "alpine:3.16" "alpine-3.16"
+  pull_from_ecr "confluentinc/cp-kafka:7.5.0" "cp-kafka-7.5.0"
+  pull_from_ecr "ghcr.io/shopify/toxiproxy:latest" "toxiproxy-latest"
+else
+  echo "Failed to access $ECR_ACCOUNT or pull image from this ECR, will not pull base images from this ECR"
+  # com.rfs.framework.SearchClusterContainer Images
+  pull_docker_image "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2"
+  pull_docker_image "docker.elastic.co/elasticsearch/elasticsearch:7.17.22"
+  pull_docker_image "docker.elastic.co/elasticsearch/elasticsearch:6.8.23"
+  pull_docker_image "opensearchproject/opensearch:1.3.16"
+  pull_docker_image "opensearchproject/opensearch:2.14.0"
+  # org.opensearch.migrations.trafficcapture.proxyserver.testcontainers.HttpdContainerTestBase
+  pull_docker_image "httpd:alpine"
+  pull_docker_image "alpine:3.16"
+  # org.opensearch.migrations.trafficcapture.proxyserver.testcontainers.KafkaContainerTestBase
+  pull_docker_image "confluentinc/cp-kafka:7.5.0"
+  # org.opensearch.migrations.trafficcapture.proxyserver.testcontainers.ToxiproxyContainerTestBase
+  pull_docker_image "ghcr.io/shopify/toxiproxy:latest"
+  # TrafficCapture/dockerSolution/src/main/docker/elasticsearchTestConsole/Dockerfile
+  pull_docker_image "amazonlinux:2023"
+fi
+
+echo "Images pulled successfully. Continuing to unit tests."
+
 # Run unit tests
 echo "Running unit tests"
 
 # Get reference for source folder
 source_dir="$(cd $PWD/../source; pwd -P)"
 coverage_reports_top_path=$source_dir/test/coverage-reports
 
-run_gradle_tests "$source_dir/opensearch-migrations/TrafficCapture" "TrafficCapture"
-check_test_failure "TrafficCapture"
+run_gradle_tests "$source_dir/opensearch-migrations" "opensearch-migrations" "build copyDependencies mergeJacocoReports -x spotlessCheck"
+check_test_failure "opensearch-migrations"
 
-run_python_tests "$source_dir/opensearch-migrations/FetchMigration/python" "FetchMigration"
-check_test_failure "FetchMigration"
+run_python_tests "$source_dir/opensearch-migrations/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link" "ConsoleLibrary"
+check_test_failure "ConsoleLibrary"
 
 # Test packages from /source directory
 declare -a packages=(

deployment/run-unit-tests.sh

@@ -28,7 +28,7 @@ const getProps = (): SolutionsInfrastructureStackProps => {
 
   // Uncomment for local testing
   // const codeBucket = 'unknown';
-  // const solutionVersion = "1.0.0";
+  // const solutionVersion = "2.0.0";
   // const solutionId = 'SO0290';
   // const solutionName = 'migration-assistant-for-amazon-opensearch';
   // const description = `(${solutionId}) - The AWS CloudFormation template for deployment of the ${solutionName}. Version ${solutionVersion}`;
@@ -49,4 +49,4 @@ new SolutionsInfrastructureStack(app, 'OSMigrations-Bootstrap', {
     generateBootstrapVersionRule: false
   }),
   ...infraProps
-});
+});

source/solutions-infrastructure/bin/app.ts

@@ -1,11 +1,53 @@
 #!/bin/bash
 
+usage() {
+  echo "Usage: $0 [--tag <tag_name>] [--branch <branch_name>]"
+  exit 1
+}
+
+tag=""
+branch=""
+
+# Parse options
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    --tag)
+      if [ -n "$branch" ]; then
+        echo "Error: You cannot specify both --tag and --branch."
+        usage
+      fi
+      tag="$2"
+      shift
+      ;;
+    --branch)
+      if [ -n "$tag" ]; then
+        echo "Error: You cannot specify both --tag and --branch."
+        usage
+      fi
+      branch="$2"
+      shift
+      ;;
+    *)
+      echo "Unknown parameter passed: $1"
+      usage
+      ;;
+  esac
+  shift
+done
+
 yum update && yum install -y git java-11-amazon-corretto-devel docker nodejs https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm
 systemctl start docker
 git init
-release_tag=$(curl -s https://api.github.com/repos/opensearch-project/opensearch-migrations/releases/latest | jq -r ".tag_name")
-git remote add -f origin https://github.com/opensearch-project/opensearch-migrations.git
-git checkout tags/$release_tag
+git remote | grep "origin" || git remote add -f origin https://github.com/opensearch-project/opensearch-migrations.git
+
+if [ -n "$branch" ]; then
+  git checkout $branch
+elif [ -n "$tag" ]; then
+  git checkout tags/$tag
+else
+  latest_release_tag=$(curl -s https://api.github.com/repos/opensearch-project/opensearch-migrations/releases/latest | jq -r ".tag_name")
+  git checkout tags/$latest_release_tag
+fi
 
 cd deployment/cdk/opensearch-service-migration || exit
 npm install -g aws-cdk

source/solutions-infrastructure/initBootstrap.sh

@@ -148,7 +148,7 @@ export class SolutionsInfrastructureStack extends Stack {
     })
     const solutionsUserAgent = `AwsSolution/${props.solutionId}/${props.solutionVersion}`
     const cfnInitConfig : InitElement[] = [
-      InitCommand.shellCommand(`echo "export MIGRATIONS_APP_REGISTRY_ARN=${appRegistryAppARN}; export CUSTOM_REPLAYER_USER_AGENT=${solutionsUserAgent}" > /etc/profile.d/solutionsEnv.sh`),
+      InitCommand.shellCommand(`echo "export MIGRATIONS_APP_REGISTRY_ARN=${appRegistryAppARN}; export MIGRATIONS_USER_AGENT=${solutionsUserAgent}" > /etc/profile.d/solutionsEnv.sh`),
       bootstrapFile
     ]
 
@@ -166,7 +166,7 @@ export class SolutionsInfrastructureStack extends Stack {
     new Instance(this, 'BootstrapEC2Instance', {
       vpc: vpc,
       instanceName: `bootstrap-${stageParameter.valueAsString}-instance`,
-      instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.LARGE),
+      instanceType: InstanceType.of(InstanceClass.T3, InstanceSize.LARGE),
       machineImage: MachineImage.latestAmazonLinux2023(),
       role: bootstrapRole,
       blockDevices: [

source/solutions-infrastructure/lib/solutions-infrastructure-stack.ts

@@ -30,4 +30,4 @@
     "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.105.0-alpha.0",
     "source-map-support": "^0.5.21"
   }
-}
+}