Add OpenSearch backend

.ci/opensearch/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker:stable
+
+RUN apk add --update bash
+
+COPY run-opensearch.sh /run-opensearch.sh
+
+ENTRYPOINT ["/run-opensearch.sh"]

.ci/opensearch/action.yml

@@ -0,0 +1,33 @@
+name: 'Run OpenSearch'
+description: 'This action spins up an Opensearch instance that can be accessed and used in your subsequent steps.'
+
+inputs:
+  opensearch-version:
+    description: 'The version of the OpenSearch you want to run'
+    required: true
+  security-enabled:
+    description: 'Enable or disable HTTPS, enabled by default'
+    default: 'false'
+    required: false
+  nodes:
+    description: 'Number of nodes in the cluster'
+    required: false
+    default: 1
+  port:
+    description: 'Port where you want to run OpenSearch'
+    required: false
+    default: 9200
+  opensearch-initial-admin-password:
+    description: 'The password for the user admin in your cluster'
+    required: false
+    default: 'myStrongPassword_123'
+
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  env:
+    OPENSEARCH_VERSION: ${{ inputs.opensearch-version }}
+    NODES: ${{ inputs.nodes }}
+    PORT: ${{ inputs.port }}
+    SECURITY_ENABLED: ${{ inputs.security-enabled }}
+    OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${{ inputs.opensearch-initial-admin-password }}

.ci/opensearch/functions/imports.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+# From https://github.com/opensearch-project/opensearch-py/blob/main/.ci/functions/imports.sh
+
+source ./.ci/opensearch/functions/wait-for-container.sh

.ci/opensearch/functions/wait-for-container.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+# From https://github.com/opensearch-project/opensearch-py/blob/main/.ci/functions/wait-for-container.sh
+
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 4;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1}\033[0m"
+    return 0
+  fi
+}

.ci/opensearch/run-opensearch.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+source ./.ci/opensearch/functions/imports.sh
+set -euxo pipefail
+
+if [[ -z $OPENSEARCH_VERSION ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [OPENSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+
+OPENSEARCH_REQUIRED_VERSION="latest"
+# Starting in 2.12.0, security demo configuration script requires an initial admin password
+if [ "$OPENSEARCH_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin"
+fi
+
+for (( node=1; node<=${NODES-1}; node++ ))
+do
+  port=$((PORT + $node - 1))
+
+  if [[ "$SECURITY_ENABLED" == "true" ]]; then
+    healthcmd="curl -vvv -s --insecure -u admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD --fail https://localhost:$port/_cluster/health || exit 1"
+    security=($(cat <<-END
+
+END
+  ))
+  elif [[ "$SECURITY_ENABLED" == "false" ]]; then
+    healthcmd="curl -vvv -s --fail http://localhost:$port/_cluster/health || exit 1"
+    security=($(cat <<-END
+      --env plugins.security.disabled=true
+END
+  ))
+  fi
+
+  docker run \
+    --rm \
+    --detach \
+    --name="os${node}" \
+    --env "cluster.name=docker-opensearch" \
+    --env "http.port=${port}" \
+    --env discovery.type=single-node \
+    --env bootstrap.memory_lock=true \
+    --env "OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g" \
+    --env OPENSEARCH_INITIAL_ADMIN_PASSWORD=$OPENSEARCH_INITIAL_ADMIN_PASSWORD \
+    "${security[@]}" \
+    --publish "${port}:${port}" \
+    --ulimit nofile=65536:65536 \
+    --ulimit memlock=-1:-1 \
+    --health-cmd="$(echo $healthcmd)" \
+    --health-interval=2s \
+    --health-retries=20 \
+    --health-timeout=2s \
+    opensearchproject/opensearch:${OPENSEARCH_VERSION}
+
+  if wait_for_container "os$node"; then
+    echo -e "\033[32;1mSUCCESS:\033[0m OpenSearch up and running\033[0m"
+  fi
+done

.ci/opensearch/test.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euxo pipefail
+
+echo $script_path/functions/imports.sh

.github/workflows/ci.yml

@@ -7,12 +7,29 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        python-version: ["3.10", "3.11", "3.12"]
+        opensearch: ["latest"]
+        security-enabled: ["true", "false"]
 
     steps:
       - uses: actions/checkout@v4
 
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+
+      - name: Start OpenSearch
+        uses: ./.ci/opensearch
+        with:
+          port: 9250
+          opensearch-version: ${{ matrix.opensearch }}
+          security-enabled: ${{ matrix.security-enabled }}
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -29,6 +46,10 @@ jobs:
       - name: Run unit tests
         run: |
           poetry run pytest --cov=abcd --cov-report xml --cov-report term:skip-covered
+        env:
+          port: 9250
+          security_enabled: ${{ matrix.security-enabled }}
+          opensearch-version: ${{ matrix.opensearch }}
 
       - name: Upload coverage reports to Codecov
         uses: codecov/[email protected]

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.9.6
+  rev: v0.11.2
   hooks:
     # Run the linter.
     - id: ruff

README.md

@@ -12,75 +12,145 @@ Main features:
 - Configurations that consist of atom positions, elements, forces, and various metadata are stored as a dictionary by a MongoDB backend.
 - There is no predefined schema, any combination of keys are allowed for all configurations.
 - Two modes: "discovery" and "download". Both use filter-type queries, but in "discovery" mode, summary statistics of the configurations that pass the filter are reported. In "download" mode, the matching configurations are downloaded and exported to a file.
-- The "discovery" mode can be used to learn what keys exist in the set of configurations that have passed the current quiery filter. The user can use this to refine the query.
+- The "discovery" mode can be used to learn what keys exist in the set of configurations that have passed the current query filter. The user can use this to refine the query.
 - Complex queries on dictionary key-value pairs are allowed, and their logical combinations.
 
 ## Installation
 
+### General Setup
+
 creating tables and views
-```
+
+```sh
 $ pip install git+https://github.com/libAtoms/abcd.git
 ```
 
-## Setup
+Example Docker installation on Ubuntu:
+
+```sh
+sudo apt-get update
+sudo apt upgrade
+sudo apt install docker.io
+sudo groupadd docker
+sudo usermod -aG docker $USER
+newgrp docker # or exit and log in
+```
+
+Docker can be tested by running:
+
+```sh
+docker run hello-world
+```
+
+Example Python setup on Ubuntu (pip must be updated for poetry to be used successfully):
+
+```sh
+sudo apt install software-properties-common
+sudo add-apt-repository ppa:deadsnakes/ppa
+sudo apt install python3.10
+sudo apt-get install python3.10-distutils
+sudo apt install python3-virtualenv
+virtualenv -p /usr/bin/python3.10 venv_10
+source venv_10/bin/activate
+curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+```
 
-If you have an already running mongo server, or install your own, they you are ready to go. Alternatively,
+Building and installing ABCD dependencies via poetry:
 
+```sh
+git clone https://github.com/libAtoms/abcd.git
+curl -sSL https://install.python-poetry.org | python3 -
+export PATH="/home/ubuntu/.local/bin:$PATH"
+cd abcd
+poetry install
+poetry build
 ```
+
+### MongoDB
+
+If you have an already running MongoDB server, or install your own, then you are ready to go. Alternatively,
+
+```sh
 docker run -d --rm --name abcd-mongodb -v <path-on-your-machine-to-store-database>:/data/db -p 27017:27017 mongo
 ```
 
 will download and install a docker and run a database in it.
 
 To connect to a mongodb that is already running, use
-```
+
+```sh
 abcd login mongodb://localhost
 ```
 
 If you are running `abcd` inside a docker, and want to connect to a mongodb outside that docker use something like this (example is for Mac OS):
 
-```
+```sh
 abcd login mongodb://docker.for.mac.localhost
 ```
 
 The above login command will place create an `~/.abcd` file with the following contents:
 
-```
+```sh
 {"url": "mongodb://localhost"}
 ```
 
-# Remote access
+### OpenSearch
+If you have an already running OpenSearch server, or installed your own, then you are ready to go. Alternatively,
 
-You can set up an `abcd` user on your machine where the database is running, and then access it remotely for discovering data. Make sure you have the `~/.abcd` file created for this user, then put this in the `.ssh/authorized_keys` file (substituting your public key for the last part):
+```sh
+sudo swapoff -a # optional
+sudo sysctl -w vm.swappiness=1 # optional
+sudo sysctl -w fs.file-max=262144 # optional
+sudo sysctl -w vm.max_map_count=262144
+docker run -d --name abcd-opensearch -v <path-on-your-machine-to-store-database>:/data/db -p 9200:9200 -e "discovery.type=single-node" -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=<PASSWORD>" -it opensearchproject/opensearch:latest
+```
+
+will download and install an OpenSearch image and run it. The connection can be tested with:
+
+```sh
+curl -vvv -s --insecure -u admin:<PASSWORD> --fail https://localhost:9200
+```
+
+To connect to an OpenSearch database that is already running, use
+
+```sh
+abcd login opensearch://<USER>:<PASSWORD>@localhost
 ```
+
+## Remote access
+
+You can set up an `abcd` user on your machine where the database is running, and then access it remotely for discovering data. Make sure you have the `~/.abcd` file created for this user, then put this in the `.ssh/authorized_keys` file (substituting your public key for the last part):
+
+```sh
 command="/path/to/abcd --remote  ${SSH_ORIGINAL_COMMAND}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty ssh-rsa <public-key> your@email
 ```
 
 Then you'll be able to access the database remotely using, e.g.
-```
+
+```sh
 ssh [email protected] summary
 ```
 
-# GUI through a browser + visualisation
+## GUI through a browser + visualisation
 
 The database has a simple GUI, coupled with a visualiser. Data for now needs to be uploaded on the command line, but query can be done through the browsers. Instructions below (they include running `abcd` from a docker too, but of course you can run it outside the docker as well. )
 
 
-#### Usage in docker
+## Usage in docker
 Currently a manual uploaded image is available, that was built on 7/2/2020 by Tamas K. Stenczel.
 To access it:
 1. pull the image
-    ```
+    ```sh
     docker pull stenczelt/projection-abcd:latest
     ```
 
 2. create a docker network, which enables the containers to communicate with each other and the outside world as well
-    ```
+    ```sh
     docker network create --driver bridge abcd-network
     ```
 
 3. run the mongo (ABCD) and the visualiser as well
-    ```
+    ```sh
     docker run -d --rm --name abcd-mongodb-net -v <path-on-your-machine-to-store-database>:/data/db -p 27017:27017 --network abcd-network mongo
 
     docker run -it --rm --name visualiser-dev -p 9999:9999 --network abcd-network stenczelt/projection-abcd
@@ -91,8 +161,17 @@ To access it:
 This will start the visualiser with ABCD integration! Have fun!
 
 After usage, for cleanup:
-```
+
+```sh
 docker stop visualiser-dev abcd-mongodb-net         # stop the containers
 docker rm visualiser-dev abcd-mongodb-net           # remove them if --rm did not
 docker network rm abcd-network                      # remove the docker network
 ```
+
+## Testing
+
+Unit tests are automatically run on push and creation of pull requests. Unit testing using mock databases can also be run in the command line using:
+
+```sh
+python -m unittest tests
+```