Enable PSI with Anolis (#161)

* Add support for Ubuntu 20.04 and gramine v1.2

* Update dockerfiles in common folder

* Update image

* Enable psi in anolisos (#135)

* enable psi in anolisos

* fixup! enable psi in anolisos

* fixup! enable psi in anolisos

Co-authored-by: Hsy-Intel <[email protected]>
Co-authored-by: Liang Ma <[email protected]>

Author: 3 people

cczoo/common/docker/gramine/build_docker_image.sh

@@ -16,10 +16,11 @@
 #!/bin/bash
 set -e
 
-if  [ -n "$1" ] ; then
+if  [ "$1" == "anolisos" ] ; then
     base_image=$1
 else
     base_image=ubuntu:20.04
+
 fi
 
 if  [ -n "$2" ] ; then
@@ -34,14 +35,25 @@ proxy_server="" # your http proxy server
 
 cd `dirname $0`
 
+if [ ${base_image} == "anolisos" ] ; then
+DOCKER_BUILDKIT=0 docker build \
+    --build-arg no_proxy=${no_proxy} \
+    --build-arg http_proxy=${proxy_server} \
+    --build-arg https_proxy=${proxy_server} \
+    --build-arg base_image=${base_image} \
+    --build-arg BASE_IMAGE=${base_image} \
+    -f gramine-sgx-dev:v1.2-anolisos.dockerfile \
+    -t gramine-sgx-dev:v1.2-anolisos \
+    .
+else
 DOCKER_BUILDKIT=0 docker build \
     --build-arg no_proxy=${no_proxy} \
     --build-arg http_proxy=${proxy_server} \
     --build-arg https_proxy=${proxy_server} \
     --build-arg base_image=${base_image} \
     --build-arg BASE_IMAGE=${base_image} \
     -f gramine-sgx-dev.dockerfile \
-    -t ${image_tag} \
+    -t gramine-sgx-dev:v1.2-ubuntu20.04-latest \
     .
-
+fi
 cd -

cczoo/common/docker/gramine/gramine-sgx-dev.dockerfile

@@ -77,11 +77,13 @@ ENV ISGX_DRIVER_PATH=${GRAMINEDIR}/driver
 ENV WERROR=1
 ENV SGX=1
 
+
 RUN apt-get update && apt-get install -y bison gawk nasm python3-click python3-jinja2 ninja-build pkg-config \
     libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf protobuf-c-compiler \
     libgmp-dev libmpfr-dev libmpc-dev libisl-dev
 
 RUN pip3 install --upgrade pip \
+
     && pip3 install 'meson>=0.56' 'toml>=0.10' cryptography
 
 #todo: use github gramine
@@ -137,6 +139,10 @@ RUN gramine-sgx-gen-private-key
 
 COPY configs /
 
+RUN gramine-sgx-gen-private-key
+
+COPY configs /
+
 # Use it to ignore packages authenticate in apt-get
 # ENV apt_arg="-o Acquire::AllowInsecureRepositories=true \
 #              -o Acquire::AllowDowngradeToInsecureRepositories=true"

cczoo/common/docker/gramine/gramine-sgx-dev:v1.2-anolisos.dockerfile

@@ -0,0 +1,101 @@
+
+# Copyright (c) 2022 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# https://github.com/oscarlab/graphene/blob/master/Tools/gsc/images/graphene_aks.latest.dockerfile
+FROM openanolis/anolisos:8.4-x86_64 AS Anolisos
+
+ENV INSTALL_PREFIX=/usr/local
+ENV LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib64:${LD_LIBRARY_PATH}
+ENV PATH=${INSTALL_PREFIX}/bin:${LD_LIBRARY_PATH}:${PATH}
+# Add steps here to set up dependencies
+RUN yum -y install \
+    openssl-devel \
+    libcurl-devel \
+    protobuf-devel \
+    yum-utils.noarch \
+    python3 \
+    wget
+
+# Intel SGX
+RUN mkdir /opt/intel && cd /opt/intel \
+    && wget https://mirrors.openanolis.cn/inclavare-containers/bin/anolis8.4/sgx-2.15.1/sgx_rpm_local_repo.tar.gz \
+    && sha256sum sgx_rpm_local_repo.tar.gz \
+    && tar xvf sgx_rpm_local_repo.tar.gz \
+    && yum-config-manager --add-repo file:///opt/intel/sgx_rpm_local_repo \
+    && yum -y --nogpgcheck install libsgx-urts libsgx-launch libsgx-epid libsgx-quote-ex libsgx-dcap-ql libsgx-uae-service libsgx-dcap-quote-verify-devel 
+    && yum -y groupinstall 'Development Tools'
+
+# COPY patches/libsgx_dcap_quoteverify.so  /usr/lib64/
+RUN yum -y install --nogpgcheck sgx-dcap-pccs libsgx-dcap-default-qpl
+
+# Gramine
+ENV GRAMINEDIR=/gramine
+ENV SGX_DCAP_VERSION=DCAP_1.11
+ENV GRAMINE_VERSION=v1.2
+ENV ISGX_DRIVER_PATH=${GRAMINEDIR}/driver
+ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig/
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV WERROR=1
+ENV SGX=1
+ENV GRAMINE_PKGLIBDIR=/usr/local/lib64/gramine
+ENV ARCH_LIBDIR=/lib64
+
+RUN yum -y install gawk bison python3-click python3-jinja2 golang ninja-build 
+RUN yum -y install openssl-devel protobuf-c-devel python3-protobuf protobuf-c-compiler
+RUN yum -y install gmp-devel mpfr-devel libmpc-devel isl-devel nasm python3-devel mailcap
+#install gramine
+RUN ln -s /usr/bin/python3 /usr/bin/python \
+    && python3 -m pip install --upgrade pip \
+    && python3 -m pip install toml meson wheel cryptography paramiko \
+    && git clone https://github.com/gramineproject/gramine.git ${GRAMINEDIR} \
+    && cd ${GRAMINEDIR} \
+    && git checkout ${GRAMINE_VERSION}
+
+RUN git clone https://github.com/intel/SGXDataCenterAttestationPrimitives.git ${ISGX_DRIVER_PATH} \
+    && cd ${ISGX_DRIVER_PATH} \
+    && git checkout ${SGX_DCAP_VERSION}
+
+ENV LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib64:${LD_LIBRARY_PATH}
+RUN cd ${GRAMINEDIR} \
+    && LD_LIBRARY_PATH="" meson setup build/ --buildtype=debug -Dprefix=${INSTALL_PREFIX} -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled -Dsgx_driver=dcap1.10 -Dsgx_driver_include_path=${ISGX_DRIVER_PATH}/driver/linux/include \
+    && LD_LIBRARY_PATH="" ninja -C build/ \
+    && LD_LIBRARY_PATH="" ninja -C build/ install
+RUN gramine-sgx-gen-private-key
+
+FROM Anolisos AS Psi_tensorflow
+# Install mbedtls
+RUN cd ${GRAMINEDIR}/build/subprojects/mbedtls-mbedtls* \
+    && cp -r *_gramine.a ${INSTALL_PREFIX}/lib \
+    && cd ${GRAMINEDIR}/subprojects/mbedtls-mbedtls*/mbedtls-mbedtls* \
+    && cp -r include/mbedtls ${INSTALL_PREFIX}/include
+
+# Install cJSON
+RUN cd ${GRAMINEDIR}/subprojects/cJSON*/ \
+    && make static \
+    && cp -r *.a ${INSTALL_PREFIX}/lib \
+    && mkdir -p ${INSTALL_PREFIX}/include/cjson \
+    && cp -r *.h ${INSTALL_PREFIX}/include/cjson
+
+RUN echo "enabled=0" > /etc/default/apport
+RUN echo "exit 0" > /usr/sbin/policy-rc.d
+
+# Clean tmp files
+RUN yum -y clean all && rm -rf /var/cache
+
+COPY configs /
+
+# Workspace
+ENV WORK_SPACE_PATH=${GRAMINEDIR}
+WORKDIR ${WORK_SPACE_PATH}

cczoo/psi/README.md

@@ -55,8 +55,8 @@ images for developing the gRPC RA-TLS application.
 
 ### Prerequisites
 
-- Ubuntu 18.04. This solution should work on other Linux distributions as well,
-  but for simplicity we provide the steps for Ubuntu 18.04 only.
+- Ubuntu 20.04. This solution should work on other Linux distributions as well,
+  but for simplicity we provide the steps for Ubuntu 20.04 only.
 
 - Docker Engine. Docker Engine is an open source containerization technology for
   building and containerizing your applications.
@@ -85,7 +85,7 @@ After Intel SGX DCAP is setup, verify the Intel Architectural Enclave Service Ma
    - Gramine
         ```bash
         cd cczoo/common/docker/gramine
-        ./build_docker_image.sh
+        ./build_docker_image.sh [ubuntu/anolisos]
         ```
 
 ### Setup develop environment of gRPC RA-TLS based on Intel SGX
@@ -94,7 +94,7 @@ After Intel SGX DCAP is setup, verify the Intel Architectural Enclave Service Ma
         ```bash
         cd -
         cd cczoo/psi/gramine
-        ./build_docker_image.sh
+        ./build_docker_image.sh [ubuntu/anolisos]
         ```
 
 ## Run PSI examples

cczoo/psi/gramine/CI-Examples/psi/cpp/anolisos.diff

@@ -0,0 +1,13 @@
+diff --git a./grpc.manifest.template b./grpc.manifest.template
+index c29ed9d..fc8a3fb 100644
+--- a./grpc.manifest.template
++++ b./grpc.manifest.template
+@@ -58,7 +58,7 @@ sgx.trusted_files = [
+   "file:{{ gramine.runtimedir() }}/",
+   "file:{{ arch_libdir }}/",
+   "file:/usr{{ arch_libdir }}/",
+-  "file:/etc/ssl/certs/ca-certificates.crt",
++  "file:/etc/ssl/certs/ca-bundle.crt",
+ ]
+ 
+ sgx.allowed_files = [

cczoo/psi/gramine/CI-Examples/psi/cpp/grpc.manifest.template

@@ -14,9 +14,9 @@
 # limitations under the License.
 
 libos.entrypoint = "{{ entrypoint }}"
-libos.check_invalid_pointers = false
+# libos.check_invalid_pointers = false
 
-loader.preload = "file:{{ gramine.libos }}"
+loader.entrypoint = "file:{{ gramine.libos }}"
 loader.log_level = "{{ log_level }}"
 # loader.log_file = ""
 
@@ -32,42 +32,29 @@ loader.env.GRPC_VERBOSITY = "ERROR"
 sys.enable_sigterm_injection = true
 sys.stack.size = "16M"
 
-fs.mount.lib.type = "chroot"
-fs.mount.lib.path = "/lib"
-fs.mount.lib.uri = "file:{{ gramine.runtimedir() }}"
-
-fs.mount.lib2.type = "chroot"
-fs.mount.lib2.path = "{{ arch_libdir }}"
-fs.mount.lib2.uri = "file:{{ arch_libdir }}"
-
-fs.mount.lib3.type = "chroot"
-fs.mount.lib3.path = "/usr{{ arch_libdir }}"
-fs.mount.lib3.uri = "file:/usr{{ arch_libdir }}"
-
-fs.mount.etc.type = "chroot"
-fs.mount.etc.path = "/etc"
-fs.mount.etc.uri = "file:/etc"
-
-# Note that Gramine also creates an internal
-# thread for handling inter-process communication (IPC), and potentially another
-# thread for asynchronous events. Therefore, the actual number of threads that
-# the application can create is (sgx.thread_num - 2).
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { path = "/usr", uri = "file:/usr" },
+  { type = "tmpfs", path = "/var/tmp" },
+]
 
 sgx.debug = false
 sgx.remote_attestation = true
+sgx.enable_stats = false
 sgx.nonpie_binary = true
-
 sgx.enclave_size = "2G"
 sgx.thread_num = 16
 
-sgx.require_avx = true
+# sgx.require_avx = true
 # sgx.require_avx512 = true
 # sgx.require_amx = true
 
-sgx.enable_stats = false
 
 sgx.trusted_files = [
   "file:{{ entrypoint }}",
+  "file:{{ gramine.libos }}",
   "file:{{ gramine.runtimedir() }}/",
   "file:{{ arch_libdir }}/",
   "file:/usr{{ arch_libdir }}/",

cczoo/psi/gramine/CI-Examples/psi/python/Makefile

@@ -15,15 +15,13 @@
 
 ################################# CONSTANTS ###################################
 
-GRAMINEDIR ?= ../../../..
-SGX_SIGNER_KEY ?= $(GRAMINEDIR)/Pal/src/host/Linux-SGX/signer/enclave-key.pem
-
-include $(GRAMINEDIR)/Scripts/Makefile.configs
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+# ENTRYPOINT ?= $(realpath $(shell sh -c "command -v python3"))
 
 ifeq ($(DEBUG),1)
-GRAPHENE_LOG_LEVEL = debug
+GRAMINE_LOG_LEVEL = debug
 else
-GRAPHENE_LOG_LEVEL = error
+GRAMINE_LOG_LEVEL = error
 endif
 
 .PHONY: all
@@ -32,22 +30,23 @@ ifeq ($(SGX),1)
 all: python.manifest.sgx python.sig python.token
 endif
 
-################################ gRPC MANIFEST ###############################
+################################ MANIFEST ###############################
 
 python.manifest: python.manifest.template
 	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
 		-Dentrypoint=$(realpath $(shell sh -c "command -v python3")) \
 		-Darch_libdir=$(ARCH_LIBDIR) \
-		-Dlog_level=$(GRAPHENE_LOG_LEVEL) \
 		$< >$@
 
-python.manifest.sgx: python.manifest
+python.manifest.sgx python.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: python.manifest
 	gramine-sgx-sign \
-		--key $(SGX_SIGNER_KEY) \
 		--manifest $< \
-		--output $@
-
-python.sig: python.manifest.sgx
+		--output $<.sgx
 
 python.token: python.sig
 	gramine-sgx-get-token --output $@ --sig $<
@@ -60,4 +59,4 @@ clean:
 	$(RM) -r scripts/__pycache__
 
 .PHONY: distclean
-distclean: clean
+distclean: clean

cczoo/psi/gramine/CI-Examples/psi/python/anolisos.diff

@@ -0,0 +1,33 @@
+diff --git a./python.manifest.template b./python.manifest.template
+index 057cffe..480dd78 100644
+--- a./python.manifest.template
++++ b./python.manifest.template
+@@ -37,9 +37,10 @@ fs.mounts = [
+   { path = "/usr", uri = "file:/usr" },
+   { type = "tmpfs", path = "/var/tmp" },
+   { path = "{{ python.stdlib }}", uri = "file:{{ python.stdlib }}" },
+-  { path = "{{ python.distlib }}", uri = "file:{{ python.distlib }}" },
++  # { path = "{{ python.distlib }}", uri = "file:{{ python.distlib }}" },
+   # { path = "/usr/local/lib/python3.6/dist-packages/", uri = "file:/usr/local/lib/python3.6/dist-packages/" }, # for ubuntu:18.04
+-  { path = "/usr/local/lib/python3.8/dist-packages/", uri = "file:/usr/local/lib/python3.8/dist-packages/" }, # for ubuntu:20.04
++  # { path = "/usr/local/lib/python3.8/dist-packages/", uri = "file:/usr/local/lib/python3.8/dist-packages/" }, # for ubuntu:20.04
++  { path = "/usr/local/lib/python3.6/site-packages/", uri = "file:/usr/local/lib/python3.6/site-packages/" }, # for anolisos
+ ]
+ 
+ sgx.debug = false
+@@ -60,10 +61,12 @@ sgx.trusted_files = [
+   "file:{{ arch_libdir }}/",
+   "file:/usr/{{ arch_libdir }}/",
+   "file:{{ python.stdlib }}/",
+-  "file:{{ python.distlib }}/",
++  # "file:{{ python.distlib }}/",
+   # "file:/usr/local/lib/python3.6/dist-packages/",  # for ubuntu:18.04
+-  "file:/usr/local/lib/python3.8/dist-packages/",  # for ubuntu:20.04
+-  "file:/etc/ssl/certs/ca-certificates.crt",
++  # "file:/usr/local/lib/python3.8/dist-packages/",  # for ubuntu:20.04
++  "file:/usr/local/lib/python3.6/site-packages/", 
++  "file:/usr/local/lib64/python3.6/site-packages/", 
++  "file:/etc/ssl/certs/ca-bundle.crt",
+   "file:server.py",
+   "file:data_provider1.py",
+   "file:data_provider2.py",