DRAFT compiler support U24.04

on-going draft with compiler support U24.04

Signed-off-by: SaeHie Park <[email protected]>

Author: seanshpark

.github/workflows/run-onecc-build.yml

@@ -45,12 +45,14 @@ jobs:
     strategy:
       matrix:
         type: [ Debug, Release ]
-        ubuntu_code: [ focal, jammy ]
+        ubuntu_code: [ focal, jammy ] #, noble ]
         include:
           - ubuntu_code: focal
             ubuntu_ver: 20.04
           - ubuntu_code: jammy
             ubuntu_ver: 22.04
+          #- ubuntu_code: noble
+          #  ubuntu_ver: 24.04
     runs-on: one-x64-linux
     container:
       image: samsungonedev.azurecr.io/nnfw/one-devtools:${{ matrix.ubuntu_code }}
@@ -67,9 +69,15 @@ jobs:
       # dalgona uses pybind11, but pybind11 cannot bind packages in virtualenv.
       # So we need to install packages for dalgona-test globally.
       - name: Install required packages
+        if: ${{ matrix.ubuntu_code == 'focal' || matrix.ubuntu_code == 'jammy' }}
         run: |
           python3 -m pip install numpy h5py==3.11.0 flatbuffers==23.5.26
 
+      - name: Install required packages
+        if: ${{ matrix.ubuntu_code == 'noble' }}
+        run: |
+          apt-get install -y python3-numpy python3-h5py python3-flatbuffers
+
       - name: Caching externals
         uses: actions/cache@v4
         with:

.github/workflows/run-onert-android-build.yml

@@ -21,13 +21,6 @@ on:
       - master
       - release/*
     paths:
-      - '.github/workflows/run-onert-android-build.yml'
-      - 'nn*'
-      - 'Makefile.template'
-      - 'infra/buildtool/**'
-      - 'infra/cmake/**'
-      - 'infra/nncc/**'
-      - 'infra/nnfw/**'
       - 'runtime/**'
       - '!runtime/contrib/**'
       - '!**/*.md'

.github/workflows/run-onert-cross-build.yml

@@ -21,13 +21,6 @@ on:
       - master
       - release/*
     paths:
-      - '.github/workflows/run-onert-cross-build.yml'
-      - 'nn*'
-      - 'Makefile.template'
-      - 'infra/buildtool/**'
-      - 'infra/cmake/**'
-      - 'infra/nncc/**'
-      - 'infra/nnfw/**'
       - 'runtime/**'
       - '!runtime/contrib/**'
       - '!**/*.md'

.github/workflows/run-onert-gbs-build.yml

@@ -24,15 +24,6 @@ on:
       - master
       - release/*
     paths:
-      - '.github/workflows/run-onert-gbs-build.yml'
-      - 'nncc'
-      - 'nnfw'
-      - 'compiler/luci/**'
-      - 'compiler/loco/**'
-      - 'compiler/logo/**'
-      - 'compute/**'
-      - 'infra/nnfw/**'
-      - 'infra/cmake/**'
       - 'runtime/**'
       - '!runtime/contrib/**'
       - 'packaging/**'

.github/workflows/run-onert-native-build.yml

@@ -21,13 +21,6 @@ on:
       - master
       - release/*
     paths:
-      - '.github/workflows/run-onert-native-build.yml'
-      - 'nn*'
-      - 'Makefile.template'
-      - 'infra/buildtool/**'
-      - 'infra/cmake/**'
-      - 'infra/nncc/**'
-      - 'infra/nnfw/**'
       - 'runtime/**'
       - '!runtime/contrib/**'
       - '!**/*.md'

compiler/CMakeLists.txt

@@ -3,7 +3,7 @@
 set(ONE_UBUNTU_CODENAME "focal")
 find_program(LSB_RELEASE_EXEC lsb_release)
 if(LSB_RELEASE_EXEC)
-  # output should be one of 'bionic', 'focal', 'jammy'
+  # output should be one of 'bionic', 'focal', 'jammy', 'noble'
   # others are not tested
   execute_process(COMMAND "${LSB_RELEASE_EXEC}" --short --codename
                   OUTPUT_VARIABLE ONE_UBUNTU_CODENAME
@@ -14,6 +14,10 @@ endif()
 
 if(${ONE_UBUNTU_CODENAME} STREQUAL "bionic")
   set(ONE_UBUNTU_CODENAME_BIONIC TRUE)
+elseif(${ONE_UBUNTU_CODENAME} STREQUAL "focal")
+  set(ONE_UBUNTU_CODENAME_FOCAL TRUE)
+elseif(${ONE_UBUNTU_CODENAME} STREQUAL "jammy")
+  set(ONE_UBUNTU_CODENAME_JAMMY TRUE)
 elseif(${ONE_UBUNTU_CODENAME} STREQUAL "noble")
   set(ONE_UBUNTU_CODENAME_NOBLE TRUE)
 endif()

compiler/one-cmds/CMakeLists.txt

@@ -2,7 +2,7 @@
 #   Ubuntu18.04; explictly installed python3.8 (default is python3.6)
 #   Ubuntu20.04; default python3.8
 #   Ubuntu22.04; default python3.10
-#   Ubuntu24.04; explicitly installed python3.8 (default is python3.12)
+#   Ubuntu24.04; default python3.12
 #   refer https://github.com/Samsung/ONE/issues/9962
 if(CMAKE_VERSION VERSION_LESS 3.12)
   find_package(PythonInterp 3.8 QUIET)
@@ -24,10 +24,10 @@ else()
   endif()
 
   # tensorflow 2.12.1 supports Python 3.8 ~ 3.11
-  if(Python_VERSION VERSION_GREATER_EQUAL 3.12)
-    message(STATUS "Build one-cmds: FALSE (Python version 3.12 or higher is not supported yet)")
-    return()
-  endif()
+  #if(Python_VERSION VERSION_GREATER_EQUAL 3.12)
+  #  message(STATUS "Build one-cmds: FALSE (Python version 3.12 or higher is not supported yet)")
+  #  return()
+  #endif()
   if(Python_VERSION VERSION_LESS 3.8)
     message(STATUS "Build one-cmds: FAILED (Install Python version 3.8 or 3.10)")
     return()
@@ -67,17 +67,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
   #      and install build/../one-prepare-venv file
   list(APPEND ONE_COMMAND_FILES one-prepare-venv.aarch64)
 else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
-  if(ONE_UBUNTU_CODENAME_BIONIC)
-    # NOTE copy one-prepare-venv.u1804 as build/../one-prepare-venv
-    #      and install build/../one-prepare-venv file
-    list(APPEND ONE_COMMAND_FILES one-prepare-venv.u1804)
-  elseif(ONE_UBUNTU_CODENAME_NOBLE)
-    # NOTE copy one-prepare-venv.u2404 as build/../one-prepare-venv
-    #      and install build/../one-prepare-venv file
-    list(APPEND ONE_COMMAND_FILES one-prepare-venv.u2404)
-  else()
-    list(APPEND ONE_COMMAND_FILES one-prepare-venv)
-  endif()
+  list(APPEND ONE_COMMAND_FILES one-prepare-venv)
 endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
 
 # pytorch importer is an experimental feature, it is not used in default configuration

compiler/one-cmds/one-import-onnx

@@ -24,7 +24,13 @@ import os
 import sys
 import tempfile
 import onnx
-import onnx_tf
+
+# onnx_tf may not exist for latest python
+try:
+    import onnx_tf
+    _onnx_tf_enabled = True
+except ImportError:
+    _onnx_tf_enabled = False
 
 # ONNX legalizer is an optional feature
 # It enables conversion of some operations, but in experimental phase for now
@@ -124,6 +130,10 @@ class TidyIONames:
                 self.onnx_model.graph.output[i].name = self.remap_outputs[idx]
 
 
+def _check_onnx_tf():
+    import onnx
+
+
 def get_driver_cfg_section():
     return "one-import-onnx"
 
@@ -330,6 +340,9 @@ def _convert(args):
             if _force_ext(args):
                 run_default_import = False
 
+        if not _onnx_tf_enabled:
+            run_default_import = False
+
         res_conv = -1
         if run_default_import:
             if _force_ext(args):

compiler/one-cmds/one-prepare-venv

@@ -54,6 +54,7 @@ source /etc/lsb-release
 # - https://github.com/onnx/onnx/blob/master/docs/Versioning.md
 # - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md
 
+# default version values
 VER_TENSORFLOW=2.12.1
 VER_TF_PROB=0.20.1
 VER_TF_ADON=0.20.0
@@ -63,18 +64,31 @@ VER_ONNX_TF=1.10.0
 VER_PYDOT=1.4.2
 VER_TORCH="2.1.2+cpu"
 VER_PROTOBUF=4.23.3
-
-# Install tensorflow
+VER_NUMPY=1.24.3
 
 echo "Setting version for '$DISTRIB_CODENAME'"
 if [[ "$DISTRIB_CODENAME" == "bionic" ]]; then
-  : # use as is
+  : # use default
 elif [[ "$DISTRIB_CODENAME" == "focal" ]]; then
-  : # use as is
+  : # use default
 elif [[ "$DISTRIB_CODENAME" == "jammy" ]]; then
-  : # TODO change vesions
+  :
+  #VER_TENSORFLOW=2.19.0
+  #VER_TF_PROB=0.25.0
+  #VER_TF_ADON=0.23.0
+  #VER_ONNX=1.17.0
+  #VER_ONNXRUNTIME=1.21.1
+  #VER_TORCH="2.7.0+cpu"
+  #VER_NUMPY=1.24.4
 elif [[ "$DISTRIB_CODENAME" == "noble" ]]; then
-  : # TODO change vesions
+  VER_TENSORFLOW=2.19.0
+  VER_TF_PROB=0
+  VER_TF_ADON=0 # EOL, don't exist in python3.12~
+  VER_ONNX=1.17.0
+  VER_ONNXRUNTIME=1.21.0
+  VER_ONNX_TF=0
+  VER_TORCH="2.7.0+cpu"
+  VER_NUMPY=1.26.4
 else
   echo "Error one-prepare-venv: Unsupported codename $DISTRIB_CODENAME"
   exit -1
@@ -97,15 +111,18 @@ if [[ ! -z "$ONE_PREPVENV_PIP_OPTION" ]]; then
 fi
 
 ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools
-if [ -n "${EXT_TENSORFLOW_WHL}" ]; then
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_TENSORFLOW_WHL}
-else
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow-cpu==${VER_TENSORFLOW}
+
+PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} "
+
+PYTHON_PACKAGES+="Pillow "
+
+if [[ "$VER_TF_PROB" != "0" ]]; then
+  # Fix version to that of TF release date
+  PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} "
+fi
+if [[ "$VER_TF_ADON" != "0" ]]; then
+  PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} "
 fi
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install Pillow
-# Fix version to that of TF release date
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==${VER_TF_PROB}
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==${VER_TF_ADON}
 
 # Install PyTorch and ONNX related
 # NOTE set ONE_PREPVENV_TORCH_SOURCE to override options for source URL.
@@ -115,24 +132,26 @@ if [[ ! -z "$ONE_PREPVENV_TORCH_SOURCE" ]]; then
 fi
 # TODO remove torch message
 echo "Torch from '${ONE_PREPVENV_TORCH_SOURCE}' -> '${TORCH_SOURCE_OPTION}'"
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install torch==${VER_TORCH} ${TORCH_SOURCE_OPTION}
+PYTHON_PACKAGES+="torch==${VER_TORCH} "
 
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx==${VER_ONNX}
+PYTHON_PACKAGES+="onnx==${VER_ONNX} "
 
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnxruntime==${VER_ONNXRUNTIME}
+PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} "
 
-# Provide install of custom onnx-tf
-if [ -n "${EXT_ONNX_TF_WHL}" ]; then
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL}
-else
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF}
+if [[ "$VER_ONNX_TF" != "0" ]]; then
+  PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} "
 fi
 
 # Fix version to that of TF release date
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==${VER_PROTOBUF}
+PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} "
 
-# Fix version with fsspec error on U20.04
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade fsspec==2024.6.1
+# Fix version with fsspec error
+PYTHON_PACKAGES+="fsspec==2024.6.1 "
 
 # Install pydot for visq
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install pydot==${VER_PYDOT}
+PYTHON_PACKAGES+="pydot==${VER_PYDOT} "
+
+# Install numpy 1.x
+PYTHON_PACKAGES+="numpy==${VER_NUMPY} "
+
+${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade ${PYTHON_PACKAGES} ${TORCH_SOURCE_OPTION}

compiler/one-cmds/one-prepare-venv.u2404

@@ -26,19 +26,19 @@ VENV_PYTHON=${DRIVER_PATH}/venv/bin/python
 
 if [ ! -f ${VENV_ACTIVATE} ]; then
   # Create python virtual enviornment
-  python3.8 -m venv "${DRIVER_PATH}/venv"
+  python3 -m venv "${DRIVER_PATH}/venv"
 fi
 
 # NOTE version
 # - https://github.com/onnx/onnx/blob/master/docs/Versioning.md
 # - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md
 
-VER_TENSORFLOW=2.12.1
-VER_ONNX=1.16.0
-VER_ONNXRUNTIME=1.18.0
-VER_ONNX_TF=1.10.0
+VER_TENSORFLOW=2.19.0 # 2025-03-12
+VER_ONNX=1.17.0
+VER_ONNXRUNTIME=1.20.1
+#VER_ONNX_TF=1.10.0
 VER_PYDOT=1.4.2
-VER_TORCH="2.1.2+cpu"
+VER_TORCH="2.6.0+cpu"
 
 # Install tensorflow
 
@@ -66,8 +66,8 @@ else
 fi
 ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install Pillow
 # Fix version to that of TF release date
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==0.20.1
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==0.20.0
+#${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_probability==0.25.0
+#${VENV_PYTHON} -m pip ${PIP_OPTIONS} install tensorflow_addons==0.23.0
 
 # Install PyTorch and ONNX related
 # NOTE set ONE_PREPVENV_TORCH_SOURCE to override options for source URL.
@@ -84,14 +84,14 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx==${VER_ONNX}
 ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnxruntime==${VER_ONNXRUNTIME}
 
 # Provide install of custom onnx-tf
-if [ -n "${EXT_ONNX_TF_WHL}" ]; then
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL}
-else
-  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF}
-fi
+#if [ -n "${EXT_ONNX_TF_WHL}" ]; then
+#  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install ${EXT_ONNX_TF_WHL}
+#else
+#  ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install onnx-tf==${VER_ONNX_TF}
+#fi
 
 # Fix version to that of TF release date
-${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==4.23.3
+${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade protobuf==5.29.4
 
 # Install pydot for visq
 ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install pydot==${VER_PYDOT}