[PyOV] Enable Python 3.10 on Azure CI

.ci/azure/android_arm64.yml

@@ -63,6 +63,9 @@ jobs:
       curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
       whoami
       uname -a
+      echo ls /usr/bin/python3.10
+      rm -rf /usr/bin/python3
+      sudo ln -s /usr/bin/python3.10 /usr/bin/python3
       echo Python3 info ; which python3 ; python3 --version
       echo Python info ; which python ; python --version
       echo Java info ; which java ; java -version

.ci/azure/linux.yml

@@ -62,7 +62,7 @@ jobs:
         CMAKE_CPACK_GENERATOR:
         SAMPLES_INSTALL_DIR: $(INSTALL_DIR)/samples
         PYTHON_SAMPLES_INSTALL_DIR: $(SAMPLES_INSTALL_DIR)/python
-        RUN_PREFIX: . $(SETUPVARS) -pyver 3.8 &&
+        RUN_PREFIX: . $(SETUPVARS) -pyver 3.10 &&
     maxParallel: 2
 
   # About 150% of total time
@@ -91,14 +91,17 @@ jobs:
     TMP_DIR: /mnt/tmp
     SHARE_DIR: /mount/cinfsshare/onnxtestdata
     CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux
+    BUILD_PYTHON: $(WORK_DIR)/build_python
+    INSTALL_PYTHON: $(INSTALL_OPENVINO)/extras/python
+    PY_VENV: $(WORK_DIR)/.venv
+    PYTHON_VERSION: 3.10.7
 
   steps:
   - script: |
       curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
       whoami
       uname -a
       echo Python3 info ; which python3 ; python3 --version
-      echo Python info ; which python ; python --version
       echo Java info ; which java ; java -version
       echo gcc info ; which gcc ; gcc --version
       echo cmake info ; which cmake ; cmake --version
@@ -144,8 +147,21 @@ jobs:
       $(REPO_DIR)/install_build_dependencies.sh
       # Move jdk into contrib
       # 'clang' compiler is to check that samples can be built using it
-      sudo apt --assume-yes install openjdk-11-jdk clang
+      sudo apt --assume-yes install openjdk-11-jdk libbz2-dev clang
       # For opencv-python: python3-setuptools and pip upgrade
+      wget https://www.python.org/ftp/python/"$PYTHON_VERSION"/Python-"$PYTHON_VERSION".tgz
+      tar -xzf Python-"$PYTHON_VERSION".tgz
+      cd Python-"$PYTHON_VERSION" || exit
+      ./configure --enable-optimizations --prefix=/usr/local --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
+      make python
+      sudo make install
+      export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+      /usr/local/bin/python3.10 -m venv $(PY_VENV)
+      . $(PY_VENV)/bin/activate
+      echo Python3 info ; which python3 ; python3 --version
+      python3 --version
+      python3.10 --version
+      curl https://bootstrap.pypa.io/get-pip.py | python3
       python3 -m pip install --upgrade pip
       python3 -m pip install -r $(REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements.txt
       python3 -m pip install -r $(REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt
@@ -185,7 +201,7 @@ jobs:
         -DENABLE_PYTHON=ON
         -DBUILD_SHARED_LIBS=$(CMAKE_BUILD_SHARED_LIBS)
         -DENABLE_ONEDNN_FOR_GPU=$(CMAKE_BUILD_SHARED_LIBS)
-        -DPYTHON_EXECUTABLE=/usr/bin/python3.8
+        -DPYTHON_EXECUTABLE=$(PY_VENV)/bin/python3.10
         -DENABLE_TESTS=ON
         -DENABLE_OV_ONNX_FRONTEND=ON
         -DENABLE_FASTER_BUILD=ON
@@ -225,8 +241,8 @@ jobs:
     inputs:
       cmakeArgs: >
         -GNinja
-        -S $(REPO_DIR)/tests/layer_tests
-        -B $(BUILD_LAYER_TESTS_DIR)
+        $(REPO_DIR)/tests/layer_tests
+      workingDirectory: $(BUILD_LAYER_TESTS_DIR)
 
   - script: ninja -C $(BUILD_LAYER_TESTS_DIR)
     displayName: 'Build Layer Tests'
@@ -235,26 +251,24 @@ jobs:
     displayName: 'Remove debian dependencies'
     condition: eq(variables['CMAKE_CPACK_GENERATOR'], 'DEB')
     continueOnError: false
-
   - script: cmake -DCOMPONENT=python_wheels -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P $(BUILD_DIR)/cmake_install.cmake
     displayName: 'Install wheel packages'
-
   - script: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P $(BUILD_LAYER_TESTS_DIR)/cmake_install.cmake
     displayName: 'Install Layer Tests'
-
-  - script: python3 -m pip install openvino-dev --find-links=$(INSTALL_DIR)/tools
+  - script: |
+      . $(PY_VENV)/bin/activate
+      python3 -m pip install openvino-dev --find-links=$(INSTALL_DIR)/tools
     displayName: 'Install python wheels'
-
   - script: |
       set -e
       cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -DCOMPONENT=tests -P $(BUILD_DIR)/cmake_install.cmake
     displayName: 'Install tests'
-
   - script: ls -alR $(INSTALL_DIR)
     displayName: 'List install test files'
 
     # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
   - script: |
+      . $(PY_VENV)/bin/activate
       export DATA_PATH=$(MODELS_PATH)
       export MODELS_PATH=$(MODELS_PATH)
       # because of static build libgna is needed for python binary
@@ -268,6 +282,7 @@ jobs:
 
     # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
   - script: |
+      . $(PY_VENV)/bin/activate
       export DATA_PATH=$(MODELS_PATH)
       export MODELS_PATH=$(MODELS_PATH)
       # because of static build libgna is needed for python binary and mock_py frontend library
@@ -285,6 +300,7 @@ jobs:
     continueOnError: false
 
   - script: |
+      . $(PY_VENV)/bin/activate
       # because of static build libgna is needed for python binary
       export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.00.00.1815.1/linux/x64:$LD_LIBRARY_PATH
       python3 -m pytest -s $(INSTALL_TEST_DIR)/mo/unit_tests --junitxml=$(INSTALL_TEST_DIR)/TEST-ModelOptimizer.xml
@@ -434,17 +450,23 @@ jobs:
     inputs:
       cmakeArgs: >
         -GNinja
-        -S $(REPO_DIR)/tests/samples_tests
-        -B $(BUILD_SAMPLES_TESTS_DIR)
+        $(REPO_DIR)/tests/samples_tests
+      workingDirectory: $(BUILD_SAMPLES_TESTS_DIR)
 
-  - script: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P $(BUILD_SAMPLES_TESTS_DIR)/cmake_install.cmake
+  - script: cmake -DCOMPONENT=tests -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
+    workingDirectory: $(BUILD_SAMPLES_TESTS_DIR)
     displayName: 'Install Samples Tests'
 
-  - script: python3 -m pip install -r $(INSTALL_TEST_DIR)/smoke_tests/requirements.txt
+  - script: |
+      . $(PY_VENV)/bin/activate
+      python3 -m pip install -r $(INSTALL_TEST_DIR)/smoke_tests/requirements.txt
+    workingDirectory: $(INSTALL_TEST_DIR)
+
     displayName: 'Install dependencies for samples smoke tests'
     continueOnError: false
 
   - script: |
+      . $(PY_VENV)/bin/activate
       export PATH=$HOME/.local/bin:$PATH
       export IE_APP_PATH=$(INSTALL_DIR)/samples_bin
       export LD_LIBRARY_PATH=$IE_APP_PATH:$LD_LIBRARY_PATH
@@ -456,6 +478,7 @@ jobs:
     continueOnError: false
 
   - script: |
+      . $(PY_VENV)/bin/activate
       python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt
       export PYTHONPATH=$(REPO_DIR)/tools/mo/:$(LAYER_TESTS_DIR):$PYTHONPATH
       export OV_FRONTEND_PATH=$(INSTALL_DIR)/runtime/lib/intel64

.ci/azure/linux_arm64.yml

@@ -46,8 +46,8 @@ jobs:
     system.debug: true
     VSTS_HTTP_RETRY: 5
     VSTS_HTTP_TIMEOUT: 200
-    PYTHON_ARM_VERSION: "3.8.12"
-    PYTHON_EXEC: "python3.8"
+    PYTHON_ARM_VERSION: "3.10.6"
+    PYTHON_EXEC: "python3.10"
     OPENVINO_ARCH: 'aarch64'
     NUM_PROC: 1
     BUILD_TYPE: Release
@@ -79,7 +79,9 @@ jobs:
       curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
       whoami
       uname -a
-      echo Python3 info ; which python3 ; python3 --version
+      echo ls /usr/bin/python3.10
+      rm -rf /usr/bin/python3
+      sudo ln -s /usr/bin/python3.10 /usr/bin/python3      echo Python3 info ; which python3 ; python3 --version
       echo Python info ; which python ; python --version
       echo Java info ; which java ; java -version
       echo gcc info ; which gcc ; gcc --version
@@ -146,6 +148,9 @@ jobs:
       export OPENCV_REPO_DIR=$(OPENCV_REPO_DIR)
       export BUILD_OPENCV=$(BUILD_OPENCV)
       export INSTALL_OPENVINO=$(INSTALL_OPENVINO)
+      sed -i 's;make -j "$NUM_PROC" python Parser/pgen;make -j "$NUM_PROC" python;' $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
+      sed -i 's;cp -r Parser/pgen "$BUILD_PYTHON";;' $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
+      sed -i 's;    HOSTPGEN="$BUILD_PYTHON"/Parser/pgen;;' $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
       $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
     workingDirectory: $(BUILD_OPENVINO)
     displayName: 'Install dependencies'
@@ -158,8 +163,8 @@ jobs:
         -DCMAKE_VERBOSE_MAKEFILE=ON
         -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF
         -DOpenCV_DIR=$(INSTALL_OPENCV)/cmake
-        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8
-        -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so
+        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.10
+        -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.10.so
         -DENABLE_PYTHON=ON
         -DPYTHON_MODULE_EXTENSION=".so"
         -DENABLE_TESTS=ON
@@ -204,10 +209,10 @@ jobs:
         -DOpenVINODeveloperPackage_DIR=$(BUILD_OPENVINO)
         -DENABLE_PYTHON=ON
         -DTHREADING=TBB
-        -DPYTHON_EXECUTABLE=$(INSTALL_PYTHON)/bin/python3.8
-        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8
+        -DPYTHON_EXECUTABLE=$(INSTALL_PYTHON)/bin/python3.10
+        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.10
         -DPYTHON_LIBRARIES=$(INSTALL_PYTHON)/lib
-        -DPYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.8/site-packages/numpy/core/include
+        -DPYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.10/site-packages/numpy/core/include
         -DPYTHON_MODULE_EXTENSION=".so"
         -DPYBIND11_FINDPYTHON=OFF
         -DPYBIND11_NOPYTHON=OFF
@@ -245,8 +250,8 @@ jobs:
         -GNinja
         -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
         -DENABLE_PYTHON=ON
-        -DPYTHON_EXECUTABLE=/usr/local/bin/python3.8
-        -DPYTHON_INCLUDE_DIR=$(INSTALL_PYTHON)/include/python3.8
+        -DPYTHON_EXECUTABLE=/usr/local/bin/python3.10
+        -DPYTHON_INCLUDE_DIR=$(INSTALL_PYTHON)/include/python3.10
         -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib
         -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
         -DOpenVINO_DIR=$(BUILD_OPENVINO)

.ci/azure/linux_conditional_compilation.yml

@@ -79,6 +79,19 @@ jobs:
   - script: |
       set -e
       $(REPO_DIR)/install_build_dependencies.sh
+      wget https://www.python.org/ftp/python/3.10.6/Python-3.10.6.tgz
+      tar -xzf Python-3.10.6.tgz
+      cd Python-3.10.6 || exit
+      ./configure --enable-optimizations --prefix=/usr/local --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
+      make -j 1 python
+      sudo make -j 1 install
+      export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+      /usr/local/bin/python3.10 -m venv $(WORK_DIR)/.venv
+      . $(WORK_DIR)/.venv/bin/activate
+      echo Python3 info ; which python3 ; python3 --version
+      python3 --version
+      python3.10 --version
+      curl https://bootstrap.pypa.io/get-pip.py | python3
       python3 -m pip install -r $(REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements.txt
       # Speed up build
       wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
@@ -97,6 +110,7 @@ jobs:
         -DENABLE_FASTER_BUILD=ON
         -DENABLE_PROFILING_ITT=ON
         -DSELECTIVE_BUILD=COLLECT
+        -DPYTHON_EXECUTABLE=$(WORK_DIR)/.venv/bin/python3.10
         $(REPO_DIR)
       workingDirectory: $(BUILD_DIR)
 

.ci/azure/linux_coverity.yml

@@ -35,6 +35,9 @@ jobs:
       curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
       whoami
       uname -a
+      echo ls /usr/bin/python3.10
+      rm -rf /usr/bin/python3
+      sudo ln -s /usr/bin/python3.10 /usr/bin/python3
       echo Python3 info ; which python3 ; python3 --version
       echo Python info ; which python ; python --version
       echo Java info ; which java ; java -version

.ci/azure/linux_ngraph_onnx.yml

@@ -80,6 +80,7 @@ jobs:
       rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
       sudo mkdir -p $(MODELS_DIR)
       sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
+      sudo apt install nfs-common -y
       sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(MODELS_DIR) -o vers=4,minorversion=1,sec=sys
       mkdir -p $(MODELS_DIR)/models_data
     displayName: 'Make dirs'
@@ -92,17 +93,36 @@ jobs:
 
   - script: |
       set -e
+      apt-get update && apt-get install -y lsb-release && apt-get clean all
+      sudo apt-get -y install libssl-dev zlib1g-dev
+      sudo apt-get -y install build-essential
+      wget https://www.python.org/ftp/python/3.10.7/Python-3.10.7.tgz
+      tar -xzf Python-3.10.7.tgz
+      cd Python-3.10.7 || exit
+      ./configure --enable-optimizations --prefix=/usr/local --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
+      make python
+      sudo make install
+      export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+      /usr/local/bin/python3.10 -m venv $(WORK_DIR)/.venv
+      . $(WORK_DIR)/.venv/bin/activate
+      echo Python3 info ; which python3 ; python3 --version
+      python3 --version
+      python3.10 --version
+      curl https://bootstrap.pypa.io/get-pip.py | python3
       sudo apt --assume-yes install git-lfs uidmap
       curl -fsSL https://get.docker.com -o get-docker.sh
       sudo sh get-docker.sh
     workingDirectory: $(WORK_DIR)
     displayName: 'Install dependencies'
 
-  - script: src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
+  - script:
+      . $(WORK_DIR)/.venv/bin/activate
+      src/bindings/python/tests/test_onnx/model_zoo_preprocess.sh -d $(MODELS_DIR)/models_data -o -s "$(ONNX_MODEL_ZOO_SHA)"
     displayName: 'Update models'
     condition: ne(variables['BUILD_TYPE'], 'Debug')
 
-  - script: sudo docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg PROTOBUF_LITE=$(PROTOBUF_LITE) .
+  - script: 
+      sudo docker build --tag=openvino-onnx-ci-image --file=.ci/openvino-onnx/Dockerfile --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg PROTOBUF_LITE=$(PROTOBUF_LITE) .
     displayName: 'Docker build $(BUILD_TYPE)'
 
   - script: sudo fallocate -l 64G /swapfile ; sudo mkswap /swapfile ; sudo swapon /swapfile ; df ; free -h