[App] Restore cpp_rtvm native inference tool on Relax VM and the tvm-ffi runtime

apps/cpp_rtvm/CMakeLists.txt

@@ -0,0 +1,86 @@
+cmake_policy(SET CMP0069 NEW) # suppress cmake warning about IPO
+
+set(RTVM_SOURCES
+  main.cc
+  tvm_runner.cc
+)
+set(TVM_RUNNER_SOURCES
+  tvm_runner.cc
+)
+
+set(RTVM_LINKER_LIBS "")
+
+# Set output to same directory as the other TVM libs
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+add_executable(rtvm ${RTVM_SOURCES})
+add_library(tvm_runner_objs OBJECT ${TVM_RUNNER_SOURCES})
+add_library(tvm_runner SHARED $<TARGET_OBJECTS:tvm_runner_objs>)
+
+include(CheckIPOSupported)
+check_ipo_supported(RESULT result OUTPUT output)
+if(result)
+  set_property(TARGET rtvm PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
+endif()
+
+if(WIN32)
+  target_compile_definitions(rtvm PUBLIC -DNOMINMAX)
+endif()
+
+if (OS)
+   if (OS STREQUAL "Linux")
+      set_property(TARGET rtvm PROPERTY LINK_FLAGS -lpthread)
+      set_property(TARGET tvm_runner PROPERTY LINK_FLAGS -lpthread)
+   endif()
+endif()
+
+if(USE_OPENCL)
+   if (ANDROID_ABI)
+     if(DEFINED ENV{ANDROID_NDK_MAJOR})
+       if($ENV{ANDROID_NDK_MAJOR} VERSION_LESS "23")
+         set_property(TARGET rtvm PROPERTY LINK_FLAGS -fuse-ld=gold)
+         set_property(TARGET tvm_runner PROPERTY LINK_FLAGS -fuse-ld=gold)
+       endif()
+     endif()
+   endif()
+endif()
+
+target_include_directories(rtvm PUBLIC "../../include")
+target_link_libraries(rtvm PUBLIC tvm_ffi_header)
+target_include_directories(tvm_runner_objs PUBLIC "../../include")
+target_link_libraries(tvm_runner_objs PUBLIC tvm_ffi_header)
+
+if (BUILD_FOR_ANDROID AND USE_HEXAGON)
+  get_hexagon_sdk_property("${USE_HEXAGON_SDK}" "${USE_HEXAGON_ARCH}"
+    DSPRPC_LIB DSPRPC_LIB_DIRS
+  )
+  if(DSPRPC_LIB_DIRS)
+    link_directories(${DSPRPC_LIB_DIRS})
+  else()
+    message(WARNING "Could not locate some Hexagon SDK components")
+  endif()
+  list(APPEND RTVM_LINKER_LIBS cdsprpc log)
+endif()
+
+# Device backends (OpenCL, CUDA, Vulkan, ...) are separate shared runtime libraries
+# that register their DeviceAPI at load time, so they must stay linked even though
+# rtvm references no symbol from them (hence --no-as-needed).
+if(BUILD_STATIC_RUNTIME)
+  foreach(lib tvm_runtime tvm_ffi_static)
+    if(MSVC)
+      list(APPEND RTVM_LINKER_LIBS "/WHOLEARCHIVE:$<TARGET_FILE:${lib}>")
+    elseif(APPLE)
+      list(APPEND RTVM_LINKER_LIBS "-Wl,-force_load,$<TARGET_FILE:${lib}>")
+    else()
+      list(APPEND RTVM_LINKER_LIBS "-Wl,--whole-archive" "${lib}" "-Wl,--no-whole-archive")
+    endif()
+  endforeach()
+else()
+  if(NOT MSVC AND NOT APPLE)
+    list(APPEND RTVM_LINKER_LIBS tvm_runtime "-Wl,--no-as-needed" ${TVM_RUNTIME_BACKEND_LIBS} "-Wl,--as-needed")
+  else()
+    list(APPEND RTVM_LINKER_LIBS tvm_runtime ${TVM_RUNTIME_BACKEND_LIBS})
+  endif()
+endif()
+
+target_link_libraries(rtvm PRIVATE ${RTVM_LINKER_LIBS})
+target_link_libraries(tvm_runner PRIVATE ${RTVM_LINKER_LIBS})

apps/cpp_rtvm/README.md

@@ -0,0 +1,77 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+
+# Native Inference application for CPP Native
+
+Native inference tool ```rtvm``` helps in deploying TVM compiled models from a standalone cpp environment.
+Overall process starts from getting a model from a framework all the way up to running on target device using `rtvm` tool.
+
+
+### Compile the model
+
+Compilation step generates TVM compiler output artifacts which need to be taken to target device for deployment.
+The inputs and params are need to be dump as numpy files in a folder, which will be need to run on device.
+
+Below command will generate the same
+
+
+```bash
+python3  scripts/download_models.py
+```
+
+### Deployment Run
+
+Now we will verify the deployment run of the compiled model using ```rtvm``` tool on target device.
+
+We need to copy the artifacts and inputs folder under Android temp folder at ```/data/local/tmp/```
+
+Also copy the cross compiled tool ```rtvm```, ```libtvm_runtime.so```, ```libtvm_ffi.so``` and the device
+backend library (e.g. ```libtvm_runtime_opencl.so```) to ```data/local/tmp/```
+
+```rtvm``` usage can be quired as below
+```bash
+Android:/data/local/tmp $ LD_LIBRARY_PATH=./ ./rtvm
+Command line usage
+--model        - The tvm artifacts(mod.so)
+--device       - The target device to use {llvm, opencl, cpu, cuda, metal, rocm, vpi, oneapi}
+--input        - Input folder with input Numpy files where file name need to match with TVM mod main() param
+--output       - Output folder to dump the model output as numpy
+--dump-meta    - Dump model meta information
+--profile      - Profile over all execution
+--dry-run      - Profile after given dry runs, default 10
+--run-count    - Profile for given runs, default 50
+
+  Example
+  LD_LIBRARY_PATH=. ./rtvm --model=./opencl_vm_mod.so --input=./inputs --device=opencl --output=./out
+```
+
+# Performnace Profiling Options
+The tool has added few options to measure wall clock performance of the given model on Target natively.
+--profile : Can turn on the profiling
+--dry-run : The number of times dry run the model before mearuring the performance. Default value os 10
+--run-count : The number times to run the model and take an average. Default value is 50.
+
+Performance profile options dumps information summary as given below.
+     Module Load              :27 ms
+     Graph Runtime Create     :11 ms
+     Params Read              :15 ms
+     Params Set               :41 ms
+     Pre Compiled Progs Load  :24 ms
+Total Load Time     :118 ms
+Average ExecTime    :27 ms
+Unload Time         :35.9236 ms