[MNN:Sync] Sync internal gitlab

CMakeLists.txt

@@ -73,9 +73,9 @@ option(MNN_BUILD_MINI "Build MNN-MINI that just supports fixed shape models." OF
 option(MNN_USE_SSE "Use SSE optimization for x86 if possiable" ON)
 option(MNN_BUILD_CODEGEN "Build with codegen" OFF)
 option(MNN_ENABLE_COVERAGE "Build with coverage enable" OFF)
-option(MNN_MODEL_AUTH "Build with model authentication enable" OFF)
 option(MNN_BUILD_PROTOBUFFER "Build with protobuffer in MNN" ON)
 option(MNN_BUILD_OPENCV "Build OpenCV api in MNN." OFF)
+option(MNN_INTERNAL "Build with MNN internal features, such as model authentication, metrics logging" OFF)
 
 IF (NOT DEFINED MNN_USE_SPARSE_COMPUTE)
   IF (TARGET_OS_IPHONE OR TARGET_OS_SIMULATOR)
@@ -204,7 +204,6 @@ message(STATUS "\tOpenMP: ${MNN_OPENMP}")
 message(STATUS "\tBF16: ${MNN_SUPPORT_BF16}")
 message(STATUS "\tThreadPool: ${MNN_USE_THREAD_POOL}")
 message(STATUS "\tHidden: ${MNN_HIDDEN}")
-message(STATUS "\tModel Auth: ${MNN_MODEL_AUTH}")
 message(STATUS "\tBuild Path: ${CMAKE_CURRENT_BINARY_DIR}")
 
 if(MSVC)
@@ -386,6 +385,7 @@ list(APPEND MNN_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/Matrix.h")
 list(APPEND MNN_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/Rect.h")
 list(APPEND MNN_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/MNNForwardType.h")
 list(APPEND MNN_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/AutoTime.hpp")
+list(APPEND MNN_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/MNNSharedContext.h")
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/Expr.hpp")
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/ExprCreator.hpp")
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/MathOp.hpp")
@@ -394,17 +394,22 @@ list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/Opti
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/Executor.hpp")
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/Module.hpp")
 list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/NeuralNetWorkOp.hpp")
+list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/ExecutorScope.hpp")
+list(APPEND MNN_EXPR_PUB_HDRS "${CMAKE_CURRENT_SOURCE_DIR}/include/MNN/expr/Scope.hpp")
 
 set(MNN_DEPS "")
 set(MNN_EXTRA_DEPENDS "")
 
+# Add Thread dependency
+find_package(Threads)
+list(APPEND MNN_EXTRA_DEPENDS ${CMAKE_THREAD_LIBS_INIT})
+
 if (NOT APPLE)
   if(MNN_OPENMP)
       message(STATUS "[*] Checking OpenMP")
       find_package(OpenMP)
       # For CMake < 3.9, we need to make the target ourselves
       if(NOT TARGET OpenMP::OpenMP_CXX)
-          find_package(Threads REQUIRED)
           add_library(OpenMP::OpenMP_CXX IMPORTED INTERFACE)
           set_property(TARGET OpenMP::OpenMP_CXX
               PROPERTY INTERFACE_COMPILE_OPTIONS ${OpenMP_CXX_FLAGS})
@@ -447,12 +452,6 @@ if(MNN_WITH_PLUGIN)
     include(${CMAKE_CURRENT_LIST_DIR}/source/plugin/CMakeLists.txt)
 endif()
 
-# Model Auth. Disabled for Open Source MNN.
-if (MNN_MODEL_AUTH)
-    target_compile_options(MNNCore PRIVATE -DMNN_MODEL_AUTH)
-    include(${CMAKE_CURRENT_LIST_DIR}/source/auth/CMakeLists.txt)
-endif()
-
 # Metal
 if(MNN_METAL AND APPLE)
     target_compile_options(MNNCore PRIVATE -DMNN_METAL_ENABLED=1)
@@ -556,6 +555,14 @@ ELSE()
    list(APPEND MNN_OBJECTS_TO_LINK $<TARGET_OBJECTS:MNN_Express>)
 ENDIF()
 
+# Model Internal. Enable MNN internal features such as model authentication and metrics logging.
+if (MNN_INTERNAL)
+    target_compile_options(MNNCore PRIVATE -DMNN_INTERNAL_ENABLED)
+    target_compile_options(MNN_Express PRIVATE -DMNN_INTERNAL_ENABLED)
+    include(${CMAKE_CURRENT_LIST_DIR}/source/internal/auth/CMakeLists.txt)
+    include(${CMAKE_CURRENT_LIST_DIR}/source/internal/logging/CMakeLists.txt)
+endif()
+
 # Train
 IF(MNN_BUILD_TRAIN OR MNN_BUILD_QUANTOOLS)
   add_subdirectory(tools/train)

MNN.podspec

@@ -33,10 +33,6 @@ Pod::Spec.new do |s|
   s.requires_arc = true
 
   #s.source =  { :git => "[email protected]:alibaba/MNN.git", :branch => 'master' }
-  s.prepare_command = <<-CMD
-                          schema/generate.sh
-                          python source/backend/metal/MetalCodeGen.py source/backend/metal/ source/backend/metal/MetalOPRegister.mm
-                      CMD
   s.source = {:git => "/Users/zhang/Development/AliNNPrivate/",:branch=> 'head'}
   s.frameworks = 'Metal', 'Accelerate'
   s.library = 'c++'
@@ -62,6 +58,6 @@ Pod::Spec.new do |s|
   'express/**/*.{hpp,cpp}'
   s.header_mappings_dir = 'include'
 
-  s.pod_target_xcconfig = {'METAL_LIBRARY_FILE_BASE' => 'mnn', 'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/include" "$(PODS_TARGET_SRCROOT)/3rd_party/flatbuffers/include" "$(PODS_TARGET_SRCROOT)/source" "$(PODS_TARGET_SRCROOT)/3rd_party/half"', 'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) MNN_CODEGEN_REGISTER=1 MNN_SUPPORT_TFLITE_QUAN=1 MNN_METAL_ENABLED=1 MNN_SUPPORT_BF16=1 MNN_MODEL_AUTH=1'}
+  s.pod_target_xcconfig = {'METAL_LIBRARY_FILE_BASE' => 'mnn', 'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/include" "$(PODS_TARGET_SRCROOT)/3rd_party/flatbuffers/include" "$(PODS_TARGET_SRCROOT)/source" "$(PODS_TARGET_SRCROOT)/3rd_party/half"', 'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) MNN_CODEGEN_REGISTER=1 MNN_SUPPORT_TFLITE_QUAN=1 MNN_METAL_ENABLED=1 MNN_SUPPORT_BF16=1'}
   s.user_target_xcconfig = { 'OTHER_LDFLAGS' => '-force_load $(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)/MNN/libMNN.a', 'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/include"' }
 end

README.md

@@ -7,7 +7,7 @@
 ## Intro
 MNN is a highly efficient and lightweight deep learning framework. It supports inference and training of deep learning models, and has industry leading performance for inference and training on-device. At present, MNN has been integrated in more than 20 apps of Alibaba Inc, such as Taobao, Tmall, Youku, Dingtalk, Xianyu and etc., covering more than 70 usage scenarios such as live broadcast, short video capture, search recommendation, product searching by image, interactive marketing, equity distribution, security risk control. In addition, MNN is also used on embedded devices, such as IoT.
 
-The design principles and performance data of MNN has been published in an MLSys 2020 paper [here](https://proceedings.mlsys.org/static/paper_files/mlsys/2020/7-Paper.pdf). Please cite MNN in your publications if it helps your research:
+The design principles and performance data of MNN has been published in an MLSys 2020 paper [here](https://arxiv.org/pdf/2002.12418.pdf). Please cite MNN in your publications if it helps your research:
 
     @inproceedings{alibaba2020mnn,
       author = {Jiang, Xiaotang and Wang, Huan and Chen, Yiliu and Wu, Ziqi and Wang, Lichuan and Zou, Bin and Yang, Yafeng and Cui, Zongyang and Cai, Yu and Yu, Tianhang and Lv, Chengfei and Wu, Zhihua},

README_CN.md

@@ -7,7 +7,7 @@
 ## 简介
 MNN是一个高效、轻量的深度学习框架。它支持深度模型推理与训练，尤其在端侧的推理与训练性能在业界处于领先地位。目前，MNN已经在阿里巴巴的手机淘宝、手机天猫、优酷、钉钉、闲鱼等20多个App中使用，覆盖直播、短视频、搜索推荐、商品图像搜索、互动营销、权益发放、安全风控等70多个场景。此外，IoT等场景下也有若干应用。
 
-MNN的架构设计理念与性能数据在MLSys 2020上面发表。Paper [在此处](https://proceedings.mlsys.org/static/paper_files/mlsys/2020/7-Paper.pdf)。如果MNN对你的研究有所助益，欢迎引用MNN的论文：
+MNN的架构设计理念与性能数据在MLSys 2020上面发表。Paper [在此处](https://arxiv.org/pdf/2002.12418.pdf)。如果MNN对你的研究有所助益，欢迎引用MNN的论文：
 
     @inproceedings{alibaba2020mnn,
       author = {Jiang, Xiaotang and Wang, Huan and Chen, Yiliu and Wu, Ziqi and Wang, Lichuan and Zou, Bin and Yang, Yafeng and Cui, Zongyang and Cai, Yu and Yu, Tianhang and Lv, Chengfei and Wu, Zhihua},

demo/iOS/Podfile.lock

@@ -1,5 +1,5 @@
 PODS:
-  - MNN (1.0.0)
+  - MNN (1.2.0)
 
 DEPENDENCIES:
   - MNN (from `../../`)
@@ -9,8 +9,8 @@ EXTERNAL SOURCES:
     :path: "../../"
 
 SPEC CHECKSUMS:
-  MNN: 31075cbcadf73e96c1bf29cccc97e4ef131e0650
+  MNN: 572edcad498d35f845f5a98c27165b6032ed09f8
 
 PODFILE CHECKSUM: b0491e2fa8f04fdaec2683a1c6c9de3a1d483842
 
-COCOAPODS: 1.5.3
+COCOAPODS: 1.9.1

demo/iOS/playground.xcodeproj/project.pbxproj

@@ -7,6 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		48BDF97427579425001E99A7 /* Pretreat.metal in Sources */ = {isa = PBXBuildFile; fileRef = 48BDF97327579425001E99A7 /* Pretreat.metal */; };
 		922CC9BE2206B7DC0085D404 /* synset_words.txt in Resources */ = {isa = PBXBuildFile; fileRef = 922CC9B92206B7DC0085D404 /* synset_words.txt */; };
 		922CC9BF2206B7DC0085D404 /* testcat.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 922CC9BA2206B7DC0085D404 /* testcat.jpg */; };
 		922CC9C02206B7DC0085D404 /* squeezenet.txt in Resources */ = {isa = PBXBuildFile; fileRef = 922CC9BC2206B7DC0085D404 /* squeezenet.txt */; };
@@ -28,6 +29,7 @@
 /* Begin PBXFileReference section */
 		19D6949C3BE113930E7EB826 /* libPods-simple.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-simple.a"; sourceTree = BUILT_PRODUCTS_DIR; };
 		443959A142C9F5A9260064A1 /* libPods-playground.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-playground.a"; sourceTree = BUILT_PRODUCTS_DIR; };
+		48BDF97327579425001E99A7 /* Pretreat.metal */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.metal; path = Pretreat.metal; sourceTree = "<group>"; };
 		78072E65338A363D5816A269 /* Pods-simple.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-simple.release.xcconfig"; path = "Pods/Target Support Files/Pods-simple/Pods-simple.release.xcconfig"; sourceTree = "<group>"; };
 		823F14839C64B7FBF3FE2122 /* Pods-simple.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-simple.debug.xcconfig"; path = "Pods/Target Support Files/Pods-simple/Pods-simple.debug.xcconfig"; sourceTree = "<group>"; };
 		83E77ED818CBE7A7F809B82C /* Pods-playground.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-playground.debug.xcconfig"; path = "Pods/Target Support Files/Pods-playground/Pods-playground.debug.xcconfig"; sourceTree = "<group>"; };
@@ -131,6 +133,7 @@
 		AE2CFE1D21FFE7390053943C /* playground */ = {
 			isa = PBXGroup;
 			children = (
+				48BDF97327579425001E99A7 /* Pretreat.metal */,
 				AE2CFE1E21FFE7390053943C /* AppDelegate.h */,
 				AE2CFE1F21FFE7390053943C /* AppDelegate.m */,
 				AE2CFE2121FFE7390053943C /* ViewController.h */,
@@ -278,6 +281,7 @@
 			files = (
 				AE2CFE2321FFE7390053943C /* ViewController.mm in Sources */,
 				AE2CFE2E21FFE73B0053943C /* main.m in Sources */,
+				48BDF97427579425001E99A7 /* Pretreat.metal in Sources */,
 				AE2CFE2021FFE7390053943C /* AppDelegate.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -422,14 +426,15 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
-				DEVELOPMENT_TEAM = "";
+				DEVELOPMENT_TEAM = 6G7464HHUS;
+				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = playground/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 9.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.taobao.mnndemo;
+				PRODUCT_BUNDLE_IDENTIFIER = com.taobao.mnndemo34;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
@@ -441,15 +446,16 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
-				DEVELOPMENT_TEAM = "";
+				DEVELOPMENT_TEAM = 6G7464HHUS;
+				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = playground/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 9.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
 				ONLY_ACTIVE_ARCH = YES;
-				PRODUCT_BUNDLE_IDENTIFIER = com.taobao.mnndemo;
+				PRODUCT_BUNDLE_IDENTIFIER = com.taobao.mnndemo34;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};

demo/iOS/playground/Pretreat.metal

@@ -0,0 +1,32 @@
+#include <metal_stdlib>
+
+using namespace metal;
+
+struct PretreatInfo {
+    int4 outputSize;
+    float4 mean;
+    float4 normal;
+    float4 inputSize;
+    float4x4 matrix;
+};
+
+kernel void pretreat(device half4* output  [[buffer(0)]],
+                     texture2d<float> input [[texture(0)]],
+                     constant PretreatInfo& info[[buffer(1)]],
+                   uint2 gid[[thread_position_in_grid]]) {
+    constexpr sampler linearSampler(mip_filter::none,
+                                    mag_filter::linear,
+                                    min_filter::linear);
+    if ((int)gid.x < (int)info.outputSize.x && (int)gid.y < (int)info.outputSize.y) {
+#ifdef COMMON_MATRIX
+        float3 pos = float3((float)gid.x, (float)gid.y, 1.0);
+        pos = pos * info.matrix;
+        float4 color = input.sample(linearSampler, pos.xy / pos.z);
+#else
+        float2 pos = float2((float)(gid.x - 1) / (float)(info.outputSize.x - 1), (float)(gid.y - 1) / (float)(info.outputSize.y - 1));
+        float4 color = input.sample(linearSampler, pos);
+#endif
+        color = (color * float4(255) - info.mean) * info.normal;
+        output[gid.y * info.outputSize.x + gid.x] = half4(color);
+    }
+}