add superpoint

README.md

@@ -4,7 +4,7 @@
 
 Tensor library for machine learning
 
-***Note that this project is under active development. \
+***Note that this project is under active development.
 Some of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos***
 
 ## Features
@@ -39,6 +39,7 @@ Some of the development is currently happening in the [llama.cpp](https://github
 - [X] Example of ChatGLM inference [li-plus/chatglm.cpp](https://github.com/li-plus/chatglm.cpp)
 - [X] Example of Stable Diffusion inference [leejet/stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp)
 - [X] Example of Qwen inference [QwenLM/qwen.cpp](https://github.com/QwenLM/qwen.cpp)
+- [X] Example of superpoint inference [examples/superpoint]()
 - [X] Example of YOLO inference [examples/yolo](https://github.com/ggerganov/ggml/tree/master/examples/yolo)
 - [X] Example of ViT inference [staghado/vit.cpp](https://github.com/staghado/vit.cpp)
 - [X] Example of multiple LLMs inference [foldl/chatllm.cpp](https://github.com/foldl/chatllm.cpp)
@@ -51,8 +52,8 @@ With ggml you can efficiently run [Whisper](examples/whisper) inference on the C
 Memory requirements:
 
 | Model | Disk | Mem |
-| --- | --- | ---  |
-| tiny |  75 MB | ~280 MB |
+| ------ | ------ | ------- |
+| tiny | 75 MB  | ~280 MB |
 | base | 142 MB | ~430 MB |
 | small | 466 MB | ~1.0 GB |
 | medium | 1.5 GB | ~2.6 GB |
@@ -92,13 +93,13 @@ python3 ../examples/gpt-2/convert-cerebras-to-ggml.py /path/to/Cerebras-GPT-111M
 The inference speeds that I get for the different models on my 32GB MacBook M1 Pro are as follows:
 
 | Model | Size | Time / Token |
-| --- | --- | ---  |
-| GPT-2 |  117M | 5 ms |
-| GPT-2 |  345M | 12 ms |
-| GPT-2 |  774M | 23 ms |
-| GPT-2 | 1558M |  42 ms |
-| --- | --- | --- |
-| GPT-J | 6B | 125 ms |
+| ----- | ----- | ------------ |
+| GPT-2 | 117M  | 5 ms  |
+| GPT-2 | 345M  | 12 ms  |
+| GPT-2 | 774M  | 23 ms  |
+| GPT-2 | 1558M | 42 ms  |
+| --- | --- | ---  |
+| GPT-J | 6B | 125 ms  |
 
 For more information, checkout the corresponding programs in the [examples](examples) folder.
 
@@ -127,6 +128,7 @@ cmake -DGGML_CUBLAS=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.1/bin/nvcc ..
 ```bash
 cmake -DGGML_CLBLAST=ON ..
 ```
+
 ## Compiling for Android
 
 Download and unzip the NDK from this download [page](https://developer.android.com/ndk/downloads). Set the NDK_ROOT_PATH environment variable or provide the absolute path to the CMAKE_ANDROID_NDK in the command below.

examples/CMakeLists.txt

@@ -24,5 +24,6 @@ add_subdirectory(whisper)
 add_subdirectory(mnist)
 add_subdirectory(sam)
 add_subdirectory(yolo)
+add_subdirectory(superpoint)
 add_subdirectory(simple)
 add_subdirectory(magika)

examples/superpoint/CMakeLists.txt

@@ -0,0 +1,6 @@
+#
+# yolov3-tiny
+
+set(TEST_TARGET superpoint)
+add_executable(${TEST_TARGET} run_superpoint.cpp superpoint-image.cpp)
+target_link_libraries(${TEST_TARGET} PRIVATE ggml common)

examples/superpoint/README.md

@@ -0,0 +1,63 @@
+# Superpoint.cpp
+
+This project shows how to implement Superpoint point extraction and description with ggml using pretrained model weights.
+
+The feature is no thirdparty dependencies needed~
+
+If you want to integrate superpoint to your project and refuse to use thirdparty libs like libtorch, tensorrt, etc, superpoint.cpp is an option!
+
+TODO:
+
+* image has to been preprocessed to size of 480,640
+* image loading is complex and dirty
+* acceleration ...
+
+Download the model weights:(optional)
+
+```bash
+$ wget https://github.com/magicleap/SuperPointPretrainedNetwork/blob/master/superpoint_v1.pth
+
+```
+
+compile the project and generate the executable file
+
+```bash
+$ mkdir build
+$ cd build
+$ cmake ..
+$ make
+$ mv bin/superpoint ../examples/superpoint
+
+```
+
+Convert the weights to GGUF format (optional): since the superpoint.gguf is uploaded to the folder superpoint, this step could be skipped.
+
+```bash
+$ cd /examples/superpoint
+$ ./convert-pth-ggml.py
+```
+
+inference
+
+```bash
+$ ./superpoint -i dog_color.jpg
+```
+
+# Result
+
+feature extration
+
+![yolodog](result.jpg)
+
+matching performance
+
+![matches](matches.png)
+
+# Reference
+
+https://github.com/ggerganov/ggml
+
+https://github.com/magicleap/SuperPointPretrainedNetwork
+
+
+https://github.com/adityamwagh/SuperSLAM

examples/superpoint/convert_pth_ggml.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+'''
+convert superpoint pth paramters to gguf parameters
+
+'''
+
+import sys
+import gguf
+import numpy as np
+from demo_superpoint import *
+from benchmark import *
+
+
+
+def list_params(net):
+ params_list = []
+ list_vars = net.state_dict()
+ for name in list_vars.keys():
+ data = list_vars[name].numpy()
+ print("Processing variable: " + name + " with shape: ", data.shape)
+ params_list.append((name, data))
+ return params_list
+
+
+
+def onnx_export(net):
+ # Let's create a dummy input tensor
+ dummy_input = torch.randn(1, 1,480, 640, requires_grad=True)
+
+ # Export the model
+ torch.onnx.export(net, # model being run
+ dummy_input, # model input (or a tuple for multiple inputs)
+ "superpoint.onnx", # where to save the model
+ export_params=True, # store the trained parameter weights inside the model file
+ opset_version=10, # the ONNX version to export the model to
+ do_constant_folding=True, # whether to execute constant folding for optimization
+ input_names = ['modelInput'], # the model's input names
+ output_names = ['output1', 'output2'], # the model's output names
+ dynamic_axes={'modelInput' : {0 : 'batch_size'}, # variable length axes
+ 'modelOutput' : {0 : 'batch_size'}})
+ return
+
+def isBias(name):
+ return (name.strip().split(".")[1] == "bias")
+
+def isConv(name):
+ return (name.strip().split(".")[1] == "weight")
+
+def save_conv2d_layer( name , conv, index, gguf_writer):
+ layername = "l" + str(index) + "_weights"
+ print(f"save {layername} with shape {conv.shape}")
+ ## ggml doesn't support f32 convolution yet, use f16 instead
+
+ flat_conv = conv.astype(np.float16).flatten()
+ # print(type(conv))
+ # exit(0)
+ gguf_writer.add_tensor(layername, flat_conv, raw_shape= conv.shape)
+ return
+
+def save_bias_layer( name ,biases, index, gguf_writer):
+ filters = biases.shape[0]
+ layername = "l" + str(index) + "_biases"
+ print(f"save {layername} with shape {biases.shape}")
+ gguf_writer.add_tensor(layername, biases, raw_shape=(1, filters, 1, 1))
+ return
+
+
+
+if __name__ == '__main__':
+ #hyper parameters provided by superpoint#
+ weights_path = 'superpoint_v1.pth'
+ nms_dist = 4
+ conf_thresh = 0.015
+ nn_thresh = 0.7
+
+ outfile = "superpoint.gguf"
+ gguf_writer = gguf.GGUFWriter(outfile, 'superpoint')
+
+ fe = SuperPointFrontend(weights_path= weights_path,
+ nms_dist= nms_dist,
+ conf_thresh= conf_thresh,
+ nn_thresh= nn_thresh,
+ cuda= False)
+ conv_list = list_params(fe.net)
+ conv_idx = 0
+ bias_idx = 0
+ for name, layer in conv_list:
+ print(f"processing {name}")
+ if(isConv(name)):
+ # if(conv_idx==10):
+ # print(f"name: {name}: {layer.flatten()[:10]}")
+ # if(conv_idx==11):
+ # print(f"name: {name}: {layer.flatten()[:10]}")
+ # # exit(0)
+
+
+ save_conv2d_layer(name, layer, conv_idx, gguf_writer)
+ conv_idx+=1
+ elif(isBias(name)):
+ save_bias_layer(name, layer, bias_idx, gguf_writer)
+ bias_idx+=1
+
+ gguf_writer.write_header_to_file()
+ gguf_writer.write_kv_data_to_file()
+ gguf_writer.write_tensors_to_file()
+ gguf_writer.close()
+ print("In total {} conv layers, {} bias ".format(conv_idx, bias_idx))
+ print("{} converted to {}".format(weights_path, outfile))