refactor the C and Go wrapper build system

- moved "nimbus/api" to "wrappers"
- renamed files
- replaced the build scripts with Makefile targets
- set the rpath relative to the test binary's location so it can look
  for libnimbus.so there at runtime
- libnimbus.so.0 required on Linux, apparently
- compiled all the Nimbus code with `--app:lib`, not just one file (this
  required skipping a proc in "nimbus/config.nim" because it uses an API
  that's unavailable in libraries)
- removed static linking from the Go wrapper. It doesn't make sense at a
  global level, when using a shared Nimbus library. To selectively link
  static libraries, we should probably be specifying them as *.a. I did
  build a static libnimbus.a, as a test, but it insisted on dlopen-ing a
  shared version of itself which looked too ugly to continue.

.gitignore

@@ -1,4 +1,4 @@
-nimcache
+/nimcache
 
 # Executables shall be put in an ignored build/ directory
 /build

Makefile

@@ -49,7 +49,7 @@ TOOLS_DIRS := premix tests
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
-.PHONY: all $(TOOLS) deps sanity-checks github-ssh build-nim update status ntags ctags nimbus testsuite test clean mrproper fetch-dlls test-libp2p-daemon nat-libs libminiupnpc.a libnatpmp.a go-checks
+.PHONY: all $(TOOLS) deps sanity-checks github-ssh build-nim update status ntags ctags nimbus testsuite test clean mrproper fetch-dlls test-libp2p-daemon nat-libs libminiupnpc.a libnatpmp.a go-checks libnimbus.so wrappers
 
 # default target, because it's the first one that doesn't start with '.'
 all: $(TOOLS) nimbus
@@ -128,7 +128,7 @@ test-reproducibility:
 
 # usual cleaning
 clean:
- rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc,*.exe} vendor/go/bin \
+ rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc,*.exe,*.so,*.so.0,*_wrapper_test} vendor/go/bin \
  $(NIMBLE_DIR) $(NIM_BINARY) $(NIM_DIR)/nimcache nimcache
  + $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc clean $(HANDLE_OUTPUT)
  + $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp clean $(HANDLE_OUTPUT)
@@ -197,6 +197,18 @@ test-libp2p-daemon: | vendor/go/bin/p2pd deps
  $(ENV_SCRIPT) nim c -r $(NIM_PARAMS) tests/testdaemon.nim && \
  rm -f tests/testdaemon
 
+libnimbus.so: | build deps nat-libs
+ echo -e $(BUILD_MSG) "build/$@" && \
+ $(ENV_SCRIPT) nim c --app:lib --noMain -d:"chronicles_sinks=textlines" --debuginfo --opt:speed --lineTrace:off $(NIM_PARAMS) -o:build/$@.0 wrappers/wrapper.nim && \
+ rm -f build/$@ && \
+ ln -s $@.0 build/$@
+
+wrappers: | build deps nat-libs libnimbus.so go-checks
+ echo -e $(BUILD_MSG) "build/C_wrapper_test" && \
+ $(CC) wrappers/wrapper.c -Wl,-rpath,'$$ORIGIN' -Lbuild -lnimbus -lm -g -o build/C_wrapper_test
+ echo -e $(BUILD_MSG) "build/go_wrapper_test" && \
+ go build -o build/go_wrapper_test wrappers/wrapper.go
+
 # https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory"
 ntags:
  ntags -R .

nim.cfg

@@ -17,3 +17,7 @@
 --excessiveStackTrace:on
 -d:metrics # enable metric collection
 
+# Required to make up for the compiler's inability to invalidate the C file
+# (and object) cache on different command line arguments.
+--forceBuild
+

nimbus/config.nim

@@ -752,60 +752,61 @@ LOGGING AND DEBUGGING OPTIONS:
  $ord(defaultNetwork)
  ]
 
-proc processArguments*(msg: var string): ConfigStatus =
- ## Process command line argument and update `NimbusConfiguration`.
- let config = getConfiguration()
-
- # At this point `config.net.bootnodes` is likely populated with network default
- # bootnodes. We want to override those if at least one custom bootnode is
- # specified on the command line. We temporarily set `config.net.bootNodes`
- # to empty seq, and in the end restore it if no bootnodes were spricified on
- # the command line.
- # TODO: This is pretty hacky and it's better to refactor it to make a clear
- # distinction between default and custom bootnodes.
- var tempBootNodes: seq[ENode]
- swap(tempBootNodes, config.net.bootNodes)
-
- # The same trick is done to discPort
- config.net.discPort = 0
-
- var opt = initOptParser()
- var length = 0
- for kind, key, value in opt.getopt():
- result = Error
- case kind
- of cmdArgument:
- discard
- of cmdLongOption, cmdShortOption:
- inc(length)
- case key.toLowerAscii()
- of "help", "h":
- msg = getHelpString()
- result = Success
- break
- of "version", "ver", "v":
- msg = NimbusVersion
- result = Success
- break
- else:
- processArgument processEthArguments, key, value, msg
- processArgument processRpcArguments, key, value, msg
- processArgument processNetArguments, key, value, msg
- processArgument processShhArguments, key, value, msg
- processArgument processDebugArguments, key, value, msg
- if result != Success:
- msg = "Unknown option: '" & key & "'."
- break
- of cmdEnd:
- doAssert(false) # we're never getting this kind here
-
- if config.net.bootNodes.len == 0:
- # No custom bootnodes were specified on the command line, restore to
- # previous values
+when declared(os.paramCount): # not available with `--app:lib`
+ proc processArguments*(msg: var string): ConfigStatus =
+ ## Process command line argument and update `NimbusConfiguration`.
+ let config = getConfiguration()
+
+ # At this point `config.net.bootnodes` is likely populated with network default
+ # bootnodes. We want to override those if at least one custom bootnode is
+ # specified on the command line. We temporarily set `config.net.bootNodes`
+ # to empty seq, and in the end restore it if no bootnodes were spricified on
+ # the command line.
+ # TODO: This is pretty hacky and it's better to refactor it to make a clear
+ # distinction between default and custom bootnodes.
+ var tempBootNodes: seq[ENode]
  swap(tempBootNodes, config.net.bootNodes)
 
- if config.net.discPort == 0:
- config.net.discPort = config.net.bindPort
+ # The same trick is done to discPort
+ config.net.discPort = 0
+
+ var opt = initOptParser()
+ var length = 0
+ for kind, key, value in opt.getopt():
+ result = Error
+ case kind
+ of cmdArgument:
+ discard
+ of cmdLongOption, cmdShortOption:
+ inc(length)
+ case key.toLowerAscii()
+ of "help", "h":
+ msg = getHelpString()
+ result = Success
+ break
+ of "version", "ver", "v":
+ msg = NimbusVersion
+ result = Success
+ break
+ else:
+ processArgument processEthArguments, key, value, msg
+ processArgument processRpcArguments, key, value, msg
+ processArgument processNetArguments, key, value, msg
+ processArgument processShhArguments, key, value, msg
+ processArgument processDebugArguments, key, value, msg
+ if result != Success:
+ msg = "Unknown option: '" & key & "'."
+ break
+ of cmdEnd:
+ doAssert(false) # we're never getting this kind here
+
+ if config.net.bootNodes.len == 0:
+ # No custom bootnodes were specified on the command line, restore to
+ # previous values
+ swap(tempBootNodes, config.net.bootNodes)
+
+ if config.net.discPort == 0:
+ config.net.discPort = config.net.bindPort
 
 proc processConfiguration*(pathname: string): ConfigStatus =
  ## Process configuration file `pathname` and update `NimbusConfiguration`.

wrappers/README.md

@@ -0,0 +1,21 @@
+This folder contains an experimental C wrapper for using parts of the Nimbus
+code from C/Go in the Status console client:
+
+https://github.com/status-im/status-console-client/
+
+It serves mainly as a proof-of-concept for now - there are several unresolved
+issues surrounding threading, inter-language communication, callbacks etc.
+
+To build the wrappers and the test programs, run from the top level directory:
+
+```bash
+make wrappers
+```
+
+Now you can run the test programs:
+
+```bash
+build/C_wrapper_test
+build/go_wrapper_test
+```
+

nimbus/api/status_api.c → wrappers/wrapper.c

@@ -4,7 +4,7 @@
 #include <unistd.h>
 #include <time.h>
 
-#include "status_api.h"
+#include "wrapper.h"
 
 void NimMain();
 

nimbus/api/main.go → wrappers/wrapper.go

@@ -6,8 +6,8 @@ import (
  "time"
 )
 
-// #cgo LDFLAGS: /home/oskarth/git/nimbus/nimbus/libnimbus_api.so -lm
-// #include "libnim.h"
+// #cgo LDFLAGS: -Wl,-rpath,'$ORIGIN' -L${SRCDIR}/../build -lnimbus -lm
+// #include "wrapper.h"
 import "C"
 
 // Arrange that main.main runs on main thread.

nimbus/api/status_api.nim → wrappers/wrapper.nim

@@ -12,7 +12,7 @@ import
  nimcrypto/[bcmode, hmac, rijndael, pbkdf2, sha2, sysrand, utils, keccak, hash],
  eth/[keys, rlp, p2p], eth/p2p/rlpx_protocols/[whisper_protocol],
  eth/p2p/[discovery, enode, peer_pool], chronicles,
- ../config
+ ../nimbus/config
 
 type
  CReceivedMessage* = object