feat: add host arg. (#273)

Author: b4rtaz

README.md

@@ -100,6 +100,7 @@ Worker, API
 
 | Argument                     | Description                       | Example           |
 | ---------------------------- | --------------------------------- | ----------------- |
+| `--host <addr>`              | Binding address.                  | `127.0.0.1`       |
 | `--port <port>`              | Binding port.                     | `9999`            |
 
 Inference

docs/HOW_TO_RUN_RASPBERRYPI.md

@@ -78,6 +78,7 @@ sudo nice -n -20 ./dllama inference \
 
 ```sh
 sudo nice -n -20 ./dllama-api \
+  --host 0.0.0.0 \
   --port 9999 \
   --model models/llama3_2_3b_instruct_q40/dllama_model_llama3_2_3b_instruct_q40.m \
   --tokenizer models/llama3_2_3b_instruct_q40/dllama_tokenizer_llama3_2_3b_instruct_q40.t \

src/app.cpp

@@ -35,6 +35,7 @@ AppCliArgs AppCliArgs::parse(int argc, char* *argv, bool requireMode) {
     args.nWorkers = 0;
     args.workerHosts = nullptr;
     args.workerPorts = nullptr;
+    args.host = "0.0.0.0";
     args.port = 9990;
     args.temperature = 0.8f;
     args.topp = 0.9f;
@@ -97,6 +98,8 @@ AppCliArgs AppCliArgs::parse(int argc, char* *argv, bool requireMode) {
             i += count - 1;
         } else if (std::strcmp(name, "--port") == 0) {
             args.port = atoi(value);
+        } else if (std::strcmp(name, "--host") == 0) {
+            args.host = value;
         } else if (std::strcmp(name, "--nthreads") == 0) {
             args.nThreads = atoi(value);
         } else if (std::strcmp(name, "--steps") == 0) {
@@ -302,7 +305,7 @@ void runInferenceApp(AppCliArgs *args, void (*handler)(AppInferenceContext *cont
 
 void runWorkerApp(AppCliArgs *args) {
     while (true) {
-        std::unique_ptr<NnNetwork> networkPtr = NnNetwork::serve(args->port);
+        std::unique_ptr<NnNetwork> networkPtr = NnNetwork::serve(args->host, args->port);
         NnNetwork *network = networkPtr.get();
 
         NnWorkerConfigReader configReader(network);

src/app.hpp

@@ -35,7 +35,8 @@ class AppCliArgs {
     int gpuSegmentFrom;
     int gpuSegmentTo;
 
-    // worker
+    // binding
+    const char *host;
     NnUint port;
 
     static AppCliArgs parse(int argc, char **argv, bool hasMode);

src/dllama-api.cpp

@@ -534,14 +534,16 @@ void handleModelsRequest(HttpRequest& request, const char* modelPath) {
 }
 
 static void server(AppInferenceContext *context) {
-    NnSocket serverSocket(createServerSocket(context->args->port));
+    NnSocket serverSocket(createServerSocket(context->args->host, context->args->port));
 
     TokenizerChatStops stops(context->tokenizer);
     ChatTemplateGenerator templateGenerator(context->args->chatTemplateType, context->tokenizer->chatTemplate, stops.stops[0]);
     EosDetector eosDetector(stops.nStops, context->tokenizer->eosTokenIds.data(), stops.stops, stops.maxStopLength, stops.maxStopLength);
     ApiServer api(context->inference, context->tokenizer, context->sampler, context->args, context->header, &eosDetector, &templateGenerator);
 
-    printf("Server URL: http://127.0.0.1:%d/v1/\n", context->args->port);
+    if (strcmp(context->args->host, "0.0.0.0") == 0 ||
+        strcmp(context->args->host, "127.0.0.1") == 0)
+        printf("Server URL: http://localhost:%d/v1/\n", context->args->port);
 
     std::vector<Route> routes = {
         {
@@ -577,7 +579,7 @@ static void server(AppInferenceContext *context) {
 #endif
 
 void usage() {
-    fprintf(stderr, "Usage: %s {--model <path>} {--tokenizer <path>} [--port <p>]\n", EXECUTABLE_NAME);
+    fprintf(stderr, "Usage: %s {--model <path>} {--tokenizer <path>} [--host <addr>] [--port <p>]\n", EXECUTABLE_NAME);
     fprintf(stderr, "        [--buffer-float-type {f32|f16|q40|q80}]\n");
     fprintf(stderr, "        [--weights-float-type {f32|f16|q40|q80}]\n");
     fprintf(stderr, "        [--max-seq-len <max>]\n");

src/nn/nn-network.cpp

@@ -172,8 +172,7 @@ static inline int connectSocket(char *host, int port) {
     return sock;
 }
 
-int createServerSocket(int port) {
-    const char *host = "0.0.0.0";
+int createServerSocket(const char *host, const int port) {
     struct sockaddr_in serverAddr;
 
     int serverSocket = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
@@ -185,6 +184,10 @@ int createServerSocket(int port) {
     serverAddr.sin_family = AF_INET;
     serverAddr.sin_port = htons(port);
     serverAddr.sin_addr.s_addr = inet_addr(host);
+    if (serverAddr.sin_addr.s_addr == INADDR_NONE) {
+        destroySocket(serverSocket);
+        throw std::runtime_error("Invalid bind host");
+    }
 
     int bindResult;
     #ifdef _WIN32
@@ -289,8 +292,8 @@ int NnSocket::release() {
     return fd;
 }
 
-std::unique_ptr<NnNetwork> NnNetwork::serve(int port) {
-    NnSocket socketSocket(createServerSocket(port));
+std::unique_ptr<NnNetwork> NnNetwork::serve(const char *host, const int port) {
+    NnSocket socketSocket(createServerSocket(host, port));
 
     NnUint nSockets;
     NnUint nodeIndex;

src/nn/nn-network.hpp

@@ -11,7 +11,7 @@ int acceptSocket(int serverSocket);
 void setReuseAddr(int socket);
 void writeSocket(int socket, const void* data, NnSize size);
 void readSocket(int socket, void* data, NnSize size);
-int createServerSocket(int port);
+int createServerSocket(const char *host, const int port);
 void destroySocket(int serverSocket);
 
 class NnConnectionSocketException : public std::runtime_error {
@@ -48,7 +48,7 @@ class NnNetwork {
     NnSize *recvBytes;
 
 public:
-    static std::unique_ptr<NnNetwork> serve(int port);
+    static std::unique_ptr<NnNetwork> serve(const char *host, const int port);
     static std::unique_ptr<NnNetwork> connect(NnUint nSockets, char **hosts, NnUint *ports);
 
     NnUint nSockets;