1.0.6

CMakeLists.txt

@@ -24,8 +24,12 @@ FetchContent_MakeAvailable(httplib)
 
 set(CMAKE_BUILD_TYPE Release)
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+target_compile_options(fasttext-static PRIVATE
+    -march=x86-64 -mno-avx512f)
 
 add_executable(fasttext_lid_server main.cpp)
+target_compile_options(fasttext_lid_server PRIVATE
+    -march=x86-64 -mno-avx512f)
 target_include_directories(fasttext_lid_server PRIVATE ${fasttext_SOURCE_DIR}/src)
 
 target_link_libraries(fasttext_lid_server

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim AS builder
+FROM debian:bookworm-slim AS builder
 
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -14,7 +14,7 @@ RUN mkdir build-docker \
     && cmake -DCMAKE_BUILD_TYPE=Release .. \
     && make -j$(nproc)
 
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y \
     libstdc++6 \
@@ -23,6 +23,6 @@ RUN apt-get update && apt-get install -y \
 
 WORKDIR /app
 COPY --from=builder /app/build-docker/fasttext_lid_server .
-COPY lid.176.ftz .
+RUN curl --remote-name https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
 
-CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.ftz} ${PORT:-8080}"]
+CMD ["sh", "-c", "exec ./fasttext_lid_server ${MODEL_FILE:-lid.176.bin} ${PORT:-8080}"]

README.md

@@ -1,3 +1,7 @@
+## This is a fork
+
+Forked from [badlogic/fasttext-lid-server](https://github.com/badlogic/fasttext-lid-server/), this repository uses a newer debian version (bookworm), uses the [bigger model](https://fasttext.cc/docs/en/language-identification) (126MB) and gives you the top 3 results instead of only 1.
+
 # FastText Language ID Server
 
 HTTP server that provides language identification using FastText models.
@@ -6,7 +10,7 @@ HTTP server that provides language identification using FastText models.
 
 ```bash
 # Pull and run
-docker run -p 8080:8080 mariozechner/fasttext-lid-server
+docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server
 
 # Test
 curl "http://localhost:8080/detect?text=Hello%20world"
@@ -18,10 +22,20 @@ curl "http://localhost:8080/detect?text=Hello%20world"
 - **Parameter**: `text` (required)
 - **Response**: JSON with language code and probability
 ```json
-{
-    "language": "en",
-    "probability": 0.987
-}
+[
+	{
+		"language": "en",
+		"probability": 0.987
+	},
+	{
+		"language": "de",
+		"probability": 0.234
+	},
+	{
+		"language": "nl",
+		"probability": 0.123
+	}
+]
 ```
 - **Errors**:
   - 400: Missing text parameter
@@ -32,14 +46,14 @@ curl "http://localhost:8080/detect?text=Hello%20world"
 ### Using docker run
 ```bash
 # Default settings
-docker run -p 8080:8080 mariozechner/fasttext-lid-server
+docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server
 
 # Custom settings
 docker run -p 9000:9000 \
   -e PORT=9000 \
   -e MODEL_FILE=/models/custom.ftz \
   -v $(pwd)/models:/models \
-  mariozechner/fasttext-lid-server
+  suspiciousactivity/fasttext-lid-server
 ```
 
 ### Using docker compose
@@ -48,7 +62,7 @@ version: '3.8'
 
 services:
   lid-server:
-    image: mariozechner/fasttext-lid-server
+    image: suspiciousactivity/fasttext-lid-server
     ports:
         - "${PORT:-8080}:${PORT:-8080}"
     environment:

main.cpp

@@ -3,6 +3,9 @@
 #include <string>
 #include <iostream>
 #include <sstream>
+#include <csignal>
+
+httplib::Server *g_svr = nullptr;
 
 std::string stripLabel(const std::string &label) {
 	const std::string prefix = "__label__";
@@ -33,7 +36,7 @@ int main(int argc, char *argv[]) {
 		auto text = req.get_param_value("text");
 		std::istringstream iss(text);
 		std::vector<std::pair<fasttext::real, std::string>> predictions;
-		ft.predictLine(iss, predictions, 1, 0.0);
+		ft.predictLine(iss, predictions, 3, 0.0);
 
 		if (predictions.empty()) {
 			res.status = 500;
@@ -41,13 +44,31 @@ int main(int argc, char *argv[]) {
 			return;
 		}
 
-		std::string result = "{\"language\":\"" + stripLabel(predictions[0].second) +
-							 "\",\"probability\":" + std::to_string(predictions[0].first) + "}";
+		std::string result = "[";
+
+		bool first = true;
+		for (const auto& prediction : predictions) {
+			std::string predictionResult = "{\"language\":\"" + stripLabel(prediction.second) +
+				"\",\"probability\":" + std::to_string(prediction.first) + "}";
+			if (!first) {
+				result += ",";
+			}
+			result += predictionResult;
+			first = false;
+		}
+
+		result += "]";
+		
 		res.set_content(result, "application/json");
 	});
 
+	g_svr = &svr;
+	std::signal(SIGINT,  [](int) { if (g_svr) g_svr->stop(); });
+	std::signal(SIGTERM, [](int) { if (g_svr) g_svr->stop(); });
+
 	auto port = std::stoi(argv[2]);
 	std::cout << "Starting server on port " << port << std::endl;
 	svr.listen("0.0.0.0", port);
+	std::cout << "Server stopped" << std::endl;
 	return 0;
 }

publish.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 set -e
 
-version=1.0.3
-image_name="mariozechner/fasttext-lid-server"
+version=1.0.6
+image_name="suspiciousactivity/fasttext-lid-server"
 
 # Check git status
 if [ -n "$(git status --porcelain)" ]; then
@@ -19,18 +19,12 @@ fi
 docker build --platform linux/amd64 -t ${image_name}:${version}-amd64 .
 docker push ${image_name}:${version}-amd64
 
-# Build ARM64
-docker build --platform linux/arm64 -t ${image_name}:${version}-arm64 .
-docker push ${image_name}:${version}-arm64
-
 # Create and push manifest
 docker manifest create ${image_name}:${version} \
-    ${image_name}:${version}-amd64 \
-    ${image_name}:${version}-arm64
+    ${image_name}:${version}-amd64
 
 docker manifest create ${image_name}:latest \
-    ${image_name}:${version}-amd64 \
-    ${image_name}:${version}-arm64
+    ${image_name}:${version}-amd64
 
 docker manifest push ${image_name}:${version}
 docker manifest push ${image_name}:latest