1.0.6

CMakeLists.txt

@@ -24,8 +24,12 @@ FetchContent_MakeAvailable(httplib)
 
 set(CMAKE_BUILD_TYPE Release)
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+target_compile_options(fasttext-static PRIVATE
+    -march=x86-64 -mno-avx512f)
 
 add_executable(fasttext_lid_server main.cpp)
+target_compile_options(fasttext_lid_server PRIVATE
+    -march=x86-64 -mno-avx512f)
 target_include_directories(fasttext_lid_server PRIVATE ${fasttext_SOURCE_DIR}/src)
 
 target_link_libraries(fasttext_lid_server

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim AS builder
+FROM debian:bookworm-slim AS builder
 
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -14,14 +14,15 @@ RUN mkdir build-docker \
     && cmake -DCMAKE_BUILD_TYPE=Release .. \
     && make -j$(nproc)
 
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y \
     libstdc++6 \
+    curl \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 COPY --from=builder /app/build-docker/fasttext_lid_server .
-COPY lid.176.ftz .
+RUN curl --remote-name https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
 
-CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.ftz} ${PORT:-8080}"]
+CMD ["sh", "-c", "exec ./fasttext_lid_server ${MODEL_FILE:-lid.176.bin} ${PORT:-8080}"]

README.md

@@ -1,194 +1,16 @@
+## This is a fork
+
+Forked from [badlogic/fasttext-lid-server](https://github.com/badlogic/fasttext-lid-server/), this repository uses a newer debian version (bookworm), uses the [bigger model](https://fasttext.cc/docs/en/language-identification) (126MB) and gives you the top 3 results instead of only 1.
+
 # FastText Language ID Server
 
-HTTP server that provides language identification using [FastText language identification models](https://fasttext.cc/docs/en/language-identification.html). The default build comes with lid.176.ftz built-in and supports 176 languages:
+HTTP server that provides language identification using FastText models.
-
-| Code | Language |
-|--------|-----------|
-| af | Afrikaans |
-| als | Alemannic |
-| am | Amharic |
-| an | Aragonese |
-| ar | Arabic |
-| arz | Egyptian Arabic |
-| as | Assamese |
-| ast | Asturian |
-| av | Avaric |
-| az | Azerbaijani |
-| azb | South Azerbaijani |
-| ba | Bashkir |
-| bar | Bavarian |
-| bcl | Central Bikol |
-| be | Belarusian |
-| bg | Bulgarian |
-| bh | Bhojpuri |
-| bn | Bengali |
-| bo | Tibetan |
-| bpy | Bishnupriya |
-| br | Breton |
-| bs | Bosnian |
-| bxr | Buryat |
-| ca | Catalan |
-| cbk | Chavacano |
-| ce | Chechen |
-| ceb | Cebuano |
-| ckb | Central Kurdish |
-| co | Corsican |
-| cs | Czech |
-| cv | Chuvash |
-| cy | Welsh |
-| da | Danish |
-| de | German |
-| diq | Zazaki |
-| dsb | Lower Sorbian |
-| dty | Doteli |
-| dv | Divehi |
-| el | Greek |
-| eml | Emilian-Romagnol |
-| en | English |
-| eo | Esperanto |
-| es | Spanish |
-| et | Estonian |
-| eu | Basque |
-| fa | Persian |
-| fi | Finnish |
-| fr | French |
-| frr | North Frisian |
-| fy | West Frisian |
-| ga | Irish |
-| gd | Scottish Gaelic |
-| gl | Galician |
-| gn | Guarani |
-| gom | Goan Konkani |
-| gu | Gujarati |
-| gv | Manx |
-| he | Hebrew |
-| hi | Hindi |
-| hif | Fiji Hindi |
-| hr | Croatian |
-| hsb | Upper Sorbian |
-| ht | Haitian Creole |
-| hu | Hungarian |
-| hy | Armenian |
-| ia | Interlingua |
-| id | Indonesian |
-| ie | Interlingue |
-| ilo | Iloko |
-| io | Ido |
-| is | Icelandic |
-| it | Italian |
-| ja | Japanese |
-| jbo | Lojban |
-| jv | Javanese |
-| ka | Georgian |
-| kk | Kazakh |
-| km | Khmer |
-| kn | Kannada |
-| ko | Korean |
-| krc | Karachay-Balkar |
-| ku | Kurdish |
-| kv | Komi |
-| kw | Cornish |
-| ky | Kyrgyz |
-| la | Latin |
-| lb | Luxembourgish |
-| lez | Lezgian |
-| li | Limburgish |
-| lmo | Lombard |
-| lo | Lao |
-| lrc | Northern Luri |
-| lt | Lithuanian |
-| lv | Latvian |
-| mai | Maithili |
-| mg | Malagasy |
-| mhr | Eastern Mari |
-| min | Minangkabau |
-| mk | Macedonian |
-| ml | Malayalam |
-| mn | Mongolian |
-| mr | Marathi |
-| mrj | Western Mari |
-| ms | Malay |
-| mt | Maltese |
-| mwl | Mirandese |
-| my | Burmese |
-| myv | Erzya |
-| mzn | Mazanderani |
-| nah | Nahuatl |
-| nap | Neapolitan |
-| nds | Low German |
-| ne | Nepali |
-| new | Newari |
-| nl | Dutch |
-| nn | Norwegian Nynorsk |
-| no | Norwegian |
-| oc | Occitan |
-| or | Odia |
-| os | Ossetian |
-| pa | Punjabi |
-| pam | Pampanga |
-| pfl | Palatine German |
-| pl | Polish |
-| pms | Piedmontese |
-| pnb | Western Punjabi |
-| ps | Pashto |
-| pt | Portuguese |
-| qu | Quechua |
-| rm | Romansh |
-| ro | Romanian |
-| ru | Russian |
-| rue | Rusyn |
-| sa | Sanskrit |
-| sah | Sakha |
-| sc | Sardinian |
-| scn | Sicilian |
-| sco | Scots |
-| sd | Sindhi |
-| sh | Serbo-Croatian |
-| si | Sinhala |
-| sk | Slovak |
-| sl | Slovenian |
-| so | Somali |
-| sq | Albanian |
-| sr | Serbian |
-| su | Sundanese |
-| sv | Swedish |
-| sw | Swahili |
-| ta | Tamil |
-| te | Telugu |
-| tg | Tajik |
-| th | Thai |
-| tk | Turkmen |
-| tl | Tagalog |
-| tr | Turkish |
-| tt | Tatar |
-| tyv | Tuvan |
-| ug | Uyghur |
-| uk | Ukrainian |
-| ur | Urdu |
-| uz | Uzbek |
-| vec | Venetian |
-| vep | Veps |
-| vi | Vietnamese |
-| vls | West Flemish |
-| vo | Volapük |
-| wa | Walloon |
-| war | Waray |
-| wuu | Wu Chinese |
-| xal | Kalmyk |
-| xmf | Mingrelian |
-| yi | Yiddish |
-| yo | Yoruba |
-| yue | Cantonese |
-| zh | Chinese |
 
 ## Quick Start
 
 ```bash
-# Run with default settings
+# Pull and run
-docker compose up
+docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server
-
-# Run with custom port
-PORT=9000 docker compose up
 
 # Test
 curl "http://localhost:8080/detect?text=Hello%20world"
@@ -200,47 +22,38 @@ curl "http://localhost:8080/detect?text=Hello%20world"
 - **Parameter**: `text` (required)
 - **Response**: JSON with language code and probability
 ```json
-{
+[
+	{
 		"language": "en",
 		"probability": 0.987
-}
+	},
+	{
+		"language": "de",
+		"probability": 0.234
+	},
+	{
+		"language": "nl",
+		"probability": 0.123
+	}
+]
 ```
 - **Errors**:
   - 400: Missing text parameter
   - 500: Prediction failed
 
-## Building
-
-Prerequisites: CMake 3.14+, C++17 compiler
-
-```bash
-mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-make
-```
-
-## Running
-
-```bash
-./fasttext_lid_server [model_path] [port]
-```
-
 ## Docker
 
 ### Using docker run
 ```bash
-# Build
+# Default settings
-docker build -t fasttext-lid-server .
+docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server
 
-# Run with defaults
+# Custom settings
-docker run -p 8080:8080 fasttext-lid-server
-
-# Run with custom port and local model file
 docker run -p 9000:9000 \
   -e PORT=9000 \
   -e MODEL_FILE=/models/custom.ftz \
   -v $(pwd)/models:/models \
-  fasttext-lid-server
+  suspiciousactivity/fasttext-lid-server
 ```
 
 ### Using docker compose
@@ -249,14 +62,19 @@ version: '3.8'
 
 services:
   lid-server:
-    build: .
+    image: suspiciousactivity/fasttext-lid-server
     ports:
         - "${PORT:-8080}:${PORT:-8080}"
     environment:
         - MODEL_FILE=${MODEL_FILE:-lid.176.ftz}
         - PORT=${PORT:-8080}
     volumes:
-      - ./models:/models  # Optional: mount local models directory
+        - ./models:/models
+    healthcheck:
+        test: ["CMD", "curl", "-f", "http://localhost:${PORT:-8080}/detect?text=Hello%20world"]
+        interval: 5s
+        timeout: 5s
+        retries: 3
 ```
 
 Run with:
@@ -270,3 +88,18 @@ PORT=9000 docker compose up
 # Custom model
 MODEL_FILE=/models/custom.ftz docker compose up
 ```
+
+## Building from Source
+
+Prerequisites: CMake 3.14+, C++17 compiler
+
+```bash
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make
+```
+
+Run:
+```bash
+./fasttext_lid_server [model_path] [port]
+```

main.cpp

@@ -3,6 +3,9 @@
 #include <string>
 #include <iostream>
 #include <sstream>
+#include <csignal>
+
+httplib::Server *g_svr = nullptr;
 
 std::string stripLabel(const std::string &label) {
 	const std::string prefix = "__label__";
@@ -33,7 +36,7 @@ int main(int argc, char *argv[]) {
 		auto text = req.get_param_value("text");
 		std::istringstream iss(text);
 		std::vector<std::pair<fasttext::real, std::string>> predictions;
-		ft.predictLine(iss, predictions, 1, 0.0);
+		ft.predictLine(iss, predictions, 3, 0.0);
 
 		if (predictions.empty()) {
 			res.status = 500;
@@ -41,13 +44,31 @@ int main(int argc, char *argv[]) {
 			return;
 		}
 
-		std::string result = "{\"language\":\"" + stripLabel(predictions[0].second) +
+		std::string result = "[";
-							 "\",\"probability\":" + std::to_string(predictions[0].first) + "}";
+
+		bool first = true;
+		for (const auto& prediction : predictions) {
+			std::string predictionResult = "{\"language\":\"" + stripLabel(prediction.second) +
+				"\",\"probability\":" + std::to_string(prediction.first) + "}";
+			if (!first) {
+				result += ",";
+			}
+			result += predictionResult;
+			first = false;
+		}
+
+		result += "]";
+		
 		res.set_content(result, "application/json");
 	});
 
+	g_svr = &svr;
+	std::signal(SIGINT,  [](int) { if (g_svr) g_svr->stop(); });
+	std::signal(SIGTERM, [](int) { if (g_svr) g_svr->stop(); });
+
 	auto port = std::stoi(argv[2]);
 	std::cout << "Starting server on port " << port << std::endl;
 	svr.listen("0.0.0.0", port);
+	std::cout << "Server stopped" << std::endl;
 	return 0;
 }

publish.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 set -e
 
-version=1.0.0
+version=1.0.6
+image_name="suspiciousactivity/fasttext-lid-server"
 
+# Check git status
 if [ -n "$(git status --porcelain)" ]; then
     echo "Uncommitted changes exist"
     exit 1
@@ -13,12 +15,20 @@ if git rev-parse "v${version}" >/dev/null 2>&1; then
     exit 1
 fi
 
-docker build --no-cache -t fasttext-lid-server .
+# Build AMD64
+docker build --platform linux/amd64 -t ${image_name}:${version}-amd64 .
+docker push ${image_name}:${version}-amd64
 
+# Create and push manifest
+docker manifest create ${image_name}:${version} \
+    ${image_name}:${version}-amd64
+
+docker manifest create ${image_name}:latest \
+    ${image_name}:${version}-amd64
+
+docker manifest push ${image_name}:${version}
+docker manifest push ${image_name}:latest
+
+# Tag git release
 git tag "v${version}"
 git push origin "v${version}"
-
-docker tag fasttext-lid-server mariozechner/fasttext-lid-server:latest
-docker push mariozechner/fasttext-lid-server:latest
-docker tag fasttext-lid-server mariozechner/fasttext-lid-server:${version}
-docker push mariozechner/fasttext-lid-server:${version}