commit b6520082e71945d607837851a81452b489514fcc Author: Mario Zechner Date: Mon Dec 2 16:17:18 2024 +0100 Initial commit. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..378eac2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..081c2cf --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 3.14) +project(fasttext_server) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Threads REQUIRED) + +include(FetchContent) + +FetchContent_Declare( + fastText + GIT_REPOSITORY https://github.com/facebookresearch/fastText.git + GIT_TAG v0.9.2 +) +FetchContent_MakeAvailable(fastText) + +FetchContent_Declare( + httplib + GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git + GIT_TAG v0.14.1 +) +FetchContent_MakeAvailable(httplib) + +set(CMAKE_BUILD_TYPE Release) +set(CMAKE_CXX_FLAGS_RELEASE "-O3") + +add_executable(fasttext_lid_server main.cpp) +target_include_directories(fasttext_lid_server PRIVATE ${fasttext_SOURCE_DIR}/src) + +target_link_libraries(fasttext_lid_server + fasttext-static + httplib::httplib + Threads::Threads +) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f6dad0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:bullseye-slim AS builder + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY . . + +RUN mkdir build-docker \ + && cd build-docker \ + && cmake -DCMAKE_BUILD_TYPE=Release .. \ + && make -j$(nproc) + +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY --from=builder /app/build-docker/fasttext_lid_server . +COPY lid.176.ftz . + +CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.ftz} ${PORT:-8080}"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ed719e1 --- /dev/null +++ b/README.md @@ -0,0 +1,272 @@ +# FastText Language ID Server + +HTTP server that provides language identification using [FastText language identification models](https://fasttext.cc/docs/en/language-identification.html). The default build comes with lid.176.ftz built-in and supports 176 languages: + +| Code | Language | +|--------|-----------| +| af | Afrikaans | +| als | Alemannic | +| am | Amharic | +| an | Aragonese | +| ar | Arabic | +| arz | Egyptian Arabic | +| as | Assamese | +| ast | Asturian | +| av | Avaric | +| az | Azerbaijani | +| azb | South Azerbaijani | +| ba | Bashkir | +| bar | Bavarian | +| bcl | Central Bikol | +| be | Belarusian | +| bg | Bulgarian | +| bh | Bhojpuri | +| bn | Bengali | +| bo | Tibetan | +| bpy | Bishnupriya | +| br | Breton | +| bs | Bosnian | +| bxr | Buryat | +| ca | Catalan | +| cbk | Chavacano | +| ce | Chechen | +| ceb | Cebuano | +| ckb | Central Kurdish | +| co | Corsican | +| cs | Czech | +| cv | Chuvash | +| cy | Welsh | +| da | Danish | +| de | German | +| diq | Zazaki | +| dsb | Lower Sorbian | +| dty | Doteli | +| dv | Divehi | +| el | Greek | +| eml | Emilian-Romagnol | +| en | English | +| eo | Esperanto | +| es | Spanish | +| et | Estonian | +| eu | Basque | +| fa | Persian | +| fi | Finnish | +| fr | French | +| frr | North Frisian | +| fy | West Frisian | +| ga | Irish | +| gd | Scottish Gaelic | +| gl | Galician | +| gn | Guarani | +| gom | Goan Konkani | +| gu | Gujarati | +| gv | Manx | +| he | Hebrew | +| hi | Hindi | +| hif | Fiji Hindi | +| hr | Croatian | +| hsb | Upper Sorbian | +| ht | Haitian Creole | +| hu | Hungarian | +| hy | Armenian | +| ia | Interlingua | +| id | Indonesian | +| ie | Interlingue | +| ilo | Iloko | +| io | Ido | +| is | Icelandic | +| it | Italian | +| ja | Japanese | +| jbo | Lojban | +| jv | Javanese | +| ka | Georgian | +| kk | Kazakh | +| km | Khmer | +| kn | Kannada | +| ko | Korean | +| krc | Karachay-Balkar | +| ku | Kurdish | +| kv | Komi | +| kw | Cornish | +| ky | Kyrgyz | +| la | Latin | +| lb | Luxembourgish | +| lez | Lezgian | +| li | Limburgish | +| lmo | Lombard | +| lo | Lao | +| lrc | Northern Luri | +| lt | Lithuanian | +| lv | Latvian | +| mai | Maithili | +| mg | Malagasy | +| mhr | Eastern Mari | +| min | Minangkabau | +| mk | Macedonian | +| ml | Malayalam | +| mn | Mongolian | +| mr | Marathi | +| mrj | Western Mari | +| ms | Malay | +| mt | Maltese | +| mwl | Mirandese | +| my | Burmese | +| myv | Erzya | +| mzn | Mazanderani | +| nah | Nahuatl | +| nap | Neapolitan | +| nds | Low German | +| ne | Nepali | +| new | Newari | +| nl | Dutch | +| nn | Norwegian Nynorsk | +| no | Norwegian | +| oc | Occitan | +| or | Odia | +| os | Ossetian | +| pa | Punjabi | +| pam | Pampanga | +| pfl | Palatine German | +| pl | Polish | +| pms | Piedmontese | +| pnb | Western Punjabi | +| ps | Pashto | +| pt | Portuguese | +| qu | Quechua | +| rm | Romansh | +| ro | Romanian | +| ru | Russian | +| rue | Rusyn | +| sa | Sanskrit | +| sah | Sakha | +| sc | Sardinian | +| scn | Sicilian | +| sco | Scots | +| sd | Sindhi | +| sh | Serbo-Croatian | +| si | Sinhala | +| sk | Slovak | +| sl | Slovenian | +| so | Somali | +| sq | Albanian | +| sr | Serbian | +| su | Sundanese | +| sv | Swedish | +| sw | Swahili | +| ta | Tamil | +| te | Telugu | +| tg | Tajik | +| th | Thai | +| tk | Turkmen | +| tl | Tagalog | +| tr | Turkish | +| tt | Tatar | +| tyv | Tuvan | +| ug | Uyghur | +| uk | Ukrainian | +| ur | Urdu | +| uz | Uzbek | +| vec | Venetian | +| vep | Veps | +| vi | Vietnamese | +| vls | West Flemish | +| vo | Volapük | +| wa | Walloon | +| war | Waray | +| wuu | Wu Chinese | +| xal | Kalmyk | +| xmf | Mingrelian | +| yi | Yiddish | +| yo | Yoruba | +| yue | Cantonese | +| zh | Chinese | + +## Quick Start + +```bash +# Run with default settings +docker compose up + +# Run with custom port +PORT=9000 docker compose up + +# Test +curl "http://localhost:8080/detect?text=Hello%20world" +``` + +## API + +### GET /detect +- **Parameter**: `text` (required) +- **Response**: JSON with language code and probability +```json +{ + "language": "en", + "probability": 0.987 +} +``` +- **Errors**: + - 400: Missing text parameter + - 500: Prediction failed + +## Building + +Prerequisites: CMake 3.14+, C++17 compiler + +```bash +mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +make +``` + +## Running + +```bash +./fasttext_lid_server [model_path] [port] +``` + +## Docker + +### Using docker run +```bash +# Build +docker build -t fasttext-lid-server . + +# Run with defaults +docker run -p 8080:8080 fasttext-lid-server + +# Run with custom port and local model file +docker run -p 9000:9000 \ + -e PORT=9000 \ + -e MODEL_FILE=/models/custom.ftz \ + -v $(pwd)/models:/models \ + fasttext-lid-server +``` + +### Using docker compose +```yaml +version: '3.8' + +services: + lid-server: + build: . + ports: + - "${PORT:-8080}:${PORT:-8080}" + environment: + - MODEL_FILE=${MODEL_FILE:-lid.176.ftz} + - PORT=${PORT:-8080} + volumes: + - ./models:/models # Optional: mount local models directory +``` + +Run with: +```bash +# Default settings +docker compose up + +# Custom port +PORT=9000 docker compose up + +# Custom model +MODEL_FILE=/models/custom.ftz docker compose up +``` \ No newline at end of file diff --git a/lid.176.ftz b/lid.176.ftz new file mode 100644 index 0000000..1fb85b3 Binary files /dev/null and b/lid.176.ftz differ diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..cdc86d6 --- /dev/null +++ b/main.cpp @@ -0,0 +1,53 @@ +#include +#include "fasttext.h" +#include +#include +#include + +std::string stripLabel(const std::string &label) { + const std::string prefix = "__label__"; + if (label.substr(0, prefix.length()) == prefix) { + return label.substr(prefix.length()); + } + return label; +} + +int main(int argc, char *argv[]) { + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + fasttext::FastText ft; + ft.loadModel(argv[1]); + + httplib::Server svr; + + svr.Get("/detect", [&ft](const httplib::Request &req, httplib::Response &res) { + if (!req.has_param("text")) { + res.status = 400; + res.set_content("Missing text parameter", "text/plain"); + return; + } + + auto text = req.get_param_value("text"); + std::istringstream iss(text); + std::vector> predictions; + ft.predictLine(iss, predictions, 1, 0.0); + + if (predictions.empty()) { + res.status = 500; + res.set_content("Prediction failed", "text/plain"); + return; + } + + std::string result = "{\"language\":\"" + stripLabel(predictions[0].second) + + "\",\"probability\":" + std::to_string(predictions[0].first) + "}"; + res.set_content(result, "application/json"); + }); + + auto port = std::stoi(argv[2]); + std::cout << "Starting server on port " << port << std::endl; + svr.listen("0.0.0.0", port); + return 0; +} \ No newline at end of file diff --git a/publish.sh b/publish.sh new file mode 100755 index 0000000..1fcdf51 --- /dev/null +++ b/publish.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +version=1.0.0 + +if git rev-parse "v${version}" >/dev/null 2>&1; then + echo "Tag v${version} already exists" + exit 1 +fi + +docker build --no-cache -t fasttext-lid-server . + +git tag "v${version}" +git push origin "v${version}" + +docker tag fasttext-lid-server mariozechner/fasttext-lid-server:latest +docker push mariozechner/fasttext-lid-server:latest +docker tag fasttext-lid-server mariozechner/fasttext-lid-server:${version} +docker push mariozechner/fasttext-lid-server:${version} \ No newline at end of file