EaZyCode/fasttext-lid-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit.

Browse Source
This commit is contained in:
Mario Zechner committed 2024-12-02 16:17:18 +01:00
commit b6520082e7
7 files changed
+407
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1
View File
@@ -0,0 +1 @@
build
CMakeLists.txt
+35
View File
@@ -0,0 +1,35 @@
cmake_minimum_required(VERSION 3.14)
project(fasttext_server)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
find_package(Threads REQUIRED)
include(FetchContent)
FetchContent_Declare(
fastText
GIT_REPOSITORY https://github.com/facebookresearch/fastText.git
GIT_TAG v0.9.2
)
FetchContent_MakeAvailable(fastText)
FetchContent_Declare(
httplib
GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git
GIT_TAG v0.14.1
)
FetchContent_MakeAvailable(httplib)
set(CMAKE_BUILD_TYPE Release)
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
add_executable(fasttext_lid_server main.cpp)
target_include_directories(fasttext_lid_server PRIVATE ${fasttext_SOURCE_DIR}/src)
target_link_libraries(fasttext_lid_server
fasttext-static
httplib::httplib
Threads::Threads
)
Dockerfile
+27
View File
@@ -0,0 +1,27 @@
FROM debian:bullseye-slim AS builder
RUN apt-get update && apt-get install -y \
build-essential \
cmake \
git \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY . .
RUN mkdir build-docker \
&& cd build-docker \
&& cmake -DCMAKE_BUILD_TYPE=Release .. \
&& make -j$(nproc)
FROM debian:bullseye-slim
RUN apt-get update && apt-get install -y \
libstdc++6 \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=builder /app/build-docker/fasttext_lid_server .
COPY lid.176.ftz .
CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.ftz} ${PORT:-8080}"]
README.md
+272
View File
@@ -0,0 +1,272 @@
# FastText Language ID Server
HTTP server that provides language identification using [FastText language identification models](https://fasttext.cc/docs/en/language-identification.html). The default build comes with lid.176.ftz built-in and supports 176 languages:
| Code | Language |
|--------|-----------|
| af | Afrikaans |
| als | Alemannic |
| am | Amharic |
| an | Aragonese |
| ar | Arabic |
| arz | Egyptian Arabic |
| as | Assamese |
| ast | Asturian |
| av | Avaric |
| az | Azerbaijani |
| azb | South Azerbaijani |
| ba | Bashkir |
| bar | Bavarian |
| bcl | Central Bikol |
| be | Belarusian |
| bg | Bulgarian |
| bh | Bhojpuri |
| bn | Bengali |
| bo | Tibetan |
| bpy | Bishnupriya |
| br | Breton |
| bs | Bosnian |
| bxr | Buryat |
| ca | Catalan |
| cbk | Chavacano |
| ce | Chechen |
| ceb | Cebuano |
| ckb | Central Kurdish |
| co | Corsican |
| cs | Czech |
| cv | Chuvash |
| cy | Welsh |
| da | Danish |
| de | German |
| diq | Zazaki |
| dsb | Lower Sorbian |
| dty | Doteli |
| dv | Divehi |
| el | Greek |
| eml | Emilian-Romagnol |
| en | English |
| eo | Esperanto |
| es | Spanish |
| et | Estonian |
| eu | Basque |
| fa | Persian |
| fi | Finnish |
| fr | French |
| frr | North Frisian |
| fy | West Frisian |
| ga | Irish |
| gd | Scottish Gaelic |
| gl | Galician |
| gn | Guarani |
| gom | Goan Konkani |
| gu | Gujarati |
| gv | Manx |
| he | Hebrew |
| hi | Hindi |
| hif | Fiji Hindi |
| hr | Croatian |
| hsb | Upper Sorbian |
| ht | Haitian Creole |
| hu | Hungarian |
| hy | Armenian |
| ia | Interlingua |
| id | Indonesian |
| ie | Interlingue |
| ilo | Iloko |
| io | Ido |
| is | Icelandic |
| it | Italian |
| ja | Japanese |
| jbo | Lojban |
| jv | Javanese |
| ka | Georgian |
| kk | Kazakh |
| km | Khmer |
| kn | Kannada |
| ko | Korean |
| krc | Karachay-Balkar |
| ku | Kurdish |
| kv | Komi |
| kw | Cornish |
| ky | Kyrgyz |
| la | Latin |
| lb | Luxembourgish |
| lez | Lezgian |
| li | Limburgish |
| lmo | Lombard |
| lo | Lao |
| lrc | Northern Luri |
| lt | Lithuanian |
| lv | Latvian |
| mai | Maithili |
| mg | Malagasy |
| mhr | Eastern Mari |
| min | Minangkabau |
| mk | Macedonian |
| ml | Malayalam |
| mn | Mongolian |
| mr | Marathi |
| mrj | Western Mari |
| ms | Malay |
| mt | Maltese |
| mwl | Mirandese |
| my | Burmese |
| myv | Erzya |
| mzn | Mazanderani |
| nah | Nahuatl |
| nap | Neapolitan |
| nds | Low German |
| ne | Nepali |
| new | Newari |
| nl | Dutch |
| nn | Norwegian Nynorsk |
| no | Norwegian |
| oc | Occitan |
| or | Odia |
| os | Ossetian |
| pa | Punjabi |
| pam | Pampanga |
| pfl | Palatine German |
| pl | Polish |
| pms | Piedmontese |
| pnb | Western Punjabi |
| ps | Pashto |
| pt | Portuguese |
| qu | Quechua |
| rm | Romansh |
| ro | Romanian |
| ru | Russian |
| rue | Rusyn |
| sa | Sanskrit |
| sah | Sakha |
| sc | Sardinian |
| scn | Sicilian |
| sco | Scots |
| sd | Sindhi |
| sh | Serbo-Croatian |
| si | Sinhala |
| sk | Slovak |
| sl | Slovenian |
| so | Somali |
| sq | Albanian |
| sr | Serbian |
| su | Sundanese |
| sv | Swedish |
| sw | Swahili |
| ta | Tamil |
| te | Telugu |
| tg | Tajik |
| th | Thai |
| tk | Turkmen |
| tl | Tagalog |
| tr | Turkish |
| tt | Tatar |
| tyv | Tuvan |
| ug | Uyghur |
| uk | Ukrainian |
| ur | Urdu |
| uz | Uzbek |
| vec | Venetian |
| vep | Veps |
| vi | Vietnamese |
| vls | West Flemish |
| vo | Volapük |
| wa | Walloon |
| war | Waray |
| wuu | Wu Chinese |
| xal | Kalmyk |
| xmf | Mingrelian |
| yi | Yiddish |
| yo | Yoruba |
| yue | Cantonese |
| zh | Chinese |
## Quick Start
```bash
# Run with default settings
docker compose up
# Run with custom port
PORT=9000 docker compose up
# Test
curl "http://localhost:8080/detect?text=Hello%20world"
```
## API
### GET /detect
- **Parameter**: `text` (required)
- **Response**: JSON with language code and probability
```json
{
"language": "en",
"probability": 0.987
}
```
- **Errors**:
- 400: Missing text parameter
- 500: Prediction failed
## Building
Prerequisites: CMake 3.14+, C++17 compiler
```bash
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make
```
## Running
```bash
./fasttext_lid_server [model_path] [port]
```
## Docker
### Using docker run
```bash
# Build
docker build -t fasttext-lid-server .
# Run with defaults
docker run -p 8080:8080 fasttext-lid-server
# Run with custom port and local model file
docker run -p 9000:9000 \
-e PORT=9000 \
-e MODEL_FILE=/models/custom.ftz \
-v $(pwd)/models:/models \
fasttext-lid-server
```
### Using docker compose
```yaml
version: '3.8'
services:
lid-server:
build: .
ports:
- "${PORT:-8080}:${PORT:-8080}"
environment:
- MODEL_FILE=${MODEL_FILE:-lid.176.ftz}
- PORT=${PORT:-8080}
volumes:
- ./models:/models # Optional: mount local models directory
```
Run with:
```bash
# Default settings
docker compose up
# Custom port
PORT=9000 docker compose up
# Custom model
MODEL_FILE=/models/custom.ftz docker compose up
```
lid.176.ftz
BIN
View File
Binary file not shown.
main.cpp
+53
View File
@@ -0,0 +1,53 @@
#include <httplib.h>
#include "fasttext.h"
#include <string>
#include <iostream>
#include <sstream>
std::string stripLabel(const std::string &label) {
const std::string prefix = "__label__";
if (label.substr(0, prefix.length()) == prefix) {
return label.substr(prefix.length());
}
return label;
}
int main(int argc, char *argv[]) {
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <model_path> <port>" << std::endl;
return 1;
}
fasttext::FastText ft;
ft.loadModel(argv[1]);
httplib::Server svr;
svr.Get("/detect", [&ft](const httplib::Request &req, httplib::Response &res) {
if (!req.has_param("text")) {
res.status = 400;
res.set_content("Missing text parameter", "text/plain");
return;
}
auto text = req.get_param_value("text");
std::istringstream iss(text);
std::vector<std::pair<fasttext::real, std::string>> predictions;
ft.predictLine(iss, predictions, 1, 0.0);
if (predictions.empty()) {
res.status = 500;
res.set_content("Prediction failed", "text/plain");
return;
}
std::string result = "{\"language\":\"" + stripLabel(predictions[0].second) +
"\",\"probability\":" + std::to_string(predictions[0].first) + "}";
res.set_content(result, "application/json");
});
auto port = std::stoi(argv[2]);
std::cout << "Starting server on port " << port << std::endl;
svr.listen("0.0.0.0", port);
return 0;
}
publish.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -e
version=1.0.0
if git rev-parse "v${version}" >/dev/null 2>&1; then
echo "Tag v${version} already exists"
exit 1
fi
docker build --no-cache -t fasttext-lid-server .
git tag "v${version}"
git push origin "v${version}"
docker tag fasttext-lid-server mariozechner/fasttext-lid-server:latest
docker push mariozechner/fasttext-lid-server:latest
docker tag fasttext-lid-server mariozechner/fasttext-lid-server:${version}
docker push mariozechner/fasttext-lid-server:${version}