diff --git a/Dockerfile b/Dockerfile index 4fba385..53cf6a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS builder +FROM debian:bookworm-slim AS builder RUN apt-get update && apt-get install -y \ build-essential \ @@ -14,7 +14,7 @@ RUN mkdir build-docker \ && cmake -DCMAKE_BUILD_TYPE=Release .. \ && make -j$(nproc) -FROM debian:bullseye-slim +FROM debian:bookworm-slim RUN apt-get update && apt-get install -y \ libstdc++6 \ @@ -23,6 +23,6 @@ RUN apt-get update && apt-get install -y \ WORKDIR /app COPY --from=builder /app/build-docker/fasttext_lid_server . -COPY lid.176.ftz . +RUN curl --remote-name https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.ftz} ${PORT:-8080}"] \ No newline at end of file +CMD ["sh", "-c", "./fasttext_lid_server ${MODEL_FILE:-lid.176.bin} ${PORT:-8080}"] \ No newline at end of file diff --git a/README.md b/README.md index 195eb89..47858d9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +## This is a fork + +Forked from [badlogic/fasttext-lid-server](https://github.com/badlogic/fasttext-lid-server/), this repository uses a newer debian version (bookworm), uses the [bigger model](https://fasttext.cc/docs/en/language-identification) (126MB) and gives you the top 3 results instead of only 1. + # FastText Language ID Server HTTP server that provides language identification using FastText models. @@ -6,7 +10,7 @@ HTTP server that provides language identification using FastText models. ```bash # Pull and run -docker run -p 8080:8080 mariozechner/fasttext-lid-server +docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server # Test curl "http://localhost:8080/detect?text=Hello%20world" @@ -18,10 +22,20 @@ curl "http://localhost:8080/detect?text=Hello%20world" - **Parameter**: `text` (required) - **Response**: JSON with language code and probability ```json -{ - "language": "en", - "probability": 0.987 -} +[ + { + "language": "en", + "probability": 0.987 + }, + { + "language": "de", + "probability": 0.234 + }, + { + "language": "nl", + "probability": 0.123 + } +] ``` - **Errors**: - 400: Missing text parameter @@ -32,14 +46,14 @@ curl "http://localhost:8080/detect?text=Hello%20world" ### Using docker run ```bash # Default settings -docker run -p 8080:8080 mariozechner/fasttext-lid-server +docker run -p 8080:8080 suspiciousactivity/fasttext-lid-server # Custom settings docker run -p 9000:9000 \ -e PORT=9000 \ -e MODEL_FILE=/models/custom.ftz \ -v $(pwd)/models:/models \ - mariozechner/fasttext-lid-server + suspiciousactivity/fasttext-lid-server ``` ### Using docker compose @@ -48,7 +62,7 @@ version: '3.8' services: lid-server: - image: mariozechner/fasttext-lid-server + image: suspiciousactivity/fasttext-lid-server ports: - "${PORT:-8080}:${PORT:-8080}" environment: diff --git a/lid.176.ftz b/lid.176.ftz deleted file mode 100644 index 1fb85b3..0000000 Binary files a/lid.176.ftz and /dev/null differ diff --git a/main.cpp b/main.cpp index cdc86d6..df73f12 100644 --- a/main.cpp +++ b/main.cpp @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) { auto text = req.get_param_value("text"); std::istringstream iss(text); std::vector> predictions; - ft.predictLine(iss, predictions, 1, 0.0); + ft.predictLine(iss, predictions, 3, 0.0); if (predictions.empty()) { res.status = 500; @@ -41,8 +41,21 @@ int main(int argc, char *argv[]) { return; } - std::string result = "{\"language\":\"" + stripLabel(predictions[0].second) + - "\",\"probability\":" + std::to_string(predictions[0].first) + "}"; + std::string result = "["; + + bool first = true; + for (const auto& prediction : predictions) { + std::string predictionResult = "{\"language\":\"" + stripLabel(prediction.second) + + "\",\"probability\":" + std::to_string(prediction.first) + "}"; + if (!first) { + result += ","; + } + result += predictionResult; + first = false; + } + + result += "]"; + res.set_content(result, "application/json"); }); diff --git a/publish.sh b/publish.sh index b0cac72..ded18e6 100755 --- a/publish.sh +++ b/publish.sh @@ -1,8 +1,8 @@ #!/bin/sh set -e -version=1.0.3 -image_name="mariozechner/fasttext-lid-server" +version=1.0.0 +image_name="suspiciousactivity/fasttext-lid-server" # Check git status if [ -n "$(git status --porcelain)" ]; then