XLMR Document Classification Server + Console (facebookresearch#1358)

Summary:
Pull Request resolved: facebookresearch#1358

This diff adds open source support for our XLM-R document classification server along with a simple console to visualize instances and provide additional annotations.

Reviewed By: geof90

Differential Revision: D21482314

fbshipit-source-id: e5e3fd5695f17876ec5db5329d0875b5a8c2afd5

Author: Akshat Shrivastava

demo/xlm_server/README.md

@@ -0,0 +1,35 @@
+# TorchScript Server for XLM-R
+In this directory, we provide a model server and a client console for DocNN and XLM-R text classifications models. These models were trained with PyTorch and exported to TorchScript
+
+
+## Server
+For monolingual DocNN,
+```
+$ mkdir build && cd build
+$ # Copy the downloaded models into this directory
+$ echo -e 'FROM pytext/predictor_service_torchscript:who\nCOPY *.torchscript /app/\nCMD ["./server","mono.model.pt.torchscript"]' >> Dockerfile
+$ docker build -t server .
+$ docker run -it -p 8080:8080 server
+$ curl -d '{"text": "hi"}' -H 'Content-Type: application/json' localhost:8080
+```
+For multilingual XLM-R,
+```
+echo -e 'FROM pytext/predictor_service_torchscript:who\nCOPY *.torchscript /app/\nCMD ["./server","multi.model.pt.torchscript", "multi.vocab.model.pt.torchscript"]' >> Dockerfile
+```
+
+
+## Console
+The console provides a front-end webpage to view the predictions of the classification model. It also allows for exploring model predictions interactively, and for logging corrections back to the server.
+
+### Console Setup
+
+```
+$ python3 -m venv env
+$ source env/bin/activate
+$ (env) pip install -r requirements.txt
+```
+
+running the server:
+```
+$ python server.py --modelserver http://localhost:8080
+```

demo/xlm_server/console/requirements.txt

@@ -0,0 +1,20 @@
+aniso8601==8.0.0
+certifi==2020.4.5.1
+chardet==3.0.4
+click==7.1.1
+Flask==1.1.2
+Flask-Cors==3.0.8
+Flask-RESTful==0.3.8
+idna==2.9
+itsdangerous==1.1.0
+Jinja2==2.11.1
+MarkupSafe==1.1.1
+numpy==1.18.2
+pandas==1.0.3
+python-dateutil==2.8.1
+pytz==2019.3
+requests==2.23.0
+six==1.14.0
+tqdm==4.45.0
+urllib3==1.25.8
+Werkzeug==1.0.1

demo/xlm_server/console/server.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import argparse
+import csv
+import json
+import logging
+import threading
+from typing import Any, Dict, Tuple
+
+import requests
+from flask import Blueprint, Flask, jsonify, render_template, request
+from flask_cors import CORS
+
+
+# a lock for writing to the output
+# data file
+DATA_FILE_LOCK = threading.Lock()
+
+
+def get_args() -> argparse.ArgumentParser:
+    """
+    Return CLI configuration for running server
+    """
+    parser = argparse.ArgumentParser(description="Run the html hosting server")
+    parser.add_argument(
+        "--port", default=5000, type=int, help="the port to launch server on"
+    )
+
+    parser.add_argument("--modelserver", default="http://localhost:8080")
+
+    parser.add_argument(
+        "--debug", action="store_true", help="Launch debug version of server"
+    )
+
+    parser.add_argument(
+        "--datafile", default="data.csv", help="the file to write data to"
+    )
+
+    parser.add_argument(
+        "--console_address",
+        default="http://localhost:5000",
+        help="the location of the console server",
+    )
+    return parser
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_key_from_data(data, key):
+    if key in data:
+        return data[key]
+    return None
+
+
+def create_app(config_filename: str):
+    api_bp = Blueprint("api", __name__)
+    app = Flask(__name__, static_url_path="/static")
+    CORS(app, resources=r"/api/*")
+
+    app.register_blueprint(api_bp, url_prefix="/api")
+
+    return app
+
+
+def setup_app(app: Flask, args: object):
+    @app.route("/api/model/", methods=["GET"])
+    def get_predictions():
+        query = get_key_from_data(request.args, "query")
+        payload = {"text": str(query)}
+        r = requests.post(args.modelserver, json=payload)
+        response = json.loads(r.text)
+        intent_scores = response["intent_ranking"]
+
+        intent_scores = list(filter(lambda inp: inp, intent_scores))
+
+        def convert_to_tuple(intent_score: Dict[str, Any]) -> Tuple[str, float]:
+            intent_name = intent_score["name"]
+            intent_score = intent_score["confidence"]
+            return intent_name, intent_score
+
+        intent_scores = list(map(convert_to_tuple, intent_scores))
+        intent_scores = sorted(intent_scores, reverse=True, key=lambda tup: tup[1])
+        return (
+            jsonify(
+                {
+                    "query": query,
+                    "prediction": intent_scores[0],
+                    "raw_scores": intent_scores,
+                }
+            ),
+            200,
+        )
+
+    @app.route("/api/add_data/", methods=["GET"])
+    def upload_data():
+        data_point = get_key_from_data(request.args, "data_point")
+        query, label = data_point.split(",")
+        with DATA_FILE_LOCK:
+            with open(args.datafile, "a+") as csv_file:
+                csvwriter = csv.writer(csv_file, delimiter="\t")
+                csvwriter.writerow([query, label])
+
+        return jsonify({"query": query, "label": label}), 200
+
+    @app.route("/", methods=["GET"])
+    def root():
+        return render_template("index.html", console_address=args.console_address)
+
+
+def main():
+    args = get_args().parse_args()
+
+    app = create_app("config")
+    setup_app(app, args)
+    try:
+        app.run(host="0.0.0.0", debug=args.debug, port=args.port)
+    except KeyboardInterrupt:
+        print("Received Keyboard interrupt, exiting")
+
+
+if __name__ == "__main__":
+    main()

demo/xlm_server/console/templates/index.html

@@ -0,0 +1,159 @@
+<doctype html>
+
+    <head>
+        <title>Document Classification Console</title>
+
+        <style>
+            .button {
+                margin-top: 20px;
+                height: 30px;
+                width: 100px;
+                font-size: 12pt;
+            }
+
+            #query {
+                height: 50px;
+                width: 700px;
+                font-size: 14pt;
+                padding-left: 10px;
+            }
+
+            #submit_button {
+                height: 50px;
+                width: 70px;
+                font-size: 14pt;
+            }
+
+            .selection_area {
+                padding: 20px;
+            }
+
+            input[type=radio]:checked+label {
+                color: red;
+            }
+        </style>
+
+        <script>
+            var PYTHON_SERVER_ADDRESS = "{{ console_address }}";
+            window.onload = function () {
+                function makeScoreString(raw_scores) {
+                    return raw_scores[0] + ": " + (raw_scores[1] * 100).toFixed(2) + "%";
+                }
+                function makeUL(query, array) {
+                    // Create the list element:
+                    var prediction_container = document.createElement('div');
+                    var list = document.createElement('ul');
+
+                    var header = document.createElement("h4");
+                    header.appendChild(document.createTextNode("Help us gather more data!"));
+                    prediction_container.appendChild(header);
+                    prediction_container.appendChild(document.createTextNode("Select the correct prediction from below and submit to provide us more data to learn from!"))
+
+                    // prediction_container.appendChild(list);
+
+                    var selection_area = document.createElement("div");
+                    selection_area.setAttribute("class", "selection_area");
+
+
+                    for (var i = 0; i < array.length; i++) {
+                        // Create the list item:
+                        var item = document.createElement('li');
+                        var intent_name = array[i][0];
+                        var intent_score = array[i][1];
+
+                        var choiceSelection = document.createElement('input')
+                        choiceSelection.setAttribute('type', 'radio');
+                        choiceSelection.setAttribute('name', 'choice');
+                        choiceSelection.setAttribute('value', query + "," + intent_name);
+                        choiceSelection.setAttribute('id', "choice_" + intent_name);
+
+                        var choiceLabel = document.createElement('label')
+                        choiceLabel.setAttribute('for', "choice_" + intent_name);
+                        choiceLabel.appendChild(document.createTextNode(makeScoreString(array[i])));
+
+                        selection_area.appendChild(choiceSelection);
+                        selection_area.appendChild(choiceLabel);
+                        selection_area.appendChild(document.createElement("br"));
+                    }
+
+                    // add submit button
+
+                    var submit_data_button = document.createElement("button");
+                    submit_data_button.setAttribute("type", "submit");
+                    submit_data_button.setAttribute("class", "button");
+                    submit_data_button.onclick = function () {
+                        var data_point = document.querySelector('input[name="choice"]:checked').value;
+                        var xhr = new XMLHttpRequest();
+                        xhr.onreadystatechange = function () {
+                            if (this.readyState == 4) {
+                                if (this.status == 200) {
+                                    var data = JSON.parse(xhr.responseText);
+                                    // Display the returned data in browser
+                                    var success_message = "The data point (query=" + data.query + ", label=" + data.label + ") has been added, thank you for your help!";
+                                    document.getElementById('raw_scores').innerHTML = success_message;
+                                } else {
+                                    console.error('Error: ' + this.status);
+                                }
+                            }
+                        };
+
+                        xhr.open('GET', PYTHON_SERVER_ADDRESS + 'api/add_data/?data_point=' + encodeURI(data_point));
+                        xhr.send();
+                    };
+
+                    submit_data_button.appendChild(document.createTextNode("Submit"));
+
+                    selection_area.appendChild(submit_data_button);
+                    prediction_container.appendChild(selection_area);
+                    return prediction_container;
+                }
+
+                // Add the contents of options[0] to #foo:
+                document.getElementById('submit_button').onclick = function () {
+                    var query_box_val = document.getElementById('query').value;
+                    var xhr = new XMLHttpRequest();
+                    xhr.onreadystatechange = function () {
+                        if (this.readyState == 4) {
+                            if (this.status == 200) {
+                                var data = JSON.parse(xhr.responseText);
+                                // Display the returned data in browser
+                                document.getElementById("best_prediction").innerHTML = makeScoreString(data.prediction);
+                                document.getElementById('raw_scores').innerHTML = "";
+                                document.getElementById('raw_scores').appendChild(makeUL(data.query, data.raw_scores));
+                            } else {
+                                console.error('Error: ' + this.status);
+                            }
+                        }
+                    };
+
+                    xhr.open('GET', PYTHON_SERVER_ADDRESS + "api/model/?query=" + encodeURI(query_box_val));
+                    xhr.send();
+                };
+
+                // Query on enter
+                var query_box = document.getElementById("query");
+                query_box.addEventListener("keyup", function (event) {
+                    // Number 13 is the "Enter" key on the keyboard
+                    if (event.keyCode === 13) {
+                        event.preventDefault();
+                        document.getElementById("submit_button").click();
+                    }
+                });
+            };
+        </script>
+    </head>
+
+    <body>
+
+        <h2 id="title">Document Classification Console</h2>
+        <input type="text" name="doc" id="query" value="Sample query">
+        <button type="submit" id="submit_button">Enter</button>
+
+        <br />
+        <h3>Top Model Prediction</h3>
+        <div id="best_prediction"></div>
+
+        <br />
+        <div id="raw_scores"></div>
+    </body>
+</doctype>