From 3046b1ee7b881db5cf5a4c11528f0ead9d7b9a3e Mon Sep 17 00:00:00 2001 From: Firq Date: Mon, 30 Sep 2024 20:40:26 +0200 Subject: [PATCH] Initial Version --- .forgejo/workflows/build_release.yml | 72 ++++++++++++++++++++++++ .forgejo/workflows/default.yml | 20 +++++++ .gitignore | 6 ++ Dockerfile | 5 ++ MANIFEST.in | 1 + README.md | 19 +++++++ pyproject.toml | 38 +++++++++++++ support_formatter/__init__.py | 2 + support_formatter/app.py | 38 +++++++++++++ support_formatter/config/__init__.py | 1 + support_formatter/config/settings.py | 35 ++++++++++++ support_formatter/logic/__init__.py | 0 support_formatter/logic/csv_processor.py | 44 +++++++++++++++ support_formatter/models/__init__.py | 0 support_formatter/models/interface.py | 19 +++++++ support_formatter/pages/error.html | 14 +++++ support_formatter/pages/index.html | 34 +++++++++++ support_formatter/routes/__init__.py | 7 +++ support_formatter/routes/interface.py | 38 +++++++++++++ support_formatter/routes/upload.py | 70 +++++++++++++++++++++++ support_formatter/server.py | 28 +++++++++ 21 files changed, 491 insertions(+) create mode 100644 .forgejo/workflows/build_release.yml create mode 100644 .forgejo/workflows/default.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 support_formatter/__init__.py create mode 100644 support_formatter/app.py create mode 100644 support_formatter/config/__init__.py create mode 100644 support_formatter/config/settings.py create mode 100644 support_formatter/logic/__init__.py create mode 100644 support_formatter/logic/csv_processor.py create mode 100644 support_formatter/models/__init__.py create mode 100644 support_formatter/models/interface.py create mode 100644 support_formatter/pages/error.html create mode 100644 support_formatter/pages/index.html create mode 100644 support_formatter/routes/__init__.py create mode 100644 support_formatter/routes/interface.py create mode 100644 support_formatter/routes/upload.py create mode 100644 support_formatter/server.py diff --git a/.forgejo/workflows/build_release.yml b/.forgejo/workflows/build_release.yml new file mode 100644 index 0000000..bc5e10d --- /dev/null +++ b/.forgejo/workflows/build_release.yml @@ -0,0 +1,72 @@ +on: + push: + tags: + - '[0-9]+\.[0-9]+\.[0-9]+-c\.[0-9]+' + - '[0-9]+\.[0-9]+\.[0-9]+-a\.[0-9]+' + - '[0-9]+\.[0-9]+\.[0-9]' + +jobs: + build-artifacts: + runs-on: docker + container: nikolaik/python-nodejs:python3.11-nodejs21 + steps: + - name: Checkout source code + uses: https://code.forgejo.org/actions/checkout@v3 + - name: Install packages + run: pip install build + - name: Build package + run: python -m build + - name: Save build artifacts + uses: https://code.forgejo.org/actions/upload-artifact@v3 + with: + name: packages + path: dist/* + + publish-artifacts: + needs: ["build-artifacts"] + runs-on: docker + container: nikolaik/python-nodejs:python3.11-nodejs21 + steps: + - name: Downloading static site artifacts + uses: actions/download-artifact@v3 + with: + name: packages + path: dist + - name: Install Dependencies + run: pip install twine + - name: Upload package to registry + run: python -m twine upload --repository-url ${{ secrets.REPOSITORY_URL }} -u ${{ secrets.TWINE_DEPLOY_USER }} -p ${{ secrets.TWINE_DEPLOY_PASSWORD }} dist/* + + build-and-push-container: + needs: [ "publish-artifacts" ] + runs-on: dind + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log into Docker Package Registry + uses: docker/login-action@v3 + with: + registry: forgejo.neshweb.net + username: ${{ secrets.FORGEJO_USERNAME }} + password: ${{ secrets.FORGEJO_TOKEN }} + - name: Build and push to Docker Package Registry + uses: docker/build-push-action@v5 + with: + build-args: | + PACKAGE_VERSION=${{ github.ref_name }} + push: true + tags: forgejo.neshweb.net/firq/support-formatter:${{ github.ref_name }} + + release: + needs: [ build-and-push-container, publish-artifacts ] + if: success() + runs-on: docker + steps: + - name: Release New Version + uses: actions/forgejo-release@v1 + with: + direction: upload + url: https://forgejo.neshweb.net + release-dir: release + token: ${{ secrets.FORGEJO_TOKEN }} + tag: ${{ github.ref_name }} \ No newline at end of file diff --git a/.forgejo/workflows/default.yml b/.forgejo/workflows/default.yml new file mode 100644 index 0000000..33a1b8e --- /dev/null +++ b/.forgejo/workflows/default.yml @@ -0,0 +1,20 @@ +on: + push: + branches: "**" + +jobs: + backend-pylint: + runs-on: docker + container: nikolaik/python-nodejs:python3.11-nodejs21 + steps: + - name: Checkout source code + uses: https://code.forgejo.org/actions/checkout@v3 + - name: Install packages + run: | + pip install -e . -q + python -m pip list --format=columns --disable-pip-version-check + pip install pylint~=2.17.7 --disable-pip-version-check -q + - name: Run pylint + run: | + pylint --version + pylint **/*.py --exit-zero diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9decf27 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv/ +.vscode/ +**/.temp/* +__pycache__/ +*.egg-info +dist/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..62b66cb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM forgejo.neshweb.net/ci-docker-images/python-neshweb:3.11 +ARG PACKAGE_VERSION=0.1.0 +RUN pip install support_formatter==${PACKAGE_VERSION} + +CMD [ "support-formatter" ] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..791a35b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include support_formatter/pages/*.html \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f1264f --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# support-organizer + +## Setup + +Set up the venv and requirements + +```shell +py -m venv .venv +.venv/Scripts/activate +pip install -e . +``` + +## Running + +Start the server using + +```shell +py -m support-formatter +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0379ac8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "support_formatter" +version = "0.1.0-a.1" +requires-python = ">= 3.10" +authors = [{name = "Firq", email = "firelp42@gmail.com"}] +maintainers = [{name = "Firq", email = "firelp42@gmail.com"}] +description = "Tool to manage requests for supports" +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Flask", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "flask~=2.3.3", + "flask-smorest~=0.42.1", + "marshmallow~=3.20.1", + "gevent~=23.9.1", +] + +[project.scripts] +support-formatter = "support_formatter.server:run_server" + +[tool.setuptools.packages.find] +where = ["."] +include = ["support_formatter*"] + +[tool.setuptools.package-data] +support_formatter = ["*.html"] + +[tool.pylint."MAIN"] +disable = [ "line-too-long", "missing-module-docstring" ] + +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" diff --git a/support_formatter/__init__.py b/support_formatter/__init__.py new file mode 100644 index 0000000..31866f2 --- /dev/null +++ b/support_formatter/__init__.py @@ -0,0 +1,2 @@ +import importlib.metadata +__version__ = importlib.metadata.version(__package__ or "support_formatter") diff --git a/support_formatter/app.py b/support_formatter/app.py new file mode 100644 index 0000000..4a3cab3 --- /dev/null +++ b/support_formatter/app.py @@ -0,0 +1,38 @@ +# pylint: disable=too-few-public-methods + +from datetime import datetime +from flask import Flask +from flask_smorest import Api +from .config import APISettings + +class IsSingletonException(Exception): + pass + +class Application: + """ + This is a singleton that can be accessed using get_instance() + + It has 2 properties + - app: Used for WSGI servers and such + - api: Used for Blueprints + """ + + __instance = None + app = Flask(__name__) + app.config.from_object(APISettings) + app.json.sort_keys = False + api = None + alive_since = None + + @staticmethod + def get_instance(): + if Application.__instance is None: + Application() + return Application.__instance + + def __init__(self): + if Application.__instance is not None: + raise IsSingletonException("This class is a singleton") + Application.__instance = self + self.api = Api(self.app) + self.alive_since = datetime.now() diff --git a/support_formatter/config/__init__.py b/support_formatter/config/__init__.py new file mode 100644 index 0000000..2ad3ed5 --- /dev/null +++ b/support_formatter/config/__init__.py @@ -0,0 +1 @@ +from .settings import APISettings, ServerSettings diff --git a/support_formatter/config/settings.py b/support_formatter/config/settings.py new file mode 100644 index 0000000..b9b4929 --- /dev/null +++ b/support_formatter/config/settings.py @@ -0,0 +1,35 @@ +# pylint: disable=too-few-public-methods +import os +from pathlib import Path + +from .. import __version__ + +class ServerSettings: + HOSTNAME = os.environ.get("SUPPORT_FORMATTER_HOST", "localhost") + PORT = int(os.environ.get("SUPPORT_FORMATTER_PORT", 5000)) + +class APISettings: + API_TITLE = "Support Organizer" + API_VERSION = __version__ + OPENAPI_VERSION = "3.1.0" + + # openapi.json settings + OPENAPI_URL_PREFIX = "/" + OPENAPI_JSON_PATH = "openapi.json" + + # swagger settings + OPENAPI_SWAGGER_UI_PATH = "/swagger" + OPENAPI_SWAGGER_UI_URL = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.10.0/" + SWAGGER_UI_DOC_EXPANSION = "list" + + # Info settings + API_SPEC_OPTIONS = { + 'info': { + 'description': 'Support Organizer for FGO' + } + } + + PAGES_DIRECTORY = Path(__file__).parents[1] / "pages" + + ALLOWED_EXTENSIONS = { 'csv', 'txt' } + FILE_SAVE_DIRECTORY = Path(__file__).parents[1] / ".temp" diff --git a/support_formatter/logic/__init__.py b/support_formatter/logic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/support_formatter/logic/csv_processor.py b/support_formatter/logic/csv_processor.py new file mode 100644 index 0000000..137428c --- /dev/null +++ b/support_formatter/logic/csv_processor.py @@ -0,0 +1,44 @@ +import csv +import pathlib + +class FileTypeInvalidError(ValueError): + pass + +def determine_type(row): + if set(row) == set(["servant", "level", "np_level", "skills", "bce"]): + return "servants" + if set(row) == set(["ce", "ce_level", "mlb"]): + return "ces" + return "unknown" + +def process_csv(path: pathlib.Path): + data, entries = {}, [] + + with open(path, "r") as csv_file: + csv_reader = csv.DictReader(csv_file) + csv_type = determine_type(csv_reader.fieldnames) + + if csv_type == "unknown": + raise FileTypeInvalidError() + + for row in csv_reader: + entry = row.copy() + if csv_type == "servants": + entry.update({ + "level": int(entry['level']), + "np_level": int(entry['np_level']), + "bce": bool(entry["bce"]) + }) + else: + entry.update({ + "ce_level": int(entry['ce_level']), + "mlb": bool(entry["mlb"]) + }) + entries.append(entry) + + if csv_type == "servants": + data = { "support_list": entries } + else: + data = { "ce_list": entries } + + return data diff --git a/support_formatter/models/__init__.py b/support_formatter/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/support_formatter/models/interface.py b/support_formatter/models/interface.py new file mode 100644 index 0000000..56a8e29 --- /dev/null +++ b/support_formatter/models/interface.py @@ -0,0 +1,19 @@ +from enum import Enum +import marshmallow as ma + +class HealthStatus(Enum): + OK = 0 + WARNING = 1 + ERROR = 2 + CRITICAL = 3 + +class HealthGet(ma.Schema): + alive_since = ma.fields.String() + alive_for = ma.fields.String() + status = ma.fields.Enum(HealthStatus, type=ma.fields.String) + +class ApiVersionGet(ma.Schema): + version = ma.fields.String(example="0.1") + +class OpenAPIGet(ma.Schema): + pass diff --git a/support_formatter/pages/error.html b/support_formatter/pages/error.html new file mode 100644 index 0000000..ca5dcdb --- /dev/null +++ b/support_formatter/pages/error.html @@ -0,0 +1,14 @@ + + + + + + + Support File Converter - Error + + + +

Error

+ + + \ No newline at end of file diff --git a/support_formatter/pages/index.html b/support_formatter/pages/index.html new file mode 100644 index 0000000..1feca42 --- /dev/null +++ b/support_formatter/pages/index.html @@ -0,0 +1,34 @@ + + + + + + + Support File Converter + + + +

Convert to Nerone-compatible data

+
+ + +
+ + +
+
+ +
+ +
+
+ +
+ +
+
+ +
+ + + \ No newline at end of file diff --git a/support_formatter/routes/__init__.py b/support_formatter/routes/__init__.py new file mode 100644 index 0000000..2af0a36 --- /dev/null +++ b/support_formatter/routes/__init__.py @@ -0,0 +1,7 @@ +# pylint: disable=wrong-import-position,cyclic-import +from flask_smorest import Blueprint + +interface_routes = Blueprint("interface", "interface", url_prefix="/", description="") +formatter_routes = Blueprint("formatter", "formatter", url_prefix="/", description="") + +from . import interface, upload # avoids circular imports problem diff --git a/support_formatter/routes/interface.py b/support_formatter/routes/interface.py new file mode 100644 index 0000000..e9d211e --- /dev/null +++ b/support_formatter/routes/interface.py @@ -0,0 +1,38 @@ +from datetime import datetime + +from flask import redirect, url_for +from flask.views import MethodView + +from ..app import Application +from ..config import APISettings +from ..models.interface import ApiVersionGet, HealthGet, HealthStatus, OpenAPIGet + +from . import interface_routes as blp + +APP = Application.get_instance() + +@blp.route("/health") +class ApiVersion(MethodView): + @blp.doc(summary="Returns the status and alive-time of the server") + @blp.response(200, HealthGet, description="Successful operation") + def get(self): + response = { + "alive_since": datetime.strftime(APP.alive_since, "%d.%m.%Y %H:%M:%S"), + "alive_for": str(datetime.now() - APP.alive_since), + "status": HealthStatus.OK + } + return response + +@blp.route("/openapi") +class ApiVersion(MethodView): + @blp.doc(summary="Get the OpenAPI spec as a JSON.") + @blp.response(200, OpenAPIGet, description="Successful operation") + def get(self): + return redirect(url_for('api-docs.openapi_json')) + +@blp.route("/version") +class ApiVersion(MethodView): + @blp.doc(summary="Get the REST interface version identification.") + @blp.response(200, ApiVersionGet, description="Successful operation") + def get(self): + return { "version": APISettings.API_VERSION } diff --git a/support_formatter/routes/upload.py b/support_formatter/routes/upload.py new file mode 100644 index 0000000..42e3b88 --- /dev/null +++ b/support_formatter/routes/upload.py @@ -0,0 +1,70 @@ +import os +import pathlib +from typing import List +from flask import send_from_directory +import marshmallow as ma +from flask_smorest.fields import Upload +from werkzeug.datastructures import FileStorage + +import uuid + +from ..app import Application +from ..config import APISettings +from ..logic.csv_processor import process_csv, FileTypeInvalidError +from . import formatter_routes as blp + +APP = Application.get_instance() + +def allowed_file(filename): + return '.' in filename and filename.rsplit('.', 1)[1].lower() in APISettings.ALLOWED_EXTENSIONS + +class MultipartFormSchema(ma.Schema): + username = ma.fields.String(required=True) + friendcode = ma.fields.String(required=False) + +class MultipartFileSchema(ma.Schema): + servantdata = Upload() + cedata = Upload() + +@blp.route("/", methods=["POST"]) +@blp.arguments(MultipartFormSchema, location="form") +@blp.arguments(MultipartFileSchema, location="files") +@blp.response(200) +def upload_file(form: dict[str, str], files: dict[str, FileStorage]): + filepaths: List[pathlib.Path] = [] + returndata = { + "username": form["username"], + "friendcode": form.get("friendcode", "") + } + + for name, file in files.items(): + if name not in ("servantdata", "cedata"): + return send_from_directory(APISettings.PAGES_DIRECTORY, "error.html") + + filepath = APISettings.FILE_SAVE_DIRECTORY / f"{uuid.uuid4()}.csv" + file.save(filepath) + + if os.stat(filepath).st_size < 1 or not allowed_file(file.filename): + filepath.unlink() + continue + + filepaths.append(filepath) + + if len(filepaths) == 0: + return send_from_directory(APISettings.PAGES_DIRECTORY, "error.html") + + try: + for f in filepaths: + result = process_csv(f) + returndata = returndata | result + f.unlink() + except FileTypeInvalidError: + for f in filepaths: + f.unlink() + return send_from_directory(APISettings.PAGES_DIRECTORY, "error.html") + + return returndata + +@blp.route("/", methods=["GET"]) +def file_dialog(): + return send_from_directory(APISettings.PAGES_DIRECTORY, "index.html") diff --git a/support_formatter/server.py b/support_formatter/server.py new file mode 100644 index 0000000..853e3e4 --- /dev/null +++ b/support_formatter/server.py @@ -0,0 +1,28 @@ +# pylint: disable=multiple-statements,wrong-import-position,wrong-import-order +from gevent.monkey import patch_all; patch_all() +from gevent.pywsgi import WSGIServer + +from .app import Application +from .routes import interface_routes, formatter_routes +from .config import APISettings, ServerSettings + +APISettings.FILE_SAVE_DIRECTORY.mkdir(parents=True, exist_ok=True) + +APP = Application.get_instance() +app, api = APP.app, APP.api +api.register_blueprint(interface_routes) +api.register_blueprint(formatter_routes) + +HOSTNAME, PORT = ServerSettings.HOSTNAME, ServerSettings.PORT + +def run_server(): + http_Server = WSGIServer((HOSTNAME, PORT), app) + try: + print(f"Server available on http://{HOSTNAME}:{PORT}/") + print(f"View docs on http://{HOSTNAME}:{PORT}/swagger") + http_Server.serve_forever() + except KeyboardInterrupt: + print("Keyboard interrupt received, stopping...") + +if __name__ == '__main__': + run_server()