HTTP long polling before upgrade

.forgejo/workflows/build-release.yaml

@@ -56,3 +56,37 @@ jobs:
         run: pip install twine
       - name: Upload package to registry
         run: python -m twine upload --repository-url ${{ secrets.REPOSITORY_URL }} -u ${{ secrets.TWINE_DEPLOY_USER }} -p ${{ secrets.TWINE_DEPLOY_PASSWORD }} dist/*
+
+  build-and-push-container:
+    needs: [ "publish-artifacts" ]
+    runs-on: dind
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log into Docker Package Registry
+        uses: docker/login-action@v3
+        with:
+          registry: forgejo.neshweb.net
+          username: ${{ secrets.FORGEJO_USERNAME }}
+          password: ${{ secrets.FORGEJO_TOKEN }}
+      - name: Build and push to Docker Package Registry
+        uses: docker/build-push-action@v5
+        with:
+          build-args: |
+            PACKAGE_VERSION=${{ github.ref_name }}
+          push: true
+          tags: forgejo.neshweb.net/firq/dockge-cli:${{ github.ref_name }}
+
+  release:
+    needs: [ build-and-push-container, publish-artifacts ]
+    if: success()
+    runs-on: docker
+    steps:
+      - name: Release New Version
+        uses: actions/forgejo-release@v1
+        with:
+          direction: upload
+          url: https://forgejo.neshweb.net
+          release-dir: release
+          token: ${{ secrets.FORGEJO_TOKEN }}
+          tag: ${{  github.ref_name }}

Dockerfile

@@ -0,0 +1,4 @@
+FROM forgejo.neshweb.net/ci-docker-images/python-neshweb:3.11
+
+ARG PACKAGE_VERSION=0.1.0
+RUN pip install dockge-cli==${PACKAGE_VERSION}

README.md

@@ -1,3 +1,67 @@
 # dockge-cli
 
 A simple CLI application written in Python for communicating with Dockge using websockets
+
+## Background
+
+Dockge (spoken dock-ge or dockage) is a tool to manage docker-compose stacks from a web ui. It is developed by louislam, who also develops UptimeKuma.
+
+Dockge itself doesn't offer any kind of API or programmatic access, as it is just intended for managing stacks via UI.
+
+My current deployment solution for firq.dev and fgo-ta.com is based on Dockge, and I was over it always having to reload the stack whenever I pushed an update. Instead, I wanted to have this as a separate CI step, automatically redeploying a givens stack.
+
+As Dockge is using a websocket-based system under the hood, it was easy to take a look at how communication occurs. In general, communication is achieved by leveraging socket.io for the data. Since Python already offers a solution for socket.io, it is just a matter of emulating the calls the webui sends and receives.
+
+In the end, this is the current result that works pretty well for my understanding. I am still trying to improve upon some issues (login times out, stability, features), but in general this works as a fine solution for automatic stack updating.
+
+## Installation
+
+Install it from the custom package index using
+
+```shell
+pip install --extra-index-url https://forgejo.neshweb.net/api/packages/Firq/pypi/simple/ dockge-cli
+```
+
+Alternativly, install it using this repository. When installing for development, make sure to install with the additional dependencies
+
+```shell
+pip install -e .[lint,typing]
+```
+
+## Usage
+
+Call the CLI using `dockge-cli` or `dockge`.
+
+```shell
+usage: dockge_cli [-h] [--version] {host,login,logout,list,status,restart,start,stop,down,update,exit,help}
+
+CLI interface for interacting with Dockge
+
+positional arguments:
+  {host,login,logout,list,status,restart,start,stop,down,update,exit,help}
+
+options:
+  -h, --help            show this help message and exit
+  --version             show program's version number and exit
+```
+
+Help for each individual command can be invoked by calling `dockge-cli help <command>`
+
+## The magic behind this
+
+Generally, this makes use of the underlying Websockets API that the Dockge frontend uses to communicate with the server. By analyzing the traffic and looking into the codebase, I was able to reverse most of the packets that are being sent. This allows me to then contruct, send and receive my own packets, making the whole thing work.
+
+There are some things that need to be taken into account for this: For one, dockge uses socket.io for the websocket communication. This meant I had to find the corresponding socket.io version to get the correct version of python-socketio. In addition, I had to find out how the authorization mechanism behind this works.
+
+After finishing up the first prototype, the workings are as follows:
+
+1. A websocket session is established using socket.io - this happens automatically
+2. After the session is ready, the `login` command is sent together with a provided username and password
+3. Once the CLI is authorized, the selected command is sent
+4. The CLI waits for any response values and exits once the command has executed successfully
+
+To provide a smooth experience, both the credentials and the remote host URI are stored on disk. just like the `docker` cli, the credentials are not encrypted, meaning it is advised to either clear the credentials after use OR to use the `--username` and `--password` parameters. This is especially recommended for CI applications.
+
+## Known issues
+
+This CLI does not work when Mullvad is used, as Mullvad actively blocks port forwarding (which python-socketio uses)

dockge_cli/.gitignore

@@ -1 +0,0 @@
-.temp/*

dockge_cli/client/commandprovider/factory.py

@@ -1,7 +0,0 @@
-from ...models import Command
-from .descriptors import mapping
-
-commands: dict[str, Command] = {}
-
-for descriptor in mapping:
-    commands.update({ descriptor.cmd: descriptor  })

dockge_cli/client/commands/__init__.py

@@ -0,0 +1,4 @@
+from ...models import Command
+from .mappings import mapping
+
+commands: dict[str, Command] = { c.cmd: c for c in mapping }

dockge_cli/client/commands/functions.py

@@ -1,18 +1,22 @@
 from urllib.parse import urlparse
 from getpass import getpass
+import re
 
 from ...models import Credentials
 from ...service import storage
-from ...service.communicate import DockgeConnection
-from ..utils import stack_formatter, status_formatter, generic_formatter, get_credential_parser
+from ...service.connection import DockgeConnection
+from ..utils import stack_formatter, status_formatter, generic_formatter, credential_parser_factory
 
-class ExecutionCommands():
+class FunctionBindings():
     """
     Helper class that provides all the static methods in an organized way
     This is an abstraction layer of the CLI, as those functions only do little preprocessing before calling the actural DockgeConnection
     """
     @staticmethod
     def __setup():
+        """
+        Creates a connection and logs into Dockge
+        """
         con = DockgeConnection()
         con.connect_and_login()
         return con
@@ -23,10 +27,12 @@ class ExecutionCommands():
         host command binding
         """
         if len(extra_args) > 0:
-            res = urlparse(extra_args[0])
+            mat = re.search(r"((\w+\.)?\w+\.\w+(\/.+)?)", extra_args[0], re.IGNORECASE)
+            if mat is None:
+                raise ValueError("Given host did not match regex")
+            res = urlparse(f"https://{mat[0]}")
             if all([res.scheme, res.netloc]):
-                host = extra_args[0].rstrip("/").replace("https://", "").replace("wss://", "")
-                storage.put("host", host)
+                storage.put("host", mat[0])
             else:
                 raise ValueError(f"Malformed URL {extra_args[0]}")
         print(storage.get("host"))
@@ -36,8 +42,9 @@ class ExecutionCommands():
         """
         login command binding
         """
+        print(f"WARNING! These credentials will be saved unencrypted in {storage._file.absolute()}")
         if len(extra_args) > 0:
-            credentials = get_credential_parser().parse_args(extra_args, namespace=Credentials)
+            credentials = credential_parser_factory().parse_args(extra_args, namespace=Credentials)
             storage.put("username", credentials.username, encoded=True)
             storage.put("password", credentials.password, encoded=True)
             return
@@ -64,7 +71,7 @@ class ExecutionCommands():
         """
         list command binding
         """
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         stack_formatter(con.list_stacks())
         con.disconnect()
 
@@ -73,9 +80,7 @@ class ExecutionCommands():
         """
         status command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         status_formatter(con.list_stack(extra_args[0]))
         con.disconnect()
 
@@ -84,9 +89,7 @@ class ExecutionCommands():
         """
         restart command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         generic_formatter(con.restart(extra_args[0]))
         con.disconnect()
 
@@ -95,9 +98,7 @@ class ExecutionCommands():
         """
         update command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         generic_formatter(con.update(extra_args[0]))
         con.disconnect()
 
@@ -106,9 +107,7 @@ class ExecutionCommands():
         """
         stop command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         generic_formatter(con.stop(extra_args[0]))
         con.disconnect()
 
@@ -117,9 +116,7 @@ class ExecutionCommands():
         """
         start command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         generic_formatter(con.start(extra_args[0]))
         con.disconnect()
 
@@ -128,9 +125,7 @@ class ExecutionCommands():
         """
         down command binding
         """
-        if extra_args is None:
-            raise ValueError
-        con = ExecutionCommands.__setup()
+        con = FunctionBindings.__setup()
         generic_formatter(con.down(extra_args[0]))
         con.disconnect()
 

dockge_cli/client/commands/mappings.py

@@ -1,6 +1,6 @@
 from typing import List
 from ...models import Command
-from .bindings import ExecutionCommands
+from .functions import FunctionBindings
 
 mapping: List[Command] = [
     Command(
@@ -8,83 +8,83 @@ mapping: List[Command] = [
         description="Sets and gets the URI of the dockge instance. Remove any unnecessary subdomains/protocols from the URI",
         args=1,
         optional=True,
-        bind=ExecutionCommands.host
+        func=FunctionBindings.host
     ),
     Command(
         cmd="login",
         description="Logs into a given dockge account, either with an interactive dialogue or by passing --user and --password",
         args=2,
         optional=True,
-        bind=ExecutionCommands.login
+        func=FunctionBindings.login
     ),
     Command(
         cmd="logout",
         description="Removes the credentials from the local storage.",
         args=0,
         optional=False,
-        bind=ExecutionCommands.logout
+        func=FunctionBindings.logout
     ),
     Command(
         cmd="list",
         description="Lists all available stacks with their status",
         args=0,
         optional=False,
-        bind=ExecutionCommands.list
+        func=FunctionBindings.list
     ),
     Command(
         cmd="status",
         description="Returns the status of one stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.status
+        func=FunctionBindings.status
     ),
     Command(
         cmd="restart",
         description="Restarts a given stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.restart
+        func=FunctionBindings.restart
     ),
     Command(
         cmd="start",
         description="Starts a given stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.start
+        func=FunctionBindings.start
     ),
     Command(
         cmd="stop",
         description="Stops a given stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.stop
+        func=FunctionBindings.stop
     ),
     Command(
         cmd="down",
         description="Stop & Downs a given stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.down
+        func=FunctionBindings.down
     ),
     Command(
         cmd="update",
         description="Updates a stack",
         args=1,
         optional=False,
-        bind=ExecutionCommands.update
+        func=FunctionBindings.update
     ),
     Command(
         cmd="exit",
         description="Exits the CLI - this will reset all settings, including credentials and host",
         args=0,
         optional=False,
-        bind=ExecutionCommands.exit
+        func=FunctionBindings.exit
     ),
     Command(
         cmd="help",
         description="Displays helping hints for commands",
         args=1,
         optional=True,
-        bind=ExecutionCommands.help
+        func=FunctionBindings.help
     )
 ]

dockge_cli/client/parser.py

@@ -3,7 +3,7 @@ import sys
 
 from .. import __version__
 from ..models import Arguments
-from .commandprovider.factory import commands
+from .commands import commands
 
 def parse_arguments():
     """

dockge_cli/client/run.py

@@ -1,4 +1,4 @@
-from .commandprovider.factory import commands
+from .commands import commands
 
 def display_help(extra_args):
     """
@@ -32,4 +32,4 @@ def run(command, args):
         display_help(args)
         return
 
-    c.bind(args)
+    c.func(args)

dockge_cli/client/utils.py

@@ -2,7 +2,7 @@ import argparse
 from tabulate import tabulate
 from ..models import StackStatus
 
-def get_credential_parser():
+def credential_parser_factory():
     """
     Creates a new parser for login credentials
     """
@@ -23,7 +23,7 @@ def stack_formatter(stacks):
 
     table, headers = [], ["Stackname", "Status"]
     for key, val in stacks["stackList"].items():
-        table.append([key, StackStatus(val['status']).name])
+        table.append([key, StackStatus(val['status']).name.lower()])
 
     print(tabulate(table, headers=headers, tablefmt="github"), "\n")
 

dockge_cli/models/__init__.py

@@ -1,3 +1,3 @@
 from .codes import StackStatus
-from .commands import Command
+from .command import Command
 from .parser import Arguments, Credentials

dockge_cli/models/codes.py

@@ -2,9 +2,8 @@ from enum import Enum
 
 class StackStatus(Enum):
     """
-    mapping codes for status vs text
+    mapping for plaintext vs statuscode
     """
-    # pylint: disable=invalid-name
-    inactive = 1
-    running = 3
-    exited = 4
+    INACTIVE = 1
+    RUNNING = 3
+    EXITED = 4

dockge_cli/models/command.py

@@ -6,7 +6,7 @@ class Command(BaseModel):
     Basic command structure for the CLI to automatically generate valid commands
     """
     cmd: str
-    bind: Callable
+    func: Callable
     args: int
     optional: bool
     description: str

dockge_cli/service/.gitignore

@@ -0,0 +1 @@
+.storage/*

dockge_cli/service/connection.py

@@ -32,7 +32,7 @@ class DockgeConnection:
     def _init_events(self):
         @self._sio.event
         def connect():
-            self.connect()
+            self.login()
             print("Connected!")
 
         @self._sio.event
@@ -63,6 +63,7 @@ class DockgeConnection:
             success = True
         else:
             print("Issue with login procedure")
+            print(data)
         return success
 
     # Functions
@@ -70,10 +71,11 @@ class DockgeConnection:
         """
         Connect to the websocket
         """
-        self._sio.connect(f"https://{self._host}/socket.io/", transports=['websocket'])
-        self.connect()
+        # Dockge uses Socket.io for the websockets, so this URI and params are always the same
+        self._sio.connect(f"https://{self._host}/socket.io/")
+        self.login()
 
-    def connect(self):
+    def login(self):
         """
         Log into dockge using basicauth
         Retries 5 times when timeouts occur
@@ -84,6 +86,11 @@ class DockgeConnection:
         data = None
         retry, count = True, 0
 
+        if not storage.exists("username"):
+            raise ValueError("Missing username")
+        if not storage.exists("password"):
+            raise ValueError("Missing password")
+
         while retry and count < 5:
             try:
                 data = self._sio.call(
@@ -91,12 +98,13 @@ class DockgeConnection:
                     {
                         "username": storage.get("username", encoded=True),
                         "password": storage.get("password", encoded=True),
-                        "token":""
+                        "token": ""
                     },
-                    timeout=5
+                    timeout=10
                 )
                 retry = False
             except socketio.exceptions.TimeoutError:
+                print("Reached timeout for login, retrying ...")
                 retry = True
                 count += 1
 
@@ -119,42 +127,42 @@ class DockgeConnection:
         """
         Lists status for a stack
         """
-        ret = self._sio.call("agent", ("", "serviceStatusList", name), timeout=5)
+        ret = self._sio.call("agent", ("", "serviceStatusList", name), timeout=10)
         return ret
 
     def restart(self, name):
         """
         Restarts a given stack
         """
-        ret = self._sio.call("agent", ("", "restartStack", name), timeout=10)
+        ret = self._sio.call("agent", ("", "restartStack", name), timeout=30)
         return ret
 
     def update(self, name):
         """
         Updates a given stack
         """
-        ret = self._sio.call("agent", ("", "updateStack", name), timeout=10)
+        ret = self._sio.call("agent", ("", "updateStack", name), timeout=30)
         return ret
 
     def stop(self, name):
         """
         Stops a given stack
         """
-        ret = self._sio.call("agent", ("", "stopStack", name), timeout=10)
+        ret = self._sio.call("agent", ("", "stopStack", name), timeout=30)
         return ret
 
     def start(self, name):
         """
         Starts a given stack
         """
-        ret = self._sio.call("agent", ("", "startStack", name), timeout=10)
+        ret = self._sio.call("agent", ("", "startStack", name), timeout=30)
         return ret
 
     def down(self, name):
         """
         Stops and downs a given stack
         """
-        ret = self._sio.call("agent", ("", "downStack", name), timeout=10)
+        ret = self._sio.call("agent", ("", "downStack", name), timeout=30)
         return ret
 
     def disconnect(self):

dockge_cli/service/storage.py

@@ -3,28 +3,44 @@ import pathlib
 import base64
 import yaml
 
-_storagepath = pathlib.Path(__file__).parents[1] / ".temp"
+_storagepath = pathlib.Path(__file__).parent / ".storage"
+_storagepath.mkdir(exist_ok=True, parents=True)
 _file = _storagepath / "storage.yaml"
 
-_storagepath.mkdir(exist_ok=True, parents=True)
-
-def fileexists():
+def create_file_when_missing():
     """
     Checks if storage file does exist, creates it when necessary
     """
+    if _file.exists():
+        return
+    with open(_file, 'a', encoding="utf-8"):
+        os.utime(_file, None)
+
+def exists(key: str) -> bool:
+    """
+    Checks if a given key exists in the storage file
+    """
     if not _file.exists():
-        with open(_file, 'a', encoding="utf-8"):
-            os.utime(_file, None)
+        return False
+
+    with open(_file, "r", encoding="utf-8") as file:
+        content: dict[str, str] = yaml.load(file, Loader=yaml.SafeLoader)
+    
+    return key in content
+
 
 def put(key: str, value: str, encoded=False):
     """
     Puts a given value with a given key into the storage file
     Encodes the data as base64 when encoded is set to true
     """
-    fileexists()
-    with open(_file, "r+", encoding="utf-8") as file:
+    if not _file.exists():
+        create_file_when_missing()
+
+    with open(_file, "r", encoding="utf-8") as file:
         content: dict[str, str] = yaml.load(file, Loader=yaml.SafeLoader) or {}
         content.update({ key: str(base64.b64encode(value.encode()), "utf-8") if encoded else value })
+
     with open(_file, "w+", encoding="utf-8") as file:
         yaml.dump(content, file, Dumper=yaml.SafeDumper)
 
@@ -32,10 +48,13 @@ def remove(key: str):
     """
     Removed a given key from the storage file
     """
-    fileexists()
+    if not _file.exists():
+        create_file_when_missing()
+
     with open(_file, "r", encoding="utf-8") as file:
         content: dict[str, str] = yaml.load(file, Loader=yaml.SafeLoader) or {}
         content.pop(key, None)
+
     with open(_file, "w+", encoding="utf-8") as file:
         yaml.dump(content, file, Dumper=yaml.SafeDumper)
 
@@ -47,9 +66,11 @@ def get(key: str, encoded=False):
     value: str | None = None
     if not _file.exists():
         return None
+
     with open(_file, "r", encoding="utf-8") as file:
         content: dict[str, str] = yaml.load(file, Loader=yaml.SafeLoader)
         value = content.get(key, None)
+
     if value is None:
         return None
     return base64.b64decode(value.encode()).decode() if encoded else value
@@ -58,4 +79,6 @@ def clear():
     """
     Deletes the storage file
     """
+    if not _file.exists():
+        return
     _file.unlink()

pyproject.toml

@@ -1,9 +1,10 @@
 [project]
 name = "dockge_cli"
-version = "0.1.0-c.2"
+version = "0.1.2"
 dependencies = [
     "pyyaml~=6.0.1",
     "pydantic~=2.8.0",
+    "requests~=2.32.3",
     "python-socketio~=5.11.3",
     "websocket-client~=1.8.0",
     "tabulate ~=0.9.0",
@@ -11,7 +12,7 @@ dependencies = [
 requires-python = ">= 3.10"
 authors = [{name = "Firq", email = "firelp42@gmail.com"}]
 maintainers = [{name = "Firq", email = "firelp42@gmail.com"}]
-description = "CLi for interacting with dockge"
+description = "CLI for interacting with dockge"
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Programming Language :: Python :: 3",