From 7000fce72fbec34c6f4957a59d4146cc7148ee59 Mon Sep 17 00:00:00 2001 From: Declan Hoare Date: Thu, 16 Apr 2020 22:58:21 +1000 Subject: [PATCH] Initial Release --- README | 18 +++ auth.py | 99 +++++++++++++++++ create_tables.py | 32 ++++++ create_test_account.py | 21 ++++ database.py | 206 ++++++++++++++++++++++++++++++++++ mapping.py | 52 +++++++++ mappings/ClientSave | 3 + mappings/PongHighscore | 3 + mappings/Save | 17 +++ mappings/UserSave | 3 + messagehandler.py | 30 +++++ mud.py | 87 +++++++++++++++ myconfig.py | 10 ++ netclass/__init__.py | 2 + netclass/binaryformatter.py | 128 +++++++++++++++++++++ netclass/jsonconverter.py | 26 +++++ netclass/netclass.py | 53 +++++++++ netobject.py | 23 ++++ payload.py | 30 +++++ pong.py | 216 ++++++++++++++++++++++++++++++++++++ save.py | 75 +++++++++++++ server.py | 11 ++ servermessage.py | 24 ++++ unite.py | 203 +++++++++++++++++++++++++++++++++ unite.wsgi | 4 + 25 files changed, 1376 insertions(+) create mode 100644 README create mode 100644 auth.py create mode 100644 create_tables.py create mode 100644 create_test_account.py create mode 100644 database.py create mode 100644 mapping.py create mode 100644 mappings/ClientSave create mode 100644 mappings/PongHighscore create mode 100644 mappings/Save create mode 100644 mappings/UserSave create mode 100644 messagehandler.py create mode 100644 mud.py create mode 100644 myconfig.py create mode 100644 netclass/__init__.py create mode 100644 netclass/binaryformatter.py create mode 100644 netclass/jsonconverter.py create mode 100644 netclass/netclass.py create mode 100644 netobject.py create mode 100644 payload.py create mode 100644 pong.py create mode 100644 save.py create mode 100644 server.py create mode 100644 servermessage.py create mode 100644 unite.py create mode 100644 unite.wsgi diff --git a/README b/README new file mode 100644 index 0000000..6b0d0b3 --- /dev/null +++ b/README @@ -0,0 +1,18 @@ +Shift Gears is a new server for ShiftOS 1.0. + +Currently focusing on the available source and binary of 1.0 Beta 2.5.2 +but ultimately it would be nice to support as many client versions as +possible. + +To run it you need a config.json file that looks something like + +``` +{ + "dbaddr": "mysql://root:password@localhost/shiftos" +} +``` + +dependencies: sqlalchemy, twisted, flask, netfleece +You need a WSGI Web server to run unite.wsgi for the Unite server. +You can just execute server.py to run the MUD server. + diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..a1ec562 --- /dev/null +++ b/auth.py @@ -0,0 +1,99 @@ + +from datetime import datetime, timedelta +import io +import secrets +import socket + +from sqlalchemy.orm.exc import NoResultFound + +from database import AuthSession +from mapping import map_from, map_to +from messagehandler import handler +from myconfig import config, mappings +from netclass.jsonconverter import from_json + +class InvalidTokenError(Exception): + pass + +class Auth: + def __init__(self, dbsession, addr): + self.dbsession = dbsession + self.addr = addr + + def session(self, token): + try: + session = self.dbsession.query(AuthSession).filter_by(Token = token).one() + except NoResultFound: + raise InvalidTokenError() + session.LastUsed = datetime.utcnow() + session.LastIP = self.addr + return session + + def create_session(self, user, app_name, app_description, version): + session = AuthSession() + session.Token = secrets.token_urlsafe() + session.User = user + session.AppName = app_name + session.AppDesc = app_description + session.Version = version + + session.Created = session.LastUsed = datetime.utcnow() + session.LastIP = self.addr + self.dbsession.add(session) + return session + + +@handler("mud_token_login") +def mud_token_login(connection, contents): + if connection.authsession is not None: + connection.error("You're already logged in and tried to log in again.") + return + try: + connection.authsession = connection.auth.session(contents) + user = connection.authsession.User + save = user.Save + if save is None: + connection.send_message("mud_login_denied") + else: + data = {"Upgrades": {u.Name: u.Installed for u in save.Upgrades}, + "CurrentLegions": [], #NYI + "UniteAuthToken": connection.authsession.Token, + "StoriesExperienced": [s.Name for s in save.StoriesExperienced], + "Users": [map_to(u, mappings["ClientSave"]) for u in save.Users]} + data.update(map_to(user, mappings["UserSave"])) + data.update(map_to(save, mappings["Save"])) + connection.send_message("mud_savefile", data) + except InvalidTokenError: + connection.send_message("mud_login_denied") + +@handler("mud_save", dict) +def mud_save(connection, contents): + user = connection.authsession.User + save = user.Save + if save is None: + save = Save() + user.Save = save + map_from(user, mappings["UserSave"], contents) + map_from(save, mappings["Save"], contents) + connection.dbsession.query(Upgrade).filter_by(Save = save).delete() + if contents["Upgrades"] is not None: + for k, v in contents["Upgrades"].items(): + connection.dbsession.add(Upgrade(Name = k, Installed = v, Save = save)) + connection.dbsession.query(StoryExperienced).filter_by(Save = save).delete() + if contents["StoriesExperienced"] is not None: + for v in contents["StoriesExperienced"]: + connection.dbsession.add(StoryExperienced(Name = v, Save = save)) + if contents["Users"] is not None: + users_new = {u["Username"]: u for u in contents["Users"]} + for usr in connection.dbsession.query(ClientSave).filter_by(Save = save): + if usr.Username in users_new: + map_from(usr, mappings["ClientSave"], users_new[usr.Username]) + del users_new[usr.Username] + else: + connection.dbsession.delete(usr) + for data in users_new.values(): + usr = ClientSave(Save = save) + map_from(usr, mappings["ClientSave"], data) + connection.dbsession.add(usr) + + diff --git a/create_tables.py b/create_tables.py new file mode 100644 index 0000000..ff4b2e4 --- /dev/null +++ b/create_tables.py @@ -0,0 +1,32 @@ +import database +database.Base.metadata.create_all(database.engine) +session = database.DbSession() + +systems = [("DevX", "mud", ["sys", "DevX"]), + ("hacker101", "undisclosed", ["hacker101"]), + ("victortran", "theos", ["victortran"])] + +# create the system account + +for i, (displayname, sysname, users) in enumerate(systems): + + user = database.User() + user.dontvalidate = True + user.ID = "00000000-0000-0000-0000-%012d" % i + user.Email = f"{sysname}@system.invalid" + user.DisplayName = displayname + user.SysName = sysname + session.add(user) + save = database.Save() + save.User = user + save.IsMUDAdmin = True + session.add(save) + print(repr(user.Save)) + for username in users: + clientsave = database.ClientSave() + clientsave.Username = username + clientsave.Save = save + session.add(clientsave) + +session.commit() +session.close() \ No newline at end of file diff --git a/create_test_account.py b/create_test_account.py new file mode 100644 index 0000000..d3bf136 --- /dev/null +++ b/create_test_account.py @@ -0,0 +1,21 @@ + +import base64 +import json +import uuid + +import requests + +uniq = str(uuid.uuid4())[:8].upper() + +displayname = f"u{uniq}" +sysname = f"s{uniq}" +email = f"{uniq}@getshiftos.ml" +password = "P@ssw0rd" + +auth = "Basic " + base64.b64encode(f"{email}:{password}".encode()).decode() + +token = requests.get("http://getshiftos.ml/Auth/Register", + params = {"appname": "ShiftGears", "appdesc": "ShiftGears testing software", "version": "45", "displayname": displayname, "sysname": sysname}, + headers = {"Authentication": auth}).text.strip() + +print(token) diff --git a/database.py b/database.py new file mode 100644 index 0000000..6fd8056 --- /dev/null +++ b/database.py @@ -0,0 +1,206 @@ +# SQL Tables + +# This shits a mess but hey it's python... + +import contextlib +import enum +import os + +from sqlalchemy import Boolean, Column, create_engine, DateTime, ForeignKey, Integer, MetaData, String, Table, Text, Unicode, UnicodeText +from sqlalchemy.dialects.mysql import BIGINT, DOUBLE +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import backref, relationship, RelationshipProperty, sessionmaker, validates +from sqlalchemy.sql import func + +from mapping import Direction +from myconfig import config, mappings + +Base = declarative_base() + +engine = create_engine(config["dbaddr"]) +DbSession = sessionmaker(bind = engine) + +class ValidationError(Exception): + pass + +def validated(col, validator = None, strip = False, min_length = None, min_value = None, max_value = None): + + def validate_string(row, key, val, encoding = None): + if not isinstance(val, str): + raise ValidationError(f"{col.name} must be a string") + if encoding is not None: + try: + val.encode(encoding) + except UnicodeEncodeError: # this exc is also used for ASCII! + raise ValidationError(f"{col.name} must be a valid {encoding} string") + + if strip: + val = val.strip() + if min_length is not None and len(val) < min_length: + raise ValidationError(f"{col.name} must be at least {min_length} characters long") + if col.type.length is not None and len(val) > col.type.length: + raise ValidationError(f"{col.name} cannot be more than {col.type.length} characters long") + return val + + def validate_number(row, key, val, min, max, t): + if not isinstance(val, t): + raise ValidationError(f"{val} must be {t.__name__}") + if min is not None and val < min: + raise ValidationError(f"{col.name} cannot be less than {min}") + if max is not None and val > max: + raise ValidationError(f"{col.name} cannot be greater than {max}") + return val + + @validates(col.name) + def validate(row, key, val): + if hasattr(row, "dontvalidate"): + return val + if validator is not None: + val = validator(row, key, val) + if val is None: + if col.nullable: + return None + else: + raise ValidationError(f"{col.name} cannot be null") + + min = min_value + max = max_value + + if isinstance(col.type, String) or isinstance(col.type, Text): + val = validate_string(row, key, val, "ASCII") + elif isinstance(col.type, Unicode) or isinstance(col.type, UnicodeText): + val = validate_string(row, key, val, "UTF-8") + + elif isinstance(col.type, Boolean) and not isinstance(val, bool): + raise ValidationError(f"{col.name} must be a boolean") + + + elif isinstance(col.type, Integer): + if min is None: + min = -(1 << 31) + if max is None: + max = (1 << 31) - 1 + val = validate_number(row, key, val, min, max, int) + elif isinstance(col.type, BIGINT): + if min is None: + min = 0 if col.type.unsigned else -(1 << 63) + if max is None: + max = 1 << (63 if col.type.unsigned else 64) + val = validate_number(row, key, val, min, max, int) + elif isinstance(col.type, DOUBLE): + val = validate_number(row, key, val, min, max, float) + return val + return col, validate + +def validate_email(row, key, val): + if val.count("@") == 1 and ".." not in val.split("@")[-1]: + return val + else: + raise ValidationError("Invalid email") + +class User(Base): + __tablename__ = "User" + + # This is a GUID + ID = Column(String(36), nullable = False, primary_key = True) + + Email, ValidateEmail = validated(Column("Email", Unicode(254), nullable = False, unique = True), + validate_email, strip = True) + Password = Column(String(60)) + + DisplayName, ValidateDisplayName = validated( + Column("DisplayName", Unicode(255), unique = True, nullable = False), + strip = True, min_length = 1) + + SysName, ValidateSysName = validated( + Column("SysName", Unicode(255), unique = True, nullable = False), + strip = True, min_length = 5) + + FullName, ValidateFullName = validated( + Column("FullName", Unicode(255), nullable = False, default = ""), + strip = True) + + Codepoints, ValidateCodepoints = validated(Column("Codepoints", BIGINT(unsigned = True), nullable = False, default = 0)) + PongLevel, ValidatePongLevel = validated(Column("PongLevel", Integer), min_value = 1, max_value = 42) + PongCP, ValidatePongCP = validated(Column("PongCP", BIGINT(unsigned = True))) + + Sessions = relationship("AuthSession", back_populates = "User") + +class AuthSession(Base): + __tablename__ = "AuthSession" + Token = Column(String(43), primary_key = True) + UserID = Column(String(36), ForeignKey("User.ID")) + User = relationship("User", back_populates = "Sessions") + AppName, ValidateAppName = validated(Column("AppName", Unicode(255), nullable = False), strip = True) + AppDesc, ValidateAppDesc = validated(Column("AppDesc", Unicode(255), nullable = False), strip = True) + Version, ValidateVersion = validated(Column("Version", Unicode(255), nullable = False), strip = True) + Created = Column(DateTime) + LastUsed = Column(DateTime) + LastIP = Column(String(39), nullable = False) + +# Save is out to a separate table cause the player can delete it without +# deleting their whole account +class Save(Base): + __tablename__ = "Save" + + ID = Column(Integer, primary_key = True) + + UserID = Column(String(36), ForeignKey("User.ID")) + User = relationship("User", backref = backref("Save", uselist = False)) + + MusicVolume, ValidateMusicVolume = validated(Column("MusicVolume", Integer, nullable = False, default = 0)) + SfxVolume, ValidateSfxVolume = validated(Column("SfxVolume", Integer, nullable = False, default = 0)) + + Upgrades = relationship("Upgrade", backref = "Save") + + # I think if the StoryPosition is changed back to 0 after the Oobe then ShiftOS will softlock. + # Keep an eye on this... + StoryPosition, ValidateStoryPosition = validated(Column("StoryPosition", Integer, nullable = False, default = 0)) + + Language, ValidateLanguage = validated(Column("Language", Unicode(255))) + MyShop, ValidateMyShop = validated(Column("MyShop", Unicode(255))) + + MajorVersion, ValidateMajorVersion = validated(Column("MajorVersion", Integer, nullable = False, default = 0)) + MinorVersion, ValidateMinorVersion = validated(Column("MinorVersion", Integer, nullable = False, default = 0)) + Revision, ValidateRevision = validated(Column("Revision", Integer, nullable = False, default = 0)) + + IsPatreon = Column(Boolean, nullable = False, default = False) + Class, ValidateClass = validated(Column("Class", Integer, nullable = False, default = 0), + min_value = 0, max_value = 8) + RawReputation, ValidateRawReputation = validated(Column("RawReputation", DOUBLE, nullable = False, default = 0)) + + Password, ValidatePassword = validated(Column("Password", Unicode(255))) + PasswordHashed, ValidatePasswordHashed = validated(Column("PasswordHashed", Boolean, nullable = False, default = False)) + + ShiftnetSubscription, ValidateShiftnetSubscription = validated(Column("ShiftnetSubscription", Integer, nullable = False, default = False), + min_value = 0, max_value = 3) + + IsMUDAdmin = Column(Boolean, nullable = False, default = False) + + LastMonthPaid, ValidateLastMonthPaid = validated(Column("LastMonthPaid", Integer, nullable = False, default = 0)) + + StoriesExperienced = relationship("StoryExperienced", backref = "Save") + Users = relationship("ClientSave", backref = "Save") + +class Upgrade(Base): + __tablename__ = "Upgrade" + ID = Column(Integer, primary_key = True) + SaveID = Column(Integer, ForeignKey("Save.ID")) + Name, ValidateName = validated(Column("Name", Unicode(255))) + Installed, ValidateInstalled = validated(Column("Installed", Boolean, nullable = False)) + +class StoryExperienced(Base): + __tablename__ = "StoryExperienced" + ID = Column(Integer, primary_key = True) + SaveID = Column(Integer, ForeignKey("Save.ID")) + Name, ValidateName = validated(Column("Name", Unicode(255))) + Name = Column(Unicode(255)) + +class ClientSave(Base): + __tablename__ = "ClientSave" + ID = Column(Integer, primary_key = True) + SaveID = Column(Integer, ForeignKey("Save.ID")) + Username, ValidateUsername = validated(Column("Username", Unicode(255))) + Password, ValidatePassword = validated(Column("Password", Unicode(255))) + Permissions, ValidatePermissions = validated(Column("Permissions", Integer, nullable = False, default = 0), min_value = 0, max_value = 3) + diff --git a/mapping.py b/mapping.py new file mode 100644 index 0000000..a01652f --- /dev/null +++ b/mapping.py @@ -0,0 +1,52 @@ +import enum +import os + +# A mapping defines how to convert part of a row to a dictionary for +# an API response, and how to insert data from a received dictionary +# back into the row. In other words it allows you to quickly select +# and rename a lot of fields. This module deals with parsing them. +# They are used in database and the path is chosen in myconfig. + +class Direction(enum.Flag): + TO = enum.auto() + FROM = enum.auto() + +Direction.BOTH = Direction.TO|Direction.FROM + +_direction_symbols = [("<", Direction.TO), (">", Direction.FROM)] + +def load_mappings(path): + mappings = {} + for fn in os.listdir(path): + with open(os.path.join(path, fn)) as f: + mapping = [] + for l in f: + l = l.strip() + if l == "" or l.startswith("#"): + continue + cols = l.split() + dictname = cols.pop(0) + if cols: + direction = Direction(0) + dirstring = cols.pop(0) + for sym, val in _direction_symbols: + if sym in dirstring: + direction |= val + else: + direction = Direction.BOTH + if cols: + tablename = cols.pop(0) + else: + tablename = dictname + mapping.append((dictname, direction, tablename)) + mappings[fn] = mapping + return mappings + +def map_to(row, mapping): + return {dictname: getattr(row, tablename) for dictname, direction, tablename in mapping if direction & direction.TO} + +def map_from(row, mapping, data): + for dictname, direction, tablename in mapping: + if direction & direction.FROM: + setattr(row, tablename, data[dictname]) + diff --git a/mappings/ClientSave b/mappings/ClientSave new file mode 100644 index 0000000..ee1afca --- /dev/null +++ b/mappings/ClientSave @@ -0,0 +1,3 @@ +Username +Password +Permissions diff --git a/mappings/PongHighscore b/mappings/PongHighscore new file mode 100644 index 0000000..9ec2b98 --- /dev/null +++ b/mappings/PongHighscore @@ -0,0 +1,3 @@ +UserId < ID +Level < PongLevel +CodepointsCashout < PongCP diff --git a/mappings/Save b/mappings/Save new file mode 100644 index 0000000..6210073 --- /dev/null +++ b/mappings/Save @@ -0,0 +1,17 @@ +MusicVolume +SfxVolume +StoryPosition +Language +MyShop +MajorVersion +MinorVersion +Revision +IsPatreon < +Class +RawReputation +Password +PasswordHashed +ShiftnetSubscription +ID < UserID +IsMUDAdmin < +LastMonthPaid diff --git a/mappings/UserSave b/mappings/UserSave new file mode 100644 index 0000000..9b70b5d --- /dev/null +++ b/mappings/UserSave @@ -0,0 +1,3 @@ +Username <> Email +Codepoints < +SystemName <> SysName diff --git a/messagehandler.py b/messagehandler.py new file mode 100644 index 0000000..dd061cc --- /dev/null +++ b/messagehandler.py @@ -0,0 +1,30 @@ + +from netclass.jsonconverter import from_json, to_json +from servermessage import ServerMessage + +handlers = {} +forwardhandlers = {} + +def handler(name, t = None, dct = handlers): + def decorator(fun): + dct[name] = (fun if t is None + else lambda conn, c, *a: fun(conn, from_json(t, c), *a)) + return fun + return decorator + +def forwardhandler(name, t = None): + return handler(name, t, dct = forwardhandlers) + +# "Forward" messages are meant to be sent to other clients, and the +# original server did this without validating them at all. This +# gives clients all the same power over each other as the server has +# over them, all the way up to arbitrary code execution...not a good +# idea. Instead forwards are treated as just a different kind of +# message for the server to handle. Forward-handlers get the GUID +# which in this case is a target, unlike normal handlers, where it is +# discarded because it is on the source connection. +@handler("mud_forward", ServerMessage) +def mud_forward(connection, contents): + if contents.Name in forwardhandlers: + return forwardhandlers[contents.Name](connection, contents.Contents, contents.GUID) + diff --git a/mud.py b/mud.py new file mode 100644 index 0000000..9a06023 --- /dev/null +++ b/mud.py @@ -0,0 +1,87 @@ + +import traceback +import uuid + +from twisted.internet.protocol import Factory + +from auth import Auth +from database import DbSession +from netclass import netclass_root, jsonconverter +from servermessage import ServerMessage, ServerMessageStream + +import pong +import save +from messagehandler import handlers + +class MudConnection(ServerMessageStream): + def __init__(self, factory, addr): + self.factory = factory + self.addr = addr + self.authsession = None + super().__init__() + def connectionMade(self): + self.dbsession = DbSession() + + self.pong = pong.PongState(self) + + self.auth = Auth(self.dbsession, self.addr.host) + print(f"{self.addr.host} connected.") + self.send_message("Welcome", str(uuid.uuid4())) + + # If the server shuts down, all the clients that were left open + # will reconnect as soon as it comes back on. But, they don't + # bother to re-authenticate on their own, so the server has to + # prompt them to save to get the copy of the auth token that is + # in the save. When the client joins on its own startup, + # and authenticates anyway, hopefully this won't matter... + #self.run_command("sos.save") + # but it is commented out cause the lua script doesn't work + def connectionLost(self, reason): + print(f"{self.addr.host} disconnected.") + self.closing = True + self.pong.leave() + self.dbsession.close() + def serverMessageReceived(self, message): + if message.Name not in ["pong_mp_setballpos", "pong_mp_setopponenty"]: + print(f"{self.addr.host}: {message.Name}({repr(message.Contents)})") + if message.Name in handlers: + try: + handlers[message.Name](self, message.Contents) + self.dbsession.commit() + except: + self.dbsession.rollback() + self.error(traceback.format_exc()) + traceback.print_exc() + else: + self.error(f"Unimplemented message {message.Name}. Thanks for using Shift Gears!") + + def send_message(self, name, contents = None, guid = None): + if contents is not None and not isinstance(contents, str): + contents = jsonconverter.to_json(contents) + guid = str(guid) + self.sendServerMessage(ServerMessage(name, contents, guid)) + + def error(self, message): + # The content of the Error message is deserialised at the other + # end as an Exception, but only the Message member is read. + self.send_message("Error", {"ClassName":"System.Exception","Message":message,"Data":None,"InnerException":None,"HelpURL":None,"StackTraceString":None,"RemoteStackTraceString":None,"RemoteStackIndex":0,"ExceptionMethod":None,"HResult":-2146233088,"Source":None}) + + # executes Lua code on the client...its that easy + def run(self, script): + self.send_message("run", {"script": script}) + + # executes a ShiftOS command on the client + def run_command(self, cmd): + # We don't use trm_invokecommand because it expects the prompt + # to be sent to it before the command and it's not always + # feasible to figure out what the prompt is. + self.run(f"sos.runCommand({repr(cmd)})") + + def infobox(self, msg, title = "MUD"): + self.invoke_command("infobox.show" + jsonconverter.to_json({"title": title, "msg": msg})) + +class MudConnectionFactory(Factory): + def __init__(self): + self.pong = pong.PongMatchmaking() + def buildProtocol(self, addr): + return MudConnection(self, addr) diff --git a/myconfig.py b/myconfig.py new file mode 100644 index 0000000..2297a26 --- /dev/null +++ b/myconfig.py @@ -0,0 +1,10 @@ +import os +import json + +import mapping + +os.chdir(os.path.dirname(os.path.realpath(__file__))) +with open("config.json") as f: + config = json.load(f) + +mappings = mapping.load_mappings("mappings") diff --git a/netclass/__init__.py b/netclass/__init__.py new file mode 100644 index 0000000..8837c48 --- /dev/null +++ b/netclass/__init__.py @@ -0,0 +1,2 @@ + +from .netclass import netclass, netclass_root diff --git a/netclass/binaryformatter.py b/netclass/binaryformatter.py new file mode 100644 index 0000000..00fd327 --- /dev/null +++ b/netclass/binaryformatter.py @@ -0,0 +1,128 @@ +# Copyright 2020 Declan Hoare + +# This should really be split into two layers: +# - binaryformatter.py deals with transforming Python objects into +# NRBF meta-dictionaries and vice versa. +# - nrbf.py reads and writes the binary structure. +# To do this, netfleece should ideally be patched, because as-is, it +# changes the meta-dictionaries somewhat from how they are in the file: +# in particular, after reading *AndTypes records it will consume the +# following records and move them to a 'Values' member on the lead +# record. +# Anyway, I think the interface of this module is fine, so it will do +# as a black box for now. + +import struct + +import netfleece +from netfleece.netfleece import RecordTypeEnum + +from .netclass import netclass_root, classes + +def deserialise(stream): + def extract_value(meta): #Recover the underlying structure from NRBF meta-dictionary + if "Value" in meta: + return meta["Value"] + elif "Values" in meta: + return classes[meta["ClassInfo"]["Name"]].from_dict( + {n.split("<")[1].split(">")[0]: extract_value(v) # They look like this: k__BackingField for some reason + for n, v in zip(meta["ClassInfo"]["MemberNames"], meta["Values"])}) + elif meta["RecordTypeEnum"] == "ObjectNull": + return None + else: + raise ValueError(f"Unknown value format: {meta}") + dnb = netfleece.DNBinary(stream, expand = True) + dnb.parse() + meta = dnb.backfill() + return extract_value(meta) + +# Serialisation is limited, mostly only supporting the features needed +# for ShiftOS +_nrbf_header = b"\0\x01\0\0\0\xFF\xFF\xFF\xFF\x01\0\0\0\0\0\0\0" +_nrbf_footer = b"\x0B" + +s32 = struct.Struct(" 0x7FFFFFFF: + raise ValueError(f"String is too long ({n} bytes)") + while n > 0x7F: + write_byte((n & 0x7F) | 0x80) + n >>= 7 + write_byte(n) + stream.write(data) + + def next_id(): + nonlocal last_id + last_id += 1 + return last_id + + def write_record_type(typ): + write_byte(typ.value) + + def write_library(name): + library_id = len(assemblies) + write_record_type(RecordTypeEnum.BinaryLibrary) + write_s32(library_id) + write_string(name) + assemblies.append(name) + return library_id + + def library(name): + return assemblies.index(name) + + def write_object_string(object_id, val): + write_record_type(RecordTypeEnum.BinaryObjectString) + write_s32(object_id) + write_string(val) + + def write_object_null(): + write_record_type(RecordTypeEnum.ObjectNull) + + def write_class_with_members(object_id, val): + library_id = library(val._assembly) + write_record_type(RecordTypeEnum.ClassWithMembers) + write_s32(object_id) + write_string(val._name) + write_s32(len(val._members)) + for typ, name in val._members: + write_string(f"<{name}>k__BackingField") + write_s32(library_id) + + def walk_library(it): + if isinstance(it, netclass_root): + if it._assembly not in assemblies: + write_library(it._assembly) + for typ, name in it._members: + walk_library(it._contents[name]) + + def walk(it): + object_id = next_id() + if isinstance(it, netclass_root): + write_class_with_members(object_id, it) + for typ, name in it._members: + walk(it._contents[name]) + elif isinstance(it, str): + write_object_string(object_id, it) + elif it is None: + write_object_null() + else: + raise TypeError("Type not supported!") + return object_id + + stream.write(_nrbf_header) + walk_library(obj) + walk(obj) + stream.write(_nrbf_footer) + diff --git a/netclass/jsonconverter.py b/netclass/jsonconverter.py new file mode 100644 index 0000000..8029fcd --- /dev/null +++ b/netclass/jsonconverter.py @@ -0,0 +1,26 @@ + +import decimal +import json + +from .netclass import netclass_root + +class _encoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, netclass_root): + return obj._contents + elif isinstance(obj, decimal.Decimal): + return float(obj) + else: + return super().default(obj) + +def from_json(t, j): + print(repr(j)) + val = json.loads(j) + if issubclass(t, netclass_root): + val = t.from_dict(val) + if not isinstance(val, t): + raise ValueError(f"the JSON value was of type {type(val)}, not {t}") + return val + +def to_json(obj): + return json.dumps(obj, cls=_encoder) diff --git a/netclass/netclass.py b/netclass/netclass.py new file mode 100644 index 0000000..988bc77 --- /dev/null +++ b/netclass/netclass.py @@ -0,0 +1,53 @@ + +# The netclasses derive this so they can be identified. +class netclass_root: + pass + +classes = {} + +def netclass(name, assembly, members): + member_names = [n for _, n in members] + class proxy(netclass_root): + def __init__(self, *args, **kwargs): + if args == (): + super().__setattr__("_contents", kwargs) + else: + super().__setattr__("_contents", dict(zip(member_names, args))) + self._validate() + + def _validate(self): + if set(member_names) != set(self._contents.keys()): + raise TypeError("The instance does not have the correct members") + for typ, name in members: + val = self._contents[name] + if not isinstance(val, typ): + raise TypeError(f"{name} is {val}, must be {typ}") + + @staticmethod + def from_dict(d): + cleaned = {} + for typ, name in members: + val = d[name] + if isinstance(val, dict) and issubclass(typ, netclass_root): + val = typ.from_dict(val) + cleaned[name] = val + return proxy(**cleaned) + + def __getattr__(self, member): + try: + return self._contents[member] + except KeyError: + raise AttributeError(f"No such member {member}") + + def __setattr__(self, member, value): + if member in self._members: + self._contents[member] = value + else: + raise AttributeError(f"No such member {member}") + + proxy._name = name + proxy._assembly = assembly + proxy._members = members + proxy.__name__ = name.split(".")[-1] + classes[name] = proxy + return proxy diff --git a/netobject.py b/netobject.py new file mode 100644 index 0000000..87673d9 --- /dev/null +++ b/netobject.py @@ -0,0 +1,23 @@ + +import io + +from netclass import netclass, binaryformatter +from payload import PayloadStream + +NetObject = netclass("NetSockets.NetObject", + "NetSockets, Version=1.1.0.0, Culture=neutral, PublicKeyToken=null", + [(str, "Name"), (object, "Object")]) + +class NetObjectStream(PayloadStream): + def payloadReceived(self, payload): + with io.BytesIO(payload) as buf: + obj = binaryformatter.deserialise(buf) + if not isinstance(obj, NetObject): + raise TypeError(f"An object was received on the NetObjectStream of type {type(obj)}") + self.netObjectReceived(obj) + def sendNetObject(self, obj): + if not isinstance(obj, NetObject): + raise TypeError(f"The NetObjectStream can only send NetObject, not {type(obj)}") + with io.BytesIO() as buf: + binaryformatter.serialise(buf, obj) + self.sendPayload(buf.getvalue()) diff --git a/payload.py b/payload.py new file mode 100644 index 0000000..36f33ba --- /dev/null +++ b/payload.py @@ -0,0 +1,30 @@ + +import struct + +from twisted.internet.protocol import Protocol + +s32 = struct.Struct("= 4: + self.__length = s32.unpack(self.__buffer[:4])[0] + self.__buffer = self.__buffer[4:] + if self.__length < 0: + raise ValueError(f"Invalid (negative) payload length {self.__length}") + if self.__length is not None and len(self.__buffer) >= self.__length: + payload = self.__buffer[:self.__length] + self.__buffer = self.__buffer[self.__length:] + self.__length = None + self.payloadReceived(payload) + continue + break + def sendPayload(self, data): + self.transport.write(s32.pack(len(data)) + data) + diff --git a/pong.py b/pong.py new file mode 100644 index 0000000..0a8307c --- /dev/null +++ b/pong.py @@ -0,0 +1,216 @@ +# The original server didn't even know about the pong multiplayer +# protocol, it just relayed "Forward" messages between clients and let +# them sort it all out. In practice this worked but if anybody with +# a sort of twisted Joker mind ever read the ShiftOS source then this +# could have led to mischief and malice. +# This implementation mediates the protocol to make sure messages are +# only going where they should. That makes it more complicated but +# on the plus side it's hopefully also more reliable even when +# everyone's playing by the rules. + +import datetime + +from messagehandler import forwardhandler, handler + +class PongMatchmaking: + def __init__(self): + self.players = {} + + self.current_heartbeat = 0 + def join(self, newcomer): + for opponent in self.players.values(): + opponent.connection.send_message("pong_handshake_matchmake", newcomer.name) + newcomer.connection.send_message("pong_handshake_matchmake", opponent.name) + self.players[newcomer.name] = newcomer + def leave(self, player): + try: + del self.players[player.name] + except KeyError: + return + for opponent in self.players.values(): + opponent.connection.send_message("pong_handshake_left", player.name) + def heartbeat(self): + for player in list(self.players.values()): # copy to be able to delete + if self.current_heartbeat - player.last_heartbeat >= 3: + player.connection.infobox("Your connection to Pong has timed out.", "Timed out") + player.leave() + else: + player.send_heartbeat() + self.current_heartbeat += 1 + def handshake(self, leader, follower_name): + try: + follower = self.players[follower_name] + except KeyError: + return False + if follower.opponent is not None: + return False + + leader.opponent = follower + leader.is_leader = True + follower.opponent = leader + follower.is_leader = False + follower.handshake_complete = False + + # If the opponent GUID is null-or-whitespace then the client + # won't send pong_mp_left. Other than that, the value doesn't + # matter. + follower.connection.send_message("pong_handshake_chosen", "_") + return True + +class PongPlayer: + def __init__(self, connection): + self.connection = connection + + # We have to store this, because it's used as the key sent to + # the clients to identify each other but it can change at any + # time. + self.name = self.connection.authsession.User.DisplayName + + # The heartbeat 'freezes' while the server is waiting for the + # client to finish the handshake. This means that a + # mischievous client can't leave someone else hanging by + # responding to the heartbeats and deliberately ignoring the + # handshake. As a bonus it clears up the connection a little. + self.frozen = False + self.heartbeat() + + self.y = 0 + + self.connected = True + self.connection.factory.pong.join(self) + + self.opponent = None + def leave(self): + self.connected = False + self.connection.factory.pong.leave(self) + if self.opponent is not None: + self.opponent.opponent = None + + # In the special situation that the follower leaves before + # the handshake is complete, the leader is still fit to + # choose another follower. + if not self.is_leader and not self.handshake_complete: + self.opponent.connection.infobox(f"{self.name} is no longer available.", "Matchmaking failed") + + # in all other circumstances, including leader leaving + # part-way through handshake, the other player's state has + # changed and can't be recovered, so Pong has to quit. + else: + self.opponent.connection.send_message("pong_mp_left") + + def send_heartbeat(self): + if not self.frozen: + self.connection.send_message("pong_handshake_resendid") + def heartbeat(self): + if not self.frozen: + self.last_heartbeat = self.connection.factory.pong.current_heartbeat + def freeze(self): + self.frozen = True + def unfreeze(self): + self.frozen = False + self.heartbeat() + + def complete_handshake(self): + if self.is_leader: + raise ValueError("The handshake can only be completed by a follower") + + self.handshake_complete = True + self.opponent.connection.send_message("pong_handshake_complete", "_") + + +class PongState: + def __init__(self, connection): + self.connection = connection + self.player = None + def busy(self): + return self.player is not None and self.player.connected + def join(self): + if self.busy(): + self.connection.infobox("You can't play multiple simultaneous games of online Pong...nobody is that good") + return + self.player = PongPlayer(self.connection) + def leave(self): + if self.busy(): + self.player.leave() + def heartbeat(self): + if self.busy(): + self.player.heartbeat() + +# Note that this is a normal handler. This is sent as a normal +# message only once at the start of matchmaking; after that, it's resent +# as a forward. +@handler("pong_handshake_matchmake") +def pong_handshake_matchmake(connection, contents): + connection.pong.join() + +# We send out a pong_handshake_resendid to every matchmaking player +# every 5 seconds as a heartbeat, and a working client will respond with +# a pong_handshake_matchmake immediately. +@forwardhandler("pong_handshake_matchmake") +def pong_handshake_matchmake_forward(connection, contents, name): + connection.pong.heartbeat() + +@forwardhandler("pong_handshake_chosen") +def pong_handshake_chosen(connection, contents, name): + if not connection.factory.pong.handshake(connection.pong.player, name): + connection.infobox(f"{name} is no longer available.", "Matchmaking failed") + +@forwardhandler("pong_handshake_complete") +def pong_handshake_complete(connection, contents, name): + connection.pong.player.complete_handshake() + +@forwardhandler("pong_handshake_left") +def pong_handshake_left(connection, contents, name): + connection.factory.pong.leave(connection.pong.player) + +@forwardhandler("pong_mp_setopponenty", int) +def pong_mp_setopponenty(connection, y, name): + try: + assert -2147483648 <= y <= 2147483647 + except: + connection.error("Y co-ordinate out of range") + + connection.pong.player.y = y + +@forwardhandler("pong_mp_setballpos", str) +def pong_mp_setballpos(connection, point, name): + if not connection.pong.player.is_leader: + connection.error("That message can only be used by the leader") + return + + try: + x, y = point.split(",") + x = int(x) + y = int(y) + assert -2147483648 <= x <= 2147483647 + assert -2147483648 <= y <= 2147483647 + except: + connection.error("Invalid Point") # har har + + connection.pong.player.opponent.connection.send_message("pong_mp_setballpos", f'"{x},{y}"') + connection.pong.player.opponent.connection.send_message("pong_mp_setopponenty", connection.pong.player.y) + connection.pong.player.connection.send_message("pong_mp_setopponenty", connection.pong.player.opponent.y) + +@forwardhandler("pong_mp_left") +def pong_mp_left(connection, contents, name): + connection.pong.leave() + +@forwardhandler("pong_mp_youwin") +def pong_mp_youwin(connection, contents, name): + connection.pong.player.opponent.connection.send_message("pong_mp_youwin") + +@forwardhandler("pong_mp_youlose") +def pong_mp_youlose(connection, contents, name): + connection.pong.player.opponent.connection.send_message("pong_mp_youlose") + +@forwardhandler("pong_mp_cashedout") +def pong_mp_cashedout(connection, contents, name): + # The client is already out of here immediately after sending the + # message. + opponent = connection.pong.player.opponent + if opponent is not None: + opponent.connection.send_message("pong_mp_cashedout") + opponent.opponent = None + connection.pong.player.opponent = None + + diff --git a/save.py b/save.py new file mode 100644 index 0000000..f632160 --- /dev/null +++ b/save.py @@ -0,0 +1,75 @@ +# mud save/load + +from auth import Auth, InvalidTokenError +from database import ClientSave, Save, StoryExperienced, Upgrade +from mapping import map_from, map_to +from messagehandler import handler +from myconfig import mappings +from netclass.jsonconverter import from_json + +@handler("mud_token_login") +def mud_token_login(connection, contents): + try: + connection.authsession = connection.auth.session(contents) + user = connection.authsession.User + save = user.Save + if save is None: + connection.send_message("mud_login_denied") + else: + data = {"Upgrades": {u.Name: u.Installed for u in save.Upgrades}, + "CurrentLegions": [], #NYI + "UniteAuthToken": connection.authsession.Token, + "StoriesExperienced": [s.Name for s in save.StoriesExperienced], + "Users": [map_to(u, mappings["ClientSave"]) for u in save.Users]} + data.update(map_to(user, mappings["UserSave"])) + data.update(map_to(save, mappings["Save"])) + connection.send_message("mud_savefile", data) + except InvalidTokenError: + connection.send_message("mud_login_denied") + +@handler("mud_save", dict) +def mud_save(connection, contents): + + # No token means the save isn't ready yet + if contents["UniteAuthToken"] is None or contents["UniteAuthToken"].strip() == "": + return + + # If the server shuts down, all the clients that were left open + # will reconnect as soon as it comes back on. But, they don't + # bother to re-authenticate on their own, so the server has to + # prompt them to save to get the copy of the auth token that is + # in the save. + if connection.authsession is None: + try: + connection.authsession = connection.auth.session(contents["UniteAuthToken"]) + except InvalidTokenError: + connection.error("Your token is incorrect and you could not be re-authenticated with the server") + return + + user = connection.authsession.User + save = user.Save + if save is None: + save = Save() + user.Save = save + map_from(user, mappings["UserSave"], contents) + map_from(save, mappings["Save"], contents) + connection.dbsession.query(Upgrade).filter_by(Save = save).delete() + if contents["Upgrades"] is not None: + for k, v in contents["Upgrades"].items(): + connection.dbsession.add(Upgrade(Name = k, Installed = v, Save = save)) + connection.dbsession.query(StoryExperienced).filter_by(Save = save).delete() + if contents["StoriesExperienced"] is not None: + for v in contents["StoriesExperienced"]: + connection.dbsession.add(StoryExperienced(Name = v, Save = save)) + if contents["Users"] is not None: + users_new = {u["Username"]: u for u in contents["Users"]} + for usr in connection.dbsession.query(ClientSave).filter_by(Save = save): + if usr.Username in users_new: + map_from(usr, mappings["ClientSave"], users_new[usr.Username]) + del users_new[usr.Username] + else: + connection.dbsession.delete(usr) + for data in users_new.values(): + usr = ClientSave(Save = save) + map_from(usr, mappings["ClientSave"], data) + connection.dbsession.add(usr) diff --git a/server.py b/server.py new file mode 100644 index 0000000..1f0451a --- /dev/null +++ b/server.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +from twisted.internet.endpoints import TCP4ServerEndpoint +from twisted.internet import reactor + +from mud import MudConnectionFactory + +endpoint = TCP4ServerEndpoint(reactor, 13370) +endpoint.listen(MudConnectionFactory()) +reactor.run() + + diff --git a/servermessage.py b/servermessage.py new file mode 100644 index 0000000..d53e254 --- /dev/null +++ b/servermessage.py @@ -0,0 +1,24 @@ + +from netclass import netclass +from netobject import NetObject, NetObjectStream + +ServerMessage = netclass("ShiftOS.Objects.ServerMessage", + "ShiftOS.Objects, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + [(str, "Name"), (str, "Contents"), (str, "GUID")]) + +class ServerMessageStream(NetObjectStream): + def netObjectReceived(self, obj): + if not isinstance(obj.Object, ServerMessage): + raise TypeError(f"An object was received on the ServerMessageStream of type {type(obj.Object)}") + + self.serverMessageReceived(obj.Object) + + def sendServerMessage(self, message): + if not isinstance(message, ServerMessage): + raise TypeError(f"The ServerMessageStream can only send ServerMessage, not {type(obj)}") + + # Although the real ShiftOS fills in the Name field on the + # NetObject, it does not ever read it, so it's not really + # part of the protocol. + self.sendNetObject(NetObject(None, message)) + diff --git a/unite.py b/unite.py new file mode 100644 index 0000000..b5515cc --- /dev/null +++ b/unite.py @@ -0,0 +1,203 @@ +import base64 +import re +import traceback +import uuid + +import bcrypt +import flask +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import NoResultFound + +from auth import Auth, InvalidTokenError +from database import User, DbSession, ValidationError +import mapping +from myconfig import mappings + +app = flask.Flask(__name__) + +def cb_decorator(fn): + return lambda *args, **kwargs: lambda cb: fn(*args, **kwargs, cb = cb) + +@cb_decorator +def db_route(path, cb): + @app.route(path, endpoint = path) + def db_handler(*args, **kwargs): + dbsession = DbSession() + try: + ret = cb(*args, **kwargs, dbsession = dbsession) + except (ValidationError, NoResultFound): + traceback.print_exc() + return flask.Response(status = 400) + try: + dbsession.commit() + except IntegrityError as ex: + # 1062 is the SQL error code for duplicate entry. + # That's a user input error, but if we get some other kind + # of integrity error, then it's a server error. + if ex.orig.args[0] == 1062: + return flask.Response(status = 400) + else: + raise + dbsession.close() + return ret + return db_handler + +# /Auth/ endpoints use basic authentication and create new tokens. +@cb_decorator +def auth_route(path, cb): + @db_route(f"/Auth{path}") + def auth_handler(*args, dbsession, **kwargs): + try: + kind, value = flask.request.headers["Authentication"].split(" ") + assert kind == "Basic" + email, password = base64.b64decode(value).decode().split(":") + except (KeyError, ValueError, AssertionError): + print("no basic auth") + return flask.Response(status = 401) + email = email.strip() + return cb(*args, **kwargs, dbsession = dbsession, email = email, password = password) + return auth_handler + +# /API/ endpoints use token authentication and operate as a +# Unite user. +@cb_decorator +def api_route(path, cb): + @db_route(f"/API{path}") + def api_handler(*args, dbsession, **kwargs): + try: + kind, token = flask.request.headers["Authentication"].split(" ") + assert kind == "Token" + auth = Auth(dbsession, flask.request.remote_addr) + authsession = auth.session(token) + except (KeyError, ValueError, AssertionError, InvalidTokenError): + return flask.Response(status = 401) + return cb(*args, **kwargs, dbsession = dbsession, authsession = authsession) + return api_handler + +@auth_route("/Register") +def register(email, password, dbsession): + + def error(message): + return flask.jsonify({"ClassName":"System.Exception","Message":message,"Data":None,"InnerException":None,"HelpURL":None,"StackTraceString":None,"RemoteStackTraceString":None,"RemoteStackIndex":0,"ExceptionMethod":None,"HResult":-2146233088,"Source":None}) + + try: + displayname = flask.request.args["displayname"] + sysname = flask.request.args["sysname"] + appname = flask.request.args["appname"][:255] + appdesc = flask.request.args["appdesc"][:255] + version = flask.request.args["version"][:255] + except KeyError as ex: + return error(str(ex)) + + # Additional constraints for password + if len(password) < 7: + return error("Password too short") + requirements = [".*[A-Z].*", ".*[a-z].*", ".*[0-9].*"] + for expr in list(requirements): + if re.compile(expr).match(password) is None: + return error("Password does not meet the requirements") + + password = bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode() + + try: + user = User(ID = str(uuid.uuid4()), Email = email, Password = password, DisplayName = displayname, SysName = sysname) + except ValidationError: + return error(str(ex)) + dbsession.add(user) + + try: + dbsession.commit() + except IntegrityError as ex: + if ex.orig.args[0] == 1062: + dbsession.rollback() + return error(ex.orig.args[1]) + else: + raise + + auth = Auth(dbsession, flask.request.remote_addr) + return auth.create_session(user, appname, appdesc, version).Token + +@auth_route("/Login") +def login(email, password, dbsession): + if email == "" or password == "": + return flask.Response(status = 400) + + try: + appname = flask.request.args["appname"][:255] + appdesc = flask.request.args["appdesc"][:255] + version = flask.request.args["version"][:255] + except KeyError: + return flask.Response(status = 400) + + try: + user = dbsession.query(User).filter_by(Email = email).one() + except NoResultFound: + print("acct does not exist") + return flask.Response(status = 401) + + if user.Password is None: # login disabled + print("login disabled") + return flask.Response(status = 401) + + if not bcrypt.checkpw(password.encode(), user.Password.encode()): + print("password incorrect") + return flask.Response(status = 401) + + auth = Auth(dbsession, flask.request.remote_addr) + return auth.create_session(user, appname, appdesc, version).Token + + +@api_route("/GetDisplayName/") +def get_display_name(uid, dbsession, authsession): + return dbsession.query(User).filter_by(ID = uid).one().DisplayName + +@api_route("/GetPongHighscores") +def get_pong_highscores(dbsession, authsession): + return flask.jsonify({"Pages": 0, # unused + "Highscores": [mapping.map_to(user, mappings["PongHighscore"]) + for user in dbsession.query(User).all() + if user.PongLevel is not None and user.PongCP is not None]}) + +@api_route("/GetEmail") +def get_email(dbsession, authsession): + return authsession.User.Email + +def gettersetters(name, convert = lambda v: v): + @api_route(f"/Get{name}") + def getter(dbsession, authsession): + return str(getattr(authsession.User, name)) + @api_route(f"/Set{name}/") + def setter(value, dbsession, authsession): + setattr(authsession.User, name, convert(value)) + return flask.Response(status = 200) + +@api_route("/GetPongCP") +def get_pong_cp(dbsession, authsession): + return str(authsession.User.PongCP or 0) + +@api_route("/SetPongCP/") +def set_pong_cp(value, dbsession, authsession): + value = int(value) + if authsession.User.PongCP is not None and value <= authsession.User.PongCP: + return flask.Response(status = 400) + authsession.User.PongCP = value + return flask.Response(status = 200) + +@api_route("/GetPongLevel") +def get_pong_level(dbsession, authsession): + return str(authsession.User.PongLevel or 0) + +@api_route("/SetPongLevel/") +def set_pong_cp(value, dbsession, authsession): + value = int(value) + if authsession.User.PongLevel is not None and value <= authsession.User.PongLevel: + return flask.Response(status = 400) + authsession.User.PongLevel = value + return flask.Response(status = 200) + +gettersetters("SysName") +gettersetters("DisplayName") +gettersetters("FullName") +gettersetters("Codepoints", convert = int) + + diff --git a/unite.wsgi b/unite.wsgi new file mode 100644 index 0000000..ba5e26b --- /dev/null +++ b/unite.wsgi @@ -0,0 +1,4 @@ +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +from unite import app as application