Python Forum
Thread Rating:
  • 1 Vote(s) - 5 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Mini-Web Framework
#1
So I've had an idea running around my head for a while, and I decided to finally do something about it and try it out.  The basics of the mini framework are simple: async by default, and routing is handled via annotations.  I like Flask a lot, but using decorators for url routing is exceptionally ugly to me.  You're applying meta-information in a decorator... the only reason people have done that in the past, was because it was the most practical method available.  Now we have a better alternative, and I wanted to explore that to see if it could actually work as nicely as I thought it could.

While I'm at it, I thought setting this up to be namespaced by default would help to encourage modular design, such that each group of related controllers is within its own namespace (ie: /user/edit, /user/add, /user/list).

This is just an experiment.  It is obviously not meant for production usage.  It's a toy.  Play with it if you want to.
def index() -> "/":
   # routing is handled through an annotation
   # you can read it "function_name 'maps to' url_route"
   return "hi"


async def user(id: int = None) -> "/user/{id}":
   # annotations in arguments causes web.py to auto-cast url args
   # cast failrues (id: int = "bob") get redirected to the namespaced 404
   # urls are written in pseudo-string format syntax...
   # ...web.py converts the pseudo-string to a regex for speed
   return repr(id)


async def user_error(id: None = None) -> "/user/{id}":
   # this will work, but PLEASE don't do that.  You'll confuse anyone reading your code
   # that said, any int id will be routed to user(), while anything else goes
   # to user_error()
   return "you wrong, sucka"


async def all_users() -> "/users":
   # Controllers (I refer to them as Handlers a lot) can be plain functions, or coroutines
   # plain functions will be wrapped into coroutines by asyncio.
   # probably best to just define your controllers as async, unless that's too ugly for you
   # TODO: test if async has any speed impact, or if wrapping native
   # functions is Actively Bad
   return "admin users"


async def not_found():
   return "derp"

if __name__ == "__main__":
   import web

   # make a website!
   site = web.site()

   # / -> index()
   site.route(index)
   # /user/12345 -> user(12345)
   site.route(user)
   # /does-not-exist -> not_found()
   site._404(not_found)

   # bind some handlers to a "namespace"
   # ...all routes will be attached under this top-level-uri
   with site.namespace("/admin") as admin_area:
       # /admin/users -> all_users()
       admin_area.route(all_users)

   # run the website!
   site.run()
That works, though without templating or a decent web server, it's still just a half-baked toy.  Here's the actual web.py:
import asyncio
import re


class Route:
   def __init__(self, path="/", handler=None, formatters={}):
       self.path = path
       self.formatters = formatters
       self.regex = self.compile(path)
       handler = handler if handler else lambda: None
       # fake async for non-async handlers
       self.handler = asyncio.coroutine(handler)

   def compile(self, path="", formatters={}):
       if not isinstance(path, str):
           # feel free to write your own regex
           return path

       # TODO: uri type-intelligent arguments
       path = re.sub(r"\{([^/]+)\}", r"([^/]+)", path)

       path = "^{0}$".format(path)
       return re.compile(path)

   def __call__(self):
       return self.handler()

   #... probably not useful
   def __contains__(self, uri="/"):
       return self.regex.match(uri) is not None

   def __getitem__(self, uri):
       match = self.regex.match(uri)
       if not match:
           return None
       # format data
       return match

   def __repr__(self):
       return self.path


class Namespace:
   # what we call a namespace, other frameworks refer to as "apps" or "modules"
   # a namespace is a self-contained collection of endpoints, bound to a url
   # path
   def __init__(self, site=None, attached="/"):
       self.site = site
       self.attachment_point = attached
       self.routes = []

   def __enter__(self):
       return self

   def __exit__(self, *args):
       self.site.register_namespace(self.attachment_point, self.routes)

   def route(self, route):
       path = None
       anno = route.__annotations__
       if "return" in anno:
           path = anno["return"]
           del anno["return"]

       # TODO: lists should be allowed in annotations so a controller can
       # listen to multiple uris
       self.routes.append(Route(path=path, handler=route, formatters=anno))

       # allow for route chaining
       return self

   def _404(self, handler):
       self.errors = {"404": Route(handler=handler)}

   def namespace(self, base):
       return Namespace(self, base)

   def register_namespace(self, attachment_point, routes):
       for route in routes:
           new_path = "{0}{1}".format(attachment_point, route.path)
           new_route = Route(
               path=new_path,
               handler=route.handler,
               formatters=route.formatters
           )
           self.routes.append(new_route)


class Site(Namespace):
   # a site is just the root node of a namespace heirarchy
   # only the root node (ie: the Site) can actually be run
   # sub-namespaces are dependent upon their parent
   def run(self):
       import http.server as http

       class Handler(http.BaseHTTPRequestHandler):
           def do_GET(server):
               content = None
               for route in self.routes:
                   data = route[server.path]
                   if data:
                       content = self.handle(route, data)
                       break
               else:
                   if "404" in self.errors:
                       content = self.handle(self.errors["404"])
               if content:
                   server.wfile.write(content.encode())

       with http.HTTPServer(("", 8080), Handler) as httpd:
           print("Starting test server...")
           httpd.serve_forever()

   # TODO: have an event loop that processes incomplete requests
   # TODO: handle() should just add things to the event loop
   # TODO: ...actually pass url parameters to controller
   def handle(self, route, data={}):
       runner = route()
       try:
           while True:
               runner.send(None)
       except StopIteration as resp:
           return resp.value


# nice things should look nice :)
site = Site
To me, the most interesting thing currently is the namespacing.  For example, this snippet (already listed above):
   # bind some handlers to a "namespace"
  # ...all routes will be attached under this top-level-uri
  with site.namespace("/admin") as admin_area:
      # /admin/users -> all_users()
      admin_area.route(all_users)
The admin_area controller only lists it's route as "/users", but because it's routed to the /admin namespace, it will only respond to "/admin/users"... the controller doesn't have any concept of where it's actually mounted, it only knows what it needs to.  The idea here, is to have a class of controllers that only refer to things like /edit or /add, so you can share that module easily, and it can be mounted easily anywhere.  The request object which will get passed to each controller will then contain the current namespace, so the controller's view can build accurate urls.

Roughly, something like this:
class User:
   def index(self) -> "/":
       pass

   def add(self) -> "/add":
       pass

   def edit(self, id) -> "/edit/{id}":
       pass

   def route(self, mount):
       mount.route(self.index).route(self.add).route(self.edit)

site = web.site()
with site.namespace("/user") as ns_user:
   user = User()
   user.route(ns_user)
So anyway, here's a little something I've been messing around with.  The routing works, and a server starts up to serve content based on the routing, but that's all that's done.
I'd still like to do:
-type-dependent uri routing, based on annotations (and auto-casting to those types before dispatching)
-actually using asyncio.  I mostly just faked it long enough to have the test script work, but it's still trash.
-move the http.server.HTTPServer to a backend module that handles the binding to a port and communicating with a socket
-create another backend using Twisted using the same interface, except... actually usable for production environments
-wrap jinja in a nice interface for simple templating
Reply
#2
For more info on why I wanted to try routing via annotations, let's start with what other frameworks/languages do, and why I think most of them are bad.

In the beginning, there was php/perl, where your urls were decided by your file names. If you later want to refactor your code, you need to leave "stub" files with the original file names that do nothing but call the new files, so old bookmarks to your site continue to work. That's terrible.

Then people started using frameworks. Cakephp, CodeIgniter, Ruby on Rails, Asp.net and many others all follow the same pattern. Which is to say, your urls are decided for you based on your class/method names. Which, again, makes refactoring your code difficult, if not just completely infeasible.

Django, I feel, was a good step in the right direction. By having all the urls defined in a urls.py file, which then mapped to whatever end point you wanted to handle that url, you could change the underlying application at any time, without fear of breaking any urls. The only problem, was that the urls were defined in urls.py... so far away from the controller (they call it a view) which handles that url. In the real world, you link pages together with page navigation, so knowing what the url is actually matters, and should be easily identifiable for a particular controller.

Which is why I like Flask. The url is detached from the file structure, while still being physically close to the method which actually handles the request, so writing the view is easy since you can easily see which urls go where.

But Flask uses a decorator to apply the routing. I view a controller's url as little more than metadata for that controller, so wrapping the endpoint in a decorator feels like using a sledge hammer where a rubber mallet is much more suitable. Things that mutate the controller, should be things that actually mutate the in/output, not things that just keep track of metadata. Before annotations were added, however, decorators were the cleanest way to handle it.

But now we have annotations. So now I wanted to try to utilize them, to show semantic meaning to the site I'm building. So far, I think it doesn't look that bad.
Reply
#3
Hm!
As I know you annotations are applicable to type you may create. To a class for example?
"As they say in Mexico 'dosvidaniya'. That makes two vidaniyas."
https://freedns.afraid.org
Reply
#4
An annotation can be anything, as long as it's legal python. So it could be a custom class, a string, a type, a tuple of types, a range, etc. Python doesn't actually care what they are, and doesn't do any checking whatsoever that what's passed to the function matches the annotation... it's basically just a comment that you can use basic reflection to read.
Reply
#5
Yeah, the work I'm doing currently (in Java) uses annotations for paths. I agree 100% about annotations making more sense that decorators.

I'm curious though what your thoughts are when multiple annotations need to be present. Java's syntax for annotations is similar to Python's decorator syntax. For example,
@POST
@Path("/users/{userUUID}/preferences")
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Produces(MediaType.APPLICATION_JSON)
is what it might look like in Java (this is using a Java standard interface). How would you model HTTP methods using Python's annotations? It seems that decorators could be used in parallel with annotations, but that decorators are still not the write thing to be using, and trying to make the annotation fit more things in one line doesn't seem like it's going to work out in the long run.
Reply
#6
The cool thing about annotations is they can be any python object.  So just to get started, I was using strings, but I was planning on having a string default to GET, while if you wanted something else, you could use something else.  ie:

# GET only
def index() -> "/":
   pass

# POST only
def submitted(data: Payload) -> POST("/submit"):
   pass

# Either, and also returning json data
# this is a tuple of multiple endpoints that this controller listens to...
# I could also see using an Enum type interface so you could OR Get and Post together, defining the actual uri only once
@format.json
def get_data() -> ("/data", POST("/data")):
   return {"spam": "eggs"}
Reply
#7
I actually really like Flask.  And... value my time.  And have no desire to maintain a web framework.  So I put a little bit of effort into taking the bit that I like (...and that actually works), and changed it to basically be a translation layer between annotations and Flask.  It doesn't add anything to the runtime of flask, everything this component does is completely at startup, so there's zero performance impact.

On the (minor) downside, async doesn't work with Flask, so that part of what I had before no longer works.  Oh well.

Also, it's on github now: https://github.com/nilamo/flask-routing

Example app:
def index() -> "/":
   # routing is handled through an annotation
   # you can read it "function_name 'maps to' url_route"
   return "hi"


def user(id: int = None) -> "/user/{id}":
   # annotations in arguments causes web.py to auto-cast url args
   # cast failrues (id: int = "bob") get redirected to the namespaced 404
   # urls are written in pseudo-string format syntax...
   # ...web.py converts the pseudo-string to a regex for speed
   return repr(id)


def all_users() -> "/users":
   # Controllers (I refer to them as Handlers a lot) can be plain functions, or coroutines
   # plain functions will be wrapped into coroutines by asyncio.
   # probably best to just define your controllers as async, unless that's too ugly for you
   # TODO: test if async has any speed impact, or if wrapping native
   # functions is Actively Bad
   return "admin users"


def not_found(error):
   return "derp", 404


if __name__ == "__main__":
   import routing
   from flask import Flask

   # make a website!
   app = Flask(__name__)
   with routing.Root(app) as root:
       # / -> index()
       root.route(index)
       # /user/12345 -> user(12345)
       root.route(user)
       # /does-not-exist -> not_found()
       root.error(404, not_found)

       # bind some handlers to a "namespace"
       # ...all routes will be attached under this top-level-uri
       with root.namespace("/admin") as admin_area:
           # /admin/users -> all_users()
           admin_area.route(all_users)

   # run the flask app!
   app.run()
And the actual wrapper around flask:
import flask


class Route:
   def __init__(self, path="/", handler=None, formatters={}):
       self.path = self.translate(path, formatters)
       self.handler = handler if handler else lambda: None

   # flask expects url params to be wrapped in angle brackets.
   # we allow for string formatting syntax, and translate into something flask understands.
   # furthermore, if there's an annotation, we add the flask-compatable type
   # to the url
   def translate(self, path, formatters):
       type_map = {
           int: "int",
           float: "float",
           str: "string",
           #"path": "path",
           #"any": "any",
           #"uuid": "uuid"
       }

       for url_param in formatters:
           formatter = formatters[url_param]
           if formatter in type_map:
               formatter = type_map[formatter]
           _from = "{{{0}}}".format(url_param)
           to = "<{0}:{1}>".format(formatter, url_param)
           path = path.replace(_from, to)

       return path

   def __repr__(self):
       return "{0} => {1}".format(self.path, self.handler)


class Namespace:
   # what we call a namespace, other frameworks refer to as "apps" or "modules"
   # a namespace is a self-contained collection of endpoints, bound to a url
   # path
   def __init__(self, site=None, attached="/"):
       self.site = site
       self.attachment_point = attached
       self.routes = []
       self.errors = []

   def __enter__(self):
       return self

   def __exit__(self, *args):
       self.site.register_namespace(self.attachment_point, self.routes)

   def error(self, error_code, handler):
       self.errors.append({"code": error_code, "handler": handler})

   def route(self, route):
       path = None
       anno = route.__annotations__
       if "return" in anno:
           path = anno["return"]
           del anno["return"]

       # TODO: lists should be allowed in annotations so a controller can
       # listen to multiple uris
       self.routes.append(Route(path=path, handler=route, formatters=anno))

       # allow for route chaining
       return self

   def namespace(self, base):
       return Namespace(self, base)

   def register_namespace(self, attachment_point, routes):
       for route in routes:
           new_path = "{0}{1}".format(attachment_point, route.path)
           new_route = Route(
               path=new_path,
               handler=route.handler
           )
           self.routes.append(new_route)


class Root:
   def __init__(self, app: flask.Flask):
       self.app = app
       self.root = None

   def __enter__(self):
       self.root = Namespace()
       return self.root

   def __exit__(self, *_):
       for route in self.root.routes:
           # flask uses decorators for the routing.
           # so let flask's routing know what would-have-been decorated.
           self.app.route(route.path)(route.handler)
       for err in self.root.errors:
           self.app.errorhandler(err["code"])(err["handler"])
Reply
#8
Could you make a tutorial about all of this? Annotations I mean.
"As they say in Mexico 'dosvidaniya'. That makes two vidaniyas."
https://freedns.afraid.org
Reply
#9
Sure, I suppose I've messed around with them enough that I could do that comfortably.
The important thing to keep in mind, is that they 100% don't matter. Python almost completely ignores them. I feel that they should be used almost exclusively as a stronger hint about paramaters/return values than just using comments. Or for things like what I did here, where they're short cuts to generating code or event handlers or something, or maybe for an automated testing framework.
Reply
#10
This part is what I am interested to: -> some_function
The syntax is the easy one.
"As they say in Mexico 'dosvidaniya'. That makes two vidaniyas."
https://freedns.afraid.org
Reply


Possibly Related Threads…
Thread Author Replies Views Last Post
  Guess the dice roll mini-game tawnnx 6 7,249 May-22-2018, 02:12 PM
Last Post: malonn
  5 mini programming projects for the python beginner kerzol81 4 36,112 Sep-26-2017, 02:36 PM
Last Post: VeenaReddy

Forum Jump:

User Panel Messages

Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020