How network games work...

Highly handy comment from PlayerController.uc in the Unreal Development Kit: Multiplayer net games 101..

/*
========================================================================
Here's how player movement prediction, replication and correction works in network games:

Every tick, the PlayerTick() function is called.  It calls the PlayerMove() function (which is implemented
in various states).  PlayerMove() figures out the acceleration and rotation, and then calls ProcessMove()
(for single player or listen servers), or ReplicateMove() (if its a network client).

ReplicateMove() saves the move (in the PendingMove list), calls ProcessMove(), and then replicates the move
to the server by calling the replicated function ServerMove() - passing the movement parameters, the client's
resultant position, and a timestamp.

ServerMove() is executed on the server.  It decodes the movement parameters and causes the appropriate movement
to occur.  It then looks at the resulting position and if enough time has passed since the last response, or the
position error is significant enough, the server calls ClientAdjustPosition(), a replicated function.

ClientAdjustPosition() is executed on the client.  The client sets its position to the servers version of position,
and sets the bUpdatePosition flag to true.

When PlayerTick() is called on the client again, if bUpdatePosition is true, the client will call
ClientUpdatePosition() before calling PlayerMove().  ClientUpdatePosition() replays all the moves in the pending
move list which occured after the timestamp of the move the server was adjusting.
*/

Unrealscript Mode for Emacs Reloaded

I actually started picking up Unreal again, and Unrealscript. My previous attempt at an Unrealscript mode was badly flawed. It was based on CC-mode, which I did not fully pursuade to understand the syntax of Unrealscript. This time, I've gone back to first principles and created simple progmode not derived from cc-mode. This means a loss of functionality, but it does mean correct indenting, and true case insensitivity. I found CC mode would have to be patched to handle a case insensitive language like UnrealScript. Anyway, without further ado, here it is. I reccomend it over the previous ones.

(defvar unrealscript-mode-hook nil)

(defvar unrealscript-mode-map
  (let ((unrealscript-mode-map (make-sparse-keymap)))
 (define-key unrealscript-mode-map "\C-j" 'newline-and-indent)
 unrealscript-mode-map)
  "Keymap for UNREALSCRIPT major mode")


(defconst unrealscript-font-lock-keywords-1
  (list
   '("\\<\\(?:break\\|c\\(?:\\(?:as\\|ontinu\\)e\\)\\|do\\|e\\(?:lse\\|xtends\\)\\|for\\(?:each\\)?\\|i\\(?:f\\|nterface\\)\\|new\\|return\\|switch\\|var\\|while\\|class\\)\\>" . font-lock-keyword-face))
  "Minimal highlighting expressions for UNREALSCRIPT mode.")

(defconst unrealscript-font-lock-keywords-2
  (append unrealscript-font-lock-keywords-1
    (list
     '("\\<\\(?:array\\|b\\(?:ool\\|yte\\)\\|c\\(?:lass\\|o\\(?:erce\\|lor\\|ords\\)\\)\\|de\\(?:faultproperties\\|legate\\)\\|e\\(?:num\\|vent\\)\\|f\\(?:alse\\|loat\\|unction\\)\\|int\\|local\\|name\\|o\\(?:ptional\\|ut\\)\\|plane\\|r\\(?:egion\\|otator\\)\\|st\\(?:ate\\|r\\(?:ing\\|uct\\)\\)\\|true\\|v\\(?:\\(?:a\\|ecto\\)r\\)\\)\\>" . font-lock-keyword-face)))
  "Additional Keywords to highlight in UNREALSCRIPT mode.")


(defconst unrealscript-font-lock-keywords-3
  (append unrealscript-font-lock-keywords-2
    (list
     '("\\<\\(?:A\\(?:bstract\\|llowAbstract\\|uto\\(?:Comment\\|ExpandCategories\\)\\)\\|Co\\(?:llapseCategories\\|nfig\\)\\|D\\(?:ep\\(?:endsOn\\|recated\\)\\|isplayName\\|ontCollapseCategories\\)\\|Edit\\(?:Condition\\|InlineNew\\)\\|FriendlyName\\|Hide\\(?:Categories\\|DropDown\\)\\|I\\(?:\\(?:mplemen\\|nheri\\)ts\\)\\|N\\(?:ative\\(?:Replication\\)?\\|o\\(?:Export\\|nTransient\\|t\\(?:EditInlineNew\\|Placeable\\)\\)\\)\\|P\\(?:erObject\\(?:Config\\|Localized\\)\\|laceable\\)\\|ShowCategories\\|T\\(?:oolTip\\|ransient\\)\\|Within\\)\\>" . font-lock-type-face)
     '("\\<\\(?:auto\\|c\\(?:lient\\|on\\(?:fig\\|st\\)\\)\\|d\\(?:atabinding\\|eprecated\\|uplicatetransient\\)\\|e\\(?:dit\\(?:const\\|fixedsize\\|inline\\(?:use\\)?\\|oronly\\)\\|x\\(?:ec\\|port\\)\\)\\|globalconfig\\|i\\(?:gnores\\|n\\(?:it\\|put\\|stanced\\|terp\\)\\|terator\\)\\|l\\(?:atent\\|ocalized\\)\\|n\\(?:ative\\(?:replication\\)?\\|o\\(?:clear\\|export\\|import\\|ntransactional\\|tforconsole\\)\\)\\|operator\\|p\\(?:o\\(?:\\(?:inte\\|stoperato\\)r\\)\\|r\\(?:eoperator\\|ivate\\|otected\\)\\|ublic\\)\\|re\\(?:liable\\|pnotify\\)\\|s\\(?:erver\\|i\\(?:mulated\\|ngular\\)\\|kip\\|tatic\\)\\|tra\\(?:nsient\\|vel\\)\\|unreliable\\)\\>" . font-lock-keyword-face)
     '("\\<\\(?:A\\(?:bs\\|cos\\|dd\\(?:Item\\)?\\|llActors\\|s\\(?:c\\|in\\)\\|tan\\)\\|B\\(?:asedActors\\|egin\\(?:Play\\|State\\)\\)\\|C\\(?:aps\\|eil\\|h\\(?:ildActors\\|r\\)\\|l\\(?:amp\\|earTimer\\)\\|o\\(?:\\(?:llidingActor\\)?s\\)\\)\\|D\\(?:estroyed\\|i\\(?:\\(?:sabl\\|vid\\)e\\)\\|ynamicActors\\)\\|E\\(?:mpty\\|n\\(?:\\(?:abl\\|dStat\\)e\\)\\|val\\|xp\\)\\|F\\(?:Clamp\\|M\\(?:ax\\|in\\)\\|Rand\\|astTrace\\|in\\(?:d\\|ish\\(?:Anim\\|Interpolation\\)\\)\\)\\|G\\(?:etTimer\\(?:Count\\|Rate\\)\\|oTo\\(?:State\\)?\\)\\|I\\(?:n\\(?:Str\\|itGame\\|sert\\(?:Item\\)?\\|vert\\)\\|s\\(?:\\(?:InStat\\|TimerActiv\\)e\\)\\)\\|L\\(?:e\\(?:ft\\|n\\|rp\\)\\|o\\(?:cs\\|ge\\)\\)\\|M\\(?:ax\\|i\\(?:rrorVectorByNormal\\|[dn]\\)\\)\\|Normal\\|OverlappingActors\\|P\\(?:o\\(?:pState\\|stBeginPlay\\)\\|reBeginPlay\\|ushState\\)\\|R\\(?:and\\|e\\(?:move\\(?:I\\(?:ndex\\|tem\\)\\)?\\|pl\\(?:ace\\)?\\)\\|ight\\|ound\\)\\|S\\(?:et\\(?:State\\|Timer\\)\\|in\\|leep\\|merp\\|p\\(?:awn\\|lit\\)\\|q\\(?:rt\\|uare\\)\\)\\|T\\(?:an\\|ick\\|ouchingActors\\|race\\(?:Actors\\)?\\)\\|V\\(?:Rand\\|Size\\|isible\\(?:\\(?:Colliding\\)?Actors\\)\\)\\|\\(?:ro\\|vec\\)t\\)\\>" . font-lock-function-name-face)))
  "Balls-out highlighting in UNREALSCRIPT mode.")

(defvar unrealscript-font-lock-keywords unrealscript-font-lock-keywords-3
  "Default highlighting expressions for UNREALSCRIPT mode.")


(defvar unrealscript-indent-width 4)

(defun looking-at-unrealscript-indent-keyword ()
  (or (looking-at "^[\t ]*while") (looking-at "^[\t ]*if") (looking-at "^[\t ]*else") (looking-at "^[\t ]*for")))

(defun looking-at-unrealscript-block-end ()  
  (or (looking-at "^[\t ]*end") (and (looking-at "^.*}[ \t]*$") (not (looking-at "^.*{")))))

(defun looking-at-unrealscript-block-start ()
  (or (looking-at "^[\t ]*begin") (and (looking-at "^.*{") (not (looking-at "^.*}[ \t]*$")))))

;; Function to control indenting.
(defun unrealscript-indent-line ()
  "Indent current line as Unrealscript code"
  (interactive)
  ;; Set the point to beginning of line.
  (beginning-of-line)
  (if (bobp)
   (indent-line-to 0)
 (let ((not-indented t) (lines-back 0) cur-indent)
   (if  (looking-at-unrealscript-block-end) ; Check for closing brace
    ;; if we are at the end of a block
    (progn
   (save-excursion
     (forward-line -1)
     (setq lines-back (+ lines-back 1))
     (setq cur-indent (- (current-indentation) unrealscript-indent-width)))
   ;; Safety check to make sure we don't indent negative.
   (if (< cur-indent 0)
    (setq cur-indent 0)))
  ;; else scan backward 
  (save-excursion
    (if (looking-at-unrealscript-block-start) ; Opening block     
     (progn
    (forward-line -1)
    (setq lines-back (+ lines-back 1))
    (setq cur-indent (current-indentation))
    (setq not-indented nil))
   (while not-indented
     (forward-line -1)
     (setq lines-back (+ lines-back 1))
     (if (looking-at-unrealscript-block-end) ;; Closing Block
      (progn
     (setq cur-indent (current-indentation))
     (setq not-indented nil))
    (if (looking-at-unrealscript-block-start)
     (progn
       (setq cur-indent (+ (current-indentation) unrealscript-indent-width))
       (setq not-indented nil))
      (if (looking-at-unrealscript-indent-keyword)
       (progn
      (setq cur-indent (current-indentation))
      (forward-line 1)
      (setq lines-back (- lines-back 1))
      (if (looking-at-unrealscript-block-start)
       (setq not-indented nil) ;; has block
        (if (zerop lines-back) ;; no block
         (progn
        (setq cur-indent (+ cur-indent unrealscript-indent-width))       
        (setq not-indented nil))
       (setq not-indented nil))))
     (if (bobp)
      (setq not-indented nil)))))))))
   (if cur-indent
    (indent-line-to cur-indent)
  (indent-line-to 0))))) 

(defun unrealscript-populate-syntax-table (table)
  "Populate the given syntax table as necessary for a C-like language.
This includes setting ' and \" as string delimiters, and setting up
the comment syntax to handle both line style \"//\" and block style
\"/*\" \"*/\" comments."

  (modify-syntax-entry ?_  "_"    table)
  (modify-syntax-entry ?\\ "\\"    table)
  (modify-syntax-entry ?+  "."    table)
  (modify-syntax-entry ?-  "."    table)
  (modify-syntax-entry ?=  "."    table)
  (modify-syntax-entry ?%  "."    table)
  (modify-syntax-entry ?<  "."    table)
  (modify-syntax-entry ?>  "."    table)
  (modify-syntax-entry ?&  "."    table)
  (modify-syntax-entry ?|  "."    table)
  (modify-syntax-entry ?\' "\""    table)
  (modify-syntax-entry ?\240 "."  table)

  ;; Set up block and line oriented comments.  The new C
  ;; standard mandates both comment styles even in C, so since
  ;; all languages now require dual comments, we make this the
  ;; default.
  (modify-syntax-entry ?/  ". 124b" table)
  (modify-syntax-entry ?*  ". 23" table)

  (modify-syntax-entry ?\n "> b" table)
  ;; Give CR the same syntax as newline, for selective-display
  (modify-syntax-entry ?\^m "> b" table)
  table)

(defvar unrealscript-mode-syntax-table
  (let ((unrealscript-mode-syntax-table (unrealscript-populate-syntax-table (make-syntax-table))))
 unrealscript-mode-syntax-table)
  "Syntax table for unrealscript-mode")

(defun unrealscript-mode ()
  "Major mode for editing Unrealscript files"
  (interactive)
  (kill-all-local-variables)
  (set-syntax-table unrealscript-mode-syntax-table)
  (use-local-map unrealscript-mode-map)
  (setq indent-line-function 'unrealscript-indent-line)
  (setq font-lock-defaults '(unrealscript-font-lock-keywords nil t))
  (setq major-mode 'unrealscript-mode)
  (setq mode-name "UNREALSCRIPT")
  (run-hooks 'unrealscript-mode-hook)
  (setq case-fold-search t)
  (setq font-lock-keywords-case-fold-search t))


(provide 'unrealscript-mode)

A micro manual for Lisp in C++

It was with some interest that I read A Micro Manual for Lisp in C. One thing I have noticed about such efforts is that they are nearly always implemented in idiomatic C. I thought (out of idle curiosity, mainly) I'd see what a similar thing implemented in idomatic C++ would look like.

The main practical advantage that it offered in the end was the fact that boost::shared_ptr gave me rudimentary garbage collection for free. I used boost::variant to give me polymorphic lisp objects and thus did not have to use naked pointers anywhere, which seems to me to be an improvement in code safety. The price of this is a nosier syntax. You pay your money and you takes your choice..

As a concrete example here is what the implementation of CONS looked like.

boost::shared_ptr<lisp::object> fn_cons(boost::shared_ptr<lisp::object> args,
                                        boost::shared_ptr<lisp::object> env) {

    boost::shared_ptr<lisp::object> list = make_cons(car(args), boost::shared_ptr<lisp::object>());
    args = car(cdr(args));

    while ((args != NULL) && (args->which() == lisp::e_CONS)) {
        append(list, car(args));
        args = cdr(args);
    }
    return list;
}

Its probably also worth noting that avoiding naked pointers meant using boost::function for function objects, thusly..

    enum kind {
        e_ATOM = 0,
        e_CONS,
        e_FUNC,
        e_LAMBDA
    };

    struct atom {
        std::string name;

        atom(const std::string& n) : name(n) {
        }
    };

    struct cons {
        boost::shared_ptr<object> car;
        boost::shared_ptr<object> cdr;

        cons(boost::shared_ptr<object> first, boost::shared_ptr<object> second) : car(first), cdr(second) {

        }
    };

    typedef boost::function< boost::shared_ptr<object> (boost::shared_ptr<object>,boost::shared_ptr<object>) > lisp_func;

    struct func {
        lisp_func fn;

        func(const lisp_func& f) : fn(f) {

        }
    };

    struct lambda {
        boost::shared_ptr<object> args;
        boost::shared_ptr<object> sexp;

        lambda(boost::shared_ptr<object> a, boost::shared_ptr<object> s) : args(a), sexp(s) {

        }
    };

    typedef boost::variant< atom, cons, func, lambda > base_object;

And creating the environment like so...

boost::shared_ptr<lisp::object> init_env() {

    boost::shared_ptr<lisp::object> nul;

    boost::shared_ptr<lisp::object> a_quote(new lisp::object(std::string("QUOTE")));
    boost::shared_ptr<lisp::object> f_quote(new lisp::object(fn_quote));

    boost::shared_ptr<lisp::object> a_car(new lisp::object(std::string("CAR")));
    boost::shared_ptr<lisp::object> f_car(new lisp::object(fn_car));

    boost::shared_ptr<lisp::object> a_cdr(new lisp::object(std::string("CDR")));
    boost::shared_ptr<lisp::object> f_cdr(new lisp::object(fn_cdr));

    boost::shared_ptr<lisp::object> a_cons(new lisp::object(std::string("CONS")));
    boost::shared_ptr<lisp::object> f_cons(new lisp::object(fn_cons));

    boost::shared_ptr<lisp::object> a_equal(new lisp::object(std::string("EQUAL")));
    boost::shared_ptr<lisp::object> f_equal(new lisp::object(fn_equal));

    boost::shared_ptr<lisp::object> a_atom(new lisp::object(std::string("ATOM")));
    boost::shared_ptr<lisp::object> f_atom(new lisp::object(fn_atom));

    boost::shared_ptr<lisp::object> a_cond(new lisp::object(std::string("COND")));
    boost::shared_ptr<lisp::object> f_cond(new lisp::object(fn_cond));

    boost::shared_ptr<lisp::object> a_lambda(new lisp::object(std::string("LAMBDA")));
    boost::shared_ptr<lisp::object> f_lambda(new lisp::object(fn_lambda));

    boost::shared_ptr<lisp::object> a_label(new lisp::object(std::string("LABEL")));
    boost::shared_ptr<lisp::object> f_label(new lisp::object(fn_label));


    boost::shared_ptr<lisp::object> env = make_cons(make_cons(a_quote,make_cons(f_quote,nul)),nul);


    append(env,make_cons(a_car,   make_cons(f_car,nul)));
    append(env,make_cons(a_cdr,   make_cons(f_cdr,nul)));
    append(env,make_cons(a_cons,  make_cons(f_cons,nul)));
    append(env,make_cons(a_equal, make_cons(f_equal,nul)));
    append(env,make_cons(a_atom,  make_cons(f_atom,nul)));
    append(env,make_cons(a_cond,  make_cons(f_cond,nul)));
    append(env,make_cons(a_lambda, make_cons(f_lambda,nul)));
    append(env,make_cons(a_label,  make_cons(f_label,nul)));

    boost::shared_ptr<lisp::object> a_tee(new lisp::object(std::string("#T")));
    tee = a_tee;
    nil = make_cons(nul,nul);

    return env;
}

OpenGL Development on Windows with Slime

I've been playing with OpenGL, Common Lisp (SBCL, CCL, Lispworks Personal) and SLIME again. Mostly I've been establishing a S-Expression exporter for Blender so I can export entire scenes to the engine I'm working on.

I did discover some wrinkles of working on OpenGL development with SLIME in Windows. CCL uses the :spawn communication style in SLIME which uses it's multi-threaded nature to run each repl request in a seperate thread from the executing lisp code. So if you have your OpenGL app running in the background as you develop it, if you write an experimental form at the repl that includes OpenGL calls, its going to fail, since it will not be executing in the main OpenGL thread.

The solution is to fall back on the nil communication style, which is not threaded and is blocking. This has the less than desirable effect of blocking your repl until the executing app finishes. This can be circumvented by calling swank-handle-requests - like so

(iterate 
 (initially (progn ,init-forms))
 (with-simple-restart 
  (skip-photons-loop "Skip photons loop body")
  ,@body)
 #+photons-debug 
 (with-simple-restart
  (skip-swank-request "Skip swank evaluation")
  (let ((connection
   (or swank::*emacs-connection* (swank::default-connection))))
 (swank::handle-requests connection t)))
 (glfwSwapBuffers)
 (until ,exit-test)
 (finally (progn ,exit-forms))))

There are actually some subtelties here. The skip-photons-loop restart allows you to recover from an error in the loop body, by recompiling the offending form, when the handle-requests function is called, while the skip-swank-request restart allows you to recover from entering an invalid form at the repl without losing the application.

Rooms at last

You might have noticed a hiatus on this project, but finally I returned to it to finish off the floorplan: here it is. The rooms were generated mainly by using polar coordinates around the space defined by the corridoor end points, and turned out to be relatively straightforward.

At this point, the project really needs to switch away from pyglet to something more useful, that can do the needed 3d extrusion more easily...

Dungeon Generation Part 4: Clearing space for the rooms

Here we are, we now have spaces where the rooms are going to go: my initial attempts at creating rooms did not go so well, so I had to backtrack and refactor the code, in order to get this to work.



Its actually a fairly simple computation: work out the length of the corridor, scale it, and re-adjust the co-ordinates of each end. This requires some book keeping in terms of data structures: knowing which end terminates at which room, so that you don't accidentally srhink the worong end.

Dungeon Generation Part 3 : Doors

Thinking about key points in positioning a room, I decided doors were a good idea, so I tried to code up a routine that would reasonably place doors. I interpolated about 30% up along a line and placed a small black perpendicular line prependicular to the line along the centre of the corridor. I went with the simple rule of thumb that a point that connected more than two corridors was the centre of the room. I got this:

Not bad, but not quite right. Probably the best way forward is to decide on an overall radius that will establish the size of the room and fit the doors to that.

Dungeon Generation Part Two: Corridors

And once the network is complete, we do corridors. This is straigtforward geometry: extrude two lines at either end of our connecting line, perpendicular to said line, then join them up. When you do that you get this:

A floorplan which should extrude nicely into the third dimension. However there's one more thing to do before we tackle this, and that is the actual rooms. Which is where things get a tad harder..

Dungeon generation for fun and..well, fun...

Recently I was inspired by the PixelCity series of articles. I was impressed what could be achieved by judicious application of basic OpenGL and some clever aesthetic decisions. I'm inspired enough to try the same thing, only with dungeons. My weapons of choice are pyglet and Python.

The first and most obvious thing to try was to generate a network of corridors. Here's what I ended up with.

I generated this mainly by picking a point and then connecting it to one of it's closest neighbours. The points are initially distributed on a grid and peturbed by a random amount to make the layout less "manhattan" like. The order the points are picked in is related to the grid order, which is why you see a "spine" at the side of the dungeon. I thought this would make for interersting gameplay: monster - infested branches connected by a relatively safe trunk. So this is what I went with. In the next part, we will get some corridoor walls. Once I have a decent looking dungeon, I will release the source.

Give it a kick, it might still be going..

Just to prove I haven't been completely inactive on the project front since my flirtation with Lisp (which isn't over yet, either, just maturing slowly), here's a screenshot of yet another thing. Basically this is a mesh importer for the excellent OGRE engine, powered by wxWidgets and The Open Asset Import Library. It doesn't handle textures or bones yet, but it does more than enough for now as the Open Asset Import Library imports an ungodly number of formats, it should be possible to magpie bits of free art from the web in whatever format and assemble them into a passable scene using this thing..

More Emacs Help

As well as keying the help provided by Google in Emacs to the major-mode, it's possible to use it with multiple libraries as well. For instance, the following snippet lets me look up documentation for one library (OGRE) by pressing 'o' and for another by pressing 'a' , rather like the Slime selector, discussed previously. It's a huseful way of pulling together doxygen API documentation when working on clonk-loads of libraries, as is all to typical of C++ or Java development.

(defun search-site-url (keyword &optional site inurl lucky)
 "Do a Google search for KEYWORD. Restrict to SITE and INURL, if specified.
Jump to best match (I Feel Lucky) if LUCKY set.
"
;;  (message (list keyword site inurl lucky))
 (concat "http://www.google.com/"
         (format "search?q=%s" (url-hexify-string keyword))
         (if site (format "+site:%s" (url-hexify-string site)))
         (if inurl (format "+inurl:%s" (url-hexify-string inurl)))
         (if lucky "&btnI")))


(defun help-selector ()
 (interactive)
 (message "Select [%s]: "
          (apply #'string (mapcar #'car help-selector-methods)))
 (let* ((ch (save-window-excursion
              (select-window (minibuffer-window))
              (read-char)))
        (params (find ch help-selector-methods :key #'car)))
   (browse-url (apply #'search-site-url (thing-at-point 'symbol) (list   (third params) (fourth params) (fifth params))))))

(defvar help-selector-methods '(( ?a "Assimp"
                                    "assimp.sourceforge.net" "/lib_html/" t)
                               ( ?o "Ogre"
                                    "www.ogre3d.org"  "/docs/api/html/" t)
                               ( ?w "WxWidgets"
                                     "docs.wxwidgets.org" "/stable/" t)))

(global-set-key [(shift f1)] 'help-selector)

Lambda of the Daleks

As promised, the Daleks appear. They have been loaded dynamically from an md2 whilst the framework was running. Sometimes I find myself fighting my C++ reflexes. There's no need to bring the framework down in order to change it. It still seems like magic when I edit a function definition, compile it and *bam* the change appears in the game window...






...one day, all software will be developed this way. Onwards..

And now for a bit of swank..

Now my testbed talks happily to emacs via slime. That's the "exploratory" part of "exploratory game developement.". Next up: Daleks...

And now for something completley different.

You guessed it - another testbed. This time, in 100% Common Lisp (SBCL), just for comparison.

More testbed

I've done some more playing around with that testbed, integrating ECL and G3Ds Shape API. It works ok, apart from a few wrinkles. It might even be the beginnings of a nice educational app for learning 3d transforms and the like. The main improvement is that the REPL gives you error feedback rather than crashing the app when you give it a bad form. A distinct must for interactivity...

What next?

Embedding ECLS in a 3D Engine - G3D - proved remarkably easy. The question is - now what. What should live on the Lisp side and what should live on the C++ side? Geometry for C++? AI for Lisp..what about collisions, then? Hmm..

Using Google for Context Sensitive Help in Emacs

Here's a hack that combines three of the most useful things in the known Universe: Google, Emacs and Firefox. Emacs 22.x has a standard method for invoking a browser: the browse URL function. To customise it use:

M-x customize-group browse-url

We are interested in the settings: Browse Url Browser Function which should be set to

browse-url-firefox

Then there is Browse Url Firefox Program which should be set to the full path of wherever firefox lives on your machine, and Browse Url Firefox New Window Is Tab which should be on to prevent multiple firefoxen cluttering up your windows when you try this.

Now that you have customized everything, you drop a crafted function into your .emacs and bind it to a key:

;; -- INTEGRATED HELP
(require 'url)
(defun search-site-url (site url keyword)
(concat "http://www.google.com/"
    (format "search?q=%s+site:%s+inurl:%s&btnI"  
        (url-hexify-string keyword)  
        (url-hexify-string site)  
        (url-hexify-string url))))

(defun wxhelp ()
"Open a window showing the wxWidgets documentation for the word under the point"
(interactive)
(browse-url (search-site-url  "docs.wxwidgets.org" "2\\\\.8\\\\.6" 
               (thing-at-point 'symbol))))

(global-set-key "\C-h\C-w" 'wxhelp)

Now, if you type Control H Control W while the cursor is over a symbol, Emacs will look it up via Google. For instance, if your cursor is over wxApp, by the magic of "I'm feeling Lucky", Google will find http://docs.wxwidgets.org/2.8.6/wx_wxapp.html which is the manual page for wxApp

Obviously the interesting function here is

site-search-url

which munges up an URL to feed to google to make it search a specific site for a keyword, filtering URLs that do not contain a certian string. The highly escaped \\\\ is to ensure that 2\.8\.6 appears in the final URL so that Google treats . as a literal.

The great utility of this hack is its universiality. It will work for any language or library with a sanely organised reference web site. Language specific lookup functions based on sites like cppreference.com are left as an exercise for the reader...

Emacs Selectors

One of the nicer things that comes with slime is the slime-selector: it's just a little buffer switching thing, that lets you press r to switch to the repl buffer, d to switch to the debugger buffer, s to switch to the scratch buffer and so forth. Bound to a function key, it's an extraordinarily convenient way of switching to a specific buffer.

Of course, it's geared to SLIME and Lisp, but when coding C++ on the day job I often find myself switching between code, the debugger, shell and compilation buffer, so without further ado, here is the C equivalent, which is easily hacked up to include other buffers, too: actually, I'm beginning to think something like this functionality should be in emacs by default, as it eliminates a lot of keyboard gymnastics.

;; slime selector clone for c  ----------------------------------

(defvar c-selector-methods nil)

(defun c-selector ()
  (interactive)
  (message "Select [%s]: "
           (apply #'string (mapcar #'car c-selector-methods)))
  (let* ((ch (save-window-excursion
               (select-window (minibuffer-window))
                              (read-char)))
         (method (find ch c-selector-methods :key #'car)))
    (cond ((null method)
           (message (format "No method for charachter: %s" ch))
           (ding)
           (sleep-for 1)
           (discard-input)
           (c-selector))
          (t 
           (funcall (third method))))))
          
         
(defmacro def-c-selector-method (key description &rest body)
  `(setq c-selector-methods
         (sort* (cons (list ,key ,description
                            (lambda () 
                              (let ((buffer (progn ,@body)))
                                (cond ((get-buffer buffer)
                                       (switch-to-buffer buffer))
                                      (t
                                       (message  "No such buffer")
                                       (ding))))))
                      (remove* ,key c-selector-methods :key #'car))
                #'< :key #'car)))
  
(def-c-selector-method ?d "GDB debugger buffer"
  gud-comint-buffer)

(def-c-selector-method ?g "Grep buffer"
  grep-last-buffer)

(def-c-selector-method ?l "Buffer List"
  (let ((result (get-buffer "*buffer-selection*")))
    (if result
        result
      ((bs-show)
       (get-buffer "*buffer-selection*")))))

(def-c-selector-method ?c "Compilation buffer"
  (let ((result (get-buffer "*compilation*")))
    (if result
        result
      (compilation-find-buffer))))

(def-c-selector-method ?s "Shell buffer"
  (get-buffer "*shell*"))

ASDF for the slightly confused

Well, you have a shiny new lisp compiler/interpreter/environment on your machine, and like an eager and smart newbie you want to do something with it. Right there and then many lisp newbies go SPLAT and end up as a bug on a windscreen, later to be scraped off and fed to Guido's pet snake. So here is ASDF for confused newbies (a very different proposition from a dummy, who should go back to reading bright yellow paperbacks and programming Java in the enterprise shop window).

ASDF is about 500 lines of lisp source, and a dot asd file is the lisp equivalent of the ubqutious make; you can't do anything in a serious modern lisp environment without it, so lets take you through it step by step. An asd file describes a system that contains modules and components. Components are usually individual files. Modules are collections of files, usually in subdirectories. Let's look at an example dot asd file: this one is for cowl - a simple opengl gui written by William Robinson who is working on Cityscape, a promising
game found at this address.

(defsystem #:cowl-ftgl
 :depends-on (#:cffi)
 :description "FTGL Common Lisp wrapper for Cowl."
 :components ((:module "src"
              :components 
             ((:file "cowl-ftgl")))))

The defsystem form then defines a system with one component which is a module called src which in turn contains one component in a file called cowl-ftgl, which will be a lisp file, naturally: since it's in a module called src, it will be loaded in the src directory below the directory that holds the dot asd file. Sometimes it is necessary to wrap the defsystem form in it's own package using defpackage and a throwaway package name. We do not do that here because we do not define any symbols that we would need to refer to externally, but we would need to if we were defining new component classes or methods to be used in the system, examples of which will follow later in the article.

Components, modules and systems are defined as CLOS objects in asdf.lisp, so of course they can be operated upon with generic functions, and there are quite a few that come with asdf. The key one is named with Arc-like gnomicness: "oos" - which I imagine is short for "Operate on System". It takes at least two parameters: the operation to perform and the system to perform it on.

The operation you are most likely to perfom is load-op: this loads and compiles a system, and the form is simply:

(asdf:oos 'asdf:load-op 'system-name)

Other operations are compile-op, and test-op: the latter will only have meaning if the author of your system has provided tests, as some packages do.

ASDF will search for systems using the list of functions referred to by *system-definition-search-functions* used to search for .asd files in the filesystem. By default this contains only one function, (sysdef-central-registry-search) which is perfectly adequate for file systems with symbolic links. The modus operandi on such system is to have a share/common-lisp/systems directory populated with symbolic links to dot asd files which may be downloaded and untarred in any part of the fs tree to which the user has access. Then the (sysdef-central-registry-search) scans the list of directories referred to by *central-registry* for dot asd files containing the system which it needs to operate on.

To put it another way, procedurally this translates to:

1. Download your asdf package


2. Unzip - say to ~/thinngy/cl-blab


3. Note that ~/thinngy/cl-blab/cl-blab.asd is the system.


4. Create a ~/share/common-lisp/systems directory for systems
   

5. cd ~/share/common-lisp/system
   

6. ln -s ~/thinggy/cl-blab/cl-blab.asd cl-blab.asd
   

7. now fire up your lisp and enter..
   

   
   ;; for systems that don't do (require 'asdf)
   (load #"/path/to/asdf.lisp")
   
   ;; the / on the end of the path is important, don't forget it
   (push #"/home/me/share/common-lisp/systems/" asdf:*central-registry*)
   
   (asdf:oos 'asdf:load-op 'cl-blab)
   

For Windows the situation is sligtly more complicated: either you write your own search function *or*, more practically apply a patch that lets you use shortcuts in place of symlinks.

You'd be right to point out that this is on the laborious side, but the purpose of this article is to demonstrate how asdf *works*. There are two packages for automating lisp package and asdf system installation: asdf-install and clbuild, which I encourage newbies to investigate, in the usual sadistic exercise for the reader.

Why go through all this palaver with asdf? Well the beauty of asdf is that systems, components and modules are in fact CLOS objects, so we can do tricks like this:

(cffi:load-foreign-library 
 (make-pathname 
  :name "cftgl" 
  :type "so" 
  :directory (directory-namestring (asdf:component-pathname 
                                    (asdf:find-system :cowl-ftgl)))))

..which loads "clftgl.so" which is found in the same directory as the dot asd file containing the system definition. Highly useful when distributing a C shared lib that wraps a C++ library so that it can be called into from Lisp.

The fact that source files are component, lets us write specialist methods for different kinds of source file such as c-source-file, java-source-file, html-file (all defined in asdf.lisp) allows asdf to work as a make replacement, operating on non-lisp source, thus:

(defmethod output-files ((op compile-op) (c c-source-file))
    (list (make-pathname :type "o" :defaults
                                                 (component-pathname c))))

(defmethod perform ((op compile-op) (c c-source-file))
    (unless
       (= 0 (run-shell-command "/usr/bin/gcc -fPIC -o ~S -c ~S"
                               (unix-name (car (output-files op c)))
                               (unix-name (component-pathname c))))
     (error 'operation-error :operation op :component c)))

(defmethod perform ((operation load-op) (c c-source-file))
    t)   

There is an extended example of this kind of thing where asdf is extended to work with a fortran to lisp converter here.

To conclude: like many things in Lisp, asdf is prickly for newbies and takes time to get to know and use well. However, when you do you can do things with it for which you'd normally need another build system, independent of your language. As Brucio would observe: Lisp does not have this limitation and Common Lispers laugh at things like ant.

Hippie Expand and Autocompletion in Emacs

The Emacs way of doing auto - completion is known as Hippie Expand which is a function that uses a variety of methods to try and expand an incomplete word near the point. Why it's called Hippie Expand I have no idea. The name is a bit misleading; for a long time I thought it was related to zippy or pinhead. Perhaps it's something to do with Emacs being used and written by hippies, who knows?

The nice thing about hippie expand is that like many things in Emacs, it can be customized and expanded, via the make-hippie-expand function. For instance, here is the hippie-expand function I use for c-mode

(make-hippie-expand-function
'(try-expand-dabbrev-visible
 try-expand-dabbrev-from-kill
 try-expand-dabbrev-all-buffers
 try-complete-file-name-partially
 try-complete-file-name)))))

This function first tries to expand the thing found at the point as a dynamic abbreviation, which is based on a scan of existing text for words or symbols with the same beginning as the one at the point. For instance, if "catalogue" is in the buffer, and you expand "cat", then "catalogue" will be one of the expansions offered, which is useful if "catalogue" is a variable name you use a lot.

The hippie expand function is bound to a key (I use M-/) and each press of that key will cycle through the possible completions for you. It's not Intellisense, but it does cut down on typing if you use descriptive variable names.

It's also fairly trivial to write your own hippie-expand function. Here is an example of a hippie expand function which does completion based on a list of current tags - which are sometimes more useful than dabbrevs.

;; This is a simple function to return the point at the beginning of the symbol to be completed
(defun he-tag-beg ()
(let ((p
      (save-excursion
        (backward-word 1)
        (point))))
 p))

;; The actual expansion function
(defun try-expand-tag (old)
;; old is true if we have already attempted an expansion
(unless  old
 ;; he-init-string is used to capture the string we are trying to complete
 (he-init-string (he-tag-beg) (point))
 ;; he-expand list is the list of possible expansions
 (setq he-expand-list (sort
                       (all-completions he-search-string 'tags-complete-tag) 'string-lessp)))
;; now we go through the list, looking for an expansion that isn't in the table of previously
;; tried expansions
(while (and he-expand-list
           (he-string-member (car he-expand-list) he-tried-table))
           (setq he-expand-list (cdr he-expand-list)))
;; if we didn't have any expansions left, reset the expansion list
(if (null he-expand-list)
   (progn
     (when old (he-reset-string))
     ())
 ;; otherwise offer the expansion at the head of the list
 (he-substitute-string (car he-expand-list))
 ;; and put that expansion into the tried expansions list
 (setq he-expand-list (cdr he-expand-list))
 t))
;; done, now we just use it as a clause in our make-hippie-expand-function (as above)

Hippie-expand isn't the only auto-completion gizmo I work with in Emacs; there is also slime-complete-symbol which does symbol completion for my while I am working in SLIME. This can be neatly integrated into hippie-expand, like so..

;; hippie expand slime symbol
(defun he-slime-symbol-beg ()
(let ((p
      (slime-symbol-start-pos)))
 p))

(defun try-expand-slime-symbol (old)
(unless  old
 (he-init-string (he-slime-symbol-beg) (point))
 (setq he-expand-list (sort
                       (car (slime-contextual-completions (slime-symbol-start-pos) (slime-symbol-end-pos))) 'string-lessp)))
(while (and he-expand-list
           (he-string-member (car he-expand-list) he-tried-table))
           (setq he-expand-list (cdr he-expand-list)))
(if (null he-expand-list)
   (progn
     (when old (he-reset-string))
     ())
 (he-substitute-string (car he-expand-list))
 (setq he-expand-list (cdr he-expand-list))
 t))

EDIT: With newer (CVS and 3.0) versions of SLIME the api for symbol completion
has changed and the hippie-expand function for slime symbol completion is..

;; hippie expand slime symbol
(defun he-slime-symbol-beg ()
(let ((p
      (slime-symbol-start-pos)))
 p))

(defun try-expand-slime-symbol (old)
(unless  old
 (he-init-string (he-slime-symbol-beg) (point))
 (setq he-expand-list (sort
                       (car (slime-simple-completions
                             (buffer-substring-no-properties (slime-symbol-start-pos) (slime-symbol-end-pos))))
                       'string-lessp)))
(while (and he-expand-list
           (he-string-member (car he-expand-list) he-tried-table))
           (setq he-expand-list (cdr he-expand-list)))
(if (null he-expand-list)
   (progn
     (when old (he-reset-string))
     ())
 (he-substitute-string (car he-expand-list))
 (setq he-expand-list (cdr he-expand-list))
 t))