Monday, July 1, 2013

Writing a real time single page application - server side

In the first part I highlighted how to take care of the frontend of a real time web app. Now I'll explain how to face the backend. As usual I will not dive into details, follow the links if you search in depth explanations.

Server side

I wrote the server side of my application using node.js and a very lightweight framework called express.js
The most important feature of this framework is the middleware system.
This is a middleware:

function middleware(req, res, next){
...
}

A middleware is a sort of russian doll. You can put a middleware inside another middleware (the "next" argument).

A middleware can basically:
  • call the next middleware in the chain ( next() )
  • call the output function ( res() )
  • change the input (req)
  • overwrite the output function (res)

You can use a middleware for:
  • authenticate and get user informations
  • route to a specific middleware using the URL (req.url) and the method (req.method)
  • add a specific header to the HTTP response
  • etc.

Using middlewares is very common because it allows to build simpler and reusable components.

Express.js gives you already a lot of middlewares but I used passportjs to get a complete solution for authentication.
In this example I will store users into couchdb using nano

var config = require('./config'), // I used a config.json to store configuration parameters
    db_url = config.db_protocol + "://" + config.db_user + ":" + config.db_password + "@" + config.db_url;
    nano = require('nano')(db_url), 
    setupAuth = require('./auth'),
    MemoryStore = express.session.MemoryStore,
    sessionStore = new MemoryStore(), // nano will store user id in this session
    passport = setupAuth(userdb);

var app = express(),
    server = http.createServer(app);
    
// configure Express
app.configure(function() {
    app.set('views', __dirname + '/views');
    app.set('view engine', 'ejs');
    app.use(express.logger());
    app.use(express.cookieParser());
    app.use(express.bodyParser());
    app.use(express.methodOverride());
    app.use(express.session({ store: sessionStore, secret: config.session_secret, key: config.session_key }))
    // Initialize Passport!  Also use passport.session() middleware, to support
    // persistent login sessions (recommended).
    app.use(flash());
    app.use(passport.initialize());
    app.use(passport.session());
    app.use(app.router);
    app.use(express.static(__dirname + '/' + config.static_dir));
});

var ensureAuthenticated = function(req, res, next) {
    if (req.isAuthenticated()) {
        return next();
    }
    res.redirect('/login');
};

// I check if a user is authenticated before accessing this URL
app.get('/', ensureAuthenticated, function(req, res){
  res.render('index', { user: req.user});
});

// login
app.get('/login', function(req, res){
    res.render('login', { user: req.user });
});

app.post('/login', 
    passport.authenticate('local', { failureRedirect: '/login', failureFlash: true }),
function(req, res) {
    res.redirect('/');
});

app.get('/logout', function(req, res){
  req.logout();
  res.redirect('/login');
});

server.listen(config.http_port);

The auth module contains functions used by passport:

var passport = require('passport'),
    LocalStrategy = require('passport-local').Strategy,
    crypto = require('crypto');


module.exports = function (userdb){
    var getUserFromId = function(id, done) {
        userdb.get(id, { revs_info: false }, function(err, body) {
            if (!err){
                return done(null, body);
            }
            else {
                return done(null, false, { message: 'Invalid credentials' });
            }
        });
    };

    passport.getUserFromId = getUserFromId;
    
    passport.serializeUser(function(user, done) {
        done(null, user._id);
    });
        
    passport.deserializeUser(getUserFromId);

    passport.use(new LocalStrategy(
        function(username, password, done) {
            var shasum = crypto.createHash('sha1').update(password),
                key = [username, shasum.digest('hex')]

            // this view's keys are [username, password]
            // the password is hashed of course
            userdb.view("user", "login", { keys: [key] }, function (err, body){
                if (!err) {
                    if (body.rows.length){
                        // logged in !!!
                        return done(null, body.rows[0].value);
                    }
                    else {
                        return done(null, false, { message: 'Invalid credentials' });
                    }
                }
                else {
                    console.error(err);
                    return done(null, false, { message: 'Db error, try later' });
                }
            });        

        })
    );

    
    return passport;
};

This example is explained in the passport documentation.
If you are careful you will notice that the getUserFromId function is called every time to get the complete object from the database (couchdb in this case).
This is not very optimal and it's better to cache users for some time. I used this nice memoization module:

    var memoize = require('memoizee');
    
    // cache this function for optimal performance (2 minutes)
    // https://npmjs.org/package/memoizee
    getUserFromId = memoize(getUserFromId, { maxAge: 120000,  async: true});

Backbone.io in the server

At this point I will define a backbone.io backend (as explained here: https://github.com/scttnlsn/backbone.io)

var items_backend = backboneio.createBackend();

A backend is very similar to an express.js middleware.

var backendMiddleware1 = function(req, res, next) {
   ...
};

items_backend.use(backendMiddleware1);
items_backend.use(backendMiddleware2);

Once defined, I will connect the backend.

var io = backboneio.listen(server, {items: items_backend});

The io object which is returned by the listen method is a socket.io object.

When a websocket is connected for the first time it makes a basic handshake. This phase can be used to perform the passport authentication.

var cookie = require('cookie'),
    cookiesig = require('cookie-signature');
// socket io authorization
io.set('authorization', function (data, accept) {
    if (data.headers.cookie) {
        data.cookie = cookie.parse(data.headers.cookie);

        // cookies are signed for better security:
        //
        // s:name:signature
        //
        // s: is a prefix for signed cookies
        // name is the cookie name
        // signature is an hmac of the value
        // doing this the client cannot change the cookie value
        // without invalidate the cookie
        if (data.cookie[session_key].indexOf('s:') === 0){
            data.sessionID = cookiesig.unsign(data.cookie[session_key].slice(2), session_secret);
        }
        else {
            data.sessionID = data.cookie[session_key];
        }
        
        // (literally) get the session data from the session store
        sessionStore.get(data.sessionID, function (err, session) {
            
            if (err || !session) {
                // if we cannot grab a session, turn down the connection
                accept('Cannot get sessionid', false);
            } else {
                // save the session data and accept the connection
                data.session = session;
                if("passport" in session && "user" in session.passport){
                    passport.getUserFromId(session.passport.user, function (err, user, message){
                        if(err || !user){
                            accept('Cannot find user', false);
                        }
                        else {
                            try{
                                data.user = user;

                                accept(null, true);
                            }
                            catch (e){
                                accept('Error: ' + e.toString(), false);
                            }

                        }
                        
                    });
                }
                else {
                    accept('Session does not contain userid', false);
                }
            }
        });
    } else {
       return accept('No cookie transmitted.', false);
    }
});

The tricky part here is to extract the session from the (signed) cookie. The passport and sessionStore objects are the same defined before for normal authentication.
The backend authentication middleware can get the object through req.socket.handshake object:

var authMiddleware = function(req, res, next) {
    var user = req.socket.handshake.user;

    if (!req.user){
        next(new Error('Unauthorized'));
    }
    else {
        req.user = user;
        next();
    }

};

Backend events and channels

When a backend changes something backbone.io automatically broadcasts the change to every node connected (and triggers the events I talked before).
You often need to notify only a subset of clients. For this reason you can define channels. Every changes will be notified to clients connected to a certain channel.
The channel can be defined client side but I added a useful feature to define channel server side, during the handshake.

There is also another case where you need to detect whether couchdb has been changed by another application.
In this case I recommend you to use my fork of backbone.io  because it supports channels.

Database and flow control with promises

The last piece of the application is talking to the database. In my application I used couchdb but it is really not important which database will you use.
In the first paragraph I have underlined that a single resource operation could cause many operations in the backend. For this reason is very important using a smarter way to control the flow. I have chosen promises.

Promises are a standard pattern to manage asynchronous tasks. I used this library: https://github.com/kriskowal/q
The advantage of promises is to avoid the "pyramid of doom" of nested callbacks:

step1(function (value1) {
    step2(value1, function(value2) {
        step3(value2, function(value3) {
            step4(value3, function(value4) {
                // Do something with value4
            });
        });
    });
});

And transform that in something more manageable:

Q.fcall(step1)
.then(step2)
.then(step3)
.then(step4)
.then(function (value4) {
    // Do something with value4
}, function (error) {
    // Handle any error from step1 through step4
})
.done();

With promises, managing errors is very easy.
This is an example using nano:

var getUser = function (id) {
    var deferred = Q.defer();

    userdb.get(id, {}, function(err, body) {
        if (err) {
            deferred.reject(new Error('Not found'));
        }
        else {
            deferred.resolve(body);
        }
    });
    return deferred.promise;
};

var getGroup = function (user) {
    var deferred = Q.defer();

    userdb.get(user.groupid, {}, function(err, body) {
        if (err) {
            deferred.reject(new Error('Not found'));
        }
        else {
            deferred.resolve(body);
        }
    });
    return deferred.promise;
};

function getGroupFromUserId(id, callback){
    getUser(id)
    .then(getGroup)
    .then(function (group){
        callback(group);
    })
    .done();
}

Backbone.io and databases

Backbone.io has some ready to use backend and it is quite easy to write your own following the examples (I added the couchdb backend).

The end?

This ends this whirlwind tour. I hope someone will find this useful.

Friday, June 28, 2013

Writing a real time single page application - client side

Introduction

In this two parts guide I will explain how to use backbone.js, node.js and socket.io (and many others libraries) to write a single page/real time application.
The guide's goal is to explain a system and I will not dive into details of every component, for this reason I suggest you to follow the links to get more informations.
The topic is quite hot, so let's get started !

First step: defining resources

This part is the most boring but it is very important.
In my application I should model every client server exchange using REST principles  (even though I will eventually use websockets instead of HTTP protocol).

Following these principles I need to define resources.
A resource is a well defined piece of information

Every resource should:
  • have an id (the URL)
  • be atomic (every exchange will be stateless)
A very important detail here is that resources don't need to match exactly with backend models (database schemas). For example a resource can be the result of a join between tables.

I will leave to node.js backend to manage the differences between the actual db models and resources.
The backend can update more than one model in a reliable way (and, in case, rolling back changes) while it's not so reliable do this directly from the client.

From this point when I talk about models in the client I am talking about an HTTP resources.

Client side

For the front end I will use backbone.js. This library's task is basically to organize homogeneous models (they are resources!) into collections and keep these data synchronized with the UI (view) and the backend.
This is a model:

var Item = Backbone.Model.extend({
    defaults: {
       "name": "",
       "description": "",
    },
    validate: function(attrs, options) {
        // return a string is attributes are wrong
    }
});

This is a simple collection:

var Items = Backbone.Collection.extend({
    model: Item,
    initialize: function (){
        //...
    }
});

This collection contains a group of Items.
A backbone collection usually has an url attribute that is used to download data from the server. So I'll rewrite the last collection:

var Items = Backbone.Collection.extend({
    url: "/items"
    model: Item,
    initialize: function (){
        //...
    }
});

Backbone.js uses AJAX by default to download and update collections. It uses the standard HTTP methods: GET POST PUT DELETE (optionally PATCH if supported).
If the collection's url is "/items" the single model urls will be "/items/model_id".

Backbone.IO

One of the nicest feature of Backbone.js is that you can use different ways to save your data (overwriting the Backbone.sync function).
In this example I will use backbone.io. This component sends data to the backend using a websocket or another available socket like transport (it uses socket.io under the hood).

Backbone.io.connect();

var Items = Backbone.Collection.extend({
    backend: 'items',
    model: Item,
    initialize: function (){
        this.bindBackend();        
        //...
    }
});

Backbone.io.connect is used to initialize websockets. I replaced the url attribute with a backend identifier.
I have also launched "bindBackend". This method connect backbone.io events to the collection events. So when the server broadcasts that something is changed the collection triggers the event and views are refreshed.

Views

Backbone.js main feature is to be unopinionated. Views, for example, are mostly boilerplate code. For this reason, you often need to build a more high level framework on top of it.
It is usually a good idea to use something like Marionette.
Anyway I tried to build something simpler. This is a model's view:

var ModelView = Backbone.View.extend({
    initialize: function (){
        this.initialEvents();
    },
    initialEvents: function (){
        if (this.model){
          this.listenTo(this.model, "change", this.render);
        }
    },
    serialize: function (){
        var attrs = _.clone(this.model.attributes);
        attrs.id = attrs[this.model.idAttribute];
        return attrs;
    },
    render: function (){
        this.preRender();
        this.$el.html(this.template(this.serialize()));
        this.postRender();
        return this;
    },
    preRender: function (){
        // this runs before the rendering
    },
    postRender: function (){
        // this runs after the rendering
    }
});

I also build a collection's view:

var CollectionView = Backbone.View.extend({
    contentSelector: '.append-item-here',
    // this is the class where I append the model views
    viewOptions: {},
    initialize: function (){
        this.children = {}; // this will contain the models views
        this.initialEvents();
    },
    initialEvents: function (){
        if (this.collection){
          this.listenTo(this.collection, "add", this.addChildView);
          this.listenTo(this.collection, "remove", this.removeChildView);
          this.listenTo(this.collection, "reset", this.render);
          this.listenTo(this.collection, "sort", this.render);
        }
    },   
    render: function (){
        var that = this;
        this.preRender();

        // I remove the old models views
        _.each(this.children, function (view){
            view.remove();
        });

        this.children = {};

        // I rebuild the models views
        this.collection.chain()
        .each(function(item, index){
            // every model view have a reference to its collection view
            var options = _.extend({index: index, parentView: that}, that.viewOptions);
            that.addChildView(item, that.collection, options);
        });

        this.postRender();
        return this;
    },
    // I can use this to filter some view if it is necessary
    filterView: function (model){
        return true;
    },
    addChildView: function (item, collection, options){
        // add a view to this.children trying to respect the sorting order
        var index;
        if (!this.filterView(item)){
            return;
        }
        options.model = item;
        if (options && options.index){
            index = options.index;
        }
        else {
            index = collection.chain().filter(this.filterView, this).indexOf(item).value();
            if (index === -1){
                index = 0;
            }
        }

        var view = new this.itemView(options);
        view.render();

        this.children[item.cid] = view;
        this.addToView(view, index);
        
    },
    removeChildView: function (item, collection, options){
        this.children[item.cid].remove();
        delete this.children[item.cid];
    },

    addToView: function (view, index){
        // append a model view to the collection view in the correct sorting order
        var $el = this.$el.find(this.contentSelector),
            $children = $el.children();
        if (index >= $children.length ){
            $el.append(view.el);
        }
        else {
            $children.eq(index).before(view.el);
        }
    },
    preRender: function (){
        // this runs before the rendering
    },
    postRender: function (){
        // this runs after the rendering
    }

});

Extending these base views I can write a very simple item view:

var ItemView = ModelView.extend({
    template: _.template(html_item),
    tagName: 'div',
    events: {
        //...
    }
});

and, of course, a very simple collection view:

var ItemsView =  CollectionView.extend({
    itemView: ItemView,
    el: $("#items"),
    events: {
        //...
    }
});

You should notice that I am using the underscore template engine (_.template).

Routing and bootstrapping

Backbone uses a router object to keep the state of your application. The state is represented by the current URL.

var Router = Backbone.Router.extend({

    routes: {
        "item/:id":        "changeitem",  // #item/id
    },
    changeitem: function(id) {
        // this is called when the URL is changed
    }
});

Like others backbone primitives you can react to an URL change handling an event (inside a view for example):

var ItemsView =  CollectionView.extend({
    itemView: ItemView,
    el: $("#items"),
    initialize: function (){
        this.listenTo(router, 'route:changeitem', this.update);
    },
    update: function (){
        // ...
    },
    events: {
        //...
    }
});

When every component is in place you can instance it and bootstrap the application.

var router = new Router(); // router instance
 
var items = new Items(); // collection instance
var itemsView = new ItemsView({collection: items}); // collection view instance
    
Backbone.history.start(); // this initialize the router

items.fetch(); // this loads initially the collection from the server

The fetch method is not your best option if you want to load models for the first time. It's better to load models inline.


I hope everything is clear. Next part: the server side!

Friday, April 26, 2013

Objects in HTML5 canvas (part 3: interacting with images)

This article is the third of a series of simple recipes that explains how to interact with objects in HTML5 canvas.
  • part 1 (Drawing shapes)
  • part 2 (Interacing with objects)
In this part we will solve the last problem: how to interact with objects with irregular shapes.

The simple answer is that you don't have to. You can wrap any image inside a rectangular box and detect where the image is transparent.

Introducing offscreen canvas

An offscreen canvas is a canvas outside the DOM. It's very useful for 2 reasons:
  • you can do complex drawing once and reuse the result with just an instruction
  • you can use it to check where an image is transparent

Let's draw

First of all I start with an images array (I used inline images, but you can use images urls):

var objs = [
    {name:'firefox',
     src:'data:image/png;base64,iVBORw0KGgoAAAA.....',
     x:30,
     y:50,
     angle:Math.PI/2,
     },
    {name:'opera',
     src:'data:image/png;base64,iVBORw0KGgoAAAA...',
     x:40,
     y:30,
     angle:Math.PI/4,
     },
    {name:'chrome',
     src:'data:image/png;base64,iVBORw0...',
     x:60,
     y:100,
     angle:-Math.PI/4,
     }
];

Then I draw each image inside an offscreen canvas and, then the offscreen canvas in the main canvas:

var i, obj;
ctx.clearRect(0, 0, canvas.width, canvas.height);
for (i = 0;i < objs.length;i++){
    obj = objs[i];
    (function (obj){
        var img = new Image();   // Creating a new img element
        img.onload = function(){
            // It's very important to wait until the image loads!!!
            // otherwise I have no clue of the real size of the image 
            ctx.save();

            ctx.translate(obj.x, obj.y);
            ctx.rotate(obj.angle);
            obj.width = img.width;
            obj.height = img.height;
            //I attach the offscreen canvas to the object
            obj.canvas = document.createElement("canvas");
            obj.canvas.width = img.width;
            obj.canvas.height = img.height;
            obj.ctx = obj.canvas.getContext('2d');
            // I draw the image on a off-screen canvas
            obj.ctx.drawImage(img, 0 , 0);
            // I draw the off-screen canvas on the main canvas
            ctx.drawImage(obj.canvas, -img.width/2 , -img.height/2);
            
            ctx.restore();
        };
        img.src = obj.src; // Set source path
    }(objs[i]));
}

Pay attention! I have applied transformations on the main canvas and not on each  off-screen canvases.

Detecting mouse clicks

Let's finish detecting on which shape a user is pointing. As usual I check each object in reverse order and transform pointer coordinates. Then I must retrieve the image from the offscreen canvas and test the color on the resulting coordinates. If the alpha value is opaque the pointer is on the image.

    // I get a single pixel
    imageData = obj.ctx.getImageData(p.x+(obj.width/2), p.y+(obj.height/2), 1, 1);
    // imageData contains r,g,b,a
    if(imageData.data[3] > 50){ // 50 is the threeshold
        log.innerHTML = 'clicked on ' + obj.name;
        return;
    }            

I have taken a pixel from the canvas using getImageData. This function returns  an array that include 4 values for each pixel (just one in my case).
The values are Red, Green, Blue and Alpha (transparency) and they have values between 0 and 255.
I set a threeshold of 50 on alpha.
This is the whole code:

canvas.addEventListener('click', function (evt){
    var rect = canvas.getBoundingClientRect(),
        posx = evt.clientX - rect.left,
        posy = evt.clientY - rect.top,
        i = objs.length,
        obj,p;
        for (; i > 0; i--){
            obj = objs[i-1];
            // translate coordinates
            p = new Point(posx, posy);
            p.translate(-obj.x, -obj.y);
            p.rotate(-obj.angle);

            imageData = obj.ctx.getImageData(p.x+(obj.width/2), p.y+(obj.height/2), 1, 1);

            if(imageData.data[3] > 50){
                log.innerHTML = 'clicked on ' + obj.name;
                return;
            }            
            
        }
        log.innerHTML = '';

}, false);


And this is the results:
  Your browser doesn't support canvas!


I hope this is helpful!