The trick was to fetch an image at a URL, save it into GridFS, then be able to serve it back out – all without temp files or creating individual files for every image. Here’s a recipe that does it all!

I didn’t have much luck using the plain StringIO class and it’s tricky to get the right interfaces working together – some stream readers are more demanding than others…

… Enjoy!

/*
 * Template backbone.js code that encapsulates a 
 * datatable.net table within a view for a hypothetical ;)
 * auction listings application.
 */
 
var HomeRouter = Backbone.Router.extend({
  
  routes: {
    'refresh'     : 'refresh',   
    'newest'      : 'newest', 
    'endingsoon'  : 'endingsoon',
    'recentsold'  : 'recentsold', 
  },
    
});


var HomeView = Backbone.View.extend({
  
  // our default table view
  'present' : 'newest',
  
  'router' : new HomeRouter(),
  
  'events': {
    
    // users can enter a keyword,
    // but the table will only refresh after
    // the user can laid off the keys for 500ms
    'keyup #searchbox': _.debounce(function() {
      this.refresh();   
    }, 500 /* ms delay until taking real action */),
    
  },
  
  'newest': function() {
    this.present = 'newest';
    this.refresh();
  },
  
  'endingsoon': function() {
    this.present = 'endingsoon';
    this.refresh();   
  },
  
  'recentsold': function() {    
    this.present = 'recentsold';
    this.refresh();       
  },
  
  'refresh': function() {
    this.table.fnReloadAjax();    
  },

  
  'initialize': function() {
    
    var view = this;
    
    // setup routes
    //
    view.router.on('route:refresh', function() {      
      view.refresh();     
    });
    view.router.on('route:newest', function() {     
      view.newest();      
    });
    view.router.on('route:endingsoon', function() {     
      view.endingsoon();      
    });
    view.router.on('route:recentsold', function() {     
      view.recentsold();      
    });

    // setup table
    //
    view.table = $('#example').dataTable({
      "bProcessing": true,
      "bServerSide": true,
      "sAjaxSource": "/api/data",
  
      // override this, we supply our own keyword inputbox
      "sDom"       : 'p<"#toolbar">rtp',
  
      "aoColumns": [

        /* table declarations */

      ],
      
      "fnServerParams": function (aoData) {
        // before sending the request for data
        // we'll add our custom parameters for 'present' and 'keyword'
        if (view.present) {
          aoData.push( { "name": "present", "value": view.present } );
        }
        aoData.push( { "name": "keyword", "value": $('#searchbox').val() } );
      },      
    });   
  },  
});

/**
 * reload ajax plugin - available at http://datatables.net/plug-ins/api#fnReloadAjax
 */
$.fn.dataTableExt.oApi.fnReloadAjax = function ( oSettings, sNewSource, fnCallback, bStandingRedraw )
{

  if ( typeof sNewSource != 'undefined' && sNewSource != null ) {
    oSettings.sAjaxSource = sNewSource;
  }
 
  // Server-side processing should just call fnDraw
  if ( oSettings.oFeatures.bServerSide ) {
    this.fnDraw();
    return;
  }
 
  this.oApi._fnProcessingDisplay( oSettings, true );
  var that = this;
  var iStart = oSettings._iDisplayStart;
  var aData = [];
  
  this.oApi._fnServerParams( oSettings, aData );
    
  oSettings.fnServerData.call( oSettings.oInstance, oSettings.sAjaxSource, aData, function(json) {
    /* Clear the old information from the table */
    that.oApi._fnClearTable( oSettings );
      
    /* Got the data - add it to the table */
    var aData =  (oSettings.sAjaxDataProp !== "") ?
      that.oApi._fnGetObjectDataFn( oSettings.sAjaxDataProp )( json ) : json;
      
    for ( var i=0 ; i<aData.length ; i++ )
    {
      that.oApi._fnAddData( oSettings, aData[i] );
    }
      
    oSettings.aiDisplay = oSettings.aiDisplayMaster.slice();
      
    if ( typeof bStandingRedraw != 'undefined' && bStandingRedraw === true )
    {
      oSettings._iDisplayStart = iStart;
      that.fnDraw( false );
    }
    else
    {
      that.fnDraw();
    }
      
    that.oApi._fnProcessingDisplay( oSettings, false );
      
    /* Callback user function - for event handlers etc */
    if ( typeof fnCallback == 'function' && fnCallback != null )
    {
      fnCallback( oSettings );
    }
  }, oSettings );
};

/*
 * create the app view and start the browser history
 */
$(document).ready(function() {
    
  // app view
  new HomeView({el: $('body')});
  
  // get this party started
  Backbone.history.start();

} );

A couple of interesting things going on here programmatically: 1) using decorators that are static methods of a class that maintains some state for the decorated operations and 2) decorating class methods that are generators – the typical decorator can’t handle a yielded result…

Here’s a class that defines some static methods for us: scrape.route and scrape.tag. We’ll use these as decorators in another class that’s specific to our scraping task. These decorators will declare URL patterns that our decorated methods will handle when called using scrape.nav().

Now we’ll create a class that’s specific to our scraping task. We’ll use the scrape.route decorator to capture a web page for a specific URL pattern with: @scrape.route('/newreleases'). When we navigate to a URL patching this pattern, the class method weekly will be called.

Notice that when weekly is first called, it captures the total number of pages. We’ll use this information later on subsequent round-trips.

Now for the main event! We instantiate NewComics and pass the instance to the scrape constructor. This instance is our context object when we call the scrape.nav(url) method.

When nav(url) is called and matches a route we declared with the scrape.route decorator, the handler weekly(url, d) is called. ‘d‘ is our PyQuery object which can be used to traverse the document.

Route handlers like weekly, can be generators or return anything that suitable for the list() constructor. In this example, we yield the titles of upcoming comic book releases compliments of MyComicShop.com. Go grab the full source at GitHub!

TL;DR; Go right to the Gist on GitHub.

It’s even more apparent that NO ‘metadata’ is being used when Google Maps map tiles make it in the results…

It’s REALLY hard to deliver relevant images similarity results without any context…

tl;dr; live demo and gist

The header view below will initialize and bind a model to our view’s element using Rivets. Updating the model will cause Rivets to refresh our view based on data- attributes in the HTML, and the updating of the model is triggered by an ‘updateHeader’ event bound to the BODY element. This means that any sub-view can trigger updates to our header.

The application view will setup event handlers on buttons so we can toggle between status states. These button click events trigger an ‘updateHeader’ event sending new data for our model, which will bubble-up to the handler defined in the header view.

The HTML contains the Rivets data- directives to hide and show alternative status text.

See the live demo or browse the gist…

When the event source sub-view triggers events (e.g. clicking a link), it does so on the delegate. Event names are specified in the HTML using a data-event attribute on the link. Enjoy!

When we initialize the application view, the control (source) and subview (target) are created.

The control view listens to ‘subscribe’ events and sets the delegate subview, then responds to clicks from links with data-event attributes sending them to the delegate subview.

The target view triggers a ‘subscribe’ event, sending itself to the control for delegation and listens for those events described in the control’s view.

Output

http://example.com/page/1

http://example.com/page/2

http://example.com/page/3

http://example.com/page/4