Edit in JSFiddle

Result
JavaScript
CSS
HTML

$("input").click(function(){
    var $count = $("input:checked").length;
    var $not = $('input').not(':checked');

    if($count >= 3) {
        $not.attr("disabled",true);
    }else{
        $not.attr("disabled",false);
    }
});

<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1"><br />
<input type="checkbox" value="1">