how to Filter lookup in CRM 2011 by javascript in Client Side.

You can set any lookup with custom view in client.
Here is a simple javascript code build dfor CRM 2011.This code run in onLoad of  Response  Custom entity.
relation of the entities EntA 1---N EntB 1---N EntC 1-- N EntD for the sample.

Filter rule is; Lookup gets only selected EntAs for the EntD

 function onLoad() {  
   //filter questions for the current EntC  
   var EntCLookup = Xrm.Page.getAttribute("field of the EntityReference of EntC").getValue();  
   if (EntCLookup != null) {  
     var EntCId = EntCLookup[0].id;  
     var viewId = "{8F03FB3E-0EFF-4417-B704-80CE1711B8D2}";/static guid id.you can obtain by Visual Studio tool   
     var entityName = "EntA";  
     var viewDisplayName = "EntA Filtered View by EntC";  
     var fetchXML = "<fetch version='1.0' output-format='xml-platform' mapping='logical' distinct='true'>" +  
               "<entity name='EntA'>" +  
                 "<attribute name='EntAid' />" +  
                 "<attribute name='new_name' />" +  
                 "<attribute name='createdon' />" +  
                 "<order attribute='new_name' descending='false' />" +  
                 "<link-entity name='EntB' from='foreign key in EntD form' to='EntBid' alias='aa'>" +  
                   "<link-entity name='EntC' from='foreign key in EntB' to='Referred Primary Key in EntC' alias='ab'>" +  
                     "<filter type='and'>" +  
                       "<condition attribute='EntCid' operator='eq' uitype='EntC' value='" + EntCId + "' /> " +  
                     "</filter>" +  
                   "</link-entity>" +  
                 "</link-entity>" +  
               "</entity>" +  
             "</fetch>";  
     var layoutXML =   
                 "<grid name ='result' " +  
                   "object='10032' " +  
                   "jump='new_name' " +  
                   "select='1' " +  
                   "icon = '1' " +  
                   "preview = '1'>" +  
                   "<row name = 'result' id = 'EntAid'>" +  
                     "<cell name='new_name' width='200' />" +  
                     "<cell name='createdon' width='100' />" +  
                   "</row>" +  
                 "</grid>" ;  
     Xrm.Page.getControl("EntCid").addCustomView(viewId, entityName, viewDisplayName, fetchXML, layoutXML, true);  
   }  

good lucks,