When the records are selected, call a JavaScript method using onrowselection. Add the selected rows to a list variable. Then using length property in JavaScript, we can get the count of selected rows from lightning:datatable in Salesforce. 

Sample Code:

Component:

<aura:component implements="force:appHostable" controller="AccountListController">  
      
    <aura:attribute type="Account[]" name="acctList"/>  
    <aura:attribute name="mycolumns" type="List"/>  
    <aura:attribute name="selectedAccts" type="List"/>  
      
    <aura:handler name="init" value="{!this}" action="{!c.init}"/>  
      
    <div class="slds-box slds-theme_default">  
          
        <lightning:datatable data="{! v.acctList }"   
                             columns="{! v.mycolumns }"   
                             keyField="id"  
                             onrowselection="{!c.handleSelect}"/>  
        <br/>  
        <center>  
            <lightning:button onclick="{!c.showSelectedName}" label="Alert Size" variant="brand"/>  
        </center>  
          
    </div>  
      
</aura:component> 

JS Controller:

({  
      
    init : function( component, event, helper ) {  
          
        component.set('v.mycolumns', [  
            {label: 'Account Name', fieldName: 'linkName', type: 'url',   
            typeAttributes: {label: { fieldName: 'Name' }, target: '_blank'}},  
            {label: 'Industry', fieldName: 'Industry', type: 'text'},  
            {label: 'Type', fieldName: 'Type', type: 'Text'}  
        ]);  
          
        helper.fetchAccounts( component, event );  
          
    },  
      
    handleSelect : function( component, event, helper ) {  
          
        var selectedRows = event.getParam( 'selectedRows' );   
        var setRows = [];  
        for ( var i = 0; i < selectedRows.length; i++ ) {  
              
            setRows.push( selectedRows[ i ] );  
  
        }  
        component.set( "v.selectedAccts", setRows );  
          
    },  
      
    showSelectedName : function( component, event, helper ) {  
          
        var records = component.get( "v.selectedAccts" );  
        alert( records.length );  
          
    }  
  
}) 

JS Helper:

({  
      
 fetchAccounts : function(component, event) {  
    
        var action = component.get("c.fetchAccts");  
          
        action.setCallback(this, function(response) {  
              
            var state = response.getState();  
              
            if (state === "SUCCESS") {  
                  
                var records = response.getReturnValue();  
                records.forEach(function(record) {  
                      
                    record.linkName = '/' + record.Id;  
                    record.CheckBool = false;  
                      
                });     
                  
                component.set("v.acctList", records);  
                  
            }              
              
        });  
          
        $A.enqueueAction(action);  
          
 }  
      
}) 

Apex Class:

public class AccountListController {  
      
    @AuraEnabled  
    public static List < Account > fetchAccts() {  
          
        return [ SELECT Id, Name, Industry, Type FROM Account LIMIT 5 ];  
          
    }  
      
} 

Output: