This code in VB.NET works for me.
Dim pFlayer As ESRI.ArcGIS.Carto.IFeatureLayer
Dim pFlayer1 As ESRI.ArcGIS.Carto.IFeatureLayer
pFlayer = pMxDoc.FocusMap.Layer(1)
pFlayer1 = pMxDoc.FocusMap.Layer(0)
SelectbyLocation(pFlayer, pFlayer1, "INTERSECT", "NEW_SELECTION")
Friend Sub SelectbyLocation(ByVal InputLayer As ESRI.ArcGIS.Carto.IFeatureLayer2, ByVal SelectionLayer As ESRI.ArcGIS.Carto.IFeatureLayer2, ByVal OverlapType As String, ByVal SelectionType As String)
Dim SelByLoc As New ESRI.ArcGIS.DataManagementTools.SelectLayerByLocation
Dim Result As ESRI.ArcGIS.Geoprocessing.IGeoProcessorResult2
Try
Using releaser As New ESRI.ArcGIS.ADF.ComReleaser
releaser.ManageLifetime(SelByLoc)
SelByLoc.in_layer = InputLayer
SelByLoc.overlap_type = OverlapType
SelByLoc.select_features = SelectionLayer
SelByLoc.selection_type = SelectionType
Result = RunTool(SelByLoc, Nothing)
If Result Is Nothing Then
System.Windows.Forms.MessageBox.Show("Could not select features")
End If
End Using
Catch ex As Exception
System.Windows.Forms.MessageBox.Show(ex.ToString, "Select Layer by Location")
End Try
End Sub
Private Sub ReturnMessages(ByVal pResult As ESRI.ArcGIS.Geoprocessing.IGeoProcessorResult2, ByVal Title As String)
Dim ErrorMessage As String
If pResult.MessageCount > 0 Then
For Count As Integer = 0 To pResult.MessageCount - 1
ErrorMessage += pResult.GetMessage(Count)
Next
End If
System.Windows.Forms.MessageBox.Show(ErrorMessage, Title)
End Sub
Friend Function RunTool(ByVal Process As ESRI.ArcGIS.Geoprocessor.IGPProcess, ByVal TC As ESRI.ArcGIS.esriSystem.ITrackCancel2) As ESRI.ArcGIS.Geoprocessing.IGeoProcessorResult2
Dim Result As ESRI.ArcGIS.Geoprocessing.IGeoProcessorResult2
Dim GP As New ESRI.ArcGIS.Geoprocessor.Geoprocessor
Try
Result = CType(GP.Execute(Process, Nothing), ESRI.ArcGIS.Geoprocessing.IGeoProcessorResult2)
If Result.Status <> ESRI.ArcGIS.esriSystem.esriJobStatus.esriJobSucceeded Then ReturnMessages(Result, "Geoprocessing Error")
GP.ClearMessages()
Catch ex As Exception
ReturnMessages(Result, "Fail")
System.Windows.Forms.MessageBox.Show(ex.ToString, "Run Geoprocessor")
End Try
Return Result
End Function