Is it possible to read in specific bands from a multi-band raster with gdal or rasterio?

You can read specific bands in a single call using rasterio by passing a list/tuple of band numbers (Following the GDAL convention, bands are indexed from 1):

import rasterio
rasterio.__version__
'1.0a8'

dataset = rasterio.open('multiband.tif')

dataset.count
4

dataset.read((1,2)) #read 1st two bands into an array.

array([[[ 85,  98,  75, ...,  53,  55,  55],
        [ 84,  94,  76, ...,  54,  55,  54],
        [ 68,  60,  55, ...,  53,  54,  53],
        ...,
        [ 67,  67,  66, ...,  63,  63,  62],
        [255, 255, 255, ..., 255, 255, 255],
        [255, 255, 255, ..., 255, 255, 255]],

       [[ 78,  88,  65, ...,  41,  43,  45],
        [ 77,  84,  66, ...,  42,  43,  44],
        [ 61,  51,  46, ...,  41,  42,  43],
        ...,
        [ 77,  77,  77, ...,  71,  70,  69],
        [255, 255, 255, ..., 255, 207, 255],
        [255, 255, 255, ..., 191,   0, 135]]], dtype=uint8)

This appears to work with rasterio

import rasterio
import numpy as np

vstack = '/path/to/virtual_stack.vrt'
bands = [2,4,6,14,35]

# define cropping window ((row_start, row_stop), (col_start, col_stop))
window = ((10, 50), (30, 40))

with rasterio.open(vstack) as src:
    # Create zero array (you may want to set dtype too)
    array = np.zeros((window[0][1] - window[0][0],
                      window[1][1] - window[1][0],
                      len(bands)))
    # Fill the array
    for i, band in enumerate(bands):
        array[:,:,i] = src.read(band, window=window)