How to select entire matrix except certain rows and columns?

Use functions row and col together with logical operations. The following works because R's matrices are in column-first order.

mat <- matrix(seq.int(5*4), nrow = 5)
mat[ !(row(mat) %in% 2:4) | !(col(mat) %in% 2:3) ] <- 0
mat
#     [,1] [,2] [,3] [,4]
#[1,]    0    0    0    0
#[2,]    0    7   12    0
#[3,]    0    8   13    0
#[4,]    0    9   14    0
#[5,]    0    0    0    0

Another option with replace

replace(mat * 0, as.matrix(expand.grid(2:4, 2:3)), mat[2:4, 2:3])

-output

#      [,1] [,2] [,3] [,4]
#[1,]    0    0    0    0
#[2,]    0    7   12    0
#[3,]    0    8   13    0
#[4,]    0    9   14    0
#[5,]    0    0    0    0

data

mat <- matrix(seq.int(5*4), nrow = 5)

Here is a two-step base R option

rs <- 2:4
cs <- 2:3
`<-`(mat[setdiff(seq(nrow(mat)), rs), ], 0)
`<-`(mat[, setdiff(seq(ncol(mat)), cs)], 0)

which gives

> mat
     [,1] [,2] [,3] [,4]
[1,]    0    0    0    0
[2,]    0    7   12    0
[3,]    0    8   13    0
[4,]    0    9   14    0
[5,]    0    0    0    0

Data

mat <- matrix(seq.int(5*4), nrow = 5)