Resizing NSWindow to match view controller size in storyboard

This solution for 'toolbar style' tab view controllers does animate and supports the nice crossfade effect. In the storyboard designer, add 'TabViewController' in the custom class name field of the NSTabViewController. Don't forget to assign a title to each viewController, this is used as a key value.

import Cocoa

class TabViewController: NSTabViewController {

    private lazy var tabViewSizes: [String : NSSize] = [:]

    override func viewDidLoad() {
        // Add size of first tab to tabViewSizes
        if let viewController = self.tabViewItems.first?.viewController, let title = viewController.title {
            tabViewSizes[title] = viewController.view.frame.size
        }
        super.viewDidLoad()
    }

    override func transition(from fromViewController: NSViewController, to toViewController: NSViewController, options: NSViewController.TransitionOptions, completionHandler completion: (() -> Void)?) {

        NSAnimationContext.runAnimationGroup({ context in
            context.duration = 0.5
            self.updateWindowFrameAnimated(viewController: toViewController)
            super.transition(from: fromViewController, to: toViewController, options: [.crossfade, .allowUserInteraction], completionHandler: completion)
        }, completionHandler: nil)
    }

    func updateWindowFrameAnimated(viewController: NSViewController) {

        guard let title = viewController.title, let window = view.window else {
            return
        }

        let contentSize: NSSize

        if tabViewSizes.keys.contains(title) {
            contentSize = tabViewSizes[title]!
        }
        else {
            contentSize = viewController.view.frame.size
            tabViewSizes[title] = contentSize
        }

        let newWindowSize = window.frameRect(forContentRect: NSRect(origin: NSPoint.zero, size: contentSize)).size

        var frame = window.frame
        frame.origin.y += frame.height
        frame.origin.y -= newWindowSize.height
        frame.size = newWindowSize
        window.animator().setFrame(frame, display: false)
    }
}

The auto layout answer is half of it. You need to set the preferredContentSize in your ViewController for each tab to the fitting size (if you wanted the tab to size to the smallest size satisfying all constraints).

override func viewWillAppear() {
        super.viewWillAppear()
        preferredContentSize = view.fittingSize
}

If your constraints are causing an issue below try first with a fixed size, the example below sets this in the tab item's view controller's viewWillAppear function (Swift used here, but the Objective-C version works just as well).

override func viewWillAppear() {
        super.viewWillAppear()
        preferredContentSize = NSSize(width: 400, height: 280)
}

If that works, fiddle with your constraints to figure out what's going on

The window containing a toolbar style tab view controller does resize without any code if you have auto layout constraints in your storyboard tab views (macOS 11.1, Xcode 12.3). I haven't tried other style tab view controllers.

If you want to resize with animation as in Finder, it is sufficient to add one override in your tab view controller. It will resize the window with system-calculated resize animation time and will hide the tab view during resize animation:

class PreferencesTabViewController: NSTabViewController {

    override func transition(from fromViewController: NSViewController, to toViewController: NSViewController, options: NSViewController.TransitionOptions = [], completionHandler completion: (() -> Void)? = nil) {

        guard let window = view.window else {
            super.transition(from: fromViewController, to: toViewController, options: options, completionHandler: completion)
            return
        }

        let fromSize = window.frame.size
        let toSize = window.frameRect(forContentRect: toViewController.view.frame).size
        let widthDelta = toSize.width - fromSize.width
        let heightDelta = toSize.height - fromSize.height
        var toOrigin = window.frame.origin
        toOrigin.x += widthDelta / 2
        toOrigin.y -= heightDelta
        let toFrame = NSRect(origin: toOrigin, size: toSize)

        NSAnimationContext.runAnimationGroup { context in
            context.duration = window.animationResizeTime(toFrame)
            view.isHidden = true
            window.animator().setFrame(toFrame, display: false)
            super.transition(from: fromViewController, to: toViewController, options: options, completionHandler: completion)
        } completionHandler: { [weak self] in
            self?.view.isHidden = false
        }

    }

}

Please adjust closure syntax if you are using Swift versions older than 5.3.