# AVKit patterns

Advanced playback recipes for iOS 26 and Swift 6.3.

## Contents

- [Custom player with AVPlayerLayer](#custom-player-with-avplayerlayer)
- [Interstitials](#interstitials)
- [Background playback](#background-playback)
- [Errors](#errors)
- [Observing playback](#observing-playback)
- [Video frame analysis](#video-frame-analysis)
- [HDR](#hdr)
- [SwiftUI player model](#swiftui-player-model)
- [Wrapping AVPlayerViewController for SwiftUI](#wrapping-avplayerviewcontroller-for-swiftui)

## Custom player with AVPlayerLayer

Drawing video through `AVPlayerLayer` means giving up the system transport
controls in exchange for complete control over the UI.

```swift
import AVFoundation
import UIKit

@MainActor
final class FilmSurface: UIView {
    override static var layerClass: AnyClass { AVPlayerLayer.self }

    var filmLayer: AVPlayerLayer {
        layer as! AVPlayerLayer
    }

    var player: AVPlayer? {
        get { filmLayer.player }
        set { filmLayer.player = newValue }
    }

    override init(frame rect: CGRect) {
        super.init(frame: rect)
        filmLayer.videoGravity = .resizeAspect
    }

    required init?(coder decoder: NSCoder) {
        super.init(coder: decoder)
        filmLayer.videoGravity = .resizeAspect
    }
}
```

The force cast is safe because `layerClass` guarantees the layer type.

### Adding PiP to it

```swift
import AVFoundation
import AVKit
import UIKit

@MainActor
final class CustomCinemaController: UIViewController, @preconcurrency AVPictureInPictureControllerDelegate {
    private let surface = FilmSurface()
    private let player = AVPlayer()
    private var pip: AVPictureInPictureController?
    private let overlayControls = UIStackView()

    override func loadView() {
        view = surface
    }

    func load(_ url: URL) {
        surface.player = player
        if AVPictureInPictureController.isPictureInPictureSupported() {
            let controller = AVPictureInPictureController(playerLayer: surface.filmLayer)
            controller?.delegate = self
            controller?.canStartPictureInPictureAutomaticallyFromInline = true
            pip = controller
        }
        let episode = AVPlayerItem(url: url)
        player.replaceCurrentItem(with: episode)
        player.play()
    }

    @objc func pipButtonTapped() {
        guard let pip, pip.isPictureInPicturePossible else { return }
        pip.startPictureInPicture()
    }

    func pictureInPictureControllerWillStartPictureInPicture(_ controller: AVPictureInPictureController) {
        overlayControls.isHidden = true
    }

    func pictureInPictureControllerDidStopPictureInPicture(_ controller: AVPictureInPictureController) {
        overlayControls.isHidden = false
    }

    func pictureInPictureController(_ controller: AVPictureInPictureController,
                                    restoreUserInterfaceForPictureInPictureStopWithCompletionHandler finish: @escaping (Bool) -> Void) {
        finish(true)
    }
}
```

- Create the PiP controller only when the device supports PiP.
- Set its delegate and the automatic-inline flag, then load the item and play.
- The button action re-checks `isPictureInPicturePossible` before starting.
- Hide custom controls when PiP starts, show them again when it stops, and
  always call `completionHandler(true)` in the restore callback.

### PiP from sample buffers

When the app renders frames itself into an `AVSampleBufferDisplayLayer`, build
the controller from a content source:

```swift
import AVFoundation
import AVKit
import CoreMedia

@MainActor
final class RemoteDesktopPiP: NSObject, @preconcurrency AVPictureInPictureSampleBufferPlaybackDelegate {
    let frameLayer = AVSampleBufferDisplayLayer()
    private(set) var pip: AVPictureInPictureController?
    private var paused = false

    func prepare() {
        if !AVPictureInPictureController.isPictureInPictureSupported() { return }
        let source = AVPictureInPictureController.ContentSource(sampleBufferDisplayLayer: frameLayer, playbackDelegate: self)
        pip = AVPictureInPictureController(contentSource: source)
    }

    func userTappedPiP() {
        guard let pip, pip.isPictureInPicturePossible else { return }
        pip.startPictureInPicture()
    }

    func pictureInPictureController(_ controller: AVPictureInPictureController, setPlaying playing: Bool) {
        paused = !playing
    }

    func pictureInPictureControllerTimeRangeForPlayback(_ controller: AVPictureInPictureController) -> CMTimeRange {
        CMTimeRange(start: .negativeInfinity, duration: .positiveInfinity)
    }

    func pictureInPictureControllerIsPlaybackPaused(_ controller: AVPictureInPictureController) -> Bool {
        paused
    }

    func pictureInPictureController(_ controller: AVPictureInPictureController,
                                    didTransitionToRenderSize newRenderSize: CMVideoDimensions) {}

    func pictureInPictureController(_ controller: AVPictureInPictureController,
                                    skipByInterval skipInterval: CMTime,
                                    completion completionHandler: @escaping () -> Void) {
        completionHandler()
    }

    func pictureInPictureControllerShouldProhibitBackgroundAudioPlayback(_ controller: AVPictureInPictureController) -> Bool {
        false
    }
}
```

- Offer a PiP button only once support is confirmed.
- On tap, check `isPictureInPicturePossible` again; the layer, item, format or
  playback state can still rule PiP out. Start only from the user's tap.
- The playback delegate must answer the system's questions about timing and
  play state: `setPlaying`, the playable time range (an infinite range here, for
  live content), whether playback is paused, render-size changes, skip requests
  (call the completion), and whether background audio should be blocked.

## Interstitials

Interstitials cover ads, legal notices and any stretch of the timeline with
special rules. `AVPlayerViewController` marks them on the scrubber and tells its
delegate when one starts and ends.

```swift
import AVFoundation
import AVKit

@MainActor
func adBreaks(in item: AVPlayerItem) -> [CMTimeRange] {
    item.interstitialTimeRanges.map(\.timeRange)
}

@MainActor
func scheduleMidroll(on player: AVPlayer, adURL: URL, at seconds: Double) -> AVPlayerInterstitialEventController? {
    guard let main = player.currentItem else { return nil }
    let controller = AVPlayerInterstitialEventController(primaryPlayer: player)
    let midroll = AVPlayerInterstitialEvent(
        primaryItem: main,
        time: CMTime(seconds: seconds, preferredTimescale: 600)
    )
    midroll.templateItems = [AVPlayerItem(asset: AVURLAsset(url: adURL))]
    midroll.restrictions = [.requiresPlaybackAtPreferredRateForAdvancement]
    controller.events = [midroll]
    return controller
}

@MainActor
final class AdGate: NSObject, @preconcurrency AVPlayerViewControllerDelegate {
    func playerViewController(_ viewer: AVPlayerViewController, willPresent adBreak: AVInterstitialTimeRange) {
        viewer.requiresLinearPlayback = true
    }

    func playerViewController(_ viewer: AVPlayerViewController, didPresent adBreak: AVInterstitialTimeRange) {
        viewer.requiresLinearPlayback = false
    }
}
```

- Read stream-defined breaks by iterating `interstitialTimeRanges` and taking
  each `timeRange`.
- On iOS there are two valid sources: interstitials defined in the stream, or
  an `AVPlayerInterstitialEventController` schedule. Assigning
  `interstitialTimeRanges` is not a scheduling mechanism.
- Once `AVPlayerInterstitialEventController(primaryPlayer:)` sets a schedule,
  playback ignores interstitial events embedded in the source media. Use it only
  when the app owns the ad schedule.
- Lock seeking in `willPresent` and unlock it in `didPresent`.

## Background playback

- Info.plist: `UIBackgroundModes` array containing `audio`.
- Set the `.playback` category, activate when playback starts, and hold the
  `AVPlayer` strongly for as long as audio should continue.
- iOS normally pauses video when its scene moves to the background unless the
  video goes into PiP. Use PiP for video that keeps playing, and the background
  audio mode for sound that should keep going once the app is no longer in front.

## Errors

```swift
import AVFoundation

enum PlaybackProblem {
    case offline, timedOut, media(Error), unknown
}

@MainActor
func watchStatus(of item: AVPlayerItem, report: @escaping @MainActor (PlaybackProblem?) -> Void) -> NSKeyValueObservation {
    item.observe(\.status, options: [.new]) { observed, _ in
        MainActor.assumeIsolated {
            switch observed.status {
            case .readyToPlay:
                report(nil)
            case .failed:
                report(classify(observed.error))
            case .unknown:
                return
            @unknown default:
                report(.unknown)
            }
        }
    }
}

func classify(_ error: Error?) -> PlaybackProblem {
    guard let error = error as NSError? else { return .unknown }
    switch (error.domain, error.code) {
    case (NSURLErrorDomain, NSURLErrorNotConnectedToInternet): return .offline
    case (NSURLErrorDomain, NSURLErrorTimedOut): return .timedOut
    case (AVFoundationErrorDomain, _): return .media(error)
    default: return .media(error)
    }
}

@MainActor
func watchStalls(of player: AVPlayer, spinner: @escaping @MainActor (Bool) -> Void) -> NSKeyValueObservation {
    player.observe(\.timeControlStatus, options: [.new]) { observed, _ in
        MainActor.assumeIsolated {
            switch observed.timeControlStatus {
            case .waitingToPlayAtSpecifiedRate:
                spinner(observed.reasonForWaitingToPlay == .toMinimizeStalls)
            case .playing, .paused:
                spinner(false)
            @unknown default:
                spinner(false)
            }
        }
    }
}
```

- `AVPlayerItem.status` moves from `.unknown` to `.readyToPlay` or `.failed`;
  on failure read `item.error`.
- Split network errors (`NSURLErrorDomain`: show an offline message for
  `NSURLErrorNotConnectedToInternet`, offer a retry for `NSURLErrorTimedOut`)
  from media errors in `AVFoundationErrorDomain`.
- `AVPlayer.timeControlStatus` is `.paused`, `.playing` or
  `.waitingToPlayAtSpecifiedRate`. While waiting, `reasonForWaitingToPlay` says
  why: `.toMinimizeStalls` (show a buffering indicator),
  `.evaluatingBufferingRate` or `.noItemToPlay`.

`AVKitError` codes:

| Code | Meaning |
|---|---|
| `.pictureInPictureStartFailed` | PiP could not start; verify the playback category, the `audio` background mode, device support, `isPictureInPicturePossible`, and that the layer or content source feeding PiP is still active |
| `.contentRatingUnknown` | the content's rating could not be determined |
| `.contentDisallowedByPasscode` | blocked by parental restrictions |
| `.contentDisallowedByProfile` | blocked by a device management profile |
| `.unknown` | unspecified failure |

## Observing playback

```swift
import AVFoundation

@MainActor
final class ChapterClock {
    private let player: AVPlayer
    private var ticker: Any?
    private var markers: Any?
    private var endObserver: NSObjectProtocol?

    init(player: AVPlayer) {
        self.player = player
    }

    func start(chapters: [CMTime], onTick: @escaping @Sendable (CMTime) -> Void, onChapter: @escaping @Sendable () -> Void) {
        let halfSecond = CMTime(value: 1, timescale: 2)
        ticker = player.addPeriodicTimeObserver(forInterval: halfSecond,
                                                queue: .main, using: onTick)
        markers = player.addBoundaryTimeObserver(forTimes: chapters.map { NSValue(time: $0) },
                                                 queue: .main, using: onChapter)
        let center = NotificationCenter.default
        endObserver = center.addObserver(forName: .AVPlayerItemDidPlayToEndTime, object: player.currentItem,
                                         queue: .main) { _ in onChapter() }
    }

    func stop() {
        if let ticker { player.removeTimeObserver(ticker) }
        if let markers { player.removeTimeObserver(markers) }
        if let endObserver { NotificationCenter.default.removeObserver(endObserver) }
        ticker = nil
        markers = nil
        endObserver = nil
    }
}
```

- `addPeriodicTimeObserver(forInterval:queue:using:)` returns a token; pass it
  to `removeTimeObserver(_:)` when done.
- `addBoundaryTimeObserver(forTimes:queue:using:)` fires at the listed times,
  wrapped with `NSValue(time:)`.
- The end of an item is announced with `.AVPlayerItemDidPlayToEndTime`, scoped
  to that item.

## Video frame analysis

`allowsVideoFrameAnalysis` (default `true`, iOS 16+) lets the user interact
with a paused frame: select text, look things up, lift a subject. Narrow it with
`videoFrameAnalysisTypes` (iOS 17+), whose options include `.text`,
`.visualSearch`, `.subject` and `.machineReadableCode`, plus `.default`.

```swift
import AVKit

@MainActor
func onlyTextAndCodes(_ viewer: AVPlayerViewController) {
    viewer.allowsVideoFrameAnalysis = true
    viewer.videoFrameAnalysisTypes = [.text, .machineReadableCode]
}
```

## HDR

`AVPlayerViewController.preferredDisplayDynamicRange` (iOS 26+) picks how HDR
content is shown:

| Value | Behavior |
|---|---|
| `.automatic` | the system decides (default) |
| `.standard` | force SDR |
| `.high` | full HDR |
| `.constrainedHigh` | limited HDR, useful when HDR sits next to SDR content |

```swift
import AVKit

@MainActor
func calmHDR(_ viewer: AVPlayerViewController) {
    viewer.preferredDisplayDynamicRange = .constrainedHigh
}
```

## SwiftUI player model

```swift
import AVFoundation
import AVKit
import Observation
import SwiftUI

@Observable
@MainActor
final class EpisodePlayer {
    private(set) var player: AVPlayer?
    private(set) var position: Double = 0
    private(set) var isPlaying = false
    private(set) var isBuffering = false
    private(set) var length: Double = 0
    private(set) var failure: String?

    @ObservationIgnored private var tick: Any?
    @ObservationIgnored private var observations: [NSKeyValueObservation] = []

    func load(_ url: URL) {
        tearDown()
        let item = AVPlayerItem(asset: AVURLAsset(url: url))
        let fresh = AVPlayer(playerItem: item)
        player = fresh

        observations.append(item.observe(\.status) { [weak self] observed, _ in
            let status = observed.status
            let seconds = CMTimeGetSeconds(observed.duration)
            let message = observed.error?.localizedDescription
            Task { @MainActor [weak self] in
                guard let model = self else { return }
                if status == .readyToPlay { model.length = seconds.isFinite ? seconds : 0 }
                if status == .failed { model.failure = message }
            }
        })
        observations.append(fresh.observe(\.timeControlStatus) { [weak self] observed, _ in
            let state = observed.timeControlStatus
            Task { @MainActor in
                self?.isPlaying = state == .playing
                self?.isBuffering = state == .waitingToPlayAtSpecifiedRate
            }
        })
        let every = CMTime(value: 1, timescale: 2)
        tick = fresh.addPeriodicTimeObserver(forInterval: every, queue: .main) { [weak self] now in
            let seconds = now.seconds
            Task { @MainActor in self?.position = seconds }
        }
    }

    func toggle() {
        guard let player else { return }
        isPlaying ? player.pause() : player.play()
    }

    func jump(to seconds: Double) {
        player?.seek(to: CMTime(seconds: seconds, preferredTimescale: 600), toleranceBefore: .zero, toleranceAfter: .zero)
    }

    func tearDown() {
        if let tick, let player { player.removeTimeObserver(tick) }
        tick = nil
        observations.forEach { $0.invalidate() }
        observations.removeAll()
        player = nil
        isPlaying = false
        isBuffering = false
        position = 0
        length = 0
        failure = nil
    }

    isolated deinit {
        if let tick, let player { player.removeTimeObserver(tick) }
    }
}

struct EpisodeScreen: View {
    let episodeURL: URL
    @State private var model = EpisodePlayer()

    var body: some View {
        VStack(spacing: 12) {
            if let current = model.player {
                VideoPlayer(player: current)
                    .aspectRatio(16 / 9, contentMode: .fit)
            }
            HStack {
                Button(model.isPlaying ? "Pause" : "Play") { model.toggle() }
                Text("\(Int(model.position))s / \(Int(model.length))s").monospacedDigit()
                if model.isBuffering { ProgressView() }
            }
            if let failure = model.failure {
                Text(failure).foregroundStyle(.red)
            }
        }
        .task { model.load(episodeURL) }
    }
}
```

- `load(_:)` clears previous state, creates the item and player, and installs
  the status observation, the `timeControlStatus` observation and a periodic
  time observer.
- Seeking uses zero tolerance on both sides for frame-accurate jumps, with a
  timescale of 600.
- `tearDown()` removes the time observer, invalidates each
  `NSKeyValueObservation` and resets the published state; the isolated
  `deinit` removes the time observer as well.
- Observer closures capture `self` weakly and hop to the main actor with
  `Task { @MainActor in ... }`. Duration is read with `CMTimeGetSeconds` once
  the item is ready; buffering is `.waitingToPlayAtSpecifiedRate`.

## Wrapping AVPlayerViewController for SwiftUI

When the app needs the delegate, PiP settings or speed options, a representable
around `AVPlayerViewController` gives more control than `VideoPlayer`.

```swift
import AVKit
import SwiftUI

struct FullPlayer: UIViewControllerRepresentable {
    let player: AVPlayer
    var speeds = AVPlaybackSpeed.systemDefaultSpeeds
    var allowsPiP = true

    func makeCoordinator() -> Coordinator { Coordinator() }

    func makeUIViewController(context ctx: Context) -> AVPlayerViewController {
        let viewer = AVPlayerViewController()
        viewer.player = player
        viewer.allowsPictureInPicturePlayback = allowsPiP
        viewer.canStartPictureInPictureAutomaticallyFromInline = allowsPiP
        viewer.speeds = speeds
        viewer.delegate = ctx.coordinator
        return viewer
    }

    func updateUIViewController(_ viewer: AVPlayerViewController, context: Context) {
        viewer.player = player
    }

    @MainActor
    final class Coordinator: NSObject, @preconcurrency AVPlayerViewControllerDelegate {
        func playerViewController(_ viewer: AVPlayerViewController,
                                  willBeginFullScreenPresentationWithAnimationCoordinator transition: any UIViewControllerTransitionCoordinator) {}

        func playerViewController(_ viewer: AVPlayerViewController,
                                  willEndFullScreenPresentationWithAnimationCoordinator transition: any UIViewControllerTransitionCoordinator) {}

        func playerViewController(_ viewer: AVPlayerViewController,
                                  restoreUserInterfaceForPictureInPictureStopWithCompletionHandler done: @escaping (Bool) -> Void) {
            done(true)
        }

        func playerViewController(_ viewer: AVPlayerViewController,
                                  failedToStartPictureInPictureWithError reason: any Error) {}
    }
}
```