;; NOTE: Assertions have been generated by update_lit_checks.py --all-items and should not be edited.

;; RUN: wasm-opt %s -all --heap2local -S -o - | filecheck %s

(module
  (type $i32 (struct (field (mut i32))))
  (type $i64 (struct (field (mut i64))))
  ;; CHECK:      (type $struct (struct (field (mut (ref null $struct)))))
  (type $struct (struct (field (mut (ref null $struct)))))
  ;; CHECK:      (type $1 (func (result i32)))

  ;; CHECK:      (type $2 (func (result i64)))

  ;; CHECK:      (type $3 (func (param (ref null $struct)) (result (ref null $struct))))

  ;; CHECK:      (type $4 (func (param (ref null $struct) (ref null $struct)) (result (ref null $struct))))

  ;; CHECK:      (type $5 (func (param i32) (result i32)))

  ;; CHECK:      (type $arr (array (mut i32)))
  (type $arr (array (mut i32)))

  ;; CHECK:      (func $escape-rmw (type $3) (param $0 (ref null $struct)) (result (ref null $struct))
  ;; CHECK-NEXT:  (struct.atomic.rmw.xchg $struct 0
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:   (struct.new_default $struct)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $escape-rmw (param (ref null $struct)) (result (ref null $struct))
    ;; Allocations that flow into RMW modification values can be written and escape.
    (struct.atomic.rmw.xchg $struct 0
      (local.get 0)
      (struct.new_default $struct)
    )
  )

  ;; CHECK:      (func $escape-cmpxchg (type $3) (param $0 (ref null $struct)) (result (ref null $struct))
  ;; CHECK-NEXT:  (struct.atomic.rmw.cmpxchg $struct 0
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:   (struct.new_default $struct)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $escape-cmpxchg (param (ref null $struct)) (result (ref null $struct))
    ;; Similarly, allocations that flow into cmpxchg replacement values can escape.
    (struct.atomic.rmw.cmpxchg $struct 0
      (local.get 0)
      (local.get 0)
      (struct.new_default $struct)
    )
  )

  ;; CHECK:      (func $no-escape-cmpxchg-expected (type $3) (param $0 (ref null $struct)) (result (ref null $struct))
  ;; CHECK-NEXT:  (local $1 (ref null $struct))
  ;; CHECK-NEXT:  (struct.atomic.rmw.cmpxchg $struct 0
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $1
  ;; CHECK-NEXT:     (ref.null none)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $no-escape-cmpxchg-expected (param (ref null $struct)) (result (ref null $struct))
    ;; Allocations that flow into the cmpxchg `expected` operand do not escape
    ;; and can be optimized, but do not require any fixups of the cmpxchg.
    (struct.atomic.rmw.cmpxchg $struct 0
      (local.get 0)
      (struct.new_default $struct)
      (local.get 0)
    )
  )

  ;; CHECK:      (func $rmw-add-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.add
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-add-i32 (result i32)
    (struct.atomic.rmw.add $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-sub-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.sub
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-sub-i32 (result i32)
    (struct.atomic.rmw.sub $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-and-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.and
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-and-i32 (result i32)
    (struct.atomic.rmw.and $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-or-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.or
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-or-i32 (result i32)
    (struct.atomic.rmw.or $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-xor-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.xor
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-xor-i32 (result i32)
    (struct.atomic.rmw.xor $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-xchg-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (local.get $2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-xchg-i32 (result i32)
    (struct.atomic.rmw.xchg $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $rmw-cmpxchg-i32 (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (local $3 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $3
  ;; CHECK-NEXT:   (i32.const 2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (if
  ;; CHECK-NEXT:   (i32.eq
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (then
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $3)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-cmpxchg-i32 (result i32)
    (struct.atomic.rmw.cmpxchg $i32 0
      (struct.new_default $i32)
      (i32.const 1)
      (i32.const 2)
    )
  )

  ;; CHECK:      (func $rmw-add-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i64.add
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-add-i64 (result i64)
    (struct.atomic.rmw.add $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-sub-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i64.sub
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-sub-i64 (result i64)
    (struct.atomic.rmw.sub $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-and-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i64.and
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-and-i64 (result i64)
    (struct.atomic.rmw.and $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-or-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i64.or
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-or-i64 (result i64)
    (struct.atomic.rmw.or $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-xor-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i64.xor
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-xor-i64 (result i64)
    (struct.atomic.rmw.xor $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-xchg-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (local.get $2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-xchg-i64 (result i64)
    (struct.atomic.rmw.xchg $i64 0
      (struct.new_default $i64)
      (i64.const 1)
    )
  )

  ;; CHECK:      (func $rmw-cmpxchg-i64 (type $2) (result i64)
  ;; CHECK-NEXT:  (local $0 i64)
  ;; CHECK-NEXT:  (local $1 i64)
  ;; CHECK-NEXT:  (local $2 i64)
  ;; CHECK-NEXT:  (local $3 i64)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i64.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i64.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $3
  ;; CHECK-NEXT:   (i64.const 2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (if
  ;; CHECK-NEXT:   (i64.eq
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (then
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $3)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-cmpxchg-i64 (result i64)
    (struct.atomic.rmw.cmpxchg $i64 0
      (struct.new_default $i64)
      (i64.const 1)
      (i64.const 2)
    )
  )

  ;; CHECK:      (func $rmw-xchg-ref (type $3) (param $0 (ref null $struct)) (result (ref null $struct))
  ;; CHECK-NEXT:  (local $1 (ref null $struct))
  ;; CHECK-NEXT:  (local $2 (ref null $struct))
  ;; CHECK-NEXT:  (local $3 (ref null $struct))
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $1
  ;; CHECK-NEXT:     (ref.null none)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $3
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (local.get $1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $3)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $2)
  ;; CHECK-NEXT: )
  (func $rmw-xchg-ref (param (ref null $struct)) (result (ref null $struct))
    (struct.atomic.rmw.xchg $struct 0
      (struct.new_default $struct)
      (local.get 0)
    )
  )

  ;; CHECK:      (func $rmw-cmpxchg-ref (type $4) (param $0 (ref null $struct)) (param $1 (ref null $struct)) (result (ref null $struct))
  ;; CHECK-NEXT:  (local $2 (ref null $struct))
  ;; CHECK-NEXT:  (local $3 (ref null $struct))
  ;; CHECK-NEXT:  (local $4 (ref null $struct))
  ;; CHECK-NEXT:  (local $5 (ref null $struct))
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $2
  ;; CHECK-NEXT:     (ref.null none)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $4
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $5
  ;; CHECK-NEXT:   (local.get $1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $3
  ;; CHECK-NEXT:   (local.get $2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (if
  ;; CHECK-NEXT:   (ref.eq
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:    (local.get $4)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (then
  ;; CHECK-NEXT:    (local.set $2
  ;; CHECK-NEXT:     (local.get $5)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $3)
  ;; CHECK-NEXT: )
  (func $rmw-cmpxchg-ref (param (ref null $struct) (ref null $struct)) (result (ref null $struct))
    (struct.atomic.rmw.cmpxchg $struct 0
      (struct.new_default $struct)
      (local.get 0)
      (local.get 1)
    )
  )

  ;; CHECK:      (func $rmw-acqrel (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.add
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $rmw-acqrel (result i32)
    (struct.atomic.rmw.add acqrel acqrel $i32 0
      (struct.new_default $i32)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $cmpxchg-acqrel (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (local $3 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $2
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $3
  ;; CHECK-NEXT:   (i32.const 2)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $1
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (if
  ;; CHECK-NEXT:   (i32.eq
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $2)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (then
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $3)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $1)
  ;; CHECK-NEXT: )
  (func $cmpxchg-acqrel (result i32)
    (struct.atomic.rmw.cmpxchg acqrel acqrel $i32 0
      (struct.new_default $i32)
      (i32.const 1)
      (i32.const 2)
    )
  )

  ;; CHECK:      (func $rmw-unreachable-value (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (block ;; (replaces unreachable StructRMW we can't emit)
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (block (result nullref)
  ;; CHECK-NEXT:     (local.set $0
  ;; CHECK-NEXT:      (i32.const 0)
  ;; CHECK-NEXT:     )
  ;; CHECK-NEXT:     (ref.null none)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (unreachable)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (unreachable)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $rmw-unreachable-value (result i32)
    ;; When the value is unreachable, the whole expression is unreachable.
    ;; We should not attempt to optimize this (it would hit an assertion
    ;; on type == field.type since unreachable != i32).
    (struct.atomic.rmw.add $i32 0
      (struct.new_default $i32)
      (unreachable)
    )
  )

  ;; CHECK:      (func $cmpxchg-unreachable-expected (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (block ;; (replaces unreachable StructCmpxchg we can't emit)
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (block (result nullref)
  ;; CHECK-NEXT:     (local.set $0
  ;; CHECK-NEXT:      (i32.const 0)
  ;; CHECK-NEXT:     )
  ;; CHECK-NEXT:     (ref.null none)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (unreachable)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (drop
  ;; CHECK-NEXT:    (i32.const 1)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (unreachable)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $cmpxchg-unreachable-expected (result i32)
    ;; When the expected operand is unreachable, the whole expression is
    ;; unreachable. We should not attempt to optimize this.
    (struct.atomic.rmw.cmpxchg $i32 0
      (struct.new_default $i32)
      (unreachable)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $array-rmw-add (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (local $3 i32)
  ;; CHECK-NEXT:  (local $4 i32)
  ;; CHECK-NEXT:  (local $5 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $2
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $3
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $2)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $1
  ;; CHECK-NEXT:     (local.get $3)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $5
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $4
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $0
  ;; CHECK-NEXT:   (i32.add
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $5)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $4)
  ;; CHECK-NEXT: )
  (func $array-rmw-add (result i32)
    ;; Array atomic RMW on a non-escaping fixed-size array should be
    ;; optimized: the array is converted to a struct, then to locals.
    (array.atomic.rmw.add $arr
      (array.new_fixed $arr 2
        (i32.const 0)
        (i32.const 0)
      )
      (i32.const 0)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $array-cmpxchg (type $1) (result i32)
  ;; CHECK-NEXT:  (local $0 i32)
  ;; CHECK-NEXT:  (local $1 i32)
  ;; CHECK-NEXT:  (local $2 i32)
  ;; CHECK-NEXT:  (local $3 i32)
  ;; CHECK-NEXT:  (local $4 i32)
  ;; CHECK-NEXT:  (local $5 i32)
  ;; CHECK-NEXT:  (local $6 i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (local.set $2
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $3
  ;; CHECK-NEXT:     (i32.const 0)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $2)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (local.set $1
  ;; CHECK-NEXT:     (local.get $3)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $5
  ;; CHECK-NEXT:   (i32.const 10)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $6
  ;; CHECK-NEXT:   (i32.const 20)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.set $4
  ;; CHECK-NEXT:   (local.get $0)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (if
  ;; CHECK-NEXT:   (i32.eq
  ;; CHECK-NEXT:    (local.get $0)
  ;; CHECK-NEXT:    (local.get $5)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (then
  ;; CHECK-NEXT:    (local.set $0
  ;; CHECK-NEXT:     (local.get $6)
  ;; CHECK-NEXT:    )
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (local.get $4)
  ;; CHECK-NEXT: )
  (func $array-cmpxchg (result i32)
    ;; Array atomic cmpxchg on a non-escaping fixed-size array should be
    ;; optimized similarly.
    (array.atomic.rmw.cmpxchg $arr
      (array.new_fixed $arr 2
        (i32.const 0)
        (i32.const 0)
      )
      (i32.const 0)
      (i32.const 10)
      (i32.const 20)
    )
  )

  ;; CHECK:      (func $array-rmw-nonconstant-index (type $5) (param $idx i32) (result i32)
  ;; CHECK-NEXT:  (array.atomic.rmw.add $arr
  ;; CHECK-NEXT:   (array.new_fixed $arr 2
  ;; CHECK-NEXT:    (i32.const 0)
  ;; CHECK-NEXT:    (i32.const 0)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:   (local.get $idx)
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT: )
  (func $array-rmw-nonconstant-index (param $idx i32) (result i32)
    ;; A non-constant index prevents the optimization, since Array2Struct
    ;; needs to know which struct field to access at compile time.
    (array.atomic.rmw.add $arr
      (array.new_fixed $arr 2
        (i32.const 0)
        (i32.const 0)
      )
      (local.get $idx)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $array-rmw-oob (type $1) (result i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (i32.const 1)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (unreachable)
  ;; CHECK-NEXT: )
  (func $array-rmw-oob (result i32)
    ;; An out-of-bounds index on a zero-size array. The access will always
    ;; trap, so we emit drops for operands and an unreachable.
    (array.atomic.rmw.add $arr
      (array.new_default $arr
        (i32.const 0)
      )
      (i32.const 0)
      (i32.const 1)
    )
  )

  ;; CHECK:      (func $array-cmpxchg-oob (type $1) (result i32)
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (block (result nullref)
  ;; CHECK-NEXT:    (ref.null none)
  ;; CHECK-NEXT:   )
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (i32.const 10)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (drop
  ;; CHECK-NEXT:   (i32.const 20)
  ;; CHECK-NEXT:  )
  ;; CHECK-NEXT:  (unreachable)
  ;; CHECK-NEXT: )
  (func $array-cmpxchg-oob (result i32)
    ;; As above, but for cmpxchg with an out-of-bounds index.
    (array.atomic.rmw.cmpxchg $arr
      (array.new_default $arr
        (i32.const 0)
      )
      (i32.const 0)
      (i32.const 10)
      (i32.const 20)
    )
  )
)