;; Machine description for optimization of RVV auto-vectorization.
;; Copyright (C) 2023-2024 Free Software Foundation, Inc.
;; Contributed by Juzhe Zhong (juzhe.zhong@rivai.ai), RiVAI Technologies Ltd.

;; This file is part of GCC.

;; GCC is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3, or (at your option)
;; any later version.

;; GCC is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3.  If not see
;; <http://www.gnu.org/licenses/>.

;; -----------------------------------------------------------------------------
;; ---- Integer Compare Instructions Simplification
;; -----------------------------------------------------------------------------
;; Simplify OP(V, V) Instructions to VMCLR.m Includes:
;; - 1.  VMSNE
;; - 2.  VMSLT
;; - 3.  VMSLTU
;; - 4.  VMSGT
;; - 5.  VMSGTU
;; -----------------------------------------------------------------------------
;; Simplify OP(V, V) Instructions to VMSET.m Includes:
;; - 1.  VMSEQ
;; - 2.  VMSLE
;; - 3.  VMSLEU
;; - 4.  VMSGE
;; - 5.  VMSGEU
;; -----------------------------------------------------------------------------

(define_split
  [(set (match_operand:VB      0 "register_operand")
	(if_then_else:VB
	  (unspec:VB
	    [(match_operand:VB 1 "vector_all_trues_mask_operand")
	     (match_operand    4 "vector_length_operand")
	     (match_operand    5 "const_int_operand")
	     (match_operand    6 "const_int_operand")
	     (reg:SI VL_REGNUM)
	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
	  (match_operand:VB    3 "vector_move_operand")
	  (match_operand:VB    2 "vector_undef_operand")))]
  "TARGET_VECTOR"
  [(const_int 0)]
  {
    emit_insn (gen_pred_mov (<MODE>mode, operands[0], CONST1_RTX (<MODE>mode),
			     RVV_VUNDEF (<MODE>mode), operands[3],
			     operands[4], operands[5]));
    DONE;
  }
)

;; -------------------------------------------------------------------------
;; ---- Sign-extension for vmv.x.s.
;; -------------------------------------------------------------------------
(define_insn "*pred_extract_first_sextdi<mode>"
  [(set (match_operand:DI 0 "register_operand"		"=r")
	(sign_extend:DI
          (unspec:<VEL>
	    [(vec_select:<VEL>
	       (match_operand:VI_QHS 1 "register_operand""vr")
	       (parallel [(const_int 0)]))
	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)))]
  "TARGET_VECTOR && Pmode == DImode"
  "vmv.x.s\t%0,%1"
  [(set_attr "type" "vimovvx")
   (set_attr "mode" "<MODE>")])

(define_insn "*pred_extract_first_sextsi<mode>"
  [(set (match_operand:SI 0 "register_operand"		  "=r")
	(sign_extend:SI
          (unspec:<VEL>
	    [(vec_select:<VEL>
	       (match_operand:VI_QH 1 "register_operand"  "vr")
	       (parallel [(const_int 0)]))
	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)))]
  "TARGET_VECTOR && Pmode == SImode"
  "vmv.x.s\t%0,%1"
  [(set_attr "type" "vimovvx")
   (set_attr "mode" "<MODE>")])

;; =============================================================================
;; All combine patterns for combine pass.
;; =============================================================================

;; =============================================================================
;; Combine op + vmerge to cond_op
;; =============================================================================

;; Combine <op> and vcond_mask generated by midend into cond_len_<op>
;; Currently supported operations:
;;   abs(FP)
(define_insn_and_split "*cond_abs<mode>"
  [(set (match_operand:V_VLSF 0 "register_operand")
        (if_then_else:V_VLSF
          (match_operand:<VM> 1 "register_operand")
          (abs:V_VLSF (match_operand:V_VLSF 2 "nonmemory_operand"))
          (match_operand:V_VLSF 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred (ABS, <MODE>mode);
  riscv_vector::expand_cond_unop (icode, operands);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine vfsqrt.v and cond_mask
(define_insn_and_split "*cond_<optab><mode>"
  [(set (match_operand:V_VLSF 0 "register_operand")
     (if_then_else:V_VLSF
       (match_operand:<VM> 1 "register_operand")
       (any_float_unop:V_VLSF
         (match_operand:V_VLSF 2 "register_operand"))
       (match_operand:V_VLSF 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred (<CODE>, <MODE>mode);
  riscv_vector::expand_cond_unop (icode, operands);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf2) and vcond_mask
(define_insn_and_split "*cond_<optab><v_double_trunc><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
        (if_then_else:VWEXTI
          (match_operand:<VM> 1 "register_operand")
          (any_extend:VWEXTI (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
          (match_operand:VWEXTI 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_vf2 (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf4) and vcond_mask
(define_insn_and_split "*cond_<optab><v_quad_trunc><mode>"
  [(set (match_operand:VQEXTI 0 "register_operand")
        (if_then_else:VQEXTI
          (match_operand:<VM> 1 "register_operand")
          (any_extend:VQEXTI (match_operand:<V_QUAD_TRUNC> 2 "register_operand"))
          (match_operand:VQEXTI 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_vf4 (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf8) and vcond_mask
(define_insn_and_split "*cond_<optab><v_oct_trunc><mode>"
  [(set (match_operand:VOEXTI 0 "register_operand")
        (if_then_else:VOEXTI
          (match_operand:<VM> 1 "register_operand")
          (any_extend:VOEXTI (match_operand:<V_OCT_TRUNC> 2 "register_operand"))
          (match_operand:VOEXTI 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_vf8 (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf2) and vcond_mask_len
(define_insn_and_split "*cond_len_<optab><v_double_trunc><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
    (if_then_else:VWEXTI
      (unspec:<VM>
        [(match_operand 4 "vector_length_operand")
         (match_operand 5 "const_int_operand")
         (match_operand 6 "const_int_operand")
         (reg:SI VL_REGNUM)
         (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
      (vec_merge:VWEXTI
        (any_extend:VWEXTI (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
        (match_operand:VWEXTI 1 "vector_merge_operand")
	(match_operand:<VM> 3 "register_operand"))
      (match_dup 1)))]
  "TARGET_VECTOR"
  "#"
  "&& 1"
  [(const_int 0)]
{
  emit_insn (gen_pred_<optab><mode>_vf2 (operands[0], operands[3], operands[1], operands[2],
                                         operands[4], operands[5], operands[6], CONST0_RTX (Pmode)));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf4) and vcond_mask_len
(define_insn_and_split "*cond_len_<optab><v_quad_trunc><mode>"
  [(set (match_operand:VQEXTI 0 "register_operand")
    (if_then_else:VQEXTI
      (unspec:<VM>
        [(match_operand 4 "vector_length_operand")
         (match_operand 5 "const_int_operand")
         (match_operand 6 "const_int_operand")
         (reg:SI VL_REGNUM)
         (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
      (vec_merge:VQEXTI
        (any_extend:VQEXTI (match_operand:<V_QUAD_TRUNC> 2 "register_operand"))
        (match_operand:VQEXTI 1 "vector_merge_operand")
	(match_operand:<VM> 3 "register_operand"))
      (match_dup 1)))]
  "TARGET_VECTOR"
  "#"
  "&& 1"
  [(const_int 0)]
{
  emit_insn (gen_pred_<optab><mode>_vf4 (operands[0], operands[3], operands[1], operands[2],
                                         operands[4], operands[5], operands[6], CONST0_RTX (Pmode)));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine sign_extend/zero_extend(vf8) and vcond_mask_len
(define_insn_and_split "*cond_len_<optab><v_oct_trunc><mode>"
  [(set (match_operand:VOEXTI 0 "register_operand")
    (if_then_else:VOEXTI
      (unspec:<VM>
        [(match_operand 4 "vector_length_operand")
         (match_operand 5 "const_int_operand")
         (match_operand 6 "const_int_operand")
         (reg:SI VL_REGNUM)
         (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
      (vec_merge:VOEXTI
        (any_extend:VOEXTI (match_operand:<V_OCT_TRUNC> 2 "register_operand"))
        (match_operand:VOEXTI 1 "vector_merge_operand")
	(match_operand:<VM> 3 "register_operand"))
      (match_dup 1)))]
  "TARGET_VECTOR"
  "#"
  "&& 1"
  [(const_int 0)]
{
  emit_insn (gen_pred_<optab><mode>_vf8 (operands[0], operands[3], operands[1], operands[2],
                                         operands[4], operands[5], operands[6], CONST0_RTX (Pmode)));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine trunc(vf2) + vcond_mask
(define_insn_and_split "*cond_trunc<mode><v_double_trunc>"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
    (if_then_else:<V_DOUBLE_TRUNC>
          (match_operand:<VM> 1 "register_operand")
          (truncate:<V_DOUBLE_TRUNC>
            (match_operand:VWEXTI 2 "register_operand"))
          (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_trunc (<MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine FP extend(vf2) and vcond_mask
(define_insn_and_split "*cond_extend<v_double_trunc><mode>"
  [(set (match_operand:VWEXTF_ZVFHMIN 0 "register_operand")
        (if_then_else:VWEXTF_ZVFHMIN
          (match_operand:<VM> 1 "register_operand")
          (float_extend:VWEXTF_ZVFHMIN (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
          (match_operand:VWEXTF_ZVFHMIN 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_extend (<MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine FP extend(vf2) and vcond_mask_len
(define_insn_and_split "*cond_len_extend<v_double_trunc><mode>"
  [(set (match_operand:VWEXTF_ZVFHMIN 0 "register_operand")
    (if_then_else:VWEXTF_ZVFHMIN
      (unspec:<VM>
        [(match_operand 4 "vector_length_operand")
         (match_operand 5 "const_int_operand")
         (match_operand 6 "const_int_operand")
         (reg:SI VL_REGNUM)
         (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
      (vec_merge:VWEXTF_ZVFHMIN
        (float_extend:VWEXTF_ZVFHMIN (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
        (match_operand:VWEXTF_ZVFHMIN 1 "vector_merge_operand")
	(match_operand:<VM> 3 "register_operand"))
      (match_dup 1)))]
  "TARGET_VECTOR"
  "#"
  "&& 1"
  [(const_int 0)]
{
  emit_insn (gen_pred_extend<mode> (operands[0], operands[3], operands[1], operands[2],
                                    operands[4], operands[5], operands[6], CONST0_RTX (Pmode)));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine FP trunc(vf2) + vcond_mask
(define_insn_and_split "*cond_trunc<mode><v_double_trunc>"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
    (if_then_else:<V_DOUBLE_TRUNC>
          (match_operand:<VM> 1 "register_operand")
          (float_truncate:<V_DOUBLE_TRUNC>
            (match_operand:VWEXTF_ZVFHMIN 2 "register_operand"))
          (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_trunc (<MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(FP->INT) + vcond_mask
(define_insn_and_split "*cond_<optab><mode><vconvert>"
  [(set (match_operand:<VCONVERT> 0 "register_operand")
        (if_then_else:<VCONVERT>
          (match_operand:<VM> 1 "register_operand")
          (any_fix:<VCONVERT>
            (match_operand:V_VLSF 2 "register_operand"))
          (match_operand:<VCONVERT> 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(INT->FP) + vcond_mask
(define_insn_and_split "*cond_<float_cvt><vconvert><mode>"
  [(set (match_operand:V_VLSF 0 "register_operand")
        (if_then_else:V_VLSF
          (match_operand:<VM> 1 "register_operand")
          (any_float:V_VLSF
            (match_operand:<VCONVERT> 2 "register_operand"))
          (match_operand:V_VLSF 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(FP->2xINT) + vcond_mask
(define_insn_and_split "*cond_<optab><vnconvert><mode>"
  [(set (match_operand:VWCONVERTI 0 "register_operand")
        (if_then_else:VWCONVERTI
          (match_operand:<VM> 1 "register_operand")
	  (any_fix:VWCONVERTI
	    (match_operand:<VNCONVERT> 2 "register_operand"))
          (match_operand:VWCONVERTI 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(INT->2xFP) + vcond_mask
(define_insn_and_split "*cond_<float_cvt><vnconvert><mode>"
  [(set (match_operand:V_VLSF 0 "register_operand")
        (if_then_else:V_VLSF
          (match_operand:<VM> 1 "register_operand")
          (any_float:V_VLSF
            (match_operand:<VNCONVERT> 2 "register_operand"))
          (match_operand:V_VLSF 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_widen (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(2xFP->INT) + vcond_mask
(define_insn_and_split "*cond_<optab><mode><vnconvert>"
  [(set (match_operand:<VNCONVERT> 0 "register_operand")
        (if_then_else:<VNCONVERT>
          (match_operand:<VM> 1 "register_operand")
          (any_fix:<VNCONVERT>
            (match_operand:V_VLSF 2 "register_operand"))
          (match_operand:<VNCONVERT> 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine convert(2xINT->FP) + vcond_mask
(define_insn_and_split "*cond_<float_cvt><mode><vnconvert>2"
  [(set (match_operand:<VNCONVERT> 0 "register_operand")
        (if_then_else:<VNCONVERT>
          (match_operand:<VM> 1 "register_operand")
	  (any_float:<VNCONVERT>
	    (match_operand:VWCONVERTI 2 "register_operand"))
          (match_operand:<VNCONVERT> 3 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_narrow (<CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_unop (icode, ops);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine vnsra + vcond_mask
(define_insn_and_split "*cond_v<any_shiftrt:optab><any_extend:optab>trunc<mode>"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
     (if_then_else:<V_DOUBLE_TRUNC>
       (match_operand:<VM> 1 "register_operand")
       (truncate:<V_DOUBLE_TRUNC>
         (any_shiftrt:VWEXTI
           (match_operand:VWEXTI 2 "register_operand")
	   (any_extend:VWEXTI
             (match_operand:<V_DOUBLE_TRUNC> 3 "vector_shift_operand"))))
       (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_narrow (<any_shiftrt:CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3], operands[4],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_binop (icode, ops);
  DONE;
}
 [(set_attr "type" "vnshift")])

(define_insn_and_split "*cond_<any_shiftrt:optab>trunc<mode>"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
     (if_then_else:<V_DOUBLE_TRUNC>
       (match_operand:<VM> 1 "register_operand")
       (truncate:<V_DOUBLE_TRUNC>
         (any_shiftrt:VWEXTI
           (match_operand:VWEXTI 2 "register_operand")
	   (match_operand:<VEL> 3 "csr_operand")))
       (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_narrow_scalar (<any_shiftrt:CODE>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], gen_lowpart (Pmode, operands[3]),
               operands[4], gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_binop (icode, ops);
  DONE;
}
 [(set_attr "type" "vnshift")])

;; Combine vmulh.vv/vmulhu.vv + vcond_mask
(define_insn_and_split "*cond_<mulh_table><mode>3_highpart"
   [(set (match_operand:VFULLI 0 "register_operand")
    (if_then_else:VFULLI
      (match_operand:<VM> 1 "register_operand")
      (mulh:VFULLI
        (match_operand:VFULLI 2 "register_operand")
        (match_operand:VFULLI 3 "register_operand"))
      (match_operand:VFULLI 4 "register_operand")))]
   "TARGET_VECTOR && can_create_pseudo_p ()"
   "#"
   "&& 1"
   [(const_int 0)]
{
  insn_code icode = code_for_pred_mulh (<MULH_UNSPEC>, <MODE>mode);
  rtx ops[] = {operands[0], operands[1], operands[2], operands[3], operands[4],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_cond_len_binop (icode, ops);
   DONE;
}
[(set_attr "type" "vector")])

;; =============================================================================
;; Combine binop + trunc to narrow_binop
;; =============================================================================

;; Combine vsr[la].vv + trunc to vnsr[la].wv
(define_insn_and_split "*narrow_<any_shiftrt:optab><any_extend:optab><mode>"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand"       "=vr,vr")
    (truncate:<V_DOUBLE_TRUNC>
      (any_shiftrt:VWEXTI
        (match_operand:VWEXTI 1 "register_operand"                 " vr,vr")
	(any_extend:VWEXTI
          (match_operand:<V_DOUBLE_TRUNC> 2 "vector_shift_operand" " vr,vk")))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_narrow (<any_shiftrt:CODE>, <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
 [(set_attr "type" "vnshift")])

;; Combine vsr[la].w[xi] + trunc to vnsr[la].w[xi]
(define_insn_and_split "*narrow_<any_shiftrt:optab><mode>_scalar"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand" "=vr")
    (truncate:<V_DOUBLE_TRUNC>
      (any_shiftrt:VWEXTI
        (match_operand:VWEXTI 1 "register_operand"           " vr")
	(match_operand:<VEL> 2 "csr_operand"                 " rK"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  operands[2] = gen_lowpart (Pmode, operands[2]);
  insn_code icode = code_for_pred_narrow_scalar (<any_shiftrt:CODE>, <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
 [(set_attr "type" "vnshift")])

;; =============================================================================
;; Combine extend + binop to widen_binop
;; =============================================================================

(define_insn_and_split "*dual_widen_<any_widen_binop:optab><any_extend:su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(any_widen_binop:VWEXTI
	  (any_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
	  (any_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_dual_widen (<any_widen_binop:CODE>,
                                              <any_extend:CODE>,
                                              <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "vector")])

(define_insn_and_split "*single_widen_sub<any_extend:su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(minus:VWEXTI
	  (match_operand:VWEXTI 1 "register_operand")
	  (any_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_single_widen_sub (<any_extend:CODE>,
                                                    <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "viwalu")])

(define_insn_and_split "*single_widen_add<any_extend:su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(plus:VWEXTI
	  (any_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_single_widen_add (<any_extend:CODE>,
                                                    <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "viwalu")])

;; This combine pattern does not correspond to an single instruction,
;; i.e. there is no vwmul.wv instruction. This is a temporary pattern
;; produced by a combine pass and if there is no further combine into
;; vwmul.vv pattern, then fall back to extend pattern and vmul.vv pattern.
(define_insn_and_split "*single_widen_mul<any_extend:su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(mult:VWEXTI
	  (any_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code extend_icode = code_for_pred_vf2 (<any_extend:CODE>, <MODE>mode);
  rtx tmp = gen_reg_rtx (<MODE>mode);
  rtx extend_ops[] = {tmp, operands[2]};
  riscv_vector::emit_vlmax_insn (extend_icode, riscv_vector::UNARY_OP, extend_ops);

  rtx ops[] = {operands[0], operands[1], tmp};
  insn_code icode = code_for_pred (MULT, <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
  DONE;
}
[(set_attr "type" "viwmul")])

(define_insn_and_split "*dual_widen_mulsu<mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(mult:VWEXTI
	  (sign_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
	  (zero_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_widen_mulsu (<MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "vector")])

(define_insn_and_split "*dual_widen_mulus<mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(mult:VWEXTI
          (zero_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	  (sign_extend:VWEXTI
	    (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_widen_mulsu (<MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "vector")])

(define_insn_and_split "*dual_widen_<optab><mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
	(any_widen_binop:VWEXTF
	  (float_extend:VWEXTF
	    (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
	  (float_extend:VWEXTF
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_dual_widen (<CODE>, <MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_FRM_DYN, operands);
  DONE;
}
[(set_attr "type" "vector")])

(define_insn_and_split "*single_widen_add<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
	(plus:VWEXTF
	  (float_extend:VWEXTF
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_single_widen_add (<MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_FRM_DYN, operands);
  DONE;
}
[(set_attr "type" "vfwalu")])

(define_insn_and_split "*single_widen_sub<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
	(minus:VWEXTF
          (match_operand:VWEXTF 1 "register_operand")
	  (float_extend:VWEXTF
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code icode = code_for_pred_single_widen_sub (<MODE>mode);
  riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_FRM_DYN, operands);
  DONE;
}
[(set_attr "type" "vfwalu")])

;; This combine pattern does not correspond to an single instruction,
;; i.e. there is no vfwmul.wv instruction. This is a temporary pattern
;; produced by a combine pass and if there is no further combine into
;; vfwmul.vv pattern, then fall back to extend pattern and vfmul.vv pattern.
(define_insn_and_split "*single_widen_mul<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
	(mult:VWEXTF
	  (float_extend:VWEXTF
	    (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  insn_code extend_icode = code_for_pred_extend (<MODE>mode);
  rtx tmp = gen_reg_rtx (<MODE>mode);
  rtx extend_ops[] = {tmp, operands[2]};
  riscv_vector::emit_vlmax_insn (extend_icode, riscv_vector::UNARY_OP, extend_ops);

  rtx ops[] = {operands[0], operands[1], tmp};
  riscv_vector::emit_vlmax_insn (code_for_pred (MULT, <MODE>mode),
				 riscv_vector::BINARY_OP_FRM_DYN, ops);
  DONE;
}
[(set_attr "type" "vfwmul")])

;; Combine extend + vredsum to vwredsum[u]
(define_insn_and_split "*widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (any_extend:<V_DOUBLE_EXTEND>
            (match_operand:VI_QHS_NO_M8 1 "register_operand"))
        ] UNSPEC_REDUC_SUM))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  riscv_vector::expand_reduction (<WREDUC_UNSPEC>, riscv_vector::REDUCE_OP,
                                  operands,
                                  CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine extend + vfredusum to vfwredusum
(define_insn_and_split "*widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (float_extend:<V_DOUBLE_EXTEND>
            (match_operand:VF_HS_NO_M8 1 "register_operand"))
        ] UNSPEC_REDUC_SUM_UNORDERED))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  riscv_vector::expand_reduction (UNSPEC_WREDUC_SUM_UNORDERED,
                                  riscv_vector::REDUCE_OP_FRM_DYN,
                                  operands,
                                  CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine extend + vfredosum to vfwredosum
(define_insn_and_split "*fold_left_widen_plus_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (float_extend:<V_DOUBLE_EXTEND>
            (match_operand:VF_HS_NO_M8 1 "register_operand"))
          (match_operand:<V_DOUBLE_EXTEND_VEL> 2 "register_operand")
        ] UNSPEC_REDUC_SUM_ORDERED))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  riscv_vector::expand_reduction (UNSPEC_WREDUC_SUM_ORDERED,
                                  riscv_vector::REDUCE_OP_FRM_DYN,
                                  operands, operands[2]);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine extend + mask vfredosum to mask vfwredosum
(define_insn_and_split "*mask_len_fold_left_widen_plus_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (float_extend:<V_DOUBLE_EXTEND>
            (match_operand:VF_HS_NO_M8 2 "register_operand"))
          (match_operand:<V_DOUBLE_EXTEND_VEL> 1 "register_operand")
          (match_operand:<VM> 3 "vector_mask_operand")
          (match_operand 4 "autovec_length_operand")
          (match_operand 5 "const_0_operand")
        ] UNSPEC_REDUC_SUM_ORDERED))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  if (rtx_equal_p (operands[4], const0_rtx))
    emit_move_insn (operands[0], operands[1]);
  else
    {
      rtx ops[] = {operands[0], operands[2], operands[3], operands[4]};
      riscv_vector::expand_reduction (UNSPEC_WREDUC_SUM_ORDERED,
                                      riscv_vector::REDUCE_OP_M_FRM_DYN,
                                      ops, operands[1]);
    }
  DONE;
}
[(set_attr "type" "vector")])

;; =============================================================================
;; Combine extend + ternop to widen_ternop
;; =============================================================================

;; Combine ext + fma(vmacc,vmadd) to widen_fma (vwmacc[u])
(define_insn_and_split "*dual_widen_fma<su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(plus:VWEXTI
	  (mult:VWEXTI
            (any_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
            (any_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_plus (<CODE>, <MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP, ops);
    DONE;
  }
  [(set_attr "type" "viwmuladd")])

;; Combine sign_extend + zero_extend + fma to widen_fma (vwmaccsu)
(define_insn_and_split "*dual_widen_fmasu<mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(plus:VWEXTI
	  (mult:VWEXTI
	    (sign_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	    (zero_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_plussu (<MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP, ops);
    DONE;
  }
  [(set_attr "type" "viwmuladd")])

;; Combine zero_extend + sign_extend + fma to widen_fma (vwmaccsu)
(define_insn_and_split "*dual_widen_fmaus<mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(plus:VWEXTI
	  (mult:VWEXTI
	    (zero_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand"))
	    (sign_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_plussu (<MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP, ops);
    DONE;
  }
  [(set_attr "type" "viwmuladd")])

;; This combine pattern does not correspond to an single instruction.
;; This is a temporary pattern produced by a combine pass and if there
;; is no further combine into widen pattern, then fall back to extend
;; pattern and non-widen fma pattern.
(define_insn_and_split "*single_widen_fma<su><mode>"
  [(set (match_operand:VWEXTI 0 "register_operand")
	(plus:VWEXTI
	  (mult:VWEXTI
            (any_extend:VWEXTI
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	    (match_operand:VWEXTI 3 "register_operand"))
	  (match_operand:VWEXTI 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code extend_icode = code_for_pred_vf2 (<CODE>, <MODE>mode);
    rtx tmp = gen_reg_rtx (<MODE>mode);
    rtx extend_ops[] = {tmp, operands[2]};
    riscv_vector::emit_vlmax_insn (extend_icode, riscv_vector::UNARY_OP,
                                   extend_ops);

    rtx ops[] = {operands[0], tmp, operands[3], operands[1],
                 RVV_VUNDEF(<MODE>mode)};
    riscv_vector::emit_vlmax_insn (code_for_pred_mul_plus (<MODE>mode),
				   riscv_vector::TERNARY_OP, ops);
    DONE;
  }
  [(set_attr "type" "viwmuladd")])

;; Combine extend + fma to widen_fma (vfwmacc)
(define_insn_and_split "*dual_widen_fma<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (plus:VWEXTF
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul (PLUS, <MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; This combine pattern does not correspond to an single instruction.
;; This is a temporary pattern produced by a combine pass and if there
;; is no further combine into widen pattern, then fall back to extend
;; pattern and non-widen fma pattern.
(define_insn_and_split "*single_widen_fma<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (plus:VWEXTF
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	    (match_operand:VWEXTF 3 "register_operand"))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_extend (<MODE>mode);
    rtx tmp = gen_reg_rtx (<MODE>mode);
    rtx ext_ops[] = {tmp, operands[2]};
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, ext_ops);

    rtx ops[] = {operands[0], tmp, operands[3], operands[1],
                 RVV_VUNDEF(<MODE>mode)};
    riscv_vector::emit_vlmax_insn (code_for_pred_mul (PLUS, <MODE>mode),
				   riscv_vector::TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; Combine extend + fnma to widen_fnma (vfwnmsac)
(define_insn_and_split "*dual_widen_fnma<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
          (match_operand:VWEXTF 1 "register_operand")
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_neg (PLUS, <MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; This combine pattern does not correspond to an single instruction.
;; This is a temporary pattern produced by a combine pass and if there
;; is no further combine into widen pattern, then fall back to extend
;; pattern and non-widen fnma pattern.
(define_insn_and_split "*single_widen_fnma<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
          (match_operand:VWEXTF 1 "register_operand")
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	    (match_operand:VWEXTF 3 "register_operand"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_extend (<MODE>mode);
    rtx tmp = gen_reg_rtx (<MODE>mode);
    rtx ext_ops[] = {tmp, operands[2]};
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, ext_ops);

    rtx ops[] = {operands[0], tmp, operands[3], operands[1],
                 RVV_VUNDEF(<MODE>mode)};
    riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (PLUS, <MODE>mode),
				   riscv_vector::TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; Combine extend + fms to widen_fms (vfwmsac)
(define_insn_and_split "*dual_widen_fms<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul (MINUS, <MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; This combine pattern does not correspond to an single instruction.
;; This is a temporary pattern produced by a combine pass and if there
;; is no further combine into widen pattern, then fall back to extend
;; pattern and non-widen fms pattern.
(define_insn_and_split "*single_widen_fms<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
	  (mult:VWEXTF
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))
	    (match_operand:VWEXTF 3 "register_operand"))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_extend (<MODE>mode);
    rtx tmp = gen_reg_rtx (<MODE>mode);
    rtx ext_ops[] = {tmp, operands[2]};
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, ext_ops);

    rtx ops[] = {operands[0], tmp, operands[3], operands[1],
                 RVV_VUNDEF(<MODE>mode)};
    riscv_vector::emit_vlmax_insn (code_for_pred_mul (MINUS, <MODE>mode),
				   riscv_vector::TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; Combine extend + fnms to widen_fnms (vfwnmacc)
(define_insn_and_split "*dual_widen_fnms<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
	  (mult:VWEXTF
            (neg:VWEXTF
              (float_extend:VWEXTF
	        (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_neg (MINUS, <MODE>mode),
				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; This combine pattern does not correspond to an single instruction.
;; This is a temporary pattern produced by a combine pass and if there
;; is no further combine into widen pattern, then fall back to extend
;; pattern and non-widen fnms pattern.
(define_insn_and_split "*single_widen_fnms<mode>"
  [(set (match_operand:VWEXTF 0 "register_operand")
        (minus:VWEXTF
          (mult:VWEXTF
	    (neg:VWEXTF
              (match_operand:VWEXTF 3 "register_operand"))
            (float_extend:VWEXTF
	      (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
	  (match_operand:VWEXTF 1 "register_operand")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_extend (<MODE>mode);
    rtx tmp = gen_reg_rtx (<MODE>mode);
    rtx ext_ops[] = {tmp, operands[2]};
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::UNARY_OP, ext_ops);

    rtx ops[] = {operands[0], tmp, operands[3], operands[1],
                 RVV_VUNDEF(<MODE>mode)};
    riscv_vector::emit_vlmax_insn (code_for_pred_mul_neg (MINUS, <MODE>mode),
				   riscv_vector::TERNARY_OP_FRM_DYN, ops);
    DONE;
  }
  [(set_attr "type" "vfwmuladd")])

;; Combine mask_extend + vredsum to mask_vwredsum[u]
;; where the mrege of mask_extend is vector const 0
(define_insn_and_split "*cond_widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (if_then_else:<V_DOUBLE_EXTEND>
            (match_operand:<VM> 1 "register_operand")
            (any_extend:<V_DOUBLE_EXTEND>
              (match_operand:VI_QHS_NO_M8 2 "register_operand"))
            (if_then_else:<V_DOUBLE_EXTEND>
              (unspec:<VM> [
                (match_operand:<VM> 3 "vector_all_trues_mask_operand")
                (match_operand 6 "vector_length_operand")
                (match_operand 7 "const_int_operand")
                (match_operand 8 "const_int_operand")
                (match_operand 9 "const_1_or_2_operand")
                (reg:SI VL_REGNUM)
                (reg:SI VTYPE_REGNUM)
              ] UNSPEC_VPREDICATE)
              (match_operand:<V_DOUBLE_EXTEND> 5 "vector_const_0_operand")
              (match_operand:<V_DOUBLE_EXTEND> 4 "vector_merge_operand")))
        ] UNSPEC_REDUC_SUM))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  rtx ops[] = {operands[0], operands[2], operands[1],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_reduction (<WREDUC_UNSPEC>,
                                  riscv_vector::REDUCE_OP_M,
                                  ops, CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine mask_len_extend + vredsum to mask_vwredsum[u]
;; where the mrege of mask_len_extend is vector const 0
(define_insn_and_split "*cond_len_widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (if_then_else:<V_DOUBLE_EXTEND>
            (unspec:<VM> [
              (match_operand 2 "vector_length_operand")
              (const_int 0)
              (const_int 0)
              (reg:SI VL_REGNUM)
              (reg:SI VTYPE_REGNUM)
            ] UNSPEC_VPREDICATE)
            (vec_merge:<V_DOUBLE_EXTEND>
              (any_extend:<V_DOUBLE_EXTEND>
                (match_operand:VI_QHS_NO_M8 3 "register_operand"))
              (if_then_else:<V_DOUBLE_EXTEND>
                (unspec:<VM> [
                  (match_operand:<VM> 4 "vector_all_trues_mask_operand")
                  (match_operand 5 "vector_length_operand")
                  (match_operand 6 "const_int_operand")
                  (match_operand 7 "const_int_operand")
                  (match_operand 8 "const_1_or_2_operand")
                  (reg:SI VL_REGNUM)
                  (reg:SI VTYPE_REGNUM)
                ] UNSPEC_VPREDICATE)
                (match_operand:<V_DOUBLE_EXTEND> 9 "vector_const_0_operand")
                (match_operand:<V_DOUBLE_EXTEND> 10 "vector_merge_operand"))
              (match_operand:<VM> 1 "register_operand"))
            (if_then_else:<V_DOUBLE_EXTEND>
              (unspec:<VM> [
                (match_dup 4)
                (match_dup 5)
                (match_dup 6)
                (match_dup 7)
                (match_dup 8)
                (reg:SI VL_REGNUM)
                (reg:SI VTYPE_REGNUM)
              ] UNSPEC_VPREDICATE)
              (match_dup 9)
              (match_dup 10)))
        ] UNSPEC_REDUC_SUM))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  rtx ops[] = {operands[0], operands[3], operands[1], operands[2]};
  riscv_vector::expand_reduction (<WREDUC_UNSPEC>,
                                  riscv_vector::REDUCE_OP_M,
                                  ops, CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine mask_extend + vfredsum to mask_vfwredusum
;; where the mrege of mask_extend is vector const 0
(define_insn_and_split "*cond_widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (if_then_else:<V_DOUBLE_EXTEND>
            (match_operand:<VM> 1 "register_operand")
            (float_extend:<V_DOUBLE_EXTEND>
              (match_operand:VF_HS_NO_M8 2 "register_operand"))
            (if_then_else:<V_DOUBLE_EXTEND>
              (unspec:<VM> [
                (match_operand:<VM> 3 "vector_all_trues_mask_operand")
                (match_operand 6 "vector_length_operand")
                (match_operand 7 "const_int_operand")
                (match_operand 8 "const_int_operand")
                (match_operand 9 "const_1_or_2_operand")
                (reg:SI VL_REGNUM)
                (reg:SI VTYPE_REGNUM)
              ] UNSPEC_VPREDICATE)
              (match_operand:<V_DOUBLE_EXTEND> 5 "vector_const_0_operand")
              (match_operand:<V_DOUBLE_EXTEND> 4 "vector_merge_operand")))
        ] UNSPEC_REDUC_SUM_UNORDERED))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  rtx ops[] = {operands[0], operands[2], operands[1],
               gen_int_mode (GET_MODE_NUNITS (<MODE>mode), Pmode)};
  riscv_vector::expand_reduction (UNSPEC_WREDUC_SUM_UNORDERED,
                                  riscv_vector::REDUCE_OP_M_FRM_DYN,
                                  ops, CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; Combine mask_len_extend + vredsum to mask_vwredsum[u]
;; where the mrege of mask_len_extend is vector const 0
(define_insn_and_split "*cond_len_widen_reduc_plus_scal_<mode>"
  [(set (match_operand:<V_DOUBLE_EXTEND_VEL> 0 "register_operand")
        (unspec:<V_DOUBLE_EXTEND_VEL> [
          (if_then_else:<V_DOUBLE_EXTEND>
            (unspec:<VM> [
              (match_operand 2 "vector_length_operand")
              (const_int 0)
              (const_int 0)
              (reg:SI VL_REGNUM)
              (reg:SI VTYPE_REGNUM)
            ] UNSPEC_VPREDICATE)
            (vec_merge:<V_DOUBLE_EXTEND>
              (float_extend:<V_DOUBLE_EXTEND>
                (match_operand:VF_HS_NO_M8 3 "register_operand"))
              (if_then_else:<V_DOUBLE_EXTEND>
                (unspec:<VM> [
                  (match_operand:<VM> 4 "vector_all_trues_mask_operand")
                  (match_operand 5 "vector_length_operand")
                  (match_operand 6 "const_int_operand")
                  (match_operand 7 "const_int_operand")
                  (match_operand 8 "const_1_or_2_operand")
                  (reg:SI VL_REGNUM)
                  (reg:SI VTYPE_REGNUM)
                ] UNSPEC_VPREDICATE)
                (match_operand:<V_DOUBLE_EXTEND> 9 "vector_const_0_operand")
                (match_operand:<V_DOUBLE_EXTEND> 10 "vector_merge_operand"))
              (match_operand:<VM> 1 "register_operand"))
            (if_then_else:<V_DOUBLE_EXTEND>
              (unspec:<VM> [
                (match_dup 4)
                (match_dup 5)
                (match_dup 6)
                (match_dup 7)
                (match_dup 8)
                (reg:SI VL_REGNUM)
                (reg:SI VTYPE_REGNUM)
              ] UNSPEC_VPREDICATE)
              (match_dup 9)
              (match_dup 10)))
        ] UNSPEC_REDUC_SUM_UNORDERED))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  rtx ops[] = {operands[0], operands[3], operands[1], operands[2]};
  riscv_vector::expand_reduction (UNSPEC_WREDUC_SUM_UNORDERED,
                                  riscv_vector::REDUCE_OP_M_FRM_DYN,
                                  ops, CONST0_RTX (<V_DOUBLE_EXTEND_VEL>mode));
  DONE;
}
[(set_attr "type" "vector")])

;; =============================================================================
;; Misc combine patterns
;; =============================================================================

;; Combine neg + vfsgnj to vfsgnjn
(define_insn_and_split "*copysign<mode>_neg"
  [(set (match_operand:VF 0 "register_operand")
        (neg:VF
          (unspec:VF [
            (match_operand:VF 1 "register_operand")
            (match_operand:VF 2 "register_operand")
          ] UNSPEC_VCOPYSIGN)))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
{
  riscv_vector::emit_vlmax_insn (code_for_pred_ncopysign (<MODE>mode),
                                  riscv_vector::BINARY_OP, operands);
  DONE;
}
[(set_attr "type" "vector")])

;; Combine vmand/vmor + vmnot to vmandnot/vmornot
(define_insn_and_split "*<optab>not<mode>"
  [(set (match_operand:VB_VLS 0 "register_operand"           "=vr")
	(bitmanip_bitwise:VB_VLS
	  (not:VB_VLS (match_operand:VB_VLS 2 "register_operand" " vr"))
	  (match_operand:VB_VLS 1 "register_operand"         " vr")))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_not (<CODE>, <MODE>mode);
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_MASK_OP, operands);
    DONE;
  }
  [(set_attr "type" "vmalu")])

;; Combine vmnot + vmand/vmor/vmxor to vmnand/vmnor/vmxnor
(define_insn_and_split "*n<optab><mode>"
  [(set (match_operand:VB_VLS 0 "register_operand"     "=vr")
	(not:VB_VLS
	  (any_bitwise:VB_VLS
	    (match_operand:VB_VLS 1 "register_operand" " vr")
	    (match_operand:VB_VLS 2 "register_operand" " vr"))))]
  "TARGET_VECTOR && can_create_pseudo_p ()"
  "#"
  "&& 1"
  [(const_int 0)]
  {
    insn_code icode = code_for_pred_n (<CODE>, <MODE>mode);
    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_MASK_OP, operands);
    DONE;
  }
  [(set_attr "type" "vmalu")])