LLVM API Documentation

X86GenDAGISel.inc

Go to the documentation of this file.
00001 //===- TableGen'erated file -------------------------------------*- C++ -*-===//
00002 //
00003 // DAG Instruction Selector for the X86 target
00004 //
00005 // Automatically generated file, do not edit!
00006 //
00007 //===----------------------------------------------------------------------===//
00008 
00009 // *** NOTE: This file is #included into the middle of the target
00010 // *** instruction selector class.  These functions are really methods.
00011 
00012 #if defined(__GNUC__) && \
00013     ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)))
00014 #define NOINLINE __attribute__((noinline))
00015 #else
00016 
00017 #define NOINLINE
00018 
00019 #endif
00020 
00021 // Instance var to keep track of multiply used nodes that have 
00022 // already been selected.
00023 std::map<SDOperand, SDOperand> CodeGenMap;
00024 // Instance var to keep track of mapping of chain generating nodes
00025 // and their place handle nodes.
00026 std::map<SDOperand, SDOperand> HandleMap;
00027 // Instance var to keep track of mapping of place handle nodes
00028 // and their replacement nodes.
00029 std::map<SDOperand, SDOperand> ReplaceMap;
00030 // Keep track of nodes that are currently being selecte and therefore
00031 // should not be folded.
00032 std::set<SDNode*> InFlightSet;
00033 
00034 static void findNonImmUse(SDNode* Use, SDNode* Def, bool &found, std::set<SDNode *> &Visited) {
00035   if (found || !Visited.insert(Use).second) return;
00036   for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
00037     SDNode *N = Use->getOperand(i).Val;
00038     if (N != Def) {
00039       findNonImmUse(N, Def, found, Visited);
00040     } else {
00041       found = true;
00042       break;
00043     }
00044   }
00045 }
00046 
00047 static bool isNonImmUse(SDNode* Use, SDNode* Def) {
00048   std::set<SDNode *> Visited;
00049   bool found = false;
00050   for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
00051     SDNode *N = Use->getOperand(i).Val;
00052     if (N != Def) {
00053       findNonImmUse(N, Def, found, Visited);
00054       if (found) break;
00055     }
00056   }
00057   return found;
00058 }
00059 
00060 // AddHandleReplacement - Note the pending replacement node for a
00061 // handle node in ReplaceMap.
00062 void AddHandleReplacement(SDNode *H, unsigned HNum, SDNode *R, unsigned RNum) {
00063   SDOperand N(H, HNum);
00064   std::map<SDOperand, SDOperand>::iterator HMI = HandleMap.find(N);
00065   if (HMI != HandleMap.end()) {
00066     ReplaceMap[HMI->second] = SDOperand(R, RNum);
00067     HandleMap.erase(N);
00068   }
00069 }
00070 
00071 // SelectDanglingHandles - Select replacements for all `dangling`
00072 // handles.Some handles do not yet have replacements because the
00073 // nodes they replacements have only dead readers.
00074 void SelectDanglingHandles() {
00075   for (std::map<SDOperand, SDOperand>::iterator I = HandleMap.begin(),
00076          E = HandleMap.end(); I != E; ++I) {
00077     SDOperand N = I->first;
00078     SDOperand R;
00079     Select(R, N.getValue(0));
00080     AddHandleReplacement(N.Val, N.ResNo, R.Val, R.ResNo);
00081   }
00082 }
00083 
00084 // ReplaceHandles - Replace all the handles with the real target
00085 // specific nodes.
00086 void ReplaceHandles() {
00087   for (std::map<SDOperand, SDOperand>::iterator I = ReplaceMap.begin(),
00088         E = ReplaceMap.end(); I != E; ++I) {
00089     SDOperand From = I->first;
00090     SDOperand To   = I->second;
00091     for (SDNode::use_iterator UI = From.Val->use_begin(), E = From.Val->use_end(); UI != E; ++UI) {
00092       SDNode *Use = *UI;
00093       std::vector<SDOperand> Ops;
00094       for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i){
00095         SDOperand O = Use->getOperand(i);
00096         if (O.Val == From.Val)
00097           Ops.push_back(To);
00098         else
00099           Ops.push_back(O);
00100       }
00101       SDOperand U = SDOperand(Use, 0);
00102       CurDAG->UpdateNodeOperands(U, Ops);
00103     }
00104   }
00105 }
00106 
00107 // SelectRoot - Top level entry to DAG isel.
00108 SDOperand SelectRoot(SDOperand N) {
00109   SDOperand ResNode;
00110   Select(ResNode, N);
00111   SelectDanglingHandles();
00112   ReplaceHandles();
00113   ReplaceMap.clear();
00114   return ResNode;
00115 }
00116 
00117 // Node transformations.
00118 inline SDOperand Transform_SHUFFLE_get_shuf_imm(SDNode *N) {
00119 
00120   return getI8Imm(X86::getShuffleSHUFImmediate(N));
00121 
00122 }
00123 inline SDOperand Transform_SHUFFLE_get_pshuflw_imm(SDNode *N) {
00124 
00125   return getI8Imm(X86::getShufflePSHUFLWImmediate(N));
00126 
00127 }
00128 inline SDOperand Transform_SHUFFLE_get_pshufhw_imm(SDNode *N) {
00129 
00130   return getI8Imm(X86::getShufflePSHUFHWImmediate(N));
00131 
00132 }
00133 inline SDOperand Transform_PSxLDQ_imm(SDNode *inN) {
00134   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00135 
00136   // Transformation function: imm >> 3
00137   return getI32Imm(N->getValue() >> 3);
00138 
00139 }
00140 
00141 // Predicate functions.
00142 inline bool Predicate_MOVHLPS_shuffle_mask(SDNode *N) {
00143 
00144   return X86::isMOVHLPSMask(N);
00145 
00146 }
00147 inline bool Predicate_MOVHP_shuffle_mask(SDNode *N) {
00148 
00149   return X86::isMOVHPMask(N);
00150 
00151 }
00152 inline bool Predicate_MOVLP_shuffle_mask(SDNode *N) {
00153 
00154   return X86::isMOVLPMask(N);
00155 
00156 }
00157 inline bool Predicate_MOVL_shuffle_mask(SDNode *N) {
00158 
00159   return X86::isMOVLMask(N);
00160 
00161 }
00162 inline bool Predicate_MOVSHDUP_shuffle_mask(SDNode *N) {
00163 
00164   return X86::isMOVSHDUPMask(N);
00165 
00166 }
00167 inline bool Predicate_MOVSLDUP_shuffle_mask(SDNode *N) {
00168 
00169   return X86::isMOVSLDUPMask(N);
00170 
00171 }
00172 inline bool Predicate_PSHUFD_binary_shuffle_mask(SDNode *N) {
00173 
00174   return X86::isSHUFPMask(N);
00175 
00176 }
00177 inline bool Predicate_PSHUFD_shuffle_mask(SDNode *N) {
00178 
00179   return X86::isPSHUFDMask(N);
00180 
00181 }
00182 inline bool Predicate_PSHUFHW_shuffle_mask(SDNode *N) {
00183 
00184   return X86::isPSHUFHWMask(N);
00185 
00186 }
00187 inline bool Predicate_PSHUFLW_shuffle_mask(SDNode *N) {
00188 
00189   return X86::isPSHUFLWMask(N);
00190 
00191 }
00192 inline bool Predicate_SHUFP_shuffle_mask(SDNode *N) {
00193 
00194   return X86::isSHUFPMask(N);
00195 
00196 }
00197 inline bool Predicate_SHUFP_unary_shuffle_mask(SDNode *N) {
00198 
00199   return X86::isPSHUFDMask(N);
00200 
00201 }
00202 inline bool Predicate_SSE_splat_mask(SDNode *N) {
00203 
00204   return X86::isSplatMask(N);
00205 
00206 }
00207 inline bool Predicate_SSE_splat_v2_mask(SDNode *N) {
00208 
00209   return X86::isSplatMask(N);
00210 
00211 }
00212 inline bool Predicate_UNPCKH_shuffle_mask(SDNode *N) {
00213 
00214   return X86::isUNPCKHMask(N);
00215 
00216 }
00217 inline bool Predicate_UNPCKL_shuffle_mask(SDNode *N) {
00218 
00219   return X86::isUNPCKLMask(N);
00220 
00221 }
00222 inline bool Predicate_UNPCKL_v_undef_shuffle_mask(SDNode *N) {
00223 
00224   return X86::isUNPCKL_v_undef_Mask(N);
00225 
00226 }
00227 inline bool Predicate_fp32imm0(SDNode *inN) {
00228   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00229 
00230   return N->isExactlyValue(+0.0);
00231 
00232 }
00233 inline bool Predicate_fp64imm0(SDNode *inN) {
00234   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00235 
00236   return N->isExactlyValue(+0.0);
00237 
00238 }
00239 inline bool Predicate_fp64imm1(SDNode *inN) {
00240   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00241 
00242   return N->isExactlyValue(+1.0);
00243 
00244 }
00245 inline bool Predicate_fp64immneg0(SDNode *inN) {
00246   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00247 
00248   return N->isExactlyValue(-0.0);
00249 
00250 }
00251 inline bool Predicate_fp64immneg1(SDNode *inN) {
00252   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00253 
00254   return N->isExactlyValue(-1.0);
00255 
00256 }
00257 inline bool Predicate_i16immSExt8(SDNode *inN) {
00258   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00259 
00260   // i16immSExt8 predicate - True if the 16-bit immediate fits in a 8-bit
00261   // sign extended field.
00262   return (int16_t)N->getValue() == (int8_t)N->getValue();
00263 
00264 }
00265 inline bool Predicate_i32immSExt8(SDNode *inN) {
00266   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00267 
00268   // i32immSExt8 predicate - True if the 32-bit immediate fits in a 8-bit
00269   // sign extended field.
00270   return (int32_t)N->getValue() == (int8_t)N->getValue();
00271 
00272 }
00273 inline bool Predicate_immAllOnes(SDNode *inN) {
00274   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00275  return N->isAllOnesValue(); 
00276 }
00277 inline bool Predicate_immAllOnesV(SDNode *N) {
00278 
00279   return ISD::isBuildVectorAllOnes(N);
00280 
00281 }
00282 inline bool Predicate_immAllOnesV_bc(SDNode *N) {
00283 
00284   return ISD::isBuildVectorAllOnes(N);
00285 
00286 }
00287 inline bool Predicate_immAllZerosV(SDNode *N) {
00288 
00289   return ISD::isBuildVectorAllZeros(N);
00290 
00291 }
00292 inline bool Predicate_vtFP(SDNode *inN) {
00293   VTSDNode *N = cast<VTSDNode>(inN);
00294  return MVT::isFloatingPoint(N->getVT()); 
00295 }
00296 inline bool Predicate_vtInt(SDNode *inN) {
00297   VTSDNode *N = cast<VTSDNode>(inN);
00298  return MVT::isInteger(N->getVT()); 
00299 }
00300 
00301 
00302 void Emit_0(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0) NOINLINE {
00303   SDOperand Tmp0(0, 0);
00304   SDNode *ResNode = NULL;
00305   Tmp0 = N0;
00306   if (N.Val->hasOneUse()) {
00307     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
00308   } else {
00309     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
00310     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
00311     Result = SDOperand(ResNode, 0);
00312   }
00313   return;
00314 }
00315 void Emit_1(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0) NOINLINE {
00316   SDOperand Tmp0(0, 0);
00317   SDNode *ResNode = NULL;
00318   Select(Tmp0, N0);
00319   if (N.Val->hasOneUse()) {
00320     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
00321   } else {
00322     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
00323     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
00324     Result = SDOperand(ResNode, 0);
00325   }
00326   return;
00327 }
00328 void Select_X86Wrapper(SDOperand &Result, SDOperand N) {
00329   SDOperand N0(0, 0);
00330   SDOperand Tmp0(0, 0);
00331   SDNode *ResNode = NULL;
00332   N0 = N.getOperand(0);
00333 
00334   // Pattern: (X86Wrapper:i32 (tconstpool:i32):$dst)
00335   // Emits: (MOV32ri:i32 (tconstpool:i32):$dst)
00336   // Pattern complexity = 4  cost = 1  size = 3
00337   if (N0.getOpcode() == ISD::TargetConstantPool &&
00338       N.Val->getValueType(0) == MVT::i32) {
00339     Emit_0(Result, N, X86::MOV32ri, MVT::i32, N0);
00340     return;
00341   }
00342 
00343   // Pattern: (X86Wrapper:i32 (tjumptable:i32):$dst)
00344   // Emits: (MOV32ri:i32 (tjumptable:i32):$dst)
00345   // Pattern complexity = 4  cost = 1  size = 3
00346   if (N0.getOpcode() == ISD::TargetJumpTable &&
00347       N.Val->getValueType(0) == MVT::i32) {
00348     Emit_1(Result, N, X86::MOV32ri, MVT::i32, N0);
00349     return;
00350   }
00351 
00352   // Pattern: (X86Wrapper:i32 (tglobaladdr:i32):$dst)
00353   // Emits: (MOV32ri:i32 (tglobaladdr:i32):$dst)
00354   // Pattern complexity = 4  cost = 1  size = 3
00355   if (N0.getOpcode() == ISD::TargetGlobalAddress &&
00356       N.Val->getValueType(0) == MVT::i32) {
00357     Emit_0(Result, N, X86::MOV32ri, MVT::i32, N0);
00358     return;
00359   }
00360 
00361   // Pattern: (X86Wrapper:i32 (texternalsym:i32):$dst)
00362   // Emits: (MOV32ri:i32 (texternalsym:i32):$dst)
00363   // Pattern complexity = 4  cost = 1
00364   if (N0.getOpcode() == ISD::TargetExternalSymbol &&
00365       N.Val->getValueType(0) == MVT::i32) {
00366     Emit_0(Result, N, X86::MOV32ri, MVT::i32, N0);
00367     return;
00368   }
00369   std::cerr << "Cannot yet select: ";
00370   N.Val->dump(CurDAG);
00371   std::cerr << '\n';
00372   abort();
00373 }
00374 
00375 void Emit_2(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
00376   SDOperand InFlag(0, 0);
00377   SDOperand Tmp1(0, 0);
00378   SDNode *ResNode = NULL;
00379   Select(Tmp1, N1);
00380   Select(Chain, Chain);
00381   Select(InFlag, N.getOperand(3));
00382   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Chain, InFlag);
00383   Chain = SDOperand(ResNode, 0);
00384   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
00385   Result = SDOperand(ResNode, N.ResNo);
00386   return;
00387 }
00388 void Select_X86brcond(SDOperand &Result, SDOperand N) {
00389   SDOperand Chain(0, 0);
00390   SDOperand InFlag(0, 0);
00391   SDOperand N1(0, 0);
00392   SDOperand N2(0, 0);
00393   SDOperand Tmp1(0, 0);
00394   SDNode *ResNode = NULL;
00395   Chain = N.getOperand(0);
00396   N1 = N.getOperand(1);
00397   if (N1.getOpcode() == ISD::BasicBlock) {
00398     N2 = N.getOperand(2);
00399     if (isa<ConstantSDNode>(N2)) {
00400       int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
00401 
00402       // Pattern: (X86brcond:void (bb:Other):$dst, 4:i8)
00403       // Emits: (JE:void (bb:Other):$dst)
00404       // Pattern complexity = 5  cost = 1  size = 3
00405       if (CN0 == 4) {
00406         Emit_2(Result, N, X86::JE, Chain, N1, N2);
00407         return;
00408       }
00409 
00410       // Pattern: (X86brcond:void (bb:Other):$dst, 9:i8)
00411       // Emits: (JNE:void (bb:Other):$dst)
00412       // Pattern complexity = 5  cost = 1  size = 3
00413       if (CN0 == 9) {
00414         Emit_2(Result, N, X86::JNE, Chain, N1, N2);
00415         return;
00416       }
00417 
00418       // Pattern: (X86brcond:void (bb:Other):$dst, 7:i8)
00419       // Emits: (JL:void (bb:Other):$dst)
00420       // Pattern complexity = 5  cost = 1  size = 3
00421       if (CN0 == 7) {
00422         Emit_2(Result, N, X86::JL, Chain, N1, N2);
00423         return;
00424       }
00425 
00426       // Pattern: (X86brcond:void (bb:Other):$dst, 8:i8)
00427       // Emits: (JLE:void (bb:Other):$dst)
00428       // Pattern complexity = 5  cost = 1  size = 3
00429       if (CN0 == 8) {
00430         Emit_2(Result, N, X86::JLE, Chain, N1, N2);
00431         return;
00432       }
00433 
00434       // Pattern: (X86brcond:void (bb:Other):$dst, 5:i8)
00435       // Emits: (JG:void (bb:Other):$dst)
00436       // Pattern complexity = 5  cost = 1  size = 3
00437       if (CN0 == 5) {
00438         Emit_2(Result, N, X86::JG, Chain, N1, N2);
00439         return;
00440       }
00441 
00442       // Pattern: (X86brcond:void (bb:Other):$dst, 6:i8)
00443       // Emits: (JGE:void (bb:Other):$dst)
00444       // Pattern complexity = 5  cost = 1  size = 3
00445       if (CN0 == 6) {
00446         Emit_2(Result, N, X86::JGE, Chain, N1, N2);
00447         return;
00448       }
00449 
00450       // Pattern: (X86brcond:void (bb:Other):$dst, 2:i8)
00451       // Emits: (JB:void (bb:Other):$dst)
00452       // Pattern complexity = 5  cost = 1  size = 3
00453       if (CN0 == 2) {
00454         Emit_2(Result, N, X86::JB, Chain, N1, N2);
00455         return;
00456       }
00457 
00458       // Pattern: (X86brcond:void (bb:Other):$dst, 3:i8)
00459       // Emits: (JBE:void (bb:Other):$dst)
00460       // Pattern complexity = 5  cost = 1  size = 3
00461       if (CN0 == 3) {
00462         Emit_2(Result, N, X86::JBE, Chain, N1, N2);
00463         return;
00464       }
00465 
00466       // Pattern: (X86brcond:void (bb:Other):$dst, 0:i8)
00467       // Emits: (JA:void (bb:Other):$dst)
00468       // Pattern complexity = 5  cost = 1  size = 3
00469       if (CN0 == 0) {
00470         Emit_2(Result, N, X86::JA, Chain, N1, N2);
00471         return;
00472       }
00473 
00474       // Pattern: (X86brcond:void (bb:Other):$dst, 1:i8)
00475       // Emits: (JAE:void (bb:Other):$dst)
00476       // Pattern complexity = 5  cost = 1  size = 3
00477       if (CN0 == 1) {
00478         Emit_2(Result, N, X86::JAE, Chain, N1, N2);
00479         return;
00480       }
00481 
00482       // Pattern: (X86brcond:void (bb:Other):$dst, 15:i8)
00483       // Emits: (JS:void (bb:Other):$dst)
00484       // Pattern complexity = 5  cost = 1  size = 3
00485       if (CN0 == 15) {
00486         Emit_2(Result, N, X86::JS, Chain, N1, N2);
00487         return;
00488       }
00489 
00490       // Pattern: (X86brcond:void (bb:Other):$dst, 12:i8)
00491       // Emits: (JNS:void (bb:Other):$dst)
00492       // Pattern complexity = 5  cost = 1  size = 3
00493       if (CN0 == 12) {
00494         Emit_2(Result, N, X86::JNS, Chain, N1, N2);
00495         return;
00496       }
00497 
00498       // Pattern: (X86brcond:void (bb:Other):$dst, 14:i8)
00499       // Emits: (JP:void (bb:Other):$dst)
00500       // Pattern complexity = 5  cost = 1  size = 3
00501       if (CN0 == 14) {
00502         Emit_2(Result, N, X86::JP, Chain, N1, N2);
00503         return;
00504       }
00505 
00506       // Pattern: (X86brcond:void (bb:Other):$dst, 11:i8)
00507       // Emits: (JNP:void (bb:Other):$dst)
00508       // Pattern complexity = 5  cost = 1  size = 3
00509       if (CN0 == 11) {
00510         Emit_2(Result, N, X86::JNP, Chain, N1, N2);
00511         return;
00512       }
00513 
00514       // Pattern: (X86brcond:void (bb:Other):$dst, 13:i8)
00515       // Emits: (JO:void (bb:Other):$dst)
00516       // Pattern complexity = 5  cost = 1  size = 3
00517       if (CN0 == 13) {
00518         Emit_2(Result, N, X86::JO, Chain, N1, N2);
00519         return;
00520       }
00521 
00522       // Pattern: (X86brcond:void (bb:Other):$dst, 10:i8)
00523       // Emits: (JNO:void (bb:Other):$dst)
00524       // Pattern complexity = 5  cost = 1
00525       if (CN0 == 10) {
00526         Emit_2(Result, N, X86::JNO, Chain, N1, N2);
00527         return;
00528       }
00529     }
00530   }
00531   std::cerr << "Cannot yet select: ";
00532   N.Val->dump(CurDAG);
00533   std::cerr << '\n';
00534   abort();
00535 }
00536 
00537 void Emit_3(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, bool HasInFlag) NOINLINE {
00538   SDOperand InFlag(0, 0);
00539   SDOperand Tmp0(0, 0);
00540   SDNode *ResNode = NULL;
00541   HasInFlag = (N.getOperand(N.getNumOperands()-1).getValueType() == MVT::Flag);
00542   std::vector<SDOperand> Ops;
00543   Tmp0 = N1;
00544   Select(Chain, Chain);
00545   if (HasInFlag)
00546     Select(InFlag, N.getOperand(N.getNumOperands()-1));
00547   Ops.push_back(Tmp0);
00548   for (unsigned i = 2, e = N.getNumOperands()-(HasInFlag?1:0); i != e; ++i) {
00549     SDOperand VarOp(0, 0);
00550     Select(VarOp, N.getOperand(i));
00551     Ops.push_back(VarOp);
00552   }
00553   Ops.push_back(Chain);
00554   if (HasInFlag)
00555     Ops.push_back(InFlag);
00556   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Ops);
00557   Chain = SDOperand(ResNode, 0);
00558   InFlag = SDOperand(ResNode, 1);
00559   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
00560   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00561   Result = SDOperand(ResNode, N.ResNo);
00562   return;
00563 }
00564 void Emit_4(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, bool HasInFlag) NOINLINE {
00565   SDOperand InFlag(0, 0);
00566   SDOperand Tmp0(0, 0);
00567   SDNode *ResNode = NULL;
00568   Chain = N.getOperand(0);
00569   N1 = N.getOperand(1);
00570   HasInFlag = (N.getOperand(N.getNumOperands()-1).getValueType() == MVT::Flag);
00571   std::vector<SDOperand> Ops;
00572   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
00573   Select(Tmp0, N1);
00574   Select(Chain, Chain);
00575   if (HasInFlag)
00576     Select(InFlag, N.getOperand(N.getNumOperands()-1));
00577   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
00578   Ops.push_back(Tmp0);
00579   for (unsigned i = 2, e = N.getNumOperands()-(HasInFlag?1:0); i != e; ++i) {
00580     SDOperand VarOp(0, 0);
00581     Select(VarOp, N.getOperand(i));
00582     Ops.push_back(VarOp);
00583   }
00584   Ops.push_back(Chain);
00585   if (HasInFlag)
00586     Ops.push_back(InFlag);
00587   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Ops);
00588   Chain = SDOperand(ResNode, 0);
00589   InFlag = SDOperand(ResNode, 1);
00590   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
00591   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00592   Result = SDOperand(ResNode, N.ResNo);
00593   return;
00594 }
00595 void Select_X86call(SDOperand &Result, SDOperand N) {
00596   SDOperand Chain(0, 0);
00597   SDOperand InFlag(0, 0);
00598   SDOperand N1(0, 0);
00599   SDOperand Tmp0(0, 0);
00600   SDNode *ResNode = NULL;
00601   bool HasInFlag = false;
00602   {
00603     Chain = N.getOperand(0);
00604     N1 = N.getOperand(1);
00605 
00606     // Pattern: (X86call:void (tglobaladdr:i32):$dst)
00607     // Emits: (CALLpcrel32:void (tglobaladdr:i32):$dst)
00608     // Pattern complexity = 4  cost = 1  size = 3
00609     if (N1.getOpcode() == ISD::TargetGlobalAddress) {
00610       Emit_3(Result, N, X86::CALLpcrel32, Chain, N1, HasInFlag);
00611       return;
00612     }
00613 
00614     // Pattern: (X86call:void (texternalsym:i32):$dst)
00615     // Emits: (CALLpcrel32:void (texternalsym:i32):$dst)
00616     // Pattern complexity = 4  cost = 1
00617     if (N1.getOpcode() == ISD::TargetExternalSymbol) {
00618       Emit_3(Result, N, X86::CALLpcrel32, Chain, N1, HasInFlag);
00619       return;
00620     }
00621   }
00622 
00623   // Pattern: (X86call:void GR32:i32:$dst)
00624   // Emits: (CALL32r:void GR32:i32:$dst)
00625   // Pattern complexity = 2  cost = 1
00626   Emit_4(Result, N, X86::CALL32r, Chain, N1, HasInFlag);
00627   return;
00628 }
00629 
00630 void Emit_5(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
00631   SDOperand InFlag(0, 0);
00632   SDOperand Tmp0(0, 0);
00633   SDOperand Tmp1(0, 0);
00634   SDNode *ResNode = NULL;
00635   Tmp0 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
00636   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N2)->getValue()), MVT::i32);
00637   Select(Chain, Chain);
00638   Select(InFlag, N.getOperand(3));
00639   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Chain, InFlag);
00640   Chain = SDOperand(ResNode, 0);
00641   InFlag = SDOperand(ResNode, 1);
00642   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
00643   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00644   Result = SDOperand(ResNode, N.ResNo);
00645   return;
00646 }
00647 void Select_X86callseq_end(SDOperand &Result, SDOperand N) {
00648   SDOperand Chain(0, 0);
00649   SDOperand InFlag(0, 0);
00650   SDOperand N1(0, 0);
00651   SDOperand N2(0, 0);
00652   SDOperand Tmp0(0, 0);
00653   SDOperand Tmp1(0, 0);
00654   SDNode *ResNode = NULL;
00655   Chain = N.getOperand(0);
00656   N1 = N.getOperand(1);
00657   if (N1.getOpcode() == ISD::Constant) {
00658     N2 = N.getOperand(2);
00659     if (N2.getOpcode() == ISD::Constant) {
00660       Emit_5(Result, N, X86::ADJCALLSTACKUP, Chain, N1, N2);
00661       return;
00662     }
00663   }
00664   std::cerr << "Cannot yet select: ";
00665   N.Val->dump(CurDAG);
00666   std::cerr << '\n';
00667   abort();
00668 }
00669 
00670 void Emit_6(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
00671   SDOperand Tmp0(0, 0);
00672   SDNode *ResNode = NULL;
00673   Tmp0 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
00674   Select(Chain, Chain);
00675   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Chain);
00676   Chain = SDOperand(ResNode, 0);
00677   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
00678   Result = SDOperand(ResNode, N.ResNo);
00679   return;
00680 }
00681 void Select_X86callseq_start(SDOperand &Result, SDOperand N) {
00682   SDOperand Chain(0, 0);
00683   SDOperand N1(0, 0);
00684   SDOperand Tmp0(0, 0);
00685   SDNode *ResNode = NULL;
00686   Chain = N.getOperand(0);
00687   N1 = N.getOperand(1);
00688   if (N1.getOpcode() == ISD::Constant) {
00689     Emit_6(Result, N, X86::ADJCALLSTACKDOWN, Chain, N1);
00690     return;
00691   }
00692   std::cerr << "Cannot yet select: ";
00693   N.Val->dump(CurDAG);
00694   std::cerr << '\n';
00695   abort();
00696 }
00697 
00698 void Emit_7(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11, SDOperand &N2) NOINLINE {
00699   SDOperand InFlag(0, 0);
00700   SDOperand Tmp1(0, 0);
00701   SDOperand Tmp2(0, 0);
00702   SDOperand Tmp3(0, 0);
00703   SDOperand Tmp4(0, 0);
00704   SDOperand Tmp5(0, 0);
00705   SDNode *ResNode = NULL;
00706   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
00707   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
00708   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
00709   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
00710   Select(Tmp1, CPTmp1);
00711   Select(Tmp2, CPTmp2);
00712   Select(Tmp3, CPTmp3);
00713   Select(Tmp4, CPTmp4);
00714   Select(Tmp5, N0);
00715   Select(Chain1, Chain1);
00716   Select(InFlag, N.getOperand(3));
00717   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00718   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
00719   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
00720   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
00721   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
00722   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00723   Chain1 = SDOperand(ResNode, 1);
00724   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00725   InFlag = SDOperand(ResNode, 2);
00726   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00727   AddHandleReplacement(N1.Val, 1, ResNode, 1);
00728   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00729   if (N.ResNo < 1)
00730     Result = SDOperand(ResNode, N.ResNo);
00731   else
00732     Result = SDOperand(ResNode, N.ResNo+1);
00733   return;
00734 }
00735 void Emit_8(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
00736   SDOperand InFlag(0, 0);
00737   SDOperand Tmp1(0, 0);
00738   SDOperand Tmp2(0, 0);
00739   SDNode *ResNode = NULL;
00740   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00741   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
00742   Select(Tmp1, N0);
00743   Select(Tmp2, N1);
00744   Select(InFlag, N.getOperand(3));
00745   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00746   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
00747   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp1, Tmp2, InFlag);
00748   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00749   InFlag = SDOperand(ResNode, 1);
00750   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00751   Result = SDOperand(ResNode, N.ResNo);
00752   return;
00753 }
00754 void Emit_9(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
00755   SDOperand InFlag(0, 0);
00756   SDOperand Tmp0(0, 0);
00757   SDOperand Tmp1(0, 0);
00758   SDOperand Tmp2(0, 0);
00759   SDNode *ResNode = NULL;
00760   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00761   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
00762   Select(Tmp0, N0);
00763   Select(Tmp1, N1);
00764   Tmp2 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N2)->getValue()), MVT::i8);
00765   Select(InFlag, N.getOperand(3));
00766   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00767   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
00768   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
00769   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00770   InFlag = SDOperand(ResNode, 1);
00771   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00772   Result = SDOperand(ResNode, N.ResNo);
00773   return;
00774 }
00775 void Select_X86cmov(SDOperand &Result, SDOperand N) {
00776   SDOperand CPTmp1(0, 0);
00777   SDOperand CPTmp2(0, 0);
00778   SDOperand CPTmp3(0, 0);
00779   SDOperand CPTmp4(0, 0);
00780   SDOperand Chain1(0, 0);
00781   SDOperand InFlag(0, 0);
00782   SDOperand N0(0, 0);
00783   SDOperand N1(0, 0);
00784   SDOperand N11(0, 0);
00785   SDOperand N2(0, 0);
00786   SDOperand Tmp0(0, 0);
00787   SDOperand Tmp1(0, 0);
00788   SDOperand Tmp2(0, 0);
00789   SDOperand Tmp3(0, 0);
00790   SDOperand Tmp4(0, 0);
00791   SDOperand Tmp5(0, 0);
00792   SDNode *ResNode = NULL;
00793   {
00794     N0 = N.getOperand(0);
00795     N1 = N.getOperand(1);
00796     if (N1.getOpcode() == ISD::LOAD &&
00797         InFlightSet.count(N1.Val) == 0 &&
00798         N1.hasOneUse() &&
00799         !CodeGenMap.count(N1.getValue(0)) &&
00800         N.Val->isOnlyUse(N1.Val) &&
00801         !isNonImmUse(N.Val, N1.Val)) {
00802       Chain1 = N1.getOperand(0);
00803       N11 = N1.getOperand(1);
00804       N2 = N.getOperand(2);
00805       if (isa<ConstantSDNode>(N2)) {
00806         int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
00807         if (CN0 == 2) {
00808 
00809           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 2:i8)
00810           // Emits: (CMOVB16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00811           // Pattern complexity = 17  cost = 1  size = 3
00812           if (N.Val->getValueType(0) == MVT::i16) {
00813             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00814             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00815             if (!Match) {
00816               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00817             }
00818             if (Match) {
00819               Emit_7(Result, N, X86::CMOVB16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00820               return;
00821             }
00822           }
00823 
00824           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 2:i8)
00825           // Emits: (CMOVB32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00826           // Pattern complexity = 17  cost = 1
00827           if (N.Val->getValueType(0) == MVT::i32) {
00828             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00829             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00830             if (!Match) {
00831               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00832             }
00833             if (Match) {
00834               Emit_7(Result, N, X86::CMOVB32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00835               return;
00836             }
00837           }
00838         }
00839         if (CN0 == 1) {
00840 
00841           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 1:i8)
00842           // Emits: (CMOVAE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00843           // Pattern complexity = 17  cost = 1  size = 3
00844           if (N.Val->getValueType(0) == MVT::i16) {
00845             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00846             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00847             if (!Match) {
00848               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00849             }
00850             if (Match) {
00851               Emit_7(Result, N, X86::CMOVAE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00852               return;
00853             }
00854           }
00855 
00856           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 1:i8)
00857           // Emits: (CMOVAE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00858           // Pattern complexity = 17  cost = 1
00859           if (N.Val->getValueType(0) == MVT::i32) {
00860             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00861             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00862             if (!Match) {
00863               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00864             }
00865             if (Match) {
00866               Emit_7(Result, N, X86::CMOVAE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00867               return;
00868             }
00869           }
00870         }
00871         if (CN0 == 4) {
00872 
00873           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 4:i8)
00874           // Emits: (CMOVE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00875           // Pattern complexity = 17  cost = 1  size = 3
00876           if (N.Val->getValueType(0) == MVT::i16) {
00877             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00878             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00879             if (!Match) {
00880               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00881             }
00882             if (Match) {
00883               Emit_7(Result, N, X86::CMOVE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00884               return;
00885             }
00886           }
00887 
00888           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 4:i8)
00889           // Emits: (CMOVE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00890           // Pattern complexity = 17  cost = 1
00891           if (N.Val->getValueType(0) == MVT::i32) {
00892             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00893             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00894             if (!Match) {
00895               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00896             }
00897             if (Match) {
00898               Emit_7(Result, N, X86::CMOVE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00899               return;
00900             }
00901           }
00902         }
00903         if (CN0 == 9) {
00904 
00905           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 9:i8)
00906           // Emits: (CMOVNE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00907           // Pattern complexity = 17  cost = 1  size = 3
00908           if (N.Val->getValueType(0) == MVT::i16) {
00909             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00910             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00911             if (!Match) {
00912               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00913             }
00914             if (Match) {
00915               Emit_7(Result, N, X86::CMOVNE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00916               return;
00917             }
00918           }
00919 
00920           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 9:i8)
00921           // Emits: (CMOVNE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00922           // Pattern complexity = 17  cost = 1
00923           if (N.Val->getValueType(0) == MVT::i32) {
00924             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00925             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00926             if (!Match) {
00927               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00928             }
00929             if (Match) {
00930               Emit_7(Result, N, X86::CMOVNE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00931               return;
00932             }
00933           }
00934         }
00935         if (CN0 == 3) {
00936 
00937           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 3:i8)
00938           // Emits: (CMOVBE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00939           // Pattern complexity = 17  cost = 1  size = 3
00940           if (N.Val->getValueType(0) == MVT::i16) {
00941             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00942             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00943             if (!Match) {
00944               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00945             }
00946             if (Match) {
00947               Emit_7(Result, N, X86::CMOVBE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00948               return;
00949             }
00950           }
00951 
00952           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 3:i8)
00953           // Emits: (CMOVBE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00954           // Pattern complexity = 17  cost = 1
00955           if (N.Val->getValueType(0) == MVT::i32) {
00956             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00957             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00958             if (!Match) {
00959               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00960             }
00961             if (Match) {
00962               Emit_7(Result, N, X86::CMOVBE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00963               return;
00964             }
00965           }
00966         }
00967         if (CN0 == 0) {
00968 
00969           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 0:i8)
00970           // Emits: (CMOVA16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
00971           // Pattern complexity = 17  cost = 1  size = 3
00972           if (N.Val->getValueType(0) == MVT::i16) {
00973             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00974             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00975             if (!Match) {
00976               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00977             }
00978             if (Match) {
00979               Emit_7(Result, N, X86::CMOVA16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00980               return;
00981             }
00982           }
00983 
00984           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 0:i8)
00985           // Emits: (CMOVA32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
00986           // Pattern complexity = 17  cost = 1
00987           if (N.Val->getValueType(0) == MVT::i32) {
00988             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
00989             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
00990             if (!Match) {
00991               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
00992             }
00993             if (Match) {
00994               Emit_7(Result, N, X86::CMOVA32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
00995               return;
00996             }
00997           }
00998         }
00999         if (CN0 == 7) {
01000 
01001           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 7:i8)
01002           // Emits: (CMOVL16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01003           // Pattern complexity = 17  cost = 1  size = 3
01004           if (N.Val->getValueType(0) == MVT::i16) {
01005             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01006             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01007             if (!Match) {
01008               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01009             }
01010             if (Match) {
01011               Emit_7(Result, N, X86::CMOVL16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01012               return;
01013             }
01014           }
01015 
01016           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 7:i8)
01017           // Emits: (CMOVL32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01018           // Pattern complexity = 17  cost = 1
01019           if (N.Val->getValueType(0) == MVT::i32) {
01020             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01021             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01022             if (!Match) {
01023               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01024             }
01025             if (Match) {
01026               Emit_7(Result, N, X86::CMOVL32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01027               return;
01028             }
01029           }
01030         }
01031         if (CN0 == 6) {
01032 
01033           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 6:i8)
01034           // Emits: (CMOVGE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01035           // Pattern complexity = 17  cost = 1  size = 3
01036           if (N.Val->getValueType(0) == MVT::i16) {
01037             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01038             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01039             if (!Match) {
01040               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01041             }
01042             if (Match) {
01043               Emit_7(Result, N, X86::CMOVGE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01044               return;
01045             }
01046           }
01047 
01048           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 6:i8)
01049           // Emits: (CMOVGE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01050           // Pattern complexity = 17  cost = 1
01051           if (N.Val->getValueType(0) == MVT::i32) {
01052             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01053             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01054             if (!Match) {
01055               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01056             }
01057             if (Match) {
01058               Emit_7(Result, N, X86::CMOVGE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01059               return;
01060             }
01061           }
01062         }
01063         if (CN0 == 8) {
01064 
01065           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 8:i8)
01066           // Emits: (CMOVLE16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01067           // Pattern complexity = 17  cost = 1  size = 3
01068           if (N.Val->getValueType(0) == MVT::i16) {
01069             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01070             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01071             if (!Match) {
01072               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01073             }
01074             if (Match) {
01075               Emit_7(Result, N, X86::CMOVLE16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01076               return;
01077             }
01078           }
01079 
01080           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 8:i8)
01081           // Emits: (CMOVLE32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01082           // Pattern complexity = 17  cost = 1
01083           if (N.Val->getValueType(0) == MVT::i32) {
01084             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01085             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01086             if (!Match) {
01087               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01088             }
01089             if (Match) {
01090               Emit_7(Result, N, X86::CMOVLE32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01091               return;
01092             }
01093           }
01094         }
01095         if (CN0 == 5) {
01096 
01097           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 5:i8)
01098           // Emits: (CMOVG16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01099           // Pattern complexity = 17  cost = 1  size = 3
01100           if (N.Val->getValueType(0) == MVT::i16) {
01101             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01102             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01103             if (!Match) {
01104               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01105             }
01106             if (Match) {
01107               Emit_7(Result, N, X86::CMOVG16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01108               return;
01109             }
01110           }
01111 
01112           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 5:i8)
01113           // Emits: (CMOVG32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01114           // Pattern complexity = 17  cost = 1
01115           if (N.Val->getValueType(0) == MVT::i32) {
01116             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01117             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01118             if (!Match) {
01119               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01120             }
01121             if (Match) {
01122               Emit_7(Result, N, X86::CMOVG32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01123               return;
01124             }
01125           }
01126         }
01127         if (CN0 == 15) {
01128 
01129           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 15:i8)
01130           // Emits: (CMOVS16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01131           // Pattern complexity = 17  cost = 1  size = 3
01132           if (N.Val->getValueType(0) == MVT::i16) {
01133             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01134             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01135             if (!Match) {
01136               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01137             }
01138             if (Match) {
01139               Emit_7(Result, N, X86::CMOVS16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01140               return;
01141             }
01142           }
01143 
01144           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 15:i8)
01145           // Emits: (CMOVS32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01146           // Pattern complexity = 17  cost = 1
01147           if (N.Val->getValueType(0) == MVT::i32) {
01148             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01149             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01150             if (!Match) {
01151               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01152             }
01153             if (Match) {
01154               Emit_7(Result, N, X86::CMOVS32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01155               return;
01156             }
01157           }
01158         }
01159         if (CN0 == 12) {
01160 
01161           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 12:i8)
01162           // Emits: (CMOVNS16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01163           // Pattern complexity = 17  cost = 1  size = 3
01164           if (N.Val->getValueType(0) == MVT::i16) {
01165             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01166             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01167             if (!Match) {
01168               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01169             }
01170             if (Match) {
01171               Emit_7(Result, N, X86::CMOVNS16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01172               return;
01173             }
01174           }
01175 
01176           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 12:i8)
01177           // Emits: (CMOVNS32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01178           // Pattern complexity = 17  cost = 1
01179           if (N.Val->getValueType(0) == MVT::i32) {
01180             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01181             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01182             if (!Match) {
01183               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01184             }
01185             if (Match) {
01186               Emit_7(Result, N, X86::CMOVNS32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01187               return;
01188             }
01189           }
01190         }
01191         if (CN0 == 14) {
01192 
01193           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 14:i8)
01194           // Emits: (CMOVP16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01195           // Pattern complexity = 17  cost = 1  size = 3
01196           if (N.Val->getValueType(0) == MVT::i16) {
01197             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01198             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01199             if (!Match) {
01200               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01201             }
01202             if (Match) {
01203               Emit_7(Result, N, X86::CMOVP16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01204               return;
01205             }
01206           }
01207 
01208           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 14:i8)
01209           // Emits: (CMOVP32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01210           // Pattern complexity = 17  cost = 1
01211           if (N.Val->getValueType(0) == MVT::i32) {
01212             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01213             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01214             if (!Match) {
01215               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01216             }
01217             if (Match) {
01218               Emit_7(Result, N, X86::CMOVP32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01219               return;
01220             }
01221           }
01222         }
01223         if (CN0 == 11) {
01224 
01225           // Pattern: (X86cmov:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2), 11:i8)
01226           // Emits: (CMOVNP16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
01227           // Pattern complexity = 17  cost = 1  size = 3
01228           if (N.Val->getValueType(0) == MVT::i16) {
01229             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01230             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01231             if (!Match) {
01232               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01233             }
01234             if (Match) {
01235               Emit_7(Result, N, X86::CMOVNP16rm, MVT::i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01236               return;
01237             }
01238           }
01239 
01240           // Pattern: (X86cmov:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2), 11:i8)
01241           // Emits: (CMOVNP32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
01242           // Pattern complexity = 17  cost = 1
01243           if (N.Val->getValueType(0) == MVT::i32) {
01244             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01245             bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
01246             if (!Match) {
01247               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01248             }
01249             if (Match) {
01250               Emit_7(Result, N, X86::CMOVNP32rm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11, N2);
01251               return;
01252             }
01253           }
01254         }
01255       }
01256     }
01257   }
01258   if ((!Subtarget->hasSSE2())) {
01259     N0 = N.getOperand(0);
01260     N1 = N.getOperand(1);
01261     N2 = N.getOperand(2);
01262     if (isa<ConstantSDNode>(N2)) {
01263       int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
01264 
01265       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 2:i8)
01266       // Emits: (FpCMOVB:f64 RFP:f64:$src1, RFP:f64:$src2)
01267       // Pattern complexity = 5  cost = 1  size = 0
01268       if (CN0 == 2 &&
01269           N.Val->getValueType(0) == MVT::f64) {
01270         Emit_8(Result, N, X86::FpCMOVB, MVT::f64, N0, N1, N2);
01271         return;
01272       }
01273 
01274       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 3:i8)
01275       // Emits: (FpCMOVBE:f64 RFP:f64:$src1, RFP:f64:$src2)
01276       // Pattern complexity = 5  cost = 1  size = 0
01277       if (CN0 == 3 &&
01278           N.Val->getValueType(0) == MVT::f64) {
01279         Emit_8(Result, N, X86::FpCMOVBE, MVT::f64, N0, N1, N2);
01280         return;
01281       }
01282 
01283       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 4:i8)
01284       // Emits: (FpCMOVE:f64 RFP:f64:$src1, RFP:f64:$src2)
01285       // Pattern complexity = 5  cost = 1  size = 0
01286       if (CN0 == 4 &&
01287           N.Val->getValueType(0) == MVT::f64) {
01288         Emit_8(Result, N, X86::FpCMOVE, MVT::f64, N0, N1, N2);
01289         return;
01290       }
01291 
01292       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 14:i8)
01293       // Emits: (FpCMOVP:f64 RFP:f64:$src1, RFP:f64:$src2)
01294       // Pattern complexity = 5  cost = 1  size = 0
01295       if (CN0 == 14 &&
01296           N.Val->getValueType(0) == MVT::f64) {
01297         Emit_8(Result, N, X86::FpCMOVP, MVT::f64, N0, N1, N2);
01298         return;
01299       }
01300 
01301       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 1:i8)
01302       // Emits: (FpCMOVNB:f64 RFP:f64:$src1, RFP:f64:$src2)
01303       // Pattern complexity = 5  cost = 1  size = 0
01304       if (CN0 == 1 &&
01305           N.Val->getValueType(0) == MVT::f64) {
01306         Emit_8(Result, N, X86::FpCMOVNB, MVT::f64, N0, N1, N2);
01307         return;
01308       }
01309 
01310       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 0:i8)
01311       // Emits: (FpCMOVNBE:f64 RFP:f64:$src1, RFP:f64:$src2)
01312       // Pattern complexity = 5  cost = 1  size = 0
01313       if (CN0 == 0 &&
01314           N.Val->getValueType(0) == MVT::f64) {
01315         Emit_8(Result, N, X86::FpCMOVNBE, MVT::f64, N0, N1, N2);
01316         return;
01317       }
01318 
01319       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 9:i8)
01320       // Emits: (FpCMOVNE:f64 RFP:f64:$src1, RFP:f64:$src2)
01321       // Pattern complexity = 5  cost = 1  size = 0
01322       if (CN0 == 9 &&
01323           N.Val->getValueType(0) == MVT::f64) {
01324         Emit_8(Result, N, X86::FpCMOVNE, MVT::f64, N0, N1, N2);
01325         return;
01326       }
01327 
01328       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 11:i8)
01329       // Emits: (FpCMOVNP:f64 RFP:f64:$src1, RFP:f64:$src2)
01330       // Pattern complexity = 5  cost = 1
01331       if (CN0 == 11 &&
01332           N.Val->getValueType(0) == MVT::f64) {
01333         Emit_8(Result, N, X86::FpCMOVNP, MVT::f64, N0, N1, N2);
01334         return;
01335       }
01336     }
01337   }
01338   N0 = N.getOperand(0);
01339   N1 = N.getOperand(1);
01340   N2 = N.getOperand(2);
01341   if (isa<ConstantSDNode>(N2)) {
01342     int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
01343     if (CN0 == 2) {
01344 
01345       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 2:i8)
01346       // Emits: (CMOVB16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01347       // Pattern complexity = 5  cost = 1  size = 3
01348       if (N.Val->getValueType(0) == MVT::i16) {
01349         Emit_8(Result, N, X86::CMOVB16rr, MVT::i16, N0, N1, N2);
01350         return;
01351       }
01352 
01353       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 2:i8)
01354       // Emits: (CMOVB32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01355       // Pattern complexity = 5  cost = 1
01356       if (N.Val->getValueType(0) == MVT::i32) {
01357         Emit_8(Result, N, X86::CMOVB32rr, MVT::i32, N0, N1, N2);
01358         return;
01359       }
01360     }
01361     if (CN0 == 1) {
01362 
01363       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 1:i8)
01364       // Emits: (CMOVAE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01365       // Pattern complexity = 5  cost = 1  size = 3
01366       if (N.Val->getValueType(0) == MVT::i16) {
01367         Emit_8(Result, N, X86::CMOVAE16rr, MVT::i16, N0, N1, N2);
01368         return;
01369       }
01370 
01371       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 1:i8)
01372       // Emits: (CMOVAE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01373       // Pattern complexity = 5  cost = 1
01374       if (N.Val->getValueType(0) == MVT::i32) {
01375         Emit_8(Result, N, X86::CMOVAE32rr, MVT::i32, N0, N1, N2);
01376         return;
01377       }
01378     }
01379     if (CN0 == 4) {
01380 
01381       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 4:i8)
01382       // Emits: (CMOVE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01383       // Pattern complexity = 5  cost = 1  size = 3
01384       if (N.Val->getValueType(0) == MVT::i16) {
01385         Emit_8(Result, N, X86::CMOVE16rr, MVT::i16, N0, N1, N2);
01386         return;
01387       }
01388 
01389       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 4:i8)
01390       // Emits: (CMOVE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01391       // Pattern complexity = 5  cost = 1
01392       if (N.Val->getValueType(0) == MVT::i32) {
01393         Emit_8(Result, N, X86::CMOVE32rr, MVT::i32, N0, N1, N2);
01394         return;
01395       }
01396     }
01397     if (CN0 == 9) {
01398 
01399       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 9:i8)
01400       // Emits: (CMOVNE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01401       // Pattern complexity = 5  cost = 1  size = 3
01402       if (N.Val->getValueType(0) == MVT::i16) {
01403         Emit_8(Result, N, X86::CMOVNE16rr, MVT::i16, N0, N1, N2);
01404         return;
01405       }
01406 
01407       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 9:i8)
01408       // Emits: (CMOVNE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01409       // Pattern complexity = 5  cost = 1
01410       if (N.Val->getValueType(0) == MVT::i32) {
01411         Emit_8(Result, N, X86::CMOVNE32rr, MVT::i32, N0, N1, N2);
01412         return;
01413       }
01414     }
01415     if (CN0 == 3) {
01416 
01417       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 3:i8)
01418       // Emits: (CMOVBE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01419       // Pattern complexity = 5  cost = 1  size = 3
01420       if (N.Val->getValueType(0) == MVT::i16) {
01421         Emit_8(Result, N, X86::CMOVBE16rr, MVT::i16, N0, N1, N2);
01422         return;
01423       }
01424 
01425       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 3:i8)
01426       // Emits: (CMOVBE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01427       // Pattern complexity = 5  cost = 1
01428       if (N.Val->getValueType(0) == MVT::i32) {
01429         Emit_8(Result, N, X86::CMOVBE32rr, MVT::i32, N0, N1, N2);
01430         return;
01431       }
01432     }
01433     if (CN0 == 0) {
01434 
01435       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 0:i8)
01436       // Emits: (CMOVA16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01437       // Pattern complexity = 5  cost = 1  size = 3
01438       if (N.Val->getValueType(0) == MVT::i16) {
01439         Emit_8(Result, N, X86::CMOVA16rr, MVT::i16, N0, N1, N2);
01440         return;
01441       }
01442 
01443       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 0:i8)
01444       // Emits: (CMOVA32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01445       // Pattern complexity = 5  cost = 1
01446       if (N.Val->getValueType(0) == MVT::i32) {
01447         Emit_8(Result, N, X86::CMOVA32rr, MVT::i32, N0, N1, N2);
01448         return;
01449       }
01450     }
01451     if (CN0 == 7) {
01452 
01453       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 7:i8)
01454       // Emits: (CMOVL16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01455       // Pattern complexity = 5  cost = 1  size = 3
01456       if (N.Val->getValueType(0) == MVT::i16) {
01457         Emit_8(Result, N, X86::CMOVL16rr, MVT::i16, N0, N1, N2);
01458         return;
01459       }
01460 
01461       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 7:i8)
01462       // Emits: (CMOVL32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01463       // Pattern complexity = 5  cost = 1
01464       if (N.Val->getValueType(0) == MVT::i32) {
01465         Emit_8(Result, N, X86::CMOVL32rr, MVT::i32, N0, N1, N2);
01466         return;
01467       }
01468     }
01469     if (CN0 == 6) {
01470 
01471       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 6:i8)
01472       // Emits: (CMOVGE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01473       // Pattern complexity = 5  cost = 1  size = 3
01474       if (N.Val->getValueType(0) == MVT::i16) {
01475         Emit_8(Result, N, X86::CMOVGE16rr, MVT::i16, N0, N1, N2);
01476         return;
01477       }
01478 
01479       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 6:i8)
01480       // Emits: (CMOVGE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01481       // Pattern complexity = 5  cost = 1
01482       if (N.Val->getValueType(0) == MVT::i32) {
01483         Emit_8(Result, N, X86::CMOVGE32rr, MVT::i32, N0, N1, N2);
01484         return;
01485       }
01486     }
01487     if (CN0 == 8) {
01488 
01489       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 8:i8)
01490       // Emits: (CMOVLE16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01491       // Pattern complexity = 5  cost = 1  size = 3
01492       if (N.Val->getValueType(0) == MVT::i16) {
01493         Emit_8(Result, N, X86::CMOVLE16rr, MVT::i16, N0, N1, N2);
01494         return;
01495       }
01496 
01497       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 8:i8)
01498       // Emits: (CMOVLE32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01499       // Pattern complexity = 5  cost = 1
01500       if (N.Val->getValueType(0) == MVT::i32) {
01501         Emit_8(Result, N, X86::CMOVLE32rr, MVT::i32, N0, N1, N2);
01502         return;
01503       }
01504     }
01505     if (CN0 == 5) {
01506 
01507       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 5:i8)
01508       // Emits: (CMOVG16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01509       // Pattern complexity = 5  cost = 1  size = 3
01510       if (N.Val->getValueType(0) == MVT::i16) {
01511         Emit_8(Result, N, X86::CMOVG16rr, MVT::i16, N0, N1, N2);
01512         return;
01513       }
01514 
01515       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 5:i8)
01516       // Emits: (CMOVG32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01517       // Pattern complexity = 5  cost = 1
01518       if (N.Val->getValueType(0) == MVT::i32) {
01519         Emit_8(Result, N, X86::CMOVG32rr, MVT::i32, N0, N1, N2);
01520         return;
01521       }
01522     }
01523     if (CN0 == 15) {
01524 
01525       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 15:i8)
01526       // Emits: (CMOVS16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01527       // Pattern complexity = 5  cost = 1  size = 3
01528       if (N.Val->getValueType(0) == MVT::i16) {
01529         Emit_8(Result, N, X86::CMOVS16rr, MVT::i16, N0, N1, N2);
01530         return;
01531       }
01532 
01533       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 15:i8)
01534       // Emits: (CMOVS32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01535       // Pattern complexity = 5  cost = 1
01536       if (N.Val->getValueType(0) == MVT::i32) {
01537         Emit_8(Result, N, X86::CMOVS32rr, MVT::i32, N0, N1, N2);
01538         return;
01539       }
01540     }
01541     if (CN0 == 12) {
01542 
01543       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 12:i8)
01544       // Emits: (CMOVNS16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01545       // Pattern complexity = 5  cost = 1  size = 3
01546       if (N.Val->getValueType(0) == MVT::i16) {
01547         Emit_8(Result, N, X86::CMOVNS16rr, MVT::i16, N0, N1, N2);
01548         return;
01549       }
01550 
01551       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 12:i8)
01552       // Emits: (CMOVNS32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01553       // Pattern complexity = 5  cost = 1
01554       if (N.Val->getValueType(0) == MVT::i32) {
01555         Emit_8(Result, N, X86::CMOVNS32rr, MVT::i32, N0, N1, N2);
01556         return;
01557       }
01558     }
01559     if (CN0 == 14) {
01560 
01561       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 14:i8)
01562       // Emits: (CMOVP16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01563       // Pattern complexity = 5  cost = 1  size = 3
01564       if (N.Val->getValueType(0) == MVT::i16) {
01565         Emit_8(Result, N, X86::CMOVP16rr, MVT::i16, N0, N1, N2);
01566         return;
01567       }
01568 
01569       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 14:i8)
01570       // Emits: (CMOVP32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01571       // Pattern complexity = 5  cost = 1
01572       if (N.Val->getValueType(0) == MVT::i32) {
01573         Emit_8(Result, N, X86::CMOVP32rr, MVT::i32, N0, N1, N2);
01574         return;
01575       }
01576     }
01577     if (CN0 == 11) {
01578 
01579       // Pattern: (X86cmov:i16 GR16:i16:$src1, GR16:i16:$src2, 11:i8)
01580       // Emits: (CMOVNP16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
01581       // Pattern complexity = 5  cost = 1  size = 3
01582       if (N.Val->getValueType(0) == MVT::i16) {
01583         Emit_8(Result, N, X86::CMOVNP16rr, MVT::i16, N0, N1, N2);
01584         return;
01585       }
01586 
01587       // Pattern: (X86cmov:i32 GR32:i32:$src1, GR32:i32:$src2, 11:i8)
01588       // Emits: (CMOVNP32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
01589       // Pattern complexity = 5  cost = 1
01590       if (N.Val->getValueType(0) == MVT::i32) {
01591         Emit_8(Result, N, X86::CMOVNP32rr, MVT::i32, N0, N1, N2);
01592         return;
01593       }
01594     }
01595   }
01596   if (N2.getOpcode() == ISD::Constant) {
01597 
01598     // Pattern: (X86cmov:f32 FR32:f32:$t, FR32:f32:$f, (imm:i8):$cond)
01599     // Emits: (CMOV_FR32:f32 FR32:f32:$t, FR32:f32:$f, (imm:i8):$cond)
01600     // Pattern complexity = 4  cost = 11  size = 3
01601     if (N.Val->getValueType(0) == MVT::f32) {
01602       Emit_9(Result, N, X86::CMOV_FR32, MVT::f32, N0, N1, N2);
01603       return;
01604     }
01605 
01606     // Pattern: (X86cmov:f64 FR64:f64:$t, FR64:f64:$f, (imm:i8):$cond)
01607     // Emits: (CMOV_FR64:f64 FR64:f64:$t, FR64:f64:$f, (imm:i8):$cond)
01608     // Pattern complexity = 4  cost = 11  size = 3
01609     if (N.Val->getValueType(0) == MVT::f64) {
01610       Emit_9(Result, N, X86::CMOV_FR64, MVT::f64, N0, N1, N2);
01611       return;
01612     }
01613 
01614     // Pattern: (X86cmov:v4f32 VR128:v4f32:$t, VR128:v4f32:$f, (imm:i8):$cond)
01615     // Emits: (CMOV_V4F32:v4f32 VR128:v4f32:$t, VR128:v4f32:$f, (imm:i8):$cond)
01616     // Pattern complexity = 4  cost = 11  size = 3
01617     if (N.Val->getValueType(0) == MVT::v4f32) {
01618       Emit_9(Result, N, X86::CMOV_V4F32, MVT::v4f32, N0, N1, N2);
01619       return;
01620     }
01621 
01622     // Pattern: (X86cmov:v2f64 VR128:v2f64:$t, VR128:v2f64:$f, (imm:i8):$cond)
01623     // Emits: (CMOV_V2F64:v2f64 VR128:v2f64:$t, VR128:v2f64:$f, (imm:i8):$cond)
01624     // Pattern complexity = 4  cost = 11  size = 3
01625     if (N.Val->getValueType(0) == MVT::v2f64) {
01626       Emit_9(Result, N, X86::CMOV_V2F64, MVT::v2f64, N0, N1, N2);
01627       return;
01628     }
01629 
01630     // Pattern: (X86cmov:v2i64 VR128:v2i64:$t, VR128:v2i64:$f, (imm:i8):$cond)
01631     // Emits: (CMOV_V2I64:v2i64 VR128:v2i64:$t, VR128:v2i64:$f, (imm:i8):$cond)
01632     // Pattern complexity = 4  cost = 11
01633     if (N.Val->getValueType(0) == MVT::v2i64) {
01634       Emit_9(Result, N, X86::CMOV_V2I64, MVT::v2i64, N0, N1, N2);
01635       return;
01636     }
01637   }
01638   std::cerr << "Cannot yet select: ";
01639   N.Val->dump(CurDAG);
01640   std::cerr << '\n';
01641   abort();
01642 }
01643 
01644 void Emit_10(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
01645   SDOperand InFlag(0, 0);
01646   SDOperand Tmp0(0, 0);
01647   SDOperand Tmp1(0, 0);
01648   SDOperand Tmp2(0, 0);
01649   SDOperand Tmp3(0, 0);
01650   SDOperand Tmp4(0, 0);
01651   SDNode *ResNode = NULL;
01652   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
01653   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
01654   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
01655   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
01656   Select(Tmp0, CPTmp0);
01657   Select(Tmp1, CPTmp1);
01658   Select(Tmp2, CPTmp2);
01659   Select(Tmp3, CPTmp3);
01660   Tmp4 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
01661   Select(Chain0, Chain0);
01662   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
01663   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
01664   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
01665   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
01666   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
01667   Chain0 = SDOperand(ResNode, 0);
01668   InFlag = SDOperand(ResNode, 1);
01669   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
01670   AddHandleReplacement(N0.Val, 1, ResNode, 0);
01671   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01672   Result = SDOperand(ResNode, N.ResNo+1);
01673   return;
01674 }
01675 void Emit_11(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
01676   SDOperand InFlag(0, 0);
01677   SDOperand Tmp0(0, 0);
01678   SDOperand Tmp1(0, 0);
01679   SDOperand Tmp2(0, 0);
01680   SDOperand Tmp3(0, 0);
01681   SDOperand Tmp4(0, 0);
01682   SDNode *ResNode = NULL;
01683   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
01684   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
01685   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
01686   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
01687   Select(Tmp0, CPTmp0);
01688   Select(Tmp1, CPTmp1);
01689   Select(Tmp2, CPTmp2);
01690   Select(Tmp3, CPTmp3);
01691   Tmp4 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
01692   Select(Chain0, Chain0);
01693   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
01694   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
01695   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
01696   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
01697   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
01698   Chain0 = SDOperand(ResNode, 0);
01699   InFlag = SDOperand(ResNode, 1);
01700   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
01701   AddHandleReplacement(N0.Val, 1, ResNode, 0);
01702   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01703   Result = SDOperand(ResNode, N.ResNo+1);
01704   return;
01705 }
01706 void Emit_12(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
01707   SDOperand InFlag(0, 0);
01708   SDOperand Tmp0(0, 0);
01709   SDOperand Tmp1(0, 0);
01710   SDOperand Tmp2(0, 0);
01711   SDOperand Tmp3(0, 0);
01712   SDOperand Tmp4(0, 0);
01713   SDNode *ResNode = NULL;
01714   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
01715   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
01716   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
01717   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
01718   Select(Tmp0, CPTmp0);
01719   Select(Tmp1, CPTmp1);
01720   Select(Tmp2, CPTmp2);
01721   Select(Tmp3, CPTmp3);
01722   Tmp4 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N1)->getValue()), MVT::i8);
01723   Select(Chain0, Chain0);
01724   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
01725   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
01726   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
01727   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
01728   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
01729   Chain0 = SDOperand(ResNode, 0);
01730   InFlag = SDOperand(ResNode, 1);
01731   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
01732   AddHandleReplacement(N0.Val, 1, ResNode, 0);
01733   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01734   Result = SDOperand(ResNode, N.ResNo+1);
01735   return;
01736 }
01737 void Emit_13(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
01738   SDOperand InFlag(0, 0);
01739   SDOperand Tmp0(0, 0);
01740   SDOperand Tmp1(0, 0);
01741   SDOperand Tmp2(0, 0);
01742   SDOperand Tmp3(0, 0);
01743   SDOperand Tmp4(0, 0);
01744   SDNode *ResNode = NULL;
01745   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
01746   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
01747   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
01748   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
01749   Select(Tmp0, CPTmp0);
01750   Select(Tmp1, CPTmp1);
01751   Select(Tmp2, CPTmp2);
01752   Select(Tmp3, CPTmp3);
01753   Select(Tmp4, N1);
01754   Select(Chain0, Chain0);
01755   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
01756   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
01757   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
01758   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
01759   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
01760   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
01761   Chain0 = SDOperand(ResNode, 0);
01762   InFlag = SDOperand(ResNode, 1);
01763   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
01764   AddHandleReplacement(N0.Val, 1, ResNode, 0);
01765   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01766   Result = SDOperand(ResNode, N.ResNo+1);
01767   return;
01768 }
01769 void Emit_14(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
01770   SDOperand InFlag(0, 0);
01771   SDOperand Tmp0(0, 0);
01772   SDOperand Tmp1(0, 0);
01773   SDOperand Tmp2(0, 0);
01774   SDOperand Tmp3(0, 0);
01775   SDOperand Tmp4(0, 0);
01776   SDNode *ResNode = NULL;
01777   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
01778   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
01779   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
01780   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
01781   Select(Tmp0, CPTmp0);
01782   Select(Tmp1, CPTmp1);
01783   Select(Tmp2, CPTmp2);
01784   Select(Tmp3, CPTmp3);
01785   Select(Tmp4, N0);
01786   Select(Chain1, Chain1);
01787   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01788   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
01789   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
01790   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
01791   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
01792   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
01793   Chain1 = SDOperand(ResNode, 0);
01794   InFlag = SDOperand(ResNode, 1);
01795   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
01796   AddHandleReplacement(N1.Val, 1, ResNode, 0);
01797   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01798   Result = SDOperand(ResNode, N.ResNo+1);
01799   return;
01800 }
01801 void Emit_15(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &N0, SDOperand &N1) NOINLINE {
01802   SDOperand InFlag(0, 0);
01803   SDOperand Tmp0(0, 0);
01804   SDOperand Tmp1(0, 0);
01805   SDNode *ResNode = NULL;
01806   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01807   Select(Tmp0, N0);
01808   Tmp1 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
01809   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01810   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, Tmp1);
01811   InFlag = SDOperand(ResNode, 0);
01812   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01813   Result = SDOperand(ResNode, N.ResNo);
01814   return;
01815 }
01816 void Emit_16(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &N0, SDOperand &N1) NOINLINE {
01817   SDOperand InFlag(0, 0);
01818   SDOperand Tmp0(0, 0);
01819   SDOperand Tmp1(0, 0);
01820   SDNode *ResNode = NULL;
01821   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01822   Select(Tmp0, N0);
01823   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
01824   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01825   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, Tmp1);
01826   InFlag = SDOperand(ResNode, 0);
01827   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01828   Result = SDOperand(ResNode, N.ResNo);
01829   return;
01830 }
01831 void Emit_17(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &N0, SDOperand &N1) NOINLINE {
01832   SDOperand InFlag(0, 0);
01833   SDOperand Tmp0(0, 0);
01834   SDOperand Tmp1(0, 0);
01835   SDNode *ResNode = NULL;
01836   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01837   Select(Tmp0, N0);
01838   Tmp1 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N1)->getValue()), MVT::i8);
01839   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01840   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, Tmp1);
01841   InFlag = SDOperand(ResNode, 0);
01842   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01843   Result = SDOperand(ResNode, N.ResNo);
01844   return;
01845 }
01846 void Emit_18(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &N0, SDOperand &N1) NOINLINE {
01847   SDOperand InFlag(0, 0);
01848   SDOperand Tmp0(0, 0);
01849   SDOperand Tmp1(0, 0);
01850   SDNode *ResNode = NULL;
01851   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
01852   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
01853   Select(Tmp0, N0);
01854   Select(Tmp1, N1);
01855   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
01856   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
01857   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, Tmp1);
01858   InFlag = SDOperand(ResNode, 0);
01859   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
01860   Result = SDOperand(ResNode, N.ResNo);
01861   return;
01862 }
01863 void Select_X86cmp(SDOperand &Result, SDOperand N) {
01864   SDOperand CPTmp0(0, 0);
01865   SDOperand CPTmp1(0, 0);
01866   SDOperand CPTmp2(0, 0);
01867   SDOperand CPTmp3(0, 0);
01868   SDOperand Chain0(0, 0);
01869   SDOperand Chain1(0, 0);
01870   SDOperand InFlag(0, 0);
01871   SDOperand N0(0, 0);
01872   SDOperand N01(0, 0);
01873   SDOperand N1(0, 0);
01874   SDOperand N11(0, 0);
01875   SDOperand Tmp0(0, 0);
01876   SDOperand Tmp1(0, 0);
01877   SDOperand Tmp2(0, 0);
01878   SDOperand Tmp3(0, 0);
01879   SDOperand Tmp4(0, 0);
01880   SDNode *ResNode = NULL;
01881   {
01882     N0 = N.getOperand(0);
01883     if (N0.getOpcode() == ISD::LOAD &&
01884         InFlightSet.count(N0.Val) == 0 &&
01885         N0.hasOneUse() &&
01886         !CodeGenMap.count(N0.getValue(0)) &&
01887         N.Val->isOnlyUse(N0.Val) &&
01888         !isNonImmUse(N.Val, N0.Val)) {
01889       Chain0 = N0.getOperand(0);
01890       N01 = N0.getOperand(1);
01891       N1 = N.getOperand(1);
01892       if (N1.getOpcode() == ISD::Constant) {
01893 
01894         // Pattern: (X86cmp:void (load:i16 addr:iPTR:$src1), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
01895         // Emits: (CMP16mi8:void addr:iPTR:$src1, (imm:i16):$src2)
01896         // Pattern complexity = 17  cost = 1  size = 3
01897         if (Predicate_i16immSExt8(N1.Val) &&
01898             N0.Val->getValueType(0) == MVT::i16) {
01899           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01900           if (Match) {
01901             Emit_10(Result, N, X86::CMP16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01902             return;
01903           }
01904         }
01905 
01906         // Pattern: (X86cmp:void (load:i32 addr:iPTR:$src1), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
01907         // Emits: (CMP32mi8:void addr:iPTR:$src1, (imm:i32):$src2)
01908         // Pattern complexity = 17  cost = 1  size = 3
01909         if (Predicate_i32immSExt8(N1.Val) &&
01910             N0.Val->getValueType(0) == MVT::i32) {
01911           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01912           if (Match) {
01913             Emit_11(Result, N, X86::CMP32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01914             return;
01915           }
01916         }
01917 
01918         // Pattern: (X86cmp:void (load:i8 addr:iPTR:$src1), (imm:i8):$src2)
01919         // Emits: (CMP8mi:void addr:iPTR:$src1, (imm:i8):$src2)
01920         // Pattern complexity = 16  cost = 1  size = 3
01921         if (N0.Val->getValueType(0) == MVT::i8) {
01922           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01923           if (Match) {
01924             Emit_12(Result, N, X86::CMP8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01925             return;
01926           }
01927         }
01928 
01929         // Pattern: (X86cmp:void (load:i16 addr:iPTR:$src1), (imm:i16):$src2)
01930         // Emits: (CMP16mi:void addr:iPTR:$src1, (imm:i16):$src2)
01931         // Pattern complexity = 16  cost = 1  size = 3
01932         if (N0.Val->getValueType(0) == MVT::i16) {
01933           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01934           if (Match) {
01935             Emit_10(Result, N, X86::CMP16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01936             return;
01937           }
01938         }
01939 
01940         // Pattern: (X86cmp:void (load:i32 addr:iPTR:$src1), (imm:i32):$src2)
01941         // Emits: (CMP32mi:void addr:iPTR:$src1, (imm:i32):$src2)
01942         // Pattern complexity = 16  cost = 1
01943         if (N0.Val->getValueType(0) == MVT::i32) {
01944           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01945           if (Match) {
01946             Emit_11(Result, N, X86::CMP32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01947             return;
01948           }
01949         }
01950       }
01951 
01952       // Pattern: (X86cmp:void (load:i8 addr:iPTR:$src1), GR8:i8:$src2)
01953       // Emits: (CMP8mr:void addr:iPTR:$src1, GR8:i8:$src2)
01954       // Pattern complexity = 14  cost = 1  size = 3
01955       if (N0.Val->getValueType(0) == MVT::i8) {
01956         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
01957         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01958         if (!Match) {
01959           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
01960         }
01961         if (Match) {
01962           Emit_13(Result, N, X86::CMP8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01963           return;
01964         }
01965       }
01966 
01967       // Pattern: (X86cmp:void (load:i16 addr:iPTR:$src1), GR16:i16:$src2)
01968       // Emits: (CMP16mr:void addr:iPTR:$src1, GR16:i16:$src2)
01969       // Pattern complexity = 14  cost = 1  size = 3
01970       if (N0.Val->getValueType(0) == MVT::i16) {
01971         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
01972         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01973         if (!Match) {
01974           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
01975         }
01976         if (Match) {
01977           Emit_13(Result, N, X86::CMP16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01978           return;
01979         }
01980       }
01981 
01982       // Pattern: (X86cmp:void (load:i32 addr:iPTR:$src1), GR32:i32:$src2)
01983       // Emits: (CMP32mr:void addr:iPTR:$src1, GR32:i32:$src2)
01984       // Pattern complexity = 14  cost = 1
01985       if (N0.Val->getValueType(0) == MVT::i32) {
01986         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
01987         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
01988         if (!Match) {
01989           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
01990         }
01991         if (Match) {
01992           Emit_13(Result, N, X86::CMP32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
01993           return;
01994         }
01995       }
01996     }
01997     N1 = N.getOperand(1);
01998     if (N1.getOpcode() == ISD::LOAD &&
01999         InFlightSet.count(N1.Val) == 0 &&
02000         N1.hasOneUse() &&
02001         !CodeGenMap.count(N1.getValue(0)) &&
02002         N.Val->isOnlyUse(N1.Val) &&
02003         !isNonImmUse(N.Val, N1.Val)) {
02004       Chain1 = N1.getOperand(0);
02005       N11 = N1.getOperand(1);
02006 
02007       // Pattern: (X86cmp:void GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
02008       // Emits: (CMP8rm:void GR8:i8:$src1, addr:iPTR:$src2)
02009       // Pattern complexity = 14  cost = 1  size = 3
02010       if (N0.Val->getValueType(0) == MVT::i8) {
02011         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02012         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02013         if (!Match) {
02014           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02015         }
02016         if (Match) {
02017           Emit_14(Result, N, X86::CMP8rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02018           return;
02019         }
02020       }
02021 
02022       // Pattern: (X86cmp:void GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
02023       // Emits: (CMP16rm:void GR16:i16:$src1, addr:iPTR:$src2)
02024       // Pattern complexity = 14  cost = 1  size = 3
02025       if (N0.Val->getValueType(0) == MVT::i16) {
02026         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02027         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02028         if (!Match) {
02029           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02030         }
02031         if (Match) {
02032           Emit_14(Result, N, X86::CMP16rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02033           return;
02034         }
02035       }
02036 
02037       // Pattern: (X86cmp:void GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
02038       // Emits: (CMP32rm:void GR32:i32:$src1, addr:iPTR:$src2)
02039       // Pattern complexity = 14  cost = 1
02040       if (N0.Val->getValueType(0) == MVT::i32) {
02041         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02042         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02043         if (!Match) {
02044           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02045         }
02046         if (Match) {
02047           Emit_14(Result, N, X86::CMP32rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02048           return;
02049         }
02050       }
02051     }
02052   }
02053 
02054   // Pattern: (X86cmp:void FR32:f32:$src1, (load:f32 addr:iPTR:$src2))
02055   // Emits: (UCOMISSrm:void FR32:f32:$src1, addr:iPTR:$src2)
02056   // Pattern complexity = 14  cost = 1  size = 3
02057   if ((Subtarget->hasSSE1())) {
02058     N0 = N.getOperand(0);
02059     N1 = N.getOperand(1);
02060     if (N1.getOpcode() == ISD::LOAD &&
02061         InFlightSet.count(N1.Val) == 0 &&
02062         N1.hasOneUse() &&
02063         !CodeGenMap.count(N1.getValue(0)) &&
02064         N.Val->isOnlyUse(N1.Val) &&
02065         !isNonImmUse(N.Val, N1.Val)) {
02066       Chain1 = N1.getOperand(0);
02067       N11 = N1.getOperand(1);
02068       if (N0.Val->getValueType(0) == MVT::f32) {
02069         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02070         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02071         if (!Match) {
02072           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02073         }
02074         if (Match) {
02075           Emit_14(Result, N, X86::UCOMISSrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02076           return;
02077         }
02078       }
02079     }
02080   }
02081 
02082   // Pattern: (X86cmp:void FR64:f64:$src1, (load:f64 addr:iPTR:$src2))
02083   // Emits: (UCOMISDrm:void FR64:f64:$src1, addr:iPTR:$src2)
02084   // Pattern complexity = 14  cost = 1  size = 3
02085   if ((Subtarget->hasSSE2())) {
02086     N0 = N.getOperand(0);
02087     N1 = N.getOperand(1);
02088     if (N1.getOpcode() == ISD::LOAD &&
02089         InFlightSet.count(N1.Val) == 0 &&
02090         N1.hasOneUse() &&
02091         !CodeGenMap.count(N1.getValue(0)) &&
02092         N.Val->isOnlyUse(N1.Val) &&
02093         !isNonImmUse(N.Val, N1.Val)) {
02094       Chain1 = N1.getOperand(0);
02095       N11 = N1.getOperand(1);
02096       if (N0.Val->getValueType(0) == MVT::f64) {
02097         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02098         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02099         if (!Match) {
02100           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02101         }
02102         if (Match) {
02103           Emit_14(Result, N, X86::UCOMISDrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02104           return;
02105         }
02106       }
02107     }
02108   }
02109   {
02110     N0 = N.getOperand(0);
02111     N1 = N.getOperand(1);
02112     if (N1.getOpcode() == ISD::Constant) {
02113 
02114       // Pattern: (X86cmp:void GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
02115       // Emits: (CMP16ri8:void GR16:i16:$src1, (imm:i16):$src2)
02116       // Pattern complexity = 5  cost = 1  size = 3
02117       if (Predicate_i16immSExt8(N1.Val) &&
02118           N0.Val->getValueType(0) == MVT::i16) {
02119         Emit_15(Result, N, X86::CMP16ri8, N0, N1);
02120         return;
02121       }
02122 
02123       // Pattern: (X86cmp:void GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
02124       // Emits: (CMP32ri8:void GR32:i32:$src1, (imm:i32):$src2)
02125       // Pattern complexity = 5  cost = 1  size = 3
02126       if (Predicate_i32immSExt8(N1.Val) &&
02127           N0.Val->getValueType(0) == MVT::i32) {
02128         Emit_16(Result, N, X86::CMP32ri8, N0, N1);
02129         return;
02130       }
02131 
02132       // Pattern: (X86cmp:void GR8:i8:$src1, (imm:i8):$src2)
02133       // Emits: (CMP8ri:void GR8:i8:$src1, (imm:i8):$src2)
02134       // Pattern complexity = 4  cost = 1  size = 3
02135       if (N0.Val->getValueType(0) == MVT::i8) {
02136         Emit_17(Result, N, X86::CMP8ri, N0, N1);
02137         return;
02138       }
02139 
02140       // Pattern: (X86cmp:void GR16:i16:$src1, (imm:i16):$src2)
02141       // Emits: (CMP16ri:void GR16:i16:$src1, (imm:i16):$src2)
02142       // Pattern complexity = 4  cost = 1  size = 3
02143       if (N0.Val->getValueType(0) == MVT::i16) {
02144         Emit_15(Result, N, X86::CMP16ri, N0, N1);
02145         return;
02146       }
02147 
02148       // Pattern: (X86cmp:void GR32:i32:$src1, (imm:i32):$src2)
02149       // Emits: (CMP32ri:void GR32:i32:$src1, (imm:i32):$src2)
02150       // Pattern complexity = 4  cost = 1
02151       if (N0.Val->getValueType(0) == MVT::i32) {
02152         Emit_16(Result, N, X86::CMP32ri, N0, N1);
02153         return;
02154       }
02155     }
02156   }
02157 
02158   // Pattern: (X86cmp:void RFP:f64:$lhs, RFP:f64:$rhs)
02159   // Emits: (FpUCOMIr:void RFP:f64:$lhs, RFP:f64:$rhs)
02160   // Pattern complexity = 2  cost = 1  size = 0
02161   if ((!Subtarget->hasSSE2())) {
02162     N0 = N.getOperand(0);
02163     N1 = N.getOperand(1);
02164     if (N0.Val->getValueType(0) == MVT::f64) {
02165       Emit_18(Result, N, X86::FpUCOMIr, N0, N1);
02166       return;
02167     }
02168   }
02169   {
02170     N0 = N.getOperand(0);
02171     N1 = N.getOperand(1);
02172 
02173     // Pattern: (X86cmp:void GR8:i8:$src1, GR8:i8:$src2)
02174     // Emits: (CMP8rr:void GR8:i8:$src1, GR8:i8:$src2)
02175     // Pattern complexity = 2  cost = 1  size = 3
02176     if (N0.Val->getValueType(0) == MVT::i8) {
02177       Emit_18(Result, N, X86::CMP8rr, N0, N1);
02178       return;
02179     }
02180 
02181     // Pattern: (X86cmp:void GR16:i16:$src1, GR16:i16:$src2)
02182     // Emits: (CMP16rr:void GR16:i16:$src1, GR16:i16:$src2)
02183     // Pattern complexity = 2  cost = 1  size = 3
02184     if (N0.Val->getValueType(0) == MVT::i16) {
02185       Emit_18(Result, N, X86::CMP16rr, N0, N1);
02186       return;
02187     }
02188 
02189     // Pattern: (X86cmp:void GR32:i32:$src1, GR32:i32:$src2)
02190     // Emits: (CMP32rr:void GR32:i32:$src1, GR32:i32:$src2)
02191     // Pattern complexity = 2  cost = 1
02192     if (N0.Val->getValueType(0) == MVT::i32) {
02193       Emit_18(Result, N, X86::CMP32rr, N0, N1);
02194       return;
02195     }
02196   }
02197 
02198   // Pattern: (X86cmp:void FR32:f32:$src1, FR32:f32:$src2)
02199   // Emits: (UCOMISSrr:void FR32:f32:$src1, FR32:f32:$src2)
02200   // Pattern complexity = 2  cost = 1  size = 3
02201   if ((Subtarget->hasSSE1())) {
02202     N0 = N.getOperand(0);
02203     N1 = N.getOperand(1);
02204     if (N0.Val->getValueType(0) == MVT::f32) {
02205       Emit_18(Result, N, X86::UCOMISSrr, N0, N1);
02206       return;
02207     }
02208   }
02209 
02210   // Pattern: (X86cmp:void FR64:f64:$src1, FR64:f64:$src2)
02211   // Emits: (UCOMISDrr:void FR64:f64:$src1, FR64:f64:$src2)
02212   // Pattern complexity = 2  cost = 1
02213   if ((Subtarget->hasSSE2())) {
02214     N0 = N.getOperand(0);
02215     N1 = N.getOperand(1);
02216     if (N0.Val->getValueType(0) == MVT::f64) {
02217       Emit_18(Result, N, X86::UCOMISDrr, N0, N1);
02218       return;
02219     }
02220   }
02221   std::cerr << "Cannot yet select: ";
02222   N.Val->dump(CurDAG);
02223   std::cerr << '\n';
02224   abort();
02225 }
02226 
02227 void Select_X86comi(SDOperand &Result, SDOperand N) {
02228   SDOperand CPTmp0(0, 0);
02229   SDOperand CPTmp1(0, 0);
02230   SDOperand CPTmp2(0, 0);
02231   SDOperand CPTmp3(0, 0);
02232   SDOperand Chain1(0, 0);
02233   SDOperand InFlag(0, 0);
02234   SDOperand N0(0, 0);
02235   SDOperand N1(0, 0);
02236   SDOperand N11(0, 0);
02237   SDOperand Tmp0(0, 0);
02238   SDOperand Tmp1(0, 0);
02239   SDOperand Tmp2(0, 0);
02240   SDOperand Tmp3(0, 0);
02241   SDOperand Tmp4(0, 0);
02242   SDNode *ResNode = NULL;
02243 
02244   // Pattern: (X86comi:void VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
02245   // Emits: (Int_COMISSrm:void VR128:v4f32:$src1, addr:iPTR:$src2)
02246   // Pattern complexity = 14  cost = 1  size = 3
02247   if ((Subtarget->hasSSE1())) {
02248     N0 = N.getOperand(0);
02249     N1 = N.getOperand(1);
02250     if (N1.getOpcode() == ISD::LOAD &&
02251         InFlightSet.count(N1.Val) == 0 &&
02252         N1.hasOneUse() &&
02253         !CodeGenMap.count(N1.getValue(0)) &&
02254         N.Val->isOnlyUse(N1.Val) &&
02255         !isNonImmUse(N.Val, N1.Val)) {
02256       Chain1 = N1.getOperand(0);
02257       N11 = N1.getOperand(1);
02258       if (N0.Val->getValueType(0) == MVT::v4f32) {
02259         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02260         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02261         if (!Match) {
02262           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02263         }
02264         if (Match) {
02265           Emit_14(Result, N, X86::Int_COMISSrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02266           return;
02267         }
02268       }
02269     }
02270   }
02271 
02272   // Pattern: (X86comi:void VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
02273   // Emits: (Int_COMISDrm:void VR128:v2f64:$src1, addr:iPTR:$src2)
02274   // Pattern complexity = 14  cost = 1  size = 3
02275   if ((Subtarget->hasSSE2())) {
02276     N0 = N.getOperand(0);
02277     N1 = N.getOperand(1);
02278     if (N1.getOpcode() == ISD::LOAD &&
02279         InFlightSet.count(N1.Val) == 0 &&
02280         N1.hasOneUse() &&
02281         !CodeGenMap.count(N1.getValue(0)) &&
02282         N.Val->isOnlyUse(N1.Val) &&
02283         !isNonImmUse(N.Val, N1.Val)) {
02284       Chain1 = N1.getOperand(0);
02285       N11 = N1.getOperand(1);
02286       if (N0.Val->getValueType(0) == MVT::v2f64) {
02287         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02288         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02289         if (!Match) {
02290           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02291         }
02292         if (Match) {
02293           Emit_14(Result, N, X86::Int_COMISDrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02294           return;
02295         }
02296       }
02297     }
02298   }
02299 
02300   // Pattern: (X86comi:void VR128:v4f32:$src1, VR128:v4f32:$src2)
02301   // Emits: (Int_COMISSrr:void VR128:v4f32:$src1, VR128:v4f32:$src2)
02302   // Pattern complexity = 2  cost = 1  size = 3
02303   if ((Subtarget->hasSSE1())) {
02304     N0 = N.getOperand(0);
02305     N1 = N.getOperand(1);
02306     if (N0.Val->getValueType(0) == MVT::v4f32) {
02307       Emit_18(Result, N, X86::Int_COMISSrr, N0, N1);
02308       return;
02309     }
02310   }
02311 
02312   // Pattern: (X86comi:void VR128:v2f64:$src1, VR128:v2f64:$src2)
02313   // Emits: (Int_COMISDrr:void VR128:v2f64:$src1, VR128:v2f64:$src2)
02314   // Pattern complexity = 2  cost = 1
02315   if ((Subtarget->hasSSE2())) {
02316     N0 = N.getOperand(0);
02317     N1 = N.getOperand(1);
02318     if (N0.Val->getValueType(0) == MVT::v2f64) {
02319       Emit_18(Result, N, X86::Int_COMISDrr, N0, N1);
02320       return;
02321     }
02322   }
02323   std::cerr << "Cannot yet select: ";
02324   N.Val->dump(CurDAG);
02325   std::cerr << '\n';
02326   abort();
02327 }
02328 
02329 void Emit_19(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
02330   SDOperand Tmp0(0, 0);
02331   SDOperand Tmp1(0, 0);
02332   SDOperand Tmp2(0, 0);
02333   SDOperand Tmp3(0, 0);
02334   SDOperand Tmp4(0, 0);
02335   SDNode *ResNode = NULL;
02336   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
02337   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
02338   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
02339   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
02340   Select(Tmp0, CPTmp0);
02341   Select(Tmp1, CPTmp1);
02342   Select(Tmp2, CPTmp2);
02343   Select(Tmp3, CPTmp3);
02344   Select(Tmp4, N0);
02345   Select(Chain1, Chain1);
02346   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02347   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
02348   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
02349   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
02350   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
02351   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02352   Chain1 = SDOperand(ResNode, 1);
02353   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02354   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
02355   AddHandleReplacement(N1.Val, 1, ResNode, 1);
02356   Result = SDOperand(ResNode, N.ResNo);
02357   return;
02358 }
02359 void Emit_20(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
02360   SDOperand Tmp0(0, 0);
02361   SDOperand Tmp1(0, 0);
02362   SDOperand Tmp2(0, 0);
02363   SDOperand Tmp3(0, 0);
02364   SDOperand Tmp4(0, 0);
02365   SDNode *ResNode = NULL;
02366   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
02367   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
02368   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
02369   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
02370   Select(Tmp0, CPTmp0);
02371   Select(Tmp1, CPTmp1);
02372   Select(Tmp2, CPTmp2);
02373   Select(Tmp3, CPTmp3);
02374   Select(Tmp4, N1);
02375   Select(Chain0, Chain0);
02376   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02377   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
02378   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
02379   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
02380   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
02381   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
02382   Chain0 = SDOperand(ResNode, 1);
02383   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02384   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
02385   AddHandleReplacement(N0.Val, 1, ResNode, 1);
02386   Result = SDOperand(ResNode, N.ResNo);
02387   return;
02388 }
02389 void Emit_21(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
02390   SDOperand Tmp0(0, 0);
02391   SDOperand Tmp1(0, 0);
02392   SDNode *ResNode = NULL;
02393   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02394   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02395   Select(Tmp0, N0);
02396   Select(Tmp1, N1);
02397   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02398   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02399   if (N.Val->hasOneUse()) {
02400     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
02401   } else {
02402     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
02403     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
02404     Result = SDOperand(ResNode, 0);
02405   }
02406   return;
02407 }
02408 void Select_X86fand(SDOperand &Result, SDOperand N) {
02409   SDOperand CPTmp0(0, 0);
02410   SDOperand CPTmp1(0, 0);
02411   SDOperand CPTmp2(0, 0);
02412   SDOperand CPTmp3(0, 0);
02413   SDOperand Chain0(0, 0);
02414   SDOperand Chain1(0, 0);
02415   SDOperand N0(0, 0);
02416   SDOperand N01(0, 0);
02417   SDOperand N1(0, 0);
02418   SDOperand N11(0, 0);
02419   SDOperand Tmp0(0, 0);
02420   SDOperand Tmp1(0, 0);
02421   SDOperand Tmp2(0, 0);
02422   SDOperand Tmp3(0, 0);
02423   SDOperand Tmp4(0, 0);
02424   SDNode *ResNode = NULL;
02425 
02426   // Pattern: (X86fand:f32 FR32:f32:$src1, (X86loadp:f32 addr:iPTR:$src2))
02427   // Emits: (FsANDPSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
02428   // Pattern complexity = 14  cost = 1  size = 3
02429   if ((Subtarget->hasSSE1())) {
02430     N0 = N.getOperand(0);
02431     N1 = N.getOperand(1);
02432     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
02433         InFlightSet.count(N1.Val) == 0 &&
02434         N1.hasOneUse() &&
02435         !CodeGenMap.count(N1.getValue(0)) &&
02436         !isNonImmUse(N.Val, N1.Val)) {
02437       Chain1 = N1.getOperand(0);
02438       N11 = N1.getOperand(1);
02439       if (N.Val->getValueType(0) == MVT::f32) {
02440         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02441         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02442         if (!Match) {
02443           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02444         }
02445         if (Match) {
02446           Emit_19(Result, N, X86::FsANDPSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02447           return;
02448         }
02449       }
02450     }
02451   }
02452 
02453   // Pattern: (X86fand:f64 FR64:f64:$src1, (X86loadp:f64 addr:iPTR:$src2))
02454   // Emits: (FsANDPDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
02455   // Pattern complexity = 14  cost = 1  size = 3
02456   if ((Subtarget->hasSSE2())) {
02457     N0 = N.getOperand(0);
02458     N1 = N.getOperand(1);
02459     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
02460         InFlightSet.count(N1.Val) == 0 &&
02461         N1.hasOneUse() &&
02462         !CodeGenMap.count(N1.getValue(0)) &&
02463         !isNonImmUse(N.Val, N1.Val)) {
02464       Chain1 = N1.getOperand(0);
02465       N11 = N1.getOperand(1);
02466       if (N.Val->getValueType(0) == MVT::f64) {
02467         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
02468         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02469         if (!Match) {
02470           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
02471         }
02472         if (Match) {
02473           Emit_19(Result, N, X86::FsANDPDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
02474           return;
02475         }
02476       }
02477     }
02478   }
02479 
02480   // Pattern: (X86fand:f32 (X86loadp:f32 addr:iPTR:$src2), FR32:f32:$src1)
02481   // Emits: (FsANDPSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
02482   // Pattern complexity = 14  cost = 1  size = 3
02483   if ((Subtarget->hasSSE1())) {
02484     N0 = N.getOperand(0);
02485     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
02486         InFlightSet.count(N0.Val) == 0 &&
02487         N0.hasOneUse() &&
02488         !CodeGenMap.count(N0.getValue(0)) &&
02489         !isNonImmUse(N.Val, N0.Val)) {
02490       Chain0 = N0.getOperand(0);
02491       N01 = N0.getOperand(1);
02492       N1 = N.getOperand(1);
02493       if (N.Val->getValueType(0) == MVT::f32) {
02494         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02495         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02496         if (!Match) {
02497           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02498         }
02499         if (Match) {
02500           Emit_20(Result, N, X86::FsANDPSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
02501           return;
02502         }
02503       }
02504     }
02505   }
02506 
02507   // Pattern: (X86fand:f64 (X86loadp:f64 addr:iPTR:$src2), FR64:f64:$src1)
02508   // Emits: (FsANDPDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
02509   // Pattern complexity = 14  cost = 1  size = 3
02510   if ((Subtarget->hasSSE2())) {
02511     N0 = N.getOperand(0);
02512     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
02513         InFlightSet.count(N0.Val) == 0 &&
02514         N0.hasOneUse() &&
02515         !CodeGenMap.count(N0.getValue(0)) &&
02516         !isNonImmUse(N.Val, N0.Val)) {
02517       Chain0 = N0.getOperand(0);
02518       N01 = N0.getOperand(1);
02519       N1 = N.getOperand(1);
02520       if (N.Val->getValueType(0) == MVT::f64) {
02521         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02522         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02523         if (!Match) {
02524           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02525         }
02526         if (Match) {
02527           Emit_20(Result, N, X86::FsANDPDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
02528           return;
02529         }
02530       }
02531     }
02532   }
02533 
02534   // Pattern: (X86fand:f32 FR32:f32:$src1, FR32:f32:$src2)
02535   // Emits: (FsANDPSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
02536   // Pattern complexity = 2  cost = 1  size = 3
02537   if ((Subtarget->hasSSE1())) {
02538     N0 = N.getOperand(0);
02539     N1 = N.getOperand(1);
02540     if (N.Val->getValueType(0) == MVT::f32) {
02541       Emit_21(Result, N, X86::FsANDPSrr, MVT::f32, N0, N1);
02542       return;
02543     }
02544   }
02545 
02546   // Pattern: (X86fand:f64 FR64:f64:$src1, FR64:f64:$src2)
02547   // Emits: (FsANDPDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
02548   // Pattern complexity = 2  cost = 1
02549   if ((Subtarget->hasSSE2())) {
02550     N0 = N.getOperand(0);
02551     N1 = N.getOperand(1);
02552     if (N.Val->getValueType(0) == MVT::f64) {
02553       Emit_21(Result, N, X86::FsANDPDrr, MVT::f64, N0, N1);
02554       return;
02555     }
02556   }
02557   std::cerr << "Cannot yet select: ";
02558   N.Val->dump(CurDAG);
02559   std::cerr << '\n';
02560   abort();
02561 }
02562 
02563 void Emit_22(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
02564   SDOperand Tmp0(0, 0);
02565   SDOperand Tmp1(0, 0);
02566   SDOperand Tmp2(0, 0);
02567   SDOperand Tmp3(0, 0);
02568   SDNode *ResNode = NULL;
02569   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
02570   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
02571   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
02572   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
02573   Select(Tmp0, CPTmp0);
02574   Select(Tmp1, CPTmp1);
02575   Select(Tmp2, CPTmp2);
02576   Select(Tmp3, CPTmp3);
02577   Select(Chain, Chain);
02578   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
02579   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
02580   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
02581   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
02582   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
02583   Chain = SDOperand(ResNode, 1);
02584   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02585   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
02586   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
02587   Result = SDOperand(ResNode, N.ResNo);
02588   return;
02589 }
02590 void Select_X86fild(SDOperand &Result, SDOperand N) {
02591   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
02592     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
02593     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
02594     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
02595     Result = Dummy;
02596     return;
02597   }
02598   SDOperand CPTmp0(0, 0);
02599   SDOperand CPTmp1(0, 0);
02600   SDOperand CPTmp2(0, 0);
02601   SDOperand CPTmp3(0, 0);
02602   SDOperand Chain(0, 0);
02603   SDOperand N1(0, 0);
02604   SDOperand N2(0, 0);
02605   SDOperand Tmp0(0, 0);
02606   SDOperand Tmp1(0, 0);
02607   SDOperand Tmp2(0, 0);
02608   SDOperand Tmp3(0, 0);
02609   SDNode *ResNode = NULL;
02610   if ((!Subtarget->hasSSE2())) {
02611     Chain = N.getOperand(0);
02612     N1 = N.getOperand(1);
02613     N2 = N.getOperand(2);
02614 
02615     // Pattern: (X86fild:f64 addr:iPTR:$src, i16:Other)
02616     // Emits: (FpILD16m:f64 addr:iPTR:$src)
02617     // Pattern complexity = 12  cost = 1  size = 0
02618     if (cast<VTSDNode>(N2)->getVT() == MVT::i16) {
02619       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02620       if (Match) {
02621         Emit_22(Result, N, X86::FpILD16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02622         return;
02623       }
02624     }
02625 
02626     // Pattern: (X86fild:f64 addr:iPTR:$src, i32:Other)
02627     // Emits: (FpILD32m:f64 addr:iPTR:$src)
02628     // Pattern complexity = 12  cost = 1  size = 0
02629     if (cast<VTSDNode>(N2)->getVT() == MVT::i32) {
02630       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02631       if (Match) {
02632         Emit_22(Result, N, X86::FpILD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02633         return;
02634       }
02635     }
02636 
02637     // Pattern: (X86fild:f64 addr:iPTR:$src, i64:Other)
02638     // Emits: (FpILD64m:f64 addr:iPTR:$src)
02639     // Pattern complexity = 12  cost = 1
02640     if (cast<VTSDNode>(N2)->getVT() == MVT::i64) {
02641       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02642       if (Match) {
02643         Emit_22(Result, N, X86::FpILD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02644         return;
02645       }
02646     }
02647   }
02648   std::cerr << "Cannot yet select: ";
02649   N.Val->dump(CurDAG);
02650   std::cerr << '\n';
02651   abort();
02652 }
02653 
02654 void Emit_23(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
02655   SDOperand InFlag(0, 0);
02656   SDOperand Tmp0(0, 0);
02657   SDOperand Tmp1(0, 0);
02658   SDOperand Tmp2(0, 0);
02659   SDOperand Tmp3(0, 0);
02660   SDNode *ResNode = NULL;
02661   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
02662   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
02663   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
02664   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
02665   Select(Tmp0, CPTmp0);
02666   Select(Tmp1, CPTmp1);
02667   Select(Tmp2, CPTmp2);
02668   Select(Tmp3, CPTmp3);
02669   Select(Chain, Chain);
02670   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
02671   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
02672   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
02673   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
02674   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
02675   Chain = SDOperand(ResNode, 1);
02676   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02677   InFlag = SDOperand(ResNode, 2);
02678   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
02679   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
02680   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, InFlag.Val, InFlag.ResNo);
02681   Result = SDOperand(ResNode, N.ResNo);
02682   return;
02683 }
02684 void Select_X86fildflag(SDOperand &Result, SDOperand N) {
02685   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
02686     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
02687     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
02688     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
02689     Result = Dummy;
02690     return;
02691   }
02692   SDOperand CPTmp0(0, 0);
02693   SDOperand CPTmp1(0, 0);
02694   SDOperand CPTmp2(0, 0);
02695   SDOperand CPTmp3(0, 0);
02696   SDOperand Chain(0, 0);
02697   SDOperand InFlag(0, 0);
02698   SDOperand N1(0, 0);
02699   SDOperand N2(0, 0);
02700   SDOperand Tmp0(0, 0);
02701   SDOperand Tmp1(0, 0);
02702   SDOperand Tmp2(0, 0);
02703   SDOperand Tmp3(0, 0);
02704   SDNode *ResNode = NULL;
02705   Chain = N.getOperand(0);
02706   N1 = N.getOperand(1);
02707   N2 = N.getOperand(2);
02708   if (cast<VTSDNode>(N2)->getVT() == MVT::i64) {
02709     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02710     if (Match) {
02711       Emit_23(Result, N, X86::FpILD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02712       return;
02713     }
02714   }
02715   std::cerr << "Cannot yet select: ";
02716   N.Val->dump(CurDAG);
02717   std::cerr << '\n';
02718   abort();
02719 }
02720 
02721 void Select_X86fld(SDOperand &Result, SDOperand N) {
02722   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
02723     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
02724     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
02725     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
02726     Result = Dummy;
02727     return;
02728   }
02729   SDOperand CPTmp0(0, 0);
02730   SDOperand CPTmp1(0, 0);
02731   SDOperand CPTmp2(0, 0);
02732   SDOperand CPTmp3(0, 0);
02733   SDOperand Chain(0, 0);
02734   SDOperand N1(0, 0);
02735   SDOperand N2(0, 0);
02736   SDOperand Tmp0(0, 0);
02737   SDOperand Tmp1(0, 0);
02738   SDOperand Tmp2(0, 0);
02739   SDOperand Tmp3(0, 0);
02740   SDNode *ResNode = NULL;
02741   Chain = N.getOperand(0);
02742   N1 = N.getOperand(1);
02743   N2 = N.getOperand(2);
02744 
02745   // Pattern: (X86fld:f64 addr:iPTR:$src, f32:Other)
02746   // Emits: (FpLD32m:f64 addr:iPTR:$src)
02747   // Pattern complexity = 12  cost = 1  size = 0
02748   if (cast<VTSDNode>(N2)->getVT() == MVT::f32) {
02749     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02750     if (Match) {
02751       Emit_22(Result, N, X86::FpLD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02752       return;
02753     }
02754   }
02755 
02756   // Pattern: (X86fld:f64 addr:iPTR:$src, f64:Other)
02757   // Emits: (FpLD64m:f64 addr:iPTR:$src)
02758   // Pattern complexity = 12  cost = 1
02759   if (cast<VTSDNode>(N2)->getVT() == MVT::f64) {
02760     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02761     if (Match) {
02762       Emit_22(Result, N, X86::FpLD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02763       return;
02764     }
02765   }
02766   std::cerr << "Cannot yet select: ";
02767   N.Val->dump(CurDAG);
02768   std::cerr << '\n';
02769   abort();
02770 }
02771 
02772 void Emit_24(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
02773   SDOperand Tmp0(0, 0);
02774   SDOperand Tmp1(0, 0);
02775   SDOperand Tmp2(0, 0);
02776   SDOperand Tmp3(0, 0);
02777   SDOperand Tmp4(0, 0);
02778   SDNode *ResNode = NULL;
02779   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
02780   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
02781   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
02782   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
02783   Select(Tmp0, CPTmp0);
02784   Select(Tmp1, CPTmp1);
02785   Select(Tmp2, CPTmp2);
02786   Select(Tmp3, CPTmp3);
02787   Select(Tmp4, N1);
02788   Select(Chain, Chain);
02789   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02790   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
02791   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
02792   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
02793   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
02794   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
02795   Chain = SDOperand(ResNode, 0);
02796   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
02797   Result = SDOperand(ResNode, N.ResNo);
02798   return;
02799 }
02800 void Select_X86fp_to_i16mem(SDOperand &Result, SDOperand N) {
02801   SDOperand CPTmp0(0, 0);
02802   SDOperand CPTmp1(0, 0);
02803   SDOperand CPTmp2(0, 0);
02804   SDOperand CPTmp3(0, 0);
02805   SDOperand Chain(0, 0);
02806   SDOperand N1(0, 0);
02807   SDOperand N2(0, 0);
02808   SDOperand Tmp0(0, 0);
02809   SDOperand Tmp1(0, 0);
02810   SDOperand Tmp2(0, 0);
02811   SDOperand Tmp3(0, 0);
02812   SDOperand Tmp4(0, 0);
02813   SDNode *ResNode = NULL;
02814 
02815   // Pattern: (X86fp_to_i16mem:void RFP:f64:$src, addr:iPTR:$op)
02816   // Emits: (FpISTT16m:void addr:iPTR:$op, RFP:f64:$src)
02817   // Pattern complexity = 12  cost = 1  size = 0
02818   if ((Subtarget->hasSSE3())) {
02819     Chain = N.getOperand(0);
02820     N1 = N.getOperand(1);
02821     N2 = N.getOperand(2);
02822     if (N1.Val->getValueType(0) == MVT::f64) {
02823       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02824       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02825       if (!Match) {
02826         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02827       }
02828       if (Match) {
02829         Emit_24(Result, N, X86::FpISTT16m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02830         return;
02831       }
02832     }
02833   }
02834 
02835   // Pattern: (X86fp_to_i16mem:void RFP:f64:$src, addr:iPTR:$dst)
02836   // Emits: (FP_TO_INT16_IN_MEM:void addr:iPTR:$dst, RFP:f64:$src)
02837   // Pattern complexity = 12  cost = 11
02838   Chain = N.getOperand(0);
02839   N1 = N.getOperand(1);
02840   N2 = N.getOperand(2);
02841   if (N1.Val->getValueType(0) == MVT::f64) {
02842     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02843     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02844     if (!Match) {
02845       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02846     }
02847     if (Match) {
02848       Emit_24(Result, N, X86::FP_TO_INT16_IN_MEM, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02849       return;
02850     }
02851   }
02852   std::cerr << "Cannot yet select: ";
02853   N.Val->dump(CurDAG);
02854   std::cerr << '\n';
02855   abort();
02856 }
02857 
02858 void Select_X86fp_to_i32mem(SDOperand &Result, SDOperand N) {
02859   SDOperand CPTmp0(0, 0);
02860   SDOperand CPTmp1(0, 0);
02861   SDOperand CPTmp2(0, 0);
02862   SDOperand CPTmp3(0, 0);
02863   SDOperand Chain(0, 0);
02864   SDOperand N1(0, 0);
02865   SDOperand N2(0, 0);
02866   SDOperand Tmp0(0, 0);
02867   SDOperand Tmp1(0, 0);
02868   SDOperand Tmp2(0, 0);
02869   SDOperand Tmp3(0, 0);
02870   SDOperand Tmp4(0, 0);
02871   SDNode *ResNode = NULL;
02872 
02873   // Pattern: (X86fp_to_i32mem:void RFP:f64:$src, addr:iPTR:$op)
02874   // Emits: (FpISTT32m:void addr:iPTR:$op, RFP:f64:$src)
02875   // Pattern complexity = 12  cost = 1  size = 0
02876   if ((Subtarget->hasSSE3())) {
02877     Chain = N.getOperand(0);
02878     N1 = N.getOperand(1);
02879     N2 = N.getOperand(2);
02880     if (N1.Val->getValueType(0) == MVT::f64) {
02881       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02882       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02883       if (!Match) {
02884         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02885       }
02886       if (Match) {
02887         Emit_24(Result, N, X86::FpISTT32m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02888         return;
02889       }
02890     }
02891   }
02892 
02893   // Pattern: (X86fp_to_i32mem:void RFP:f64:$src, addr:iPTR:$dst)
02894   // Emits: (FP_TO_INT32_IN_MEM:void addr:iPTR:$dst, RFP:f64:$src)
02895   // Pattern complexity = 12  cost = 11
02896   Chain = N.getOperand(0);
02897   N1 = N.getOperand(1);
02898   N2 = N.getOperand(2);
02899   if (N1.Val->getValueType(0) == MVT::f64) {
02900     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02901     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02902     if (!Match) {
02903       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02904     }
02905     if (Match) {
02906       Emit_24(Result, N, X86::FP_TO_INT32_IN_MEM, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02907       return;
02908     }
02909   }
02910   std::cerr << "Cannot yet select: ";
02911   N.Val->dump(CurDAG);
02912   std::cerr << '\n';
02913   abort();
02914 }
02915 
02916 void Select_X86fp_to_i64mem(SDOperand &Result, SDOperand N) {
02917   SDOperand CPTmp0(0, 0);
02918   SDOperand CPTmp1(0, 0);
02919   SDOperand CPTmp2(0, 0);
02920   SDOperand CPTmp3(0, 0);
02921   SDOperand Chain(0, 0);
02922   SDOperand N1(0, 0);
02923   SDOperand N2(0, 0);
02924   SDOperand Tmp0(0, 0);
02925   SDOperand Tmp1(0, 0);
02926   SDOperand Tmp2(0, 0);
02927   SDOperand Tmp3(0, 0);
02928   SDOperand Tmp4(0, 0);
02929   SDNode *ResNode = NULL;
02930 
02931   // Pattern: (X86fp_to_i64mem:void RFP:f64:$src, addr:iPTR:$op)
02932   // Emits: (FpISTT64m:void addr:iPTR:$op, RFP:f64:$src)
02933   // Pattern complexity = 12  cost = 1  size = 0
02934   if ((Subtarget->hasSSE3())) {
02935     Chain = N.getOperand(0);
02936     N1 = N.getOperand(1);
02937     N2 = N.getOperand(2);
02938     if (N1.Val->getValueType(0) == MVT::f64) {
02939       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02940       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02941       if (!Match) {
02942         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02943       }
02944       if (Match) {
02945         Emit_24(Result, N, X86::FpISTT64m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02946         return;
02947       }
02948     }
02949   }
02950 
02951   // Pattern: (X86fp_to_i64mem:void RFP:f64:$src, addr:iPTR:$dst)
02952   // Emits: (FP_TO_INT64_IN_MEM:void addr:iPTR:$dst, RFP:f64:$src)
02953   // Pattern complexity = 12  cost = 11
02954   Chain = N.getOperand(0);
02955   N1 = N.getOperand(1);
02956   N2 = N.getOperand(2);
02957   if (N1.Val->getValueType(0) == MVT::f64) {
02958     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
02959     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
02960     if (!Match) {
02961       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
02962     }
02963     if (Match) {
02964       Emit_24(Result, N, X86::FP_TO_INT64_IN_MEM, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
02965       return;
02966     }
02967   }
02968   std::cerr << "Cannot yet select: ";
02969   N.Val->dump(CurDAG);
02970   std::cerr << '\n';
02971   abort();
02972 }
02973 
02974 void Emit_25(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &Chain) NOINLINE {
02975   SDOperand InFlag(0, 0);
02976   SDNode *ResNode = NULL;
02977   Chain = N.getOperand(0);
02978   Select(Chain, Chain);
02979   Select(InFlag, N.getOperand(1));
02980   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Chain, InFlag);
02981   Chain = SDOperand(ResNode, 1);
02982   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02983   InFlag = SDOperand(ResNode, 2);
02984   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
02985   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
02986   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, InFlag.Val, InFlag.ResNo);
02987   Result = SDOperand(ResNode, N.ResNo);
02988   return;
02989 }
02990 void Select_X86fpget(SDOperand &Result, SDOperand N) {
02991   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
02992     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
02993     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
02994     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
02995     Result = Dummy;
02996     return;
02997   }
02998   SDOperand Chain(0, 0);
02999   SDOperand InFlag(0, 0);
03000   SDNode *ResNode = NULL;
03001   Emit_25(Result, N, X86::FpGETRESULT, MVT::f64, Chain);
03002   return;
03003 }
03004 
03005 void Emit_26(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
03006   SDOperand InFlag(0, 0);
03007   SDOperand Tmp0(0, 0);
03008   SDNode *ResNode = NULL;
03009   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03010   Select(Tmp0, N1);
03011   Select(Chain, Chain);
03012   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03013   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Chain);
03014   Chain = SDOperand(ResNode, 0);
03015   InFlag = SDOperand(ResNode, 1);
03016   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03017   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03018   Result = SDOperand(ResNode, N.ResNo);
03019   return;
03020 }
03021 void Select_X86fpset(SDOperand &Result, SDOperand N) {
03022   SDOperand Chain(0, 0);
03023   SDOperand InFlag(0, 0);
03024   SDOperand N1(0, 0);
03025   SDOperand Tmp0(0, 0);
03026   SDNode *ResNode = NULL;
03027   Chain = N.getOperand(0);
03028   N1 = N.getOperand(1);
03029   if (N1.Val->getValueType(0) == MVT::f64) {
03030     Emit_26(Result, N, X86::FpSETRESULT, Chain, N1);
03031     return;
03032   }
03033   std::cerr << "Cannot yet select: ";
03034   N.Val->dump(CurDAG);
03035   std::cerr << '\n';
03036   abort();
03037 }
03038 
03039 void Emit_27(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
03040   SDOperand InFlag(0, 0);
03041   SDOperand Tmp0(0, 0);
03042   SDOperand Tmp1(0, 0);
03043   SDOperand Tmp2(0, 0);
03044   SDOperand Tmp3(0, 0);
03045   SDOperand Tmp4(0, 0);
03046   SDNode *ResNode = NULL;
03047   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
03048   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
03049   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
03050   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
03051   Select(Tmp0, CPTmp0);
03052   Select(Tmp1, CPTmp1);
03053   Select(Tmp2, CPTmp2);
03054   Select(Tmp3, CPTmp3);
03055   Select(Tmp4, N1);
03056   Select(Chain, Chain);
03057   Select(InFlag, N.getOperand(4));
03058   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03059   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
03060   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
03061   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
03062   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
03063   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
03064   Chain = SDOperand(ResNode, 0);
03065   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03066   Result = SDOperand(ResNode, N.ResNo);
03067   return;
03068 }
03069 void Select_X86fst(SDOperand &Result, SDOperand N) {
03070   SDOperand CPTmp0(0, 0);
03071   SDOperand CPTmp1(0, 0);
03072   SDOperand CPTmp2(0, 0);
03073   SDOperand CPTmp3(0, 0);
03074   SDOperand Chain(0, 0);
03075   SDOperand InFlag(0, 0);
03076   SDOperand N1(0, 0);
03077   SDOperand N2(0, 0);
03078   SDOperand N3(0, 0);
03079   SDOperand Tmp0(0, 0);
03080   SDOperand Tmp1(0, 0);
03081   SDOperand Tmp2(0, 0);
03082   SDOperand Tmp3(0, 0);
03083   SDOperand Tmp4(0, 0);
03084   SDNode *ResNode = NULL;
03085   Chain = N.getOperand(0);
03086   N1 = N.getOperand(1);
03087   N2 = N.getOperand(2);
03088   N3 = N.getOperand(3);
03089 
03090   // Pattern: (X86fst:void RFP:f64:$src, addr:iPTR:$op, f32:Other)
03091   // Emits: (FpST32m:void addr:iPTR:$op, RFP:f64:$src)
03092   // Pattern complexity = 12  cost = 1  size = 0
03093   if (cast<VTSDNode>(N3)->getVT() == MVT::f32 &&
03094       N1.Val->getValueType(0) == MVT::f64) {
03095     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03096     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03097     if (!Match) {
03098       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03099     }
03100     if (Match) {
03101       Emit_27(Result, N, X86::FpST32m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
03102       return;
03103     }
03104   }
03105 
03106   // Pattern: (X86fst:void RFP:f64:$src, addr:iPTR:$op, f64:Other)
03107   // Emits: (FpST64m:void addr:iPTR:$op, RFP:f64:$src)
03108   // Pattern complexity = 12  cost = 1
03109   if (cast<VTSDNode>(N3)->getVT() == MVT::f64 &&
03110       N1.Val->getValueType(0) == MVT::f64) {
03111     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03112     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03113     if (!Match) {
03114       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03115     }
03116     if (Match) {
03117       Emit_27(Result, N, X86::FpST64m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
03118       return;
03119     }
03120   }
03121   std::cerr << "Cannot yet select: ";
03122   N.Val->dump(CurDAG);
03123   std::cerr << '\n';
03124   abort();
03125 }
03126 
03127 void Select_X86fxor(SDOperand &Result, SDOperand N) {
03128   SDOperand CPTmp0(0, 0);
03129   SDOperand CPTmp1(0, 0);
03130   SDOperand CPTmp2(0, 0);
03131   SDOperand CPTmp3(0, 0);
03132   SDOperand Chain0(0, 0);
03133   SDOperand Chain1(0, 0);
03134   SDOperand N0(0, 0);
03135   SDOperand N01(0, 0);
03136   SDOperand N1(0, 0);
03137   SDOperand N11(0, 0);
03138   SDOperand Tmp0(0, 0);
03139   SDOperand Tmp1(0, 0);
03140   SDOperand Tmp2(0, 0);
03141   SDOperand Tmp3(0, 0);
03142   SDOperand Tmp4(0, 0);
03143   SDNode *ResNode = NULL;
03144 
03145   // Pattern: (X86fxor:f32 FR32:f32:$src1, (X86loadp:f32 addr:iPTR:$src2))
03146   // Emits: (FsXORPSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
03147   // Pattern complexity = 14  cost = 1  size = 3
03148   if ((Subtarget->hasSSE1())) {
03149     N0 = N.getOperand(0);
03150     N1 = N.getOperand(1);
03151     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
03152         InFlightSet.count(N1.Val) == 0 &&
03153         N1.hasOneUse() &&
03154         !CodeGenMap.count(N1.getValue(0)) &&
03155         !isNonImmUse(N.Val, N1.Val)) {
03156       Chain1 = N1.getOperand(0);
03157       N11 = N1.getOperand(1);
03158       if (N.Val->getValueType(0) == MVT::f32) {
03159         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03160         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03161         if (!Match) {
03162           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03163         }
03164         if (Match) {
03165           Emit_19(Result, N, X86::FsXORPSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
03166           return;
03167         }
03168       }
03169     }
03170   }
03171 
03172   // Pattern: (X86fxor:f64 FR64:f64:$src1, (X86loadp:f64 addr:iPTR:$src2))
03173   // Emits: (FsXORPDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
03174   // Pattern complexity = 14  cost = 1  size = 3
03175   if ((Subtarget->hasSSE2())) {
03176     N0 = N.getOperand(0);
03177     N1 = N.getOperand(1);
03178     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
03179         InFlightSet.count(N1.Val) == 0 &&
03180         N1.hasOneUse() &&
03181         !CodeGenMap.count(N1.getValue(0)) &&
03182         !isNonImmUse(N.Val, N1.Val)) {
03183       Chain1 = N1.getOperand(0);
03184       N11 = N1.getOperand(1);
03185       if (N.Val->getValueType(0) == MVT::f64) {
03186         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03187         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03188         if (!Match) {
03189           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03190         }
03191         if (Match) {
03192           Emit_19(Result, N, X86::FsXORPDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
03193           return;
03194         }
03195       }
03196     }
03197   }
03198 
03199   // Pattern: (X86fxor:f32 (X86loadp:f32 addr:iPTR:$src2), FR32:f32:$src1)
03200   // Emits: (FsXORPSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
03201   // Pattern complexity = 14  cost = 1  size = 3
03202   if ((Subtarget->hasSSE1())) {
03203     N0 = N.getOperand(0);
03204     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
03205         InFlightSet.count(N0.Val) == 0 &&
03206         N0.hasOneUse() &&
03207         !CodeGenMap.count(N0.getValue(0)) &&
03208         !isNonImmUse(N.Val, N0.Val)) {
03209       Chain0 = N0.getOperand(0);
03210       N01 = N0.getOperand(1);
03211       N1 = N.getOperand(1);
03212       if (N.Val->getValueType(0) == MVT::f32) {
03213         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03214         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03215         if (!Match) {
03216           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03217         }
03218         if (Match) {
03219           Emit_20(Result, N, X86::FsXORPSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
03220           return;
03221         }
03222       }
03223     }
03224   }
03225 
03226   // Pattern: (X86fxor:f64 (X86loadp:f64 addr:iPTR:$src2), FR64:f64:$src1)
03227   // Emits: (FsXORPDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
03228   // Pattern complexity = 14  cost = 1  size = 3
03229   if ((Subtarget->hasSSE2())) {
03230     N0 = N.getOperand(0);
03231     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
03232         InFlightSet.count(N0.Val) == 0 &&
03233         N0.hasOneUse() &&
03234         !CodeGenMap.count(N0.getValue(0)) &&
03235         !isNonImmUse(N.Val, N0.Val)) {
03236       Chain0 = N0.getOperand(0);
03237       N01 = N0.getOperand(1);
03238       N1 = N.getOperand(1);
03239       if (N.Val->getValueType(0) == MVT::f64) {
03240         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03241         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03242         if (!Match) {
03243           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03244         }
03245         if (Match) {
03246           Emit_20(Result, N, X86::FsXORPDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
03247           return;
03248         }
03249       }
03250     }
03251   }
03252 
03253   // Pattern: (X86fxor:f32 FR32:f32:$src1, FR32:f32:$src2)
03254   // Emits: (FsXORPSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
03255   // Pattern complexity = 2  cost = 1  size = 3
03256   if ((Subtarget->hasSSE1())) {
03257     N0 = N.getOperand(0);
03258     N1 = N.getOperand(1);
03259     if (N.Val->getValueType(0) == MVT::f32) {
03260       Emit_21(Result, N, X86::FsXORPSrr, MVT::f32, N0, N1);
03261       return;
03262     }
03263   }
03264 
03265   // Pattern: (X86fxor:f64 FR64:f64:$src1, FR64:f64:$src2)
03266   // Emits: (FsXORPDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
03267   // Pattern complexity = 2  cost = 1
03268   if ((Subtarget->hasSSE2())) {
03269     N0 = N.getOperand(0);
03270     N1 = N.getOperand(1);
03271     if (N.Val->getValueType(0) == MVT::f64) {
03272       Emit_21(Result, N, X86::FsXORPDrr, MVT::f64, N0, N1);
03273       return;
03274     }
03275   }
03276   std::cerr << "Cannot yet select: ";
03277   N.Val->dump(CurDAG);
03278   std::cerr << '\n';
03279   abort();
03280 }
03281 
03282 void Emit_28(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1) NOINLINE {
03283   SDOperand Tmp0(0, 0);
03284   SDOperand Tmp1(0, 0);
03285   SDOperand Tmp2(0, 0);
03286   SDOperand Tmp3(0, 0);
03287   SDNode *ResNode = NULL;
03288   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
03289   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
03290   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
03291   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
03292   Select(Tmp0, CPTmp0);
03293   Select(Tmp1, CPTmp1);
03294   Select(Tmp2, CPTmp2);
03295   Select(Tmp3, CPTmp3);
03296   Select(Chain, Chain);
03297   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
03298   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
03299   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
03300   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
03301   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03302   Chain = SDOperand(ResNode, 1);
03303   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03304   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
03305   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
03306   Result = SDOperand(ResNode, N.ResNo);
03307   return;
03308 }
03309 void Select_X86loadp(SDOperand &Result, SDOperand N) {
03310   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03311     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03312     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03313     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03314     Result = Dummy;
03315     return;
03316   }
03317   SDOperand CPTmp0(0, 0);
03318   SDOperand CPTmp1(0, 0);
03319   SDOperand CPTmp2(0, 0);
03320   SDOperand CPTmp3(0, 0);
03321   SDOperand Chain(0, 0);
03322   SDOperand N1(0, 0);
03323   SDOperand Tmp0(0, 0);
03324   SDOperand Tmp1(0, 0);
03325   SDOperand Tmp2(0, 0);
03326   SDOperand Tmp3(0, 0);
03327   SDNode *ResNode = NULL;
03328 
03329   // Pattern: (X86loadp:f32 addr:iPTR:$src)
03330   // Emits: (FsMOVAPSrm:f32 addr:iPTR:$src)
03331   // Pattern complexity = 12  cost = 1  size = 3
03332   if ((Subtarget->hasSSE1())) {
03333     Chain = N.getOperand(0);
03334     N1 = N.getOperand(1);
03335     if (N.Val->getValueType(0) == MVT::f32) {
03336       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03337       if (Match) {
03338         Emit_28(Result, N, X86::FsMOVAPSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
03339         return;
03340       }
03341     }
03342   }
03343 
03344   // Pattern: (X86loadp:f64 addr:iPTR:$src)
03345   // Emits: (FsMOVAPDrm:f64 addr:iPTR:$src)
03346   // Pattern complexity = 12  cost = 1
03347   if ((Subtarget->hasSSE2())) {
03348     Chain = N.getOperand(0);
03349     N1 = N.getOperand(1);
03350     if (N.Val->getValueType(0) == MVT::f64) {
03351       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03352       if (Match) {
03353         Emit_28(Result, N, X86::FsMOVAPDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
03354         return;
03355       }
03356     }
03357   }
03358   std::cerr << "Cannot yet select: ";
03359   N.Val->dump(CurDAG);
03360   std::cerr << '\n';
03361   abort();
03362 }
03363 
03364 void Select_X86loadu(SDOperand &Result, SDOperand N) {
03365   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03366     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03367     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03368     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03369     Result = Dummy;
03370     return;
03371   }
03372   SDOperand CPTmp0(0, 0);
03373   SDOperand CPTmp1(0, 0);
03374   SDOperand CPTmp2(0, 0);
03375   SDOperand CPTmp3(0, 0);
03376   SDOperand Chain(0, 0);
03377   SDOperand N1(0, 0);
03378   SDOperand Tmp0(0, 0);
03379   SDOperand Tmp1(0, 0);
03380   SDOperand Tmp2(0, 0);
03381   SDOperand Tmp3(0, 0);
03382   SDNode *ResNode = NULL;
03383   if ((Subtarget->hasSSE1())) {
03384     Chain = N.getOperand(0);
03385     N1 = N.getOperand(1);
03386     if (N.Val->getValueType(0) == MVT::v4f32) {
03387       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03388       if (Match) {
03389         Emit_28(Result, N, X86::MOVUPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
03390         return;
03391       }
03392     }
03393   }
03394   std::cerr << "Cannot yet select: ";
03395   N.Val->dump(CurDAG);
03396   std::cerr << '\n';
03397   abort();
03398 }
03399 
03400 void Emit_29(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
03401   SDOperand Tmp0(0, 0);
03402   SDOperand Tmp1(0, 0);
03403   SDNode *ResNode = NULL;
03404   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03405   Select(Tmp0, N0);
03406   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
03407   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03408   if (N.Val->hasOneUse()) {
03409     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
03410   } else {
03411     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
03412     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03413     Result = SDOperand(ResNode, 0);
03414   }
03415   return;
03416 }
03417 void Select_X86pextrw(SDOperand &Result, SDOperand N) {
03418   SDOperand N0(0, 0);
03419   SDOperand N1(0, 0);
03420   SDOperand Tmp0(0, 0);
03421   SDOperand Tmp1(0, 0);
03422   SDNode *ResNode = NULL;
03423   if ((Subtarget->hasSSE2())) {
03424     N0 = N.getOperand(0);
03425     N1 = N.getOperand(1);
03426     if (N1.getOpcode() == ISD::Constant &&
03427         N.Val->getValueType(0) == MVT::i32 &&
03428         N0.Val->getValueType(0) == MVT::v8i16 &&
03429         N1.Val->getValueType(0) == MVT::i32) {
03430       Emit_29(Result, N, X86::PEXTRWri, MVT::i32, N0, N1);
03431       return;
03432     }
03433   }
03434   std::cerr << "Cannot yet select: ";
03435   N.Val->dump(CurDAG);
03436   std::cerr << '\n';
03437   abort();
03438 }
03439 
03440 void Emit_30(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N2) NOINLINE {
03441   SDOperand Tmp0(0, 0);
03442   SDOperand Tmp1(0, 0);
03443   SDOperand Tmp2(0, 0);
03444   SDOperand Tmp3(0, 0);
03445   SDOperand Tmp4(0, 0);
03446   SDOperand Tmp5(0, 0);
03447   SDNode *ResNode = NULL;
03448   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
03449   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
03450   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
03451   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
03452   Select(Tmp0, CPTmp0);
03453   Select(Tmp1, CPTmp1);
03454   Select(Tmp2, CPTmp2);
03455   Select(Tmp3, CPTmp3);
03456   Select(Tmp4, N0);
03457   Tmp5 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N2)->getValue()), MVT::i32);
03458   Select(Chain10, Chain10);
03459   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03460   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
03461   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
03462   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
03463   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
03464   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain10);
03465   Chain10 = SDOperand(ResNode, 1);
03466   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03467   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
03468   AddHandleReplacement(N10.Val, 1, ResNode, 1);
03469   Result = SDOperand(ResNode, N.ResNo);
03470   return;
03471 }
03472 void Emit_31(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
03473   SDOperand Tmp0(0, 0);
03474   SDOperand Tmp1(0, 0);
03475   SDOperand Tmp2(0, 0);
03476   SDNode *ResNode = NULL;
03477   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03478   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03479   Select(Tmp0, N0);
03480   Select(Tmp1, N1);
03481   Tmp2 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N2)->getValue()), MVT::i32);
03482   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03483   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03484   if (N.Val->hasOneUse()) {
03485     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1, Tmp2);
03486   } else {
03487     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1, Tmp2);
03488     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03489     Result = SDOperand(ResNode, 0);
03490   }
03491   return;
03492 }
03493 void Select_X86pinsrw(SDOperand &Result, SDOperand N) {
03494   SDOperand CPTmp0(0, 0);
03495   SDOperand CPTmp1(0, 0);
03496   SDOperand CPTmp2(0, 0);
03497   SDOperand CPTmp3(0, 0);
03498   SDOperand Chain10(0, 0);
03499   SDOperand N0(0, 0);
03500   SDOperand N1(0, 0);
03501   SDOperand N10(0, 0);
03502   SDOperand N101(0, 0);
03503   SDOperand N2(0, 0);
03504   SDOperand Tmp0(0, 0);
03505   SDOperand Tmp1(0, 0);
03506   SDOperand Tmp2(0, 0);
03507   SDOperand Tmp3(0, 0);
03508   SDOperand Tmp4(0, 0);
03509   SDOperand Tmp5(0, 0);
03510   SDNode *ResNode = NULL;
03511   if ((Subtarget->hasSSE2())) {
03512     N0 = N.getOperand(0);
03513     N1 = N.getOperand(1);
03514 
03515     // Pattern: (X86pinsrw:v8i16 VR128:v8i16:$src1, (anyext:i32 (load:i16 addr:iPTR:$src2)), (imm:iPTR):$src3)
03516     // Emits: (PINSRWrmi:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2, (imm:i32):$src3)
03517     // Pattern complexity = 18  cost = 1  size = 3
03518     if (N1.getOpcode() == ISD::ANY_EXTEND &&
03519         InFlightSet.count(N1.Val) == 0 &&
03520         N1.hasOneUse() &&
03521         !CodeGenMap.count(N1.getValue(0))) {
03522       N10 = N1.getOperand(0);
03523       if (N10.getOpcode() == ISD::LOAD &&
03524           InFlightSet.count(N10.Val) == 0 &&
03525           N10.hasOneUse() &&
03526           !CodeGenMap.count(N10.getValue(0))) {
03527         Chain10 = N10.getOperand(0);
03528         N101 = N10.getOperand(1);
03529         N2 = N.getOperand(2);
03530         if (N2.getOpcode() == ISD::Constant &&
03531             N.Val->getValueType(0) == MVT::v8i16 &&
03532             N0.Val->getValueType(0) == MVT::v8i16 &&
03533             N1.Val->getValueType(0) == MVT::i32 &&
03534             N10.Val->getValueType(0) == MVT::i16 &&
03535             N2.Val->getValueType(0) == TLI.getPointerTy()) {
03536           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03537           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
03538           if (!Match) {
03539             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03540           }
03541           if (Match) {
03542             Emit_30(Result, N, X86::PINSRWrmi, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
03543             return;
03544           }
03545         }
03546       }
03547     }
03548 
03549     // Pattern: (X86pinsrw:v8i16 VR128:v8i16:$src1, GR32:i32:$src2, (imm:iPTR):$src3)
03550     // Emits: (PINSRWrri:v8i16 VR128:v8i16:$src1, GR32:i32:$src2, (imm:i32):$src3)
03551     // Pattern complexity = 4  cost = 1
03552     N2 = N.getOperand(2);
03553     if (N2.getOpcode() == ISD::Constant &&
03554         N.Val->getValueType(0) == MVT::v8i16 &&
03555         N0.Val->getValueType(0) == MVT::v8i16 &&
03556         N1.Val->getValueType(0) == MVT::i32 &&
03557         N2.Val->getValueType(0) == TLI.getPointerTy()) {
03558       Emit_31(Result, N, X86::PINSRWrri, MVT::v8i16, N0, N1, N2);
03559       return;
03560     }
03561   }
03562   std::cerr << "Cannot yet select: ";
03563   N.Val->dump(CurDAG);
03564   std::cerr << '\n';
03565   abort();
03566 }
03567 
03568 void Emit_32(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain) NOINLINE {
03569   SDOperand InFlag(0, 0);
03570   SDNode *ResNode = NULL;
03571   Chain = N.getOperand(0);
03572   Select(Chain, Chain);
03573   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Chain);
03574   Chain = SDOperand(ResNode, 0);
03575   InFlag = SDOperand(ResNode, 1);
03576   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03577   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03578   Result = SDOperand(ResNode, N.ResNo);
03579   return;
03580 }
03581 void Select_X86rdtsc(SDOperand &Result, SDOperand N) {
03582   SDOperand Chain(0, 0);
03583   SDOperand InFlag(0, 0);
03584   SDNode *ResNode = NULL;
03585   Emit_32(Result, N, X86::RDTSC, Chain);
03586   return;
03587 }
03588 
03589 void Emit_33(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
03590   SDOperand InFlag(0, 0);
03591   SDNode *ResNode = NULL;
03592   Select(Chain, Chain);
03593   Select(InFlag, N.getOperand(2));
03594   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Chain, InFlag);
03595   Chain = SDOperand(ResNode, 0);
03596   InFlag = SDOperand(ResNode, 1);
03597   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03598   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03599   Result = SDOperand(ResNode, N.ResNo);
03600   return;
03601 }
03602 void Select_X86rep_movs(SDOperand &Result, SDOperand N) {
03603   SDOperand Chain(0, 0);
03604   SDOperand InFlag(0, 0);
03605   SDOperand N1(0, 0);
03606   SDNode *ResNode = NULL;
03607   Chain = N.getOperand(0);
03608   N1 = N.getOperand(1);
03609 
03610   // Pattern: (X86rep_movs:void i8:Other)
03611   // Emits: (REP_MOVSB:void)
03612   // Pattern complexity = 2  cost = 1  size = 3
03613   if (cast<VTSDNode>(N1)->getVT() == MVT::i8) {
03614     Emit_33(Result, N, X86::REP_MOVSB, Chain, N1);
03615     return;
03616   }
03617 
03618   // Pattern: (X86rep_movs:void i16:Other)
03619   // Emits: (REP_MOVSW:void)
03620   // Pattern complexity = 2  cost = 1  size = 3
03621   if (cast<VTSDNode>(N1)->getVT() == MVT::i16) {
03622     Emit_33(Result, N, X86::REP_MOVSW, Chain, N1);
03623     return;
03624   }
03625 
03626   // Pattern: (X86rep_movs:void i32:Other)
03627   // Emits: (REP_MOVSD:void)
03628   // Pattern complexity = 2  cost = 1
03629   if (cast<VTSDNode>(N1)->getVT() == MVT::i32) {
03630     Emit_33(Result, N, X86::REP_MOVSD, Chain, N1);
03631     return;
03632   }
03633   std::cerr << "Cannot yet select: ";
03634   N.Val->dump(CurDAG);
03635   std::cerr << '\n';
03636   abort();
03637 }
03638 
03639 void Select_X86rep_stos(SDOperand &Result, SDOperand N) {
03640   SDOperand Chain(0, 0);
03641   SDOperand InFlag(0, 0);
03642   SDOperand N1(0, 0);
03643   SDNode *ResNode = NULL;
03644   Chain = N.getOperand(0);
03645   N1 = N.getOperand(1);
03646 
03647   // Pattern: (X86rep_stos:void i8:Other)
03648   // Emits: (REP_STOSB:void)
03649   // Pattern complexity = 2  cost = 1  size = 3
03650   if (cast<VTSDNode>(N1)->getVT() == MVT::i8) {
03651     Emit_33(Result, N, X86::REP_STOSB, Chain, N1);
03652     return;
03653   }
03654 
03655   // Pattern: (X86rep_stos:void i16:Other)
03656   // Emits: (REP_STOSW:void)
03657   // Pattern complexity = 2  cost = 1  size = 3
03658   if (cast<VTSDNode>(N1)->getVT() == MVT::i16) {
03659     Emit_33(Result, N, X86::REP_STOSW, Chain, N1);
03660     return;
03661   }
03662 
03663   // Pattern: (X86rep_stos:void i32:Other)
03664   // Emits: (REP_STOSD:void)
03665   // Pattern complexity = 2  cost = 1
03666   if (cast<VTSDNode>(N1)->getVT() == MVT::i32) {
03667     Emit_33(Result, N, X86::REP_STOSD, Chain, N1);
03668     return;
03669   }
03670   std::cerr << "Cannot yet select: ";
03671   N.Val->dump(CurDAG);
03672   std::cerr << '\n';
03673   abort();
03674 }
03675 
03676 void Emit_34(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, bool HasInFlag) NOINLINE {
03677   SDOperand InFlag(0, 0);
03678   SDNode *ResNode = NULL;
03679   HasInFlag = (N.getOperand(N.getNumOperands()-1).getValueType() == MVT::Flag);
03680   Select(Chain, Chain);
03681   if (HasInFlag)
03682     Select(InFlag, N.getOperand(N.getNumOperands()-1));
03683   ResNode = HasInFlag ? CurDAG->getTargetNode(Opc0, MVT::Other, Chain, InFlag) : CurDAG->getTargetNode(Opc0, MVT::Other, Chain);
03684   Chain = SDOperand(ResNode, 0);
03685   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03686   Result = SDOperand(ResNode, N.ResNo);
03687   return;
03688 }
03689 void Emit_35(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, bool HasInFlag) NOINLINE {
03690   SDOperand InFlag(0, 0);
03691   SDOperand Tmp0(0, 0);
03692   SDNode *ResNode = NULL;
03693   HasInFlag = (N.getOperand(N.getNumOperands()-1).getValueType() == MVT::Flag);
03694   Tmp0 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
03695   Select(Chain, Chain);
03696   if (HasInFlag)
03697     Select(InFlag, N.getOperand(N.getNumOperands()-1));
03698   ResNode = HasInFlag ? CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Chain, InFlag) : CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Chain);
03699   Chain = SDOperand(ResNode, 0);
03700   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
03701   Result = SDOperand(ResNode, N.ResNo);
03702   return;
03703 }
03704 void Select_X86retflag(SDOperand &Result, SDOperand N) {
03705   SDOperand Chain(0, 0);
03706   SDOperand InFlag(0, 0);
03707   SDOperand N1(0, 0);
03708   SDOperand Tmp0(0, 0);
03709   SDNode *ResNode = NULL;
03710   bool HasInFlag = false;
03711   Chain = N.getOperand(0);
03712   N1 = N.getOperand(1);
03713 
03714   // Pattern: (X86retflag:void 0:i16)
03715   // Emits: (RET:void)
03716   // Pattern complexity = 5  cost = 1  size = 3
03717   if (isa<ConstantSDNode>(N1)) {
03718     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
03719     if (CN0 == 0) {
03720       Emit_34(Result, N, X86::RET, Chain, N1, HasInFlag);
03721       return;
03722     }
03723   }
03724 
03725   // Pattern: (X86retflag:void (imm:i16):$amt)
03726   // Emits: (RETI:void (imm:i16):$amt)
03727   // Pattern complexity = 4  cost = 1
03728   if (N1.getOpcode() == ISD::Constant) {
03729     Emit_35(Result, N, X86::RETI, Chain, N1, HasInFlag);
03730     return;
03731   }
03732   std::cerr << "Cannot yet select: ";
03733   N.Val->dump(CurDAG);
03734   std::cerr << '\n';
03735   abort();
03736 }
03737 
03738 void Emit_36(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0) NOINLINE {
03739   SDOperand Tmp0(0, 0);
03740   SDNode *ResNode = NULL;
03741   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03742   Select(Tmp0, N0);
03743   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03744   if (N.Val->hasOneUse()) {
03745     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
03746   } else {
03747     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
03748     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03749     Result = SDOperand(ResNode, 0);
03750   }
03751   return;
03752 }
03753 void Select_X86s2vec(SDOperand &Result, SDOperand N) {
03754   SDOperand N0(0, 0);
03755   SDOperand Tmp0(0, 0);
03756   SDNode *ResNode = NULL;
03757   if ((Subtarget->hasSSE2())) {
03758     N0 = N.getOperand(0);
03759 
03760     // Pattern: (X86s2vec:v8i16 GR32:i32:$src)
03761     // Emits: (MOVDI2PDIrr:v8i16 GR32:i32:$src)
03762     // Pattern complexity = 2  cost = 1  size = 3
03763     if (N.Val->getValueType(0) == MVT::v8i16 &&
03764         N0.Val->getValueType(0) == MVT::i32) {
03765       Emit_36(Result, N, X86::MOVDI2PDIrr, MVT::v8i16, N0);
03766       return;
03767     }
03768 
03769     // Pattern: (X86s2vec:v16i8 GR32:i32:$src)
03770     // Emits: (MOVDI2PDIrr:v16i8 GR32:i32:$src)
03771     // Pattern complexity = 2  cost = 1
03772     if (N.Val->getValueType(0) == MVT::v16i8 &&
03773         N0.Val->getValueType(0) == MVT::i32) {
03774       Emit_36(Result, N, X86::MOVDI2PDIrr, MVT::v16i8, N0);
03775       return;
03776     }
03777   }
03778   std::cerr << "Cannot yet select: ";
03779   N.Val->dump(CurDAG);
03780   std::cerr << '\n';
03781   abort();
03782 }
03783 
03784 void Emit_37(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0) NOINLINE {
03785   SDOperand InFlag(0, 0);
03786   SDNode *ResNode = NULL;
03787   Select(InFlag, N.getOperand(1));
03788   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, InFlag);
03789   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03790   InFlag = SDOperand(ResNode, 1);
03791   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03792   Result = SDOperand(ResNode, N.ResNo);
03793   return;
03794 }
03795 void Select_X86setcc(SDOperand &Result, SDOperand N) {
03796   SDOperand InFlag(0, 0);
03797   SDOperand N0(0, 0);
03798   SDNode *ResNode = NULL;
03799   N0 = N.getOperand(0);
03800   if (isa<ConstantSDNode>(N0)) {
03801     int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
03802 
03803     // Pattern: (X86setcc:i8 4:i8)
03804     // Emits: (SETEr:i8)
03805     // Pattern complexity = 5  cost = 1  size = 3
03806     if (CN0 == 4) {
03807       Emit_37(Result, N, X86::SETEr, MVT::i8, N0);
03808       return;
03809     }
03810 
03811     // Pattern: (X86setcc:i8 9:i8)
03812     // Emits: (SETNEr:i8)
03813     // Pattern complexity = 5  cost = 1  size = 3
03814     if (CN0 == 9) {
03815       Emit_37(Result, N, X86::SETNEr, MVT::i8, N0);
03816       return;
03817     }
03818 
03819     // Pattern: (X86setcc:i8 7:i8)
03820     // Emits: (SETLr:i8)
03821     // Pattern complexity = 5  cost = 1  size = 3
03822     if (CN0 == 7) {
03823       Emit_37(Result, N, X86::SETLr, MVT::i8, N0);
03824       return;
03825     }
03826 
03827     // Pattern: (X86setcc:i8 6:i8)
03828     // Emits: (SETGEr:i8)
03829     // Pattern complexity = 5  cost = 1  size = 3
03830     if (CN0 == 6) {
03831       Emit_37(Result, N, X86::SETGEr, MVT::i8, N0);
03832       return;
03833     }
03834 
03835     // Pattern: (X86setcc:i8 8:i8)
03836     // Emits: (SETLEr:i8)
03837     // Pattern complexity = 5  cost = 1  size = 3
03838     if (CN0 == 8) {
03839       Emit_37(Result, N, X86::SETLEr, MVT::i8, N0);
03840       return;
03841     }
03842 
03843     // Pattern: (X86setcc:i8 5:i8)
03844     // Emits: (SETGr:i8)
03845     // Pattern complexity = 5  cost = 1  size = 3
03846     if (CN0 == 5) {
03847       Emit_37(Result, N, X86::SETGr, MVT::i8, N0);
03848       return;
03849     }
03850 
03851     // Pattern: (X86setcc:i8 2:i8)
03852     // Emits: (SETBr:i8)
03853     // Pattern complexity = 5  cost = 1  size = 3
03854     if (CN0 == 2) {
03855       Emit_37(Result, N, X86::SETBr, MVT::i8, N0);
03856       return;
03857     }
03858 
03859     // Pattern: (X86setcc:i8 1:i8)
03860     // Emits: (SETAEr:i8)
03861     // Pattern complexity = 5  cost = 1  size = 3
03862     if (CN0 == 1) {
03863       Emit_37(Result, N, X86::SETAEr, MVT::i8, N0);
03864       return;
03865     }
03866 
03867     // Pattern: (X86setcc:i8 3:i8)
03868     // Emits: (SETBEr:i8)
03869     // Pattern complexity = 5  cost = 1  size = 3
03870     if (CN0 == 3) {
03871       Emit_37(Result, N, X86::SETBEr, MVT::i8, N0);
03872       return;
03873     }
03874 
03875     // Pattern: (X86setcc:i8 0:i8)
03876     // Emits: (SETAr:i8)
03877     // Pattern complexity = 5  cost = 1  size = 3
03878     if (CN0 == 0) {
03879       Emit_37(Result, N, X86::SETAr, MVT::i8, N0);
03880       return;
03881     }
03882 
03883     // Pattern: (X86setcc:i8 15:i8)
03884     // Emits: (SETSr:i8)
03885     // Pattern complexity = 5  cost = 1  size = 3
03886     if (CN0 == 15) {
03887       Emit_37(Result, N, X86::SETSr, MVT::i8, N0);
03888       return;
03889     }
03890 
03891     // Pattern: (X86setcc:i8 12:i8)
03892     // Emits: (SETNSr:i8)
03893     // Pattern complexity = 5  cost = 1  size = 3
03894     if (CN0 == 12) {
03895       Emit_37(Result, N, X86::SETNSr, MVT::i8, N0);
03896       return;
03897     }
03898 
03899     // Pattern: (X86setcc:i8 14:i8)
03900     // Emits: (SETPr:i8)
03901     // Pattern complexity = 5  cost = 1  size = 3
03902     if (CN0 == 14) {
03903       Emit_37(Result, N, X86::SETPr, MVT::i8, N0);
03904       return;
03905     }
03906 
03907     // Pattern: (X86setcc:i8 11:i8)
03908     // Emits: (SETNPr:i8)
03909     // Pattern complexity = 5  cost = 1
03910     if (CN0 == 11) {
03911       Emit_37(Result, N, X86::SETNPr, MVT::i8, N0);
03912       return;
03913     }
03914   }
03915   std::cerr << "Cannot yet select: ";
03916   N.Val->dump(CurDAG);
03917   std::cerr << '\n';
03918   abort();
03919 }
03920 
03921 void Emit_38(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
03922   SDOperand Tmp0(0, 0);
03923   SDOperand Tmp1(0, 0);
03924   SDOperand Tmp2(0, 0);
03925   SDNode *ResNode = NULL;
03926   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03927   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03928   Select(Tmp0, N0);
03929   Select(Tmp1, N1);
03930   Tmp2 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N2)->getValue()), MVT::i8);
03931   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03932   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03933   if (N.Val->hasOneUse()) {
03934     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1, Tmp2);
03935   } else {
03936     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1, Tmp2);
03937     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03938     Result = SDOperand(ResNode, 0);
03939   }
03940   return;
03941 }
03942 void Emit_39(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &Chain, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
03943   SDOperand InFlag(0, 0);
03944   SDOperand Tmp0(0, 0);
03945   SDOperand Tmp1(0, 0);
03946   SDNode *ResNode = NULL;
03947   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
03948   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
03949   Select(Tmp0, N0);
03950   Select(Tmp1, N1);
03951   Chain = CurDAG->getEntryNode();
03952   Select(N2, N2);
03953   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N2, InFlag).Val;
03954   Chain = SDOperand(ResNode, 0);
03955   InFlag = SDOperand(ResNode, 1);
03956   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
03957   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
03958   if (N.Val->hasOneUse()) {
03959     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1, InFlag);
03960   } else {
03961     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1, InFlag);
03962     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03963     Result = SDOperand(ResNode, 0);
03964   }
03965   return;
03966 }
03967 void Select_X86shld(SDOperand &Result, SDOperand N) {
03968   SDOperand Chain(0, 0);
03969   SDOperand InFlag(0, 0);
03970   SDOperand N0(0, 0);
03971   SDOperand N1(0, 0);
03972   SDOperand N2(0, 0);
03973   SDOperand Tmp0(0, 0);
03974   SDOperand Tmp1(0, 0);
03975   SDOperand Tmp2(0, 0);
03976   SDNode *ResNode = NULL;
03977   N0 = N.getOperand(0);
03978   N1 = N.getOperand(1);
03979   N2 = N.getOperand(2);
03980   if (N2.getOpcode() == ISD::Constant) {
03981 
03982     // Pattern: (X86shld:i32 GR32:i32:$src1, GR32:i32:$src2, (imm:i8):$src3)
03983     // Emits: (SHLD32rri8:i32 GR32:i32:$src1, GR32:i32:$src2, (imm:i8):$src3)
03984     // Pattern complexity = 4  cost = 1  size = 3
03985     if (N.Val->getValueType(0) == MVT::i32 &&
03986         N2.Val->getValueType(0) == MVT::i8) {
03987       Emit_38(Result, N, X86::SHLD32rri8, MVT::i32, N0, N1, N2);
03988       return;
03989     }
03990 
03991     // Pattern: (X86shld:i16 GR16:i16:$src1, GR16:i16:$src2, (imm:i8):$src3)
03992     // Emits: (SHLD16rri8:i16 GR16:i16:$src1, GR16:i16:$src2, (imm:i8):$src3)
03993     // Pattern complexity = 4  cost = 1
03994     if (N.Val->getValueType(0) == MVT::i16 &&
03995         N2.Val->getValueType(0) == MVT::i8) {
03996       Emit_38(Result, N, X86::SHLD16rri8, MVT::i16, N0, N1, N2);
03997       return;
03998     }
03999   }
04000 
04001   // Pattern: (X86shld:i32 GR32:i32:$src1, GR32:i32:$src2, CL:i8)
04002   // Emits: (SHLD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
04003   // Pattern complexity = 2  cost = 1  size = 3
04004   if (N.Val->getValueType(0) == MVT::i32 &&
04005       N2.Val->getValueType(0) == MVT::i8) {
04006     Emit_39(Result, N, X86::SHLD32rrCL, MVT::i32, Chain, N0, N1, N2);
04007     return;
04008   }
04009 
04010   // Pattern: (X86shld:i16 GR16:i16:$src1, GR16:i16:$src2, CL:i8)
04011   // Emits: (SHLD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
04012   // Pattern complexity = 2  cost = 1
04013   if (N.Val->getValueType(0) == MVT::i16 &&
04014       N2.Val->getValueType(0) == MVT::i8) {
04015     Emit_39(Result, N, X86::SHLD16rrCL, MVT::i16, Chain, N0, N1, N2);
04016     return;
04017   }
04018   std::cerr << "Cannot yet select: ";
04019   N.Val->dump(CurDAG);
04020   std::cerr << '\n';
04021   abort();
04022 }
04023 
04024 void Select_X86shrd(SDOperand &Result, SDOperand N) {
04025   SDOperand Chain(0, 0);
04026   SDOperand InFlag(0, 0);
04027   SDOperand N0(0, 0);
04028   SDOperand N1(0, 0);
04029   SDOperand N2(0, 0);
04030   SDOperand Tmp0(0, 0);
04031   SDOperand Tmp1(0, 0);
04032   SDOperand Tmp2(0, 0);
04033   SDNode *ResNode = NULL;
04034   N0 = N.getOperand(0);
04035   N1 = N.getOperand(1);
04036   N2 = N.getOperand(2);
04037   if (N2.getOpcode() == ISD::Constant) {
04038 
04039     // Pattern: (X86shrd:i32 GR32:i32:$src1, GR32:i32:$src2, (imm:i8):$src3)
04040     // Emits: (SHRD32rri8:i32 GR32:i32:$src1, GR32:i32:$src2, (imm:i8):$src3)
04041     // Pattern complexity = 4  cost = 1  size = 3
04042     if (N.Val->getValueType(0) == MVT::i32 &&
04043         N2.Val->getValueType(0) == MVT::i8) {
04044       Emit_38(Result, N, X86::SHRD32rri8, MVT::i32, N0, N1, N2);
04045       return;
04046     }
04047 
04048     // Pattern: (X86shrd:i16 GR16:i16:$src1, GR16:i16:$src2, (imm:i8):$src3)
04049     // Emits: (SHRD16rri8:i16 GR16:i16:$src1, GR16:i16:$src2, (imm:i8):$src3)
04050     // Pattern complexity = 4  cost = 1
04051     if (N.Val->getValueType(0) == MVT::i16 &&
04052         N2.Val->getValueType(0) == MVT::i8) {
04053       Emit_38(Result, N, X86::SHRD16rri8, MVT::i16, N0, N1, N2);
04054       return;
04055     }
04056   }
04057 
04058   // Pattern: (X86shrd:i32 GR32:i32:$src1, GR32:i32:$src2, CL:i8)
04059   // Emits: (SHRD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
04060   // Pattern complexity = 2  cost = 1  size = 3
04061   if (N.Val->getValueType(0) == MVT::i32 &&
04062       N2.Val->getValueType(0) == MVT::i8) {
04063     Emit_39(Result, N, X86::SHRD32rrCL, MVT::i32, Chain, N0, N1, N2);
04064     return;
04065   }
04066 
04067   // Pattern: (X86shrd:i16 GR16:i16:$src1, GR16:i16:$src2, CL:i8)
04068   // Emits: (SHRD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
04069   // Pattern complexity = 2  cost = 1
04070   if (N.Val->getValueType(0) == MVT::i16 &&
04071       N2.Val->getValueType(0) == MVT::i8) {
04072     Emit_39(Result, N, X86::SHRD16rrCL, MVT::i16, Chain, N0, N1, N2);
04073     return;
04074   }
04075   std::cerr << "Cannot yet select: ";
04076   N.Val->dump(CurDAG);
04077   std::cerr << '\n';
04078   abort();
04079 }
04080 
04081 void Select_X86tailcall(SDOperand &Result, SDOperand N) {
04082   SDOperand Chain(0, 0);
04083   SDOperand InFlag(0, 0);
04084   SDOperand N1(0, 0);
04085   SDOperand Tmp0(0, 0);
04086   SDNode *ResNode = NULL;
04087   bool HasInFlag = false;
04088   {
04089     Chain = N.getOperand(0);
04090     N1 = N.getOperand(1);
04091 
04092     // Pattern: (X86tailcall:void (tglobaladdr:i32):$dst)
04093     // Emits: (CALLpcrel32:void (tglobaladdr:i32):$dst)
04094     // Pattern complexity = 4  cost = 1  size = 3
04095     if (N1.getOpcode() == ISD::TargetGlobalAddress) {
04096       Emit_3(Result, N, X86::CALLpcrel32, Chain, N1, HasInFlag);
04097       return;
04098     }
04099 
04100     // Pattern: (X86tailcall:void (texternalsym:i32):$dst)
04101     // Emits: (CALLpcrel32:void (texternalsym:i32):$dst)
04102     // Pattern complexity = 4  cost = 1
04103     if (N1.getOpcode() == ISD::TargetExternalSymbol) {
04104       Emit_3(Result, N, X86::CALLpcrel32, Chain, N1, HasInFlag);
04105       return;
04106     }
04107   }
04108 
04109   // Pattern: (X86tailcall:void GR32:i32:$dst)
04110   // Emits: (CALL32r:void GR32:i32:$dst)
04111   // Pattern complexity = 2  cost = 1
04112   Emit_4(Result, N, X86::CALL32r, Chain, N1, HasInFlag);
04113   return;
04114 }
04115 
04116 void Select_X86test(SDOperand &Result, SDOperand N) {
04117   SDOperand CPTmp0(0, 0);
04118   SDOperand CPTmp1(0, 0);
04119   SDOperand CPTmp2(0, 0);
04120   SDOperand CPTmp3(0, 0);
04121   SDOperand Chain0(0, 0);
04122   SDOperand Chain1(0, 0);
04123   SDOperand InFlag(0, 0);
04124   SDOperand N0(0, 0);
04125   SDOperand N01(0, 0);
04126   SDOperand N1(0, 0);
04127   SDOperand N11(0, 0);
04128   SDOperand Tmp0(0, 0);
04129   SDOperand Tmp1(0, 0);
04130   SDOperand Tmp2(0, 0);
04131   SDOperand Tmp3(0, 0);
04132   SDOperand Tmp4(0, 0);
04133   SDNode *ResNode = NULL;
04134   N0 = N.getOperand(0);
04135   if (N0.getOpcode() == ISD::LOAD &&
04136       InFlightSet.count(N0.Val) == 0 &&
04137       N0.hasOneUse() &&
04138       !CodeGenMap.count(N0.getValue(0)) &&
04139       N.Val->isOnlyUse(N0.Val) &&
04140       !isNonImmUse(N.Val, N0.Val)) {
04141     Chain0 = N0.getOperand(0);
04142     N01 = N0.getOperand(1);
04143     N1 = N.getOperand(1);
04144     if (N1.getOpcode() == ISD::Constant) {
04145 
04146       // Pattern: (X86test:void (load:i8 addr:iPTR:$src1), (imm:i8):$src2)
04147       // Emits: (TEST8mi:void addr:iPTR:$src1, (imm:i8):$src2)
04148       // Pattern complexity = 16  cost = 1  size = 3
04149       if (N0.Val->getValueType(0) == MVT::i8) {
04150         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04151         if (Match) {
04152           Emit_12(Result, N, X86::TEST8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04153           return;
04154         }
04155       }
04156 
04157       // Pattern: (X86test:void (load:i16 addr:iPTR:$src1), (imm:i16):$src2)
04158       // Emits: (TEST16mi:void addr:iPTR:$src1, (imm:i16):$src2)
04159       // Pattern complexity = 16  cost = 1  size = 3
04160       if (N0.Val->getValueType(0) == MVT::i16) {
04161         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04162         if (Match) {
04163           Emit_10(Result, N, X86::TEST16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04164           return;
04165         }
04166       }
04167 
04168       // Pattern: (X86test:void (load:i32 addr:iPTR:$src1), (imm:i32):$src2)
04169       // Emits: (TEST32mi:void addr:iPTR:$src1, (imm:i32):$src2)
04170       // Pattern complexity = 16  cost = 1
04171       if (N0.Val->getValueType(0) == MVT::i32) {
04172         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04173         if (Match) {
04174           Emit_11(Result, N, X86::TEST32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04175           return;
04176         }
04177       }
04178     }
04179 
04180     // Pattern: (X86test:void (load:i8 addr:iPTR:$src1), GR8:i8:$src2)
04181     // Emits: (TEST8mr:void addr:iPTR:$src1, GR8:i8:$src2)
04182     // Pattern complexity = 14  cost = 1  size = 3
04183     if (N0.Val->getValueType(0) == MVT::i8) {
04184       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04185       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04186       if (!Match) {
04187         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04188       }
04189       if (Match) {
04190         Emit_13(Result, N, X86::TEST8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04191         return;
04192       }
04193     }
04194 
04195     // Pattern: (X86test:void (load:i16 addr:iPTR:$src1), GR16:i16:$src2)
04196     // Emits: (TEST16mr:void addr:iPTR:$src1, GR16:i16:$src2)
04197     // Pattern complexity = 14  cost = 1  size = 3
04198     if (N0.Val->getValueType(0) == MVT::i16) {
04199       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04200       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04201       if (!Match) {
04202         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04203       }
04204       if (Match) {
04205         Emit_13(Result, N, X86::TEST16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04206         return;
04207       }
04208     }
04209 
04210     // Pattern: (X86test:void (load:i32 addr:iPTR:$src1), GR32:i32:$src2)
04211     // Emits: (TEST32mr:void addr:iPTR:$src1, GR32:i32:$src2)
04212     // Pattern complexity = 14  cost = 1
04213     if (N0.Val->getValueType(0) == MVT::i32) {
04214       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04215       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04216       if (!Match) {
04217         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04218       }
04219       if (Match) {
04220         Emit_13(Result, N, X86::TEST32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04221         return;
04222       }
04223     }
04224   }
04225   N1 = N.getOperand(1);
04226   if (N1.getOpcode() == ISD::LOAD &&
04227       InFlightSet.count(N1.Val) == 0 &&
04228       N1.hasOneUse() &&
04229       !CodeGenMap.count(N1.getValue(0)) &&
04230       N.Val->isOnlyUse(N1.Val) &&
04231       !isNonImmUse(N.Val, N1.Val)) {
04232     Chain1 = N1.getOperand(0);
04233     N11 = N1.getOperand(1);
04234 
04235     // Pattern: (X86test:void GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
04236     // Emits: (TEST8rm:void GR8:i8:$src1, addr:iPTR:$src2)
04237     // Pattern complexity = 14  cost = 1  size = 3
04238     if (N0.Val->getValueType(0) == MVT::i8) {
04239       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04240       bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04241       if (!Match) {
04242         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04243       }
04244       if (Match) {
04245         Emit_14(Result, N, X86::TEST8rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04246         return;
04247       }
04248     }
04249 
04250     // Pattern: (X86test:void GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
04251     // Emits: (TEST16rm:void GR16:i16:$src1, addr:iPTR:$src2)
04252     // Pattern complexity = 14  cost = 1  size = 3
04253     if (N0.Val->getValueType(0) == MVT::i16) {
04254       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04255       bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04256       if (!Match) {
04257         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04258       }
04259       if (Match) {
04260         Emit_14(Result, N, X86::TEST16rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04261         return;
04262       }
04263     }
04264 
04265     // Pattern: (X86test:void GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
04266     // Emits: (TEST32rm:void GR32:i32:$src1, addr:iPTR:$src2)
04267     // Pattern complexity = 14  cost = 1
04268     if (N0.Val->getValueType(0) == MVT::i32) {
04269       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04270       bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04271       if (!Match) {
04272         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04273       }
04274       if (Match) {
04275         Emit_14(Result, N, X86::TEST32rm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04276         return;
04277       }
04278     }
04279   }
04280   if (N1.getOpcode() == ISD::Constant) {
04281 
04282     // Pattern: (X86test:void GR8:i8:$src1, (imm:i8):$src2)
04283     // Emits: (TEST8ri:void GR8:i8:$src1, (imm:i8):$src2)
04284     // Pattern complexity = 4  cost = 1  size = 3
04285     if (N0.Val->getValueType(0) == MVT::i8) {
04286       Emit_17(Result, N, X86::TEST8ri, N0, N1);
04287       return;
04288     }
04289 
04290     // Pattern: (X86test:void GR16:i16:$src1, (imm:i16):$src2)
04291     // Emits: (TEST16ri:void GR16:i16:$src1, (imm:i16):$src2)
04292     // Pattern complexity = 4  cost = 1  size = 3
04293     if (N0.Val->getValueType(0) == MVT::i16) {
04294       Emit_15(Result, N, X86::TEST16ri, N0, N1);
04295       return;
04296     }
04297 
04298     // Pattern: (X86test:void GR32:i32:$src1, (imm:i32):$src2)
04299     // Emits: (TEST32ri:void GR32:i32:$src1, (imm:i32):$src2)
04300     // Pattern complexity = 4  cost = 1
04301     if (N0.Val->getValueType(0) == MVT::i32) {
04302       Emit_16(Result, N, X86::TEST32ri, N0, N1);
04303       return;
04304     }
04305   }
04306 
04307   // Pattern: (X86test:void GR8:i8:$src1, GR8:i8:$src2)
04308   // Emits: (TEST8rr:void GR8:i8:$src1, GR8:i8:$src2)
04309   // Pattern complexity = 2  cost = 1  size = 3
04310   if (N0.Val->getValueType(0) == MVT::i8) {
04311     Emit_18(Result, N, X86::TEST8rr, N0, N1);
04312     return;
04313   }
04314 
04315   // Pattern: (X86test:void GR16:i16:$src1, GR16:i16:$src2)
04316   // Emits: (TEST16rr:void GR16:i16:$src1, GR16:i16:$src2)
04317   // Pattern complexity = 2  cost = 1  size = 3
04318   if (N0.Val->getValueType(0) == MVT::i16) {
04319     Emit_18(Result, N, X86::TEST16rr, N0, N1);
04320     return;
04321   }
04322 
04323   // Pattern: (X86test:void GR32:i32:$src1, GR32:i32:$src2)
04324   // Emits: (TEST32rr:void GR32:i32:$src1, GR32:i32:$src2)
04325   // Pattern complexity = 2  cost = 1
04326   if (N0.Val->getValueType(0) == MVT::i32) {
04327     Emit_18(Result, N, X86::TEST32rr, N0, N1);
04328     return;
04329   }
04330   std::cerr << "Cannot yet select: ";
04331   N.Val->dump(CurDAG);
04332   std::cerr << '\n';
04333   abort();
04334 }
04335 
04336 void Select_X86ucomi(SDOperand &Result, SDOperand N) {
04337   SDOperand CPTmp0(0, 0);
04338   SDOperand CPTmp1(0, 0);
04339   SDOperand CPTmp2(0, 0);
04340   SDOperand CPTmp3(0, 0);
04341   SDOperand Chain1(0, 0);
04342   SDOperand InFlag(0, 0);
04343   SDOperand N0(0, 0);
04344   SDOperand N1(0, 0);
04345   SDOperand N11(0, 0);
04346   SDOperand Tmp0(0, 0);
04347   SDOperand Tmp1(0, 0);
04348   SDOperand Tmp2(0, 0);
04349   SDOperand Tmp3(0, 0);
04350   SDOperand Tmp4(0, 0);
04351   SDNode *ResNode = NULL;
04352 
04353   // Pattern: (X86ucomi:void VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
04354   // Emits: (Int_UCOMISSrm:void VR128:v4f32:$src1, addr:iPTR:$src2)
04355   // Pattern complexity = 14  cost = 1  size = 3
04356   if ((Subtarget->hasSSE1())) {
04357     N0 = N.getOperand(0);
04358     N1 = N.getOperand(1);
04359     if (N1.getOpcode() == ISD::LOAD &&
04360         InFlightSet.count(N1.Val) == 0 &&
04361         N1.hasOneUse() &&
04362         !CodeGenMap.count(N1.getValue(0)) &&
04363         N.Val->isOnlyUse(N1.Val) &&
04364         !isNonImmUse(N.Val, N1.Val)) {
04365       Chain1 = N1.getOperand(0);
04366       N11 = N1.getOperand(1);
04367       if (N0.Val->getValueType(0) == MVT::v4f32) {
04368         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04369         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04370         if (!Match) {
04371           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04372         }
04373         if (Match) {
04374           Emit_14(Result, N, X86::Int_UCOMISSrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04375           return;
04376         }
04377       }
04378     }
04379   }
04380 
04381   // Pattern: (X86ucomi:void VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
04382   // Emits: (Int_UCOMISDrm:void VR128:v2f64:$src1, addr:iPTR:$src2)
04383   // Pattern complexity = 14  cost = 1  size = 3
04384   if ((Subtarget->hasSSE2())) {
04385     N0 = N.getOperand(0);
04386     N1 = N.getOperand(1);
04387     if (N1.getOpcode() == ISD::LOAD &&
04388         InFlightSet.count(N1.Val) == 0 &&
04389         N1.hasOneUse() &&
04390         !CodeGenMap.count(N1.getValue(0)) &&
04391         N.Val->isOnlyUse(N1.Val) &&
04392         !isNonImmUse(N.Val, N1.Val)) {
04393       Chain1 = N1.getOperand(0);
04394       N11 = N1.getOperand(1);
04395       if (N0.Val->getValueType(0) == MVT::v2f64) {
04396         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04397         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04398         if (!Match) {
04399           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04400         }
04401         if (Match) {
04402           Emit_14(Result, N, X86::Int_UCOMISDrm, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04403           return;
04404         }
04405       }
04406     }
04407   }
04408 
04409   // Pattern: (X86ucomi:void VR128:v4f32:$src1, VR128:v4f32:$src2)
04410   // Emits: (Int_UCOMISSrr:void VR128:v4f32:$src1, VR128:v4f32:$src2)
04411   // Pattern complexity = 2  cost = 1  size = 3
04412   if ((Subtarget->hasSSE1())) {
04413     N0 = N.getOperand(0);
04414     N1 = N.getOperand(1);
04415     if (N0.Val->getValueType(0) == MVT::v4f32) {
04416       Emit_18(Result, N, X86::Int_UCOMISSrr, N0, N1);
04417       return;
04418     }
04419   }
04420 
04421   // Pattern: (X86ucomi:void VR128:v2f64:$src1, VR128:v2f64:$src2)
04422   // Emits: (Int_UCOMISDrr:void VR128:v2f64:$src1, VR128:v2f64:$src2)
04423   // Pattern complexity = 2  cost = 1
04424   if ((Subtarget->hasSSE2())) {
04425     N0 = N.getOperand(0);
04426     N1 = N.getOperand(1);
04427     if (N0.Val->getValueType(0) == MVT::v2f64) {
04428       Emit_18(Result, N, X86::Int_UCOMISDrr, N0, N1);
04429       return;
04430     }
04431   }
04432   std::cerr << "Cannot yet select: ";
04433   N.Val->dump(CurDAG);
04434   std::cerr << '\n';
04435   abort();
04436 }
04437 
04438 void Emit_40(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
04439   SDOperand Tmp0(0, 0);
04440   SDOperand Tmp1(0, 0);
04441   SDOperand Tmp2(0, 0);
04442   SDOperand Tmp3(0, 0);
04443   SDOperand Tmp4(0, 0);
04444   SDNode *ResNode = NULL;
04445   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
04446   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
04447   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
04448   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
04449   Select(Tmp0, CPTmp0);
04450   Select(Tmp1, CPTmp1);
04451   Select(Tmp2, CPTmp2);
04452   Select(Tmp3, CPTmp3);
04453   Select(Tmp4, N0);
04454   Select(Chain10, Chain10);
04455   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04456   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
04457   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
04458   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
04459   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
04460   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
04461   Chain10 = SDOperand(ResNode, 1);
04462   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04463   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
04464   AddHandleReplacement(N10.Val, 1, ResNode, 1);
04465   Result = SDOperand(ResNode, N.ResNo);
04466   return;
04467 }
04468 void Emit_41(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1) NOINLINE {
04469   SDOperand Tmp0(0, 0);
04470   SDOperand Tmp1(0, 0);
04471   SDOperand Tmp2(0, 0);
04472   SDOperand Tmp3(0, 0);
04473   SDOperand Tmp4(0, 0);
04474   SDNode *ResNode = NULL;
04475   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
04476   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
04477   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
04478   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
04479   Select(Tmp0, CPTmp0);
04480   Select(Tmp1, CPTmp1);
04481   Select(Tmp2, CPTmp2);
04482   Select(Tmp3, CPTmp3);
04483   Select(Tmp4, N1);
04484   Select(Chain00, Chain00);
04485   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04486   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
04487   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
04488   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
04489   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
04490   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
04491   Chain00 = SDOperand(ResNode, 1);
04492   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04493   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
04494   AddHandleReplacement(N00.Val, 1, ResNode, 1);
04495   Result = SDOperand(ResNode, N.ResNo);
04496   return;
04497 }
04498 void Emit_42(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3) NOINLINE {
04499   SDOperand Tmp0(0, 0);
04500   SDOperand Tmp1(0, 0);
04501   SDOperand Tmp2(0, 0);
04502   SDOperand Tmp3(0, 0);
04503   SDNode *ResNode = NULL;
04504   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
04505   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
04506   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
04507   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
04508   Select(Tmp0, CPTmp0);
04509   Select(Tmp1, CPTmp1);
04510   Select(Tmp2, CPTmp2);
04511   Select(Tmp3, CPTmp3);
04512   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
04513   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
04514   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
04515   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
04516   if (N.Val->hasOneUse()) {
04517     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1, Tmp2, Tmp3);
04518   } else {
04519     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1, Tmp2, Tmp3);
04520     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04521     Result = SDOperand(ResNode, 0);
04522   }
04523   return;
04524 }
04525 void Emit_43(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10) NOINLINE {
04526   SDOperand Tmp0(0, 0);
04527   SDOperand Tmp1(0, 0);
04528   SDNode *ResNode = NULL;
04529   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04530   Select(Tmp0, N0);
04531   Tmp1 = N10;
04532   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04533   if (N.Val->hasOneUse()) {
04534     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04535   } else {
04536     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04537     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04538     Result = SDOperand(ResNode, 0);
04539   }
04540   return;
04541 }
04542 void Emit_44(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10) NOINLINE {
04543   SDOperand Tmp0(0, 0);
04544   SDOperand Tmp1(0, 0);
04545   SDNode *ResNode = NULL;
04546   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04547   Select(Tmp0, N0);
04548   Select(Tmp1, N10);
04549   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04550   if (N.Val->hasOneUse()) {
04551     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04552   } else {
04553     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04554     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04555     Result = SDOperand(ResNode, 0);
04556   }
04557   return;
04558 }
04559 void Emit_45(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N1) NOINLINE {
04560   SDOperand Tmp0(0, 0);
04561   SDOperand Tmp1(0, 0);
04562   SDNode *ResNode = NULL;
04563   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04564   Select(Tmp0, N1);
04565   Tmp1 = N00;
04566   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04567   if (N.Val->hasOneUse()) {
04568     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04569   } else {
04570     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04571     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04572     Result = SDOperand(ResNode, 0);
04573   }
04574   return;
04575 }
04576 void Emit_46(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N1) NOINLINE {
04577   SDOperand Tmp0(0, 0);
04578   SDOperand Tmp1(0, 0);
04579   SDNode *ResNode = NULL;
04580   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04581   Select(Tmp0, N1);
04582   Select(Tmp1, N00);
04583   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04584   if (N.Val->hasOneUse()) {
04585     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04586   } else {
04587     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04588     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04589     Result = SDOperand(ResNode, 0);
04590   }
04591   return;
04592 }
04593 void Emit_47(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
04594   SDOperand Tmp1(0, 0);
04595   SDNode *ResNode = NULL;
04596   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04597   Select(Tmp1, N0);
04598   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04599   if (N.Val->hasOneUse()) {
04600     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1);
04601   } else {
04602     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1);
04603     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04604     Result = SDOperand(ResNode, 0);
04605   }
04606   return;
04607 }
04608 void Emit_48(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
04609   SDOperand Tmp1(0, 0);
04610   SDNode *ResNode = NULL;
04611   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04612   Select(Tmp1, N1);
04613   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04614   if (N.Val->hasOneUse()) {
04615     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1);
04616   } else {
04617     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1);
04618     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04619     Result = SDOperand(ResNode, 0);
04620   }
04621   return;
04622 }
04623 void Emit_49(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
04624   SDOperand Tmp0(0, 0);
04625   SDOperand Tmp1(0, 0);
04626   SDNode *ResNode = NULL;
04627   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04628   Select(Tmp0, N0);
04629   Tmp1 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
04630   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04631   if (N.Val->hasOneUse()) {
04632     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04633   } else {
04634     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04635     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04636     Result = SDOperand(ResNode, 0);
04637   }
04638   return;
04639 }
04640 void Emit_50(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
04641   SDOperand Tmp0(0, 0);
04642   SDOperand Tmp1(0, 0);
04643   SDNode *ResNode = NULL;
04644   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04645   Select(Tmp0, N0);
04646   Tmp1 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N1)->getValue()), MVT::i8);
04647   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04648   if (N.Val->hasOneUse()) {
04649     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
04650   } else {
04651     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
04652     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04653     Result = SDOperand(ResNode, 0);
04654   }
04655   return;
04656 }
04657 void Select_add(SDOperand &Result, SDOperand N) {
04658   SDOperand CPTmp0(0, 0);
04659   SDOperand CPTmp1(0, 0);
04660   SDOperand CPTmp2(0, 0);
04661   SDOperand CPTmp3(0, 0);
04662   SDOperand Chain0(0, 0);
04663   SDOperand Chain00(0, 0);
04664   SDOperand Chain1(0, 0);
04665   SDOperand Chain10(0, 0);
04666   SDOperand N0(0, 0);
04667   SDOperand N00(0, 0);
04668   SDOperand N001(0, 0);
04669   SDOperand N01(0, 0);
04670   SDOperand N1(0, 0);
04671   SDOperand N10(0, 0);
04672   SDOperand N101(0, 0);
04673   SDOperand N11(0, 0);
04674   SDOperand Tmp0(0, 0);
04675   SDOperand Tmp1(0, 0);
04676   SDOperand Tmp2(0, 0);
04677   SDOperand Tmp3(0, 0);
04678   SDOperand Tmp4(0, 0);
04679   SDNode *ResNode = NULL;
04680   if ((Subtarget->hasSSE2())) {
04681     N0 = N.getOperand(0);
04682     {
04683       N1 = N.getOperand(1);
04684       if (N1.getOpcode() == ISD::BIT_CONVERT &&
04685           InFlightSet.count(N1.Val) == 0 &&
04686           N1.hasOneUse() &&
04687           !CodeGenMap.count(N1.getValue(0))) {
04688         N10 = N1.getOperand(0);
04689         if (N10.getOpcode() == ISD::LOAD &&
04690             InFlightSet.count(N10.Val) == 0 &&
04691             N10.hasOneUse() &&
04692             !CodeGenMap.count(N10.getValue(0))) {
04693           Chain10 = N10.getOperand(0);
04694           N101 = N10.getOperand(1);
04695 
04696           // Pattern: (add:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
04697           // Emits: (PADDBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
04698           // Pattern complexity = 16  cost = 1  size = 3
04699           if (N.Val->getValueType(0) == MVT::v16i8 &&
04700               N10.Val->getValueType(0) == MVT::v2i64) {
04701             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04702             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04703             if (!Match) {
04704               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04705             }
04706             if (Match) {
04707               Emit_40(Result, N, X86::PADDBrm, MVT::v16i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
04708               return;
04709             }
04710           }
04711 
04712           // Pattern: (add:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
04713           // Emits: (PADDWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
04714           // Pattern complexity = 16  cost = 1  size = 3
04715           if (N.Val->getValueType(0) == MVT::v8i16 &&
04716               N10.Val->getValueType(0) == MVT::v2i64) {
04717             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04718             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04719             if (!Match) {
04720               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04721             }
04722             if (Match) {
04723               Emit_40(Result, N, X86::PADDWrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
04724               return;
04725             }
04726           }
04727 
04728           // Pattern: (add:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
04729           // Emits: (PADDDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
04730           // Pattern complexity = 16  cost = 1
04731           if (N.Val->getValueType(0) == MVT::v4i32 &&
04732               N10.Val->getValueType(0) == MVT::v2i64) {
04733             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04734             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04735             if (!Match) {
04736               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04737             }
04738             if (Match) {
04739               Emit_40(Result, N, X86::PADDDrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
04740               return;
04741             }
04742           }
04743         }
04744       }
04745     }
04746     if (N0.getOpcode() == ISD::BIT_CONVERT &&
04747         InFlightSet.count(N0.Val) == 0 &&
04748         N0.hasOneUse() &&
04749         !CodeGenMap.count(N0.getValue(0))) {
04750       N00 = N0.getOperand(0);
04751       if (N00.getOpcode() == ISD::LOAD &&
04752           InFlightSet.count(N00.Val) == 0 &&
04753           N00.hasOneUse() &&
04754           !CodeGenMap.count(N00.getValue(0))) {
04755         Chain00 = N00.getOperand(0);
04756         N001 = N00.getOperand(1);
04757         N1 = N.getOperand(1);
04758 
04759         // Pattern: (add:v16i8 (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)), VR128:v16i8:$src1)
04760         // Emits: (PADDBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
04761         // Pattern complexity = 16  cost = 1  size = 3
04762         if (N.Val->getValueType(0) == MVT::v16i8 &&
04763             N00.Val->getValueType(0) == MVT::v2i64) {
04764           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04765           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04766           if (!Match) {
04767             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04768           }
04769           if (Match) {
04770             Emit_41(Result, N, X86::PADDBrm, MVT::v16i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
04771             return;
04772           }
04773         }
04774 
04775         // Pattern: (add:v8i16 (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)), VR128:v8i16:$src1)
04776         // Emits: (PADDWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
04777         // Pattern complexity = 16  cost = 1  size = 3
04778         if (N.Val->getValueType(0) == MVT::v8i16 &&
04779             N00.Val->getValueType(0) == MVT::v2i64) {
04780           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04781           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04782           if (!Match) {
04783             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04784           }
04785           if (Match) {
04786             Emit_41(Result, N, X86::PADDWrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
04787             return;
04788           }
04789         }
04790 
04791         // Pattern: (add:v4i32 (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), VR128:v4i32:$src1)
04792         // Emits: (PADDDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
04793         // Pattern complexity = 16  cost = 1
04794         if (N.Val->getValueType(0) == MVT::v4i32 &&
04795             N00.Val->getValueType(0) == MVT::v2i64) {
04796           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04797           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04798           if (!Match) {
04799             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04800           }
04801           if (Match) {
04802             Emit_41(Result, N, X86::PADDDrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
04803             return;
04804           }
04805         }
04806       }
04807     }
04808   }
04809   {
04810     N0 = N.getOperand(0);
04811     N1 = N.getOperand(1);
04812     if (N1.getOpcode() == ISD::LOAD &&
04813         InFlightSet.count(N1.Val) == 0 &&
04814         N1.hasOneUse() &&
04815         !CodeGenMap.count(N1.getValue(0)) &&
04816         !isNonImmUse(N.Val, N1.Val)) {
04817       Chain1 = N1.getOperand(0);
04818       N11 = N1.getOperand(1);
04819 
04820       // Pattern: (add:i8 GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
04821       // Emits: (ADD8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
04822       // Pattern complexity = 14  cost = 1  size = 3
04823       if (N.Val->getValueType(0) == MVT::i8) {
04824         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04825         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04826         if (!Match) {
04827           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04828         }
04829         if (Match) {
04830           Emit_19(Result, N, X86::ADD8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04831           return;
04832         }
04833       }
04834 
04835       // Pattern: (add:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
04836       // Emits: (ADD16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
04837       // Pattern complexity = 14  cost = 1  size = 3
04838       if (N.Val->getValueType(0) == MVT::i16) {
04839         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04840         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04841         if (!Match) {
04842           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04843         }
04844         if (Match) {
04845           Emit_19(Result, N, X86::ADD16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04846           return;
04847         }
04848       }
04849 
04850       // Pattern: (add:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
04851       // Emits: (ADD32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
04852       // Pattern complexity = 14  cost = 1
04853       if (N.Val->getValueType(0) == MVT::i32) {
04854         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04855         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04856         if (!Match) {
04857           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04858         }
04859         if (Match) {
04860           Emit_19(Result, N, X86::ADD32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04861           return;
04862         }
04863       }
04864     }
04865   }
04866 
04867   // Pattern: (add:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
04868   // Emits: (PADDQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
04869   // Pattern complexity = 14  cost = 1  size = 3
04870   if ((Subtarget->hasSSE2())) {
04871     N0 = N.getOperand(0);
04872     N1 = N.getOperand(1);
04873     if (N1.getOpcode() == ISD::LOAD &&
04874         InFlightSet.count(N1.Val) == 0 &&
04875         N1.hasOneUse() &&
04876         !CodeGenMap.count(N1.getValue(0)) &&
04877         !isNonImmUse(N.Val, N1.Val)) {
04878       Chain1 = N1.getOperand(0);
04879       N11 = N1.getOperand(1);
04880       if (N.Val->getValueType(0) == MVT::v2i64) {
04881         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
04882         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04883         if (!Match) {
04884           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
04885         }
04886         if (Match) {
04887           Emit_19(Result, N, X86::PADDQrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
04888           return;
04889         }
04890       }
04891     }
04892   }
04893   {
04894     N0 = N.getOperand(0);
04895     if (N0.getOpcode() == ISD::LOAD &&
04896         InFlightSet.count(N0.Val) == 0 &&
04897         N0.hasOneUse() &&
04898         !CodeGenMap.count(N0.getValue(0)) &&
04899         !isNonImmUse(N.Val, N0.Val)) {
04900       Chain0 = N0.getOperand(0);
04901       N01 = N0.getOperand(1);
04902       N1 = N.getOperand(1);
04903 
04904       // Pattern: (add:i8 (load:i8 addr:iPTR:$src2), GR8:i8:$src1)
04905       // Emits: (ADD8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
04906       // Pattern complexity = 14  cost = 1  size = 3
04907       if (N.Val->getValueType(0) == MVT::i8) {
04908         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04909         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04910         if (!Match) {
04911           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04912         }
04913         if (Match) {
04914           Emit_20(Result, N, X86::ADD8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04915           return;
04916         }
04917       }
04918 
04919       // Pattern: (add:i16 (load:i16 addr:iPTR:$src2), GR16:i16:$src1)
04920       // Emits: (ADD16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
04921       // Pattern complexity = 14  cost = 1  size = 3
04922       if (N.Val->getValueType(0) == MVT::i16) {
04923         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04924         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04925         if (!Match) {
04926           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04927         }
04928         if (Match) {
04929           Emit_20(Result, N, X86::ADD16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04930           return;
04931         }
04932       }
04933 
04934       // Pattern: (add:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
04935       // Emits: (ADD32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
04936       // Pattern complexity = 14  cost = 1
04937       if (N.Val->getValueType(0) == MVT::i32) {
04938         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04939         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04940         if (!Match) {
04941           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04942         }
04943         if (Match) {
04944           Emit_20(Result, N, X86::ADD32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04945           return;
04946         }
04947       }
04948     }
04949   }
04950 
04951   // Pattern: (add:v2i64 (load:v2i64 addr:iPTR:$src2), VR128:v2i64:$src1)
04952   // Emits: (PADDQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
04953   // Pattern complexity = 14  cost = 1  size = 3
04954   if ((Subtarget->hasSSE2())) {
04955     N0 = N.getOperand(0);
04956     if (N0.getOpcode() == ISD::LOAD &&
04957         InFlightSet.count(N0.Val) == 0 &&
04958         N0.hasOneUse() &&
04959         !CodeGenMap.count(N0.getValue(0)) &&
04960         !isNonImmUse(N.Val, N0.Val)) {
04961       Chain0 = N0.getOperand(0);
04962       N01 = N0.getOperand(1);
04963       N1 = N.getOperand(1);
04964       if (N.Val->getValueType(0) == MVT::v2i64) {
04965         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
04966         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04967         if (!Match) {
04968           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
04969         }
04970         if (Match) {
04971           Emit_20(Result, N, X86::PADDQrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
04972           return;
04973         }
04974       }
04975     }
04976   }
04977 
04978   // Pattern: leaaddr:i32:$src
04979   // Emits: (LEA32r:i32 leaaddr:i32:$src)
04980   // Pattern complexity = 10  cost = 1  size = 3
04981   if (N.Val->getValueType(0) == MVT::i32) {
04982     bool Match = SelectLEAAddr(N, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04983     if (Match) {
04984       Emit_42(Result, N, X86::LEA32r, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
04985       return;
04986     }
04987   }
04988   {
04989     N0 = N.getOperand(0);
04990     {
04991       N1 = N.getOperand(1);
04992       if (N1.getOpcode() == X86ISD::Wrapper) {
04993         N10 = N1.getOperand(0);
04994 
04995         // Pattern: (add:i32 GR32:i32:$src1, (X86Wrapper:i32 (tconstpool:i32):$src2))
04996         // Emits: (ADD32ri:i32 GR32:i32:$src1, (tconstpool:i32):$src2)
04997         // Pattern complexity = 6  cost = 1  size = 3
04998         if (N10.getOpcode() == ISD::TargetConstantPool &&
04999             N.Val->getValueType(0) == MVT::i32) {
05000           Emit_43(Result, N, X86::ADD32ri, MVT::i32, N0, N1, N10);
05001           return;
05002         }
05003 
05004         // Pattern: (add:i32 GR32:i32:$src1, (X86Wrapper:i32 (tjumptable:i32):$src2))
05005         // Emits: (ADD32ri:i32 GR32:i32:$src1, (tjumptable:i32):$src2)
05006         // Pattern complexity = 6  cost = 1  size = 3
05007         if (N10.getOpcode() == ISD::TargetJumpTable &&
05008             N.Val->getValueType(0) == MVT::i32) {
05009           Emit_44(Result, N, X86::ADD32ri, MVT::i32, N0, N1, N10);
05010           return;
05011         }
05012 
05013         // Pattern: (add:i32 GR32:i32:$src1, (X86Wrapper:i32 (tglobaladdr:i32):$src2))
05014         // Emits: (ADD32ri:i32 GR32:i32:$src1, (tglobaladdr:i32):$src2)
05015         // Pattern complexity = 6  cost = 1  size = 3
05016         if (N10.getOpcode() == ISD::TargetGlobalAddress &&
05017             N.Val->getValueType(0) == MVT::i32) {
05018           Emit_43(Result, N, X86::ADD32ri, MVT::i32, N0, N1, N10);
05019           return;
05020         }
05021 
05022         // Pattern: (add:i32 GR32:i32:$src1, (X86Wrapper:i32 (texternalsym:i32):$src2))
05023         // Emits: (ADD32ri:i32 GR32:i32:$src1, (texternalsym:i32):$src2)
05024         // Pattern complexity = 6  cost = 1
05025         if (N10.getOpcode() == ISD::TargetExternalSymbol &&
05026             N.Val->getValueType(0) == MVT::i32) {
05027           Emit_43(Result, N, X86::ADD32ri, MVT::i32, N0, N1, N10);
05028           return;
05029         }
05030       }
05031     }
05032     if (N0.getOpcode() == X86ISD::Wrapper) {
05033       N00 = N0.getOperand(0);
05034 
05035       // Pattern: (add:i32 (X86Wrapper:i32 (tconstpool:i32):$src2), GR32:i32:$src1)
05036       // Emits: (ADD32ri:i32 GR32:i32:$src1, (tconstpool:i32):$src2)
05037       // Pattern complexity = 6  cost = 1  size = 3
05038       if (N00.getOpcode() == ISD::TargetConstantPool) {
05039         N1 = N.getOperand(1);
05040         if (N.Val->getValueType(0) == MVT::i32) {
05041           Emit_45(Result, N, X86::ADD32ri, MVT::i32, N0, N00, N1);
05042           return;
05043         }
05044       }
05045 
05046       // Pattern: (add:i32 (X86Wrapper:i32 (tjumptable:i32):$src2), GR32:i32:$src1)
05047       // Emits: (ADD32ri:i32 GR32:i32:$src1, (tjumptable:i32):$src2)
05048       // Pattern complexity = 6  cost = 1  size = 3
05049       if (N00.getOpcode() == ISD::TargetJumpTable) {
05050         N1 = N.getOperand(1);
05051         if (N.Val->getValueType(0) == MVT::i32) {
05052           Emit_46(Result, N, X86::ADD32ri, MVT::i32, N0, N00, N1);
05053           return;
05054         }
05055       }
05056 
05057       // Pattern: (add:i32 (X86Wrapper:i32 (tglobaladdr:i32):$src2), GR32:i32:$src1)
05058       // Emits: (ADD32ri:i32 GR32:i32:$src1, (tglobaladdr:i32):$src2)
05059       // Pattern complexity = 6  cost = 1  size = 3
05060       if (N00.getOpcode() == ISD::TargetGlobalAddress) {
05061         N1 = N.getOperand(1);
05062         if (N.Val->getValueType(0) == MVT::i32) {
05063           Emit_45(Result, N, X86::ADD32ri, MVT::i32, N0, N00, N1);
05064           return;
05065         }
05066       }
05067 
05068       // Pattern: (add:i32 (X86Wrapper:i32 (texternalsym:i32):$src2), GR32:i32:$src1)
05069       // Emits: (ADD32ri:i32 GR32:i32:$src1, (texternalsym:i32):$src2)
05070       // Pattern complexity = 6  cost = 1
05071       if (N00.getOpcode() == ISD::TargetExternalSymbol) {
05072         N1 = N.getOperand(1);
05073         if (N.Val->getValueType(0) == MVT::i32) {
05074           Emit_45(Result, N, X86::ADD32ri, MVT::i32, N0, N00, N1);
05075           return;
05076         }
05077       }
05078     }
05079     {
05080       N1 = N.getOperand(1);
05081       if (isa<ConstantSDNode>(N1)) {
05082         int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
05083         if (CN0 == 1) {
05084 
05085           // Pattern: (add:i16 GR16:i16:$src, 1:i16)
05086           // Emits: (INC16r:i16 GR16:i16:$src)
05087           // Pattern complexity = 5  cost = 1  size = 1
05088           if (N.Val->getValueType(0) == MVT::i16) {
05089             Emit_47(Result, N, X86::INC16r, MVT::i16, N0, N1);
05090             return;
05091           }
05092 
05093           // Pattern: (add:i32 GR32:i32:$src, 1:i32)
05094           // Emits: (INC32r:i32 GR32:i32:$src)
05095           // Pattern complexity = 5  cost = 1
05096           if (N.Val->getValueType(0) == MVT::i32) {
05097             Emit_47(Result, N, X86::INC32r, MVT::i32, N0, N1);
05098             return;
05099           }
05100         }
05101         if (CN0 == -1) {
05102 
05103           // Pattern: (add:i16 GR16:i16:$src, -1:i16)
05104           // Emits: (DEC16r:i16 GR16:i16:$src)
05105           // Pattern complexity = 5  cost = 1  size = 1
05106           if (N.Val->getValueType(0) == MVT::i16) {
05107             Emit_47(Result, N, X86::DEC16r, MVT::i16, N0, N1);
05108             return;
05109           }
05110 
05111           // Pattern: (add:i32 GR32:i32:$src, -1:i32)
05112           // Emits: (DEC32r:i32 GR32:i32:$src)
05113           // Pattern complexity = 5  cost = 1
05114           if (N.Val->getValueType(0) == MVT::i32) {
05115             Emit_47(Result, N, X86::DEC32r, MVT::i32, N0, N1);
05116             return;
05117           }
05118         }
05119       }
05120     }
05121     if (isa<ConstantSDNode>(N0)) {
05122       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
05123       if (CN0 == 1) {
05124         N1 = N.getOperand(1);
05125 
05126         // Pattern: (add:i16 1:i16, GR16:i16:$src)
05127         // Emits: (INC16r:i16 GR16:i16:$src)
05128         // Pattern complexity = 5  cost = 1  size = 1
05129         if (N.Val->getValueType(0) == MVT::i16) {
05130           Emit_48(Result, N, X86::INC16r, MVT::i16, N0, N1);
05131           return;
05132         }
05133 
05134         // Pattern: (add:i32 1:i32, GR32:i32:$src)
05135         // Emits: (INC32r:i32 GR32:i32:$src)
05136         // Pattern complexity = 5  cost = 1
05137         if (N.Val->getValueType(0) == MVT::i32) {
05138           Emit_48(Result, N, X86::INC32r, MVT::i32, N0, N1);
05139           return;
05140         }
05141       }
05142       if (CN0 == -1) {
05143         N1 = N.getOperand(1);
05144 
05145         // Pattern: (add:i16 -1:i16, GR16:i16:$src)
05146         // Emits: (DEC16r:i16 GR16:i16:$src)
05147         // Pattern complexity = 5  cost = 1  size = 1
05148         if (N.Val->getValueType(0) == MVT::i16) {
05149           Emit_48(Result, N, X86::DEC16r, MVT::i16, N0, N1);
05150           return;
05151         }
05152 
05153         // Pattern: (add:i32 -1:i32, GR32:i32:$src)
05154         // Emits: (DEC32r:i32 GR32:i32:$src)
05155         // Pattern complexity = 5  cost = 1
05156         if (N.Val->getValueType(0) == MVT::i32) {
05157           Emit_48(Result, N, X86::DEC32r, MVT::i32, N0, N1);
05158           return;
05159         }
05160       }
05161     }
05162     {
05163       N1 = N.getOperand(1);
05164       if (isa<ConstantSDNode>(N1)) {
05165         int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
05166 
05167         // Pattern: (add:i8 GR8:i8:$src, 1:i8)
05168         // Emits: (INC8r:i8 GR8:i8:$src)
05169         // Pattern complexity = 5  cost = 1  size = 2
05170         if (CN0 == 1 &&
05171             N.Val->getValueType(0) == MVT::i8) {
05172           Emit_47(Result, N, X86::INC8r, MVT::i8, N0, N1);
05173           return;
05174         }
05175 
05176         // Pattern: (add:i8 GR8:i8:$src, -1:i8)
05177         // Emits: (DEC8r:i8 GR8:i8:$src)
05178         // Pattern complexity = 5  cost = 1
05179         if (CN0 == -1 &&
05180             N.Val->getValueType(0) == MVT::i8) {
05181           Emit_47(Result, N, X86::DEC8r, MVT::i8, N0, N1);
05182           return;
05183         }
05184       }
05185     }
05186     if (isa<ConstantSDNode>(N0)) {
05187       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
05188 
05189       // Pattern: (add:i8 1:i8, GR8:i8:$src)
05190       // Emits: (INC8r:i8 GR8:i8:$src)
05191       // Pattern complexity = 5  cost = 1  size = 2
05192       if (CN0 == 1) {
05193         N1 = N.getOperand(1);
05194         if (N.Val->getValueType(0) == MVT::i8) {
05195           Emit_48(Result, N, X86::INC8r, MVT::i8, N0, N1);
05196           return;
05197         }
05198       }
05199 
05200       // Pattern: (add:i8 -1:i8, GR8:i8:$src)
05201       // Emits: (DEC8r:i8 GR8:i8:$src)
05202       // Pattern complexity = 5  cost = 1
05203       if (CN0 == -1) {
05204         N1 = N.getOperand(1);
05205         if (N.Val->getValueType(0) == MVT::i8) {
05206           Emit_48(Result, N, X86::DEC8r, MVT::i8, N0, N1);
05207           return;
05208         }
05209       }
05210     }
05211     N1 = N.getOperand(1);
05212     if (N1.getOpcode() == ISD::Constant) {
05213 
05214       // Pattern: (add:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
05215       // Emits: (ADD16ri8:i16 GR16:i16:$src1, (imm:i16):$src2)
05216       // Pattern complexity = 5  cost = 1  size = 3
05217       if (Predicate_i16immSExt8(N1.Val) &&
05218           N.Val->getValueType(0) == MVT::i16) {
05219         Emit_49(Result, N, X86::ADD16ri8, MVT::i16, N0, N1);
05220         return;
05221       }
05222 
05223       // Pattern: (add:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
05224       // Emits: (ADD32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
05225       // Pattern complexity = 5  cost = 1  size = 3
05226       if (Predicate_i32immSExt8(N1.Val) &&
05227           N.Val->getValueType(0) == MVT::i32) {
05228         Emit_29(Result, N, X86::ADD32ri8, MVT::i32, N0, N1);
05229         return;
05230       }
05231 
05232       // Pattern: (add:i8 GR8:i8:$src1, (imm:i8):$src2)
05233       // Emits: (ADD8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
05234       // Pattern complexity = 4  cost = 1  size = 3
05235       if (N.Val->getValueType(0) == MVT::i8) {
05236         Emit_50(Result, N, X86::ADD8ri, MVT::i8, N0, N1);
05237         return;
05238       }
05239 
05240       // Pattern: (add:i16 GR16:i16:$src1, (imm:i16):$src2)
05241       // Emits: (ADD16ri:i16 GR16:i16:$src1, (imm:i16):$src2)
05242       // Pattern complexity = 4  cost = 1  size = 3
05243       if (N.Val->getValueType(0) == MVT::i16) {
05244         Emit_49(Result, N, X86::ADD16ri, MVT::i16, N0, N1);
05245         return;
05246       }
05247 
05248       // Pattern: (add:i32 GR32:i32:$src1, (imm:i32):$src2)
05249       // Emits: (ADD32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
05250       // Pattern complexity = 4  cost = 1
05251       if (N.Val->getValueType(0) == MVT::i32) {
05252         Emit_29(Result, N, X86::ADD32ri, MVT::i32, N0, N1);
05253         return;
05254       }
05255     }
05256 
05257     // Pattern: (add:i8 GR8:i8:$src1, GR8:i8:$src2)
05258     // Emits: (ADD8rr:i8 GR8:i8:$src1, GR8:i8:$src2)
05259     // Pattern complexity = 2  cost = 1  size = 3
05260     if (N.Val->getValueType(0) == MVT::i8) {
05261       Emit_21(Result, N, X86::ADD8rr, MVT::i8, N0, N1);
05262       return;
05263     }
05264 
05265     // Pattern: (add:i16 GR16:i16:$src1, GR16:i16:$src2)
05266     // Emits: (ADD16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
05267     // Pattern complexity = 2  cost = 1  size = 3
05268     if (N.Val->getValueType(0) == MVT::i16) {
05269       Emit_21(Result, N, X86::ADD16rr, MVT::i16, N0, N1);
05270       return;
05271     }
05272 
05273     // Pattern: (add:i32 GR32:i32:$src1, GR32:i32:$src2)
05274     // Emits: (ADD32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
05275     // Pattern complexity = 2  cost = 1
05276     if (N.Val->getValueType(0) == MVT::i32) {
05277       Emit_21(Result, N, X86::ADD32rr, MVT::i32, N0, N1);
05278       return;
05279     }
05280   }
05281   if ((Subtarget->hasSSE2())) {
05282     N0 = N.getOperand(0);
05283     N1 = N.getOperand(1);
05284 
05285     // Pattern: (add:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
05286     // Emits: (PADDBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
05287     // Pattern complexity = 2  cost = 1  size = 3
05288     if (N.Val->getValueType(0) == MVT::v16i8) {
05289       Emit_21(Result, N, X86::PADDBrr, MVT::v16i8, N0, N1);
05290       return;
05291     }
05292 
05293     // Pattern: (add:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
05294     // Emits: (PADDWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
05295     // Pattern complexity = 2  cost = 1  size = 3
05296     if (N.Val->getValueType(0) == MVT::v8i16) {
05297       Emit_21(Result, N, X86::PADDWrr, MVT::v8i16, N0, N1);
05298       return;
05299     }
05300 
05301     // Pattern: (add:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
05302     // Emits: (PADDDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
05303     // Pattern complexity = 2  cost = 1  size = 3
05304     if (N.Val->getValueType(0) == MVT::v4i32) {
05305       Emit_21(Result, N, X86::PADDDrr, MVT::v4i32, N0, N1);
05306       return;
05307     }
05308 
05309     // Pattern: (add:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
05310     // Emits: (PADDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
05311     // Pattern complexity = 2  cost = 1
05312     if (N.Val->getValueType(0) == MVT::v2i64) {
05313       Emit_21(Result, N, X86::PADDQrr, MVT::v2i64, N0, N1);
05314       return;
05315     }
05316   }
05317   std::cerr << "Cannot yet select: ";
05318   N.Val->dump(CurDAG);
05319   std::cerr << '\n';
05320   abort();
05321 }
05322 
05323 void Emit_51(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
05324   SDOperand InFlag(0, 0);
05325   SDOperand Tmp0(0, 0);
05326   SDOperand Tmp1(0, 0);
05327   SDOperand Tmp2(0, 0);
05328   SDOperand Tmp3(0, 0);
05329   SDOperand Tmp4(0, 0);
05330   SDNode *ResNode = NULL;
05331   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05332   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05333   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05334   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05335   Select(Tmp0, CPTmp0);
05336   Select(Tmp1, CPTmp1);
05337   Select(Tmp2, CPTmp2);
05338   Select(Tmp3, CPTmp3);
05339   Select(Tmp4, N0);
05340   Select(Chain1, Chain1);
05341   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05342   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05343   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05344   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05345   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05346   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05347   Chain1 = SDOperand(ResNode, 1);
05348   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05349   InFlag = SDOperand(ResNode, 2);
05350   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05351   AddHandleReplacement(N1.Val, 1, ResNode, 1);
05352   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05353   if (N.ResNo < 1)
05354     Result = SDOperand(ResNode, N.ResNo);
05355   else
05356     Result = SDOperand(ResNode, N.ResNo+1);
05357   return;
05358 }
05359 void Emit_52(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
05360   SDOperand InFlag(0, 0);
05361   SDOperand Tmp0(0, 0);
05362   SDOperand Tmp1(0, 0);
05363   SDOperand Tmp2(0, 0);
05364   SDOperand Tmp3(0, 0);
05365   SDOperand Tmp4(0, 0);
05366   SDNode *ResNode = NULL;
05367   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05368   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05369   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05370   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05371   Select(Tmp0, CPTmp0);
05372   Select(Tmp1, CPTmp1);
05373   Select(Tmp2, CPTmp2);
05374   Select(Tmp3, CPTmp3);
05375   Select(Tmp4, N1);
05376   Select(Chain0, Chain0);
05377   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05378   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05379   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05380   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05381   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05382   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05383   Chain0 = SDOperand(ResNode, 1);
05384   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05385   InFlag = SDOperand(ResNode, 2);
05386   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05387   AddHandleReplacement(N0.Val, 1, ResNode, 1);
05388   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05389   if (N.ResNo < 1)
05390     Result = SDOperand(ResNode, N.ResNo);
05391   else
05392     Result = SDOperand(ResNode, N.ResNo+1);
05393   return;
05394 }
05395 void Emit_53(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
05396   SDOperand InFlag(0, 0);
05397   SDOperand Tmp0(0, 0);
05398   SDOperand Tmp1(0, 0);
05399   SDNode *ResNode = NULL;
05400   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05401   Select(Tmp0, N0);
05402   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
05403   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05404   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp0, Tmp1);
05405   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05406   InFlag = SDOperand(ResNode, 1);
05407   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05408   Result = SDOperand(ResNode, N.ResNo);
05409   return;
05410 }
05411 void Emit_54(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
05412   SDOperand InFlag(0, 0);
05413   SDOperand Tmp0(0, 0);
05414   SDOperand Tmp1(0, 0);
05415   SDNode *ResNode = NULL;
05416   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05417   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
05418   Select(Tmp0, N0);
05419   Select(Tmp1, N1);
05420   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05421   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05422   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp0, Tmp1);
05423   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05424   InFlag = SDOperand(ResNode, 1);
05425   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05426   Result = SDOperand(ResNode, N.ResNo);
05427   return;
05428 }
05429 void Select_addc(SDOperand &Result, SDOperand N) {
05430   SDOperand CPTmp0(0, 0);
05431   SDOperand CPTmp1(0, 0);
05432   SDOperand CPTmp2(0, 0);
05433   SDOperand CPTmp3(0, 0);
05434   SDOperand Chain0(0, 0);
05435   SDOperand Chain1(0, 0);
05436   SDOperand InFlag(0, 0);
05437   SDOperand N0(0, 0);
05438   SDOperand N01(0, 0);
05439   SDOperand N1(0, 0);
05440   SDOperand N11(0, 0);
05441   SDOperand Tmp0(0, 0);
05442   SDOperand Tmp1(0, 0);
05443   SDOperand Tmp2(0, 0);
05444   SDOperand Tmp3(0, 0);
05445   SDOperand Tmp4(0, 0);
05446   SDNode *ResNode = NULL;
05447   N0 = N.getOperand(0);
05448 
05449   // Pattern: (addc:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
05450   // Emits: (ADD32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
05451   // Pattern complexity = 14  cost = 1  size = 3
05452   {
05453     N1 = N.getOperand(1);
05454     if (N1.getOpcode() == ISD::LOAD &&
05455         InFlightSet.count(N1.Val) == 0 &&
05456         N1.hasOneUse() &&
05457         !CodeGenMap.count(N1.getValue(0)) &&
05458         N.Val->isOnlyUse(N1.Val) &&
05459         !isNonImmUse(N.Val, N1.Val)) {
05460       Chain1 = N1.getOperand(0);
05461       N11 = N1.getOperand(1);
05462       if (N.Val->getValueType(0) == MVT::i32) {
05463         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05464         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
05465         if (!Match) {
05466           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05467         }
05468         if (Match) {
05469           Emit_51(Result, N, X86::ADD32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
05470           return;
05471         }
05472       }
05473     }
05474   }
05475 
05476   // Pattern: (addc:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
05477   // Emits: (ADD32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
05478   // Pattern complexity = 14  cost = 1  size = 3
05479   if (N0.getOpcode() == ISD::LOAD &&
05480       InFlightSet.count(N0.Val) == 0 &&
05481       N0.hasOneUse() &&
05482       !CodeGenMap.count(N0.getValue(0)) &&
05483       N.Val->isOnlyUse(N0.Val) &&
05484       !isNonImmUse(N.Val, N0.Val)) {
05485     Chain0 = N0.getOperand(0);
05486     N01 = N0.getOperand(1);
05487     N1 = N.getOperand(1);
05488     if (N.Val->getValueType(0) == MVT::i32) {
05489       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
05490       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
05491       if (!Match) {
05492         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05493       }
05494       if (Match) {
05495         Emit_52(Result, N, X86::ADD32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
05496         return;
05497       }
05498     }
05499   }
05500   N1 = N.getOperand(1);
05501   if (N1.getOpcode() == ISD::Constant) {
05502 
05503     // Pattern: (addc:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
05504     // Emits: (ADD32ri8:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
05505     // Pattern complexity = 5  cost = 1  size = 3
05506     if (Predicate_i32immSExt8(N1.Val) &&
05507         N.Val->getValueType(0) == MVT::i32) {
05508       Emit_53(Result, N, X86::ADD32ri8, MVT::i32, N0, N1);
05509       return;
05510     }
05511 
05512     // Pattern: (addc:i32 GR32:i32:$src1, (imm:i32):$src2)
05513     // Emits: (ADD32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
05514     // Pattern complexity = 4  cost = 1
05515     if (N.Val->getValueType(0) == MVT::i32) {
05516       Emit_53(Result, N, X86::ADD32ri, MVT::i32, N0, N1);
05517       return;
05518     }
05519   }
05520 
05521   // Pattern: (addc:i32 GR32:i32:$src1, GR32:i32:$src2)
05522   // Emits: (ADD32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
05523   // Pattern complexity = 2  cost = 1
05524   if (N.Val->getValueType(0) == MVT::i32) {
05525     Emit_54(Result, N, X86::ADD32rr, MVT::i32, N0, N1);
05526     return;
05527   }
05528   std::cerr << "Cannot yet select: ";
05529   N.Val->dump(CurDAG);
05530   std::cerr << '\n';
05531   abort();
05532 }
05533 
05534 void Emit_55(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
05535   SDOperand InFlag(0, 0);
05536   SDOperand Tmp0(0, 0);
05537   SDOperand Tmp1(0, 0);
05538   SDOperand Tmp2(0, 0);
05539   SDOperand Tmp3(0, 0);
05540   SDOperand Tmp4(0, 0);
05541   SDNode *ResNode = NULL;
05542   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05543   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05544   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05545   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05546   Select(Tmp0, CPTmp0);
05547   Select(Tmp1, CPTmp1);
05548   Select(Tmp2, CPTmp2);
05549   Select(Tmp3, CPTmp3);
05550   Select(Tmp4, N0);
05551   Select(Chain1, Chain1);
05552   Select(InFlag, N.getOperand(2));
05553   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05554   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05555   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05556   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05557   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05558   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
05559   Chain1 = SDOperand(ResNode, 1);
05560   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05561   InFlag = SDOperand(ResNode, 2);
05562   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05563   AddHandleReplacement(N1.Val, 1, ResNode, 1);
05564   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05565   if (N.ResNo < 1)
05566     Result = SDOperand(ResNode, N.ResNo);
05567   else
05568     Result = SDOperand(ResNode, N.ResNo+1);
05569   return;
05570 }
05571 void Emit_56(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
05572   SDOperand InFlag(0, 0);
05573   SDOperand Tmp0(0, 0);
05574   SDOperand Tmp1(0, 0);
05575   SDOperand Tmp2(0, 0);
05576   SDOperand Tmp3(0, 0);
05577   SDOperand Tmp4(0, 0);
05578   SDNode *ResNode = NULL;
05579   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05580   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05581   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05582   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05583   Select(Tmp0, CPTmp0);
05584   Select(Tmp1, CPTmp1);
05585   Select(Tmp2, CPTmp2);
05586   Select(Tmp3, CPTmp3);
05587   Select(Tmp4, N1);
05588   Select(Chain0, Chain0);
05589   Select(InFlag, N.getOperand(2));
05590   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05591   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05592   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05593   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05594   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05595   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0, InFlag);
05596   Chain0 = SDOperand(ResNode, 1);
05597   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05598   InFlag = SDOperand(ResNode, 2);
05599   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05600   AddHandleReplacement(N0.Val, 1, ResNode, 1);
05601   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05602   if (N.ResNo < 1)
05603     Result = SDOperand(ResNode, N.ResNo);
05604   else
05605     Result = SDOperand(ResNode, N.ResNo+1);
05606   return;
05607 }
05608 void Emit_57(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
05609   SDOperand InFlag(0, 0);
05610   SDOperand Tmp0(0, 0);
05611   SDOperand Tmp1(0, 0);
05612   SDNode *ResNode = NULL;
05613   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05614   Select(Tmp0, N0);
05615   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
05616   Select(InFlag, N.getOperand(2));
05617   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05618   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp0, Tmp1, InFlag);
05619   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05620   InFlag = SDOperand(ResNode, 1);
05621   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05622   Result = SDOperand(ResNode, N.ResNo);
05623   return;
05624 }
05625 void Emit_58(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
05626   SDOperand InFlag(0, 0);
05627   SDOperand Tmp0(0, 0);
05628   SDOperand Tmp1(0, 0);
05629   SDNode *ResNode = NULL;
05630   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05631   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
05632   Select(Tmp0, N0);
05633   Select(Tmp1, N1);
05634   Select(InFlag, N.getOperand(2));
05635   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05636   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05637   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Flag, Tmp0, Tmp1, InFlag);
05638   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05639   InFlag = SDOperand(ResNode, 1);
05640   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
05641   Result = SDOperand(ResNode, N.ResNo);
05642   return;
05643 }
05644 void Select_adde(SDOperand &Result, SDOperand N) {
05645   SDOperand CPTmp0(0, 0);
05646   SDOperand CPTmp1(0, 0);
05647   SDOperand CPTmp2(0, 0);
05648   SDOperand CPTmp3(0, 0);
05649   SDOperand Chain0(0, 0);
05650   SDOperand Chain1(0, 0);
05651   SDOperand InFlag(0, 0);
05652   SDOperand N0(0, 0);
05653   SDOperand N01(0, 0);
05654   SDOperand N1(0, 0);
05655   SDOperand N11(0, 0);
05656   SDOperand Tmp0(0, 0);
05657   SDOperand Tmp1(0, 0);
05658   SDOperand Tmp2(0, 0);
05659   SDOperand Tmp3(0, 0);
05660   SDOperand Tmp4(0, 0);
05661   SDNode *ResNode = NULL;
05662   N0 = N.getOperand(0);
05663 
05664   // Pattern: (adde:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
05665   // Emits: (ADC32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
05666   // Pattern complexity = 14  cost = 1  size = 3
05667   {
05668     N1 = N.getOperand(1);
05669     if (N1.getOpcode() == ISD::LOAD &&
05670         InFlightSet.count(N1.Val) == 0 &&
05671         N1.hasOneUse() &&
05672         !CodeGenMap.count(N1.getValue(0)) &&
05673         N.Val->isOnlyUse(N1.Val) &&
05674         !isNonImmUse(N.Val, N1.Val)) {
05675       Chain1 = N1.getOperand(0);
05676       N11 = N1.getOperand(1);
05677       if (N.Val->getValueType(0) == MVT::i32) {
05678         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
05679         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
05680         if (!Match) {
05681           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
05682         }
05683         if (Match) {
05684           Emit_55(Result, N, X86::ADC32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
05685           return;
05686         }
05687       }
05688     }
05689   }
05690 
05691   // Pattern: (adde:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
05692   // Emits: (ADC32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
05693   // Pattern complexity = 14  cost = 1  size = 3
05694   if (N0.getOpcode() == ISD::LOAD &&
05695       InFlightSet.count(N0.Val) == 0 &&
05696       N0.hasOneUse() &&
05697       !CodeGenMap.count(N0.getValue(0)) &&
05698       N.Val->isOnlyUse(N0.Val) &&
05699       !isNonImmUse(N.Val, N0.Val)) {
05700     Chain0 = N0.getOperand(0);
05701     N01 = N0.getOperand(1);
05702     N1 = N.getOperand(1);
05703     if (N.Val->getValueType(0) == MVT::i32) {
05704       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
05705       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
05706       if (!Match) {
05707         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
05708       }
05709       if (Match) {
05710         Emit_56(Result, N, X86::ADC32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
05711         return;
05712       }
05713     }
05714   }
05715   N1 = N.getOperand(1);
05716   if (N1.getOpcode() == ISD::Constant) {
05717 
05718     // Pattern: (adde:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
05719     // Emits: (ADC32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
05720     // Pattern complexity = 5  cost = 1  size = 3
05721     if (Predicate_i32immSExt8(N1.Val) &&
05722         N.Val->getValueType(0) == MVT::i32) {
05723       Emit_57(Result, N, X86::ADC32ri8, MVT::i32, N0, N1);
05724       return;
05725     }
05726 
05727     // Pattern: (adde:i32 GR32:i32:$src1, (imm:i32):$src2)
05728     // Emits: (ADC32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
05729     // Pattern complexity = 4  cost = 1
05730     if (N.Val->getValueType(0) == MVT::i32) {
05731       Emit_57(Result, N, X86::ADC32ri, MVT::i32, N0, N1);
05732       return;
05733     }
05734   }
05735 
05736   // Pattern: (adde:i32 GR32:i32:$src1, GR32:i32:$src2)
05737   // Emits: (ADC32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
05738   // Pattern complexity = 2  cost = 1
05739   if (N.Val->getValueType(0) == MVT::i32) {
05740     Emit_58(Result, N, X86::ADC32rr, MVT::i32, N0, N1);
05741     return;
05742   }
05743   std::cerr << "Cannot yet select: ";
05744   N.Val->dump(CurDAG);
05745   std::cerr << '\n';
05746   abort();
05747 }
05748 
05749 void Emit_59(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
05750   SDOperand Tmp0(0, 0);
05751   SDOperand Tmp1(0, 0);
05752   SDOperand Tmp2(0, 0);
05753   SDOperand Tmp3(0, 0);
05754   SDOperand Tmp4(0, 0);
05755   SDNode *ResNode = NULL;
05756   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05757   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05758   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05759   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05760   Select(Tmp0, CPTmp0);
05761   Select(Tmp1, CPTmp1);
05762   Select(Tmp2, CPTmp2);
05763   Select(Tmp3, CPTmp3);
05764   Select(Tmp4, N00);
05765   Select(Chain10, Chain10);
05766   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
05767   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05768   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05769   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05770   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05771   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
05772   Chain10 = SDOperand(ResNode, 1);
05773   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05774   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
05775   AddHandleReplacement(N10.Val, 1, ResNode, 1);
05776   Result = SDOperand(ResNode, N.ResNo);
05777   return;
05778 }
05779 void Emit_60(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N00, SDOperand &N000, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
05780   SDOperand Tmp0(0, 0);
05781   SDOperand Tmp1(0, 0);
05782   SDOperand Tmp2(0, 0);
05783   SDOperand Tmp3(0, 0);
05784   SDOperand Tmp4(0, 0);
05785   SDNode *ResNode = NULL;
05786   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05787   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05788   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05789   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05790   Select(Tmp0, CPTmp0);
05791   Select(Tmp1, CPTmp1);
05792   Select(Tmp2, CPTmp2);
05793   Select(Tmp3, CPTmp3);
05794   Select(Tmp4, N000);
05795   Select(Chain10, Chain10);
05796   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N000.Val);
05797   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05798   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05799   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05800   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05801   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
05802   Chain10 = SDOperand(ResNode, 1);
05803   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05804   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
05805   AddHandleReplacement(N10.Val, 1, ResNode, 1);
05806   Result = SDOperand(ResNode, N.ResNo);
05807   return;
05808 }
05809 void Emit_61(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N00, SDOperand &N000, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
05810   SDOperand Tmp0(0, 0);
05811   SDOperand Tmp1(0, 0);
05812   SDOperand Tmp2(0, 0);
05813   SDOperand Tmp3(0, 0);
05814   SDOperand Tmp4(0, 0);
05815   SDNode *ResNode = NULL;
05816   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05817   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05818   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05819   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05820   Select(Tmp0, CPTmp0);
05821   Select(Tmp1, CPTmp1);
05822   Select(Tmp2, CPTmp2);
05823   Select(Tmp3, CPTmp3);
05824   Select(Tmp4, N01);
05825   Select(Chain10, Chain10);
05826   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
05827   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05828   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05829   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05830   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05831   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
05832   Chain10 = SDOperand(ResNode, 1);
05833   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05834   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
05835   AddHandleReplacement(N10.Val, 1, ResNode, 1);
05836   Result = SDOperand(ResNode, N.ResNo);
05837   return;
05838 }
05839 void Emit_62(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110) NOINLINE {
05840   SDOperand Tmp0(0, 0);
05841   SDOperand Tmp1(0, 0);
05842   SDOperand Tmp2(0, 0);
05843   SDOperand Tmp3(0, 0);
05844   SDOperand Tmp4(0, 0);
05845   SDNode *ResNode = NULL;
05846   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05847   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05848   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05849   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05850   Select(Tmp0, CPTmp0);
05851   Select(Tmp1, CPTmp1);
05852   Select(Tmp2, CPTmp2);
05853   Select(Tmp3, CPTmp3);
05854   Select(Tmp4, N10);
05855   Select(Chain00, Chain00);
05856   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
05857   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05858   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05859   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05860   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05861   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
05862   Chain00 = SDOperand(ResNode, 1);
05863   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05864   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
05865   AddHandleReplacement(N00.Val, 1, ResNode, 1);
05866   Result = SDOperand(ResNode, N.ResNo);
05867   return;
05868 }
05869 void Emit_63(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11) NOINLINE {
05870   SDOperand Tmp0(0, 0);
05871   SDOperand Tmp1(0, 0);
05872   SDOperand Tmp2(0, 0);
05873   SDOperand Tmp3(0, 0);
05874   SDOperand Tmp4(0, 0);
05875   SDNode *ResNode = NULL;
05876   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05877   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05878   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05879   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05880   Select(Tmp0, CPTmp0);
05881   Select(Tmp1, CPTmp1);
05882   Select(Tmp2, CPTmp2);
05883   Select(Tmp3, CPTmp3);
05884   Select(Tmp4, N11);
05885   Select(Chain00, Chain00);
05886   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
05887   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05888   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05889   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05890   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05891   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
05892   Chain00 = SDOperand(ResNode, 1);
05893   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05894   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
05895   AddHandleReplacement(N00.Val, 1, ResNode, 1);
05896   Result = SDOperand(ResNode, N.ResNo);
05897   return;
05898 }
05899 void Emit_64(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
05900   SDOperand Tmp0(0, 0);
05901   SDOperand Tmp1(0, 0);
05902   SDOperand Tmp2(0, 0);
05903   SDOperand Tmp3(0, 0);
05904   SDOperand Tmp4(0, 0);
05905   SDNode *ResNode = NULL;
05906   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05907   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05908   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05909   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05910   Select(Tmp0, CPTmp0);
05911   Select(Tmp1, CPTmp1);
05912   Select(Tmp2, CPTmp2);
05913   Select(Tmp3, CPTmp3);
05914   Select(Tmp4, N010);
05915   Select(Chain10, Chain10);
05916   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N010.Val);
05917   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05918   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05919   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05920   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05921   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
05922   Chain10 = SDOperand(ResNode, 1);
05923   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05924   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
05925   AddHandleReplacement(N10.Val, 1, ResNode, 1);
05926   Result = SDOperand(ResNode, N.ResNo);
05927   return;
05928 }
05929 void Emit_65(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11) NOINLINE {
05930   SDOperand Tmp0(0, 0);
05931   SDOperand Tmp1(0, 0);
05932   SDOperand Tmp2(0, 0);
05933   SDOperand Tmp3(0, 0);
05934   SDOperand Tmp4(0, 0);
05935   SDNode *ResNode = NULL;
05936   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05937   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05938   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05939   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05940   Select(Tmp0, CPTmp0);
05941   Select(Tmp1, CPTmp1);
05942   Select(Tmp2, CPTmp2);
05943   Select(Tmp3, CPTmp3);
05944   Select(Tmp4, N100);
05945   Select(Chain00, Chain00);
05946   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
05947   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05948   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05949   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05950   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05951   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
05952   Chain00 = SDOperand(ResNode, 1);
05953   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05954   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
05955   AddHandleReplacement(N00.Val, 1, ResNode, 1);
05956   Result = SDOperand(ResNode, N.ResNo);
05957   return;
05958 }
05959 void Emit_66(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110) NOINLINE {
05960   SDOperand Tmp0(0, 0);
05961   SDOperand Tmp1(0, 0);
05962   SDOperand Tmp2(0, 0);
05963   SDOperand Tmp3(0, 0);
05964   SDOperand Tmp4(0, 0);
05965   SDNode *ResNode = NULL;
05966   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05967   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05968   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05969   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
05970   Select(Tmp0, CPTmp0);
05971   Select(Tmp1, CPTmp1);
05972   Select(Tmp2, CPTmp2);
05973   Select(Tmp3, CPTmp3);
05974   Select(Tmp4, N110);
05975   Select(Chain00, Chain00);
05976   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
05977   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
05978   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
05979   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
05980   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
05981   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
05982   Chain00 = SDOperand(ResNode, 1);
05983   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05984   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
05985   AddHandleReplacement(N00.Val, 1, ResNode, 1);
05986   Result = SDOperand(ResNode, N.ResNo);
05987   return;
05988 }
05989 void Emit_67(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N1, SDOperand &N11) NOINLINE {
05990   SDOperand Tmp0(0, 0);
05991   SDOperand Tmp1(0, 0);
05992   SDOperand Tmp2(0, 0);
05993   SDOperand Tmp3(0, 0);
05994   SDOperand Tmp4(0, 0);
05995   SDNode *ResNode = NULL;
05996   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
05997   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
05998   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
05999   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06000   Select(Tmp0, CPTmp0);
06001   Select(Tmp1, CPTmp1);
06002   Select(Tmp2, CPTmp2);
06003   Select(Tmp3, CPTmp3);
06004   Select(Tmp4, N00);
06005   Select(Chain1, Chain1);
06006   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06007   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06008   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06009   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06010   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06011   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06012   Chain1 = SDOperand(ResNode, 1);
06013   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06014   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06015   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06016   Result = SDOperand(ResNode, N.ResNo);
06017   return;
06018 }
06019 void Emit_68(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N00, SDOperand &N000, SDOperand &N01, SDOperand &N1, SDOperand &N11) NOINLINE {
06020   SDOperand Tmp0(0, 0);
06021   SDOperand Tmp1(0, 0);
06022   SDOperand Tmp2(0, 0);
06023   SDOperand Tmp3(0, 0);
06024   SDOperand Tmp4(0, 0);
06025   SDNode *ResNode = NULL;
06026   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06027   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06028   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06029   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06030   Select(Tmp0, CPTmp0);
06031   Select(Tmp1, CPTmp1);
06032   Select(Tmp2, CPTmp2);
06033   Select(Tmp3, CPTmp3);
06034   Select(Tmp4, N01);
06035   Select(Chain1, Chain1);
06036   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06037   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06038   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06039   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06040   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06041   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06042   Chain1 = SDOperand(ResNode, 1);
06043   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06044   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06045   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06046   Result = SDOperand(ResNode, N.ResNo);
06047   return;
06048 }
06049 void Emit_69(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110) NOINLINE {
06050   SDOperand Tmp0(0, 0);
06051   SDOperand Tmp1(0, 0);
06052   SDOperand Tmp2(0, 0);
06053   SDOperand Tmp3(0, 0);
06054   SDOperand Tmp4(0, 0);
06055   SDNode *ResNode = NULL;
06056   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06057   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06058   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06059   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06060   Select(Tmp0, CPTmp0);
06061   Select(Tmp1, CPTmp1);
06062   Select(Tmp2, CPTmp2);
06063   Select(Tmp3, CPTmp3);
06064   Select(Tmp4, N10);
06065   Select(Chain0, Chain0);
06066   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06067   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06068   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06069   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06070   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06071   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06072   Chain0 = SDOperand(ResNode, 1);
06073   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06074   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06075   AddHandleReplacement(N0.Val, 1, ResNode, 1);
06076   Result = SDOperand(ResNode, N.ResNo);
06077   return;
06078 }
06079 void Emit_70(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11) NOINLINE {
06080   SDOperand Tmp0(0, 0);
06081   SDOperand Tmp1(0, 0);
06082   SDOperand Tmp2(0, 0);
06083   SDOperand Tmp3(0, 0);
06084   SDOperand Tmp4(0, 0);
06085   SDNode *ResNode = NULL;
06086   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06087   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06088   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06089   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06090   Select(Tmp0, CPTmp0);
06091   Select(Tmp1, CPTmp1);
06092   Select(Tmp2, CPTmp2);
06093   Select(Tmp3, CPTmp3);
06094   Select(Tmp4, N11);
06095   Select(Chain0, Chain0);
06096   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
06097   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06098   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06099   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06100   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06101   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06102   Chain0 = SDOperand(ResNode, 1);
06103   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06104   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06105   AddHandleReplacement(N0.Val, 1, ResNode, 1);
06106   Result = SDOperand(ResNode, N.ResNo);
06107   return;
06108 }
06109 void Emit_71(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N1, SDOperand &N11) NOINLINE {
06110   SDOperand Tmp0(0, 0);
06111   SDOperand Tmp1(0, 0);
06112   SDOperand Tmp2(0, 0);
06113   SDOperand Tmp3(0, 0);
06114   SDOperand Tmp4(0, 0);
06115   SDNode *ResNode = NULL;
06116   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06117   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06118   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06119   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06120   Select(Tmp0, CPTmp0);
06121   Select(Tmp1, CPTmp1);
06122   Select(Tmp2, CPTmp2);
06123   Select(Tmp3, CPTmp3);
06124   Select(Tmp4, N00);
06125   Select(Chain1, Chain1);
06126   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06127   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06128   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06129   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06130   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06131   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06132   Chain1 = SDOperand(ResNode, 1);
06133   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06134   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06135   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06136   Result = SDOperand(ResNode, N.ResNo);
06137   return;
06138 }
06139 void Emit_72(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N1, SDOperand &N11) NOINLINE {
06140   SDOperand Tmp0(0, 0);
06141   SDOperand Tmp1(0, 0);
06142   SDOperand Tmp2(0, 0);
06143   SDOperand Tmp3(0, 0);
06144   SDOperand Tmp4(0, 0);
06145   SDNode *ResNode = NULL;
06146   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06147   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06148   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06149   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06150   Select(Tmp0, CPTmp0);
06151   Select(Tmp1, CPTmp1);
06152   Select(Tmp2, CPTmp2);
06153   Select(Tmp3, CPTmp3);
06154   Select(Tmp4, N01);
06155   Select(Chain1, Chain1);
06156   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06157   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06158   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06159   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06160   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06161   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06162   Chain1 = SDOperand(ResNode, 1);
06163   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06164   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06165   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06166   Result = SDOperand(ResNode, N.ResNo);
06167   return;
06168 }
06169 void Emit_73(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N11) NOINLINE {
06170   SDOperand Tmp0(0, 0);
06171   SDOperand Tmp1(0, 0);
06172   SDOperand Tmp2(0, 0);
06173   SDOperand Tmp3(0, 0);
06174   SDOperand Tmp4(0, 0);
06175   SDNode *ResNode = NULL;
06176   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06177   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06178   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06179   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06180   Select(Tmp0, CPTmp0);
06181   Select(Tmp1, CPTmp1);
06182   Select(Tmp2, CPTmp2);
06183   Select(Tmp3, CPTmp3);
06184   Select(Tmp4, N10);
06185   Select(Chain0, Chain0);
06186   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06187   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06188   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06189   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06190   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06191   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06192   Chain0 = SDOperand(ResNode, 1);
06193   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06194   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06195   AddHandleReplacement(N0.Val, 1, ResNode, 1);
06196   Result = SDOperand(ResNode, N.ResNo);
06197   return;
06198 }
06199 void Emit_74(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N11) NOINLINE {
06200   SDOperand Tmp0(0, 0);
06201   SDOperand Tmp1(0, 0);
06202   SDOperand Tmp2(0, 0);
06203   SDOperand Tmp3(0, 0);
06204   SDOperand Tmp4(0, 0);
06205   SDNode *ResNode = NULL;
06206   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06207   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06208   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06209   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06210   Select(Tmp0, CPTmp0);
06211   Select(Tmp1, CPTmp1);
06212   Select(Tmp2, CPTmp2);
06213   Select(Tmp3, CPTmp3);
06214   Select(Tmp4, N11);
06215   Select(Chain0, Chain0);
06216   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
06217   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06218   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06219   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06220   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06221   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06222   Chain0 = SDOperand(ResNode, 1);
06223   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06224   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06225   AddHandleReplacement(N0.Val, 1, ResNode, 1);
06226   Result = SDOperand(ResNode, N.ResNo);
06227   return;
06228 }
06229 void Emit_75(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N00, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
06230   SDOperand Tmp0(0, 0);
06231   SDOperand Tmp1(0, 0);
06232   SDOperand Tmp2(0, 0);
06233   SDOperand Tmp3(0, 0);
06234   SDOperand Tmp4(0, 0);
06235   SDNode *ResNode = NULL;
06236   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06237   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06238   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06239   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06240   Select(Tmp0, CPTmp0);
06241   Select(Tmp1, CPTmp1);
06242   Select(Tmp2, CPTmp2);
06243   Select(Tmp3, CPTmp3);
06244   Select(Tmp4, N00);
06245   Select(Chain10, Chain10);
06246   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06247   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06248   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06249   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06250   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06251   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
06252   Chain10 = SDOperand(ResNode, 1);
06253   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06254   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
06255   AddHandleReplacement(N10.Val, 1, ResNode, 1);
06256   Result = SDOperand(ResNode, N.ResNo);
06257   return;
06258 }
06259 void Emit_76(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N10) NOINLINE {
06260   SDOperand Tmp0(0, 0);
06261   SDOperand Tmp1(0, 0);
06262   SDOperand Tmp2(0, 0);
06263   SDOperand Tmp3(0, 0);
06264   SDOperand Tmp4(0, 0);
06265   SDNode *ResNode = NULL;
06266   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
06267   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
06268   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
06269   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
06270   Select(Tmp0, CPTmp0);
06271   Select(Tmp1, CPTmp1);
06272   Select(Tmp2, CPTmp2);
06273   Select(Tmp3, CPTmp3);
06274   Select(Tmp4, N10);
06275   Select(Chain00, Chain00);
06276   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06277   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
06278   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
06279   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
06280   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
06281   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
06282   Chain00 = SDOperand(ResNode, 1);
06283   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06284   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
06285   AddHandleReplacement(N00.Val, 1, ResNode, 1);
06286   Result = SDOperand(ResNode, N.ResNo);
06287   return;
06288 }
06289 void Emit_77(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N000, SDOperand &N01, SDOperand &N1, SDOperand &N10) NOINLINE {
06290   SDOperand Tmp0(0, 0);
06291   SDOperand Tmp1(0, 0);
06292   SDNode *ResNode = NULL;
06293   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N000.Val);
06294   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06295   Select(Tmp0, N000);
06296   Select(Tmp1, N10);
06297   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N000.Val);
06298   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06299   if (N.Val->hasOneUse()) {
06300     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06301   } else {
06302     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06303     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06304     Result = SDOperand(ResNode, 0);
06305   }
06306   return;
06307 }
06308 void Emit_78(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N1, SDOperand &N10) NOINLINE {
06309   SDOperand Tmp0(0, 0);
06310   SDOperand Tmp1(0, 0);
06311   SDNode *ResNode = NULL;
06312   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N010.Val);
06313   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06314   Select(Tmp0, N010);
06315   Select(Tmp1, N10);
06316   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N010.Val);
06317   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06318   if (N.Val->hasOneUse()) {
06319     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06320   } else {
06321     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06322     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06323     Result = SDOperand(ResNode, 0);
06324   }
06325   return;
06326 }
06327 void Emit_79(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11) NOINLINE {
06328   SDOperand Tmp0(0, 0);
06329   SDOperand Tmp1(0, 0);
06330   SDNode *ResNode = NULL;
06331   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
06332   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06333   Select(Tmp0, N100);
06334   Select(Tmp1, N00);
06335   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
06336   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06337   if (N.Val->hasOneUse()) {
06338     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06339   } else {
06340     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06341     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06342     Result = SDOperand(ResNode, 0);
06343   }
06344   return;
06345 }
06346 void Emit_80(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110) NOINLINE {
06347   SDOperand Tmp0(0, 0);
06348   SDOperand Tmp1(0, 0);
06349   SDNode *ResNode = NULL;
06350   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
06351   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06352   Select(Tmp0, N110);
06353   Select(Tmp1, N00);
06354   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
06355   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06356   if (N.Val->hasOneUse()) {
06357     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06358   } else {
06359     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06360     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06361     Result = SDOperand(ResNode, 0);
06362   }
06363   return;
06364 }
06365 void Emit_81(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N1) NOINLINE {
06366   SDOperand Tmp0(0, 0);
06367   SDOperand Tmp1(0, 0);
06368   SDNode *ResNode = NULL;
06369   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06370   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
06371   Select(Tmp0, N00);
06372   Select(Tmp1, N1);
06373   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06374   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
06375   if (N.Val->hasOneUse()) {
06376     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06377   } else {
06378     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06379     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06380     Result = SDOperand(ResNode, 0);
06381   }
06382   return;
06383 }
06384 void Emit_82(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N000, SDOperand &N01, SDOperand &N1) NOINLINE {
06385   SDOperand Tmp0(0, 0);
06386   SDOperand Tmp1(0, 0);
06387   SDNode *ResNode = NULL;
06388   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
06389   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
06390   Select(Tmp0, N01);
06391   Select(Tmp1, N1);
06392   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06393   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
06394   if (N.Val->hasOneUse()) {
06395     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06396   } else {
06397     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06398     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06399     Result = SDOperand(ResNode, 0);
06400   }
06401   return;
06402 }
06403 void Emit_83(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110) NOINLINE {
06404   SDOperand Tmp0(0, 0);
06405   SDOperand Tmp1(0, 0);
06406   SDNode *ResNode = NULL;
06407   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06408   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
06409   Select(Tmp0, N10);
06410   Select(Tmp1, N0);
06411   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06412   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
06413   if (N.Val->hasOneUse()) {
06414     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06415   } else {
06416     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06417     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06418     Result = SDOperand(ResNode, 0);
06419   }
06420   return;
06421 }
06422 void Emit_84(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11) NOINLINE {
06423   SDOperand Tmp0(0, 0);
06424   SDOperand Tmp1(0, 0);
06425   SDNode *ResNode = NULL;
06426   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
06427   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
06428   Select(Tmp0, N11);
06429   Select(Tmp1, N0);
06430   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
06431   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
06432   if (N.Val->hasOneUse()) {
06433     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06434   } else {
06435     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06436     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06437     Result = SDOperand(ResNode, 0);
06438   }
06439   return;
06440 }
06441 void Emit_85(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N1) NOINLINE {
06442   SDOperand Tmp0(0, 0);
06443   SDOperand Tmp1(0, 0);
06444   SDNode *ResNode = NULL;
06445   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06446   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
06447   Select(Tmp0, N00);
06448   Select(Tmp1, N1);
06449   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06450   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
06451   if (N.Val->hasOneUse()) {
06452     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06453   } else {
06454     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06455     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06456     Result = SDOperand(ResNode, 0);
06457   }
06458   return;
06459 }
06460 void Emit_86(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N1) NOINLINE {
06461   SDOperand Tmp0(0, 0);
06462   SDOperand Tmp1(0, 0);
06463   SDNode *ResNode = NULL;
06464   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
06465   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
06466   Select(Tmp0, N01);
06467   Select(Tmp1, N1);
06468   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06469   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
06470   if (N.Val->hasOneUse()) {
06471     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06472   } else {
06473     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06474     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06475     Result = SDOperand(ResNode, 0);
06476   }
06477   return;
06478 }
06479 void Emit_87(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N11) NOINLINE {
06480   SDOperand Tmp0(0, 0);
06481   SDOperand Tmp1(0, 0);
06482   SDNode *ResNode = NULL;
06483   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06484   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
06485   Select(Tmp0, N10);
06486   Select(Tmp1, N0);
06487   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06488   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
06489   if (N.Val->hasOneUse()) {
06490     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06491   } else {
06492     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06493     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06494     Result = SDOperand(ResNode, 0);
06495   }
06496   return;
06497 }
06498 void Emit_88(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N11) NOINLINE {
06499   SDOperand Tmp0(0, 0);
06500   SDOperand Tmp1(0, 0);
06501   SDNode *ResNode = NULL;
06502   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
06503   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
06504   Select(Tmp0, N11);
06505   Select(Tmp1, N0);
06506   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
06507   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
06508   if (N.Val->hasOneUse()) {
06509     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06510   } else {
06511     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06512     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06513     Result = SDOperand(ResNode, 0);
06514   }
06515   return;
06516 }
06517 void Emit_89(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N00, SDOperand &N1, SDOperand &N10) NOINLINE {
06518   SDOperand Tmp0(0, 0);
06519   SDOperand Tmp1(0, 0);
06520   SDNode *ResNode = NULL;
06521   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06522   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06523   Select(Tmp0, N00);
06524   Select(Tmp1, N10);
06525   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06526   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06527   if (N.Val->hasOneUse()) {
06528     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
06529   } else {
06530     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
06531     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
06532     Result = SDOperand(ResNode, 0);
06533   }
06534   return;
06535 }
06536 void Select_and(SDOperand &Result, SDOperand N) {
06537   SDOperand CPTmp0(0, 0);
06538   SDOperand CPTmp1(0, 0);
06539   SDOperand CPTmp2(0, 0);
06540   SDOperand CPTmp3(0, 0);
06541   SDOperand Chain0(0, 0);
06542   SDOperand Chain00(0, 0);
06543   SDOperand Chain1(0, 0);
06544   SDOperand Chain10(0, 0);
06545   SDOperand N0(0, 0);
06546   SDOperand N00(0, 0);
06547   SDOperand N000(0, 0);
06548   SDOperand N001(0, 0);
06549   SDOperand N01(0, 0);
06550   SDOperand N010(0, 0);
06551   SDOperand N1(0, 0);
06552   SDOperand N10(0, 0);
06553   SDOperand N100(0, 0);
06554   SDOperand N101(0, 0);
06555   SDOperand N11(0, 0);
06556   SDOperand N110(0, 0);
06557   SDOperand Tmp0(0, 0);
06558   SDOperand Tmp1(0, 0);
06559   SDOperand Tmp2(0, 0);
06560   SDOperand Tmp3(0, 0);
06561   SDOperand Tmp4(0, 0);
06562   SDNode *ResNode = NULL;
06563 
06564   // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)))
06565   // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
06566   // Pattern complexity = 23  cost = 1  size = 3
06567   if ((Subtarget->hasSSE1())) {
06568     N0 = N.getOperand(0);
06569     if (N0.getOpcode() == ISD::XOR) {
06570       N00 = N0.getOperand(0);
06571       N01 = N0.getOperand(1);
06572       if (N01.getOpcode() == ISD::BIT_CONVERT) {
06573         N010 = N01.getOperand(0);
06574         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
06575             Predicate_immAllOnesV(N010.Val)) {
06576           N1 = N.getOperand(1);
06577           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06578               InFlightSet.count(N1.Val) == 0 &&
06579               N1.hasOneUse() &&
06580               !CodeGenMap.count(N1.getValue(0))) {
06581             N10 = N1.getOperand(0);
06582             if (N10.getOpcode() == ISD::LOAD &&
06583                 InFlightSet.count(N10.Val) == 0 &&
06584                 N10.hasOneUse() &&
06585                 !CodeGenMap.count(N10.getValue(0))) {
06586               Chain10 = N10.getOperand(0);
06587               N101 = N10.getOperand(1);
06588               if (N.Val->getValueType(0) == MVT::v2i64 &&
06589                   N010.Val->getValueType(0) == MVT::v4i32 &&
06590                   N10.Val->getValueType(0) == MVT::v4f32) {
06591                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06592                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06593                 if (!Match) {
06594                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06595                 }
06596                 if (Match) {
06597                   Emit_59(Result, N, X86::ANDNPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N01, N010, N1, N10, N101);
06598                   return;
06599                 }
06600               }
06601             }
06602           }
06603         }
06604       }
06605     }
06606   }
06607 
06608   // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)))
06609   // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
06610   // Pattern complexity = 23  cost = 1  size = 3
06611   if ((Subtarget->hasSSE2())) {
06612     N0 = N.getOperand(0);
06613     if (N0.getOpcode() == ISD::XOR) {
06614       N00 = N0.getOperand(0);
06615       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06616         N000 = N00.getOperand(0);
06617         N01 = N0.getOperand(1);
06618         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
06619             Predicate_immAllOnesV(N01.Val)) {
06620           N1 = N.getOperand(1);
06621           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06622               InFlightSet.count(N1.Val) == 0 &&
06623               N1.hasOneUse() &&
06624               !CodeGenMap.count(N1.getValue(0))) {
06625             N10 = N1.getOperand(0);
06626             if (N10.getOpcode() == ISD::LOAD &&
06627                 InFlightSet.count(N10.Val) == 0 &&
06628                 N10.hasOneUse() &&
06629                 !CodeGenMap.count(N10.getValue(0))) {
06630               Chain10 = N10.getOperand(0);
06631               N101 = N10.getOperand(1);
06632               if (N.Val->getValueType(0) == MVT::v2i64 &&
06633                   N000.Val->getValueType(0) == MVT::v2f64 &&
06634                   N10.Val->getValueType(0) == MVT::v2f64) {
06635                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N000.Val);
06636                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06637                 if (!Match) {
06638                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N000.Val);
06639                 }
06640                 if (Match) {
06641                   Emit_60(Result, N, X86::ANDNPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N000, N01, N1, N10, N101);
06642                   return;
06643                 }
06644               }
06645             }
06646           }
06647         }
06648       }
06649     }
06650   }
06651   if ((Subtarget->hasSSE1())) {
06652     N0 = N.getOperand(0);
06653 
06654     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)))
06655     // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
06656     // Pattern complexity = 23  cost = 1  size = 3
06657     if (N0.getOpcode() == ISD::XOR) {
06658       N00 = N0.getOperand(0);
06659       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06660         N000 = N00.getOperand(0);
06661         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06662             Predicate_immAllOnesV(N000.Val)) {
06663           N01 = N0.getOperand(1);
06664           N1 = N.getOperand(1);
06665           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06666               InFlightSet.count(N1.Val) == 0 &&
06667               N1.hasOneUse() &&
06668               !CodeGenMap.count(N1.getValue(0))) {
06669             N10 = N1.getOperand(0);
06670             if (N10.getOpcode() == ISD::LOAD &&
06671                 InFlightSet.count(N10.Val) == 0 &&
06672                 N10.hasOneUse() &&
06673                 !CodeGenMap.count(N10.getValue(0))) {
06674               Chain10 = N10.getOperand(0);
06675               N101 = N10.getOperand(1);
06676               if (N.Val->getValueType(0) == MVT::v2i64 &&
06677                   N000.Val->getValueType(0) == MVT::v4i32 &&
06678                   N10.Val->getValueType(0) == MVT::v4f32) {
06679                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
06680                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06681                 if (!Match) {
06682                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06683                 }
06684                 if (Match) {
06685                   Emit_61(Result, N, X86::ANDNPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N000, N01, N1, N10, N101);
06686                   return;
06687                 }
06688               }
06689             }
06690           }
06691         }
06692       }
06693     }
06694     if (N0.getOpcode() == ISD::BIT_CONVERT &&
06695         InFlightSet.count(N0.Val) == 0 &&
06696         N0.hasOneUse() &&
06697         !CodeGenMap.count(N0.getValue(0))) {
06698       N00 = N0.getOperand(0);
06699       if (N00.getOpcode() == ISD::LOAD &&
06700           InFlightSet.count(N00.Val) == 0 &&
06701           N00.hasOneUse() &&
06702           !CodeGenMap.count(N00.getValue(0))) {
06703         Chain00 = N00.getOperand(0);
06704         N001 = N00.getOperand(1);
06705         N1 = N.getOperand(1);
06706         if (N1.getOpcode() == ISD::XOR) {
06707           N10 = N1.getOperand(0);
06708 
06709           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
06710           // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
06711           // Pattern complexity = 23  cost = 1  size = 3
06712           {
06713             N11 = N1.getOperand(1);
06714             if (N11.getOpcode() == ISD::BIT_CONVERT) {
06715               N110 = N11.getOperand(0);
06716               if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06717                   Predicate_immAllOnesV(N110.Val) &&
06718                   N.Val->getValueType(0) == MVT::v2i64 &&
06719                   N00.Val->getValueType(0) == MVT::v4f32 &&
06720                   N110.Val->getValueType(0) == MVT::v4i32) {
06721                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06722                 bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06723                 if (!Match) {
06724                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06725                 }
06726                 if (Match) {
06727                   Emit_62(Result, N, X86::ANDNPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10, N11, N110);
06728                   return;
06729                 }
06730               }
06731             }
06732           }
06733 
06734           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)), (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
06735           // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
06736           // Pattern complexity = 23  cost = 1
06737           if (N10.getOpcode() == ISD::BIT_CONVERT) {
06738             N100 = N10.getOperand(0);
06739             if (N100.getOpcode() == ISD::BUILD_VECTOR &&
06740                 Predicate_immAllOnesV(N100.Val)) {
06741               N11 = N1.getOperand(1);
06742               if (N.Val->getValueType(0) == MVT::v2i64 &&
06743                   N00.Val->getValueType(0) == MVT::v4f32 &&
06744                   N100.Val->getValueType(0) == MVT::v4i32) {
06745                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
06746                 bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06747                 if (!Match) {
06748                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
06749                 }
06750                 if (Match) {
06751                   Emit_63(Result, N, X86::ANDNPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10, N100, N11);
06752                   return;
06753                 }
06754               }
06755             }
06756           }
06757         }
06758       }
06759     }
06760   }
06761   if ((Subtarget->hasSSE2())) {
06762     N0 = N.getOperand(0);
06763 
06764     // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)), (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)))
06765     // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
06766     // Pattern complexity = 23  cost = 1  size = 3
06767     if (N0.getOpcode() == ISD::XOR) {
06768       N00 = N0.getOperand(0);
06769       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
06770           Predicate_immAllOnesV(N00.Val)) {
06771         N01 = N0.getOperand(1);
06772         if (N01.getOpcode() == ISD::BIT_CONVERT) {
06773           N010 = N01.getOperand(0);
06774           N1 = N.getOperand(1);
06775           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06776               InFlightSet.count(N1.Val) == 0 &&
06777               N1.hasOneUse() &&
06778               !CodeGenMap.count(N1.getValue(0))) {
06779             N10 = N1.getOperand(0);
06780             if (N10.getOpcode() == ISD::LOAD &&
06781                 InFlightSet.count(N10.Val) == 0 &&
06782                 N10.hasOneUse() &&
06783                 !CodeGenMap.count(N10.getValue(0))) {
06784               Chain10 = N10.getOperand(0);
06785               N101 = N10.getOperand(1);
06786               if (N.Val->getValueType(0) == MVT::v2i64 &&
06787                   N010.Val->getValueType(0) == MVT::v2f64 &&
06788                   N10.Val->getValueType(0) == MVT::v2f64) {
06789                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N010.Val);
06790                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06791                 if (!Match) {
06792                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N010.Val);
06793                 }
06794                 if (Match) {
06795                   Emit_64(Result, N, X86::ANDNPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N01, N010, N1, N10, N101);
06796                   return;
06797                 }
06798               }
06799             }
06800           }
06801         }
06802       }
06803     }
06804     if (N0.getOpcode() == ISD::BIT_CONVERT &&
06805         InFlightSet.count(N0.Val) == 0 &&
06806         N0.hasOneUse() &&
06807         !CodeGenMap.count(N0.getValue(0))) {
06808       N00 = N0.getOperand(0);
06809       if (N00.getOpcode() == ISD::LOAD &&
06810           InFlightSet.count(N00.Val) == 0 &&
06811           N00.hasOneUse() &&
06812           !CodeGenMap.count(N00.getValue(0))) {
06813         Chain00 = N00.getOperand(0);
06814         N001 = N00.getOperand(1);
06815         N1 = N.getOperand(1);
06816         if (N1.getOpcode() == ISD::XOR) {
06817           N10 = N1.getOperand(0);
06818 
06819           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)), (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
06820           // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
06821           // Pattern complexity = 23  cost = 1  size = 3
06822           if (N10.getOpcode() == ISD::BIT_CONVERT) {
06823             N100 = N10.getOperand(0);
06824             N11 = N1.getOperand(1);
06825             if (N11.getOpcode() == ISD::BUILD_VECTOR &&
06826                 Predicate_immAllOnesV(N11.Val) &&
06827                 N.Val->getValueType(0) == MVT::v2i64 &&
06828                 N00.Val->getValueType(0) == MVT::v2f64 &&
06829                 N100.Val->getValueType(0) == MVT::v2f64) {
06830               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
06831               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06832               if (!Match) {
06833                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
06834               }
06835               if (Match) {
06836                 Emit_65(Result, N, X86::ANDNPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10, N100, N11);
06837                 return;
06838               }
06839             }
06840           }
06841 
06842           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)))
06843           // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
06844           // Pattern complexity = 23  cost = 1
06845           if (N10.getOpcode() == ISD::BUILD_VECTOR &&
06846               Predicate_immAllOnesV(N10.Val)) {
06847             N11 = N1.getOperand(1);
06848             if (N11.getOpcode() == ISD::BIT_CONVERT) {
06849               N110 = N11.getOperand(0);
06850               if (N.Val->getValueType(0) == MVT::v2i64 &&
06851                   N00.Val->getValueType(0) == MVT::v2f64 &&
06852                   N110.Val->getValueType(0) == MVT::v2f64) {
06853                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
06854                 bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06855                 if (!Match) {
06856                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
06857                 }
06858                 if (Match) {
06859                   Emit_66(Result, N, X86::ANDNPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10, N11, N110);
06860                   return;
06861                 }
06862               }
06863             }
06864           }
06865         }
06866       }
06867     }
06868     if (N0.getOpcode() == ISD::XOR) {
06869       N00 = N0.getOperand(0);
06870       {
06871         N01 = N0.getOperand(1);
06872         if (N01.getOpcode() == ISD::BIT_CONVERT) {
06873           N010 = N01.getOperand(0);
06874           if (N010.getOpcode() == ISD::BUILD_VECTOR &&
06875               Predicate_immAllOnesV(N010.Val)) {
06876             N1 = N.getOperand(1);
06877             if (N1.getOpcode() == ISD::LOAD &&
06878                 InFlightSet.count(N1.Val) == 0 &&
06879                 N1.hasOneUse() &&
06880                 !CodeGenMap.count(N1.getValue(0)) &&
06881                 !isNonImmUse(N.Val, N1.Val)) {
06882               Chain1 = N1.getOperand(0);
06883               N11 = N1.getOperand(1);
06884               if (N.Val->getValueType(0) == MVT::v2i64) {
06885 
06886                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:iPTR:$src2))
06887                 // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
06888                 // Pattern complexity = 21  cost = 1  size = 3
06889                 if (N010.Val->getValueType(0) == MVT::v4i32) {
06890                   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06891                   bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06892                   if (!Match) {
06893                     SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06894                   }
06895                   if (Match) {
06896                     Emit_67(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N01, N010, N1, N11);
06897                     return;
06898                   }
06899                 }
06900 
06901                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:iPTR:$src2))
06902                 // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
06903                 // Pattern complexity = 21  cost = 1  size = 3
06904                 if (N010.Val->getValueType(0) == MVT::v8i16) {
06905                   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06906                   bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06907                   if (!Match) {
06908                     SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06909                   }
06910                   if (Match) {
06911                     Emit_67(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N01, N010, N1, N11);
06912                     return;
06913                   }
06914                 }
06915 
06916                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:iPTR:$src2))
06917                 // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
06918                 // Pattern complexity = 21  cost = 1
06919                 if (N010.Val->getValueType(0) == MVT::v16i8) {
06920                   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
06921                   bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06922                   if (!Match) {
06923                     SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
06924                   }
06925                   if (Match) {
06926                     Emit_67(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N01, N010, N1, N11);
06927                     return;
06928                   }
06929                 }
06930               }
06931             }
06932           }
06933         }
06934       }
06935 
06936       // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:iPTR:$src2))
06937       // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
06938       // Pattern complexity = 21  cost = 1
06939       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06940         N000 = N00.getOperand(0);
06941         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06942             Predicate_immAllOnesV(N000.Val)) {
06943           N01 = N0.getOperand(1);
06944           N1 = N.getOperand(1);
06945           if (N1.getOpcode() == ISD::LOAD &&
06946               InFlightSet.count(N1.Val) == 0 &&
06947               N1.hasOneUse() &&
06948               !CodeGenMap.count(N1.getValue(0)) &&
06949               !isNonImmUse(N.Val, N1.Val)) {
06950             Chain1 = N1.getOperand(0);
06951             N11 = N1.getOperand(1);
06952             if (N.Val->getValueType(0) == MVT::v2i64 &&
06953                 N000.Val->getValueType(0) == MVT::v4i32) {
06954               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
06955               bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06956               if (!Match) {
06957                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
06958               }
06959               if (Match) {
06960                 Emit_68(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N000, N01, N1, N11);
06961                 return;
06962               }
06963             }
06964           }
06965         }
06966       }
06967     }
06968     if (N0.getOpcode() == ISD::LOAD &&
06969         InFlightSet.count(N0.Val) == 0 &&
06970         N0.hasOneUse() &&
06971         !CodeGenMap.count(N0.getValue(0)) &&
06972         !isNonImmUse(N.Val, N0.Val)) {
06973       Chain0 = N0.getOperand(0);
06974       N01 = N0.getOperand(1);
06975       N1 = N.getOperand(1);
06976       if (N1.getOpcode() == ISD::XOR) {
06977         N10 = N1.getOperand(0);
06978 
06979         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
06980         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
06981         // Pattern complexity = 21  cost = 1  size = 3
06982         {
06983           N11 = N1.getOperand(1);
06984           if (N11.getOpcode() == ISD::BIT_CONVERT) {
06985             N110 = N11.getOperand(0);
06986             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06987                 Predicate_immAllOnesV(N110.Val) &&
06988                 N.Val->getValueType(0) == MVT::v2i64 &&
06989                 N110.Val->getValueType(0) == MVT::v4i32) {
06990               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
06991               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
06992               if (!Match) {
06993                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
06994               }
06995               if (Match) {
06996                 Emit_69(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N11, N110);
06997                 return;
06998               }
06999             }
07000           }
07001         }
07002 
07003         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07004         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07005         // Pattern complexity = 21  cost = 1
07006         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07007           N100 = N10.getOperand(0);
07008           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07009               Predicate_immAllOnesV(N100.Val)) {
07010             N11 = N1.getOperand(1);
07011             if (N.Val->getValueType(0) == MVT::v2i64 &&
07012                 N100.Val->getValueType(0) == MVT::v4i32) {
07013               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
07014               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07015               if (!Match) {
07016                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
07017               }
07018               if (Match) {
07019                 Emit_70(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N100, N11);
07020                 return;
07021               }
07022             }
07023           }
07024         }
07025       }
07026     }
07027 
07028     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:iPTR:$src2))
07029     // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07030     // Pattern complexity = 21  cost = 1  size = 3
07031     if (N0.getOpcode() == ISD::XOR) {
07032       N00 = N0.getOperand(0);
07033       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07034         N000 = N00.getOperand(0);
07035         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07036             Predicate_immAllOnesV(N000.Val)) {
07037           N01 = N0.getOperand(1);
07038           N1 = N.getOperand(1);
07039           if (N1.getOpcode() == ISD::LOAD &&
07040               InFlightSet.count(N1.Val) == 0 &&
07041               N1.hasOneUse() &&
07042               !CodeGenMap.count(N1.getValue(0)) &&
07043               !isNonImmUse(N.Val, N1.Val)) {
07044             Chain1 = N1.getOperand(0);
07045             N11 = N1.getOperand(1);
07046             if (N.Val->getValueType(0) == MVT::v2i64 &&
07047                 N000.Val->getValueType(0) == MVT::v8i16) {
07048               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
07049               bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07050               if (!Match) {
07051                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
07052               }
07053               if (Match) {
07054                 Emit_68(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N000, N01, N1, N11);
07055                 return;
07056               }
07057             }
07058           }
07059         }
07060       }
07061     }
07062     if (N0.getOpcode() == ISD::LOAD &&
07063         InFlightSet.count(N0.Val) == 0 &&
07064         N0.hasOneUse() &&
07065         !CodeGenMap.count(N0.getValue(0)) &&
07066         !isNonImmUse(N.Val, N0.Val)) {
07067       Chain0 = N0.getOperand(0);
07068       N01 = N0.getOperand(1);
07069       N1 = N.getOperand(1);
07070       if (N1.getOpcode() == ISD::XOR) {
07071         N10 = N1.getOperand(0);
07072 
07073         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)))
07074         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07075         // Pattern complexity = 21  cost = 1  size = 3
07076         {
07077           N11 = N1.getOperand(1);
07078           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07079             N110 = N11.getOperand(0);
07080             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07081                 Predicate_immAllOnesV(N110.Val) &&
07082                 N.Val->getValueType(0) == MVT::v2i64 &&
07083                 N110.Val->getValueType(0) == MVT::v8i16) {
07084               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
07085               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07086               if (!Match) {
07087                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
07088               }
07089               if (Match) {
07090                 Emit_69(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N11, N110);
07091                 return;
07092               }
07093             }
07094           }
07095         }
07096 
07097         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07098         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07099         // Pattern complexity = 21  cost = 1
07100         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07101           N100 = N10.getOperand(0);
07102           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07103               Predicate_immAllOnesV(N100.Val)) {
07104             N11 = N1.getOperand(1);
07105             if (N.Val->getValueType(0) == MVT::v2i64 &&
07106                 N100.Val->getValueType(0) == MVT::v8i16) {
07107               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
07108               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07109               if (!Match) {
07110                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
07111               }
07112               if (Match) {
07113                 Emit_70(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N100, N11);
07114                 return;
07115               }
07116             }
07117           }
07118         }
07119       }
07120     }
07121 
07122     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:iPTR:$src2))
07123     // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07124     // Pattern complexity = 21  cost = 1  size = 3
07125     if (N0.getOpcode() == ISD::XOR) {
07126       N00 = N0.getOperand(0);
07127       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07128         N000 = N00.getOperand(0);
07129         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07130             Predicate_immAllOnesV(N000.Val)) {
07131           N01 = N0.getOperand(1);
07132           N1 = N.getOperand(1);
07133           if (N1.getOpcode() == ISD::LOAD &&
07134               InFlightSet.count(N1.Val) == 0 &&
07135               N1.hasOneUse() &&
07136               !CodeGenMap.count(N1.getValue(0)) &&
07137               !isNonImmUse(N.Val, N1.Val)) {
07138             Chain1 = N1.getOperand(0);
07139             N11 = N1.getOperand(1);
07140             if (N.Val->getValueType(0) == MVT::v2i64 &&
07141                 N000.Val->getValueType(0) == MVT::v16i8) {
07142               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
07143               bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07144               if (!Match) {
07145                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
07146               }
07147               if (Match) {
07148                 Emit_68(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N000, N01, N1, N11);
07149                 return;
07150               }
07151             }
07152           }
07153         }
07154       }
07155     }
07156     if (N0.getOpcode() == ISD::LOAD &&
07157         InFlightSet.count(N0.Val) == 0 &&
07158         N0.hasOneUse() &&
07159         !CodeGenMap.count(N0.getValue(0)) &&
07160         !isNonImmUse(N.Val, N0.Val)) {
07161       Chain0 = N0.getOperand(0);
07162       N01 = N0.getOperand(1);
07163       N1 = N.getOperand(1);
07164       if (N1.getOpcode() == ISD::XOR) {
07165         N10 = N1.getOperand(0);
07166 
07167         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)))
07168         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07169         // Pattern complexity = 21  cost = 1  size = 3
07170         {
07171           N11 = N1.getOperand(1);
07172           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07173             N110 = N11.getOperand(0);
07174             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07175                 Predicate_immAllOnesV(N110.Val) &&
07176                 N.Val->getValueType(0) == MVT::v2i64 &&
07177                 N110.Val->getValueType(0) == MVT::v16i8) {
07178               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
07179               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07180               if (!Match) {
07181                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
07182               }
07183               if (Match) {
07184                 Emit_69(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N11, N110);
07185                 return;
07186               }
07187             }
07188           }
07189         }
07190 
07191         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07192         // Emits: (PANDNrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
07193         // Pattern complexity = 21  cost = 1
07194         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07195           N100 = N10.getOperand(0);
07196           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07197               Predicate_immAllOnesV(N100.Val)) {
07198             N11 = N1.getOperand(1);
07199             if (N.Val->getValueType(0) == MVT::v2i64 &&
07200                 N100.Val->getValueType(0) == MVT::v16i8) {
07201               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
07202               bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07203               if (!Match) {
07204                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
07205               }
07206               if (Match) {
07207                 Emit_70(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N100, N11);
07208                 return;
07209               }
07210             }
07211           }
07212         }
07213       }
07214     }
07215     if (N0.getOpcode() == ISD::XOR) {
07216       N00 = N0.getOperand(0);
07217 
07218       // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (load:v2i64 addr:iPTR:$src2))
07219       // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07220       // Pattern complexity = 19  cost = 1  size = 3
07221       {
07222         N01 = N0.getOperand(1);
07223         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07224             Predicate_immAllOnesV(N01.Val)) {
07225           N1 = N.getOperand(1);
07226           if (N1.getOpcode() == ISD::LOAD &&
07227               InFlightSet.count(N1.Val) == 0 &&
07228               N1.hasOneUse() &&
07229               !CodeGenMap.count(N1.getValue(0)) &&
07230               !isNonImmUse(N.Val, N1.Val)) {
07231             Chain1 = N1.getOperand(0);
07232             N11 = N1.getOperand(1);
07233             if (N.Val->getValueType(0) == MVT::v2i64) {
07234               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
07235               bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07236               if (!Match) {
07237                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
07238               }
07239               if (Match) {
07240                 Emit_71(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N01, N1, N11);
07241                 return;
07242               }
07243             }
07244           }
07245         }
07246       }
07247 
07248       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1), (load:v2i64 addr:iPTR:$src2))
07249       // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07250       // Pattern complexity = 19  cost = 1
07251       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07252           Predicate_immAllOnesV(N00.Val)) {
07253         N01 = N0.getOperand(1);
07254         N1 = N.getOperand(1);
07255         if (N1.getOpcode() == ISD::LOAD &&
07256             InFlightSet.count(N1.Val) == 0 &&
07257             N1.hasOneUse() &&
07258             !CodeGenMap.count(N1.getValue(0)) &&
07259             !isNonImmUse(N.Val, N1.Val)) {
07260           Chain1 = N1.getOperand(0);
07261           N11 = N1.getOperand(1);
07262           if (N.Val->getValueType(0) == MVT::v2i64) {
07263             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N01.Val);
07264             bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07265             if (!Match) {
07266               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N01.Val);
07267             }
07268             if (Match) {
07269               Emit_72(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N00, N01, N1, N11);
07270               return;
07271             }
07272           }
07273         }
07274       }
07275     }
07276     if (N0.getOpcode() == ISD::LOAD &&
07277         InFlightSet.count(N0.Val) == 0 &&
07278         N0.hasOneUse() &&
07279         !CodeGenMap.count(N0.getValue(0)) &&
07280         !isNonImmUse(N.Val, N0.Val)) {
07281       Chain0 = N0.getOperand(0);
07282       N01 = N0.getOperand(1);
07283       N1 = N.getOperand(1);
07284       if (N1.getOpcode() == ISD::XOR) {
07285         N10 = N1.getOperand(0);
07286 
07287         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07288         // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07289         // Pattern complexity = 19  cost = 1  size = 3
07290         {
07291           N11 = N1.getOperand(1);
07292           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
07293               Predicate_immAllOnesV(N11.Val) &&
07294               N.Val->getValueType(0) == MVT::v2i64) {
07295             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
07296             bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07297             if (!Match) {
07298               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
07299             }
07300             if (Match) {
07301               Emit_73(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N11);
07302               return;
07303             }
07304           }
07305         }
07306 
07307         // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1))
07308         // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07309         // Pattern complexity = 19  cost = 1
07310         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
07311             Predicate_immAllOnesV(N10.Val)) {
07312           N11 = N1.getOperand(1);
07313           if (N.Val->getValueType(0) == MVT::v2i64) {
07314             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
07315             bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07316             if (!Match) {
07317               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
07318             }
07319             if (Match) {
07320               Emit_74(Result, N, X86::PANDNrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N10, N11);
07321               return;
07322             }
07323           }
07324         }
07325       }
07326     }
07327     if (N0.getOpcode() == ISD::BIT_CONVERT) {
07328 
07329       // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)))
07330       // Emits: (ANDPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
07331       // Pattern complexity = 18  cost = 1  size = 3
07332       {
07333         N00 = N0.getOperand(0);
07334         N1 = N.getOperand(1);
07335         if (N1.getOpcode() == ISD::BIT_CONVERT &&
07336             InFlightSet.count(N1.Val) == 0 &&
07337             N1.hasOneUse() &&
07338             !CodeGenMap.count(N1.getValue(0))) {
07339           N10 = N1.getOperand(0);
07340           if (N10.getOpcode() == ISD::LOAD &&
07341               InFlightSet.count(N10.Val) == 0 &&
07342               N10.hasOneUse() &&
07343               !CodeGenMap.count(N10.getValue(0))) {
07344             Chain10 = N10.getOperand(0);
07345             N101 = N10.getOperand(1);
07346             if (N.Val->getValueType(0) == MVT::v2i64 &&
07347                 N00.Val->getValueType(0) == MVT::v2f64 &&
07348                 N10.Val->getValueType(0) == MVT::v2f64) {
07349               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
07350               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07351               if (!Match) {
07352                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
07353               }
07354               if (Match) {
07355                 Emit_75(Result, N, X86::ANDPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N1, N10, N101);
07356                 return;
07357               }
07358             }
07359           }
07360         }
07361       }
07362 
07363       // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
07364       // Emits: (ANDPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
07365       // Pattern complexity = 18  cost = 1
07366       if (InFlightSet.count(N0.Val) == 0 &&
07367           N0.hasOneUse() &&
07368           !CodeGenMap.count(N0.getValue(0))) {
07369         N00 = N0.getOperand(0);
07370         if (N00.getOpcode() == ISD::LOAD &&
07371             InFlightSet.count(N00.Val) == 0 &&
07372             N00.hasOneUse() &&
07373             !CodeGenMap.count(N00.getValue(0))) {
07374           Chain00 = N00.getOperand(0);
07375           N001 = N00.getOperand(1);
07376           N1 = N.getOperand(1);
07377           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07378             N10 = N1.getOperand(0);
07379             if (N.Val->getValueType(0) == MVT::v2i64 &&
07380                 N00.Val->getValueType(0) == MVT::v2f64 &&
07381                 N10.Val->getValueType(0) == MVT::v2f64) {
07382               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
07383               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07384               if (!Match) {
07385                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
07386               }
07387               if (Match) {
07388                 Emit_76(Result, N, X86::ANDPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10);
07389                 return;
07390               }
07391             }
07392           }
07393         }
07394       }
07395     }
07396   }
07397   if ((Subtarget->hasSSE1())) {
07398     N0 = N.getOperand(0);
07399 
07400     // Pattern: (and:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)))
07401     // Emits: (ANDPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07402     // Pattern complexity = 16  cost = 1  size = 3
07403     {
07404       N1 = N.getOperand(1);
07405       if (N1.getOpcode() == ISD::BIT_CONVERT &&
07406           InFlightSet.count(N1.Val) == 0 &&
07407           N1.hasOneUse() &&
07408           !CodeGenMap.count(N1.getValue(0))) {
07409         N10 = N1.getOperand(0);
07410         if (N10.getOpcode() == ISD::LOAD &&
07411             InFlightSet.count(N10.Val) == 0 &&
07412             N10.hasOneUse() &&
07413             !CodeGenMap.count(N10.getValue(0))) {
07414           Chain10 = N10.getOperand(0);
07415           N101 = N10.getOperand(1);
07416           if (N.Val->getValueType(0) == MVT::v2i64 &&
07417               N10.Val->getValueType(0) == MVT::v4f32) {
07418             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
07419             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07420             if (!Match) {
07421               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
07422             }
07423             if (Match) {
07424               Emit_40(Result, N, X86::ANDPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
07425               return;
07426             }
07427           }
07428         }
07429       }
07430     }
07431 
07432     // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)), VR128:v2i64:$src1)
07433     // Emits: (ANDPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07434     // Pattern complexity = 16  cost = 1
07435     if (N0.getOpcode() == ISD::BIT_CONVERT &&
07436         InFlightSet.count(N0.Val) == 0 &&
07437         N0.hasOneUse() &&
07438         !CodeGenMap.count(N0.getValue(0))) {
07439       N00 = N0.getOperand(0);
07440       if (N00.getOpcode() == ISD::LOAD &&
07441           InFlightSet.count(N00.Val) == 0 &&
07442           N00.hasOneUse() &&
07443           !CodeGenMap.count(N00.getValue(0))) {
07444         Chain00 = N00.getOperand(0);
07445         N001 = N00.getOperand(1);
07446         N1 = N.getOperand(1);
07447         if (N.Val->getValueType(0) == MVT::v2i64 &&
07448             N00.Val->getValueType(0) == MVT::v4f32) {
07449           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
07450           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07451           if (!Match) {
07452             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
07453           }
07454           if (Match) {
07455             Emit_41(Result, N, X86::ANDPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
07456             return;
07457           }
07458         }
07459       }
07460     }
07461   }
07462   {
07463     N0 = N.getOperand(0);
07464     N1 = N.getOperand(1);
07465     if (N1.getOpcode() == ISD::LOAD &&
07466         InFlightSet.count(N1.Val) == 0 &&
07467         N1.hasOneUse() &&
07468         !CodeGenMap.count(N1.getValue(0)) &&
07469         !isNonImmUse(N.Val, N1.Val)) {
07470       Chain1 = N1.getOperand(0);
07471       N11 = N1.getOperand(1);
07472 
07473       // Pattern: (and:i8 GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
07474       // Emits: (AND8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
07475       // Pattern complexity = 14  cost = 1  size = 3
07476       if (N.Val->getValueType(0) == MVT::i8) {
07477         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
07478         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07479         if (!Match) {
07480           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
07481         }
07482         if (Match) {
07483           Emit_19(Result, N, X86::AND8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
07484           return;
07485         }
07486       }
07487 
07488       // Pattern: (and:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
07489       // Emits: (AND16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
07490       // Pattern complexity = 14  cost = 1  size = 3
07491       if (N.Val->getValueType(0) == MVT::i16) {
07492         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
07493         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07494         if (!Match) {
07495           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
07496         }
07497         if (Match) {
07498           Emit_19(Result, N, X86::AND16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
07499           return;
07500         }
07501       }
07502 
07503       // Pattern: (and:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
07504       // Emits: (AND32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
07505       // Pattern complexity = 14  cost = 1
07506       if (N.Val->getValueType(0) == MVT::i32) {
07507         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
07508         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07509         if (!Match) {
07510           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
07511         }
07512         if (Match) {
07513           Emit_19(Result, N, X86::AND32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
07514           return;
07515         }
07516       }
07517     }
07518   }
07519 
07520   // Pattern: (and:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
07521   // Emits: (PANDrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07522   // Pattern complexity = 14  cost = 1  size = 3
07523   if ((Subtarget->hasSSE2())) {
07524     N0 = N.getOperand(0);
07525     N1 = N.getOperand(1);
07526     if (N1.getOpcode() == ISD::LOAD &&
07527         InFlightSet.count(N1.Val) == 0 &&
07528         N1.hasOneUse() &&
07529         !CodeGenMap.count(N1.getValue(0)) &&
07530         !isNonImmUse(N.Val, N1.Val)) {
07531       Chain1 = N1.getOperand(0);
07532       N11 = N1.getOperand(1);
07533       if (N.Val->getValueType(0) == MVT::v2i64) {
07534         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
07535         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07536         if (!Match) {
07537           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
07538         }
07539         if (Match) {
07540           Emit_19(Result, N, X86::PANDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
07541           return;
07542         }
07543       }
07544     }
07545   }
07546   {
07547     N0 = N.getOperand(0);
07548     if (N0.getOpcode() == ISD::LOAD &&
07549         InFlightSet.count(N0.Val) == 0 &&
07550         N0.hasOneUse() &&
07551         !CodeGenMap.count(N0.getValue(0)) &&
07552         !isNonImmUse(N.Val, N0.Val)) {
07553       Chain0 = N0.getOperand(0);
07554       N01 = N0.getOperand(1);
07555       N1 = N.getOperand(1);
07556 
07557       // Pattern: (and:i8 (load:i8 addr:iPTR:$src2), GR8:i8:$src1)
07558       // Emits: (AND8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
07559       // Pattern complexity = 14  cost = 1  size = 3
07560       if (N.Val->getValueType(0) == MVT::i8) {
07561         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
07562         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07563         if (!Match) {
07564           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
07565         }
07566         if (Match) {
07567           Emit_20(Result, N, X86::AND8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
07568           return;
07569         }
07570       }
07571 
07572       // Pattern: (and:i16 (load:i16 addr:iPTR:$src2), GR16:i16:$src1)
07573       // Emits: (AND16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
07574       // Pattern complexity = 14  cost = 1  size = 3
07575       if (N.Val->getValueType(0) == MVT::i16) {
07576         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
07577         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07578         if (!Match) {
07579           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
07580         }
07581         if (Match) {
07582           Emit_20(Result, N, X86::AND16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
07583           return;
07584         }
07585       }
07586 
07587       // Pattern: (and:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
07588       // Emits: (AND32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
07589       // Pattern complexity = 14  cost = 1
07590       if (N.Val->getValueType(0) == MVT::i32) {
07591         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
07592         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07593         if (!Match) {
07594           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
07595         }
07596         if (Match) {
07597           Emit_20(Result, N, X86::AND32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
07598           return;
07599         }
07600       }
07601     }
07602   }
07603   if ((Subtarget->hasSSE2())) {
07604     N0 = N.getOperand(0);
07605 
07606     // Pattern: (and:v2i64 (load:v2i64 addr:iPTR:$src2), VR128:v2i64:$src1)
07607     // Emits: (PANDrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
07608     // Pattern complexity = 14  cost = 1  size = 3
07609     if (N0.getOpcode() == ISD::LOAD &&
07610         InFlightSet.count(N0.Val) == 0 &&
07611         N0.hasOneUse() &&
07612         !CodeGenMap.count(N0.getValue(0)) &&
07613         !isNonImmUse(N.Val, N0.Val)) {
07614       Chain0 = N0.getOperand(0);
07615       N01 = N0.getOperand(1);
07616       N1 = N.getOperand(1);
07617       if (N.Val->getValueType(0) == MVT::v2i64) {
07618         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
07619         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
07620         if (!Match) {
07621           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
07622         }
07623         if (Match) {
07624           Emit_20(Result, N, X86::PANDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
07625           return;
07626         }
07627       }
07628     }
07629     if (N0.getOpcode() == ISD::XOR) {
07630       N00 = N0.getOperand(0);
07631 
07632       // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (bitconvert:v2i64 VR128:v2f64:$src2))
07633       // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07634       // Pattern complexity = 11  cost = 1  size = 3
07635       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07636         N000 = N00.getOperand(0);
07637         N01 = N0.getOperand(1);
07638         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07639             Predicate_immAllOnesV(N01.Val)) {
07640           N1 = N.getOperand(1);
07641           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07642             N10 = N1.getOperand(0);
07643             if (N.Val->getValueType(0) == MVT::v2i64 &&
07644                 N000.Val->getValueType(0) == MVT::v2f64 &&
07645                 N10.Val->getValueType(0) == MVT::v2f64) {
07646               Emit_77(Result, N, X86::ANDNPDrr, MVT::v2i64, N0, N00, N000, N01, N1, N10);
07647               return;
07648             }
07649           }
07650         }
07651       }
07652 
07653       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)), (bitconvert:v2i64 VR128:v2f64:$src2))
07654       // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07655       // Pattern complexity = 11  cost = 1
07656       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07657           Predicate_immAllOnesV(N00.Val)) {
07658         N01 = N0.getOperand(1);
07659         if (N01.getOpcode() == ISD::BIT_CONVERT) {
07660           N010 = N01.getOperand(0);
07661           N1 = N.getOperand(1);
07662           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07663             N10 = N1.getOperand(0);
07664             if (N.Val->getValueType(0) == MVT::v2i64 &&
07665                 N010.Val->getValueType(0) == MVT::v2f64 &&
07666                 N10.Val->getValueType(0) == MVT::v2f64) {
07667               Emit_78(Result, N, X86::ANDNPDrr, MVT::v2i64, N0, N00, N01, N010, N1, N10);
07668               return;
07669             }
07670           }
07671         }
07672       }
07673     }
07674     if (N0.getOpcode() == ISD::BIT_CONVERT) {
07675       N00 = N0.getOperand(0);
07676       N1 = N.getOperand(1);
07677       if (N1.getOpcode() == ISD::XOR) {
07678         N10 = N1.getOperand(0);
07679 
07680         // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src2), (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07681         // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07682         // Pattern complexity = 11  cost = 1  size = 3
07683         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07684           N100 = N10.getOperand(0);
07685           N11 = N1.getOperand(1);
07686           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
07687               Predicate_immAllOnesV(N11.Val) &&
07688               N.Val->getValueType(0) == MVT::v2i64 &&
07689               N00.Val->getValueType(0) == MVT::v2f64 &&
07690               N100.Val->getValueType(0) == MVT::v2f64) {
07691             Emit_79(Result, N, X86::ANDNPDrr, MVT::v2i64, N0, N00, N1, N10, N100, N11);
07692             return;
07693           }
07694         }
07695 
07696         // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src2), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)))
07697         // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07698         // Pattern complexity = 11  cost = 1
07699         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
07700             Predicate_immAllOnesV(N10.Val)) {
07701           N11 = N1.getOperand(1);
07702           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07703             N110 = N11.getOperand(0);
07704             if (N.Val->getValueType(0) == MVT::v2i64 &&
07705                 N00.Val->getValueType(0) == MVT::v2f64 &&
07706                 N110.Val->getValueType(0) == MVT::v2f64) {
07707               Emit_80(Result, N, X86::ANDNPDrr, MVT::v2i64, N0, N00, N1, N10, N11, N110);
07708               return;
07709             }
07710           }
07711         }
07712       }
07713     }
07714   }
07715 
07716   // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07717   // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07718   // Pattern complexity = 9  cost = 1  size = 3
07719   if ((Subtarget->hasSSE1())) {
07720     N0 = N.getOperand(0);
07721     if (N0.getOpcode() == ISD::XOR) {
07722       N00 = N0.getOperand(0);
07723       N01 = N0.getOperand(1);
07724       if (N01.getOpcode() == ISD::BIT_CONVERT) {
07725         N010 = N01.getOperand(0);
07726         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
07727             Predicate_immAllOnesV(N010.Val)) {
07728           N1 = N.getOperand(1);
07729           if (N.Val->getValueType(0) == MVT::v2i64 &&
07730               N010.Val->getValueType(0) == MVT::v4i32) {
07731             Emit_81(Result, N, X86::ANDNPSrr, MVT::v2i64, N0, N00, N01, N010, N1);
07732             return;
07733           }
07734         }
07735       }
07736     }
07737   }
07738   if ((Subtarget->hasSSE2())) {
07739     N0 = N.getOperand(0);
07740     if (N0.getOpcode() == ISD::XOR) {
07741       N00 = N0.getOperand(0);
07742       N01 = N0.getOperand(1);
07743       if (N01.getOpcode() == ISD::BIT_CONVERT) {
07744         N010 = N01.getOperand(0);
07745         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
07746             Predicate_immAllOnesV(N010.Val)) {
07747           N1 = N.getOperand(1);
07748           if (N.Val->getValueType(0) == MVT::v2i64) {
07749 
07750             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07751             // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07752             // Pattern complexity = 9  cost = 1  size = 3
07753             if (N010.Val->getValueType(0) == MVT::v4i32) {
07754               Emit_81(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N01, N010, N1);
07755               return;
07756             }
07757 
07758             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07759             // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07760             // Pattern complexity = 9  cost = 1  size = 3
07761             if (N010.Val->getValueType(0) == MVT::v8i16) {
07762               Emit_81(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N01, N010, N1);
07763               return;
07764             }
07765 
07766             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07767             // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07768             // Pattern complexity = 9  cost = 1
07769             if (N010.Val->getValueType(0) == MVT::v16i8) {
07770               Emit_81(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N01, N010, N1);
07771               return;
07772             }
07773           }
07774         }
07775       }
07776     }
07777   }
07778   if ((Subtarget->hasSSE1())) {
07779     N0 = N.getOperand(0);
07780 
07781     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07782     // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07783     // Pattern complexity = 9  cost = 1  size = 3
07784     if (N0.getOpcode() == ISD::XOR) {
07785       N00 = N0.getOperand(0);
07786       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07787         N000 = N00.getOperand(0);
07788         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07789             Predicate_immAllOnesV(N000.Val)) {
07790           N01 = N0.getOperand(1);
07791           N1 = N.getOperand(1);
07792           if (N.Val->getValueType(0) == MVT::v2i64 &&
07793               N000.Val->getValueType(0) == MVT::v4i32) {
07794             Emit_82(Result, N, X86::ANDNPSrr, MVT::v2i64, N0, N00, N000, N01, N1);
07795             return;
07796           }
07797         }
07798       }
07799     }
07800     N1 = N.getOperand(1);
07801     if (N1.getOpcode() == ISD::XOR) {
07802       N10 = N1.getOperand(0);
07803 
07804       // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
07805       // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07806       // Pattern complexity = 9  cost = 1  size = 3
07807       {
07808         N11 = N1.getOperand(1);
07809         if (N11.getOpcode() == ISD::BIT_CONVERT) {
07810           N110 = N11.getOperand(0);
07811           if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07812               Predicate_immAllOnesV(N110.Val) &&
07813               N.Val->getValueType(0) == MVT::v2i64 &&
07814               N110.Val->getValueType(0) == MVT::v4i32) {
07815             Emit_83(Result, N, X86::ANDNPSrr, MVT::v2i64, N0, N1, N10, N11, N110);
07816             return;
07817           }
07818         }
07819       }
07820 
07821       // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07822       // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07823       // Pattern complexity = 9  cost = 1
07824       if (N10.getOpcode() == ISD::BIT_CONVERT) {
07825         N100 = N10.getOperand(0);
07826         if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07827             Predicate_immAllOnesV(N100.Val)) {
07828           N11 = N1.getOperand(1);
07829           if (N.Val->getValueType(0) == MVT::v2i64 &&
07830               N100.Val->getValueType(0) == MVT::v4i32) {
07831             Emit_84(Result, N, X86::ANDNPSrr, MVT::v2i64, N0, N1, N10, N100, N11);
07832             return;
07833           }
07834         }
07835       }
07836     }
07837   }
07838   if ((Subtarget->hasSSE2())) {
07839     N0 = N.getOperand(0);
07840 
07841     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07842     // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07843     // Pattern complexity = 9  cost = 1  size = 3
07844     if (N0.getOpcode() == ISD::XOR) {
07845       N00 = N0.getOperand(0);
07846       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07847         N000 = N00.getOperand(0);
07848         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07849             Predicate_immAllOnesV(N000.Val)) {
07850           N01 = N0.getOperand(1);
07851           N1 = N.getOperand(1);
07852           if (N.Val->getValueType(0) == MVT::v2i64 &&
07853               N000.Val->getValueType(0) == MVT::v8i16) {
07854             Emit_82(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N000, N01, N1);
07855             return;
07856           }
07857         }
07858       }
07859     }
07860     {
07861       N1 = N.getOperand(1);
07862       if (N1.getOpcode() == ISD::XOR) {
07863         N10 = N1.getOperand(0);
07864 
07865         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)))
07866         // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07867         // Pattern complexity = 9  cost = 1  size = 3
07868         {
07869           N11 = N1.getOperand(1);
07870           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07871             N110 = N11.getOperand(0);
07872             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07873                 Predicate_immAllOnesV(N110.Val) &&
07874                 N.Val->getValueType(0) == MVT::v2i64 &&
07875                 N110.Val->getValueType(0) == MVT::v8i16) {
07876               Emit_83(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N11, N110);
07877               return;
07878             }
07879           }
07880         }
07881 
07882         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07883         // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07884         // Pattern complexity = 9  cost = 1
07885         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07886           N100 = N10.getOperand(0);
07887           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07888               Predicate_immAllOnesV(N100.Val)) {
07889             N11 = N1.getOperand(1);
07890             if (N.Val->getValueType(0) == MVT::v2i64 &&
07891                 N100.Val->getValueType(0) == MVT::v8i16) {
07892               Emit_84(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N100, N11);
07893               return;
07894             }
07895           }
07896         }
07897       }
07898     }
07899 
07900     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07901     // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07902     // Pattern complexity = 9  cost = 1  size = 3
07903     if (N0.getOpcode() == ISD::XOR) {
07904       N00 = N0.getOperand(0);
07905       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07906         N000 = N00.getOperand(0);
07907         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07908             Predicate_immAllOnesV(N000.Val)) {
07909           N01 = N0.getOperand(1);
07910           N1 = N.getOperand(1);
07911           if (N.Val->getValueType(0) == MVT::v2i64 &&
07912               N000.Val->getValueType(0) == MVT::v16i8) {
07913             Emit_82(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N000, N01, N1);
07914             return;
07915           }
07916         }
07917       }
07918     }
07919     {
07920       N1 = N.getOperand(1);
07921       if (N1.getOpcode() == ISD::XOR) {
07922         N10 = N1.getOperand(0);
07923 
07924         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)))
07925         // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07926         // Pattern complexity = 9  cost = 1  size = 3
07927         {
07928           N11 = N1.getOperand(1);
07929           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07930             N110 = N11.getOperand(0);
07931             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07932                 Predicate_immAllOnesV(N110.Val) &&
07933                 N.Val->getValueType(0) == MVT::v2i64 &&
07934                 N110.Val->getValueType(0) == MVT::v16i8) {
07935               Emit_83(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N11, N110);
07936               return;
07937             }
07938           }
07939         }
07940 
07941         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07942         // Emits: (PANDNrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
07943         // Pattern complexity = 9  cost = 1
07944         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07945           N100 = N10.getOperand(0);
07946           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07947               Predicate_immAllOnesV(N100.Val)) {
07948             N11 = N1.getOperand(1);
07949             if (N.Val->getValueType(0) == MVT::v2i64 &&
07950                 N100.Val->getValueType(0) == MVT::v16i8) {
07951               Emit_84(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N100, N11);
07952               return;
07953             }
07954           }
07955         }
07956       }
07957     }
07958     if (N0.getOpcode() == ISD::XOR) {
07959       N00 = N0.getOperand(0);
07960 
07961       // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src2)
07962       // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07963       // Pattern complexity = 7  cost = 1  size = 3
07964       {
07965         N01 = N0.getOperand(1);
07966         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07967             Predicate_immAllOnesV(N01.Val)) {
07968           N1 = N.getOperand(1);
07969           if (N.Val->getValueType(0) == MVT::v2i64) {
07970             Emit_85(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N01, N1);
07971             return;
07972           }
07973         }
07974       }
07975 
07976       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1), VR128:v2i64:$src2)
07977       // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07978       // Pattern complexity = 7  cost = 1
07979       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07980           Predicate_immAllOnesV(N00.Val)) {
07981         N01 = N0.getOperand(1);
07982         N1 = N.getOperand(1);
07983         if (N.Val->getValueType(0) == MVT::v2i64) {
07984           Emit_86(Result, N, X86::PANDNrr, MVT::v2i64, N0, N00, N01, N1);
07985           return;
07986         }
07987       }
07988     }
07989     {
07990       N1 = N.getOperand(1);
07991       if (N1.getOpcode() == ISD::XOR) {
07992         N10 = N1.getOperand(0);
07993 
07994         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07995         // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07996         // Pattern complexity = 7  cost = 1  size = 3
07997         {
07998           N11 = N1.getOperand(1);
07999           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
08000               Predicate_immAllOnesV(N11.Val) &&
08001               N.Val->getValueType(0) == MVT::v2i64) {
08002             Emit_87(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N11);
08003             return;
08004           }
08005         }
08006 
08007         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1))
08008         // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08009         // Pattern complexity = 7  cost = 1
08010         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
08011             Predicate_immAllOnesV(N10.Val)) {
08012           N11 = N1.getOperand(1);
08013           if (N.Val->getValueType(0) == MVT::v2i64) {
08014             Emit_88(Result, N, X86::PANDNrr, MVT::v2i64, N0, N1, N10, N11);
08015             return;
08016           }
08017         }
08018       }
08019     }
08020 
08021     // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
08022     // Emits: (ANDPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
08023     // Pattern complexity = 6  cost = 1
08024     if (N0.getOpcode() == ISD::BIT_CONVERT) {
08025       N00 = N0.getOperand(0);
08026       N1 = N.getOperand(1);
08027       if (N1.getOpcode() == ISD::BIT_CONVERT) {
08028         N10 = N1.getOperand(0);
08029         if (N.Val->getValueType(0) == MVT::v2i64 &&
08030             N00.Val->getValueType(0) == MVT::v2f64 &&
08031             N10.Val->getValueType(0) == MVT::v2f64) {
08032           Emit_89(Result, N, X86::ANDPDrr, MVT::v2i64, N0, N00, N1, N10);
08033           return;
08034         }
08035       }
08036     }
08037   }
08038   {
08039     N0 = N.getOperand(0);
08040     N1 = N.getOperand(1);
08041     if (N1.getOpcode() == ISD::Constant) {
08042 
08043       // Pattern: (and:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
08044       // Emits: (AND16ri8:i16 GR16:i16:$src1, (imm:i16):$src2)
08045       // Pattern complexity = 5  cost = 1  size = 3
08046       if (Predicate_i16immSExt8(N1.Val) &&
08047           N.Val->getValueType(0) == MVT::i16) {
08048         Emit_49(Result, N, X86::AND16ri8, MVT::i16, N0, N1);
08049         return;
08050       }
08051 
08052       // Pattern: (and:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
08053       // Emits: (AND32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
08054       // Pattern complexity = 5  cost = 1  size = 3
08055       if (Predicate_i32immSExt8(N1.Val) &&
08056           N.Val->getValueType(0) == MVT::i32) {
08057         Emit_29(Result, N, X86::AND32ri8, MVT::i32, N0, N1);
08058         return;
08059       }
08060 
08061       // Pattern: (and:i8 GR8:i8:$src1, (imm:i8):$src2)
08062       // Emits: (AND8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
08063       // Pattern complexity = 4  cost = 1  size = 3
08064       if (N.Val->getValueType(0) == MVT::i8) {
08065         Emit_50(Result, N, X86::AND8ri, MVT::i8, N0, N1);
08066         return;
08067       }
08068 
08069       // Pattern: (and:i16 GR16:i16:$src1, (imm:i16):$src2)
08070       // Emits: (AND16ri:i16 GR16:i16:$src1, (imm:i16):$src2)
08071       // Pattern complexity = 4  cost = 1  size = 3
08072       if (N.Val->getValueType(0) == MVT::i16) {
08073         Emit_49(Result, N, X86::AND16ri, MVT::i16, N0, N1);
08074         return;
08075       }
08076 
08077       // Pattern: (and:i32 GR32:i32:$src1, (imm:i32):$src2)
08078       // Emits: (AND32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
08079       // Pattern complexity = 4  cost = 1
08080       if (N.Val->getValueType(0) == MVT::i32) {
08081         Emit_29(Result, N, X86::AND32ri, MVT::i32, N0, N1);
08082         return;
08083       }
08084     }
08085 
08086     // Pattern: (and:i8 GR8:i8:$src1, GR8:i8:$src2)
08087     // Emits: (AND8rr:i8 GR8:i8:$src1, GR8:i8:$src2)
08088     // Pattern complexity = 2  cost = 1  size = 3
08089     if (N.Val->getValueType(0) == MVT::i8) {
08090       Emit_21(Result, N, X86::AND8rr, MVT::i8, N0, N1);
08091       return;
08092     }
08093 
08094     // Pattern: (and:i16 GR16:i16:$src1, GR16:i16:$src2)
08095     // Emits: (AND16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
08096     // Pattern complexity = 2  cost = 1  size = 3
08097     if (N.Val->getValueType(0) == MVT::i16) {
08098       Emit_21(Result, N, X86::AND16rr, MVT::i16, N0, N1);
08099       return;
08100     }
08101 
08102     // Pattern: (and:i32 GR32:i32:$src1, GR32:i32:$src2)
08103     // Emits: (AND32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
08104     // Pattern complexity = 2  cost = 1
08105     if (N.Val->getValueType(0) == MVT::i32) {
08106       Emit_21(Result, N, X86::AND32rr, MVT::i32, N0, N1);
08107       return;
08108     }
08109   }
08110 
08111   // Pattern: (and:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08112   // Emits: (ANDPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08113   // Pattern complexity = 2  cost = 1  size = 3
08114   if ((Subtarget->hasSSE1())) {
08115     N0 = N.getOperand(0);
08116     N1 = N.getOperand(1);
08117     if (N.Val->getValueType(0) == MVT::v2i64) {
08118       Emit_21(Result, N, X86::ANDPSrr, MVT::v2i64, N0, N1);
08119       return;
08120     }
08121   }
08122 
08123   // Pattern: (and:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08124   // Emits: (PANDrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08125   // Pattern complexity = 2  cost = 1
08126   if ((Subtarget->hasSSE2())) {
08127     N0 = N.getOperand(0);
08128     N1 = N.getOperand(1);
08129     if (N.Val->getValueType(0) == MVT::v2i64) {
08130       Emit_21(Result, N, X86::PANDrr, MVT::v2i64, N0, N1);
08131       return;
08132     }
08133   }
08134   std::cerr << "Cannot yet select: ";
08135   N.Val->dump(CurDAG);
08136   std::cerr << '\n';
08137   abort();
08138 }
08139 
08140 void Emit_90(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01) NOINLINE {
08141   SDOperand Tmp0(0, 0);
08142   SDOperand Tmp1(0, 0);
08143   SDOperand Tmp2(0, 0);
08144   SDOperand Tmp3(0, 0);
08145   SDNode *ResNode = NULL;
08146   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
08147   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
08148   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
08149   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
08150   Select(Tmp0, CPTmp0);
08151   Select(Tmp1, CPTmp1);
08152   Select(Tmp2, CPTmp2);
08153   Select(Tmp3, CPTmp3);
08154   Select(Chain0, Chain0);
08155   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
08156   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
08157   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
08158   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
08159   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
08160   Chain0 = SDOperand(ResNode, 1);
08161   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08162   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
08163   AddHandleReplacement(N0.Val, 1, ResNode, 1);
08164   Result = SDOperand(ResNode, N.ResNo);
08165   return;
08166 }
08167 void Select_anyext(SDOperand &Result, SDOperand N) {
08168   SDOperand CPTmp0(0, 0);
08169   SDOperand CPTmp1(0, 0);
08170   SDOperand CPTmp2(0, 0);
08171   SDOperand CPTmp3(0, 0);
08172   SDOperand Chain0(0, 0);
08173   SDOperand N0(0, 0);
08174   SDOperand N01(0, 0);
08175   SDOperand Tmp0(0, 0);
08176   SDOperand Tmp1(0, 0);
08177   SDOperand Tmp2(0, 0);
08178   SDOperand Tmp3(0, 0);
08179   SDNode *ResNode = NULL;
08180   N0 = N.getOperand(0);
08181   if (N0.getOpcode() == ISD::LOAD &&
08182       InFlightSet.count(N0.Val) == 0 &&
08183       N0.hasOneUse() &&
08184       !CodeGenMap.count(N0.getValue(0))) {
08185     Chain0 = N0.getOperand(0);
08186     N01 = N0.getOperand(1);
08187 
08188     // Pattern: (anyext:i16 (load:i8 addr:iPTR:$src))
08189     // Emits: (MOVZX16rm8:i16 addr:iPTR:$src)
08190     // Pattern complexity = 14  cost = 1  size = 3
08191     if (N.Val->getValueType(0) == MVT::i16 &&
08192         N0.Val->getValueType(0) == MVT::i8) {
08193       bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08194       if (Match) {
08195         Emit_90(Result, N, X86::MOVZX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
08196         return;
08197       }
08198     }
08199     if (N.Val->getValueType(0) == MVT::i32) {
08200 
08201       // Pattern: (anyext:i32 (load:i8 addr:iPTR:$src))
08202       // Emits: (MOVZX32rm8:i32 addr:iPTR:$src)
08203       // Pattern complexity = 14  cost = 1  size = 3
08204       if (N0.Val->getValueType(0) == MVT::i8) {
08205         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08206         if (Match) {
08207           Emit_90(Result, N, X86::MOVZX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
08208           return;
08209         }
08210       }
08211 
08212       // Pattern: (anyext:i32 (load:i16 addr:iPTR:$src))
08213       // Emits: (MOVZX32rm16:i32 addr:iPTR:$src)
08214       // Pattern complexity = 14  cost = 1
08215       if (N0.Val->getValueType(0) == MVT::i16) {
08216         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08217         if (Match) {
08218           Emit_90(Result, N, X86::MOVZX32rm16, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
08219           return;
08220         }
08221       }
08222     }
08223   }
08224 
08225   // Pattern: (anyext:i16 GR8:i8:$src)
08226   // Emits: (MOVZX16rr8:i16 GR8:i8:$src)
08227   // Pattern complexity = 2  cost = 1  size = 3
08228   if (N.Val->getValueType(0) == MVT::i16 &&
08229       N0.Val->getValueType(0) == MVT::i8) {
08230     Emit_36(Result, N, X86::MOVZX16rr8, MVT::i16, N0);
08231     return;
08232   }
08233   if (N.Val->getValueType(0) == MVT::i32) {
08234 
08235     // Pattern: (anyext:i32 GR8:i8:$src)
08236     // Emits: (MOVZX32rr8:i32 GR8:i8:$src)
08237     // Pattern complexity = 2  cost = 1  size = 3
08238     if (N0.Val->getValueType(0) == MVT::i8) {
08239       Emit_36(Result, N, X86::MOVZX32rr8, MVT::i32, N0);
08240       return;
08241     }
08242 
08243     // Pattern: (anyext:i32 GR16:i16:$src)
08244     // Emits: (MOVZX32rr16:i32 GR16:i16:$src)
08245     // Pattern complexity = 2  cost = 1
08246     if (N0.Val->getValueType(0) == MVT::i16) {
08247       Emit_36(Result, N, X86::MOVZX32rr16, MVT::i32, N0);
08248       return;
08249     }
08250   }
08251   std::cerr << "Cannot yet select: ";
08252   N.Val->dump(CurDAG);
08253   std::cerr << '\n';
08254   abort();
08255 }
08256 
08257 void Emit_91(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain010, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N0101, SDOperand &N02) NOINLINE {
08258   SDOperand Tmp0(0, 0);
08259   SDOperand Tmp1(0, 0);
08260   SDOperand Tmp2(0, 0);
08261   SDOperand Tmp3(0, 0);
08262   SDNode *ResNode = NULL;
08263   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
08264   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
08265   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
08266   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
08267   Select(Tmp0, CPTmp0);
08268   Select(Tmp1, CPTmp1);
08269   Select(Tmp2, CPTmp2);
08270   Select(Tmp3, CPTmp3);
08271   Select(Chain010, Chain010);
08272   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
08273   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
08274   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
08275   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
08276   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain010);
08277   Chain010 = SDOperand(ResNode, 1);
08278   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08279   SelectionDAG::InsertISelMapEntry(CodeGenMap, N010.Val, 1, ResNode, 1);
08280   AddHandleReplacement(N010.Val, 1, ResNode, 1);
08281   Result = SDOperand(ResNode, N.ResNo);
08282   return;
08283 }
08284 void Emit_92(SDOperand &Result, SDOperand &N, SDOperand &N0) NOINLINE {
08285   SDOperand Tmp0(0, 0);
08286   Select(Tmp0, N0);
08287   Result = Tmp0;
08288   return;
08289 }
08290 void Select_bitconvert(SDOperand &Result, SDOperand N) {
08291   SDOperand CPTmp0(0, 0);
08292   SDOperand CPTmp1(0, 0);
08293   SDOperand CPTmp2(0, 0);
08294   SDOperand CPTmp3(0, 0);
08295   SDOperand Chain010(0, 0);
08296   SDOperand N0(0, 0);
08297   SDOperand N00(0, 0);
08298   SDOperand N01(0, 0);
08299   SDOperand N010(0, 0);
08300   SDOperand N0101(0, 0);
08301   SDOperand N02(0, 0);
08302   SDOperand Tmp0(0, 0);
08303   SDOperand Tmp1(0, 0);
08304   SDOperand Tmp2(0, 0);
08305   SDOperand Tmp3(0, 0);
08306   SDNode *ResNode = NULL;
08307   if ((Subtarget->hasSSE2())) {
08308     N0 = N.getOperand(0);
08309 
08310     // Pattern: (bitconvert:v2i64 (vector_shuffle:v2f64 (build_vector:v2f64)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src)), (build_vector:v2i32)<<P:Predicate_MOVL_shuffle_mask>>))
08311     // Emits: (MOVZQI2PQIrm:v2i64 addr:iPTR:$src)
08312     // Pattern complexity = 44  cost = 1  size = 3
08313     if (N0.getOpcode() == ISD::VECTOR_SHUFFLE &&
08314         InFlightSet.count(N0.Val) == 0 &&
08315         N0.hasOneUse() &&
08316         !CodeGenMap.count(N0.getValue(0))) {
08317       N00 = N0.getOperand(0);
08318       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
08319           Predicate_immAllZerosV(N00.Val)) {
08320         N01 = N0.getOperand(1);
08321         if (N01.getOpcode() == ISD::SCALAR_TO_VECTOR &&
08322             InFlightSet.count(N01.Val) == 0 &&
08323             N01.hasOneUse() &&
08324             !CodeGenMap.count(N01.getValue(0))) {
08325           N010 = N01.getOperand(0);
08326           if (N010.getOpcode() == ISD::LOAD &&
08327               InFlightSet.count(N010.Val) == 0 &&
08328               N010.hasOneUse() &&
08329               !CodeGenMap.count(N010.getValue(0))) {
08330             Chain010 = N010.getOperand(0);
08331             N0101 = N010.getOperand(1);
08332             N02 = N0.getOperand(2);
08333             if (N02.getOpcode() == ISD::BUILD_VECTOR &&
08334                 Predicate_MOVL_shuffle_mask(N02.Val) &&
08335                 N.Val->getValueType(0) == MVT::v2i64 &&
08336                 N0.Val->getValueType(0) == MVT::v2f64 &&
08337                 N010.Val->getValueType(0) == MVT::f64) {
08338               bool Match = SelectAddr(N0101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08339               if (Match) {
08340                 Emit_91(Result, N, X86::MOVZQI2PQIrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain010, N0, N00, N01, N010, N0101, N02);
08341                 return;
08342               }
08343             }
08344           }
08345         }
08346       }
08347     }
08348     if (N.Val->getValueType(0) == MVT::v2i64) {
08349 
08350       // Pattern: (bitconvert:v2i64 VR128:v4i32:$src)
08351       // Emits: VR128:v2i64:$src
08352       // Pattern complexity = 2  cost = 0  size = 0
08353       if (N0.Val->getValueType(0) == MVT::v4i32) {
08354         Emit_92(Result, N, N0);
08355         return;
08356       }
08357 
08358       // Pattern: (bitconvert:v2i64 VR128:v8i16:$src)
08359       // Emits: VR128:v2i64:$src
08360       // Pattern complexity = 2  cost = 0  size = 0
08361       if (N0.Val->getValueType(0) == MVT::v8i16) {
08362         Emit_92(Result, N, N0);
08363         return;
08364       }
08365 
08366       // Pattern: (bitconvert:v2i64 VR128:v16i8:$src)
08367       // Emits: VR128:v2i64:$src
08368       // Pattern complexity = 2  cost = 0  size = 0
08369       if (N0.Val->getValueType(0) == MVT::v16i8) {
08370         Emit_92(Result, N, N0);
08371         return;
08372       }
08373 
08374       // Pattern: (bitconvert:v2i64 VR128:v2f64:$src)
08375       // Emits: VR128:v2i64:$src
08376       // Pattern complexity = 2  cost = 0  size = 0
08377       if (N0.Val->getValueType(0) == MVT::v2f64) {
08378         Emit_92(Result, N, N0);
08379         return;
08380       }
08381 
08382       // Pattern: (bitconvert:v2i64 VR128:v4f32:$src)
08383       // Emits: VR128:v2i64:$src
08384       // Pattern complexity = 2  cost = 0
08385       if (N0.Val->getValueType(0) == MVT::v4f32) {
08386         Emit_92(Result, N, N0);
08387         return;
08388       }
08389     }
08390     if (N.Val->getValueType(0) == MVT::v4i32) {
08391 
08392       // Pattern: (bitconvert:v4i32 VR128:v2i64:$src)
08393       // Emits: VR128:v4i32:$src
08394       // Pattern complexity = 2  cost = 0  size = 0
08395       if (N0.Val->getValueType(0) == MVT::v2i64) {
08396         Emit_92(Result, N, N0);
08397         return;
08398       }
08399 
08400       // Pattern: (bitconvert:v4i32 VR128:v8i16:$src)
08401       // Emits: VR128:v4i32:$src
08402       // Pattern complexity = 2  cost = 0  size = 0
08403       if (N0.Val->getValueType(0) == MVT::v8i16) {
08404         Emit_92(Result, N, N0);
08405         return;
08406       }
08407 
08408       // Pattern: (bitconvert:v4i32 VR128:v16i8:$src)
08409       // Emits: VR128:v4i32:$src
08410       // Pattern complexity = 2  cost = 0  size = 0
08411       if (N0.Val->getValueType(0) == MVT::v16i8) {
08412         Emit_92(Result, N, N0);
08413         return;
08414       }
08415 
08416       // Pattern: (bitconvert:v4i32 VR128:v2f64:$src)
08417       // Emits: VR128:v4i32:$src
08418       // Pattern complexity = 2  cost = 0  size = 0
08419       if (N0.Val->getValueType(0) == MVT::v2f64) {
08420         Emit_92(Result, N, N0);
08421         return;
08422       }
08423 
08424       // Pattern: (bitconvert:v4i32 VR128:v4f32:$src)
08425       // Emits: VR128:v4i32:$src
08426       // Pattern complexity = 2  cost = 0
08427       if (N0.Val->getValueType(0) == MVT::v4f32) {
08428         Emit_92(Result, N, N0);
08429         return;
08430       }
08431     }
08432     if (N.Val->getValueType(0) == MVT::v8i16) {
08433 
08434       // Pattern: (bitconvert:v8i16 VR128:v2i64:$src)
08435       // Emits: VR128:v8i16:$src
08436       // Pattern complexity = 2  cost = 0  size = 0
08437       if (N0.Val->getValueType(0) == MVT::v2i64) {
08438         Emit_92(Result, N, N0);
08439         return;
08440       }
08441 
08442       // Pattern: (bitconvert:v8i16 VR128:v4i32:$src)
08443       // Emits: VR128:v8i16:$src
08444       // Pattern complexity = 2  cost = 0  size = 0
08445       if (N0.Val->getValueType(0) == MVT::v4i32) {
08446         Emit_92(Result, N, N0);
08447         return;
08448       }
08449 
08450       // Pattern: (bitconvert:v8i16 VR128:v16i8:$src)
08451       // Emits: VR128:v8i16:$src
08452       // Pattern complexity = 2  cost = 0  size = 0
08453       if (N0.Val->getValueType(0) == MVT::v16i8) {
08454         Emit_92(Result, N, N0);
08455         return;
08456       }
08457 
08458       // Pattern: (bitconvert:v8i16 VR128:v2f64:$src)
08459       // Emits: VR128:v8i16:$src
08460       // Pattern complexity = 2  cost = 0  size = 0
08461       if (N0.Val->getValueType(0) == MVT::v2f64) {
08462         Emit_92(Result, N, N0);
08463         return;
08464       }
08465 
08466       // Pattern: (bitconvert:v8i16 VR128:v4f32:$src)
08467       // Emits: VR128:v8i16:$src
08468       // Pattern complexity = 2  cost = 0
08469       if (N0.Val->getValueType(0) == MVT::v4f32) {
08470         Emit_92(Result, N, N0);
08471         return;
08472       }
08473     }
08474     if (N.Val->getValueType(0) == MVT::v16i8) {
08475 
08476       // Pattern: (bitconvert:v16i8 VR128:v2i64:$src)
08477       // Emits: VR128:v16i8:$src
08478       // Pattern complexity = 2  cost = 0  size = 0
08479       if (N0.Val->getValueType(0) == MVT::v2i64) {
08480         Emit_92(Result, N, N0);
08481         return;
08482       }
08483 
08484       // Pattern: (bitconvert:v16i8 VR128:v4i32:$src)
08485       // Emits: VR128:v16i8:$src
08486       // Pattern complexity = 2  cost = 0  size = 0
08487       if (N0.Val->getValueType(0) == MVT::v4i32) {
08488         Emit_92(Result, N, N0);
08489         return;
08490       }
08491 
08492       // Pattern: (bitconvert:v16i8 VR128:v8i16:$src)
08493       // Emits: VR128:v16i8:$src
08494       // Pattern complexity = 2  cost = 0  size = 0
08495       if (N0.Val->getValueType(0) == MVT::v8i16) {
08496         Emit_92(Result, N, N0);
08497         return;
08498       }
08499 
08500       // Pattern: (bitconvert:v16i8 VR128:v2f64:$src)
08501       // Emits: VR128:v16i8:$src
08502       // Pattern complexity = 2  cost = 0  size = 0
08503       if (N0.Val->getValueType(0) == MVT::v2f64) {
08504         Emit_92(Result, N, N0);
08505         return;
08506       }
08507 
08508       // Pattern: (bitconvert:v16i8 VR128:v4f32:$src)
08509       // Emits: VR128:v16i8:$src
08510       // Pattern complexity = 2  cost = 0
08511       if (N0.Val->getValueType(0) == MVT::v4f32) {
08512         Emit_92(Result, N, N0);
08513         return;
08514       }
08515     }
08516     if (N.Val->getValueType(0) == MVT::v4f32) {
08517 
08518       // Pattern: (bitconvert:v4f32 VR128:v2i64:$src)
08519       // Emits: VR128:v4f32:$src
08520       // Pattern complexity = 2  cost = 0  size = 0
08521       if (N0.Val->getValueType(0) == MVT::v2i64) {
08522         Emit_92(Result, N, N0);
08523         return;
08524       }
08525 
08526       // Pattern: (bitconvert:v4f32 VR128:v4i32:$src)
08527       // Emits: VR128:v4f32:$src
08528       // Pattern complexity = 2  cost = 0  size = 0
08529       if (N0.Val->getValueType(0) == MVT::v4i32) {
08530         Emit_92(Result, N, N0);
08531         return;
08532       }
08533 
08534       // Pattern: (bitconvert:v4f32 VR128:v8i16:$src)
08535       // Emits: VR128:v4f32:$src
08536       // Pattern complexity = 2  cost = 0  size = 0
08537       if (N0.Val->getValueType(0) == MVT::v8i16) {
08538         Emit_92(Result, N, N0);
08539         return;
08540       }
08541 
08542       // Pattern: (bitconvert:v4f32 VR128:v16i8:$src)
08543       // Emits: VR128:v4f32:$src
08544       // Pattern complexity = 2  cost = 0  size = 0
08545       if (N0.Val->getValueType(0) == MVT::v16i8) {
08546         Emit_92(Result, N, N0);
08547         return;
08548       }
08549 
08550       // Pattern: (bitconvert:v4f32 VR128:v2f64:$src)
08551       // Emits: VR128:v4f32:$src
08552       // Pattern complexity = 2  cost = 0
08553       if (N0.Val->getValueType(0) == MVT::v2f64) {
08554         Emit_92(Result, N, N0);
08555         return;
08556       }
08557     }
08558     if (N.Val->getValueType(0) == MVT::v2f64) {
08559 
08560       // Pattern: (bitconvert:v2f64 VR128:v2i64:$src)
08561       // Emits: VR128:v2f64:$src
08562       // Pattern complexity = 2  cost = 0  size = 0
08563       if (N0.Val->getValueType(0) == MVT::v2i64) {
08564         Emit_92(Result, N, N0);
08565         return;
08566       }
08567 
08568       // Pattern: (bitconvert:v2f64 VR128:v4i32:$src)
08569       // Emits: VR128:v2f64:$src
08570       // Pattern complexity = 2  cost = 0  size = 0
08571       if (N0.Val->getValueType(0) == MVT::v4i32) {
08572         Emit_92(Result, N, N0);
08573         return;
08574       }
08575 
08576       // Pattern: (bitconvert:v2f64 VR128:v8i16:$src)
08577       // Emits: VR128:v2f64:$src
08578       // Pattern complexity = 2  cost = 0  size = 0
08579       if (N0.Val->getValueType(0) == MVT::v8i16) {
08580         Emit_92(Result, N, N0);
08581         return;
08582       }
08583 
08584       // Pattern: (bitconvert:v2f64 VR128:v16i8:$src)
08585       // Emits: VR128:v2f64:$src
08586       // Pattern complexity = 2  cost = 0  size = 0
08587       if (N0.Val->getValueType(0) == MVT::v16i8) {
08588         Emit_92(Result, N, N0);
08589         return;
08590       }
08591 
08592       // Pattern: (bitconvert:v2f64 VR128:v4f32:$src)
08593       // Emits: VR128:v2f64:$src
08594       // Pattern complexity = 2  cost = 0
08595       if (N0.Val->getValueType(0) == MVT::v4f32) {
08596         Emit_92(Result, N, N0);
08597         return;
08598       }
08599     }
08600   }
08601   std::cerr << "Cannot yet select: ";
08602   N.Val->dump(CurDAG);
08603   std::cerr << '\n';
08604   abort();
08605 }
08606 
08607 void Emit_93(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
08608   SDOperand Tmp0(0, 0);
08609   SDNode *ResNode = NULL;
08610   Select(Tmp0, N1);
08611   Select(Chain, Chain);
08612   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Chain);
08613   Chain = SDOperand(ResNode, 0);
08614   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
08615   Result = SDOperand(ResNode, N.ResNo);
08616   return;
08617 }
08618 void Select_br(SDOperand &Result, SDOperand N) {
08619   SDOperand Chain(0, 0);
08620   SDOperand N1(0, 0);
08621   SDOperand Tmp0(0, 0);
08622   SDNode *ResNode = NULL;
08623   Chain = N.getOperand(0);
08624   N1 = N.getOperand(1);
08625   if (N1.getOpcode() == ISD::BasicBlock) {
08626     Emit_93(Result, N, X86::JMP, Chain, N1);
08627     return;
08628   }
08629   std::cerr << "Cannot yet select: ";
08630   N.Val->dump(CurDAG);
08631   std::cerr << '\n';
08632   abort();
08633 }
08634 
08635 void Emit_94(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain1, SDOperand &N1, SDOperand &N11) NOINLINE {
08636   SDOperand Tmp0(0, 0);
08637   SDOperand Tmp1(0, 0);
08638   SDOperand Tmp2(0, 0);
08639   SDOperand Tmp3(0, 0);
08640   SDNode *ResNode = NULL;
08641   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
08642   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
08643   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
08644   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
08645   Select(Tmp0, CPTmp0);
08646   Select(Tmp1, CPTmp1);
08647   Select(Tmp2, CPTmp2);
08648   Select(Tmp3, CPTmp3);
08649   Select(Chain1, Chain1);
08650   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
08651   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
08652   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
08653   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
08654   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
08655   Chain1 = SDOperand(ResNode, 0);
08656   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain1.Val, Chain1.ResNo);
08657   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
08658   AddHandleReplacement(N1.Val, 1, ResNode, 0);
08659   Result = SDOperand(ResNode, N.ResNo);
08660   return;
08661 }
08662 void Emit_95(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
08663   SDOperand Tmp0(0, 0);
08664   SDNode *ResNode = NULL;
08665   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
08666   Select(Tmp0, N1);
08667   Select(Chain, Chain);
08668   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
08669   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Chain);
08670   Chain = SDOperand(ResNode, 0);
08671   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
08672   Result = SDOperand(ResNode, N.ResNo);
08673   return;
08674 }
08675 void Select_brind(SDOperand &Result, SDOperand N) {
08676   SDOperand CPTmp0(0, 0);
08677   SDOperand CPTmp1(0, 0);
08678   SDOperand CPTmp2(0, 0);
08679   SDOperand CPTmp3(0, 0);
08680   SDOperand Chain(0, 0);
08681   SDOperand Chain1(0, 0);
08682   SDOperand N1(0, 0);
08683   SDOperand N11(0, 0);
08684   SDOperand Tmp0(0, 0);
08685   SDOperand Tmp1(0, 0);
08686   SDOperand Tmp2(0, 0);
08687   SDOperand Tmp3(0, 0);
08688   SDNode *ResNode = NULL;
08689   Chain = N.getOperand(0);
08690   N1 = N.getOperand(1);
08691 
08692   // Pattern: (brind:void (load:iPTR addr:iPTR:$dst))
08693   // Emits: (JMP32m:void addr:iPTR:$dst)
08694   // Pattern complexity = 14  cost = 1  size = 3
08695   if (N1.getOpcode() == ISD::LOAD &&
08696       InFlightSet.count(N1.Val) == 0 &&
08697       N1.hasOneUse() &&
08698       !CodeGenMap.count(N1.getValue(0)) &&
08699       (N.getNumOperands() == 1 || !isNonImmUse(N.Val, N1.Val)) &&
08700       Chain.Val == N1.Val) {
08701     Chain1 = N1.getOperand(0);
08702     N11 = N1.getOperand(1);
08703     bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08704     if (Match) {
08705       Emit_94(Result, N, X86::JMP32m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain1, N1, N11);
08706       return;
08707     }
08708   }
08709 
08710   // Pattern: (brind:void GR32:i32:$dst)
08711   // Emits: (JMP32r:void GR32:i32:$dst)
08712   // Pattern complexity = 2  cost = 1
08713   if (N1.Val->getValueType(0) == MVT::i32) {
08714     Emit_95(Result, N, X86::JMP32r, Chain, N1);
08715     return;
08716   }
08717   std::cerr << "Cannot yet select: ";
08718   N.Val->dump(CurDAG);
08719   std::cerr << '\n';
08720   abort();
08721 }
08722 
08723 void Select_bswap(SDOperand &Result, SDOperand N) {
08724   SDOperand N0(0, 0);
08725   SDOperand Tmp0(0, 0);
08726   SDNode *ResNode = NULL;
08727   N0 = N.getOperand(0);
08728   if (N.Val->getValueType(0) == MVT::i32) {
08729     Emit_36(Result, N, X86::BSWAP32r, MVT::i32, N0);
08730     return;
08731   }
08732   std::cerr << "Cannot yet select: ";
08733   N.Val->dump(CurDAG);
08734   std::cerr << '\n';
08735   abort();
08736 }
08737 
08738 void Emit_96(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0) NOINLINE {
08739   SDNode *ResNode = NULL;
08740   if (N.Val->hasOneUse()) {
08741     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0);
08742   } else {
08743     ResNode = CurDAG->getTargetNode(Opc0, VT0);
08744     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08745     Result = SDOperand(ResNode, 0);
08746   }
08747   return;
08748 }
08749 void Select_build_vector(SDOperand &Result, SDOperand N) {
08750   SDNode *ResNode = NULL;
08751 
08752   // Pattern: (build_vector:v4f32)<<P:Predicate_immAllZerosV>>
08753   // Emits: (V_SET0:v4f32)
08754   // Pattern complexity = 3  cost = 1  size = 3
08755   if ((Subtarget->hasSSE1()) &&
08756       Predicate_immAllZerosV(N.Val) &&
08757       N.Val->getValueType(0) == MVT::v4f32) {
08758     Emit_96(Result, N, X86::V_SET0, MVT::v4f32);
08759     return;
08760   }
08761   if ((Subtarget->hasSSE2())) {
08762 
08763     // Pattern: (build_vector:v2f64)<<P:Predicate_immAllOnesV>>
08764     // Emits: (V_SETALLONES:v2f64)
08765     // Pattern complexity = 3  cost = 1  size = 3
08766     if (Predicate_immAllOnesV(N.Val) &&
08767         N.Val->getValueType(0) == MVT::v2f64) {
08768       Emit_96(Result, N, X86::V_SETALLONES, MVT::v2f64);
08769       return;
08770     }
08771     if (Predicate_immAllZerosV(N.Val)) {
08772 
08773       // Pattern: (build_vector:v4i32)<<P:Predicate_immAllZerosV>>
08774       // Emits: (V_SET0:v4i32)
08775       // Pattern complexity = 3  cost = 1  size = 3
08776       if (N.Val->getValueType(0) == MVT::v4i32) {
08777         Emit_96(Result, N, X86::V_SET0, MVT::v4i32);
08778         return;
08779       }
08780 
08781       // Pattern: (build_vector:v2i64)<<P:Predicate_immAllZerosV>>
08782       // Emits: (V_SET0:v2i64)
08783       // Pattern complexity = 3  cost = 1  size = 3
08784       if (N.Val->getValueType(0) == MVT::v2i64) {
08785         Emit_96(Result, N, X86::V_SET0, MVT::v2i64);
08786         return;
08787       }
08788 
08789       // Pattern: (build_vector:v2f64)<<P:Predicate_immAllZerosV>>
08790       // Emits: (V_SET0:v2f64)
08791       // Pattern complexity = 3  cost = 1
08792       if (N.Val->getValueType(0) == MVT::v2f64) {
08793         Emit_96(Result, N, X86::V_SET0, MVT::v2f64);
08794         return;
08795       }
08796     }
08797     if (Predicate_immAllOnesV(N.Val)) {
08798 
08799       // Pattern: (build_vector:v16i8)<<P:Predicate_immAllOnesV>>
08800       // Emits: (V_SETALLONES:v16i8)
08801       // Pattern complexity = 3  cost = 1  size = 3
08802       if (N.Val->getValueType(0) == MVT::v16i8) {
08803         Emit_96(Result, N, X86::V_SETALLONES, MVT::v16i8);
08804         return;
08805       }
08806 
08807       // Pattern: (build_vector:v8i16)<<P:Predicate_immAllOnesV>>
08808       // Emits: (V_SETALLONES:v8i16)
08809       // Pattern complexity = 3  cost = 1  size = 3
08810       if (N.Val->getValueType(0) == MVT::v8i16) {
08811         Emit_96(Result, N, X86::V_SETALLONES, MVT::v8i16);
08812         return;
08813       }
08814 
08815       // Pattern: (build_vector:v4i32)<<P:Predicate_immAllOnesV>>
08816       // Emits: (V_SETALLONES:v4i32)
08817       // Pattern complexity = 3  cost = 1  size = 3
08818       if (N.Val->getValueType(0) == MVT::v4i32) {
08819         Emit_96(Result, N, X86::V_SETALLONES, MVT::v4i32);
08820         return;
08821       }
08822 
08823       // Pattern: (build_vector:v2i64)<<P:Predicate_immAllOnesV>>
08824       // Emits: (V_SETALLONES:v2i64)
08825       // Pattern complexity = 3  cost = 1
08826       if (N.Val->getValueType(0) == MVT::v2i64) {
08827         Emit_96(Result, N, X86::V_SETALLONES, MVT::v2i64);
08828         return;
08829       }
08830     }
08831   }
08832 
08833   // Pattern: (build_vector:v4f32)<<P:Predicate_immAllOnesV>>
08834   // Emits: (V_SETALLONES:v4f32)
08835   // Pattern complexity = 3  cost = 1  size = 3
08836   if ((Subtarget->hasSSE1()) &&
08837       Predicate_immAllOnesV(N.Val) &&
08838       N.Val->getValueType(0) == MVT::v4f32) {
08839     Emit_96(Result, N, X86::V_SETALLONES, MVT::v4f32);
08840     return;
08841   }
08842   if ((Subtarget->hasSSE2()) &&
08843       Predicate_immAllZerosV(N.Val)) {
08844 
08845     // Pattern: (build_vector:v16i8)<<P:Predicate_immAllZerosV>>
08846     // Emits: (V_SET0:v16i8)
08847     // Pattern complexity = 3  cost = 1  size = 3
08848     if (N.Val->getValueType(0) == MVT::v16i8) {
08849       Emit_96(Result, N, X86::V_SET0, MVT::v16i8);
08850       return;
08851     }
08852 
08853     // Pattern: (build_vector:v8i16)<<P:Predicate_immAllZerosV>>
08854     // Emits: (V_SET0:v8i16)
08855     // Pattern complexity = 3  cost = 1
08856     if (N.Val->getValueType(0) == MVT::v8i16) {
08857       Emit_96(Result, N, X86::V_SET0, MVT::v8i16);
08858       return;
08859     }
08860   }
08861   std::cerr << "Cannot yet select: ";
08862   N.Val->dump(CurDAG);
08863   std::cerr << '\n';
08864   abort();
08865 }
08866 
08867 void Select_dwarf_label(SDOperand &Result, SDOperand N) {
08868   SDOperand Chain(0, 0);
08869   SDOperand N1(0, 0);
08870   SDOperand Tmp0(0, 0);
08871   SDNode *ResNode = NULL;
08872   Chain = N.getOperand(0);
08873   N1 = N.getOperand(1);
08874   if (N1.getOpcode() == ISD::Constant &&
08875       N1.Val->getValueType(0) == MVT::i32) {
08876     Emit_6(Result, N, X86::DWARF_LABEL, Chain, N1);
08877     return;
08878   }
08879   std::cerr << "Cannot yet select: ";
08880   N.Val->dump(CurDAG);
08881   std::cerr << '\n';
08882   abort();
08883 }
08884 
08885 void Emit_97(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
08886   SDOperand Tmp0(0, 0);
08887   SDOperand Tmp1(0, 0);
08888   SDOperand Tmp2(0, 0);
08889   SDNode *ResNode = NULL;
08890   Tmp0 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
08891   Tmp1 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N2)->getValue()), MVT::i32);
08892   Tmp2 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N3)->getValue()), MVT::i32);
08893   Select(Chain, Chain);
08894   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Chain);
08895   Chain = SDOperand(ResNode, 0);
08896   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
08897   Result = SDOperand(ResNode, N.ResNo);
08898   return;
08899 }
08900 void Select_dwarf_loc(SDOperand &Result, SDOperand N) {
08901   SDOperand Chain(0, 0);
08902   SDOperand N1(0, 0);
08903   SDOperand N2(0, 0);
08904   SDOperand N3(0, 0);
08905   SDOperand Tmp0(0, 0);
08906   SDOperand Tmp1(0, 0);
08907   SDOperand Tmp2(0, 0);
08908   SDNode *ResNode = NULL;
08909   Chain = N.getOperand(0);
08910   N1 = N.getOperand(1);
08911   if (N1.getOpcode() == ISD::Constant) {
08912     N2 = N.getOperand(2);
08913     if (N2.getOpcode() == ISD::Constant) {
08914       N3 = N.getOperand(3);
08915       if (N3.getOpcode() == ISD::Constant &&
08916           N1.Val->getValueType(0) == MVT::i32 &&
08917           N2.Val->getValueType(0) == MVT::i32 &&
08918           N3.Val->getValueType(0) == MVT::i32) {
08919         Emit_97(Result, N, X86::DWARF_LOC, Chain, N1, N2, N3);
08920         return;
08921       }
08922     }
08923   }
08924   std::cerr << "Cannot yet select: ";
08925   N.Val->dump(CurDAG);
08926   std::cerr << '\n';
08927   abort();
08928 }
08929 
08930 void Emit_98(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
08931   SDOperand Tmp0(0, 0);
08932   SDOperand Tmp1(0, 0);
08933   SDOperand Tmp2(0, 0);
08934   SDOperand Tmp3(0, 0);
08935   SDNode *ResNode = NULL;
08936   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
08937   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
08938   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
08939   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
08940   Select(Tmp0, CPTmp0);
08941   Select(Tmp1, CPTmp1);
08942   Select(Tmp2, CPTmp2);
08943   Select(Tmp3, CPTmp3);
08944   Select(Chain, Chain);
08945   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
08946   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
08947   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
08948   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
08949   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
08950   Chain = SDOperand(ResNode, 1);
08951   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08952   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
08953   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
08954   Result = SDOperand(ResNode, N.ResNo);
08955   return;
08956 }
08957 void Select_extld(SDOperand &Result, SDOperand N) {
08958   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
08959     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
08960     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
08961     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
08962     Result = Dummy;
08963     return;
08964   }
08965   SDOperand CPTmp0(0, 0);
08966   SDOperand CPTmp1(0, 0);
08967   SDOperand CPTmp2(0, 0);
08968   SDOperand CPTmp3(0, 0);
08969   SDOperand Chain(0, 0);
08970   SDOperand N1(0, 0);
08971   SDOperand N2(0, 0);
08972   SDOperand N3(0, 0);
08973   SDOperand Tmp0(0, 0);
08974   SDOperand Tmp1(0, 0);
08975   SDOperand Tmp2(0, 0);
08976   SDOperand Tmp3(0, 0);
08977   SDNode *ResNode = NULL;
08978 
08979   // Pattern: (extld:f64 addr:iPTR:$src, srcvalue:Other:$dummy, f32:Other)
08980   // Emits: (FpLD32m:f64 addr:iPTR:$src)
08981   // Pattern complexity = 12  cost = 1  size = 0
08982   if ((!Subtarget->hasSSE2())) {
08983     Chain = N.getOperand(0);
08984     N1 = N.getOperand(1);
08985     N2 = N.getOperand(2);
08986     N3 = N.getOperand(3);
08987     if (cast<VTSDNode>(N3)->getVT() == MVT::f32 &&
08988         N.Val->getValueType(0) == MVT::f64) {
08989       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
08990       if (Match) {
08991         Emit_98(Result, N, X86::FpLD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
08992         return;
08993       }
08994     }
08995   }
08996 
08997   // Pattern: (extld:f64 addr:iPTR:$src, srcvalue:Other:$dummy, f32:Other)
08998   // Emits: (CVTSS2SDrm:f64 addr:iPTR:$src)
08999   // Pattern complexity = 12  cost = 1  size = 3
09000   if ((Subtarget->hasSSE2())) {
09001     Chain = N.getOperand(0);
09002     N1 = N.getOperand(1);
09003     N2 = N.getOperand(2);
09004     N3 = N.getOperand(3);
09005     if (cast<VTSDNode>(N3)->getVT() == MVT::f32 &&
09006         N.Val->getValueType(0) == MVT::f64) {
09007       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09008       if (Match) {
09009         Emit_98(Result, N, X86::CVTSS2SDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09010         return;
09011       }
09012     }
09013   }
09014   Chain = N.getOperand(0);
09015   N1 = N.getOperand(1);
09016   N2 = N.getOperand(2);
09017   N3 = N.getOperand(3);
09018   if (cast<VTSDNode>(N3)->getVT() == MVT::i1) {
09019 
09020     // Pattern: (extld:i8 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
09021     // Emits: (MOV8rm:i8 addr:iPTR:$src)
09022     // Pattern complexity = 12  cost = 1  size = 3
09023     if (N.Val->getValueType(0) == MVT::i8) {
09024       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09025       if (Match) {
09026         Emit_98(Result, N, X86::MOV8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09027         return;
09028       }
09029     }
09030 
09031     // Pattern: (extld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
09032     // Emits: (MOVZX16rm8:i16 addr:iPTR:$src)
09033     // Pattern complexity = 12  cost = 1  size = 3
09034     if (N.Val->getValueType(0) == MVT::i16) {
09035       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09036       if (Match) {
09037         Emit_98(Result, N, X86::MOVZX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09038         return;
09039       }
09040     }
09041 
09042     // Pattern: (extld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
09043     // Emits: (MOVZX32rm8:i32 addr:iPTR:$src)
09044     // Pattern complexity = 12  cost = 1
09045     if (N.Val->getValueType(0) == MVT::i32) {
09046       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09047       if (Match) {
09048         Emit_98(Result, N, X86::MOVZX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09049         return;
09050       }
09051     }
09052   }
09053   if (cast<VTSDNode>(N3)->getVT() == MVT::i8) {
09054 
09055     // Pattern: (extld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
09056     // Emits: (MOVZX16rm8:i16 addr:iPTR:$src)
09057     // Pattern complexity = 12  cost = 1  size = 3
09058     if (N.Val->getValueType(0) == MVT::i16) {
09059       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09060       if (Match) {
09061         Emit_98(Result, N, X86::MOVZX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09062         return;
09063       }
09064     }
09065 
09066     // Pattern: (extld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
09067     // Emits: (MOVZX32rm8:i32 addr:iPTR:$src)
09068     // Pattern complexity = 12  cost = 1
09069     if (N.Val->getValueType(0) == MVT::i32) {
09070       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09071       if (Match) {
09072         Emit_98(Result, N, X86::MOVZX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09073         return;
09074       }
09075     }
09076   }
09077 
09078   // Pattern: (extld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i16:Other)
09079   // Emits: (MOVZX32rm16:i32 addr:iPTR:$src)
09080   // Pattern complexity = 12  cost = 1
09081   if (cast<VTSDNode>(N3)->getVT() == MVT::i16 &&
09082       N.Val->getValueType(0) == MVT::i32) {
09083     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09084     if (Match) {
09085       Emit_98(Result, N, X86::MOVZX32rm16, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
09086       return;
09087     }
09088   }
09089   std::cerr << "Cannot yet select: ";
09090   N.Val->dump(CurDAG);
09091   std::cerr << '\n';
09092   abort();
09093 }
09094 
09095 void Select_fabs(SDOperand &Result, SDOperand N) {
09096   SDOperand N0(0, 0);
09097   SDOperand Tmp0(0, 0);
09098   SDNode *ResNode = NULL;
09099   if ((!Subtarget->hasSSE2())) {
09100     N0 = N.getOperand(0);
09101     if (N.Val->getValueType(0) == MVT::f64) {
09102       Emit_36(Result, N, X86::FpABS, MVT::f64, N0);
09103       return;
09104     }
09105   }
09106   std::cerr << "Cannot yet select: ";
09107   N.Val->dump(CurDAG);
09108   std::cerr << '\n';
09109   abort();
09110 }
09111 
09112 void Emit_99(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11, SDOperand &N12, SDOperand &N13) NOINLINE {
09113   SDOperand Tmp0(0, 0);
09114   SDOperand Tmp1(0, 0);
09115   SDOperand Tmp2(0, 0);
09116   SDOperand Tmp3(0, 0);
09117   SDOperand Tmp4(0, 0);
09118   SDNode *ResNode = NULL;
09119   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
09120   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
09121   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
09122   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
09123   Select(Tmp0, CPTmp0);
09124   Select(Tmp1, CPTmp1);
09125   Select(Tmp2, CPTmp2);
09126   Select(Tmp3, CPTmp3);
09127   Select(Tmp4, N0);
09128   Select(Chain1, Chain1);
09129   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09130   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
09131   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
09132   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
09133   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
09134   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09135   Chain1 = SDOperand(ResNode, 1);
09136   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09137   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09138   AddHandleReplacement(N1.Val, 1, ResNode, 1);
09139   Result = SDOperand(ResNode, N.ResNo);
09140   return;
09141 }
09142 void Emit_100(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11, SDOperand &N12) NOINLINE {
09143   SDOperand Tmp0(0, 0);
09144   SDOperand Tmp1(0, 0);
09145   SDOperand Tmp2(0, 0);
09146   SDOperand Tmp3(0, 0);
09147   SDOperand Tmp4(0, 0);
09148   SDNode *ResNode = NULL;
09149   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
09150   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
09151   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
09152   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
09153   Select(Tmp0, CPTmp0);
09154   Select(Tmp1, CPTmp1);
09155   Select(Tmp2, CPTmp2);
09156   Select(Tmp3, CPTmp3);
09157   Select(Tmp4, N0);
09158   Select(Chain1, Chain1);
09159   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09160   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
09161   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
09162   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
09163   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
09164   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09165   Chain1 = SDOperand(ResNode, 1);
09166   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09167   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09168   AddHandleReplacement(N1.Val, 1, ResNode, 1);
09169   Result = SDOperand(ResNode, N.ResNo);
09170   return;
09171 }
09172 void Emit_101(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N02, SDOperand &N03, SDOperand &N1) NOINLINE {
09173   SDOperand Tmp0(0, 0);
09174   SDOperand Tmp1(0, 0);
09175   SDOperand Tmp2(0, 0);
09176   SDOperand Tmp3(0, 0);
09177   SDOperand Tmp4(0, 0);
09178   SDNode *ResNode = NULL;
09179   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
09180   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
09181   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
09182   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
09183   Select(Tmp0, CPTmp0);
09184   Select(Tmp1, CPTmp1);
09185   Select(Tmp2, CPTmp2);
09186   Select(Tmp3, CPTmp3);
09187   Select(Tmp4, N1);
09188   Select(Chain0, Chain0);
09189   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09190   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
09191   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
09192   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
09193   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
09194   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09195   Chain0 = SDOperand(ResNode, 1);
09196   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09197   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09198   AddHandleReplacement(N0.Val, 1, ResNode, 1);
09199   Result = SDOperand(ResNode, N.ResNo);
09200   return;
09201 }
09202 void Emit_102(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N02, SDOperand &N1) NOINLINE {
09203   SDOperand Tmp0(0, 0);
09204   SDOperand Tmp1(0, 0);
09205   SDOperand Tmp2(0, 0);
09206   SDOperand Tmp3(0, 0);
09207   SDOperand Tmp4(0, 0);
09208   SDNode *ResNode = NULL;
09209   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
09210   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
09211   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
09212   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
09213   Select(Tmp0, CPTmp0);
09214   Select(Tmp1, CPTmp1);
09215   Select(Tmp2, CPTmp2);
09216   Select(Tmp3, CPTmp3);
09217   Select(Tmp4, N1);
09218   Select(Chain0, Chain0);
09219   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09220   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
09221   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
09222   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
09223   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
09224   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09225   Chain0 = SDOperand(ResNode, 1);
09226   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09227   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09228   AddHandleReplacement(N0.Val, 1, ResNode, 1);
09229   Result = SDOperand(ResNode, N.ResNo);
09230   return;
09231 }
09232 void Select_fadd(SDOperand &Result, SDOperand N) {
09233   SDOperand CPTmp0(0, 0);
09234   SDOperand CPTmp1(0, 0);
09235   SDOperand CPTmp2(0, 0);
09236   SDOperand CPTmp3(0, 0);
09237   SDOperand Chain0(0, 0);
09238   SDOperand Chain1(0, 0);
09239   SDOperand N0(0, 0);
09240   SDOperand N01(0, 0);
09241   SDOperand N02(0, 0);
09242   SDOperand N03(0, 0);
09243   SDOperand N1(0, 0);
09244   SDOperand N11(0, 0);
09245   SDOperand N12(0, 0);
09246   SDOperand N13(0, 0);
09247   SDOperand Tmp0(0, 0);
09248   SDOperand Tmp1(0, 0);
09249   SDOperand Tmp2(0, 0);
09250   SDOperand Tmp3(0, 0);
09251   SDOperand Tmp4(0, 0);
09252   SDNode *ResNode = NULL;
09253   if ((!Subtarget->hasSSE2())) {
09254     N0 = N.getOperand(0);
09255     {
09256       N1 = N.getOperand(1);
09257 
09258       // Pattern: (fadd:f64 RFP:f64:$src1, (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other))
09259       // Emits: (FpADD32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09260       // Pattern complexity = 14  cost = 1  size = 0
09261       if (N1.getOpcode() == ISD::EXTLOAD &&
09262           InFlightSet.count(N1.Val) == 0 &&
09263           N1.hasOneUse() &&
09264           !CodeGenMap.count(N1.getValue(0)) &&
09265           !isNonImmUse(N.Val, N1.Val)) {
09266         Chain1 = N1.getOperand(0);
09267         N11 = N1.getOperand(1);
09268         N12 = N1.getOperand(2);
09269         N13 = N1.getOperand(3);
09270         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
09271             N.Val->getValueType(0) == MVT::f64) {
09272           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09273           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09274           if (!Match) {
09275             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09276           }
09277           if (Match) {
09278             Emit_99(Result, N, X86::FpADD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12, N13);
09279             return;
09280           }
09281         }
09282       }
09283 
09284       // Pattern: (fadd:f64 RFP:f64:$src1, (load:f64 addr:iPTR:$src2))
09285       // Emits: (FpADD64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09286       // Pattern complexity = 14  cost = 1  size = 0
09287       if (N1.getOpcode() == ISD::LOAD &&
09288           InFlightSet.count(N1.Val) == 0 &&
09289           N1.hasOneUse() &&
09290           !CodeGenMap.count(N1.getValue(0)) &&
09291           !isNonImmUse(N.Val, N1.Val)) {
09292         Chain1 = N1.getOperand(0);
09293         N11 = N1.getOperand(1);
09294         if (N.Val->getValueType(0) == MVT::f64) {
09295           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09296           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09297           if (!Match) {
09298             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09299           }
09300           if (Match) {
09301             Emit_19(Result, N, X86::FpADD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09302             return;
09303           }
09304         }
09305       }
09306       if (N1.getOpcode() == X86ISD::FILD &&
09307           InFlightSet.count(N1.Val) == 0 &&
09308           N1.hasOneUse() &&
09309           !CodeGenMap.count(N1.getValue(0)) &&
09310           !isNonImmUse(N.Val, N1.Val)) {
09311         Chain1 = N1.getOperand(0);
09312         N11 = N1.getOperand(1);
09313         N12 = N1.getOperand(2);
09314 
09315         // Pattern: (fadd:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i16:Other))
09316         // Emits: (FpIADD16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09317         // Pattern complexity = 14  cost = 1  size = 0
09318         if (cast<VTSDNode>(N12)->getVT() == MVT::i16) {
09319           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09320           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09321           if (!Match) {
09322             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09323           }
09324           if (Match) {
09325             Emit_100(Result, N, X86::FpIADD16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
09326             return;
09327           }
09328         }
09329 
09330         // Pattern: (fadd:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i32:Other))
09331         // Emits: (FpIADD32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09332         // Pattern complexity = 14  cost = 1
09333         if (cast<VTSDNode>(N12)->getVT() == MVT::i32) {
09334           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09335           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09336           if (!Match) {
09337             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09338           }
09339           if (Match) {
09340             Emit_100(Result, N, X86::FpIADD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
09341             return;
09342           }
09343         }
09344       }
09345     }
09346 
09347     // Pattern: (fadd:f64 (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
09348     // Emits: (FpADD32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09349     // Pattern complexity = 14  cost = 1  size = 0
09350     if (N0.getOpcode() == ISD::EXTLOAD &&
09351         InFlightSet.count(N0.Val) == 0 &&
09352         N0.hasOneUse() &&
09353         !CodeGenMap.count(N0.getValue(0)) &&
09354         !isNonImmUse(N.Val, N0.Val)) {
09355       Chain0 = N0.getOperand(0);
09356       N01 = N0.getOperand(1);
09357       N02 = N0.getOperand(2);
09358       N03 = N0.getOperand(3);
09359       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
09360         N1 = N.getOperand(1);
09361         if (N.Val->getValueType(0) == MVT::f64) {
09362           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09363           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09364           if (!Match) {
09365             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09366           }
09367           if (Match) {
09368             Emit_101(Result, N, X86::FpADD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N03, N1);
09369             return;
09370           }
09371         }
09372       }
09373     }
09374 
09375     // Pattern: (fadd:f64 (load:f64 addr:iPTR:$src2), RFP:f64:$src1)
09376     // Emits: (FpADD64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09377     // Pattern complexity = 14  cost = 1  size = 0
09378     if (N0.getOpcode() == ISD::LOAD &&
09379         InFlightSet.count(N0.Val) == 0 &&
09380         N0.hasOneUse() &&
09381         !CodeGenMap.count(N0.getValue(0)) &&
09382         !isNonImmUse(N.Val, N0.Val)) {
09383       Chain0 = N0.getOperand(0);
09384       N01 = N0.getOperand(1);
09385       N1 = N.getOperand(1);
09386       if (N.Val->getValueType(0) == MVT::f64) {
09387         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09388         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09389         if (!Match) {
09390           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09391         }
09392         if (Match) {
09393           Emit_20(Result, N, X86::FpADD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09394           return;
09395         }
09396       }
09397     }
09398     if (N0.getOpcode() == X86ISD::FILD &&
09399         InFlightSet.count(N0.Val) == 0 &&
09400         N0.hasOneUse() &&
09401         !CodeGenMap.count(N0.getValue(0)) &&
09402         !isNonImmUse(N.Val, N0.Val)) {
09403       Chain0 = N0.getOperand(0);
09404       N01 = N0.getOperand(1);
09405       N02 = N0.getOperand(2);
09406 
09407       // Pattern: (fadd:f64 (X86fild:f64 addr:iPTR:$src2, i16:Other), RFP:f64:$src1)
09408       // Emits: (FpIADD16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09409       // Pattern complexity = 14  cost = 1  size = 0
09410       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
09411         N1 = N.getOperand(1);
09412         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09413         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09414         if (!Match) {
09415           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09416         }
09417         if (Match) {
09418           Emit_102(Result, N, X86::FpIADD16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
09419           return;
09420         }
09421       }
09422 
09423       // Pattern: (fadd:f64 (X86fild:f64 addr:iPTR:$src2, i32:Other), RFP:f64:$src1)
09424       // Emits: (FpIADD32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09425       // Pattern complexity = 14  cost = 1
09426       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
09427         N1 = N.getOperand(1);
09428         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09429         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09430         if (!Match) {
09431           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09432         }
09433         if (Match) {
09434           Emit_102(Result, N, X86::FpIADD32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
09435           return;
09436         }
09437       }
09438     }
09439   }
09440 
09441   // Pattern: (fadd:f32 FR32:f32:$src1, (load:f32 addr:iPTR:$src2))
09442   // Emits: (ADDSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
09443   // Pattern complexity = 14  cost = 1  size = 3
09444   if ((Subtarget->hasSSE1())) {
09445     N0 = N.getOperand(0);
09446     N1 = N.getOperand(1);
09447     if (N1.getOpcode() == ISD::LOAD &&
09448         InFlightSet.count(N1.Val) == 0 &&
09449         N1.hasOneUse() &&
09450         !CodeGenMap.count(N1.getValue(0)) &&
09451         !isNonImmUse(N.Val, N1.Val)) {
09452       Chain1 = N1.getOperand(0);
09453       N11 = N1.getOperand(1);
09454       if (N.Val->getValueType(0) == MVT::f32) {
09455         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09456         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09457         if (!Match) {
09458           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09459         }
09460         if (Match) {
09461           Emit_19(Result, N, X86::ADDSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09462           return;
09463         }
09464       }
09465     }
09466   }
09467 
09468   // Pattern: (fadd:f64 FR64:f64:$src1, (load:f64 addr:iPTR:$src2))
09469   // Emits: (ADDSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
09470   // Pattern complexity = 14  cost = 1  size = 3
09471   if ((Subtarget->hasSSE2())) {
09472     N0 = N.getOperand(0);
09473     N1 = N.getOperand(1);
09474     if (N1.getOpcode() == ISD::LOAD &&
09475         InFlightSet.count(N1.Val) == 0 &&
09476         N1.hasOneUse() &&
09477         !CodeGenMap.count(N1.getValue(0)) &&
09478         !isNonImmUse(N.Val, N1.Val)) {
09479       Chain1 = N1.getOperand(0);
09480       N11 = N1.getOperand(1);
09481       if (N.Val->getValueType(0) == MVT::f64) {
09482         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09483         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09484         if (!Match) {
09485           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09486         }
09487         if (Match) {
09488           Emit_19(Result, N, X86::ADDSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09489           return;
09490         }
09491       }
09492     }
09493   }
09494 
09495   // Pattern: (fadd:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
09496   // Emits: (ADDPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
09497   // Pattern complexity = 14  cost = 1  size = 3
09498   if ((Subtarget->hasSSE1())) {
09499     N0 = N.getOperand(0);
09500     N1 = N.getOperand(1);
09501     if (N1.getOpcode() == ISD::LOAD &&
09502         InFlightSet.count(N1.Val) == 0 &&
09503         N1.hasOneUse() &&
09504         !CodeGenMap.count(N1.getValue(0)) &&
09505         !isNonImmUse(N.Val, N1.Val)) {
09506       Chain1 = N1.getOperand(0);
09507       N11 = N1.getOperand(1);
09508       if (N.Val->getValueType(0) == MVT::v4f32) {
09509         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09510         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09511         if (!Match) {
09512           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09513         }
09514         if (Match) {
09515           Emit_19(Result, N, X86::ADDPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09516           return;
09517         }
09518       }
09519     }
09520   }
09521 
09522   // Pattern: (fadd:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
09523   // Emits: (ADDPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
09524   // Pattern complexity = 14  cost = 1  size = 3
09525   if ((Subtarget->hasSSE2())) {
09526     N0 = N.getOperand(0);
09527     N1 = N.getOperand(1);
09528     if (N1.getOpcode() == ISD::LOAD &&
09529         InFlightSet.count(N1.Val) == 0 &&
09530         N1.hasOneUse() &&
09531         !CodeGenMap.count(N1.getValue(0)) &&
09532         !isNonImmUse(N.Val, N1.Val)) {
09533       Chain1 = N1.getOperand(0);
09534       N11 = N1.getOperand(1);
09535       if (N.Val->getValueType(0) == MVT::v2f64) {
09536         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09537         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09538         if (!Match) {
09539           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09540         }
09541         if (Match) {
09542           Emit_19(Result, N, X86::ADDPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09543           return;
09544         }
09545       }
09546     }
09547   }
09548 
09549   // Pattern: (fadd:f32 (load:f32 addr:iPTR:$src2), FR32:f32:$src1)
09550   // Emits: (ADDSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
09551   // Pattern complexity = 14  cost = 1  size = 3
09552   if ((Subtarget->hasSSE1())) {
09553     N0 = N.getOperand(0);
09554     if (N0.getOpcode() == ISD::LOAD &&
09555         InFlightSet.count(N0.Val) == 0 &&
09556         N0.hasOneUse() &&
09557         !CodeGenMap.count(N0.getValue(0)) &&
09558         !isNonImmUse(N.Val, N0.Val)) {
09559       Chain0 = N0.getOperand(0);
09560       N01 = N0.getOperand(1);
09561       N1 = N.getOperand(1);
09562       if (N.Val->getValueType(0) == MVT::f32) {
09563         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09564         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09565         if (!Match) {
09566           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09567         }
09568         if (Match) {
09569           Emit_20(Result, N, X86::ADDSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09570           return;
09571         }
09572       }
09573     }
09574   }
09575 
09576   // Pattern: (fadd:f64 (load:f64 addr:iPTR:$src2), FR64:f64:$src1)
09577   // Emits: (ADDSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
09578   // Pattern complexity = 14  cost = 1  size = 3
09579   if ((Subtarget->hasSSE2())) {
09580     N0 = N.getOperand(0);
09581     if (N0.getOpcode() == ISD::LOAD &&
09582         InFlightSet.count(N0.Val) == 0 &&
09583         N0.hasOneUse() &&
09584         !CodeGenMap.count(N0.getValue(0)) &&
09585         !isNonImmUse(N.Val, N0.Val)) {
09586       Chain0 = N0.getOperand(0);
09587       N01 = N0.getOperand(1);
09588       N1 = N.getOperand(1);
09589       if (N.Val->getValueType(0) == MVT::f64) {
09590         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09591         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09592         if (!Match) {
09593           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09594         }
09595         if (Match) {
09596           Emit_20(Result, N, X86::ADDSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09597           return;
09598         }
09599       }
09600     }
09601   }
09602 
09603   // Pattern: (fadd:v4f32 (load:v4f32 addr:iPTR:$src2), VR128:v4f32:$src1)
09604   // Emits: (ADDPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
09605   // Pattern complexity = 14  cost = 1  size = 3
09606   if ((Subtarget->hasSSE1())) {
09607     N0 = N.getOperand(0);
09608     if (N0.getOpcode() == ISD::LOAD &&
09609         InFlightSet.count(N0.Val) == 0 &&
09610         N0.hasOneUse() &&
09611         !CodeGenMap.count(N0.getValue(0)) &&
09612         !isNonImmUse(N.Val, N0.Val)) {
09613       Chain0 = N0.getOperand(0);
09614       N01 = N0.getOperand(1);
09615       N1 = N.getOperand(1);
09616       if (N.Val->getValueType(0) == MVT::v4f32) {
09617         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09618         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09619         if (!Match) {
09620           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09621         }
09622         if (Match) {
09623           Emit_20(Result, N, X86::ADDPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09624           return;
09625         }
09626       }
09627     }
09628   }
09629 
09630   // Pattern: (fadd:v2f64 (load:v2f64 addr:iPTR:$src2), VR128:v2f64:$src1)
09631   // Emits: (ADDPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
09632   // Pattern complexity = 14  cost = 1  size = 3
09633   if ((Subtarget->hasSSE2())) {
09634     N0 = N.getOperand(0);
09635     if (N0.getOpcode() == ISD::LOAD &&
09636         InFlightSet.count(N0.Val) == 0 &&
09637         N0.hasOneUse() &&
09638         !CodeGenMap.count(N0.getValue(0)) &&
09639         !isNonImmUse(N.Val, N0.Val)) {
09640       Chain0 = N0.getOperand(0);
09641       N01 = N0.getOperand(1);
09642       N1 = N.getOperand(1);
09643       if (N.Val->getValueType(0) == MVT::v2f64) {
09644         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09645         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09646         if (!Match) {
09647           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09648         }
09649         if (Match) {
09650           Emit_20(Result, N, X86::ADDPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09651           return;
09652         }
09653       }
09654     }
09655   }
09656 
09657   // Pattern: (fadd:f64 RFP:f64:$src1, RFP:f64:$src2)
09658   // Emits: (FpADD:f64 RFP:f64:$src1, RFP:f64:$src2)
09659   // Pattern complexity = 2  cost = 1  size = 0
09660   if ((!Subtarget->hasSSE2())) {
09661     N0 = N.getOperand(0);
09662     N1 = N.getOperand(1);
09663     if (N.Val->getValueType(0) == MVT::f64) {
09664       Emit_21(Result, N, X86::FpADD, MVT::f64, N0, N1);
09665       return;
09666     }
09667   }
09668 
09669   // Pattern: (fadd:f32 FR32:f32:$src1, FR32:f32:$src2)
09670   // Emits: (ADDSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
09671   // Pattern complexity = 2  cost = 1  size = 3
09672   if ((Subtarget->hasSSE1())) {
09673     N0 = N.getOperand(0);
09674     N1 = N.getOperand(1);
09675     if (N.Val->getValueType(0) == MVT::f32) {
09676       Emit_21(Result, N, X86::ADDSSrr, MVT::f32, N0, N1);
09677       return;
09678     }
09679   }
09680 
09681   // Pattern: (fadd:f64 FR64:f64:$src1, FR64:f64:$src2)
09682   // Emits: (ADDSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
09683   // Pattern complexity = 2  cost = 1  size = 3
09684   if ((Subtarget->hasSSE2())) {
09685     N0 = N.getOperand(0);
09686     N1 = N.getOperand(1);
09687     if (N.Val->getValueType(0) == MVT::f64) {
09688       Emit_21(Result, N, X86::ADDSDrr, MVT::f64, N0, N1);
09689       return;
09690     }
09691   }
09692 
09693   // Pattern: (fadd:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
09694   // Emits: (ADDPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
09695   // Pattern complexity = 2  cost = 1  size = 3
09696   if ((Subtarget->hasSSE1())) {
09697     N0 = N.getOperand(0);
09698     N1 = N.getOperand(1);
09699     if (N.Val->getValueType(0) == MVT::v4f32) {
09700       Emit_21(Result, N, X86::ADDPSrr, MVT::v4f32, N0, N1);
09701       return;
09702     }
09703   }
09704 
09705   // Pattern: (fadd:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
09706   // Emits: (ADDPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
09707   // Pattern complexity = 2  cost = 1
09708   if ((Subtarget->hasSSE2())) {
09709     N0 = N.getOperand(0);
09710     N1 = N.getOperand(1);
09711     if (N.Val->getValueType(0) == MVT::v2f64) {
09712       Emit_21(Result, N, X86::ADDPDrr, MVT::v2f64, N0, N1);
09713       return;
09714     }
09715   }
09716   std::cerr << "Cannot yet select: ";
09717   N.Val->dump(CurDAG);
09718   std::cerr << '\n';
09719   abort();
09720 }
09721 
09722 void Select_fcos(SDOperand &Result, SDOperand N) {
09723   SDOperand N0(0, 0);
09724   SDOperand Tmp0(0, 0);
09725   SDNode *ResNode = NULL;
09726   if ((!Subtarget->hasSSE2())) {
09727     N0 = N.getOperand(0);
09728     if (N.Val->getValueType(0) == MVT::f64) {
09729       Emit_36(Result, N, X86::FpCOS, MVT::f64, N0);
09730       return;
09731     }
09732   }
09733   std::cerr << "Cannot yet select: ";
09734   N.Val->dump(CurDAG);
09735   std::cerr << '\n';
09736   abort();
09737 }
09738 
09739 void Select_fdiv(SDOperand &Result, SDOperand N) {
09740   SDOperand CPTmp0(0, 0);
09741   SDOperand CPTmp1(0, 0);
09742   SDOperand CPTmp2(0, 0);
09743   SDOperand CPTmp3(0, 0);
09744   SDOperand Chain0(0, 0);
09745   SDOperand Chain1(0, 0);
09746   SDOperand N0(0, 0);
09747   SDOperand N01(0, 0);
09748   SDOperand N02(0, 0);
09749   SDOperand N03(0, 0);
09750   SDOperand N1(0, 0);
09751   SDOperand N11(0, 0);
09752   SDOperand N12(0, 0);
09753   SDOperand N13(0, 0);
09754   SDOperand Tmp0(0, 0);
09755   SDOperand Tmp1(0, 0);
09756   SDOperand Tmp2(0, 0);
09757   SDOperand Tmp3(0, 0);
09758   SDOperand Tmp4(0, 0);
09759   SDNode *ResNode = NULL;
09760   if ((!Subtarget->hasSSE2())) {
09761     N0 = N.getOperand(0);
09762     {
09763       N1 = N.getOperand(1);
09764 
09765       // Pattern: (fdiv:f64 RFP:f64:$src1, (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other))
09766       // Emits: (FpDIV32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09767       // Pattern complexity = 14  cost = 1  size = 0
09768       if (N1.getOpcode() == ISD::EXTLOAD &&
09769           InFlightSet.count(N1.Val) == 0 &&
09770           N1.hasOneUse() &&
09771           !CodeGenMap.count(N1.getValue(0)) &&
09772           !isNonImmUse(N.Val, N1.Val)) {
09773         Chain1 = N1.getOperand(0);
09774         N11 = N1.getOperand(1);
09775         N12 = N1.getOperand(2);
09776         N13 = N1.getOperand(3);
09777         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
09778             N.Val->getValueType(0) == MVT::f64) {
09779           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09780           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09781           if (!Match) {
09782             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09783           }
09784           if (Match) {
09785             Emit_99(Result, N, X86::FpDIV32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12, N13);
09786             return;
09787           }
09788         }
09789       }
09790 
09791       // Pattern: (fdiv:f64 RFP:f64:$src1, (load:f64 addr:iPTR:$src2))
09792       // Emits: (FpDIV64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09793       // Pattern complexity = 14  cost = 1
09794       if (N1.getOpcode() == ISD::LOAD &&
09795           InFlightSet.count(N1.Val) == 0 &&
09796           N1.hasOneUse() &&
09797           !CodeGenMap.count(N1.getValue(0)) &&
09798           !isNonImmUse(N.Val, N1.Val)) {
09799         Chain1 = N1.getOperand(0);
09800         N11 = N1.getOperand(1);
09801         if (N.Val->getValueType(0) == MVT::f64) {
09802           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09803           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09804           if (!Match) {
09805             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09806           }
09807           if (Match) {
09808             Emit_19(Result, N, X86::FpDIV64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09809             return;
09810           }
09811         }
09812       }
09813     }
09814 
09815     // Pattern: (fdiv:f64 (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
09816     // Emits: (FpDIVR32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09817     // Pattern complexity = 14  cost = 1  size = 0
09818     if (N0.getOpcode() == ISD::EXTLOAD &&
09819         InFlightSet.count(N0.Val) == 0 &&
09820         N0.hasOneUse() &&
09821         !CodeGenMap.count(N0.getValue(0)) &&
09822         !isNonImmUse(N.Val, N0.Val)) {
09823       Chain0 = N0.getOperand(0);
09824       N01 = N0.getOperand(1);
09825       N02 = N0.getOperand(2);
09826       N03 = N0.getOperand(3);
09827       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
09828         N1 = N.getOperand(1);
09829         if (N.Val->getValueType(0) == MVT::f64) {
09830           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09831           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09832           if (!Match) {
09833             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09834           }
09835           if (Match) {
09836             Emit_101(Result, N, X86::FpDIVR32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N03, N1);
09837             return;
09838           }
09839         }
09840       }
09841     }
09842 
09843     // Pattern: (fdiv:f64 (load:f64 addr:iPTR:$src2), RFP:f64:$src1)
09844     // Emits: (FpDIVR64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09845     // Pattern complexity = 14  cost = 1  size = 0
09846     if (N0.getOpcode() == ISD::LOAD &&
09847         InFlightSet.count(N0.Val) == 0 &&
09848         N0.hasOneUse() &&
09849         !CodeGenMap.count(N0.getValue(0)) &&
09850         !isNonImmUse(N.Val, N0.Val)) {
09851       Chain0 = N0.getOperand(0);
09852       N01 = N0.getOperand(1);
09853       N1 = N.getOperand(1);
09854       if (N.Val->getValueType(0) == MVT::f64) {
09855         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09856         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09857         if (!Match) {
09858           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09859         }
09860         if (Match) {
09861           Emit_20(Result, N, X86::FpDIVR64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
09862           return;
09863         }
09864       }
09865     }
09866     {
09867       N1 = N.getOperand(1);
09868       if (N1.getOpcode() == X86ISD::FILD &&
09869           InFlightSet.count(N1.Val) == 0 &&
09870           N1.hasOneUse() &&
09871           !CodeGenMap.count(N1.getValue(0)) &&
09872           !isNonImmUse(N.Val, N1.Val)) {
09873         Chain1 = N1.getOperand(0);
09874         N11 = N1.getOperand(1);
09875         N12 = N1.getOperand(2);
09876 
09877         // Pattern: (fdiv:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i16:Other))
09878         // Emits: (FpIDIV16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09879         // Pattern complexity = 14  cost = 1  size = 0
09880         if (cast<VTSDNode>(N12)->getVT() == MVT::i16) {
09881           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09882           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09883           if (!Match) {
09884             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09885           }
09886           if (Match) {
09887             Emit_100(Result, N, X86::FpIDIV16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
09888             return;
09889           }
09890         }
09891 
09892         // Pattern: (fdiv:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i32:Other))
09893         // Emits: (FpIDIV32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09894         // Pattern complexity = 14  cost = 1
09895         if (cast<VTSDNode>(N12)->getVT() == MVT::i32) {
09896           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09897           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09898           if (!Match) {
09899             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09900           }
09901           if (Match) {
09902             Emit_100(Result, N, X86::FpIDIV32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
09903             return;
09904           }
09905         }
09906       }
09907     }
09908     if (N0.getOpcode() == X86ISD::FILD &&
09909         InFlightSet.count(N0.Val) == 0 &&
09910         N0.hasOneUse() &&
09911         !CodeGenMap.count(N0.getValue(0)) &&
09912         !isNonImmUse(N.Val, N0.Val)) {
09913       Chain0 = N0.getOperand(0);
09914       N01 = N0.getOperand(1);
09915       N02 = N0.getOperand(2);
09916 
09917       // Pattern: (fdiv:f64 (X86fild:f64 addr:iPTR:$src2, i16:Other), RFP:f64:$src1)
09918       // Emits: (FpIDIVR16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09919       // Pattern complexity = 14  cost = 1  size = 0
09920       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
09921         N1 = N.getOperand(1);
09922         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09923         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09924         if (!Match) {
09925           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09926         }
09927         if (Match) {
09928           Emit_102(Result, N, X86::FpIDIVR16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
09929           return;
09930         }
09931       }
09932 
09933       // Pattern: (fdiv:f64 (X86fild:f64 addr:iPTR:$src2, i32:Other), RFP:f64:$src1)
09934       // Emits: (FpIDIVR32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
09935       // Pattern complexity = 14  cost = 1
09936       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
09937         N1 = N.getOperand(1);
09938         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
09939         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09940         if (!Match) {
09941           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
09942         }
09943         if (Match) {
09944           Emit_102(Result, N, X86::FpIDIVR32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
09945           return;
09946         }
09947       }
09948     }
09949   }
09950 
09951   // Pattern: (fdiv:f32 FR32:f32:$src1, (load:f32 addr:iPTR:$src2))
09952   // Emits: (DIVSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
09953   // Pattern complexity = 14  cost = 1  size = 3
09954   if ((Subtarget->hasSSE1())) {
09955     N0 = N.getOperand(0);
09956     N1 = N.getOperand(1);
09957     if (N1.getOpcode() == ISD::LOAD &&
09958         InFlightSet.count(N1.Val) == 0 &&
09959         N1.hasOneUse() &&
09960         !CodeGenMap.count(N1.getValue(0)) &&
09961         !isNonImmUse(N.Val, N1.Val)) {
09962       Chain1 = N1.getOperand(0);
09963       N11 = N1.getOperand(1);
09964       if (N.Val->getValueType(0) == MVT::f32) {
09965         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09966         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09967         if (!Match) {
09968           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09969         }
09970         if (Match) {
09971           Emit_19(Result, N, X86::DIVSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09972           return;
09973         }
09974       }
09975     }
09976   }
09977 
09978   // Pattern: (fdiv:f64 FR64:f64:$src1, (load:f64 addr:iPTR:$src2))
09979   // Emits: (DIVSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
09980   // Pattern complexity = 14  cost = 1  size = 3
09981   if ((Subtarget->hasSSE2())) {
09982     N0 = N.getOperand(0);
09983     N1 = N.getOperand(1);
09984     if (N1.getOpcode() == ISD::LOAD &&
09985         InFlightSet.count(N1.Val) == 0 &&
09986         N1.hasOneUse() &&
09987         !CodeGenMap.count(N1.getValue(0)) &&
09988         !isNonImmUse(N.Val, N1.Val)) {
09989       Chain1 = N1.getOperand(0);
09990       N11 = N1.getOperand(1);
09991       if (N.Val->getValueType(0) == MVT::f64) {
09992         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
09993         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
09994         if (!Match) {
09995           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
09996         }
09997         if (Match) {
09998           Emit_19(Result, N, X86::DIVSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
09999           return;
10000         }
10001       }
10002     }
10003   }
10004 
10005   // Pattern: (fdiv:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
10006   // Emits: (DIVPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
10007   // Pattern complexity = 14  cost = 1  size = 3
10008   if ((Subtarget->hasSSE1())) {
10009     N0 = N.getOperand(0);
10010     N1 = N.getOperand(1);
10011     if (N1.getOpcode() == ISD::LOAD &&
10012         InFlightSet.count(N1.Val) == 0 &&
10013         N1.hasOneUse() &&
10014         !CodeGenMap.count(N1.getValue(0)) &&
10015         !isNonImmUse(N.Val, N1.Val)) {
10016       Chain1 = N1.getOperand(0);
10017       N11 = N1.getOperand(1);
10018       if (N.Val->getValueType(0) == MVT::v4f32) {
10019         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10020         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10021         if (!Match) {
10022           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10023         }
10024         if (Match) {
10025           Emit_19(Result, N, X86::DIVPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10026           return;
10027         }
10028       }
10029     }
10030   }
10031 
10032   // Pattern: (fdiv:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
10033   // Emits: (DIVPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
10034   // Pattern complexity = 14  cost = 1  size = 3
10035   if ((Subtarget->hasSSE2())) {
10036     N0 = N.getOperand(0);
10037     N1 = N.getOperand(1);
10038     if (N1.getOpcode() == ISD::LOAD &&
10039         InFlightSet.count(N1.Val) == 0 &&
10040         N1.hasOneUse() &&
10041         !CodeGenMap.count(N1.getValue(0)) &&
10042         !isNonImmUse(N.Val, N1.Val)) {
10043       Chain1 = N1.getOperand(0);
10044       N11 = N1.getOperand(1);
10045       if (N.Val->getValueType(0) == MVT::v2f64) {
10046         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10047         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10048         if (!Match) {
10049           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10050         }
10051         if (Match) {
10052           Emit_19(Result, N, X86::DIVPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10053           return;
10054         }
10055       }
10056     }
10057   }
10058 
10059   // Pattern: (fdiv:f64 RFP:f64:$src1, RFP:f64:$src2)
10060   // Emits: (FpDIV:f64 RFP:f64:$src1, RFP:f64:$src2)
10061   // Pattern complexity = 2  cost = 1  size = 0
10062   if ((!Subtarget->hasSSE2())) {
10063     N0 = N.getOperand(0);
10064     N1 = N.getOperand(1);
10065     if (N.Val->getValueType(0) == MVT::f64) {
10066       Emit_21(Result, N, X86::FpDIV, MVT::f64, N0, N1);
10067       return;
10068     }
10069   }
10070 
10071   // Pattern: (fdiv:f32 FR32:f32:$src1, FR32:f32:$src2)
10072   // Emits: (DIVSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
10073   // Pattern complexity = 2  cost = 1  size = 3
10074   if ((Subtarget->hasSSE1())) {
10075     N0 = N.getOperand(0);
10076     N1 = N.getOperand(1);
10077     if (N.Val->getValueType(0) == MVT::f32) {
10078       Emit_21(Result, N, X86::DIVSSrr, MVT::f32, N0, N1);
10079       return;
10080     }
10081   }
10082 
10083   // Pattern: (fdiv:f64 FR64:f64:$src1, FR64:f64:$src2)
10084   // Emits: (DIVSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
10085   // Pattern complexity = 2  cost = 1  size = 3
10086   if ((Subtarget->hasSSE2())) {
10087     N0 = N.getOperand(0);
10088     N1 = N.getOperand(1);
10089     if (N.Val->getValueType(0) == MVT::f64) {
10090       Emit_21(Result, N, X86::DIVSDrr, MVT::f64, N0, N1);
10091       return;
10092     }
10093   }
10094 
10095   // Pattern: (fdiv:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10096   // Emits: (DIVPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10097   // Pattern complexity = 2  cost = 1  size = 3
10098   if ((Subtarget->hasSSE1())) {
10099     N0 = N.getOperand(0);
10100     N1 = N.getOperand(1);
10101     if (N.Val->getValueType(0) == MVT::v4f32) {
10102       Emit_21(Result, N, X86::DIVPSrr, MVT::v4f32, N0, N1);
10103       return;
10104     }
10105   }
10106 
10107   // Pattern: (fdiv:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10108   // Emits: (DIVPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10109   // Pattern complexity = 2  cost = 1
10110   if ((Subtarget->hasSSE2())) {
10111     N0 = N.getOperand(0);
10112     N1 = N.getOperand(1);
10113     if (N.Val->getValueType(0) == MVT::v2f64) {
10114       Emit_21(Result, N, X86::DIVPDrr, MVT::v2f64, N0, N1);
10115       return;
10116     }
10117   }
10118   std::cerr << "Cannot yet select: ";
10119   N.Val->dump(CurDAG);
10120   std::cerr << '\n';
10121   abort();
10122 }
10123 
10124 void Select_fextend(SDOperand &Result, SDOperand N) {
10125   SDOperand N0(0, 0);
10126   SDOperand Tmp0(0, 0);
10127   SDNode *ResNode = NULL;
10128   if ((Subtarget->hasSSE2())) {
10129     N0 = N.getOperand(0);
10130     if (N.Val->getValueType(0) == MVT::f64 &&
10131         N0.Val->getValueType(0) == MVT::f32) {
10132       Emit_36(Result, N, X86::CVTSS2SDrr, MVT::f64, N0);
10133       return;
10134     }
10135   }
10136   std::cerr << "Cannot yet select: ";
10137   N.Val->dump(CurDAG);
10138   std::cerr << '\n';
10139   abort();
10140 }
10141 
10142 void Select_fmul(SDOperand &Result, SDOperand N) {
10143   SDOperand CPTmp0(0, 0);
10144   SDOperand CPTmp1(0, 0);
10145   SDOperand CPTmp2(0, 0);
10146   SDOperand CPTmp3(0, 0);
10147   SDOperand Chain0(0, 0);
10148   SDOperand Chain1(0, 0);
10149   SDOperand N0(0, 0);
10150   SDOperand N01(0, 0);
10151   SDOperand N02(0, 0);
10152   SDOperand N03(0, 0);
10153   SDOperand N1(0, 0);
10154   SDOperand N11(0, 0);
10155   SDOperand N12(0, 0);
10156   SDOperand N13(0, 0);
10157   SDOperand Tmp0(0, 0);
10158   SDOperand Tmp1(0, 0);
10159   SDOperand Tmp2(0, 0);
10160   SDOperand Tmp3(0, 0);
10161   SDOperand Tmp4(0, 0);
10162   SDNode *ResNode = NULL;
10163   if ((!Subtarget->hasSSE2())) {
10164     N0 = N.getOperand(0);
10165     {
10166       N1 = N.getOperand(1);
10167 
10168       // Pattern: (fmul:f64 RFP:f64:$src1, (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other))
10169       // Emits: (FpMUL32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10170       // Pattern complexity = 14  cost = 1  size = 0
10171       if (N1.getOpcode() == ISD::EXTLOAD &&
10172           InFlightSet.count(N1.Val) == 0 &&
10173           N1.hasOneUse() &&
10174           !CodeGenMap.count(N1.getValue(0)) &&
10175           !isNonImmUse(N.Val, N1.Val)) {
10176         Chain1 = N1.getOperand(0);
10177         N11 = N1.getOperand(1);
10178         N12 = N1.getOperand(2);
10179         N13 = N1.getOperand(3);
10180         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
10181             N.Val->getValueType(0) == MVT::f64) {
10182           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10183           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10184           if (!Match) {
10185             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10186           }
10187           if (Match) {
10188             Emit_99(Result, N, X86::FpMUL32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12, N13);
10189             return;
10190           }
10191         }
10192       }
10193 
10194       // Pattern: (fmul:f64 RFP:f64:$src1, (load:f64 addr:iPTR:$src2))
10195       // Emits: (FpMUL64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10196       // Pattern complexity = 14  cost = 1  size = 0
10197       if (N1.getOpcode() == ISD::LOAD &&
10198           InFlightSet.count(N1.Val) == 0 &&
10199           N1.hasOneUse() &&
10200           !CodeGenMap.count(N1.getValue(0)) &&
10201           !isNonImmUse(N.Val, N1.Val)) {
10202         Chain1 = N1.getOperand(0);
10203         N11 = N1.getOperand(1);
10204         if (N.Val->getValueType(0) == MVT::f64) {
10205           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10206           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10207           if (!Match) {
10208             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10209           }
10210           if (Match) {
10211             Emit_19(Result, N, X86::FpMUL64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10212             return;
10213           }
10214         }
10215       }
10216       if (N1.getOpcode() == X86ISD::FILD &&
10217           InFlightSet.count(N1.Val) == 0 &&
10218           N1.hasOneUse() &&
10219           !CodeGenMap.count(N1.getValue(0)) &&
10220           !isNonImmUse(N.Val, N1.Val)) {
10221         Chain1 = N1.getOperand(0);
10222         N11 = N1.getOperand(1);
10223         N12 = N1.getOperand(2);
10224 
10225         // Pattern: (fmul:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i16:Other))
10226         // Emits: (FpIMUL16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10227         // Pattern complexity = 14  cost = 1  size = 0
10228         if (cast<VTSDNode>(N12)->getVT() == MVT::i16) {
10229           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10230           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10231           if (!Match) {
10232             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10233           }
10234           if (Match) {
10235             Emit_100(Result, N, X86::FpIMUL16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
10236             return;
10237           }
10238         }
10239 
10240         // Pattern: (fmul:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i32:Other))
10241         // Emits: (FpIMUL32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10242         // Pattern complexity = 14  cost = 1
10243         if (cast<VTSDNode>(N12)->getVT() == MVT::i32) {
10244           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10245           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10246           if (!Match) {
10247             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10248           }
10249           if (Match) {
10250             Emit_100(Result, N, X86::FpIMUL32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
10251             return;
10252           }
10253         }
10254       }
10255     }
10256 
10257     // Pattern: (fmul:f64 (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
10258     // Emits: (FpMUL32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10259     // Pattern complexity = 14  cost = 1  size = 0
10260     if (N0.getOpcode() == ISD::EXTLOAD &&
10261         InFlightSet.count(N0.Val) == 0 &&
10262         N0.hasOneUse() &&
10263         !CodeGenMap.count(N0.getValue(0)) &&
10264         !isNonImmUse(N.Val, N0.Val)) {
10265       Chain0 = N0.getOperand(0);
10266       N01 = N0.getOperand(1);
10267       N02 = N0.getOperand(2);
10268       N03 = N0.getOperand(3);
10269       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
10270         N1 = N.getOperand(1);
10271         if (N.Val->getValueType(0) == MVT::f64) {
10272           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10273           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10274           if (!Match) {
10275             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10276           }
10277           if (Match) {
10278             Emit_101(Result, N, X86::FpMUL32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N03, N1);
10279             return;
10280           }
10281         }
10282       }
10283     }
10284 
10285     // Pattern: (fmul:f64 (load:f64 addr:iPTR:$src2), RFP:f64:$src1)
10286     // Emits: (FpMUL64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10287     // Pattern complexity = 14  cost = 1  size = 0
10288     if (N0.getOpcode() == ISD::LOAD &&
10289         InFlightSet.count(N0.Val) == 0 &&
10290         N0.hasOneUse() &&
10291         !CodeGenMap.count(N0.getValue(0)) &&
10292         !isNonImmUse(N.Val, N0.Val)) {
10293       Chain0 = N0.getOperand(0);
10294       N01 = N0.getOperand(1);
10295       N1 = N.getOperand(1);
10296       if (N.Val->getValueType(0) == MVT::f64) {
10297         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10298         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10299         if (!Match) {
10300           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10301         }
10302         if (Match) {
10303           Emit_20(Result, N, X86::FpMUL64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
10304           return;
10305         }
10306       }
10307     }
10308     if (N0.getOpcode() == X86ISD::FILD &&
10309         InFlightSet.count(N0.Val) == 0 &&
10310         N0.hasOneUse() &&
10311         !CodeGenMap.count(N0.getValue(0)) &&
10312         !isNonImmUse(N.Val, N0.Val)) {
10313       Chain0 = N0.getOperand(0);
10314       N01 = N0.getOperand(1);
10315       N02 = N0.getOperand(2);
10316 
10317       // Pattern: (fmul:f64 (X86fild:f64 addr:iPTR:$src2, i16:Other), RFP:f64:$src1)
10318       // Emits: (FpIMUL16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10319       // Pattern complexity = 14  cost = 1  size = 0
10320       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
10321         N1 = N.getOperand(1);
10322         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10323         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10324         if (!Match) {
10325           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10326         }
10327         if (Match) {
10328           Emit_102(Result, N, X86::FpIMUL16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
10329           return;
10330         }
10331       }
10332 
10333       // Pattern: (fmul:f64 (X86fild:f64 addr:iPTR:$src2, i32:Other), RFP:f64:$src1)
10334       // Emits: (FpIMUL32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
10335       // Pattern complexity = 14  cost = 1
10336       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
10337         N1 = N.getOperand(1);
10338         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10339         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10340         if (!Match) {
10341           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10342         }
10343         if (Match) {
10344           Emit_102(Result, N, X86::FpIMUL32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
10345           return;
10346         }
10347       }
10348     }
10349   }
10350 
10351   // Pattern: (fmul:f32 FR32:f32:$src1, (load:f32 addr:iPTR:$src2))
10352   // Emits: (MULSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
10353   // Pattern complexity = 14  cost = 1  size = 3
10354   if ((Subtarget->hasSSE1())) {
10355     N0 = N.getOperand(0);
10356     N1 = N.getOperand(1);
10357     if (N1.getOpcode() == ISD::LOAD &&
10358         InFlightSet.count(N1.Val) == 0 &&
10359         N1.hasOneUse() &&
10360         !CodeGenMap.count(N1.getValue(0)) &&
10361         !isNonImmUse(N.Val, N1.Val)) {
10362       Chain1 = N1.getOperand(0);
10363       N11 = N1.getOperand(1);
10364       if (N.Val->getValueType(0) == MVT::f32) {
10365         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10366         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10367         if (!Match) {
10368           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10369         }
10370         if (Match) {
10371           Emit_19(Result, N, X86::MULSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10372           return;
10373         }
10374       }
10375     }
10376   }
10377 
10378   // Pattern: (fmul:f64 FR64:f64:$src1, (load:f64 addr:iPTR:$src2))
10379   // Emits: (MULSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
10380   // Pattern complexity = 14  cost = 1  size = 3
10381   if ((Subtarget->hasSSE2())) {
10382     N0 = N.getOperand(0);
10383     N1 = N.getOperand(1);
10384     if (N1.getOpcode() == ISD::LOAD &&
10385         InFlightSet.count(N1.Val) == 0 &&
10386         N1.hasOneUse() &&
10387         !CodeGenMap.count(N1.getValue(0)) &&
10388         !isNonImmUse(N.Val, N1.Val)) {
10389       Chain1 = N1.getOperand(0);
10390       N11 = N1.getOperand(1);
10391       if (N.Val->getValueType(0) == MVT::f64) {
10392         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10393         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10394         if (!Match) {
10395           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10396         }
10397         if (Match) {
10398           Emit_19(Result, N, X86::MULSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10399           return;
10400         }
10401       }
10402     }
10403   }
10404 
10405   // Pattern: (fmul:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
10406   // Emits: (MULPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
10407   // Pattern complexity = 14  cost = 1  size = 3
10408   if ((Subtarget->hasSSE1())) {
10409     N0 = N.getOperand(0);
10410     N1 = N.getOperand(1);
10411     if (N1.getOpcode() == ISD::LOAD &&
10412         InFlightSet.count(N1.Val) == 0 &&
10413         N1.hasOneUse() &&
10414         !CodeGenMap.count(N1.getValue(0)) &&
10415         !isNonImmUse(N.Val, N1.Val)) {
10416       Chain1 = N1.getOperand(0);
10417       N11 = N1.getOperand(1);
10418       if (N.Val->getValueType(0) == MVT::v4f32) {
10419         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10420         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10421         if (!Match) {
10422           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10423         }
10424         if (Match) {
10425           Emit_19(Result, N, X86::MULPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10426           return;
10427         }
10428       }
10429     }
10430   }
10431 
10432   // Pattern: (fmul:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
10433   // Emits: (MULPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
10434   // Pattern complexity = 14  cost = 1  size = 3
10435   if ((Subtarget->hasSSE2())) {
10436     N0 = N.getOperand(0);
10437     N1 = N.getOperand(1);
10438     if (N1.getOpcode() == ISD::LOAD &&
10439         InFlightSet.count(N1.Val) == 0 &&
10440         N1.hasOneUse() &&
10441         !CodeGenMap.count(N1.getValue(0)) &&
10442         !isNonImmUse(N.Val, N1.Val)) {
10443       Chain1 = N1.getOperand(0);
10444       N11 = N1.getOperand(1);
10445       if (N.Val->getValueType(0) == MVT::v2f64) {
10446         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
10447         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10448         if (!Match) {
10449           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
10450         }
10451         if (Match) {
10452           Emit_19(Result, N, X86::MULPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
10453           return;
10454         }
10455       }
10456     }
10457   }
10458 
10459   // Pattern: (fmul:f32 (load:f32 addr:iPTR:$src2), FR32:f32:$src1)
10460   // Emits: (MULSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
10461   // Pattern complexity = 14  cost = 1  size = 3
10462   if ((Subtarget->hasSSE1())) {
10463     N0 = N.getOperand(0);
10464     if (N0.getOpcode() == ISD::LOAD &&
10465         InFlightSet.count(N0.Val) == 0 &&
10466         N0.hasOneUse() &&
10467         !CodeGenMap.count(N0.getValue(0)) &&
10468         !isNonImmUse(N.Val, N0.Val)) {
10469       Chain0 = N0.getOperand(0);
10470       N01 = N0.getOperand(1);
10471       N1 = N.getOperand(1);
10472       if (N.Val->getValueType(0) == MVT::f32) {
10473         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10474         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10475         if (!Match) {
10476           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10477         }
10478         if (Match) {
10479           Emit_20(Result, N, X86::MULSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
10480           return;
10481         }
10482       }
10483     }
10484   }
10485 
10486   // Pattern: (fmul:f64 (load:f64 addr:iPTR:$src2), FR64:f64:$src1)
10487   // Emits: (MULSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
10488   // Pattern complexity = 14  cost = 1  size = 3
10489   if ((Subtarget->hasSSE2())) {
10490     N0 = N.getOperand(0);
10491     if (N0.getOpcode() == ISD::LOAD &&
10492         InFlightSet.count(N0.Val) == 0 &&
10493         N0.hasOneUse() &&
10494         !CodeGenMap.count(N0.getValue(0)) &&
10495         !isNonImmUse(N.Val, N0.Val)) {
10496       Chain0 = N0.getOperand(0);
10497       N01 = N0.getOperand(1);
10498       N1 = N.getOperand(1);
10499       if (N.Val->getValueType(0) == MVT::f64) {
10500         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10501         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10502         if (!Match) {
10503           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10504         }
10505         if (Match) {
10506           Emit_20(Result, N, X86::MULSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
10507           return;
10508         }
10509       }
10510     }
10511   }
10512 
10513   // Pattern: (fmul:v4f32 (load:v4f32 addr:iPTR:$src2), VR128:v4f32:$src1)
10514   // Emits: (MULPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
10515   // Pattern complexity = 14  cost = 1  size = 3
10516   if ((Subtarget->hasSSE1())) {
10517     N0 = N.getOperand(0);
10518     if (N0.getOpcode() == ISD::LOAD &&
10519         InFlightSet.count(N0.Val) == 0 &&
10520         N0.hasOneUse() &&
10521         !CodeGenMap.count(N0.getValue(0)) &&
10522         !isNonImmUse(N.Val, N0.Val)) {
10523       Chain0 = N0.getOperand(0);
10524       N01 = N0.getOperand(1);
10525       N1 = N.getOperand(1);
10526       if (N.Val->getValueType(0) == MVT::v4f32) {
10527         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10528         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10529         if (!Match) {
10530           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10531         }
10532         if (Match) {
10533           Emit_20(Result, N, X86::MULPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
10534           return;
10535         }
10536       }
10537     }
10538   }
10539 
10540   // Pattern: (fmul:v2f64 (load:v2f64 addr:iPTR:$src2), VR128:v2f64:$src1)
10541   // Emits: (MULPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
10542   // Pattern complexity = 14  cost = 1  size = 3
10543   if ((Subtarget->hasSSE2())) {
10544     N0 = N.getOperand(0);
10545     if (N0.getOpcode() == ISD::LOAD &&
10546         InFlightSet.count(N0.Val) == 0 &&
10547         N0.hasOneUse() &&
10548         !CodeGenMap.count(N0.getValue(0)) &&
10549         !isNonImmUse(N.Val, N0.Val)) {
10550       Chain0 = N0.getOperand(0);
10551       N01 = N0.getOperand(1);
10552       N1 = N.getOperand(1);
10553       if (N.Val->getValueType(0) == MVT::v2f64) {
10554         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
10555         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10556         if (!Match) {
10557           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
10558         }
10559         if (Match) {
10560           Emit_20(Result, N, X86::MULPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
10561           return;
10562         }
10563       }
10564     }
10565   }
10566 
10567   // Pattern: (fmul:f64 RFP:f64:$src1, RFP:f64:$src2)
10568   // Emits: (FpMUL:f64 RFP:f64:$src1, RFP:f64:$src2)
10569   // Pattern complexity = 2  cost = 1  size = 0
10570   if ((!Subtarget->hasSSE2())) {
10571     N0 = N.getOperand(0);
10572     N1 = N.getOperand(1);
10573     if (N.Val->getValueType(0) == MVT::f64) {
10574       Emit_21(Result, N, X86::FpMUL, MVT::f64, N0, N1);
10575       return;
10576     }
10577   }
10578 
10579   // Pattern: (fmul:f32 FR32:f32:$src1, FR32:f32:$src2)
10580   // Emits: (MULSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
10581   // Pattern complexity = 2  cost = 1  size = 3
10582   if ((Subtarget->hasSSE1())) {
10583     N0 = N.getOperand(0);
10584     N1 = N.getOperand(1);
10585     if (N.Val->getValueType(0) == MVT::f32) {
10586       Emit_21(Result, N, X86::MULSSrr, MVT::f32, N0, N1);
10587       return;
10588     }
10589   }
10590 
10591   // Pattern: (fmul:f64 FR64:f64:$src1, FR64:f64:$src2)
10592   // Emits: (MULSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
10593   // Pattern complexity = 2  cost = 1  size = 3
10594   if ((Subtarget->hasSSE2())) {
10595     N0 = N.getOperand(0);
10596     N1 = N.getOperand(1);
10597     if (N.Val->getValueType(0) == MVT::f64) {
10598       Emit_21(Result, N, X86::MULSDrr, MVT::f64, N0, N1);
10599       return;
10600     }
10601   }
10602 
10603   // Pattern: (fmul:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10604   // Emits: (MULPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10605   // Pattern complexity = 2  cost = 1  size = 3
10606   if ((Subtarget->hasSSE1())) {
10607     N0 = N.getOperand(0);
10608     N1 = N.getOperand(1);
10609     if (N.Val->getValueType(0) == MVT::v4f32) {
10610       Emit_21(Result, N, X86::MULPSrr, MVT::v4f32, N0, N1);
10611       return;
10612     }
10613   }
10614 
10615   // Pattern: (fmul:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10616   // Emits: (MULPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10617   // Pattern complexity = 2  cost = 1
10618   if ((Subtarget->hasSSE2())) {
10619     N0 = N.getOperand(0);
10620     N1 = N.getOperand(1);
10621     if (N.Val->getValueType(0) == MVT::v2f64) {
10622       Emit_21(Result, N, X86::MULPDrr, MVT::v2f64, N0, N1);
10623       return;
10624     }
10625   }
10626   std::cerr << "Cannot yet select: ";
10627   N.Val->dump(CurDAG);
10628   std::cerr << '\n';
10629   abort();
10630 }
10631 
10632 void Select_fneg(SDOperand &Result, SDOperand N) {
10633   SDOperand N0(0, 0);
10634   SDOperand Tmp0(0, 0);
10635   SDNode *ResNode = NULL;
10636   if ((!Subtarget->hasSSE2())) {
10637     N0 = N.getOperand(0);
10638     if (N.Val->getValueType(0) == MVT::f64) {
10639       Emit_36(Result, N, X86::FpCHS, MVT::f64, N0);
10640       return;
10641     }
10642   }
10643   std::cerr << "Cannot yet select: ";
10644   N.Val->dump(CurDAG);
10645   std::cerr << '\n';
10646   abort();
10647 }
10648 
10649 void Select_fp_to_sint(SDOperand &Result, SDOperand N) {
10650   SDOperand CPTmp0(0, 0);
10651   SDOperand CPTmp1(0, 0);
10652   SDOperand CPTmp2(0, 0);
10653   SDOperand CPTmp3(0, 0);
10654   SDOperand Chain0(0, 0);
10655   SDOperand N0(0, 0);
10656   SDOperand N01(0, 0);
10657   SDOperand Tmp0(0, 0);
10658   SDOperand Tmp1(0, 0);
10659   SDOperand Tmp2(0, 0);
10660   SDOperand Tmp3(0, 0);
10661   SDNode *ResNode = NULL;
10662 
10663   // Pattern: (fp_to_sint:i32 (load:f32 addr:iPTR:$src))
10664   // Emits: (CVTTSS2SIrm:i32 addr:iPTR:$src)
10665   // Pattern complexity = 14  cost = 1  size = 3
10666   if ((Subtarget->hasSSE1())) {
10667     N0 = N.getOperand(0);
10668     if (N0.getOpcode() == ISD::LOAD &&
10669         InFlightSet.count(N0.Val) == 0 &&
10670         N0.hasOneUse() &&
10671         !CodeGenMap.count(N0.getValue(0))) {
10672       Chain0 = N0.getOperand(0);
10673       N01 = N0.getOperand(1);
10674       if (N.Val->getValueType(0) == MVT::i32 &&
10675           N0.Val->getValueType(0) == MVT::f32) {
10676         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10677         if (Match) {
10678           Emit_90(Result, N, X86::CVTTSS2SIrm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
10679           return;
10680         }
10681       }
10682     }
10683   }
10684 
10685   // Pattern: (fp_to_sint:i32 (load:f64 addr:iPTR:$src))
10686   // Emits: (CVTTSD2SIrm:i32 addr:iPTR:$src)
10687   // Pattern complexity = 14  cost = 1  size = 3
10688   if ((Subtarget->hasSSE2())) {
10689     N0 = N.getOperand(0);
10690     if (N0.getOpcode() == ISD::LOAD &&
10691         InFlightSet.count(N0.Val) == 0 &&
10692         N0.hasOneUse() &&
10693         !CodeGenMap.count(N0.getValue(0))) {
10694       Chain0 = N0.getOperand(0);
10695       N01 = N0.getOperand(1);
10696       if (N.Val->getValueType(0) == MVT::i32 &&
10697           N0.Val->getValueType(0) == MVT::f64) {
10698         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10699         if (Match) {
10700           Emit_90(Result, N, X86::CVTTSD2SIrm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
10701           return;
10702         }
10703       }
10704     }
10705   }
10706 
10707   // Pattern: (fp_to_sint:i32 FR32:f32:$src)
10708   // Emits: (CVTTSS2SIrr:i32 FR32:f32:$src)
10709   // Pattern complexity = 2  cost = 1  size = 3
10710   if ((Subtarget->hasSSE1())) {
10711     N0 = N.getOperand(0);
10712     if (N.Val->getValueType(0) == MVT::i32 &&
10713         N0.Val->getValueType(0) == MVT::f32) {
10714       Emit_36(Result, N, X86::CVTTSS2SIrr, MVT::i32, N0);
10715       return;
10716     }
10717   }
10718 
10719   // Pattern: (fp_to_sint:i32 FR64:f64:$src)
10720   // Emits: (CVTTSD2SIrr:i32 FR64:f64:$src)
10721   // Pattern complexity = 2  cost = 1
10722   if ((Subtarget->hasSSE2())) {
10723     N0 = N.getOperand(0);
10724     if (N.Val->getValueType(0) == MVT::i32 &&
10725         N0.Val->getValueType(0) == MVT::f64) {
10726       Emit_36(Result, N, X86::CVTTSD2SIrr, MVT::i32, N0);
10727       return;
10728     }
10729   }
10730   std::cerr << "Cannot yet select: ";
10731   N.Val->dump(CurDAG);
10732   std::cerr << '\n';
10733   abort();
10734 }
10735 
10736 void Emit_103(SDOperand &Result, SDOperand &N, unsigned Opc0, unsigned Opc1, MVT::ValueType VT0, MVT::ValueType VT1) NOINLINE {
10737   SDOperand Tmp0(0, 0);
10738   SDNode *ResNode = NULL;
10739   Tmp0 = SDOperand(CurDAG->getTargetNode(Opc0, VT0), 0);
10740   if (N.Val->hasOneUse()) {
10741     Result = CurDAG->SelectNodeTo(N.Val, Opc1, VT1, Tmp0);
10742   } else {
10743     ResNode = CurDAG->getTargetNode(Opc1, VT1, Tmp0);
10744     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10745     Result = SDOperand(ResNode, 0);
10746   }
10747   return;
10748 }
10749 void Select_fpimm(SDOperand &Result, SDOperand N) {
10750   SDOperand Tmp0(0, 0);
10751   SDNode *ResNode = NULL;
10752   if ((!Subtarget->hasSSE2())) {
10753 
10754     // Pattern: (fpimm:f64)<<P:Predicate_fp64imm0>>
10755     // Emits: (FpLD0:f64)
10756     // Pattern complexity = 3  cost = 1  size = 0
10757     if (Predicate_fp64imm0(N.Val) &&
10758         N.Val->getValueType(0) == MVT::f64) {
10759       Emit_96(Result, N, X86::FpLD0, MVT::f64);
10760       return;
10761     }
10762 
10763     // Pattern: (fpimm:f64)<<P:Predicate_fp64imm1>>
10764     // Emits: (FpLD1:f64)
10765     // Pattern complexity = 3  cost = 1
10766     if (Predicate_fp64imm1(N.Val) &&
10767         N.Val->getValueType(0) == MVT::f64) {
10768       Emit_96(Result, N, X86::FpLD1, MVT::f64);
10769       return;
10770     }
10771   }
10772 
10773   // Pattern: (fpimm:f32)<<P:Predicate_fp32imm0>>
10774   // Emits: (FsFLD0SS:f32)
10775   // Pattern complexity = 3  cost = 1  size = 3
10776   if ((Subtarget->hasSSE1()) &&
10777       Predicate_fp32imm0(N.Val) &&
10778       N.Val->getValueType(0) == MVT::f32) {
10779     Emit_96(Result, N, X86::FsFLD0SS, MVT::f32);
10780     return;
10781   }
10782 
10783   // Pattern: (fpimm:f64)<<P:Predicate_fp64imm0>>
10784   // Emits: (FsFLD0SD:f64)
10785   // Pattern complexity = 3  cost = 1  size = 3
10786   if ((Subtarget->hasSSE2()) &&
10787       Predicate_fp64imm0(N.Val) &&
10788       N.Val->getValueType(0) == MVT::f64) {
10789     Emit_96(Result, N, X86::FsFLD0SD, MVT::f64);
10790     return;
10791   }
10792   if ((!Subtarget->hasSSE2())) {
10793 
10794     // Pattern: (fpimm:f64)<<P:Predicate_fp64immneg0>>
10795     // Emits: (FpCHS:f64 (FpLD0:f64))
10796     // Pattern complexity = 3  cost = 2  size = 0
10797     if (Predicate_fp64immneg0(N.Val) &&
10798         N.Val->getValueType(0) == MVT::f64) {
10799       Emit_103(Result, N, X86::FpLD0, X86::FpCHS, MVT::f64, MVT::f64);
10800       return;
10801     }
10802 
10803     // Pattern: (fpimm:f64)<<P:Predicate_fp64immneg1>>
10804     // Emits: (FpCHS:f64 (FpLD1:f64))
10805     // Pattern complexity = 3  cost = 2
10806     if (Predicate_fp64immneg1(N.Val) &&
10807         N.Val->getValueType(0) == MVT::f64) {
10808       Emit_103(Result, N, X86::FpLD1, X86::FpCHS, MVT::f64, MVT::f64);
10809       return;
10810     }
10811   }
10812   std::cerr << "Cannot yet select: ";
10813   N.Val->dump(CurDAG);
10814   std::cerr << '\n';
10815   abort();
10816 }
10817 
10818 void Select_frameindex(SDOperand &Result, SDOperand N) {
10819   SDOperand CPTmp0(0, 0);
10820   SDOperand CPTmp1(0, 0);
10821   SDOperand CPTmp2(0, 0);
10822   SDOperand CPTmp3(0, 0);
10823   SDOperand Tmp0(0, 0);
10824   SDOperand Tmp1(0, 0);
10825   SDOperand Tmp2(0, 0);
10826   SDOperand Tmp3(0, 0);
10827   SDNode *ResNode = NULL;
10828   if (N.Val->getValueType(0) == MVT::i32) {
10829     bool Match = SelectLEAAddr(N, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10830     if (Match) {
10831       Emit_42(Result, N, X86::LEA32r, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10832       return;
10833     }
10834   }
10835   std::cerr << "Cannot yet select: ";
10836   N.Val->dump(CurDAG);
10837   std::cerr << '\n';
10838   abort();
10839 }
10840 
10841 void Select_fround(SDOperand &Result, SDOperand N) {
10842   SDOperand CPTmp0(0, 0);
10843   SDOperand CPTmp1(0, 0);
10844   SDOperand CPTmp2(0, 0);
10845   SDOperand CPTmp3(0, 0);
10846   SDOperand Chain0(0, 0);
10847   SDOperand N0(0, 0);
10848   SDOperand N01(0, 0);
10849   SDOperand Tmp0(0, 0);
10850   SDOperand Tmp1(0, 0);
10851   SDOperand Tmp2(0, 0);
10852   SDOperand Tmp3(0, 0);
10853   SDNode *ResNode = NULL;
10854   if ((Subtarget->hasSSE2())) {
10855     N0 = N.getOperand(0);
10856 
10857     // Pattern: (fround:f32 (load:f64 addr:iPTR:$src))
10858     // Emits: (CVTSD2SSrm:f32 addr:iPTR:$src)
10859     // Pattern complexity = 14  cost = 1  size = 3
10860     if (N0.getOpcode() == ISD::LOAD &&
10861         InFlightSet.count(N0.Val) == 0 &&
10862         N0.hasOneUse() &&
10863         !CodeGenMap.count(N0.getValue(0))) {
10864       Chain0 = N0.getOperand(0);
10865       N01 = N0.getOperand(1);
10866       if (N.Val->getValueType(0) == MVT::f32 &&
10867           N0.Val->getValueType(0) == MVT::f64) {
10868         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10869         if (Match) {
10870           Emit_90(Result, N, X86::CVTSD2SSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
10871           return;
10872         }
10873       }
10874     }
10875 
10876     // Pattern: (fround:f32 FR64:f64:$src)
10877     // Emits: (CVTSD2SSrr:f32 FR64:f64:$src)
10878     // Pattern complexity = 2  cost = 1
10879     if (N.Val->getValueType(0) == MVT::f32 &&
10880         N0.Val->getValueType(0) == MVT::f64) {
10881       Emit_36(Result, N, X86::CVTSD2SSrr, MVT::f32, N0);
10882       return;
10883     }
10884   }
10885   std::cerr << "Cannot yet select: ";
10886   N.Val->dump(CurDAG);
10887   std::cerr << '\n';
10888   abort();
10889 }
10890 
10891 void Select_fsin(SDOperand &Result, SDOperand N) {
10892   SDOperand N0(0, 0);
10893   SDOperand Tmp0(0, 0);
10894   SDNode *ResNode = NULL;
10895   if ((!Subtarget->hasSSE2())) {
10896     N0 = N.getOperand(0);
10897     if (N.Val->getValueType(0) == MVT::f64) {
10898       Emit_36(Result, N, X86::FpSIN, MVT::f64, N0);
10899       return;
10900     }
10901   }
10902   std::cerr << "Cannot yet select: ";
10903   N.Val->dump(CurDAG);
10904   std::cerr << '\n';
10905   abort();
10906 }
10907 
10908 void Select_fsqrt(SDOperand &Result, SDOperand N) {
10909   SDOperand CPTmp0(0, 0);
10910   SDOperand CPTmp1(0, 0);
10911   SDOperand CPTmp2(0, 0);
10912   SDOperand CPTmp3(0, 0);
10913   SDOperand Chain0(0, 0);
10914   SDOperand N0(0, 0);
10915   SDOperand N01(0, 0);
10916   SDOperand Tmp0(0, 0);
10917   SDOperand Tmp1(0, 0);
10918   SDOperand Tmp2(0, 0);
10919   SDOperand Tmp3(0, 0);
10920   SDNode *ResNode = NULL;
10921 
10922   // Pattern: (fsqrt:f32 (load:f32 addr:iPTR:$src))
10923   // Emits: (SQRTSSm:f32 addr:iPTR:$src)
10924   // Pattern complexity = 14  cost = 1  size = 3
10925   if ((Subtarget->hasSSE1())) {
10926     N0 = N.getOperand(0);
10927     if (N0.getOpcode() == ISD::LOAD &&
10928         InFlightSet.count(N0.Val) == 0 &&
10929         N0.hasOneUse() &&
10930         !CodeGenMap.count(N0.getValue(0))) {
10931       Chain0 = N0.getOperand(0);
10932       N01 = N0.getOperand(1);
10933       if (N.Val->getValueType(0) == MVT::f32) {
10934         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10935         if (Match) {
10936           Emit_90(Result, N, X86::SQRTSSm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
10937           return;
10938         }
10939       }
10940     }
10941   }
10942 
10943   // Pattern: (fsqrt:f64 (load:f64 addr:iPTR:$src))
10944   // Emits: (SQRTSDm:f64 addr:iPTR:$src)
10945   // Pattern complexity = 14  cost = 1  size = 3
10946   if ((Subtarget->hasSSE2())) {
10947     N0 = N.getOperand(0);
10948     if (N0.getOpcode() == ISD::LOAD &&
10949         InFlightSet.count(N0.Val) == 0 &&
10950         N0.hasOneUse() &&
10951         !CodeGenMap.count(N0.getValue(0))) {
10952       Chain0 = N0.getOperand(0);
10953       N01 = N0.getOperand(1);
10954       if (N.Val->getValueType(0) == MVT::f64) {
10955         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
10956         if (Match) {
10957           Emit_90(Result, N, X86::SQRTSDm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
10958           return;
10959         }
10960       }
10961     }
10962   }
10963 
10964   // Pattern: (fsqrt:f64 RFP:f64:$src)
10965   // Emits: (FpSQRT:f64 RFP:f64:$src)
10966   // Pattern complexity = 2  cost = 1  size = 0
10967   if ((!Subtarget->hasSSE2())) {
10968     N0 = N.getOperand(0);
10969     if (N.Val->getValueType(0) == MVT::f64) {
10970       Emit_36(Result, N, X86::FpSQRT, MVT::f64, N0);
10971       return;
10972     }
10973   }
10974 
10975   // Pattern: (fsqrt:f32 FR32:f32:$src)
10976   // Emits: (SQRTSSr:f32 FR32:f32:$src)
10977   // Pattern complexity = 2  cost = 1  size = 3
10978   if ((Subtarget->hasSSE1())) {
10979     N0 = N.getOperand(0);
10980     if (N.Val->getValueType(0) == MVT::f32) {
10981       Emit_36(Result, N, X86::SQRTSSr, MVT::f32, N0);
10982       return;
10983     }
10984   }
10985 
10986   // Pattern: (fsqrt:f64 FR64:f64:$src)
10987   // Emits: (SQRTSDr:f64 FR64:f64:$src)
10988   // Pattern complexity = 2  cost = 1
10989   if ((Subtarget->hasSSE2())) {
10990     N0 = N.getOperand(0);
10991     if (N.Val->getValueType(0) == MVT::f64) {
10992       Emit_36(Result, N, X86::SQRTSDr, MVT::f64, N0);
10993       return;
10994     }
10995   }
10996   std::cerr << "Cannot yet select: ";
10997   N.Val->dump(CurDAG);
10998   std::cerr << '\n';
10999   abort();
11000 }
11001 
11002 void Select_fsub(SDOperand &Result, SDOperand N) {
11003   SDOperand CPTmp0(0, 0);
11004   SDOperand CPTmp1(0, 0);
11005   SDOperand CPTmp2(0, 0);
11006   SDOperand CPTmp3(0, 0);
11007   SDOperand Chain0(0, 0);
11008   SDOperand Chain1(0, 0);
11009   SDOperand N0(0, 0);
11010   SDOperand N01(0, 0);
11011   SDOperand N02(0, 0);
11012   SDOperand N03(0, 0);
11013   SDOperand N1(0, 0);
11014   SDOperand N11(0, 0);
11015   SDOperand N12(0, 0);
11016   SDOperand N13(0, 0);
11017   SDOperand Tmp0(0, 0);
11018   SDOperand Tmp1(0, 0);
11019   SDOperand Tmp2(0, 0);
11020   SDOperand Tmp3(0, 0);
11021   SDOperand Tmp4(0, 0);
11022   SDNode *ResNode = NULL;
11023   if ((!Subtarget->hasSSE2())) {
11024     N0 = N.getOperand(0);
11025     {
11026       N1 = N.getOperand(1);
11027 
11028       // Pattern: (fsub:f64 RFP:f64:$src1, (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other))
11029       // Emits: (FpSUB32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11030       // Pattern complexity = 14  cost = 1  size = 0
11031       if (N1.getOpcode() == ISD::EXTLOAD &&
11032           InFlightSet.count(N1.Val) == 0 &&
11033           N1.hasOneUse() &&
11034           !CodeGenMap.count(N1.getValue(0)) &&
11035           !isNonImmUse(N.Val, N1.Val)) {
11036         Chain1 = N1.getOperand(0);
11037         N11 = N1.getOperand(1);
11038         N12 = N1.getOperand(2);
11039         N13 = N1.getOperand(3);
11040         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
11041             N.Val->getValueType(0) == MVT::f64) {
11042           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11043           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11044           if (!Match) {
11045             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11046           }
11047           if (Match) {
11048             Emit_99(Result, N, X86::FpSUB32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12, N13);
11049             return;
11050           }
11051         }
11052       }
11053 
11054       // Pattern: (fsub:f64 RFP:f64:$src1, (load:f64 addr:iPTR:$src2))
11055       // Emits: (FpSUB64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11056       // Pattern complexity = 14  cost = 1
11057       if (N1.getOpcode() == ISD::LOAD &&
11058           InFlightSet.count(N1.Val) == 0 &&
11059           N1.hasOneUse() &&
11060           !CodeGenMap.count(N1.getValue(0)) &&
11061           !isNonImmUse(N.Val, N1.Val)) {
11062         Chain1 = N1.getOperand(0);
11063         N11 = N1.getOperand(1);
11064         if (N.Val->getValueType(0) == MVT::f64) {
11065           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11066           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11067           if (!Match) {
11068             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11069           }
11070           if (Match) {
11071             Emit_19(Result, N, X86::FpSUB64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
11072             return;
11073           }
11074         }
11075       }
11076     }
11077 
11078     // Pattern: (fsub:f64 (extld:f64 addr:iPTR:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
11079     // Emits: (FpSUBR32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11080     // Pattern complexity = 14  cost = 1  size = 0
11081     if (N0.getOpcode() == ISD::EXTLOAD &&
11082         InFlightSet.count(N0.Val) == 0 &&
11083         N0.hasOneUse() &&
11084         !CodeGenMap.count(N0.getValue(0)) &&
11085         !isNonImmUse(N.Val, N0.Val)) {
11086       Chain0 = N0.getOperand(0);
11087       N01 = N0.getOperand(1);
11088       N02 = N0.getOperand(2);
11089       N03 = N0.getOperand(3);
11090       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
11091         N1 = N.getOperand(1);
11092         if (N.Val->getValueType(0) == MVT::f64) {
11093           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
11094           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11095           if (!Match) {
11096             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
11097           }
11098           if (Match) {
11099             Emit_101(Result, N, X86::FpSUBR32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N03, N1);
11100             return;
11101           }
11102         }
11103       }
11104     }
11105 
11106     // Pattern: (fsub:f64 (load:f64 addr:iPTR:$src2), RFP:f64:$src1)
11107     // Emits: (FpSUBR64m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11108     // Pattern complexity = 14  cost = 1  size = 0
11109     if (N0.getOpcode() == ISD::LOAD &&
11110         InFlightSet.count(N0.Val) == 0 &&
11111         N0.hasOneUse() &&
11112         !CodeGenMap.count(N0.getValue(0)) &&
11113         !isNonImmUse(N.Val, N0.Val)) {
11114       Chain0 = N0.getOperand(0);
11115       N01 = N0.getOperand(1);
11116       N1 = N.getOperand(1);
11117       if (N.Val->getValueType(0) == MVT::f64) {
11118         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
11119         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11120         if (!Match) {
11121           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
11122         }
11123         if (Match) {
11124           Emit_20(Result, N, X86::FpSUBR64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
11125           return;
11126         }
11127       }
11128     }
11129     {
11130       N1 = N.getOperand(1);
11131       if (N1.getOpcode() == X86ISD::FILD &&
11132           InFlightSet.count(N1.Val) == 0 &&
11133           N1.hasOneUse() &&
11134           !CodeGenMap.count(N1.getValue(0)) &&
11135           !isNonImmUse(N.Val, N1.Val)) {
11136         Chain1 = N1.getOperand(0);
11137         N11 = N1.getOperand(1);
11138         N12 = N1.getOperand(2);
11139 
11140         // Pattern: (fsub:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i16:Other))
11141         // Emits: (FpISUB16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11142         // Pattern complexity = 14  cost = 1  size = 0
11143         if (cast<VTSDNode>(N12)->getVT() == MVT::i16) {
11144           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11145           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11146           if (!Match) {
11147             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11148           }
11149           if (Match) {
11150             Emit_100(Result, N, X86::FpISUB16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
11151             return;
11152           }
11153         }
11154 
11155         // Pattern: (fsub:f64 RFP:f64:$src1, (X86fild:f64 addr:iPTR:$src2, i32:Other))
11156         // Emits: (FpISUB32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11157         // Pattern complexity = 14  cost = 1
11158         if (cast<VTSDNode>(N12)->getVT() == MVT::i32) {
11159           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11160           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11161           if (!Match) {
11162             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11163           }
11164           if (Match) {
11165             Emit_100(Result, N, X86::FpISUB32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N12);
11166             return;
11167           }
11168         }
11169       }
11170     }
11171     if (N0.getOpcode() == X86ISD::FILD &&
11172         InFlightSet.count(N0.Val) == 0 &&
11173         N0.hasOneUse() &&
11174         !CodeGenMap.count(N0.getValue(0)) &&
11175         !isNonImmUse(N.Val, N0.Val)) {
11176       Chain0 = N0.getOperand(0);
11177       N01 = N0.getOperand(1);
11178       N02 = N0.getOperand(2);
11179 
11180       // Pattern: (fsub:f64 (X86fild:f64 addr:iPTR:$src2, i16:Other), RFP:f64:$src1)
11181       // Emits: (FpISUBR16m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11182       // Pattern complexity = 14  cost = 1  size = 0
11183       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
11184         N1 = N.getOperand(1);
11185         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
11186         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11187         if (!Match) {
11188           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
11189         }
11190         if (Match) {
11191           Emit_102(Result, N, X86::FpISUBR16m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
11192           return;
11193         }
11194       }
11195 
11196       // Pattern: (fsub:f64 (X86fild:f64 addr:iPTR:$src2, i32:Other), RFP:f64:$src1)
11197       // Emits: (FpISUBR32m:f64 RFP:f64:$src1, addr:iPTR:$src2)
11198       // Pattern complexity = 14  cost = 1
11199       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
11200         N1 = N.getOperand(1);
11201         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
11202         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11203         if (!Match) {
11204           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
11205         }
11206         if (Match) {
11207           Emit_102(Result, N, X86::FpISUBR32m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N02, N1);
11208           return;
11209         }
11210       }
11211     }
11212   }
11213 
11214   // Pattern: (fsub:f32 FR32:f32:$src1, (load:f32 addr:iPTR:$src2))
11215   // Emits: (SUBSSrm:f32 FR32:f32:$src1, addr:iPTR:$src2)
11216   // Pattern complexity = 14  cost = 1  size = 3
11217   if ((Subtarget->hasSSE1())) {
11218     N0 = N.getOperand(0);
11219     N1 = N.getOperand(1);
11220     if (N1.getOpcode() == ISD::LOAD &&
11221         InFlightSet.count(N1.Val) == 0 &&
11222         N1.hasOneUse() &&
11223         !CodeGenMap.count(N1.getValue(0)) &&
11224         !isNonImmUse(N.Val, N1.Val)) {
11225       Chain1 = N1.getOperand(0);
11226       N11 = N1.getOperand(1);
11227       if (N.Val->getValueType(0) == MVT::f32) {
11228         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11229         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11230         if (!Match) {
11231           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11232         }
11233         if (Match) {
11234           Emit_19(Result, N, X86::SUBSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
11235           return;
11236         }
11237       }
11238     }
11239   }
11240 
11241   // Pattern: (fsub:f64 FR64:f64:$src1, (load:f64 addr:iPTR:$src2))
11242   // Emits: (SUBSDrm:f64 FR64:f64:$src1, addr:iPTR:$src2)
11243   // Pattern complexity = 14  cost = 1  size = 3
11244   if ((Subtarget->hasSSE2())) {
11245     N0 = N.getOperand(0);
11246     N1 = N.getOperand(1);
11247     if (N1.getOpcode() == ISD::LOAD &&
11248         InFlightSet.count(N1.Val) == 0 &&
11249         N1.hasOneUse() &&
11250         !CodeGenMap.count(N1.getValue(0)) &&
11251         !isNonImmUse(N.Val, N1.Val)) {
11252       Chain1 = N1.getOperand(0);
11253       N11 = N1.getOperand(1);
11254       if (N.Val->getValueType(0) == MVT::f64) {
11255         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11256         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11257         if (!Match) {
11258           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11259         }
11260         if (Match) {
11261           Emit_19(Result, N, X86::SUBSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
11262           return;
11263         }
11264       }
11265     }
11266   }
11267 
11268   // Pattern: (fsub:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
11269   // Emits: (SUBPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
11270   // Pattern complexity = 14  cost = 1  size = 3
11271   if ((Subtarget->hasSSE1())) {
11272     N0 = N.getOperand(0);
11273     N1 = N.getOperand(1);
11274     if (N1.getOpcode() == ISD::LOAD &&
11275         InFlightSet.count(N1.Val) == 0 &&
11276         N1.hasOneUse() &&
11277         !CodeGenMap.count(N1.getValue(0)) &&
11278         !isNonImmUse(N.Val, N1.Val)) {
11279       Chain1 = N1.getOperand(0);
11280       N11 = N1.getOperand(1);
11281       if (N.Val->getValueType(0) == MVT::v4f32) {
11282         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11283         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11284         if (!Match) {
11285           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11286         }
11287         if (Match) {
11288           Emit_19(Result, N, X86::SUBPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
11289           return;
11290         }
11291       }
11292     }
11293   }
11294 
11295   // Pattern: (fsub:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
11296   // Emits: (SUBPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
11297   // Pattern complexity = 14  cost = 1  size = 3
11298   if ((Subtarget->hasSSE2())) {
11299     N0 = N.getOperand(0);
11300     N1 = N.getOperand(1);
11301     if (N1.getOpcode() == ISD::LOAD &&
11302         InFlightSet.count(N1.Val) == 0 &&
11303         N1.hasOneUse() &&
11304         !CodeGenMap.count(N1.getValue(0)) &&
11305         !isNonImmUse(N.Val, N1.Val)) {
11306       Chain1 = N1.getOperand(0);
11307       N11 = N1.getOperand(1);
11308       if (N.Val->getValueType(0) == MVT::v2f64) {
11309         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
11310         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
11311         if (!Match) {
11312           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
11313         }
11314         if (Match) {
11315           Emit_19(Result, N, X86::SUBPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
11316           return;
11317         }
11318       }
11319     }
11320   }
11321 
11322   // Pattern: (fsub:f64 RFP:f64:$src1, RFP:f64:$src2)
11323   // Emits: (FpSUB:f64 RFP:f64:$src1, RFP:f64:$src2)
11324   // Pattern complexity = 2  cost = 1  size = 0
11325   if ((!Subtarget->hasSSE2())) {
11326     N0 = N.getOperand(0);
11327     N1 = N.getOperand(1);
11328     if (N.Val->getValueType(0) == MVT::f64) {
11329       Emit_21(Result, N, X86::FpSUB, MVT::f64, N0, N1);
11330       return;
11331     }
11332   }
11333 
11334   // Pattern: (fsub:f32 FR32:f32:$src1, FR32:f32:$src2)
11335   // Emits: (SUBSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
11336   // Pattern complexity = 2  cost = 1  size = 3
11337   if ((Subtarget->hasSSE1())) {
11338     N0 = N.getOperand(0);
11339     N1 = N.getOperand(1);
11340     if (N.Val->getValueType(0) == MVT::f32) {
11341       Emit_21(Result, N, X86::SUBSSrr, MVT::f32, N0, N1);
11342       return;
11343     }
11344   }
11345 
11346   // Pattern: (fsub:f64 FR64:f64:$src1, FR64:f64:$src2)
11347   // Emits: (SUBSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
11348   // Pattern complexity = 2  cost = 1  size = 3
11349   if ((Subtarget->hasSSE2())) {
11350     N0 = N.getOperand(0);
11351     N1 = N.getOperand(1);
11352     if (N.Val->getValueType(0) == MVT::f64) {
11353       Emit_21(Result, N, X86::SUBSDrr, MVT::f64, N0, N1);
11354       return;
11355     }
11356   }
11357 
11358   // Pattern: (fsub:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
11359   // Emits: (SUBPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
11360   // Pattern complexity = 2  cost = 1  size = 3
11361   if ((Subtarget->hasSSE1())) {
11362     N0 = N.getOperand(0);
11363     N1 = N.getOperand(1);
11364     if (N.Val->getValueType(0) == MVT::v4f32) {
11365       Emit_21(Result, N, X86::SUBPSrr, MVT::v4f32, N0, N1);
11366       return;
11367     }
11368   }
11369 
11370   // Pattern: (fsub:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
11371   // Emits: (SUBPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
11372   // Pattern complexity = 2  cost = 1
11373   if ((Subtarget->hasSSE2())) {
11374     N0 = N.getOperand(0);
11375     N1 = N.getOperand(1);
11376     if (N.Val->getValueType(0) == MVT::v2f64) {
11377       Emit_21(Result, N, X86::SUBPDrr, MVT::v2f64, N0, N1);
11378       return;
11379     }
11380   }
11381   std::cerr << "Cannot yet select: ";
11382   N.Val->dump(CurDAG);
11383   std::cerr << '\n';
11384   abort();
11385 }
11386 
11387 void Emit_104(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0) NOINLINE {
11388   SDOperand Tmp0(0, 0);
11389   SDNode *ResNode = NULL;
11390   Tmp0 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N)->getValue()), MVT::i8);
11391   if (N.Val->hasOneUse()) {
11392     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
11393   } else {
11394     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
11395     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11396     Result = SDOperand(ResNode, 0);
11397   }
11398   return;
11399 }
11400 void Emit_105(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0) NOINLINE {
11401   SDOperand Tmp0(0, 0);
11402   SDNode *ResNode = NULL;
11403   Tmp0 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N)->getValue()), MVT::i16);
11404   if (N.Val->hasOneUse()) {
11405     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
11406   } else {
11407     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
11408     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11409     Result = SDOperand(ResNode, 0);
11410   }
11411   return;
11412 }
11413 void Emit_106(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0) NOINLINE {
11414   SDOperand Tmp0(0, 0);
11415   SDNode *ResNode = NULL;
11416   Tmp0 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N)->getValue()), MVT::i32);
11417   if (N.Val->hasOneUse()) {
11418     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
11419   } else {
11420     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
11421     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11422     Result = SDOperand(ResNode, 0);
11423   }
11424   return;
11425 }
11426 void Select_imm(SDOperand &Result, SDOperand N) {
11427   SDOperand Tmp0(0, 0);
11428   SDNode *ResNode = NULL;
11429   if (cast<ConstantSDNode>(N)->getSignExtended() == 0) {
11430 
11431     // Pattern: 0:i8
11432     // Emits: (MOV8r0:i8)
11433     // Pattern complexity = 3  cost = 1  size = 3
11434     if (N.Val->getValueType(0) == MVT::i8) {
11435       Emit_96(Result, N, X86::MOV8r0, MVT::i8);
11436       return;
11437     }
11438 
11439     // Pattern: 0:i16
11440     // Emits: (MOV16r0:i16)
11441     // Pattern complexity = 3  cost = 1  size = 3
11442     if (N.Val->getValueType(0) == MVT::i16) {
11443       Emit_96(Result, N, X86::MOV16r0, MVT::i16);
11444       return;
11445     }
11446 
11447     // Pattern: 0:i32
11448     // Emits: (MOV32r0:i32)
11449     // Pattern complexity = 3  cost = 1
11450     if (N.Val->getValueType(0) == MVT::i32) {
11451       Emit_96(Result, N, X86::MOV32r0, MVT::i32);
11452       return;
11453     }
11454   }
11455 
11456   // Pattern: (imm:i8):$src
11457   // Emits: (MOV8ri:i8 (imm:i8):$src)
11458   // Pattern complexity = 2  cost = 1  size = 3
11459   if (N.Val->getValueType(0) == MVT::i8) {
11460     Emit_104(Result, N, X86::MOV8ri, MVT::i8);
11461     return;
11462   }
11463 
11464   // Pattern: (imm:i16):$src
11465   // Emits: (MOV16ri:i16 (imm:i16):$src)
11466   // Pattern complexity = 2  cost = 1  size = 3
11467   if (N.Val->getValueType(0) == MVT::i16) {
11468     Emit_105(Result, N, X86::MOV16ri, MVT::i16);
11469     return;
11470   }
11471 
11472   // Pattern: (imm:i32):$src
11473   // Emits: (MOV32ri:i32 (imm:i32):$src)
11474   // Pattern complexity = 2  cost = 1
11475   if (N.Val->getValueType(0) == MVT::i32) {
11476     Emit_106(Result, N, X86::MOV32ri, MVT::i32);
11477     return;
11478   }
11479   std::cerr << "Cannot yet select: ";
11480   N.Val->dump(CurDAG);
11481   std::cerr << '\n';
11482   abort();
11483 }
11484 
11485 void Emit_107(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
11486   SDOperand Tmp1(0, 0);
11487   SDOperand Tmp2(0, 0);
11488   SDOperand Tmp3(0, 0);
11489   SDOperand Tmp4(0, 0);
11490   SDOperand Tmp5(0, 0);
11491   SDNode *ResNode = NULL;
11492   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
11493   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
11494   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
11495   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
11496   Select(Tmp1, CPTmp1);
11497   Select(Tmp2, CPTmp2);
11498   Select(Tmp3, CPTmp3);
11499   Select(Tmp4, CPTmp4);
11500   Select(Tmp5, N3);
11501   Select(Chain, Chain);
11502   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11503   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
11504   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
11505   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
11506   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
11507   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11508   Chain = SDOperand(ResNode, 0);
11509   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11510   Result = SDOperand(ResNode, N.ResNo);
11511   return;
11512 }
11513 void Emit_108(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
11514   SDOperand Tmp1(0, 0);
11515   SDOperand Tmp2(0, 0);
11516   SDOperand Tmp3(0, 0);
11517   SDOperand Tmp4(0, 0);
11518   SDNode *ResNode = NULL;
11519   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
11520   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
11521   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
11522   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
11523   Select(Tmp1, CPTmp1);
11524   Select(Tmp2, CPTmp2);
11525   Select(Tmp3, CPTmp3);
11526   Select(Tmp4, CPTmp4);
11527   Select(Chain, Chain);
11528   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
11529   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
11530   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
11531   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
11532   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
11533   Chain = SDOperand(ResNode, 0);
11534   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11535   Result = SDOperand(ResNode, N.ResNo);
11536   return;
11537 }
11538 void Emit_109(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3, SDOperand &N4) NOINLINE {
11539   SDOperand InFlag(0, 0);
11540   SDOperand Tmp1(0, 0);
11541   SDOperand Tmp2(0, 0);
11542   SDNode *ResNode = NULL;
11543   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N2.Val);
11544   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11545   Select(Tmp1, N2);
11546   Select(Tmp2, N3);
11547   Select(Chain, Chain);
11548   Select(N4, N4);
11549   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::EDI, MVT::i32), N4, InFlag).Val;
11550   Chain = SDOperand(ResNode, 0);
11551   InFlag = SDOperand(ResNode, 1);
11552   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
11553   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11554   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Chain, InFlag);
11555   Chain = SDOperand(ResNode, 0);
11556   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11557   Result = SDOperand(ResNode, N.ResNo);
11558   return;
11559 }
11560 void Emit_110(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1) NOINLINE {
11561   SDNode *ResNode = NULL;
11562   Select(Chain, Chain);
11563   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Chain);
11564   Chain = SDOperand(ResNode, 0);
11565   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11566   Result = SDOperand(ResNode, N.ResNo);
11567   return;
11568 }
11569 void Emit_111(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3, SDOperand &N4) NOINLINE {
11570   SDOperand InFlag(0, 0);
11571   SDNode *ResNode = NULL;
11572   Select(Chain, Chain);
11573   Select(N2, N2);
11574   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::EAX, MVT::Other), N2, InFlag).Val;
11575   Chain = SDOperand(ResNode, 0);
11576   InFlag = SDOperand(ResNode, 1);
11577   Select(N3, N3);
11578   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::ECX, MVT::Other), N3, InFlag).Val;
11579   Chain = SDOperand(ResNode, 0);
11580   InFlag = SDOperand(ResNode, 1);
11581   Select(N4, N4);
11582   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::EDX, MVT::Other), N4, InFlag).Val;
11583   Chain = SDOperand(ResNode, 0);
11584   InFlag = SDOperand(ResNode, 1);
11585   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Chain, InFlag);
11586   Chain = SDOperand(ResNode, 0);
11587   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11588   Result = SDOperand(ResNode, N.ResNo);
11589   return;
11590 }
11591 void Emit_112(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
11592   SDOperand InFlag(0, 0);
11593   SDNode *ResNode = NULL;
11594   N2 = N.getOperand(2);
11595   N3 = N.getOperand(3);
11596   Select(Chain, Chain);
11597   Select(N2, N2);
11598   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::ECX, MVT::Other), N2, InFlag).Val;
11599   Chain = SDOperand(ResNode, 0);
11600   InFlag = SDOperand(ResNode, 1);
11601   Select(N3, N3);
11602   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::EAX, MVT::Other), N3, InFlag).Val;
11603   Chain = SDOperand(ResNode, 0);
11604   InFlag = SDOperand(ResNode, 1);
11605   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Chain, InFlag);
11606   Chain = SDOperand(ResNode, 0);
11607   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
11608   Result = SDOperand(ResNode, N.ResNo);
11609   return;
11610 }
11611 void Select_intrinsic_void(SDOperand &Result, SDOperand N) {
11612   SDOperand CPTmp1(0, 0);
11613   SDOperand CPTmp2(0, 0);
11614   SDOperand CPTmp3(0, 0);
11615   SDOperand CPTmp4(0, 0);
11616   SDOperand Chain(0, 0);
11617   SDOperand InFlag(0, 0);
11618   SDOperand N1(0, 0);
11619   SDOperand N2(0, 0);
11620   SDOperand N3(0, 0);
11621   SDOperand N4(0, 0);
11622   SDOperand Tmp1(0, 0);
11623   SDOperand Tmp2(0, 0);
11624   SDOperand Tmp3(0, 0);
11625   SDOperand Tmp4(0, 0);
11626   SDOperand Tmp5(0, 0);
11627   SDNode *ResNode = NULL;
11628 
11629   // Pattern: (intrinsic_void:void 338:iPTR, addr:iPTR:$dst, VR128:v4f32:$src)
11630   // Emits: (MOVUPSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
11631   // Pattern complexity = 15  cost = 1  size = 3
11632   if ((Subtarget->hasSSE1())) {
11633     Chain = N.getOperand(0);
11634     N1 = N.getOperand(1);
11635     if (isa<ConstantSDNode>(N1)) {
11636       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11637       if (CN0 == 338) {
11638         N2 = N.getOperand(2);
11639         N3 = N.getOperand(3);
11640         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11641         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11642         if (!Match) {
11643           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11644         }
11645         if (Match) {
11646           Emit_107(Result, N, X86::MOVUPSmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11647           return;
11648         }
11649       }
11650     }
11651   }
11652   if ((Subtarget->hasSSE2())) {
11653     Chain = N.getOperand(0);
11654     N1 = N.getOperand(1);
11655     if (isa<ConstantSDNode>(N1)) {
11656       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11657 
11658       // Pattern: (intrinsic_void:void 289:iPTR, addr:iPTR:$dst, VR128:v2f64:$src)
11659       // Emits: (MOVUPDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
11660       // Pattern complexity = 15  cost = 1  size = 3
11661       if (CN0 == 289) {
11662         N2 = N.getOperand(2);
11663         N3 = N.getOperand(3);
11664         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11665         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11666         if (!Match) {
11667           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11668         }
11669         if (Match) {
11670           Emit_107(Result, N, X86::MOVUPDmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11671           return;
11672         }
11673       }
11674 
11675       // Pattern: (intrinsic_void:void 288:iPTR, addr:iPTR:$dst, VR128:v16i8:$src)
11676       // Emits: (MOVDQUmr:void addr:iPTR:$dst, VR128:v16i8:$src)
11677       // Pattern complexity = 15  cost = 1
11678       if (CN0 == 288) {
11679         N2 = N.getOperand(2);
11680         N3 = N.getOperand(3);
11681         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11682         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11683         if (!Match) {
11684           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11685         }
11686         if (Match) {
11687           Emit_107(Result, N, X86::MOVDQUmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11688           return;
11689         }
11690       }
11691     }
11692   }
11693 
11694   // Pattern: (intrinsic_void:void 328:iPTR, addr:iPTR:$dst, VR128:v4f32:$src)
11695   // Emits: (MOVNTPSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
11696   // Pattern complexity = 15  cost = 1  size = 3
11697   if ((Subtarget->hasSSE1())) {
11698     Chain = N.getOperand(0);
11699     N1 = N.getOperand(1);
11700     if (isa<ConstantSDNode>(N1)) {
11701       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11702       if (CN0 == 328) {
11703         N2 = N.getOperand(2);
11704         N3 = N.getOperand(3);
11705         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11706         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11707         if (!Match) {
11708           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11709         }
11710         if (Match) {
11711           Emit_107(Result, N, X86::MOVNTPSmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11712           return;
11713         }
11714       }
11715     }
11716   }
11717   if ((Subtarget->hasSSE2())) {
11718     Chain = N.getOperand(0);
11719     N1 = N.getOperand(1);
11720     if (isa<ConstantSDNode>(N1)) {
11721       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11722 
11723       // Pattern: (intrinsic_void:void 236:iPTR, addr:iPTR:$dst, VR128:v2f64:$src)
11724       // Emits: (MOVNTPDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
11725       // Pattern complexity = 15  cost = 1  size = 3
11726       if (CN0 == 236) {
11727         N2 = N.getOperand(2);
11728         N3 = N.getOperand(3);
11729         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11730         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11731         if (!Match) {
11732           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11733         }
11734         if (Match) {
11735           Emit_107(Result, N, X86::MOVNTPDmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11736           return;
11737         }
11738       }
11739 
11740       // Pattern: (intrinsic_void:void 234:iPTR, addr:iPTR:$dst, VR128:v2i64:$src)
11741       // Emits: (MOVNTDQmr:void addr:iPTR:$dst, VR128:v2i64:$src)
11742       // Pattern complexity = 15  cost = 1  size = 3
11743       if (CN0 == 234) {
11744         N2 = N.getOperand(2);
11745         N3 = N.getOperand(3);
11746         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11747         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11748         if (!Match) {
11749           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11750         }
11751         if (Match) {
11752           Emit_107(Result, N, X86::MOVNTDQmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11753           return;
11754         }
11755       }
11756 
11757       // Pattern: (intrinsic_void:void 235:iPTR, addr:iPTR:$dst, GR32:i32:$src)
11758       // Emits: (MOVNTImr:void addr:iPTR:$dst, GR32:i32:$src)
11759       // Pattern complexity = 15  cost = 1  size = 3
11760       if (CN0 == 235) {
11761         N2 = N.getOperand(2);
11762         N3 = N.getOperand(3);
11763         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11764         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11765         if (!Match) {
11766           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11767         }
11768         if (Match) {
11769           Emit_107(Result, N, X86::MOVNTImr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11770           return;
11771         }
11772       }
11773 
11774       // Pattern: (intrinsic_void:void 198:iPTR, addr:iPTR:$src)
11775       // Emits: (CLFLUSH:void addr:iPTR:$src)
11776       // Pattern complexity = 15  cost = 1
11777       if (CN0 == 198) {
11778         N2 = N.getOperand(2);
11779         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11780         if (Match) {
11781           Emit_108(Result, N, X86::CLFLUSH, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
11782           return;
11783         }
11784       }
11785     }
11786   }
11787   if ((Subtarget->hasSSE1())) {
11788     Chain = N.getOperand(0);
11789     N1 = N.getOperand(1);
11790     if (isa<ConstantSDNode>(N1)) {
11791       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11792 
11793       // Pattern: (intrinsic_void:void 321:iPTR, addr:iPTR:$src)
11794       // Emits: (LDMXCSR:void addr:iPTR:$src)
11795       // Pattern complexity = 15  cost = 1  size = 3
11796       if (CN0 == 321) {
11797         N2 = N.getOperand(2);
11798         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11799         if (Match) {
11800           Emit_108(Result, N, X86::LDMXCSR, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
11801           return;
11802         }
11803       }
11804 
11805       // Pattern: (intrinsic_void:void 337:iPTR, addr:iPTR:$dst)
11806       // Emits: (STMXCSR:void addr:iPTR:$dst)
11807       // Pattern complexity = 15  cost = 1
11808       if (CN0 == 337) {
11809         N2 = N.getOperand(2);
11810         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11811         if (Match) {
11812           Emit_108(Result, N, X86::STMXCSR, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
11813           return;
11814         }
11815       }
11816     }
11817   }
11818   if ((Subtarget->hasSSE2())) {
11819     Chain = N.getOperand(0);
11820     N1 = N.getOperand(1);
11821     if (isa<ConstantSDNode>(N1)) {
11822       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11823 
11824       // Pattern: (intrinsic_void:void 287:iPTR, addr:iPTR:$dst, VR128:v4i32:$src)
11825       // Emits: (MOVLQ128mr:void addr:iPTR:$dst, VR128:v4i32:$src)
11826       // Pattern complexity = 15  cost = 1  size = 3
11827       if (CN0 == 287) {
11828         N2 = N.getOperand(2);
11829         N3 = N.getOperand(3);
11830         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N3.Val);
11831         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
11832         if (!Match) {
11833           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N3.Val);
11834         }
11835         if (Match) {
11836           Emit_107(Result, N, X86::MOVLQ128mr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
11837           return;
11838         }
11839       }
11840 
11841       // Pattern: (intrinsic_void:void 226:iPTR, VR128:v16i8:$src, VR128:v16i8:$mask, EDI:i32)
11842       // Emits: (MASKMOVDQU:void VR128:v16i8:$src, VR128:v16i8:$mask)
11843       // Pattern complexity = 5  cost = 1
11844       if (CN0 == 226) {
11845         N2 = N.getOperand(2);
11846         N3 = N.getOperand(3);
11847         N4 = N.getOperand(4);
11848         if (N4.Val->getValueType(0) == MVT::i32) {
11849           Emit_109(Result, N, X86::MASKMOVDQU, Chain, N1, N2, N3, N4);
11850           return;
11851         }
11852       }
11853     }
11854   }
11855 
11856   // Pattern: (intrinsic_void:void 334:iPTR)
11857   // Emits: (SFENCE:void)
11858   // Pattern complexity = 5  cost = 1  size = 3
11859   if ((Subtarget->hasSSE1())) {
11860     Chain = N.getOperand(0);
11861     N1 = N.getOperand(1);
11862     if (isa<ConstantSDNode>(N1)) {
11863       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11864       if (CN0 == 334) {
11865         Emit_110(Result, N, X86::SFENCE, Chain, N1);
11866         return;
11867       }
11868     }
11869   }
11870   if ((Subtarget->hasSSE2())) {
11871     Chain = N.getOperand(0);
11872     N1 = N.getOperand(1);
11873     if (isa<ConstantSDNode>(N1)) {
11874       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11875 
11876       // Pattern: (intrinsic_void:void 221:iPTR)
11877       // Emits: (LFENCE:void)
11878       // Pattern complexity = 5  cost = 1  size = 3
11879       if (CN0 == 221) {
11880         Emit_110(Result, N, X86::LFENCE, Chain, N1);
11881         return;
11882       }
11883 
11884       // Pattern: (intrinsic_void:void 229:iPTR)
11885       // Emits: (MFENCE:void)
11886       // Pattern complexity = 5  cost = 1
11887       if (CN0 == 229) {
11888         Emit_110(Result, N, X86::MFENCE, Chain, N1);
11889         return;
11890       }
11891     }
11892   }
11893   if ((Subtarget->hasSSE3())) {
11894     Chain = N.getOperand(0);
11895     N1 = N.getOperand(1);
11896     if (isa<ConstantSDNode>(N1)) {
11897       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11898 
11899       // Pattern: (intrinsic_void:void 306:iPTR, EAX:i32, ECX:i32, EDX:i32)
11900       // Emits: (MONITOR:void)
11901       // Pattern complexity = 5  cost = 1  size = 3
11902       if (CN0 == 306) {
11903         N2 = N.getOperand(2);
11904         N3 = N.getOperand(3);
11905         N4 = N.getOperand(4);
11906         if (N2.Val->getValueType(0) == MVT::i32) {
11907           Emit_111(Result, N, X86::MONITOR, Chain, N1, N2, N3, N4);
11908           return;
11909         }
11910       }
11911 
11912       // Pattern: (intrinsic_void:void 307:iPTR, ECX:i32, EAX:i32)
11913       // Emits: (MWAIT:void)
11914       // Pattern complexity = 5  cost = 1
11915       if (CN0 == 307) {
11916         Emit_112(Result, N, X86::MWAIT, Chain, N1, N2, N3);
11917         return;
11918       }
11919     }
11920   }
11921   std::cerr << "Cannot yet select: ";
11922   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
11923   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
11924   std::cerr << '\n';
11925   abort();
11926 }
11927 
11928 void Emit_113(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
11929   SDOperand Tmp1(0, 0);
11930   SDOperand Tmp2(0, 0);
11931   SDOperand Tmp3(0, 0);
11932   SDOperand Tmp4(0, 0);
11933   SDNode *ResNode = NULL;
11934   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
11935   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
11936   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
11937   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
11938   Select(Tmp1, CPTmp1);
11939   Select(Tmp2, CPTmp2);
11940   Select(Tmp3, CPTmp3);
11941   Select(Tmp4, CPTmp4);
11942   Select(Chain, Chain);
11943   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
11944   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
11945   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
11946   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
11947   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
11948   Chain = SDOperand(ResNode, 1);
11949   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11950   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
11951   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
11952   Result = SDOperand(ResNode, N.ResNo);
11953   return;
11954 }
11955 void Emit_114(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
11956   SDOperand Tmp1(0, 0);
11957   SDOperand Tmp2(0, 0);
11958   SDOperand Tmp3(0, 0);
11959   SDOperand Tmp4(0, 0);
11960   SDOperand Tmp5(0, 0);
11961   SDNode *ResNode = NULL;
11962   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
11963   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
11964   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
11965   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
11966   Select(Tmp1, CPTmp1);
11967   Select(Tmp2, CPTmp2);
11968   Select(Tmp3, CPTmp3);
11969   Select(Tmp4, CPTmp4);
11970   Select(Tmp5, N2);
11971   Select(Chain, Chain);
11972   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
11973   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
11974   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
11975   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
11976   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
11977   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
11978   Chain = SDOperand(ResNode, 1);
11979   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11980   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Chain.Val, Chain.ResNo);
11981   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, Chain.Val, Chain.ResNo);
11982   Result = SDOperand(ResNode, N.ResNo);
11983   return;
11984 }
11985 void Select_intrinsic_w_chain(SDOperand &Result, SDOperand N) {
11986   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
11987     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
11988     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
11989     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
11990     Result = Dummy;
11991     return;
11992   }
11993   SDOperand CPTmp1(0, 0);
11994   SDOperand CPTmp2(0, 0);
11995   SDOperand CPTmp3(0, 0);
11996   SDOperand CPTmp4(0, 0);
11997   SDOperand Chain(0, 0);
11998   SDOperand N1(0, 0);
11999   SDOperand N2(0, 0);
12000   SDOperand N3(0, 0);
12001   SDOperand Tmp1(0, 0);
12002   SDOperand Tmp2(0, 0);
12003   SDOperand Tmp3(0, 0);
12004   SDOperand Tmp4(0, 0);
12005   SDOperand Tmp5(0, 0);
12006   SDNode *ResNode = NULL;
12007 
12008   // Pattern: (intrinsic_w_chain:v4f32 322:iPTR, addr:iPTR:$src)
12009   // Emits: (MOVUPSrm:v4f32 addr:iPTR:$src)
12010   // Pattern complexity = 15  cost = 1  size = 3
12011   if ((Subtarget->hasSSE1())) {
12012     Chain = N.getOperand(0);
12013     N1 = N.getOperand(1);
12014     if (isa<ConstantSDNode>(N1)) {
12015       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12016       if (CN0 == 322) {
12017         N2 = N.getOperand(2);
12018         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12019         if (Match) {
12020           Emit_113(Result, N, X86::MOVUPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
12021           return;
12022         }
12023       }
12024     }
12025   }
12026   if ((Subtarget->hasSSE2())) {
12027     Chain = N.getOperand(0);
12028     N1 = N.getOperand(1);
12029     if (isa<ConstantSDNode>(N1)) {
12030       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12031 
12032       // Pattern: (intrinsic_w_chain:v2f64 225:iPTR, addr:iPTR:$src)
12033       // Emits: (MOVUPDrm:v2f64 addr:iPTR:$src)
12034       // Pattern complexity = 15  cost = 1  size = 3
12035       if (CN0 == 225) {
12036         N2 = N.getOperand(2);
12037         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12038         if (Match) {
12039           Emit_113(Result, N, X86::MOVUPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
12040           return;
12041         }
12042       }
12043 
12044       // Pattern: (intrinsic_w_chain:v16i8 224:iPTR, addr:iPTR:$src)
12045       // Emits: (MOVDQUrm:v16i8 addr:iPTR:$src)
12046       // Pattern complexity = 15  cost = 1
12047       if (CN0 == 224) {
12048         N2 = N.getOperand(2);
12049         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12050         if (Match) {
12051           Emit_113(Result, N, X86::MOVDQUrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
12052           return;
12053         }
12054       }
12055     }
12056   }
12057 
12058   // Pattern: (intrinsic_w_chain:v16i8 305:iPTR, addr:iPTR:$src)
12059   // Emits: (LDDQUrm:v16i8 addr:iPTR:$src)
12060   // Pattern complexity = 15  cost = 1  size = 3
12061   if ((Subtarget->hasSSE3())) {
12062     Chain = N.getOperand(0);
12063     N1 = N.getOperand(1);
12064     if (isa<ConstantSDNode>(N1)) {
12065       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12066       if (CN0 == 305) {
12067         N2 = N.getOperand(2);
12068         bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12069         if (Match) {
12070           Emit_113(Result, N, X86::LDDQUrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2);
12071           return;
12072         }
12073       }
12074     }
12075   }
12076   if ((Subtarget->hasSSE2())) {
12077     Chain = N.getOperand(0);
12078     N1 = N.getOperand(1);
12079     if (isa<ConstantSDNode>(N1)) {
12080       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12081 
12082       // Pattern: (intrinsic_w_chain:v2f64 222:iPTR, VR128:v2f64:$src1, addr:iPTR:$src2)
12083       // Emits: (MOVHPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
12084       // Pattern complexity = 15  cost = 1  size = 3
12085       if (CN0 == 222) {
12086         N2 = N.getOperand(2);
12087         N3 = N.getOperand(3);
12088         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N2.Val);
12089         bool Match = SelectAddr(N3, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12090         if (!Match) {
12091           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
12092         }
12093         if (Match) {
12094           Emit_114(Result, N, X86::MOVHPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
12095           return;
12096         }
12097       }
12098 
12099       // Pattern: (intrinsic_w_chain:v2f64 223:iPTR, VR128:v2f64:$src1, addr:iPTR:$src2)
12100       // Emits: (MOVLPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
12101       // Pattern complexity = 15  cost = 1
12102       if (CN0 == 223) {
12103         N2 = N.getOperand(2);
12104         N3 = N.getOperand(3);
12105         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N2.Val);
12106         bool Match = SelectAddr(N3, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12107         if (!Match) {
12108           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
12109         }
12110         if (Match) {
12111           Emit_114(Result, N, X86::MOVLPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N2, N3);
12112           return;
12113         }
12114       }
12115     }
12116   }
12117   std::cerr << "Cannot yet select: ";
12118   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
12119   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
12120   std::cerr << '\n';
12121   abort();
12122 }
12123 
12124 void Emit_115(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101) NOINLINE {
12125   SDOperand Tmp1(0, 0);
12126   SDOperand Tmp2(0, 0);
12127   SDOperand Tmp3(0, 0);
12128   SDOperand Tmp4(0, 0);
12129   SDNode *ResNode = NULL;
12130   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
12131   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
12132   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
12133   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
12134   Select(Tmp1, CPTmp1);
12135   Select(Tmp2, CPTmp2);
12136   Select(Tmp3, CPTmp3);
12137   Select(Tmp4, CPTmp4);
12138   Select(Chain10, Chain10);
12139   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
12140   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
12141   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
12142   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
12143   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
12144   Chain10 = SDOperand(ResNode, 1);
12145   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12146   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
12147   AddHandleReplacement(N10.Val, 1, ResNode, 1);
12148   Result = SDOperand(ResNode, N.ResNo);
12149   return;
12150 }
12151 void Emit_116(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
12152   SDOperand Tmp1(0, 0);
12153   SDNode *ResNode = NULL;
12154   N1 = N.getOperand(1);
12155   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12156   Select(Tmp1, N1);
12157   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12158   if (N.Val->hasOneUse()) {
12159     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1);
12160   } else {
12161     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1);
12162     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
12163     Result = SDOperand(ResNode, 0);
12164   }
12165   return;
12166 }
12167 void Emit_117(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain2, SDOperand &N0, SDOperand &N1, SDOperand &N2, SDOperand &N21, SDOperand &N3) NOINLINE {
12168   SDOperand Tmp1(0, 0);
12169   SDOperand Tmp2(0, 0);
12170   SDOperand Tmp3(0, 0);
12171   SDOperand Tmp4(0, 0);
12172   SDOperand Tmp5(0, 0);
12173   SDOperand Tmp6(0, 0);
12174   SDNode *ResNode = NULL;
12175   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
12176   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
12177   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
12178   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
12179   Select(Tmp1, CPTmp1);
12180   Select(Tmp2, CPTmp2);
12181   Select(Tmp3, CPTmp3);
12182   Select(Tmp4, CPTmp4);
12183   Select(Tmp5, N1);
12184   Tmp6 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N3)->getValue()), MVT::i8);
12185   Select(Chain2, Chain2);
12186   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12187   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
12188   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
12189   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
12190   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
12191   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Tmp6, Chain2);
12192   Chain2 = SDOperand(ResNode, 1);
12193   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12194   SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
12195   AddHandleReplacement(N2.Val, 1, ResNode, 1);
12196   Result = SDOperand(ResNode, N.ResNo);
12197   return;
12198 }
12199 void Emit_118(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain20, SDOperand &N0, SDOperand &N1, SDOperand &N2, SDOperand &N20, SDOperand &N201) NOINLINE {
12200   SDOperand Tmp1(0, 0);
12201   SDOperand Tmp2(0, 0);
12202   SDOperand Tmp3(0, 0);
12203   SDOperand Tmp4(0, 0);
12204   SDOperand Tmp5(0, 0);
12205   SDNode *ResNode = NULL;
12206   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
12207   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
12208   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
12209   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
12210   Select(Tmp1, CPTmp1);
12211   Select(Tmp2, CPTmp2);
12212   Select(Tmp3, CPTmp3);
12213   Select(Tmp4, CPTmp4);
12214   Select(Tmp5, N1);
12215   Select(Chain20, Chain20);
12216   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12217   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
12218   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
12219   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
12220   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
12221   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12222   Chain20 = SDOperand(ResNode, 1);
12223   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12224   SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12225   AddHandleReplacement(N20.Val, 1, ResNode, 1);
12226   Result = SDOperand(ResNode, N.ResNo);
12227   return;
12228 }
12229 void Emit_119(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain2, SDOperand &N0, SDOperand &N1, SDOperand &N2, SDOperand &N21) NOINLINE {
12230   SDOperand Tmp1(0, 0);
12231   SDOperand Tmp2(0, 0);
12232   SDOperand Tmp3(0, 0);
12233   SDOperand Tmp4(0, 0);
12234   SDOperand Tmp5(0, 0);
12235   SDNode *ResNode = NULL;
12236   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
12237   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
12238   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
12239   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
12240   Select(Tmp1, CPTmp1);
12241   Select(Tmp2, CPTmp2);
12242   Select(Tmp3, CPTmp3);
12243   Select(Tmp4, CPTmp4);
12244   Select(Tmp5, N1);
12245   Select(Chain2, Chain2);
12246   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12247   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
12248   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
12249   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
12250   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
12251   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
12252   Chain2 = SDOperand(ResNode, 1);
12253   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12254   SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
12255   AddHandleReplacement(N2.Val, 1, ResNode, 1);
12256   Result = SDOperand(ResNode, N.ResNo);
12257   return;
12258 }
12259 void Emit_120(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
12260   SDOperand Tmp1(0, 0);
12261   SDOperand Tmp2(0, 0);
12262   SDOperand Tmp3(0, 0);
12263   SDOperand Tmp4(0, 0);
12264   SDNode *ResNode = NULL;
12265   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
12266   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
12267   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
12268   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
12269   Select(Tmp1, CPTmp1);
12270   Select(Tmp2, CPTmp2);
12271   Select(Tmp3, CPTmp3);
12272   Select(Tmp4, CPTmp4);
12273   Select(Chain1, Chain1);
12274   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
12275   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
12276   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
12277   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
12278   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
12279   Chain1 = SDOperand(ResNode, 1);
12280   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12281   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
12282   AddHandleReplacement(N1.Val, 1, ResNode, 1);
12283   Result = SDOperand(ResNode, N.ResNo);
12284   return;
12285 }
12286 void Emit_121(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2, SDOperand &N20) NOINLINE {
12287   SDOperand Tmp1(0, 0);
12288   SDOperand Tmp2(0, 0);
12289   SDNode *ResNode = NULL;
12290   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12291   Select(Tmp1, N1);
12292   Tmp2 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N20)->getValue()), MVT::i32);
12293   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12294   if (N.Val->hasOneUse()) {
12295     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp2);
12296   } else {
12297     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp2);
12298     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
12299     Result = SDOperand(ResNode, 0);
12300   }
12301   return;
12302 }
12303 void Emit_122(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2, SDOperand &N3) NOINLINE {
12304   SDOperand Tmp1(0, 0);
12305   SDOperand Tmp2(0, 0);
12306   SDOperand Tmp3(0, 0);
12307   SDNode *ResNode = NULL;
12308   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12309   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N2.Val);
12310   Select(Tmp1, N1);
12311   Select(Tmp2, N2);
12312   Tmp3 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N3)->getValue()), MVT::i8);
12313   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12314   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
12315   if (N.Val->hasOneUse()) {
12316     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp2, Tmp3);
12317   } else {
12318     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp2, Tmp3);
12319     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
12320     Result = SDOperand(ResNode, 0);
12321   }
12322   return;
12323 }
12324 void Emit_123(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
12325   SDOperand Tmp1(0, 0);
12326   SDOperand Tmp2(0, 0);
12327   SDOperand Tmp3(0, 0);
12328   SDNode *ResNode = NULL;
12329   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12330   Select(Tmp1, N1);
12331   Tmp2 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N2)->getValue()), MVT::i32);
12332   Tmp3 = Transform_PSxLDQ_imm(Tmp2.Val);
12333   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12334   if (N.Val->hasOneUse()) {
12335     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp3);
12336   } else {
12337     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp3);
12338     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
12339     Result = SDOperand(ResNode, 0);
12340   }
12341   return;
12342 }
12343 void Emit_124(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
12344   SDOperand Tmp1(0, 0);
12345   SDOperand Tmp2(0, 0);
12346   SDNode *ResNode = NULL;
12347   N1 = N.getOperand(1);
12348   N2 = N.getOperand(2);
12349   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12350   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N2.Val);
12351   Select(Tmp1, N1);
12352   Select(Tmp2, N2);
12353   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12354   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N2.Val);
12355   if (N.Val->hasOneUse()) {
12356     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp2);
12357   } else {
12358     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp2);
12359     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
12360     Result = SDOperand(ResNode, 0);
12361   }
12362   return;
12363 }
12364 void Select_intrinsic_wo_chain(SDOperand &Result, SDOperand N) {
12365   SDOperand CPTmp1(0, 0);
12366   SDOperand CPTmp2(0, 0);
12367   SDOperand CPTmp3(0, 0);
12368   SDOperand CPTmp4(0, 0);
12369   SDOperand Chain1(0, 0);
12370   SDOperand Chain10(0, 0);
12371   SDOperand Chain2(0, 0);
12372   SDOperand Chain20(0, 0);
12373   SDOperand N0(0, 0);
12374   SDOperand N1(0, 0);
12375   SDOperand N10(0, 0);
12376   SDOperand N101(0, 0);
12377   SDOperand N11(0, 0);
12378   SDOperand N2(0, 0);
12379   SDOperand N20(0, 0);
12380   SDOperand N201(0, 0);
12381   SDOperand N21(0, 0);
12382   SDOperand N3(0, 0);
12383   SDOperand Tmp1(0, 0);
12384   SDOperand Tmp2(0, 0);
12385   SDOperand Tmp3(0, 0);
12386   SDOperand Tmp4(0, 0);
12387   SDOperand Tmp5(0, 0);
12388   SDOperand Tmp6(0, 0);
12389   SDNode *ResNode = NULL;
12390   if ((Subtarget->hasSSE2())) {
12391     N0 = N.getOperand(0);
12392     if (isa<ConstantSDNode>(N0)) {
12393       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12394       if (CN0 == 232) {
12395 
12396         // Pattern: (intrinsic_wo_chain:v4i32 232:iPTR, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src)))
12397         // Emits: (MOVZQI2PQIrm:v4i32 addr:iPTR:$src)
12398         // Pattern complexity = 39  cost = 1  size = 3
12399         {
12400           N1 = N.getOperand(1);
12401           if (N1.getOpcode() == ISD::BIT_CONVERT &&
12402               InFlightSet.count(N1.Val) == 0 &&
12403               N1.hasOneUse() &&
12404               !CodeGenMap.count(N1.getValue(0))) {
12405             N10 = N1.getOperand(0);
12406             if (N10.getOpcode() == ISD::LOAD &&
12407                 InFlightSet.count(N10.Val) == 0 &&
12408                 N10.hasOneUse() &&
12409                 !CodeGenMap.count(N10.getValue(0))) {
12410               Chain10 = N10.getOperand(0);
12411               N101 = N10.getOperand(1);
12412               if (N10.Val->getValueType(0) == MVT::v2i64) {
12413                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12414                 if (Match) {
12415                   Emit_115(Result, N, X86::MOVZQI2PQIrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain10, N0, N1, N10, N101);
12416                   return;
12417                 }
12418               }
12419             }
12420           }
12421         }
12422 
12423         // Pattern: (intrinsic_wo_chain:v4i32 232:iPTR, VR128:v4i32:$src)
12424         // Emits: (MOVZQI2PQIrr:v4i32 VR128:v4i32:$src)
12425         // Pattern complexity = 25  cost = 1
12426         Emit_116(Result, N, X86::MOVZQI2PQIrr, MVT::v4i32, N0, N1);
12427         return;
12428       }
12429     }
12430   }
12431 
12432   // Pattern: (intrinsic_wo_chain:v4f32 310:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src), (imm:i8):$cc)
12433   // Emits: (Int_CMPSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src, (imm:i8):$cc)
12434   // Pattern complexity = 19  cost = 1  size = 3
12435   if ((Subtarget->hasSSE1())) {
12436     N0 = N.getOperand(0);
12437     if (isa<ConstantSDNode>(N0)) {
12438       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12439       if (CN0 == 310) {
12440         N1 = N.getOperand(1);
12441         N2 = N.getOperand(2);
12442         if (N2.getOpcode() == ISD::LOAD &&
12443             InFlightSet.count(N2.Val) == 0 &&
12444             N2.hasOneUse() &&
12445             !CodeGenMap.count(N2.getValue(0))) {
12446           Chain2 = N2.getOperand(0);
12447           N21 = N2.getOperand(1);
12448           N3 = N.getOperand(3);
12449           if (N3.getOpcode() == ISD::Constant) {
12450             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12451             bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12452             if (!Match) {
12453               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12454             }
12455             if (Match) {
12456               Emit_117(Result, N, X86::Int_CMPSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21, N3);
12457               return;
12458             }
12459           }
12460         }
12461       }
12462     }
12463   }
12464   if ((Subtarget->hasSSE2())) {
12465     N0 = N.getOperand(0);
12466     if (isa<ConstantSDNode>(N0)) {
12467       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12468 
12469       // Pattern: (intrinsic_wo_chain:v4f32 208:iPTR, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src)))
12470       // Emits: (Int_CVTDQ2PSrm:v4f32 addr:iPTR:$src)
12471       // Pattern complexity = 19  cost = 1  size = 3
12472       if (CN0 == 208) {
12473         N1 = N.getOperand(1);
12474         if (N1.getOpcode() == ISD::BIT_CONVERT &&
12475             InFlightSet.count(N1.Val) == 0 &&
12476             N1.hasOneUse() &&
12477             !CodeGenMap.count(N1.getValue(0))) {
12478           N10 = N1.getOperand(0);
12479           if (N10.getOpcode() == ISD::LOAD &&
12480               InFlightSet.count(N10.Val) == 0 &&
12481               N10.hasOneUse() &&
12482               !CodeGenMap.count(N10.getValue(0))) {
12483             Chain10 = N10.getOperand(0);
12484             N101 = N10.getOperand(1);
12485             if (N10.Val->getValueType(0) == MVT::v2i64) {
12486               bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12487               if (Match) {
12488                 Emit_115(Result, N, X86::Int_CVTDQ2PSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain10, N0, N1, N10, N101);
12489                 return;
12490               }
12491             }
12492           }
12493         }
12494       }
12495 
12496       // Pattern: (intrinsic_wo_chain:v2f64 207:iPTR, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src)))
12497       // Emits: (Int_CVTDQ2PDrm:v2f64 addr:iPTR:$src)
12498       // Pattern complexity = 19  cost = 1
12499       if (CN0 == 207) {
12500         N1 = N.getOperand(1);
12501         if (N1.getOpcode() == ISD::BIT_CONVERT &&
12502             InFlightSet.count(N1.Val) == 0 &&
12503             N1.hasOneUse() &&
12504             !CodeGenMap.count(N1.getValue(0))) {
12505           N10 = N1.getOperand(0);
12506           if (N10.getOpcode() == ISD::LOAD &&
12507               InFlightSet.count(N10.Val) == 0 &&
12508               N10.hasOneUse() &&
12509               !CodeGenMap.count(N10.getValue(0))) {
12510             Chain10 = N10.getOperand(0);
12511             N101 = N10.getOperand(1);
12512             if (N10.Val->getValueType(0) == MVT::v2i64) {
12513               bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12514               if (Match) {
12515                 Emit_115(Result, N, X86::Int_CVTDQ2PDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain10, N0, N1, N10, N101);
12516                 return;
12517               }
12518             }
12519           }
12520         }
12521       }
12522     }
12523   }
12524 
12525   // Pattern: (intrinsic_wo_chain:v4f32 309:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src), (imm:i8):$cc)
12526   // Emits: (CMPPSrmi:v4f32 VR128:v4f32:$src1, addr:iPTR:$src, (imm:i8):$cc)
12527   // Pattern complexity = 19  cost = 1  size = 3
12528   if ((Subtarget->hasSSE1())) {
12529     N0 = N.getOperand(0);
12530     if (isa<ConstantSDNode>(N0)) {
12531       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12532       if (CN0 == 309) {
12533         N1 = N.getOperand(1);
12534         N2 = N.getOperand(2);
12535         if (N2.getOpcode() == ISD::LOAD &&
12536             InFlightSet.count(N2.Val) == 0 &&
12537             N2.hasOneUse() &&
12538             !CodeGenMap.count(N2.getValue(0))) {
12539           Chain2 = N2.getOperand(0);
12540           N21 = N2.getOperand(1);
12541           N3 = N.getOperand(3);
12542           if (N3.getOpcode() == ISD::Constant) {
12543             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12544             bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12545             if (!Match) {
12546               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12547             }
12548             if (Match) {
12549               Emit_117(Result, N, X86::CMPPSrmi, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21, N3);
12550               return;
12551             }
12552           }
12553         }
12554       }
12555     }
12556   }
12557   if ((Subtarget->hasSSE2())) {
12558     N0 = N.getOperand(0);
12559     if (isa<ConstantSDNode>(N0)) {
12560       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12561 
12562       // Pattern: (intrinsic_wo_chain:v2f64 199:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src), (imm:i8):$cc)
12563       // Emits: (CMPPDrmi:v2f64 VR128:v2f64:$src1, addr:iPTR:$src, (imm:i8):$cc)
12564       // Pattern complexity = 19  cost = 1  size = 3
12565       if (CN0 == 199) {
12566         N1 = N.getOperand(1);
12567         N2 = N.getOperand(2);
12568         if (N2.getOpcode() == ISD::LOAD &&
12569             InFlightSet.count(N2.Val) == 0 &&
12570             N2.hasOneUse() &&
12571             !CodeGenMap.count(N2.getValue(0))) {
12572           Chain2 = N2.getOperand(0);
12573           N21 = N2.getOperand(1);
12574           N3 = N.getOperand(3);
12575           if (N3.getOpcode() == ISD::Constant) {
12576             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12577             bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12578             if (!Match) {
12579               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12580             }
12581             if (Match) {
12582               Emit_117(Result, N, X86::CMPPDrmi, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21, N3);
12583               return;
12584             }
12585           }
12586         }
12587       }
12588 
12589       // Pattern: (intrinsic_wo_chain:v16i8 242:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
12590       // Emits: (PADDSBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
12591       // Pattern complexity = 19  cost = 1  size = 3
12592       if (CN0 == 242) {
12593         N1 = N.getOperand(1);
12594         N2 = N.getOperand(2);
12595         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12596             InFlightSet.count(N2.Val) == 0 &&
12597             N2.hasOneUse() &&
12598             !CodeGenMap.count(N2.getValue(0))) {
12599           N20 = N2.getOperand(0);
12600           if (N20.getOpcode() == ISD::LOAD &&
12601               InFlightSet.count(N20.Val) == 0 &&
12602               N20.hasOneUse() &&
12603               !CodeGenMap.count(N20.getValue(0))) {
12604             Chain20 = N20.getOperand(0);
12605             N201 = N20.getOperand(1);
12606             if (N20.Val->getValueType(0) == MVT::v2i64) {
12607               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12608               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12609               if (!Match) {
12610                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12611               }
12612               if (Match) {
12613                 Emit_118(Result, N, X86::PADDSBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12614                 return;
12615               }
12616             }
12617           }
12618         }
12619       }
12620 
12621       // Pattern: (intrinsic_wo_chain:v8i16 243:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12622       // Emits: (PADDSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12623       // Pattern complexity = 19  cost = 1  size = 3
12624       if (CN0 == 243) {
12625         N1 = N.getOperand(1);
12626         N2 = N.getOperand(2);
12627         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12628             InFlightSet.count(N2.Val) == 0 &&
12629             N2.hasOneUse() &&
12630             !CodeGenMap.count(N2.getValue(0))) {
12631           N20 = N2.getOperand(0);
12632           if (N20.getOpcode() == ISD::LOAD &&
12633               InFlightSet.count(N20.Val) == 0 &&
12634               N20.hasOneUse() &&
12635               !CodeGenMap.count(N20.getValue(0))) {
12636             Chain20 = N20.getOperand(0);
12637             N201 = N20.getOperand(1);
12638             if (N20.Val->getValueType(0) == MVT::v2i64) {
12639               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12640               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12641               if (!Match) {
12642                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12643               }
12644               if (Match) {
12645                 Emit_118(Result, N, X86::PADDSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12646                 return;
12647               }
12648             }
12649           }
12650         }
12651       }
12652 
12653       // Pattern: (intrinsic_wo_chain:v16i8 244:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
12654       // Emits: (PADDUSBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
12655       // Pattern complexity = 19  cost = 1  size = 3
12656       if (CN0 == 244) {
12657         N1 = N.getOperand(1);
12658         N2 = N.getOperand(2);
12659         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12660             InFlightSet.count(N2.Val) == 0 &&
12661             N2.hasOneUse() &&
12662             !CodeGenMap.count(N2.getValue(0))) {
12663           N20 = N2.getOperand(0);
12664           if (N20.getOpcode() == ISD::LOAD &&
12665               InFlightSet.count(N20.Val) == 0 &&
12666               N20.hasOneUse() &&
12667               !CodeGenMap.count(N20.getValue(0))) {
12668             Chain20 = N20.getOperand(0);
12669             N201 = N20.getOperand(1);
12670             if (N20.Val->getValueType(0) == MVT::v2i64) {
12671               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12672               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12673               if (!Match) {
12674                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12675               }
12676               if (Match) {
12677                 Emit_118(Result, N, X86::PADDUSBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12678                 return;
12679               }
12680             }
12681           }
12682         }
12683       }
12684 
12685       // Pattern: (intrinsic_wo_chain:v8i16 245:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12686       // Emits: (PADDUSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12687       // Pattern complexity = 19  cost = 1  size = 3
12688       if (CN0 == 245) {
12689         N1 = N.getOperand(1);
12690         N2 = N.getOperand(2);
12691         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12692             InFlightSet.count(N2.Val) == 0 &&
12693             N2.hasOneUse() &&
12694             !CodeGenMap.count(N2.getValue(0))) {
12695           N20 = N2.getOperand(0);
12696           if (N20.getOpcode() == ISD::LOAD &&
12697               InFlightSet.count(N20.Val) == 0 &&
12698               N20.hasOneUse() &&
12699               !CodeGenMap.count(N20.getValue(0))) {
12700             Chain20 = N20.getOperand(0);
12701             N201 = N20.getOperand(1);
12702             if (N20.Val->getValueType(0) == MVT::v2i64) {
12703               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12704               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12705               if (!Match) {
12706                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12707               }
12708               if (Match) {
12709                 Emit_118(Result, N, X86::PADDUSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12710                 return;
12711               }
12712             }
12713           }
12714         }
12715       }
12716 
12717       // Pattern: (intrinsic_wo_chain:v16i8 274:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
12718       // Emits: (PSUBSBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
12719       // Pattern complexity = 19  cost = 1  size = 3
12720       if (CN0 == 274) {
12721         N1 = N.getOperand(1);
12722         N2 = N.getOperand(2);
12723         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12724             InFlightSet.count(N2.Val) == 0 &&
12725             N2.hasOneUse() &&
12726             !CodeGenMap.count(N2.getValue(0))) {
12727           N20 = N2.getOperand(0);
12728           if (N20.getOpcode() == ISD::LOAD &&
12729               InFlightSet.count(N20.Val) == 0 &&
12730               N20.hasOneUse() &&
12731               !CodeGenMap.count(N20.getValue(0))) {
12732             Chain20 = N20.getOperand(0);
12733             N201 = N20.getOperand(1);
12734             if (N20.Val->getValueType(0) == MVT::v2i64) {
12735               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12736               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12737               if (!Match) {
12738                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12739               }
12740               if (Match) {
12741                 Emit_118(Result, N, X86::PSUBSBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12742                 return;
12743               }
12744             }
12745           }
12746         }
12747       }
12748 
12749       // Pattern: (intrinsic_wo_chain:v8i16 275:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12750       // Emits: (PSUBSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12751       // Pattern complexity = 19  cost = 1  size = 3
12752       if (CN0 == 275) {
12753         N1 = N.getOperand(1);
12754         N2 = N.getOperand(2);
12755         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12756             InFlightSet.count(N2.Val) == 0 &&
12757             N2.hasOneUse() &&
12758             !CodeGenMap.count(N2.getValue(0))) {
12759           N20 = N2.getOperand(0);
12760           if (N20.getOpcode() == ISD::LOAD &&
12761               InFlightSet.count(N20.Val) == 0 &&
12762               N20.hasOneUse() &&
12763               !CodeGenMap.count(N20.getValue(0))) {
12764             Chain20 = N20.getOperand(0);
12765             N201 = N20.getOperand(1);
12766             if (N20.Val->getValueType(0) == MVT::v2i64) {
12767               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12768               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12769               if (!Match) {
12770                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12771               }
12772               if (Match) {
12773                 Emit_118(Result, N, X86::PSUBSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12774                 return;
12775               }
12776             }
12777           }
12778         }
12779       }
12780 
12781       // Pattern: (intrinsic_wo_chain:v16i8 276:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
12782       // Emits: (PSUBUSBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
12783       // Pattern complexity = 19  cost = 1  size = 3
12784       if (CN0 == 276) {
12785         N1 = N.getOperand(1);
12786         N2 = N.getOperand(2);
12787         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12788             InFlightSet.count(N2.Val) == 0 &&
12789             N2.hasOneUse() &&
12790             !CodeGenMap.count(N2.getValue(0))) {
12791           N20 = N2.getOperand(0);
12792           if (N20.getOpcode() == ISD::LOAD &&
12793               InFlightSet.count(N20.Val) == 0 &&
12794               N20.hasOneUse() &&
12795               !CodeGenMap.count(N20.getValue(0))) {
12796             Chain20 = N20.getOperand(0);
12797             N201 = N20.getOperand(1);
12798             if (N20.Val->getValueType(0) == MVT::v2i64) {
12799               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12800               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12801               if (!Match) {
12802                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12803               }
12804               if (Match) {
12805                 Emit_118(Result, N, X86::PSUBUSBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12806                 return;
12807               }
12808             }
12809           }
12810         }
12811       }
12812 
12813       // Pattern: (intrinsic_wo_chain:v8i16 277:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12814       // Emits: (PSUBUSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12815       // Pattern complexity = 19  cost = 1  size = 3
12816       if (CN0 == 277) {
12817         N1 = N.getOperand(1);
12818         N2 = N.getOperand(2);
12819         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12820             InFlightSet.count(N2.Val) == 0 &&
12821             N2.hasOneUse() &&
12822             !CodeGenMap.count(N2.getValue(0))) {
12823           N20 = N2.getOperand(0);
12824           if (N20.getOpcode() == ISD::LOAD &&
12825               InFlightSet.count(N20.Val) == 0 &&
12826               N20.hasOneUse() &&
12827               !CodeGenMap.count(N20.getValue(0))) {
12828             Chain20 = N20.getOperand(0);
12829             N201 = N20.getOperand(1);
12830             if (N20.Val->getValueType(0) == MVT::v2i64) {
12831               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12832               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12833               if (!Match) {
12834                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12835               }
12836               if (Match) {
12837                 Emit_118(Result, N, X86::PSUBUSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12838                 return;
12839               }
12840             }
12841           }
12842         }
12843       }
12844 
12845       // Pattern: (intrinsic_wo_chain:v8i16 261:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12846       // Emits: (PMULHUWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12847       // Pattern complexity = 19  cost = 1  size = 3
12848       if (CN0 == 261) {
12849         N1 = N.getOperand(1);
12850         N2 = N.getOperand(2);
12851         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12852             InFlightSet.count(N2.Val) == 0 &&
12853             N2.hasOneUse() &&
12854             !CodeGenMap.count(N2.getValue(0))) {
12855           N20 = N2.getOperand(0);
12856           if (N20.getOpcode() == ISD::LOAD &&
12857               InFlightSet.count(N20.Val) == 0 &&
12858               N20.hasOneUse() &&
12859               !CodeGenMap.count(N20.getValue(0))) {
12860             Chain20 = N20.getOperand(0);
12861             N201 = N20.getOperand(1);
12862             if (N20.Val->getValueType(0) == MVT::v2i64) {
12863               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12864               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12865               if (!Match) {
12866                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12867               }
12868               if (Match) {
12869                 Emit_118(Result, N, X86::PMULHUWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12870                 return;
12871               }
12872             }
12873           }
12874         }
12875       }
12876 
12877       // Pattern: (intrinsic_wo_chain:v8i16 260:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12878       // Emits: (PMULHWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
12879       // Pattern complexity = 19  cost = 1  size = 3
12880       if (CN0 == 260) {
12881         N1 = N.getOperand(1);
12882         N2 = N.getOperand(2);
12883         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12884             InFlightSet.count(N2.Val) == 0 &&
12885             N2.hasOneUse() &&
12886             !CodeGenMap.count(N2.getValue(0))) {
12887           N20 = N2.getOperand(0);
12888           if (N20.getOpcode() == ISD::LOAD &&
12889               InFlightSet.count(N20.Val) == 0 &&
12890               N20.hasOneUse() &&
12891               !CodeGenMap.count(N20.getValue(0))) {
12892             Chain20 = N20.getOperand(0);
12893             N201 = N20.getOperand(1);
12894             if (N20.Val->getValueType(0) == MVT::v2i64) {
12895               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12896               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12897               if (!Match) {
12898                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12899               }
12900               if (Match) {
12901                 Emit_118(Result, N, X86::PMULHWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12902                 return;
12903               }
12904             }
12905           }
12906         }
12907       }
12908 
12909       // Pattern: (intrinsic_wo_chain:v2i64 262:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
12910       // Emits: (PMULUDQrm:v2i64 VR128:v4i32:$src1, addr:iPTR:$src2)
12911       // Pattern complexity = 19  cost = 1  size = 3
12912       if (CN0 == 262) {
12913         N1 = N.getOperand(1);
12914         N2 = N.getOperand(2);
12915         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12916             InFlightSet.count(N2.Val) == 0 &&
12917             N2.hasOneUse() &&
12918             !CodeGenMap.count(N2.getValue(0))) {
12919           N20 = N2.getOperand(0);
12920           if (N20.getOpcode() == ISD::LOAD &&
12921               InFlightSet.count(N20.Val) == 0 &&
12922               N20.hasOneUse() &&
12923               !CodeGenMap.count(N20.getValue(0))) {
12924             Chain20 = N20.getOperand(0);
12925             N201 = N20.getOperand(1);
12926             if (N20.Val->getValueType(0) == MVT::v2i64) {
12927               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12928               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12929               if (!Match) {
12930                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12931               }
12932               if (Match) {
12933                 Emit_118(Result, N, X86::PMULUDQrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12934                 return;
12935               }
12936             }
12937           }
12938         }
12939       }
12940 
12941       // Pattern: (intrinsic_wo_chain:v4i32 254:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
12942       // Emits: (PMADDWDrm:v4i32 VR128:v8i16:$src1, addr:iPTR:$src2)
12943       // Pattern complexity = 19  cost = 1  size = 3
12944       if (CN0 == 254) {
12945         N1 = N.getOperand(1);
12946         N2 = N.getOperand(2);
12947         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12948             InFlightSet.count(N2.Val) == 0 &&
12949             N2.hasOneUse() &&
12950             !CodeGenMap.count(N2.getValue(0))) {
12951           N20 = N2.getOperand(0);
12952           if (N20.getOpcode() == ISD::LOAD &&
12953               InFlightSet.count(N20.Val) == 0 &&
12954               N20.hasOneUse() &&
12955               !CodeGenMap.count(N20.getValue(0))) {
12956             Chain20 = N20.getOperand(0);
12957             N201 = N20.getOperand(1);
12958             if (N20.Val->getValueType(0) == MVT::v2i64) {
12959               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12960               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12961               if (!Match) {
12962                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12963               }
12964               if (Match) {
12965                 Emit_118(Result, N, X86::PMADDWDrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12966                 return;
12967               }
12968             }
12969           }
12970         }
12971       }
12972 
12973       // Pattern: (intrinsic_wo_chain:v16i8 246:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
12974       // Emits: (PAVGBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
12975       // Pattern complexity = 19  cost = 1  size = 3
12976       if (CN0 == 246) {
12977         N1 = N.getOperand(1);
12978         N2 = N.getOperand(2);
12979         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12980             InFlightSet.count(N2.Val) == 0 &&
12981             N2.hasOneUse() &&
12982             !CodeGenMap.count(N2.getValue(0))) {
12983           N20 = N2.getOperand(0);
12984           if (N20.getOpcode() == ISD::LOAD &&
12985               InFlightSet.count(N20.Val) == 0 &&
12986               N20.hasOneUse() &&
12987               !CodeGenMap.count(N20.getValue(0))) {
12988             Chain20 = N20.getOperand(0);
12989             N201 = N20.getOperand(1);
12990             if (N20.Val->getValueType(0) == MVT::v2i64) {
12991               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
12992               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
12993               if (!Match) {
12994                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
12995               }
12996               if (Match) {
12997                 Emit_118(Result, N, X86::PAVGBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
12998                 return;
12999               }
13000             }
13001           }
13002         }
13003       }
13004 
13005       // Pattern: (intrinsic_wo_chain:v8i16 247:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13006       // Emits: (PAVGWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13007       // Pattern complexity = 19  cost = 1  size = 3
13008       if (CN0 == 247) {
13009         N1 = N.getOperand(1);
13010         N2 = N.getOperand(2);
13011         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13012             InFlightSet.count(N2.Val) == 0 &&
13013             N2.hasOneUse() &&
13014             !CodeGenMap.count(N2.getValue(0))) {
13015           N20 = N2.getOperand(0);
13016           if (N20.getOpcode() == ISD::LOAD &&
13017               InFlightSet.count(N20.Val) == 0 &&
13018               N20.hasOneUse() &&
13019               !CodeGenMap.count(N20.getValue(0))) {
13020             Chain20 = N20.getOperand(0);
13021             N201 = N20.getOperand(1);
13022             if (N20.Val->getValueType(0) == MVT::v2i64) {
13023               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13024               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13025               if (!Match) {
13026                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13027               }
13028               if (Match) {
13029                 Emit_118(Result, N, X86::PAVGWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13030                 return;
13031               }
13032             }
13033           }
13034         }
13035       }
13036 
13037       // Pattern: (intrinsic_wo_chain:v16i8 256:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
13038       // Emits: (PMAXUBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
13039       // Pattern complexity = 19  cost = 1  size = 3
13040       if (CN0 == 256) {
13041         N1 = N.getOperand(1);
13042         N2 = N.getOperand(2);
13043         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13044             InFlightSet.count(N2.Val) == 0 &&
13045             N2.hasOneUse() &&
13046             !CodeGenMap.count(N2.getValue(0))) {
13047           N20 = N2.getOperand(0);
13048           if (N20.getOpcode() == ISD::LOAD &&
13049               InFlightSet.count(N20.Val) == 0 &&
13050               N20.hasOneUse() &&
13051               !CodeGenMap.count(N20.getValue(0))) {
13052             Chain20 = N20.getOperand(0);
13053             N201 = N20.getOperand(1);
13054             if (N20.Val->getValueType(0) == MVT::v2i64) {
13055               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13056               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13057               if (!Match) {
13058                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13059               }
13060               if (Match) {
13061                 Emit_118(Result, N, X86::PMAXUBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13062                 return;
13063               }
13064             }
13065           }
13066         }
13067       }
13068 
13069       // Pattern: (intrinsic_wo_chain:v8i16 255:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13070       // Emits: (PMAXSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13071       // Pattern complexity = 19  cost = 1  size = 3
13072       if (CN0 == 255) {
13073         N1 = N.getOperand(1);
13074         N2 = N.getOperand(2);
13075         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13076             InFlightSet.count(N2.Val) == 0 &&
13077             N2.hasOneUse() &&
13078             !CodeGenMap.count(N2.getValue(0))) {
13079           N20 = N2.getOperand(0);
13080           if (N20.getOpcode() == ISD::LOAD &&
13081               InFlightSet.count(N20.Val) == 0 &&
13082               N20.hasOneUse() &&
13083               !CodeGenMap.count(N20.getValue(0))) {
13084             Chain20 = N20.getOperand(0);
13085             N201 = N20.getOperand(1);
13086             if (N20.Val->getValueType(0) == MVT::v2i64) {
13087               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13088               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13089               if (!Match) {
13090                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13091               }
13092               if (Match) {
13093                 Emit_118(Result, N, X86::PMAXSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13094                 return;
13095               }
13096             }
13097           }
13098         }
13099       }
13100 
13101       // Pattern: (intrinsic_wo_chain:v16i8 258:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
13102       // Emits: (PMINUBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
13103       // Pattern complexity = 19  cost = 1  size = 3
13104       if (CN0 == 258) {
13105         N1 = N.getOperand(1);
13106         N2 = N.getOperand(2);
13107         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13108             InFlightSet.count(N2.Val) == 0 &&
13109             N2.hasOneUse() &&
13110             !CodeGenMap.count(N2.getValue(0))) {
13111           N20 = N2.getOperand(0);
13112           if (N20.getOpcode() == ISD::LOAD &&
13113               InFlightSet.count(N20.Val) == 0 &&
13114               N20.hasOneUse() &&
13115               !CodeGenMap.count(N20.getValue(0))) {
13116             Chain20 = N20.getOperand(0);
13117             N201 = N20.getOperand(1);
13118             if (N20.Val->getValueType(0) == MVT::v2i64) {
13119               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13120               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13121               if (!Match) {
13122                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13123               }
13124               if (Match) {
13125                 Emit_118(Result, N, X86::PMINUBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13126                 return;
13127               }
13128             }
13129           }
13130         }
13131       }
13132 
13133       // Pattern: (intrinsic_wo_chain:v8i16 257:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13134       // Emits: (PMINSWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13135       // Pattern complexity = 19  cost = 1  size = 3
13136       if (CN0 == 257) {
13137         N1 = N.getOperand(1);
13138         N2 = N.getOperand(2);
13139         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13140             InFlightSet.count(N2.Val) == 0 &&
13141             N2.hasOneUse() &&
13142             !CodeGenMap.count(N2.getValue(0))) {
13143           N20 = N2.getOperand(0);
13144           if (N20.getOpcode() == ISD::LOAD &&
13145               InFlightSet.count(N20.Val) == 0 &&
13146               N20.hasOneUse() &&
13147               !CodeGenMap.count(N20.getValue(0))) {
13148             Chain20 = N20.getOperand(0);
13149             N201 = N20.getOperand(1);
13150             if (N20.Val->getValueType(0) == MVT::v2i64) {
13151               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13152               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13153               if (!Match) {
13154                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13155               }
13156               if (Match) {
13157                 Emit_118(Result, N, X86::PMINSWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13158                 return;
13159               }
13160             }
13161           }
13162         }
13163       }
13164 
13165       // Pattern: (intrinsic_wo_chain:v2i64 263:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
13166       // Emits: (PSADBWrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
13167       // Pattern complexity = 19  cost = 1  size = 3
13168       if (CN0 == 263) {
13169         N1 = N.getOperand(1);
13170         N2 = N.getOperand(2);
13171         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13172             InFlightSet.count(N2.Val) == 0 &&
13173             N2.hasOneUse() &&
13174             !CodeGenMap.count(N2.getValue(0))) {
13175           N20 = N2.getOperand(0);
13176           if (N20.getOpcode() == ISD::LOAD &&
13177               InFlightSet.count(N20.Val) == 0 &&
13178               N20.hasOneUse() &&
13179               !CodeGenMap.count(N20.getValue(0))) {
13180             Chain20 = N20.getOperand(0);
13181             N201 = N20.getOperand(1);
13182             if (N20.Val->getValueType(0) == MVT::v2i64) {
13183               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13184               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13185               if (!Match) {
13186                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13187               }
13188               if (Match) {
13189                 Emit_118(Result, N, X86::PSADBWrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13190                 return;
13191               }
13192             }
13193           }
13194         }
13195       }
13196 
13197       // Pattern: (intrinsic_wo_chain:v8i16 267:iPTR, VR128:v8i16:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13198       // Emits: (PSLLWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13199       // Pattern complexity = 19  cost = 1  size = 3
13200       if (CN0 == 267) {
13201         N1 = N.getOperand(1);
13202         N2 = N.getOperand(2);
13203         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13204             InFlightSet.count(N2.Val) == 0 &&
13205             N2.hasOneUse() &&
13206             !CodeGenMap.count(N2.getValue(0))) {
13207           N20 = N2.getOperand(0);
13208           if (N20.getOpcode() == ISD::LOAD &&
13209               InFlightSet.count(N20.Val) == 0 &&
13210               N20.hasOneUse() &&
13211               !CodeGenMap.count(N20.getValue(0))) {
13212             Chain20 = N20.getOperand(0);
13213             N201 = N20.getOperand(1);
13214             if (N20.Val->getValueType(0) == MVT::v2i64) {
13215               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13216               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13217               if (!Match) {
13218                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13219               }
13220               if (Match) {
13221                 Emit_118(Result, N, X86::PSLLWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13222                 return;
13223               }
13224             }
13225           }
13226         }
13227       }
13228 
13229       // Pattern: (intrinsic_wo_chain:v4i32 264:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13230       // Emits: (PSLLDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13231       // Pattern complexity = 19  cost = 1  size = 3
13232       if (CN0 == 264) {
13233         N1 = N.getOperand(1);
13234         N2 = N.getOperand(2);
13235         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13236             InFlightSet.count(N2.Val) == 0 &&
13237             N2.hasOneUse() &&
13238             !CodeGenMap.count(N2.getValue(0))) {
13239           N20 = N2.getOperand(0);
13240           if (N20.getOpcode() == ISD::LOAD &&
13241               InFlightSet.count(N20.Val) == 0 &&
13242               N20.hasOneUse() &&
13243               !CodeGenMap.count(N20.getValue(0))) {
13244             Chain20 = N20.getOperand(0);
13245             N201 = N20.getOperand(1);
13246             if (N20.Val->getValueType(0) == MVT::v2i64) {
13247               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13248               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13249               if (!Match) {
13250                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13251               }
13252               if (Match) {
13253                 Emit_118(Result, N, X86::PSLLDrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13254                 return;
13255               }
13256             }
13257           }
13258         }
13259       }
13260 
13261       // Pattern: (intrinsic_wo_chain:v2i64 266:iPTR, VR128:v2i64:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13262       // Emits: (PSLLQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
13263       // Pattern complexity = 19  cost = 1  size = 3
13264       if (CN0 == 266) {
13265         N1 = N.getOperand(1);
13266         N2 = N.getOperand(2);
13267         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13268             InFlightSet.count(N2.Val) == 0 &&
13269             N2.hasOneUse() &&
13270             !CodeGenMap.count(N2.getValue(0))) {
13271           N20 = N2.getOperand(0);
13272           if (N20.getOpcode() == ISD::LOAD &&
13273               InFlightSet.count(N20.Val) == 0 &&
13274               N20.hasOneUse() &&
13275               !CodeGenMap.count(N20.getValue(0))) {
13276             Chain20 = N20.getOperand(0);
13277             N201 = N20.getOperand(1);
13278             if (N20.Val->getValueType(0) == MVT::v2i64) {
13279               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13280               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13281               if (!Match) {
13282                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13283               }
13284               if (Match) {
13285                 Emit_118(Result, N, X86::PSLLQrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13286                 return;
13287               }
13288             }
13289           }
13290         }
13291       }
13292 
13293       // Pattern: (intrinsic_wo_chain:v8i16 273:iPTR, VR128:v8i16:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13294       // Emits: (PSRLWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13295       // Pattern complexity = 19  cost = 1  size = 3
13296       if (CN0 == 273) {
13297         N1 = N.getOperand(1);
13298         N2 = N.getOperand(2);
13299         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13300             InFlightSet.count(N2.Val) == 0 &&
13301             N2.hasOneUse() &&
13302             !CodeGenMap.count(N2.getValue(0))) {
13303           N20 = N2.getOperand(0);
13304           if (N20.getOpcode() == ISD::LOAD &&
13305               InFlightSet.count(N20.Val) == 0 &&
13306               N20.hasOneUse() &&
13307               !CodeGenMap.count(N20.getValue(0))) {
13308             Chain20 = N20.getOperand(0);
13309             N201 = N20.getOperand(1);
13310             if (N20.Val->getValueType(0) == MVT::v2i64) {
13311               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13312               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13313               if (!Match) {
13314                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13315               }
13316               if (Match) {
13317                 Emit_118(Result, N, X86::PSRLWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13318                 return;
13319               }
13320             }
13321           }
13322         }
13323       }
13324 
13325       // Pattern: (intrinsic_wo_chain:v4i32 270:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13326       // Emits: (PSRLDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13327       // Pattern complexity = 19  cost = 1  size = 3
13328       if (CN0 == 270) {
13329         N1 = N.getOperand(1);
13330         N2 = N.getOperand(2);
13331         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13332             InFlightSet.count(N2.Val) == 0 &&
13333             N2.hasOneUse() &&
13334             !CodeGenMap.count(N2.getValue(0))) {
13335           N20 = N2.getOperand(0);
13336           if (N20.getOpcode() == ISD::LOAD &&
13337               InFlightSet.count(N20.Val) == 0 &&
13338               N20.hasOneUse() &&
13339               !CodeGenMap.count(N20.getValue(0))) {
13340             Chain20 = N20.getOperand(0);
13341             N201 = N20.getOperand(1);
13342             if (N20.Val->getValueType(0) == MVT::v2i64) {
13343               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13344               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13345               if (!Match) {
13346                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13347               }
13348               if (Match) {
13349                 Emit_118(Result, N, X86::PSRLDrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13350                 return;
13351               }
13352             }
13353           }
13354         }
13355       }
13356 
13357       // Pattern: (intrinsic_wo_chain:v2i64 272:iPTR, VR128:v2i64:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13358       // Emits: (PSRLQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
13359       // Pattern complexity = 19  cost = 1  size = 3
13360       if (CN0 == 272) {
13361         N1 = N.getOperand(1);
13362         N2 = N.getOperand(2);
13363         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13364             InFlightSet.count(N2.Val) == 0 &&
13365             N2.hasOneUse() &&
13366             !CodeGenMap.count(N2.getValue(0))) {
13367           N20 = N2.getOperand(0);
13368           if (N20.getOpcode() == ISD::LOAD &&
13369               InFlightSet.count(N20.Val) == 0 &&
13370               N20.hasOneUse() &&
13371               !CodeGenMap.count(N20.getValue(0))) {
13372             Chain20 = N20.getOperand(0);
13373             N201 = N20.getOperand(1);
13374             if (N20.Val->getValueType(0) == MVT::v2i64) {
13375               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13376               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13377               if (!Match) {
13378                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13379               }
13380               if (Match) {
13381                 Emit_118(Result, N, X86::PSRLQrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13382                 return;
13383               }
13384             }
13385           }
13386         }
13387       }
13388 
13389       // Pattern: (intrinsic_wo_chain:v8i16 269:iPTR, VR128:v8i16:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13390       // Emits: (PSRAWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13391       // Pattern complexity = 19  cost = 1  size = 3
13392       if (CN0 == 269) {
13393         N1 = N.getOperand(1);
13394         N2 = N.getOperand(2);
13395         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13396             InFlightSet.count(N2.Val) == 0 &&
13397             N2.hasOneUse() &&
13398             !CodeGenMap.count(N2.getValue(0))) {
13399           N20 = N2.getOperand(0);
13400           if (N20.getOpcode() == ISD::LOAD &&
13401               InFlightSet.count(N20.Val) == 0 &&
13402               N20.hasOneUse() &&
13403               !CodeGenMap.count(N20.getValue(0))) {
13404             Chain20 = N20.getOperand(0);
13405             N201 = N20.getOperand(1);
13406             if (N20.Val->getValueType(0) == MVT::v2i64) {
13407               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13408               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13409               if (!Match) {
13410                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13411               }
13412               if (Match) {
13413                 Emit_118(Result, N, X86::PSRAWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13414                 return;
13415               }
13416             }
13417           }
13418         }
13419       }
13420 
13421       // Pattern: (intrinsic_wo_chain:v4i32 268:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13422       // Emits: (PSRADrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13423       // Pattern complexity = 19  cost = 1  size = 3
13424       if (CN0 == 268) {
13425         N1 = N.getOperand(1);
13426         N2 = N.getOperand(2);
13427         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13428             InFlightSet.count(N2.Val) == 0 &&
13429             N2.hasOneUse() &&
13430             !CodeGenMap.count(N2.getValue(0))) {
13431           N20 = N2.getOperand(0);
13432           if (N20.getOpcode() == ISD::LOAD &&
13433               InFlightSet.count(N20.Val) == 0 &&
13434               N20.hasOneUse() &&
13435               !CodeGenMap.count(N20.getValue(0))) {
13436             Chain20 = N20.getOperand(0);
13437             N201 = N20.getOperand(1);
13438             if (N20.Val->getValueType(0) == MVT::v2i64) {
13439               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13440               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13441               if (!Match) {
13442                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13443               }
13444               if (Match) {
13445                 Emit_118(Result, N, X86::PSRADrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13446                 return;
13447               }
13448             }
13449           }
13450         }
13451       }
13452 
13453       // Pattern: (intrinsic_wo_chain:v16i8 248:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
13454       // Emits: (PCMPEQBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
13455       // Pattern complexity = 19  cost = 1  size = 3
13456       if (CN0 == 248) {
13457         N1 = N.getOperand(1);
13458         N2 = N.getOperand(2);
13459         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13460             InFlightSet.count(N2.Val) == 0 &&
13461             N2.hasOneUse() &&
13462             !CodeGenMap.count(N2.getValue(0))) {
13463           N20 = N2.getOperand(0);
13464           if (N20.getOpcode() == ISD::LOAD &&
13465               InFlightSet.count(N20.Val) == 0 &&
13466               N20.hasOneUse() &&
13467               !CodeGenMap.count(N20.getValue(0))) {
13468             Chain20 = N20.getOperand(0);
13469             N201 = N20.getOperand(1);
13470             if (N20.Val->getValueType(0) == MVT::v2i64) {
13471               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13472               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13473               if (!Match) {
13474                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13475               }
13476               if (Match) {
13477                 Emit_118(Result, N, X86::PCMPEQBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13478                 return;
13479               }
13480             }
13481           }
13482         }
13483       }
13484 
13485       // Pattern: (intrinsic_wo_chain:v8i16 250:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13486       // Emits: (PCMPEQWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13487       // Pattern complexity = 19  cost = 1  size = 3
13488       if (CN0 == 250) {
13489         N1 = N.getOperand(1);
13490         N2 = N.getOperand(2);
13491         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13492             InFlightSet.count(N2.Val) == 0 &&
13493             N2.hasOneUse() &&
13494             !CodeGenMap.count(N2.getValue(0))) {
13495           N20 = N2.getOperand(0);
13496           if (N20.getOpcode() == ISD::LOAD &&
13497               InFlightSet.count(N20.Val) == 0 &&
13498               N20.hasOneUse() &&
13499               !CodeGenMap.count(N20.getValue(0))) {
13500             Chain20 = N20.getOperand(0);
13501             N201 = N20.getOperand(1);
13502             if (N20.Val->getValueType(0) == MVT::v2i64) {
13503               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13504               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13505               if (!Match) {
13506                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13507               }
13508               if (Match) {
13509                 Emit_118(Result, N, X86::PCMPEQWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13510                 return;
13511               }
13512             }
13513           }
13514         }
13515       }
13516 
13517       // Pattern: (intrinsic_wo_chain:v4i32 249:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13518       // Emits: (PCMPEQDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13519       // Pattern complexity = 19  cost = 1  size = 3
13520       if (CN0 == 249) {
13521         N1 = N.getOperand(1);
13522         N2 = N.getOperand(2);
13523         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13524             InFlightSet.count(N2.Val) == 0 &&
13525             N2.hasOneUse() &&
13526             !CodeGenMap.count(N2.getValue(0))) {
13527           N20 = N2.getOperand(0);
13528           if (N20.getOpcode() == ISD::LOAD &&
13529               InFlightSet.count(N20.Val) == 0 &&
13530               N20.hasOneUse() &&
13531               !CodeGenMap.count(N20.getValue(0))) {
13532             Chain20 = N20.getOperand(0);
13533             N201 = N20.getOperand(1);
13534             if (N20.Val->getValueType(0) == MVT::v2i64) {
13535               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13536               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13537               if (!Match) {
13538                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13539               }
13540               if (Match) {
13541                 Emit_118(Result, N, X86::PCMPEQDrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13542                 return;
13543               }
13544             }
13545           }
13546         }
13547       }
13548 
13549       // Pattern: (intrinsic_wo_chain:v16i8 251:iPTR, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
13550       // Emits: (PCMPGTBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
13551       // Pattern complexity = 19  cost = 1  size = 3
13552       if (CN0 == 251) {
13553         N1 = N.getOperand(1);
13554         N2 = N.getOperand(2);
13555         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13556             InFlightSet.count(N2.Val) == 0 &&
13557             N2.hasOneUse() &&
13558             !CodeGenMap.count(N2.getValue(0))) {
13559           N20 = N2.getOperand(0);
13560           if (N20.getOpcode() == ISD::LOAD &&
13561               InFlightSet.count(N20.Val) == 0 &&
13562               N20.hasOneUse() &&
13563               !CodeGenMap.count(N20.getValue(0))) {
13564             Chain20 = N20.getOperand(0);
13565             N201 = N20.getOperand(1);
13566             if (N20.Val->getValueType(0) == MVT::v2i64) {
13567               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13568               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13569               if (!Match) {
13570                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13571               }
13572               if (Match) {
13573                 Emit_118(Result, N, X86::PCMPGTBrm, MVT::v16i8, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13574                 return;
13575               }
13576             }
13577           }
13578         }
13579       }
13580 
13581       // Pattern: (intrinsic_wo_chain:v8i16 253:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13582       // Emits: (PCMPGTWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13583       // Pattern complexity = 19  cost = 1  size = 3
13584       if (CN0 == 253) {
13585         N1 = N.getOperand(1);
13586         N2 = N.getOperand(2);
13587         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13588             InFlightSet.count(N2.Val) == 0 &&
13589             N2.hasOneUse() &&
13590             !CodeGenMap.count(N2.getValue(0))) {
13591           N20 = N2.getOperand(0);
13592           if (N20.getOpcode() == ISD::LOAD &&
13593               InFlightSet.count(N20.Val) == 0 &&
13594               N20.hasOneUse() &&
13595               !CodeGenMap.count(N20.getValue(0))) {
13596             Chain20 = N20.getOperand(0);
13597             N201 = N20.getOperand(1);
13598             if (N20.Val->getValueType(0) == MVT::v2i64) {
13599               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13600               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13601               if (!Match) {
13602                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13603               }
13604               if (Match) {
13605                 Emit_118(Result, N, X86::PCMPGTWrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13606                 return;
13607               }
13608             }
13609           }
13610         }
13611       }
13612 
13613       // Pattern: (intrinsic_wo_chain:v4i32 252:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13614       // Emits: (PCMPGTDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13615       // Pattern complexity = 19  cost = 1  size = 3
13616       if (CN0 == 252) {
13617         N1 = N.getOperand(1);
13618         N2 = N.getOperand(2);
13619         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13620             InFlightSet.count(N2.Val) == 0 &&
13621             N2.hasOneUse() &&
13622             !CodeGenMap.count(N2.getValue(0))) {
13623           N20 = N2.getOperand(0);
13624           if (N20.getOpcode() == ISD::LOAD &&
13625               InFlightSet.count(N20.Val) == 0 &&
13626               N20.hasOneUse() &&
13627               !CodeGenMap.count(N20.getValue(0))) {
13628             Chain20 = N20.getOperand(0);
13629             N201 = N20.getOperand(1);
13630             if (N20.Val->getValueType(0) == MVT::v2i64) {
13631               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13632               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13633               if (!Match) {
13634                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13635               }
13636               if (Match) {
13637                 Emit_118(Result, N, X86::PCMPGTDrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13638                 return;
13639               }
13640             }
13641           }
13642         }
13643       }
13644 
13645       // Pattern: (intrinsic_wo_chain:v8i16 240:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2f64 addr:iPTR:$src2)))
13646       // Emits: (PACKSSWBrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13647       // Pattern complexity = 19  cost = 1  size = 3
13648       if (CN0 == 240) {
13649         N1 = N.getOperand(1);
13650         N2 = N.getOperand(2);
13651         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13652             InFlightSet.count(N2.Val) == 0 &&
13653             N2.hasOneUse() &&
13654             !CodeGenMap.count(N2.getValue(0))) {
13655           N20 = N2.getOperand(0);
13656           if (N20.getOpcode() == ISD::LOAD &&
13657               InFlightSet.count(N20.Val) == 0 &&
13658               N20.hasOneUse() &&
13659               !CodeGenMap.count(N20.getValue(0))) {
13660             Chain20 = N20.getOperand(0);
13661             N201 = N20.getOperand(1);
13662             if (N20.Val->getValueType(0) == MVT::v2f64) {
13663               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13664               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13665               if (!Match) {
13666                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13667               }
13668               if (Match) {
13669                 Emit_118(Result, N, X86::PACKSSWBrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13670                 return;
13671               }
13672             }
13673           }
13674         }
13675       }
13676 
13677       // Pattern: (intrinsic_wo_chain:v4i32 239:iPTR, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
13678       // Emits: (PACKSSDWrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
13679       // Pattern complexity = 19  cost = 1  size = 3
13680       if (CN0 == 239) {
13681         N1 = N.getOperand(1);
13682         N2 = N.getOperand(2);
13683         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13684             InFlightSet.count(N2.Val) == 0 &&
13685             N2.hasOneUse() &&
13686             !CodeGenMap.count(N2.getValue(0))) {
13687           N20 = N2.getOperand(0);
13688           if (N20.getOpcode() == ISD::LOAD &&
13689               InFlightSet.count(N20.Val) == 0 &&
13690               N20.hasOneUse() &&
13691               !CodeGenMap.count(N20.getValue(0))) {
13692             Chain20 = N20.getOperand(0);
13693             N201 = N20.getOperand(1);
13694             if (N20.Val->getValueType(0) == MVT::v2i64) {
13695               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13696               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13697               if (!Match) {
13698                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13699               }
13700               if (Match) {
13701                 Emit_118(Result, N, X86::PACKSSDWrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13702                 return;
13703               }
13704             }
13705           }
13706         }
13707       }
13708 
13709       // Pattern: (intrinsic_wo_chain:v8i16 241:iPTR, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
13710       // Emits: (PACKUSWBrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
13711       // Pattern complexity = 19  cost = 1  size = 3
13712       if (CN0 == 241) {
13713         N1 = N.getOperand(1);
13714         N2 = N.getOperand(2);
13715         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13716             InFlightSet.count(N2.Val) == 0 &&
13717             N2.hasOneUse() &&
13718             !CodeGenMap.count(N2.getValue(0))) {
13719           N20 = N2.getOperand(0);
13720           if (N20.getOpcode() == ISD::LOAD &&
13721               InFlightSet.count(N20.Val) == 0 &&
13722               N20.hasOneUse() &&
13723               !CodeGenMap.count(N20.getValue(0))) {
13724             Chain20 = N20.getOperand(0);
13725             N201 = N20.getOperand(1);
13726             if (N20.Val->getValueType(0) == MVT::v2i64) {
13727               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13728               bool Match = SelectAddr(N201, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13729               if (!Match) {
13730                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13731               }
13732               if (Match) {
13733                 Emit_118(Result, N, X86::PACKUSWBrm, MVT::v8i16, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain20, N0, N1, N2, N20, N201);
13734                 return;
13735               }
13736             }
13737           }
13738         }
13739       }
13740 
13741       // Pattern: (intrinsic_wo_chain:v2f64 284:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (imm:i32):$src3)
13742       // Emits: (SHUFPDrmi:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2, (imm:i8):$src3)
13743       // Pattern complexity = 19  cost = 1
13744       if (CN0 == 284) {
13745         N1 = N.getOperand(1);
13746         N2 = N.getOperand(2);
13747         if (N2.getOpcode() == ISD::LOAD &&
13748             InFlightSet.count(N2.Val) == 0 &&
13749             N2.hasOneUse() &&
13750             !CodeGenMap.count(N2.getValue(0))) {
13751           Chain2 = N2.getOperand(0);
13752           N21 = N2.getOperand(1);
13753           N3 = N.getOperand(3);
13754           if (N3.getOpcode() == ISD::Constant) {
13755             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13756             bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13757             if (!Match) {
13758               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13759             }
13760             if (Match) {
13761               Emit_117(Result, N, X86::SHUFPDrmi, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21, N3);
13762               return;
13763             }
13764           }
13765         }
13766       }
13767     }
13768   }
13769 
13770   // Pattern: (intrinsic_wo_chain:v4f32 308:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
13771   // Emits: (Int_ADDSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
13772   // Pattern complexity = 17  cost = 1  size = 3
13773   if ((Subtarget->hasSSE1())) {
13774     N0 = N.getOperand(0);
13775     if (isa<ConstantSDNode>(N0)) {
13776       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13777       if (CN0 == 308) {
13778         N1 = N.getOperand(1);
13779         N2 = N.getOperand(2);
13780         if (N2.getOpcode() == ISD::LOAD &&
13781             InFlightSet.count(N2.Val) == 0 &&
13782             N2.hasOneUse() &&
13783             !CodeGenMap.count(N2.getValue(0))) {
13784           Chain2 = N2.getOperand(0);
13785           N21 = N2.getOperand(1);
13786           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13787           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13788           if (!Match) {
13789             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13790           }
13791           if (Match) {
13792             Emit_119(Result, N, X86::Int_ADDSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13793             return;
13794           }
13795         }
13796       }
13797     }
13798   }
13799 
13800   // Pattern: (intrinsic_wo_chain:v2f64 197:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
13801   // Emits: (Int_ADDSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
13802   // Pattern complexity = 17  cost = 1  size = 3
13803   if ((Subtarget->hasSSE2())) {
13804     N0 = N.getOperand(0);
13805     if (isa<ConstantSDNode>(N0)) {
13806       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13807       if (CN0 == 197) {
13808         N1 = N.getOperand(1);
13809         N2 = N.getOperand(2);
13810         if (N2.getOpcode() == ISD::LOAD &&
13811             InFlightSet.count(N2.Val) == 0 &&
13812             N2.hasOneUse() &&
13813             !CodeGenMap.count(N2.getValue(0))) {
13814           Chain2 = N2.getOperand(0);
13815           N21 = N2.getOperand(1);
13816           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13817           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13818           if (!Match) {
13819             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13820           }
13821           if (Match) {
13822             Emit_119(Result, N, X86::Int_ADDSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13823             return;
13824           }
13825         }
13826       }
13827     }
13828   }
13829 
13830   // Pattern: (intrinsic_wo_chain:v4f32 329:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
13831   // Emits: (Int_MULSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
13832   // Pattern complexity = 17  cost = 1  size = 3
13833   if ((Subtarget->hasSSE1())) {
13834     N0 = N.getOperand(0);
13835     if (isa<ConstantSDNode>(N0)) {
13836       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13837       if (CN0 == 329) {
13838         N1 = N.getOperand(1);
13839         N2 = N.getOperand(2);
13840         if (N2.getOpcode() == ISD::LOAD &&
13841             InFlightSet.count(N2.Val) == 0 &&
13842             N2.hasOneUse() &&
13843             !CodeGenMap.count(N2.getValue(0))) {
13844           Chain2 = N2.getOperand(0);
13845           N21 = N2.getOperand(1);
13846           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13847           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13848           if (!Match) {
13849             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13850           }
13851           if (Match) {
13852             Emit_119(Result, N, X86::Int_MULSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13853             return;
13854           }
13855         }
13856       }
13857     }
13858   }
13859 
13860   // Pattern: (intrinsic_wo_chain:v2f64 238:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
13861   // Emits: (Int_MULSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
13862   // Pattern complexity = 17  cost = 1  size = 3
13863   if ((Subtarget->hasSSE2())) {
13864     N0 = N.getOperand(0);
13865     if (isa<ConstantSDNode>(N0)) {
13866       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13867       if (CN0 == 238) {
13868         N1 = N.getOperand(1);
13869         N2 = N.getOperand(2);
13870         if (N2.getOpcode() == ISD::LOAD &&
13871             InFlightSet.count(N2.Val) == 0 &&
13872             N2.hasOneUse() &&
13873             !CodeGenMap.count(N2.getValue(0))) {
13874           Chain2 = N2.getOperand(0);
13875           N21 = N2.getOperand(1);
13876           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13877           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13878           if (!Match) {
13879             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13880           }
13881           if (Match) {
13882             Emit_119(Result, N, X86::Int_MULSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13883             return;
13884           }
13885         }
13886       }
13887     }
13888   }
13889 
13890   // Pattern: (intrinsic_wo_chain:v4f32 320:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
13891   // Emits: (Int_DIVSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
13892   // Pattern complexity = 17  cost = 1  size = 3
13893   if ((Subtarget->hasSSE1())) {
13894     N0 = N.getOperand(0);
13895     if (isa<ConstantSDNode>(N0)) {
13896       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13897       if (CN0 == 320) {
13898         N1 = N.getOperand(1);
13899         N2 = N.getOperand(2);
13900         if (N2.getOpcode() == ISD::LOAD &&
13901             InFlightSet.count(N2.Val) == 0 &&
13902             N2.hasOneUse() &&
13903             !CodeGenMap.count(N2.getValue(0))) {
13904           Chain2 = N2.getOperand(0);
13905           N21 = N2.getOperand(1);
13906           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13907           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13908           if (!Match) {
13909             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13910           }
13911           if (Match) {
13912             Emit_119(Result, N, X86::Int_DIVSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13913             return;
13914           }
13915         }
13916       }
13917     }
13918   }
13919 
13920   // Pattern: (intrinsic_wo_chain:v2f64 220:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
13921   // Emits: (Int_DIVSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
13922   // Pattern complexity = 17  cost = 1  size = 3
13923   if ((Subtarget->hasSSE2())) {
13924     N0 = N.getOperand(0);
13925     if (isa<ConstantSDNode>(N0)) {
13926       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13927       if (CN0 == 220) {
13928         N1 = N.getOperand(1);
13929         N2 = N.getOperand(2);
13930         if (N2.getOpcode() == ISD::LOAD &&
13931             InFlightSet.count(N2.Val) == 0 &&
13932             N2.hasOneUse() &&
13933             !CodeGenMap.count(N2.getValue(0))) {
13934           Chain2 = N2.getOperand(0);
13935           N21 = N2.getOperand(1);
13936           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13937           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13938           if (!Match) {
13939             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13940           }
13941           if (Match) {
13942             Emit_119(Result, N, X86::Int_DIVSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13943             return;
13944           }
13945         }
13946       }
13947     }
13948   }
13949 
13950   // Pattern: (intrinsic_wo_chain:v4f32 339:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
13951   // Emits: (Int_SUBSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
13952   // Pattern complexity = 17  cost = 1  size = 3
13953   if ((Subtarget->hasSSE1())) {
13954     N0 = N.getOperand(0);
13955     if (isa<ConstantSDNode>(N0)) {
13956       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13957       if (CN0 == 339) {
13958         N1 = N.getOperand(1);
13959         N2 = N.getOperand(2);
13960         if (N2.getOpcode() == ISD::LOAD &&
13961             InFlightSet.count(N2.Val) == 0 &&
13962             N2.hasOneUse() &&
13963             !CodeGenMap.count(N2.getValue(0))) {
13964           Chain2 = N2.getOperand(0);
13965           N21 = N2.getOperand(1);
13966           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13967           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13968           if (!Match) {
13969             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
13970           }
13971           if (Match) {
13972             Emit_119(Result, N, X86::Int_SUBSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
13973             return;
13974           }
13975         }
13976       }
13977     }
13978   }
13979 
13980   // Pattern: (intrinsic_wo_chain:v2f64 290:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
13981   // Emits: (Int_SUBSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
13982   // Pattern complexity = 17  cost = 1  size = 3
13983   if ((Subtarget->hasSSE2())) {
13984     N0 = N.getOperand(0);
13985     if (isa<ConstantSDNode>(N0)) {
13986       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13987       if (CN0 == 290) {
13988         N1 = N.getOperand(1);
13989         N2 = N.getOperand(2);
13990         if (N2.getOpcode() == ISD::LOAD &&
13991             InFlightSet.count(N2.Val) == 0 &&
13992             N2.hasOneUse() &&
13993             !CodeGenMap.count(N2.getValue(0))) {
13994           Chain2 = N2.getOperand(0);
13995           N21 = N2.getOperand(1);
13996           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
13997           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
13998           if (!Match) {
13999             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14000           }
14001           if (Match) {
14002             Emit_119(Result, N, X86::Int_SUBSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14003             return;
14004           }
14005         }
14006       }
14007     }
14008   }
14009 
14010   // Pattern: (intrinsic_wo_chain:v4f32 336:iPTR, (load:v4f32 addr:iPTR:$src))
14011   // Emits: (Int_SQRTSSm:v4f32 addr:iPTR:$src)
14012   // Pattern complexity = 17  cost = 1  size = 3
14013   if ((Subtarget->hasSSE1())) {
14014     N0 = N.getOperand(0);
14015     if (isa<ConstantSDNode>(N0)) {
14016       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14017       if (CN0 == 336) {
14018         N1 = N.getOperand(1);
14019         if (N1.getOpcode() == ISD::LOAD &&
14020             InFlightSet.count(N1.Val) == 0 &&
14021             N1.hasOneUse() &&
14022             !CodeGenMap.count(N1.getValue(0))) {
14023           Chain1 = N1.getOperand(0);
14024           N11 = N1.getOperand(1);
14025           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14026           if (Match) {
14027             Emit_120(Result, N, X86::Int_SQRTSSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14028             return;
14029           }
14030         }
14031       }
14032     }
14033   }
14034 
14035   // Pattern: (intrinsic_wo_chain:v2f64 286:iPTR, (load:v2f64 addr:iPTR:$src))
14036   // Emits: (Int_SQRTSDm:v2f64 addr:iPTR:$src)
14037   // Pattern complexity = 17  cost = 1  size = 3
14038   if ((Subtarget->hasSSE2())) {
14039     N0 = N.getOperand(0);
14040     if (isa<ConstantSDNode>(N0)) {
14041       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14042       if (CN0 == 286) {
14043         N1 = N.getOperand(1);
14044         if (N1.getOpcode() == ISD::LOAD &&
14045             InFlightSet.count(N1.Val) == 0 &&
14046             N1.hasOneUse() &&
14047             !CodeGenMap.count(N1.getValue(0))) {
14048           Chain1 = N1.getOperand(0);
14049           N11 = N1.getOperand(1);
14050           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14051           if (Match) {
14052             Emit_120(Result, N, X86::Int_SQRTSDm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14053             return;
14054           }
14055         }
14056       }
14057     }
14058   }
14059   if ((Subtarget->hasSSE1())) {
14060     N0 = N.getOperand(0);
14061     if (isa<ConstantSDNode>(N0)) {
14062       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14063 
14064       // Pattern: (intrinsic_wo_chain:v4f32 333:iPTR, (load:v4f32 addr:iPTR:$src))
14065       // Emits: (Int_RSQRTSSm:v4f32 addr:iPTR:$src)
14066       // Pattern complexity = 17  cost = 1  size = 3
14067       if (CN0 == 333) {
14068         N1 = N.getOperand(1);
14069         if (N1.getOpcode() == ISD::LOAD &&
14070             InFlightSet.count(N1.Val) == 0 &&
14071             N1.hasOneUse() &&
14072             !CodeGenMap.count(N1.getValue(0))) {
14073           Chain1 = N1.getOperand(0);
14074           N11 = N1.getOperand(1);
14075           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14076           if (Match) {
14077             Emit_120(Result, N, X86::Int_RSQRTSSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14078             return;
14079           }
14080         }
14081       }
14082 
14083       // Pattern: (intrinsic_wo_chain:v4f32 331:iPTR, (load:v4f32 addr:iPTR:$src))
14084       // Emits: (Int_RCPSSm:v4f32 addr:iPTR:$src)
14085       // Pattern complexity = 17  cost = 1  size = 3
14086       if (CN0 == 331) {
14087         N1 = N.getOperand(1);
14088         if (N1.getOpcode() == ISD::LOAD &&
14089             InFlightSet.count(N1.Val) == 0 &&
14090             N1.hasOneUse() &&
14091             !CodeGenMap.count(N1.getValue(0))) {
14092           Chain1 = N1.getOperand(0);
14093           N11 = N1.getOperand(1);
14094           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14095           if (Match) {
14096             Emit_120(Result, N, X86::Int_RCPSSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14097             return;
14098           }
14099         }
14100       }
14101 
14102       // Pattern: (intrinsic_wo_chain:v4f32 324:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14103       // Emits: (Int_MAXSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14104       // Pattern complexity = 17  cost = 1
14105       if (CN0 == 324) {
14106         N1 = N.getOperand(1);
14107         N2 = N.getOperand(2);
14108         if (N2.getOpcode() == ISD::LOAD &&
14109             InFlightSet.count(N2.Val) == 0 &&
14110             N2.hasOneUse() &&
14111             !CodeGenMap.count(N2.getValue(0))) {
14112           Chain2 = N2.getOperand(0);
14113           N21 = N2.getOperand(1);
14114           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14115           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14116           if (!Match) {
14117             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14118           }
14119           if (Match) {
14120             Emit_119(Result, N, X86::Int_MAXSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14121             return;
14122           }
14123         }
14124       }
14125     }
14126   }
14127 
14128   // Pattern: (intrinsic_wo_chain:v2f64 228:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14129   // Emits: (Int_MAXSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14130   // Pattern complexity = 17  cost = 1  size = 3
14131   if ((Subtarget->hasSSE2())) {
14132     N0 = N.getOperand(0);
14133     if (isa<ConstantSDNode>(N0)) {
14134       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14135       if (CN0 == 228) {
14136         N1 = N.getOperand(1);
14137         N2 = N.getOperand(2);
14138         if (N2.getOpcode() == ISD::LOAD &&
14139             InFlightSet.count(N2.Val) == 0 &&
14140             N2.hasOneUse() &&
14141             !CodeGenMap.count(N2.getValue(0))) {
14142           Chain2 = N2.getOperand(0);
14143           N21 = N2.getOperand(1);
14144           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14145           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14146           if (!Match) {
14147             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14148           }
14149           if (Match) {
14150             Emit_119(Result, N, X86::Int_MAXSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14151             return;
14152           }
14153         }
14154       }
14155     }
14156   }
14157 
14158   // Pattern: (intrinsic_wo_chain:v4f32 326:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14159   // Emits: (Int_MINSSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14160   // Pattern complexity = 17  cost = 1  size = 3
14161   if ((Subtarget->hasSSE1())) {
14162     N0 = N.getOperand(0);
14163     if (isa<ConstantSDNode>(N0)) {
14164       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14165       if (CN0 == 326) {
14166         N1 = N.getOperand(1);
14167         N2 = N.getOperand(2);
14168         if (N2.getOpcode() == ISD::LOAD &&
14169             InFlightSet.count(N2.Val) == 0 &&
14170             N2.hasOneUse() &&
14171             !CodeGenMap.count(N2.getValue(0))) {
14172           Chain2 = N2.getOperand(0);
14173           N21 = N2.getOperand(1);
14174           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14175           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14176           if (!Match) {
14177             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14178           }
14179           if (Match) {
14180             Emit_119(Result, N, X86::Int_MINSSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14181             return;
14182           }
14183         }
14184       }
14185     }
14186   }
14187 
14188   // Pattern: (intrinsic_wo_chain:v2f64 231:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14189   // Emits: (Int_MINSDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14190   // Pattern complexity = 17  cost = 1  size = 3
14191   if ((Subtarget->hasSSE2())) {
14192     N0 = N.getOperand(0);
14193     if (isa<ConstantSDNode>(N0)) {
14194       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14195       if (CN0 == 231) {
14196         N1 = N.getOperand(1);
14197         N2 = N.getOperand(2);
14198         if (N2.getOpcode() == ISD::LOAD &&
14199             InFlightSet.count(N2.Val) == 0 &&
14200             N2.hasOneUse() &&
14201             !CodeGenMap.count(N2.getValue(0))) {
14202           Chain2 = N2.getOperand(0);
14203           N21 = N2.getOperand(1);
14204           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14205           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14206           if (!Match) {
14207             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14208           }
14209           if (Match) {
14210             Emit_119(Result, N, X86::Int_MINSDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14211             return;
14212           }
14213         }
14214       }
14215     }
14216   }
14217 
14218   // Pattern: (intrinsic_wo_chain:i32 318:iPTR, (load:v4f32 addr:iPTR:$src))
14219   // Emits: (Int_CVTSS2SIrm:i32 addr:iPTR:$src)
14220   // Pattern complexity = 17  cost = 1  size = 3
14221   if ((Subtarget->hasSSE1())) {
14222     N0 = N.getOperand(0);
14223     if (isa<ConstantSDNode>(N0)) {
14224       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14225       if (CN0 == 318) {
14226         N1 = N.getOperand(1);
14227         if (N1.getOpcode() == ISD::LOAD &&
14228             InFlightSet.count(N1.Val) == 0 &&
14229             N1.hasOneUse() &&
14230             !CodeGenMap.count(N1.getValue(0))) {
14231           Chain1 = N1.getOperand(0);
14232           N11 = N1.getOperand(1);
14233           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14234           if (Match) {
14235             Emit_120(Result, N, X86::Int_CVTSS2SIrm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14236             return;
14237           }
14238         }
14239       }
14240     }
14241   }
14242 
14243   // Pattern: (intrinsic_wo_chain:i32 213:iPTR, (load:v2f64 addr:iPTR:$src))
14244   // Emits: (Int_CVTSD2SIrm:i32 addr:iPTR:$src)
14245   // Pattern complexity = 17  cost = 1  size = 3
14246   if ((Subtarget->hasSSE2())) {
14247     N0 = N.getOperand(0);
14248     if (isa<ConstantSDNode>(N0)) {
14249       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14250       if (CN0 == 213) {
14251         N1 = N.getOperand(1);
14252         if (N1.getOpcode() == ISD::LOAD &&
14253             InFlightSet.count(N1.Val) == 0 &&
14254             N1.hasOneUse() &&
14255             !CodeGenMap.count(N1.getValue(0))) {
14256           Chain1 = N1.getOperand(0);
14257           N11 = N1.getOperand(1);
14258           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14259           if (Match) {
14260             Emit_120(Result, N, X86::Int_CVTSD2SIrm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14261             return;
14262           }
14263         }
14264       }
14265     }
14266   }
14267 
14268   // Pattern: (intrinsic_wo_chain:i32 319:iPTR, (load:v4f32 addr:iPTR:$src))
14269   // Emits: (Int_CVTTSS2SIrm:i32 addr:iPTR:$src)
14270   // Pattern complexity = 17  cost = 1  size = 3
14271   if ((Subtarget->hasSSE1())) {
14272     N0 = N.getOperand(0);
14273     if (isa<ConstantSDNode>(N0)) {
14274       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14275       if (CN0 == 319) {
14276         N1 = N.getOperand(1);
14277         if (N1.getOpcode() == ISD::LOAD &&
14278             InFlightSet.count(N1.Val) == 0 &&
14279             N1.hasOneUse() &&
14280             !CodeGenMap.count(N1.getValue(0))) {
14281           Chain1 = N1.getOperand(0);
14282           N11 = N1.getOperand(1);
14283           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14284           if (Match) {
14285             Emit_120(Result, N, X86::Int_CVTTSS2SIrm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14286             return;
14287           }
14288         }
14289       }
14290     }
14291   }
14292 
14293   // Pattern: (intrinsic_wo_chain:i32 219:iPTR, (load:v2f64 addr:iPTR:$src))
14294   // Emits: (Int_CVTTSD2SIrm:i32 addr:iPTR:$src)
14295   // Pattern complexity = 17  cost = 1  size = 3
14296   if ((Subtarget->hasSSE2())) {
14297     N0 = N.getOperand(0);
14298     if (isa<ConstantSDNode>(N0)) {
14299       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14300       if (CN0 == 219) {
14301         N1 = N.getOperand(1);
14302         if (N1.getOpcode() == ISD::LOAD &&
14303             InFlightSet.count(N1.Val) == 0 &&
14304             N1.hasOneUse() &&
14305             !CodeGenMap.count(N1.getValue(0))) {
14306           Chain1 = N1.getOperand(0);
14307           N11 = N1.getOperand(1);
14308           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14309           if (Match) {
14310             Emit_120(Result, N, X86::Int_CVTTSD2SIrm, MVT::i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14311             return;
14312           }
14313         }
14314       }
14315     }
14316   }
14317 
14318   // Pattern: (intrinsic_wo_chain:v4f32 317:iPTR, VR128:v4f32:$src1, (load:i32 addr:iPTR:$src2))
14319   // Emits: (Int_CVTSI2SSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14320   // Pattern complexity = 17  cost = 1  size = 3
14321   if ((Subtarget->hasSSE1())) {
14322     N0 = N.getOperand(0);
14323     if (isa<ConstantSDNode>(N0)) {
14324       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14325       if (CN0 == 317) {
14326         N1 = N.getOperand(1);
14327         N2 = N.getOperand(2);
14328         if (N2.getOpcode() == ISD::LOAD &&
14329             InFlightSet.count(N2.Val) == 0 &&
14330             N2.hasOneUse() &&
14331             !CodeGenMap.count(N2.getValue(0))) {
14332           Chain2 = N2.getOperand(0);
14333           N21 = N2.getOperand(1);
14334           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14335           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14336           if (!Match) {
14337             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14338           }
14339           if (Match) {
14340             Emit_119(Result, N, X86::Int_CVTSI2SSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14341             return;
14342           }
14343         }
14344       }
14345     }
14346   }
14347   if ((Subtarget->hasSSE2())) {
14348     N0 = N.getOperand(0);
14349     if (isa<ConstantSDNode>(N0)) {
14350       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14351 
14352       // Pattern: (intrinsic_wo_chain:v4i32 211:iPTR, (load:v4f32 addr:iPTR:$src))
14353       // Emits: (Int_CVTPS2DQrm:v4i32 addr:iPTR:$src)
14354       // Pattern complexity = 17  cost = 1  size = 3
14355       if (CN0 == 211) {
14356         N1 = N.getOperand(1);
14357         if (N1.getOpcode() == ISD::LOAD &&
14358             InFlightSet.count(N1.Val) == 0 &&
14359             N1.hasOneUse() &&
14360             !CodeGenMap.count(N1.getValue(0))) {
14361           Chain1 = N1.getOperand(0);
14362           N11 = N1.getOperand(1);
14363           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14364           if (Match) {
14365             Emit_120(Result, N, X86::Int_CVTPS2DQrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14366             return;
14367           }
14368         }
14369       }
14370 
14371       // Pattern: (intrinsic_wo_chain:v4i32 218:iPTR, (load:v4f32 addr:iPTR:$src))
14372       // Emits: (Int_CVTTPS2DQrm:v4i32 addr:iPTR:$src)
14373       // Pattern complexity = 17  cost = 1  size = 3
14374       if (CN0 == 218) {
14375         N1 = N.getOperand(1);
14376         if (N1.getOpcode() == ISD::LOAD &&
14377             InFlightSet.count(N1.Val) == 0 &&
14378             N1.hasOneUse() &&
14379             !CodeGenMap.count(N1.getValue(0))) {
14380           Chain1 = N1.getOperand(0);
14381           N11 = N1.getOperand(1);
14382           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14383           if (Match) {
14384             Emit_120(Result, N, X86::Int_CVTTPS2DQrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14385             return;
14386           }
14387         }
14388       }
14389 
14390       // Pattern: (intrinsic_wo_chain:v4i32 209:iPTR, (load:v2f64 addr:iPTR:$src))
14391       // Emits: (Int_CVTPD2DQrm:v4i32 addr:iPTR:$src)
14392       // Pattern complexity = 17  cost = 1  size = 3
14393       if (CN0 == 209) {
14394         N1 = N.getOperand(1);
14395         if (N1.getOpcode() == ISD::LOAD &&
14396             InFlightSet.count(N1.Val) == 0 &&
14397             N1.hasOneUse() &&
14398             !CodeGenMap.count(N1.getValue(0))) {
14399           Chain1 = N1.getOperand(0);
14400           N11 = N1.getOperand(1);
14401           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14402           if (Match) {
14403             Emit_120(Result, N, X86::Int_CVTPD2DQrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14404             return;
14405           }
14406         }
14407       }
14408 
14409       // Pattern: (intrinsic_wo_chain:v4i32 217:iPTR, (load:v2f64 addr:iPTR:$src))
14410       // Emits: (Int_CVTTPD2DQrm:v4i32 addr:iPTR:$src)
14411       // Pattern complexity = 17  cost = 1  size = 3
14412       if (CN0 == 217) {
14413         N1 = N.getOperand(1);
14414         if (N1.getOpcode() == ISD::LOAD &&
14415             InFlightSet.count(N1.Val) == 0 &&
14416             N1.hasOneUse() &&
14417             !CodeGenMap.count(N1.getValue(0))) {
14418           Chain1 = N1.getOperand(0);
14419           N11 = N1.getOperand(1);
14420           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14421           if (Match) {
14422             Emit_120(Result, N, X86::Int_CVTTPD2DQrm, MVT::v4i32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14423             return;
14424           }
14425         }
14426       }
14427 
14428       // Pattern: (intrinsic_wo_chain:v2f64 212:iPTR, (load:v4f32 addr:iPTR:$src))
14429       // Emits: (Int_CVTPS2PDrm:v2f64 addr:iPTR:$src)
14430       // Pattern complexity = 17  cost = 1  size = 3
14431       if (CN0 == 212) {
14432         N1 = N.getOperand(1);
14433         if (N1.getOpcode() == ISD::LOAD &&
14434             InFlightSet.count(N1.Val) == 0 &&
14435             N1.hasOneUse() &&
14436             !CodeGenMap.count(N1.getValue(0))) {
14437           Chain1 = N1.getOperand(0);
14438           N11 = N1.getOperand(1);
14439           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14440           if (Match) {
14441             Emit_120(Result, N, X86::Int_CVTPS2PDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14442             return;
14443           }
14444         }
14445       }
14446 
14447       // Pattern: (intrinsic_wo_chain:v4f32 210:iPTR, (load:v2f64 addr:iPTR:$src))
14448       // Emits: (Int_CVTPD2PSrm:v4f32 addr:iPTR:$src)
14449       // Pattern complexity = 17  cost = 1  size = 3
14450       if (CN0 == 210) {
14451         N1 = N.getOperand(1);
14452         if (N1.getOpcode() == ISD::LOAD &&
14453             InFlightSet.count(N1.Val) == 0 &&
14454             N1.hasOneUse() &&
14455             !CodeGenMap.count(N1.getValue(0))) {
14456           Chain1 = N1.getOperand(0);
14457           N11 = N1.getOperand(1);
14458           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14459           if (Match) {
14460             Emit_120(Result, N, X86::Int_CVTPD2PSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14461             return;
14462           }
14463         }
14464       }
14465 
14466       // Pattern: (intrinsic_wo_chain:v2f64 215:iPTR, VR128:v2f64:$src1, (load:i32 addr:iPTR:$src2))
14467       // Emits: (Int_CVTSI2SDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14468       // Pattern complexity = 17  cost = 1  size = 3
14469       if (CN0 == 215) {
14470         N1 = N.getOperand(1);
14471         N2 = N.getOperand(2);
14472         if (N2.getOpcode() == ISD::LOAD &&
14473             InFlightSet.count(N2.Val) == 0 &&
14474             N2.hasOneUse() &&
14475             !CodeGenMap.count(N2.getValue(0))) {
14476           Chain2 = N2.getOperand(0);
14477           N21 = N2.getOperand(1);
14478           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14479           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14480           if (!Match) {
14481             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14482           }
14483           if (Match) {
14484             Emit_119(Result, N, X86::Int_CVTSI2SDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14485             return;
14486           }
14487         }
14488       }
14489 
14490       // Pattern: (intrinsic_wo_chain:v4f32 214:iPTR, VR128:v4f32:$src1, (load:v2f64 addr:iPTR:$src2))
14491       // Emits: (Int_CVTSD2SSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14492       // Pattern complexity = 17  cost = 1  size = 3
14493       if (CN0 == 214) {
14494         N1 = N.getOperand(1);
14495         N2 = N.getOperand(2);
14496         if (N2.getOpcode() == ISD::LOAD &&
14497             InFlightSet.count(N2.Val) == 0 &&
14498             N2.hasOneUse() &&
14499             !CodeGenMap.count(N2.getValue(0))) {
14500           Chain2 = N2.getOperand(0);
14501           N21 = N2.getOperand(1);
14502           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14503           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14504           if (!Match) {
14505             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14506           }
14507           if (Match) {
14508             Emit_119(Result, N, X86::Int_CVTSD2SSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14509             return;
14510           }
14511         }
14512       }
14513 
14514       // Pattern: (intrinsic_wo_chain:v2f64 216:iPTR, VR128:v2f64:$src1, (load:v4f32 addr:iPTR:$src2))
14515       // Emits: (Int_CVTSS2SDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14516       // Pattern complexity = 17  cost = 1
14517       if (CN0 == 216) {
14518         N1 = N.getOperand(1);
14519         N2 = N.getOperand(2);
14520         if (N2.getOpcode() == ISD::LOAD &&
14521             InFlightSet.count(N2.Val) == 0 &&
14522             N2.hasOneUse() &&
14523             !CodeGenMap.count(N2.getValue(0))) {
14524           Chain2 = N2.getOperand(0);
14525           N21 = N2.getOperand(1);
14526           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14527           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14528           if (!Match) {
14529             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14530           }
14531           if (Match) {
14532             Emit_119(Result, N, X86::Int_CVTSS2SDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14533             return;
14534           }
14535         }
14536       }
14537     }
14538   }
14539   if ((Subtarget->hasSSE3())) {
14540     N0 = N.getOperand(0);
14541     if (isa<ConstantSDNode>(N0)) {
14542       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14543 
14544       // Pattern: (intrinsic_wo_chain:v4f32 300:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14545       // Emits: (ADDSUBPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14546       // Pattern complexity = 17  cost = 1  size = 3
14547       if (CN0 == 300) {
14548         N1 = N.getOperand(1);
14549         N2 = N.getOperand(2);
14550         if (N2.getOpcode() == ISD::LOAD &&
14551             InFlightSet.count(N2.Val) == 0 &&
14552             N2.hasOneUse() &&
14553             !CodeGenMap.count(N2.getValue(0))) {
14554           Chain2 = N2.getOperand(0);
14555           N21 = N2.getOperand(1);
14556           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14557           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14558           if (!Match) {
14559             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14560           }
14561           if (Match) {
14562             Emit_119(Result, N, X86::ADDSUBPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14563             return;
14564           }
14565         }
14566       }
14567 
14568       // Pattern: (intrinsic_wo_chain:v2f64 299:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14569       // Emits: (ADDSUBPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14570       // Pattern complexity = 17  cost = 1
14571       if (CN0 == 299) {
14572         N1 = N.getOperand(1);
14573         N2 = N.getOperand(2);
14574         if (N2.getOpcode() == ISD::LOAD &&
14575             InFlightSet.count(N2.Val) == 0 &&
14576             N2.hasOneUse() &&
14577             !CodeGenMap.count(N2.getValue(0))) {
14578           Chain2 = N2.getOperand(0);
14579           N21 = N2.getOperand(1);
14580           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14581           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14582           if (!Match) {
14583             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14584           }
14585           if (Match) {
14586             Emit_119(Result, N, X86::ADDSUBPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14587             return;
14588           }
14589         }
14590       }
14591     }
14592   }
14593 
14594   // Pattern: (intrinsic_wo_chain:v4f32 335:iPTR, (load:v4f32 addr:iPTR:$src))
14595   // Emits: (SQRTPSm:v4f32 addr:iPTR:$src)
14596   // Pattern complexity = 17  cost = 1  size = 3
14597   if ((Subtarget->hasSSE1())) {
14598     N0 = N.getOperand(0);
14599     if (isa<ConstantSDNode>(N0)) {
14600       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14601       if (CN0 == 335) {
14602         N1 = N.getOperand(1);
14603         if (N1.getOpcode() == ISD::LOAD &&
14604             InFlightSet.count(N1.Val) == 0 &&
14605             N1.hasOneUse() &&
14606             !CodeGenMap.count(N1.getValue(0))) {
14607           Chain1 = N1.getOperand(0);
14608           N11 = N1.getOperand(1);
14609           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14610           if (Match) {
14611             Emit_120(Result, N, X86::SQRTPSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14612             return;
14613           }
14614         }
14615       }
14616     }
14617   }
14618 
14619   // Pattern: (intrinsic_wo_chain:v2f64 285:iPTR, (load:v2f64 addr:iPTR:$src))
14620   // Emits: (SQRTPDm:v2f64 addr:iPTR:$src)
14621   // Pattern complexity = 17  cost = 1  size = 3
14622   if ((Subtarget->hasSSE2())) {
14623     N0 = N.getOperand(0);
14624     if (isa<ConstantSDNode>(N0)) {
14625       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14626       if (CN0 == 285) {
14627         N1 = N.getOperand(1);
14628         if (N1.getOpcode() == ISD::LOAD &&
14629             InFlightSet.count(N1.Val) == 0 &&
14630             N1.hasOneUse() &&
14631             !CodeGenMap.count(N1.getValue(0))) {
14632           Chain1 = N1.getOperand(0);
14633           N11 = N1.getOperand(1);
14634           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14635           if (Match) {
14636             Emit_120(Result, N, X86::SQRTPDm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14637             return;
14638           }
14639         }
14640       }
14641     }
14642   }
14643   if ((Subtarget->hasSSE1())) {
14644     N0 = N.getOperand(0);
14645     if (isa<ConstantSDNode>(N0)) {
14646       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14647 
14648       // Pattern: (intrinsic_wo_chain:v4f32 332:iPTR, (load:v4f32 addr:iPTR:$src))
14649       // Emits: (RSQRTPSm:v4f32 addr:iPTR:$src)
14650       // Pattern complexity = 17  cost = 1  size = 3
14651       if (CN0 == 332) {
14652         N1 = N.getOperand(1);
14653         if (N1.getOpcode() == ISD::LOAD &&
14654             InFlightSet.count(N1.Val) == 0 &&
14655             N1.hasOneUse() &&
14656             !CodeGenMap.count(N1.getValue(0))) {
14657           Chain1 = N1.getOperand(0);
14658           N11 = N1.getOperand(1);
14659           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14660           if (Match) {
14661             Emit_120(Result, N, X86::RSQRTPSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14662             return;
14663           }
14664         }
14665       }
14666 
14667       // Pattern: (intrinsic_wo_chain:v4f32 330:iPTR, (load:v4f32 addr:iPTR:$src))
14668       // Emits: (RCPPSm:v4f32 addr:iPTR:$src)
14669       // Pattern complexity = 17  cost = 1  size = 3
14670       if (CN0 == 330) {
14671         N1 = N.getOperand(1);
14672         if (N1.getOpcode() == ISD::LOAD &&
14673             InFlightSet.count(N1.Val) == 0 &&
14674             N1.hasOneUse() &&
14675             !CodeGenMap.count(N1.getValue(0))) {
14676           Chain1 = N1.getOperand(0);
14677           N11 = N1.getOperand(1);
14678           bool Match = SelectAddr(N11, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14679           if (Match) {
14680             Emit_120(Result, N, X86::RCPPSm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain1, N0, N1, N11);
14681             return;
14682           }
14683         }
14684       }
14685 
14686       // Pattern: (intrinsic_wo_chain:v4f32 323:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14687       // Emits: (MAXPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14688       // Pattern complexity = 17  cost = 1
14689       if (CN0 == 323) {
14690         N1 = N.getOperand(1);
14691         N2 = N.getOperand(2);
14692         if (N2.getOpcode() == ISD::LOAD &&
14693             InFlightSet.count(N2.Val) == 0 &&
14694             N2.hasOneUse() &&
14695             !CodeGenMap.count(N2.getValue(0))) {
14696           Chain2 = N2.getOperand(0);
14697           N21 = N2.getOperand(1);
14698           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14699           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14700           if (!Match) {
14701             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14702           }
14703           if (Match) {
14704             Emit_119(Result, N, X86::MAXPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14705             return;
14706           }
14707         }
14708       }
14709     }
14710   }
14711 
14712   // Pattern: (intrinsic_wo_chain:v2f64 227:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14713   // Emits: (MAXPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14714   // Pattern complexity = 17  cost = 1  size = 3
14715   if ((Subtarget->hasSSE2())) {
14716     N0 = N.getOperand(0);
14717     if (isa<ConstantSDNode>(N0)) {
14718       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14719       if (CN0 == 227) {
14720         N1 = N.getOperand(1);
14721         N2 = N.getOperand(2);
14722         if (N2.getOpcode() == ISD::LOAD &&
14723             InFlightSet.count(N2.Val) == 0 &&
14724             N2.hasOneUse() &&
14725             !CodeGenMap.count(N2.getValue(0))) {
14726           Chain2 = N2.getOperand(0);
14727           N21 = N2.getOperand(1);
14728           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14729           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14730           if (!Match) {
14731             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14732           }
14733           if (Match) {
14734             Emit_119(Result, N, X86::MAXPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14735             return;
14736           }
14737         }
14738       }
14739     }
14740   }
14741 
14742   // Pattern: (intrinsic_wo_chain:v4f32 325:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14743   // Emits: (MINPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14744   // Pattern complexity = 17  cost = 1  size = 3
14745   if ((Subtarget->hasSSE1())) {
14746     N0 = N.getOperand(0);
14747     if (isa<ConstantSDNode>(N0)) {
14748       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14749       if (CN0 == 325) {
14750         N1 = N.getOperand(1);
14751         N2 = N.getOperand(2);
14752         if (N2.getOpcode() == ISD::LOAD &&
14753             InFlightSet.count(N2.Val) == 0 &&
14754             N2.hasOneUse() &&
14755             !CodeGenMap.count(N2.getValue(0))) {
14756           Chain2 = N2.getOperand(0);
14757           N21 = N2.getOperand(1);
14758           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14759           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14760           if (!Match) {
14761             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14762           }
14763           if (Match) {
14764             Emit_119(Result, N, X86::MINPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14765             return;
14766           }
14767         }
14768       }
14769     }
14770   }
14771 
14772   // Pattern: (intrinsic_wo_chain:v2f64 230:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14773   // Emits: (MINPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14774   // Pattern complexity = 17  cost = 1  size = 3
14775   if ((Subtarget->hasSSE2())) {
14776     N0 = N.getOperand(0);
14777     if (isa<ConstantSDNode>(N0)) {
14778       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14779       if (CN0 == 230) {
14780         N1 = N.getOperand(1);
14781         N2 = N.getOperand(2);
14782         if (N2.getOpcode() == ISD::LOAD &&
14783             InFlightSet.count(N2.Val) == 0 &&
14784             N2.hasOneUse() &&
14785             !CodeGenMap.count(N2.getValue(0))) {
14786           Chain2 = N2.getOperand(0);
14787           N21 = N2.getOperand(1);
14788           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14789           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14790           if (!Match) {
14791             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14792           }
14793           if (Match) {
14794             Emit_119(Result, N, X86::MINPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14795             return;
14796           }
14797         }
14798       }
14799     }
14800   }
14801   if ((Subtarget->hasSSE3())) {
14802     N0 = N.getOperand(0);
14803     if (isa<ConstantSDNode>(N0)) {
14804       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14805 
14806       // Pattern: (intrinsic_wo_chain:v4f32 302:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14807       // Emits: (HADDPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14808       // Pattern complexity = 17  cost = 1  size = 3
14809       if (CN0 == 302) {
14810         N1 = N.getOperand(1);
14811         N2 = N.getOperand(2);
14812         if (N2.getOpcode() == ISD::LOAD &&
14813             InFlightSet.count(N2.Val) == 0 &&
14814             N2.hasOneUse() &&
14815             !CodeGenMap.count(N2.getValue(0))) {
14816           Chain2 = N2.getOperand(0);
14817           N21 = N2.getOperand(1);
14818           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14819           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14820           if (!Match) {
14821             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14822           }
14823           if (Match) {
14824             Emit_119(Result, N, X86::HADDPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14825             return;
14826           }
14827         }
14828       }
14829 
14830       // Pattern: (intrinsic_wo_chain:v2f64 301:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14831       // Emits: (HADDPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14832       // Pattern complexity = 17  cost = 1  size = 3
14833       if (CN0 == 301) {
14834         N1 = N.getOperand(1);
14835         N2 = N.getOperand(2);
14836         if (N2.getOpcode() == ISD::LOAD &&
14837             InFlightSet.count(N2.Val) == 0 &&
14838             N2.hasOneUse() &&
14839             !CodeGenMap.count(N2.getValue(0))) {
14840           Chain2 = N2.getOperand(0);
14841           N21 = N2.getOperand(1);
14842           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14843           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14844           if (!Match) {
14845             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14846           }
14847           if (Match) {
14848             Emit_119(Result, N, X86::HADDPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14849             return;
14850           }
14851         }
14852       }
14853 
14854       // Pattern: (intrinsic_wo_chain:v4f32 304:iPTR, VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2))
14855       // Emits: (HSUBPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
14856       // Pattern complexity = 17  cost = 1  size = 3
14857       if (CN0 == 304) {
14858         N1 = N.getOperand(1);
14859         N2 = N.getOperand(2);
14860         if (N2.getOpcode() == ISD::LOAD &&
14861             InFlightSet.count(N2.Val) == 0 &&
14862             N2.hasOneUse() &&
14863             !CodeGenMap.count(N2.getValue(0))) {
14864           Chain2 = N2.getOperand(0);
14865           N21 = N2.getOperand(1);
14866           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14867           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14868           if (!Match) {
14869             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14870           }
14871           if (Match) {
14872             Emit_119(Result, N, X86::HSUBPSrm, MVT::v4f32, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14873             return;
14874           }
14875         }
14876       }
14877 
14878       // Pattern: (intrinsic_wo_chain:v2f64 303:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14879       // Emits: (HSUBPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
14880       // Pattern complexity = 17  cost = 1
14881       if (CN0 == 303) {
14882         N1 = N.getOperand(1);
14883         N2 = N.getOperand(2);
14884         if (N2.getOpcode() == ISD::LOAD &&
14885             InFlightSet.count(N2.Val) == 0 &&
14886             N2.hasOneUse() &&
14887             !CodeGenMap.count(N2.getValue(0))) {
14888           Chain2 = N2.getOperand(0);
14889           N21 = N2.getOperand(1);
14890           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14891           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14892           if (!Match) {
14893             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14894           }
14895           if (Match) {
14896             Emit_119(Result, N, X86::HSUBPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14897             return;
14898           }
14899         }
14900       }
14901     }
14902   }
14903   if ((Subtarget->hasSSE2())) {
14904     N0 = N.getOperand(0);
14905     if (isa<ConstantSDNode>(N0)) {
14906       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14907 
14908       // Pattern: (intrinsic_wo_chain:v2f64 297:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14909       // Emits: (UNPCKHPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
14910       // Pattern complexity = 17  cost = 1  size = 3
14911       if (CN0 == 297) {
14912         N1 = N.getOperand(1);
14913         N2 = N.getOperand(2);
14914         if (N2.getOpcode() == ISD::LOAD &&
14915             InFlightSet.count(N2.Val) == 0 &&
14916             N2.hasOneUse() &&
14917             !CodeGenMap.count(N2.getValue(0))) {
14918           Chain2 = N2.getOperand(0);
14919           N21 = N2.getOperand(1);
14920           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14921           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14922           if (!Match) {
14923             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14924           }
14925           if (Match) {
14926             Emit_119(Result, N, X86::UNPCKHPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14927             return;
14928           }
14929         }
14930       }
14931 
14932       // Pattern: (intrinsic_wo_chain:v2f64 298:iPTR, VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2))
14933       // Emits: (UNPCKLPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
14934       // Pattern complexity = 17  cost = 1  size = 3
14935       if (CN0 == 298) {
14936         N1 = N.getOperand(1);
14937         N2 = N.getOperand(2);
14938         if (N2.getOpcode() == ISD::LOAD &&
14939             InFlightSet.count(N2.Val) == 0 &&
14940             N2.hasOneUse() &&
14941             !CodeGenMap.count(N2.getValue(0))) {
14942           Chain2 = N2.getOperand(0);
14943           N21 = N2.getOperand(1);
14944           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14945           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14946           if (!Match) {
14947             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14948           }
14949           if (Match) {
14950             Emit_119(Result, N, X86::UNPCKLPDrm, MVT::v2f64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14951             return;
14952           }
14953         }
14954       }
14955 
14956       // Pattern: (intrinsic_wo_chain:v2i64 278:iPTR, VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
14957       // Emits: (PUNPCKHQDQrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
14958       // Pattern complexity = 17  cost = 1  size = 3
14959       if (CN0 == 278) {
14960         N1 = N.getOperand(1);
14961         N2 = N.getOperand(2);
14962         if (N2.getOpcode() == ISD::LOAD &&
14963             InFlightSet.count(N2.Val) == 0 &&
14964             N2.hasOneUse() &&
14965             !CodeGenMap.count(N2.getValue(0))) {
14966           Chain2 = N2.getOperand(0);
14967           N21 = N2.getOperand(1);
14968           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14969           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14970           if (!Match) {
14971             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14972           }
14973           if (Match) {
14974             Emit_119(Result, N, X86::PUNPCKHQDQrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14975             return;
14976           }
14977         }
14978       }
14979 
14980       // Pattern: (intrinsic_wo_chain:v2i64 279:iPTR, VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
14981       // Emits: (PUNPCKLQDQrm:v2i64 VR128:v16i8:$src1, addr:iPTR:$src2)
14982       // Pattern complexity = 17  cost = 1  size = 3
14983       if (CN0 == 279) {
14984         N1 = N.getOperand(1);
14985         N2 = N.getOperand(2);
14986         if (N2.getOpcode() == ISD::LOAD &&
14987             InFlightSet.count(N2.Val) == 0 &&
14988             N2.hasOneUse() &&
14989             !CodeGenMap.count(N2.getValue(0))) {
14990           Chain2 = N2.getOperand(0);
14991           N21 = N2.getOperand(1);
14992           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
14993           bool Match = SelectAddr(N21, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
14994           if (!Match) {
14995             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
14996           }
14997           if (Match) {
14998             Emit_119(Result, N, X86::PUNPCKLQDQrm, MVT::v2i64, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain2, N0, N1, N2, N21);
14999             return;
15000           }
15001         }
15002       }
15003 
15004       // Pattern: (intrinsic_wo_chain:v8i16 267:iPTR, VR128:v8i16:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15005       // Emits: (PSLLWri:v8i16 VR128:v8i16:$src1, (imm:i32):$src2)
15006       // Pattern complexity = 9  cost = 1  size = 3
15007       if (CN0 == 267) {
15008         N1 = N.getOperand(1);
15009         N2 = N.getOperand(2);
15010         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15011           N20 = N2.getOperand(0);
15012           if (N20.getOpcode() == ISD::Constant &&
15013               N20.Val->getValueType(0) == MVT::i32) {
15014             Emit_121(Result, N, X86::PSLLWri, MVT::v8i16, N0, N1, N2, N20);
15015             return;
15016           }
15017         }
15018       }
15019 
15020       // Pattern: (intrinsic_wo_chain:v4i32 264:iPTR, VR128:v4i32:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15021       // Emits: (PSLLDri:v4i32 VR128:v4i32:$src1, (imm:i32):$src2)
15022       // Pattern complexity = 9  cost = 1  size = 3
15023       if (CN0 == 264) {
15024         N1 = N.getOperand(1);
15025         N2 = N.getOperand(2);
15026         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15027           N20 = N2.getOperand(0);
15028           if (N20.getOpcode() == ISD::Constant &&
15029               N20.Val->getValueType(0) == MVT::i32) {
15030             Emit_121(Result, N, X86::PSLLDri, MVT::v4i32, N0, N1, N2, N20);
15031             return;
15032           }
15033         }
15034       }
15035 
15036       // Pattern: (intrinsic_wo_chain:v2i64 266:iPTR, VR128:v2i64:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15037       // Emits: (PSLLQri:v2i64 VR128:v2i64:$src1, (imm:i32):$src2)
15038       // Pattern complexity = 9  cost = 1  size = 3
15039       if (CN0 == 266) {
15040         N1 = N.getOperand(1);
15041         N2 = N.getOperand(2);
15042         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15043           N20 = N2.getOperand(0);
15044           if (N20.getOpcode() == ISD::Constant &&
15045               N20.Val->getValueType(0) == MVT::i32) {
15046             Emit_121(Result, N, X86::PSLLQri, MVT::v2i64, N0, N1, N2, N20);
15047             return;
15048           }
15049         }
15050       }
15051 
15052       // Pattern: (intrinsic_wo_chain:v8i16 273:iPTR, VR128:v8i16:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15053       // Emits: (PSRLWri:v8i16 VR128:v8i16:$src1, (imm:i32):$src2)
15054       // Pattern complexity = 9  cost = 1  size = 3
15055       if (CN0 == 273) {
15056         N1 = N.getOperand(1);
15057         N2 = N.getOperand(2);
15058         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15059           N20 = N2.getOperand(0);
15060           if (N20.getOpcode() == ISD::Constant &&
15061               N20.Val->getValueType(0) == MVT::i32) {
15062             Emit_121(Result, N, X86::PSRLWri, MVT::v8i16, N0, N1, N2, N20);
15063             return;
15064           }
15065         }
15066       }
15067 
15068       // Pattern: (intrinsic_wo_chain:v4i32 270:iPTR, VR128:v4i32:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15069       // Emits: (PSRLDri:v4i32 VR128:v4i32:$src1, (imm:i32):$src2)
15070       // Pattern complexity = 9  cost = 1  size = 3
15071       if (CN0 == 270) {
15072         N1 = N.getOperand(1);
15073         N2 = N.getOperand(2);
15074         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15075           N20 = N2.getOperand(0);
15076           if (N20.getOpcode() == ISD::Constant &&
15077               N20.Val->getValueType(0) == MVT::i32) {
15078             Emit_121(Result, N, X86::PSRLDri, MVT::v4i32, N0, N1, N2, N20);
15079             return;
15080           }
15081         }
15082       }
15083 
15084       // Pattern: (intrinsic_wo_chain:v2i64 272:iPTR, VR128:v2i64:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15085       // Emits: (PSRLQri:v2i64 VR128:v2i64:$src1, (imm:i32):$src2)
15086       // Pattern complexity = 9  cost = 1  size = 3
15087       if (CN0 == 272) {
15088         N1 = N.getOperand(1);
15089         N2 = N.getOperand(2);
15090         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15091           N20 = N2.getOperand(0);
15092           if (N20.getOpcode() == ISD::Constant &&
15093               N20.Val->getValueType(0) == MVT::i32) {
15094             Emit_121(Result, N, X86::PSRLQri, MVT::v2i64, N0, N1, N2, N20);
15095             return;
15096           }
15097         }
15098       }
15099 
15100       // Pattern: (intrinsic_wo_chain:v8i16 269:iPTR, VR128:v8i16:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15101       // Emits: (PSRAWri:v8i16 VR128:v8i16:$src1, (imm:i32):$src2)
15102       // Pattern complexity = 9  cost = 1  size = 3
15103       if (CN0 == 269) {
15104         N1 = N.getOperand(1);
15105         N2 = N.getOperand(2);
15106         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15107           N20 = N2.getOperand(0);
15108           if (N20.getOpcode() == ISD::Constant &&
15109               N20.Val->getValueType(0) == MVT::i32) {
15110             Emit_121(Result, N, X86::PSRAWri, MVT::v8i16, N0, N1, N2, N20);
15111             return;
15112           }
15113         }
15114       }
15115 
15116       // Pattern: (intrinsic_wo_chain:v4i32 268:iPTR, VR128:v4i32:$src1, (scalar_to_vector:v4i32 (imm:i32):$src2))
15117       // Emits: (PSRADri:v4i32 VR128:v4i32:$src1, (imm:i32):$src2)
15118       // Pattern complexity = 9  cost = 1
15119       if (CN0 == 268) {
15120         N1 = N.getOperand(1);
15121         N2 = N.getOperand(2);
15122         if (N2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
15123           N20 = N2.getOperand(0);
15124           if (N20.getOpcode() == ISD::Constant &&
15125               N20.Val->getValueType(0) == MVT::i32) {
15126             Emit_121(Result, N, X86::PSRADri, MVT::v4i32, N0, N1, N2, N20);
15127             return;
15128           }
15129         }
15130       }
15131     }
15132   }
15133   if ((Subtarget->hasSSE1())) {
15134     N0 = N.getOperand(0);
15135     if (isa<ConstantSDNode>(N0)) {
15136       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15137 
15138       // Pattern: (intrinsic_wo_chain:v4f32 310:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
15139       // Emits: (Int_CMPSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
15140       // Pattern complexity = 7  cost = 1  size = 3
15141       if (CN0 == 310) {
15142         N1 = N.getOperand(1);
15143         N2 = N.getOperand(2);
15144         N3 = N.getOperand(3);
15145         if (N3.getOpcode() == ISD::Constant) {
15146           Emit_122(Result, N, X86::Int_CMPSSrr, MVT::v4f32, N0, N1, N2, N3);
15147           return;
15148         }
15149       }
15150 
15151       // Pattern: (intrinsic_wo_chain:v4f32 309:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
15152       // Emits: (CMPPSrri:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
15153       // Pattern complexity = 7  cost = 1
15154       if (CN0 == 309) {
15155         N1 = N.getOperand(1);
15156         N2 = N.getOperand(2);
15157         N3 = N.getOperand(3);
15158         if (N3.getOpcode() == ISD::Constant) {
15159           Emit_122(Result, N, X86::CMPPSrri, MVT::v4f32, N0, N1, N2, N3);
15160           return;
15161         }
15162       }
15163     }
15164   }
15165   if ((Subtarget->hasSSE2())) {
15166     N0 = N.getOperand(0);
15167     if (isa<ConstantSDNode>(N0)) {
15168       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15169 
15170       // Pattern: (intrinsic_wo_chain:v2f64 199:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src, (imm:i8):$cc)
15171       // Emits: (CMPPDrri:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src, (imm:i8):$cc)
15172       // Pattern complexity = 7  cost = 1  size = 3
15173       if (CN0 == 199) {
15174         N1 = N.getOperand(1);
15175         N2 = N.getOperand(2);
15176         N3 = N.getOperand(3);
15177         if (N3.getOpcode() == ISD::Constant) {
15178           Emit_122(Result, N, X86::CMPPDrri, MVT::v2f64, N0, N1, N2, N3);
15179           return;
15180         }
15181       }
15182 
15183       // Pattern: (intrinsic_wo_chain:v2f64 284:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2, (imm:i32):$src3)
15184       // Emits: (SHUFPDrri:v2f64 VR128:v16i8:$src1, VR128:v16i8:$src2, (imm:i8):$src3)
15185       // Pattern complexity = 7  cost = 1  size = 3
15186       if (CN0 == 284) {
15187         N1 = N.getOperand(1);
15188         N2 = N.getOperand(2);
15189         N3 = N.getOperand(3);
15190         if (N3.getOpcode() == ISD::Constant) {
15191           Emit_122(Result, N, X86::SHUFPDrri, MVT::v2f64, N0, N1, N2, N3);
15192           return;
15193         }
15194       }
15195 
15196       // Pattern: (intrinsic_wo_chain:v2i64 265:iPTR, VR128:v2i64:$src1, (imm:i32):$src2)
15197       // Emits: (PSLLDQri:v2i64 VR128:v16i8:$src1, (PSxLDQ_imm:i32 (imm:i32):$src2))
15198       // Pattern complexity = 7  cost = 1  size = 3
15199       if (CN0 == 265) {
15200         N1 = N.getOperand(1);
15201         N2 = N.getOperand(2);
15202         if (N2.getOpcode() == ISD::Constant) {
15203           Emit_123(Result, N, X86::PSLLDQri, MVT::v2i64, N0, N1, N2);
15204           return;
15205         }
15206       }
15207 
15208       // Pattern: (intrinsic_wo_chain:v2i64 271:iPTR, VR128:v2i64:$src1, (imm:i32):$src2)
15209       // Emits: (PSRLDQri:v2i64 VR128:v16i8:$src1, (PSxLDQ_imm:i32 (imm:i32):$src2))
15210       // Pattern complexity = 7  cost = 1
15211       if (CN0 == 271) {
15212         N1 = N.getOperand(1);
15213         N2 = N.getOperand(2);
15214         if (N2.getOpcode() == ISD::Constant) {
15215           Emit_123(Result, N, X86::PSRLDQri, MVT::v2i64, N0, N1, N2);
15216           return;
15217         }
15218       }
15219     }
15220   }
15221 
15222   // Pattern: (intrinsic_wo_chain:v4f32 308:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15223   // Emits: (Int_ADDSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15224   // Pattern complexity = 5  cost = 1  size = 3
15225   if ((Subtarget->hasSSE1())) {
15226     N0 = N.getOperand(0);
15227     if (isa<ConstantSDNode>(N0)) {
15228       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15229       if (CN0 == 308) {
15230         Emit_124(Result, N, X86::Int_ADDSSrr, MVT::v4f32, N0, N1, N2);
15231         return;
15232       }
15233     }
15234   }
15235 
15236   // Pattern: (intrinsic_wo_chain:v2f64 197:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15237   // Emits: (Int_ADDSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15238   // Pattern complexity = 5  cost = 1  size = 3
15239   if ((Subtarget->hasSSE2())) {
15240     N0 = N.getOperand(0);
15241     if (isa<ConstantSDNode>(N0)) {
15242       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15243       if (CN0 == 197) {
15244         Emit_124(Result, N, X86::Int_ADDSDrr, MVT::v2f64, N0, N1, N2);
15245         return;
15246       }
15247     }
15248   }
15249 
15250   // Pattern: (intrinsic_wo_chain:v4f32 329:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15251   // Emits: (Int_MULSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15252   // Pattern complexity = 5  cost = 1  size = 3
15253   if ((Subtarget->hasSSE1())) {
15254     N0 = N.getOperand(0);
15255     if (isa<ConstantSDNode>(N0)) {
15256       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15257       if (CN0 == 329) {
15258         Emit_124(Result, N, X86::Int_MULSSrr, MVT::v4f32, N0, N1, N2);
15259         return;
15260       }
15261     }
15262   }
15263 
15264   // Pattern: (intrinsic_wo_chain:v2f64 238:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15265   // Emits: (Int_MULSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15266   // Pattern complexity = 5  cost = 1  size = 3
15267   if ((Subtarget->hasSSE2())) {
15268     N0 = N.getOperand(0);
15269     if (isa<ConstantSDNode>(N0)) {
15270       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15271       if (CN0 == 238) {
15272         Emit_124(Result, N, X86::Int_MULSDrr, MVT::v2f64, N0, N1, N2);
15273         return;
15274       }
15275     }
15276   }
15277 
15278   // Pattern: (intrinsic_wo_chain:v4f32 320:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15279   // Emits: (Int_DIVSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15280   // Pattern complexity = 5  cost = 1  size = 3
15281   if ((Subtarget->hasSSE1())) {
15282     N0 = N.getOperand(0);
15283     if (isa<ConstantSDNode>(N0)) {
15284       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15285       if (CN0 == 320) {
15286         Emit_124(Result, N, X86::Int_DIVSSrr, MVT::v4f32, N0, N1, N2);
15287         return;
15288       }
15289     }
15290   }
15291 
15292   // Pattern: (intrinsic_wo_chain:v2f64 220:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15293   // Emits: (Int_DIVSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15294   // Pattern complexity = 5  cost = 1  size = 3
15295   if ((Subtarget->hasSSE2())) {
15296     N0 = N.getOperand(0);
15297     if (isa<ConstantSDNode>(N0)) {
15298       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15299       if (CN0 == 220) {
15300         Emit_124(Result, N, X86::Int_DIVSDrr, MVT::v2f64, N0, N1, N2);
15301         return;
15302       }
15303     }
15304   }
15305 
15306   // Pattern: (intrinsic_wo_chain:v4f32 339:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15307   // Emits: (Int_SUBSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15308   // Pattern complexity = 5  cost = 1  size = 3
15309   if ((Subtarget->hasSSE1())) {
15310     N0 = N.getOperand(0);
15311     if (isa<ConstantSDNode>(N0)) {
15312       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15313       if (CN0 == 339) {
15314         Emit_124(Result, N, X86::Int_SUBSSrr, MVT::v4f32, N0, N1, N2);
15315         return;
15316       }
15317     }
15318   }
15319 
15320   // Pattern: (intrinsic_wo_chain:v2f64 290:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15321   // Emits: (Int_SUBSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15322   // Pattern complexity = 5  cost = 1  size = 3
15323   if ((Subtarget->hasSSE2())) {
15324     N0 = N.getOperand(0);
15325     if (isa<ConstantSDNode>(N0)) {
15326       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15327       if (CN0 == 290) {
15328         Emit_124(Result, N, X86::Int_SUBSDrr, MVT::v2f64, N0, N1, N2);
15329         return;
15330       }
15331     }
15332   }
15333 
15334   // Pattern: (intrinsic_wo_chain:v4f32 336:iPTR, VR128:v4f32:$src)
15335   // Emits: (Int_SQRTSSr:v4f32 VR128:v4f32:$src)
15336   // Pattern complexity = 5  cost = 1  size = 3
15337   if ((Subtarget->hasSSE1())) {
15338     N0 = N.getOperand(0);
15339     if (isa<ConstantSDNode>(N0)) {
15340       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15341       if (CN0 == 336) {
15342         Emit_116(Result, N, X86::Int_SQRTSSr, MVT::v4f32, N0, N1);
15343         return;
15344       }
15345     }
15346   }
15347 
15348   // Pattern: (intrinsic_wo_chain:v2f64 286:iPTR, VR128:v2f64:$src)
15349   // Emits: (Int_SQRTSDr:v2f64 VR128:v2f64:$src)
15350   // Pattern complexity = 5  cost = 1  size = 3
15351   if ((Subtarget->hasSSE2())) {
15352     N0 = N.getOperand(0);
15353     if (isa<ConstantSDNode>(N0)) {
15354       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15355       if (CN0 == 286) {
15356         Emit_116(Result, N, X86::Int_SQRTSDr, MVT::v2f64, N0, N1);
15357         return;
15358       }
15359     }
15360   }
15361   if ((Subtarget->hasSSE1())) {
15362     N0 = N.getOperand(0);
15363     if (isa<ConstantSDNode>(N0)) {
15364       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15365 
15366       // Pattern: (intrinsic_wo_chain:v4f32 333:iPTR, VR128:v4f32:$src)
15367       // Emits: (Int_RSQRTSSr:v4f32 VR128:v4f32:$src)
15368       // Pattern complexity = 5  cost = 1  size = 3
15369       if (CN0 == 333) {
15370         Emit_116(Result, N, X86::Int_RSQRTSSr, MVT::v4f32, N0, N1);
15371         return;
15372       }
15373 
15374       // Pattern: (intrinsic_wo_chain:v4f32 331:iPTR, VR128:v4f32:$src)
15375       // Emits: (Int_RCPSSr:v4f32 VR128:v4f32:$src)
15376       // Pattern complexity = 5  cost = 1  size = 3
15377       if (CN0 == 331) {
15378         Emit_116(Result, N, X86::Int_RCPSSr, MVT::v4f32, N0, N1);
15379         return;
15380       }
15381 
15382       // Pattern: (intrinsic_wo_chain:v4f32 324:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15383       // Emits: (Int_MAXSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15384       // Pattern complexity = 5  cost = 1
15385       if (CN0 == 324) {
15386         Emit_124(Result, N, X86::Int_MAXSSrr, MVT::v4f32, N0, N1, N2);
15387         return;
15388       }
15389     }
15390   }
15391 
15392   // Pattern: (intrinsic_wo_chain:v2f64 228:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15393   // Emits: (Int_MAXSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15394   // Pattern complexity = 5  cost = 1  size = 3
15395   if ((Subtarget->hasSSE2())) {
15396     N0 = N.getOperand(0);
15397     if (isa<ConstantSDNode>(N0)) {
15398       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15399       if (CN0 == 228) {
15400         Emit_124(Result, N, X86::Int_MAXSDrr, MVT::v2f64, N0, N1, N2);
15401         return;
15402       }
15403     }
15404   }
15405 
15406   // Pattern: (intrinsic_wo_chain:v4f32 326:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15407   // Emits: (Int_MINSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15408   // Pattern complexity = 5  cost = 1  size = 3
15409   if ((Subtarget->hasSSE1())) {
15410     N0 = N.getOperand(0);
15411     if (isa<ConstantSDNode>(N0)) {
15412       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15413       if (CN0 == 326) {
15414         Emit_124(Result, N, X86::Int_MINSSrr, MVT::v4f32, N0, N1, N2);
15415         return;
15416       }
15417     }
15418   }
15419 
15420   // Pattern: (intrinsic_wo_chain:v2f64 231:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15421   // Emits: (Int_MINSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15422   // Pattern complexity = 5  cost = 1  size = 3
15423   if ((Subtarget->hasSSE2())) {
15424     N0 = N.getOperand(0);
15425     if (isa<ConstantSDNode>(N0)) {
15426       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15427       if (CN0 == 231) {
15428         Emit_124(Result, N, X86::Int_MINSDrr, MVT::v2f64, N0, N1, N2);
15429         return;
15430       }
15431     }
15432   }
15433 
15434   // Pattern: (intrinsic_wo_chain:i32 318:iPTR, VR128:v4f32:$src)
15435   // Emits: (Int_CVTSS2SIrr:i32 VR128:v4f32:$src)
15436   // Pattern complexity = 5  cost = 1  size = 3
15437   if ((Subtarget->hasSSE1())) {
15438     N0 = N.getOperand(0);
15439     if (isa<ConstantSDNode>(N0)) {
15440       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15441       if (CN0 == 318) {
15442         Emit_116(Result, N, X86::Int_CVTSS2SIrr, MVT::i32, N0, N1);
15443         return;
15444       }
15445     }
15446   }
15447 
15448   // Pattern: (intrinsic_wo_chain:i32 213:iPTR, VR128:v2f64:$src)
15449   // Emits: (Int_CVTSD2SIrr:i32 VR128:v2f64:$src)
15450   // Pattern complexity = 5  cost = 1  size = 3
15451   if ((Subtarget->hasSSE2())) {
15452     N0 = N.getOperand(0);
15453     if (isa<ConstantSDNode>(N0)) {
15454       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15455       if (CN0 == 213) {
15456         Emit_116(Result, N, X86::Int_CVTSD2SIrr, MVT::i32, N0, N1);
15457         return;
15458       }
15459     }
15460   }
15461 
15462   // Pattern: (intrinsic_wo_chain:i32 319:iPTR, VR128:v4f32:$src)
15463   // Emits: (Int_CVTTSS2SIrr:i32 VR128:v4f32:$src)
15464   // Pattern complexity = 5  cost = 1  size = 3
15465   if ((Subtarget->hasSSE1())) {
15466     N0 = N.getOperand(0);
15467     if (isa<ConstantSDNode>(N0)) {
15468       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15469       if (CN0 == 319) {
15470         Emit_116(Result, N, X86::Int_CVTTSS2SIrr, MVT::i32, N0, N1);
15471         return;
15472       }
15473     }
15474   }
15475 
15476   // Pattern: (intrinsic_wo_chain:i32 219:iPTR, VR128:v2f64:$src)
15477   // Emits: (Int_CVTTSD2SIrr:i32 VR128:v2f64:$src)
15478   // Pattern complexity = 5  cost = 1  size = 3
15479   if ((Subtarget->hasSSE2())) {
15480     N0 = N.getOperand(0);
15481     if (isa<ConstantSDNode>(N0)) {
15482       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15483       if (CN0 == 219) {
15484         Emit_116(Result, N, X86::Int_CVTTSD2SIrr, MVT::i32, N0, N1);
15485         return;
15486       }
15487     }
15488   }
15489 
15490   // Pattern: (intrinsic_wo_chain:v4f32 317:iPTR, VR128:v4f32:$src1, GR32:i32:$src2)
15491   // Emits: (Int_CVTSI2SSrr:v4f32 VR128:v4f32:$src1, GR32:i32:$src2)
15492   // Pattern complexity = 5  cost = 1  size = 3
15493   if ((Subtarget->hasSSE1())) {
15494     N0 = N.getOperand(0);
15495     if (isa<ConstantSDNode>(N0)) {
15496       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15497       if (CN0 == 317) {
15498         Emit_124(Result, N, X86::Int_CVTSI2SSrr, MVT::v4f32, N0, N1, N2);
15499         return;
15500       }
15501     }
15502   }
15503   if ((Subtarget->hasSSE2())) {
15504     N0 = N.getOperand(0);
15505     if (isa<ConstantSDNode>(N0)) {
15506       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15507 
15508       // Pattern: (intrinsic_wo_chain:v4f32 208:iPTR, VR128:v4i32:$src)
15509       // Emits: (Int_CVTDQ2PSrr:v4f32 VR128:v4i32:$src)
15510       // Pattern complexity = 5  cost = 1  size = 3
15511       if (CN0 == 208) {
15512         Emit_116(Result, N, X86::Int_CVTDQ2PSrr, MVT::v4f32, N0, N1);
15513         return;
15514       }
15515 
15516       // Pattern: (intrinsic_wo_chain:v2f64 207:iPTR, VR128:v4i32:$src)
15517       // Emits: (Int_CVTDQ2PDrr:v2f64 VR128:v4i32:$src)
15518       // Pattern complexity = 5  cost = 1  size = 3
15519       if (CN0 == 207) {
15520         Emit_116(Result, N, X86::Int_CVTDQ2PDrr, MVT::v2f64, N0, N1);
15521         return;
15522       }
15523 
15524       // Pattern: (intrinsic_wo_chain:v4i32 211:iPTR, VR128:v4f32:$src)
15525       // Emits: (Int_CVTPS2DQrr:v4i32 VR128:v4f32:$src)
15526       // Pattern complexity = 5  cost = 1  size = 3
15527       if (CN0 == 211) {
15528         Emit_116(Result, N, X86::Int_CVTPS2DQrr, MVT::v4i32, N0, N1);
15529         return;
15530       }
15531 
15532       // Pattern: (intrinsic_wo_chain:v4i32 218:iPTR, VR128:v4f32:$src)
15533       // Emits: (Int_CVTTPS2DQrr:v4i32 VR128:v4f32:$src)
15534       // Pattern complexity = 5  cost = 1  size = 3
15535       if (CN0 == 218) {
15536         Emit_116(Result, N, X86::Int_CVTTPS2DQrr, MVT::v4i32, N0, N1);
15537         return;
15538       }
15539 
15540       // Pattern: (intrinsic_wo_chain:v4i32 209:iPTR, VR128:v2f64:$src)
15541       // Emits: (Int_CVTPD2DQrr:v4i32 VR128:v2f64:$src)
15542       // Pattern complexity = 5  cost = 1  size = 3
15543       if (CN0 == 209) {
15544         Emit_116(Result, N, X86::Int_CVTPD2DQrr, MVT::v4i32, N0, N1);
15545         return;
15546       }
15547 
15548       // Pattern: (intrinsic_wo_chain:v4i32 217:iPTR, VR128:v2f64:$src)
15549       // Emits: (Int_CVTTPD2DQrr:v4i32 VR128:v2f64:$src)
15550       // Pattern complexity = 5  cost = 1  size = 3
15551       if (CN0 == 217) {
15552         Emit_116(Result, N, X86::Int_CVTTPD2DQrr, MVT::v4i32, N0, N1);
15553         return;
15554       }
15555 
15556       // Pattern: (intrinsic_wo_chain:v2f64 212:iPTR, VR128:v4f32:$src)
15557       // Emits: (Int_CVTPS2PDrr:v2f64 VR128:v4f32:$src)
15558       // Pattern complexity = 5  cost = 1  size = 3
15559       if (CN0 == 212) {
15560         Emit_116(Result, N, X86::Int_CVTPS2PDrr, MVT::v2f64, N0, N1);
15561         return;
15562       }
15563 
15564       // Pattern: (intrinsic_wo_chain:v4f32 210:iPTR, VR128:v2f64:$src)
15565       // Emits: (Int_CVTPD2PSrr:v4f32 VR128:v2f64:$src)
15566       // Pattern complexity = 5  cost = 1  size = 3
15567       if (CN0 == 210) {
15568         Emit_116(Result, N, X86::Int_CVTPD2PSrr, MVT::v4f32, N0, N1);
15569         return;
15570       }
15571 
15572       // Pattern: (intrinsic_wo_chain:v2f64 215:iPTR, VR128:v2f64:$src1, GR32:i32:$src2)
15573       // Emits: (Int_CVTSI2SDrr:v2f64 VR128:v2f64:$src1, GR32:i32:$src2)
15574       // Pattern complexity = 5  cost = 1  size = 3
15575       if (CN0 == 215) {
15576         Emit_124(Result, N, X86::Int_CVTSI2SDrr, MVT::v2f64, N0, N1, N2);
15577         return;
15578       }
15579 
15580       // Pattern: (intrinsic_wo_chain:v4f32 214:iPTR, VR128:v4f32:$src1, VR128:v2f64:$src2)
15581       // Emits: (Int_CVTSD2SSrr:v4f32 VR128:v4f32:$src1, VR128:v2f64:$src2)
15582       // Pattern complexity = 5  cost = 1  size = 3
15583       if (CN0 == 214) {
15584         Emit_124(Result, N, X86::Int_CVTSD2SSrr, MVT::v4f32, N0, N1, N2);
15585         return;
15586       }
15587 
15588       // Pattern: (intrinsic_wo_chain:v2f64 216:iPTR, VR128:v2f64:$src1, VR128:v4f32:$src2)
15589       // Emits: (Int_CVTSS2SDrr:v2f64 VR128:v2f64:$src1, VR128:v4f32:$src2)
15590       // Pattern complexity = 5  cost = 1
15591       if (CN0 == 216) {
15592         Emit_124(Result, N, X86::Int_CVTSS2SDrr, MVT::v2f64, N0, N1, N2);
15593         return;
15594       }
15595     }
15596   }
15597   if ((Subtarget->hasSSE3())) {
15598     N0 = N.getOperand(0);
15599     if (isa<ConstantSDNode>(N0)) {
15600       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15601 
15602       // Pattern: (intrinsic_wo_chain:v4f32 300:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15603       // Emits: (ADDSUBPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15604       // Pattern complexity = 5  cost = 1  size = 3
15605       if (CN0 == 300) {
15606         Emit_124(Result, N, X86::ADDSUBPSrr, MVT::v4f32, N0, N1, N2);
15607         return;
15608       }
15609 
15610       // Pattern: (intrinsic_wo_chain:v2f64 299:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15611       // Emits: (ADDSUBPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15612       // Pattern complexity = 5  cost = 1
15613       if (CN0 == 299) {
15614         Emit_124(Result, N, X86::ADDSUBPDrr, MVT::v2f64, N0, N1, N2);
15615         return;
15616       }
15617     }
15618   }
15619 
15620   // Pattern: (intrinsic_wo_chain:v4f32 335:iPTR, VR128:v4f32:$src)
15621   // Emits: (SQRTPSr:v4f32 VR128:v4f32:$src)
15622   // Pattern complexity = 5  cost = 1  size = 3
15623   if ((Subtarget->hasSSE1())) {
15624     N0 = N.getOperand(0);
15625     if (isa<ConstantSDNode>(N0)) {
15626       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15627       if (CN0 == 335) {
15628         Emit_116(Result, N, X86::SQRTPSr, MVT::v4f32, N0, N1);
15629         return;
15630       }
15631     }
15632   }
15633 
15634   // Pattern: (intrinsic_wo_chain:v2f64 285:iPTR, VR128:v2f64:$src)
15635   // Emits: (SQRTPDr:v2f64 VR128:v2f64:$src)
15636   // Pattern complexity = 5  cost = 1  size = 3
15637   if ((Subtarget->hasSSE2())) {
15638     N0 = N.getOperand(0);
15639     if (isa<ConstantSDNode>(N0)) {
15640       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15641       if (CN0 == 285) {
15642         Emit_116(Result, N, X86::SQRTPDr, MVT::v2f64, N0, N1);
15643         return;
15644       }
15645     }
15646   }
15647   if ((Subtarget->hasSSE1())) {
15648     N0 = N.getOperand(0);
15649     if (isa<ConstantSDNode>(N0)) {
15650       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15651 
15652       // Pattern: (intrinsic_wo_chain:v4f32 332:iPTR, VR128:v4f32:$src)
15653       // Emits: (RSQRTPSr:v4f32 VR128:v4f32:$src)
15654       // Pattern complexity = 5  cost = 1  size = 3
15655       if (CN0 == 332) {
15656         Emit_116(Result, N, X86::RSQRTPSr, MVT::v4f32, N0, N1);
15657         return;
15658       }
15659 
15660       // Pattern: (intrinsic_wo_chain:v4f32 330:iPTR, VR128:v4f32:$src)
15661       // Emits: (RCPPSr:v4f32 VR128:v4f32:$src)
15662       // Pattern complexity = 5  cost = 1  size = 3
15663       if (CN0 == 330) {
15664         Emit_116(Result, N, X86::RCPPSr, MVT::v4f32, N0, N1);
15665         return;
15666       }
15667 
15668       // Pattern: (intrinsic_wo_chain:v4f32 323:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15669       // Emits: (MAXPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15670       // Pattern complexity = 5  cost = 1
15671       if (CN0 == 323) {
15672         Emit_124(Result, N, X86::MAXPSrr, MVT::v4f32, N0, N1, N2);
15673         return;
15674       }
15675     }
15676   }
15677 
15678   // Pattern: (intrinsic_wo_chain:v2f64 227:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15679   // Emits: (MAXPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15680   // Pattern complexity = 5  cost = 1  size = 3
15681   if ((Subtarget->hasSSE2())) {
15682     N0 = N.getOperand(0);
15683     if (isa<ConstantSDNode>(N0)) {
15684       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15685       if (CN0 == 227) {
15686         Emit_124(Result, N, X86::MAXPDrr, MVT::v2f64, N0, N1, N2);
15687         return;
15688       }
15689     }
15690   }
15691 
15692   // Pattern: (intrinsic_wo_chain:v4f32 325:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15693   // Emits: (MINPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15694   // Pattern complexity = 5  cost = 1  size = 3
15695   if ((Subtarget->hasSSE1())) {
15696     N0 = N.getOperand(0);
15697     if (isa<ConstantSDNode>(N0)) {
15698       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15699       if (CN0 == 325) {
15700         Emit_124(Result, N, X86::MINPSrr, MVT::v4f32, N0, N1, N2);
15701         return;
15702       }
15703     }
15704   }
15705 
15706   // Pattern: (intrinsic_wo_chain:v2f64 230:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15707   // Emits: (MINPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15708   // Pattern complexity = 5  cost = 1  size = 3
15709   if ((Subtarget->hasSSE2())) {
15710     N0 = N.getOperand(0);
15711     if (isa<ConstantSDNode>(N0)) {
15712       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15713       if (CN0 == 230) {
15714         Emit_124(Result, N, X86::MINPDrr, MVT::v2f64, N0, N1, N2);
15715         return;
15716       }
15717     }
15718   }
15719   if ((Subtarget->hasSSE3())) {
15720     N0 = N.getOperand(0);
15721     if (isa<ConstantSDNode>(N0)) {
15722       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15723 
15724       // Pattern: (intrinsic_wo_chain:v4f32 302:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15725       // Emits: (HADDPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15726       // Pattern complexity = 5  cost = 1  size = 3
15727       if (CN0 == 302) {
15728         Emit_124(Result, N, X86::HADDPSrr, MVT::v4f32, N0, N1, N2);
15729         return;
15730       }
15731 
15732       // Pattern: (intrinsic_wo_chain:v2f64 301:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15733       // Emits: (HADDPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15734       // Pattern complexity = 5  cost = 1  size = 3
15735       if (CN0 == 301) {
15736         Emit_124(Result, N, X86::HADDPDrr, MVT::v2f64, N0, N1, N2);
15737         return;
15738       }
15739 
15740       // Pattern: (intrinsic_wo_chain:v4f32 304:iPTR, VR128:v4f32:$src1, VR128:v4f32:$src2)
15741       // Emits: (HSUBPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15742       // Pattern complexity = 5  cost = 1  size = 3
15743       if (CN0 == 304) {
15744         Emit_124(Result, N, X86::HSUBPSrr, MVT::v4f32, N0, N1, N2);
15745         return;
15746       }
15747 
15748       // Pattern: (intrinsic_wo_chain:v2f64 303:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
15749       // Emits: (HSUBPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15750       // Pattern complexity = 5  cost = 1
15751       if (CN0 == 303) {
15752         Emit_124(Result, N, X86::HSUBPDrr, MVT::v2f64, N0, N1, N2);
15753         return;
15754       }
15755     }
15756   }
15757   if ((Subtarget->hasSSE2())) {
15758     N0 = N.getOperand(0);
15759     if (isa<ConstantSDNode>(N0)) {
15760       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15761 
15762       // Pattern: (intrinsic_wo_chain:v16i8 242:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15763       // Emits: (PADDSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15764       // Pattern complexity = 5  cost = 1  size = 3
15765       if (CN0 == 242) {
15766         Emit_124(Result, N, X86::PADDSBrr, MVT::v16i8, N0, N1, N2);
15767         return;
15768       }
15769 
15770       // Pattern: (intrinsic_wo_chain:v8i16 243:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15771       // Emits: (PADDSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15772       // Pattern complexity = 5  cost = 1  size = 3
15773       if (CN0 == 243) {
15774         Emit_124(Result, N, X86::PADDSWrr, MVT::v8i16, N0, N1, N2);
15775         return;
15776       }
15777 
15778       // Pattern: (intrinsic_wo_chain:v16i8 244:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15779       // Emits: (PADDUSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15780       // Pattern complexity = 5  cost = 1  size = 3
15781       if (CN0 == 244) {
15782         Emit_124(Result, N, X86::PADDUSBrr, MVT::v16i8, N0, N1, N2);
15783         return;
15784       }
15785 
15786       // Pattern: (intrinsic_wo_chain:v8i16 245:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15787       // Emits: (PADDUSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15788       // Pattern complexity = 5  cost = 1  size = 3
15789       if (CN0 == 245) {
15790         Emit_124(Result, N, X86::PADDUSWrr, MVT::v8i16, N0, N1, N2);
15791         return;
15792       }
15793 
15794       // Pattern: (intrinsic_wo_chain:v16i8 274:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15795       // Emits: (PSUBSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15796       // Pattern complexity = 5  cost = 1  size = 3
15797       if (CN0 == 274) {
15798         Emit_124(Result, N, X86::PSUBSBrr, MVT::v16i8, N0, N1, N2);
15799         return;
15800       }
15801 
15802       // Pattern: (intrinsic_wo_chain:v8i16 275:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15803       // Emits: (PSUBSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15804       // Pattern complexity = 5  cost = 1  size = 3
15805       if (CN0 == 275) {
15806         Emit_124(Result, N, X86::PSUBSWrr, MVT::v8i16, N0, N1, N2);
15807         return;
15808       }
15809 
15810       // Pattern: (intrinsic_wo_chain:v16i8 276:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15811       // Emits: (PSUBUSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15812       // Pattern complexity = 5  cost = 1  size = 3
15813       if (CN0 == 276) {
15814         Emit_124(Result, N, X86::PSUBUSBrr, MVT::v16i8, N0, N1, N2);
15815         return;
15816       }
15817 
15818       // Pattern: (intrinsic_wo_chain:v8i16 277:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15819       // Emits: (PSUBUSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15820       // Pattern complexity = 5  cost = 1  size = 3
15821       if (CN0 == 277) {
15822         Emit_124(Result, N, X86::PSUBUSWrr, MVT::v8i16, N0, N1, N2);
15823         return;
15824       }
15825 
15826       // Pattern: (intrinsic_wo_chain:v8i16 261:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15827       // Emits: (PMULHUWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15828       // Pattern complexity = 5  cost = 1  size = 3
15829       if (CN0 == 261) {
15830         Emit_124(Result, N, X86::PMULHUWrr, MVT::v8i16, N0, N1, N2);
15831         return;
15832       }
15833 
15834       // Pattern: (intrinsic_wo_chain:v8i16 260:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15835       // Emits: (PMULHWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15836       // Pattern complexity = 5  cost = 1  size = 3
15837       if (CN0 == 260) {
15838         Emit_124(Result, N, X86::PMULHWrr, MVT::v8i16, N0, N1, N2);
15839         return;
15840       }
15841 
15842       // Pattern: (intrinsic_wo_chain:v2i64 262:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
15843       // Emits: (PMULUDQrr:v2i64 VR128:v4i32:$src1, VR128:v4i32:$src2)
15844       // Pattern complexity = 5  cost = 1  size = 3
15845       if (CN0 == 262) {
15846         Emit_124(Result, N, X86::PMULUDQrr, MVT::v2i64, N0, N1, N2);
15847         return;
15848       }
15849 
15850       // Pattern: (intrinsic_wo_chain:v4i32 254:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15851       // Emits: (PMADDWDrr:v4i32 VR128:v8i16:$src1, VR128:v8i16:$src2)
15852       // Pattern complexity = 5  cost = 1  size = 3
15853       if (CN0 == 254) {
15854         Emit_124(Result, N, X86::PMADDWDrr, MVT::v4i32, N0, N1, N2);
15855         return;
15856       }
15857 
15858       // Pattern: (intrinsic_wo_chain:v16i8 246:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15859       // Emits: (PAVGBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15860       // Pattern complexity = 5  cost = 1  size = 3
15861       if (CN0 == 246) {
15862         Emit_124(Result, N, X86::PAVGBrr, MVT::v16i8, N0, N1, N2);
15863         return;
15864       }
15865 
15866       // Pattern: (intrinsic_wo_chain:v8i16 247:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15867       // Emits: (PAVGWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15868       // Pattern complexity = 5  cost = 1  size = 3
15869       if (CN0 == 247) {
15870         Emit_124(Result, N, X86::PAVGWrr, MVT::v8i16, N0, N1, N2);
15871         return;
15872       }
15873 
15874       // Pattern: (intrinsic_wo_chain:v16i8 256:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15875       // Emits: (PMAXUBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15876       // Pattern complexity = 5  cost = 1  size = 3
15877       if (CN0 == 256) {
15878         Emit_124(Result, N, X86::PMAXUBrr, MVT::v16i8, N0, N1, N2);
15879         return;
15880       }
15881 
15882       // Pattern: (intrinsic_wo_chain:v8i16 255:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15883       // Emits: (PMAXSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15884       // Pattern complexity = 5  cost = 1  size = 3
15885       if (CN0 == 255) {
15886         Emit_124(Result, N, X86::PMAXSWrr, MVT::v8i16, N0, N1, N2);
15887         return;
15888       }
15889 
15890       // Pattern: (intrinsic_wo_chain:v16i8 258:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15891       // Emits: (PMINUBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15892       // Pattern complexity = 5  cost = 1  size = 3
15893       if (CN0 == 258) {
15894         Emit_124(Result, N, X86::PMINUBrr, MVT::v16i8, N0, N1, N2);
15895         return;
15896       }
15897 
15898       // Pattern: (intrinsic_wo_chain:v8i16 257:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15899       // Emits: (PMINSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15900       // Pattern complexity = 5  cost = 1  size = 3
15901       if (CN0 == 257) {
15902         Emit_124(Result, N, X86::PMINSWrr, MVT::v8i16, N0, N1, N2);
15903         return;
15904       }
15905 
15906       // Pattern: (intrinsic_wo_chain:v2i64 263:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15907       // Emits: (PSADBWrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
15908       // Pattern complexity = 5  cost = 1  size = 3
15909       if (CN0 == 263) {
15910         Emit_124(Result, N, X86::PSADBWrr, MVT::v2i64, N0, N1, N2);
15911         return;
15912       }
15913 
15914       // Pattern: (intrinsic_wo_chain:v8i16 267:iPTR, VR128:v8i16:$src1, VR128:v4i32:$src2)
15915       // Emits: (PSLLWrr:v8i16 VR128:v8i16:$src1, VR128:v4i32:$src2)
15916       // Pattern complexity = 5  cost = 1  size = 3
15917       if (CN0 == 267) {
15918         Emit_124(Result, N, X86::PSLLWrr, MVT::v8i16, N0, N1, N2);
15919         return;
15920       }
15921 
15922       // Pattern: (intrinsic_wo_chain:v4i32 264:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
15923       // Emits: (PSLLDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
15924       // Pattern complexity = 5  cost = 1  size = 3
15925       if (CN0 == 264) {
15926         Emit_124(Result, N, X86::PSLLDrr, MVT::v4i32, N0, N1, N2);
15927         return;
15928       }
15929 
15930       // Pattern: (intrinsic_wo_chain:v2i64 266:iPTR, VR128:v2i64:$src1, VR128:v4i32:$src2)
15931       // Emits: (PSLLQrr:v2i64 VR128:v2i64:$src1, VR128:v4i32:$src2)
15932       // Pattern complexity = 5  cost = 1  size = 3
15933       if (CN0 == 266) {
15934         Emit_124(Result, N, X86::PSLLQrr, MVT::v2i64, N0, N1, N2);
15935         return;
15936       }
15937 
15938       // Pattern: (intrinsic_wo_chain:v8i16 273:iPTR, VR128:v8i16:$src1, VR128:v4i32:$src2)
15939       // Emits: (PSRLWrr:v8i16 VR128:v8i16:$src1, VR128:v4i32:$src2)
15940       // Pattern complexity = 5  cost = 1  size = 3
15941       if (CN0 == 273) {
15942         Emit_124(Result, N, X86::PSRLWrr, MVT::v8i16, N0, N1, N2);
15943         return;
15944       }
15945 
15946       // Pattern: (intrinsic_wo_chain:v4i32 270:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
15947       // Emits: (PSRLDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
15948       // Pattern complexity = 5  cost = 1  size = 3
15949       if (CN0 == 270) {
15950         Emit_124(Result, N, X86::PSRLDrr, MVT::v4i32, N0, N1, N2);
15951         return;
15952       }
15953 
15954       // Pattern: (intrinsic_wo_chain:v2i64 272:iPTR, VR128:v2i64:$src1, VR128:v4i32:$src2)
15955       // Emits: (PSRLQrr:v2i64 VR128:v2i64:$src1, VR128:v4i32:$src2)
15956       // Pattern complexity = 5  cost = 1  size = 3
15957       if (CN0 == 272) {
15958         Emit_124(Result, N, X86::PSRLQrr, MVT::v2i64, N0, N1, N2);
15959         return;
15960       }
15961 
15962       // Pattern: (intrinsic_wo_chain:v8i16 269:iPTR, VR128:v8i16:$src1, VR128:v4i32:$src2)
15963       // Emits: (PSRAWrr:v8i16 VR128:v8i16:$src1, VR128:v4i32:$src2)
15964       // Pattern complexity = 5  cost = 1  size = 3
15965       if (CN0 == 269) {
15966         Emit_124(Result, N, X86::PSRAWrr, MVT::v8i16, N0, N1, N2);
15967         return;
15968       }
15969 
15970       // Pattern: (intrinsic_wo_chain:v4i32 268:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
15971       // Emits: (PSRADrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
15972       // Pattern complexity = 5  cost = 1  size = 3
15973       if (CN0 == 268) {
15974         Emit_124(Result, N, X86::PSRADrr, MVT::v4i32, N0, N1, N2);
15975         return;
15976       }
15977 
15978       // Pattern: (intrinsic_wo_chain:v16i8 248:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
15979       // Emits: (PCMPEQBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15980       // Pattern complexity = 5  cost = 1  size = 3
15981       if (CN0 == 248) {
15982         Emit_124(Result, N, X86::PCMPEQBrr, MVT::v16i8, N0, N1, N2);
15983         return;
15984       }
15985 
15986       // Pattern: (intrinsic_wo_chain:v8i16 250:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
15987       // Emits: (PCMPEQWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15988       // Pattern complexity = 5  cost = 1  size = 3
15989       if (CN0 == 250) {
15990         Emit_124(Result, N, X86::PCMPEQWrr, MVT::v8i16, N0, N1, N2);
15991         return;
15992       }
15993 
15994       // Pattern: (intrinsic_wo_chain:v4i32 249:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
15995       // Emits: (PCMPEQDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
15996       // Pattern complexity = 5  cost = 1  size = 3
15997       if (CN0 == 249) {
15998         Emit_124(Result, N, X86::PCMPEQDrr, MVT::v4i32, N0, N1, N2);
15999         return;
16000       }
16001 
16002       // Pattern: (intrinsic_wo_chain:v16i8 251:iPTR, VR128:v16i8:$src1, VR128:v16i8:$src2)
16003       // Emits: (PCMPGTBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
16004       // Pattern complexity = 5  cost = 1  size = 3
16005       if (CN0 == 251) {
16006         Emit_124(Result, N, X86::PCMPGTBrr, MVT::v16i8, N0, N1, N2);
16007         return;
16008       }
16009 
16010       // Pattern: (intrinsic_wo_chain:v8i16 253:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
16011       // Emits: (PCMPGTWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16012       // Pattern complexity = 5  cost = 1  size = 3
16013       if (CN0 == 253) {
16014         Emit_124(Result, N, X86::PCMPGTWrr, MVT::v8i16, N0, N1, N2);
16015         return;
16016       }
16017 
16018       // Pattern: (intrinsic_wo_chain:v4i32 252:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
16019       // Emits: (PCMPGTDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
16020       // Pattern complexity = 5  cost = 1  size = 3
16021       if (CN0 == 252) {
16022         Emit_124(Result, N, X86::PCMPGTDrr, MVT::v4i32, N0, N1, N2);
16023         return;
16024       }
16025 
16026       // Pattern: (intrinsic_wo_chain:v8i16 240:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
16027       // Emits: (PACKSSWBrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16028       // Pattern complexity = 5  cost = 1  size = 3
16029       if (CN0 == 240) {
16030         Emit_124(Result, N, X86::PACKSSWBrr, MVT::v8i16, N0, N1, N2);
16031         return;
16032       }
16033 
16034       // Pattern: (intrinsic_wo_chain:v4i32 239:iPTR, VR128:v4i32:$src1, VR128:v4i32:$src2)
16035       // Emits: (PACKSSDWrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
16036       // Pattern complexity = 5  cost = 1  size = 3
16037       if (CN0 == 239) {
16038         Emit_124(Result, N, X86::PACKSSDWrr, MVT::v4i32, N0, N1, N2);
16039         return;
16040       }
16041 
16042       // Pattern: (intrinsic_wo_chain:v8i16 241:iPTR, VR128:v8i16:$src1, VR128:v8i16:$src2)
16043       // Emits: (PACKUSWBrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16044       // Pattern complexity = 5  cost = 1
16045       if (CN0 == 241) {
16046         Emit_124(Result, N, X86::PACKUSWBrr, MVT::v8i16, N0, N1, N2);
16047         return;
16048       }
16049     }
16050   }
16051   if ((Subtarget->hasSSE1())) {
16052     N0 = N.getOperand(0);
16053     if (isa<ConstantSDNode>(N0)) {
16054       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
16055 
16056       // Pattern: (intrinsic_wo_chain:i32 327:iPTR, VR128:v4f32:$src)
16057       // Emits: (MOVMSKPSrr:i32 VR128:v4f32:$src)
16058       // Pattern complexity = 5  cost = 1  size = 3
16059       if (CN0 == 327) {
16060         Emit_116(Result, N, X86::MOVMSKPSrr, MVT::i32, N0, N1);
16061         return;
16062       }
16063 
16064       // Pattern: (intrinsic_wo_chain:i32 233:iPTR, VR128:v2f64:$src)
16065       // Emits: (MOVMSKPDrr:i32 VR128:v2f64:$src)
16066       // Pattern complexity = 5  cost = 1
16067       if (CN0 == 233) {
16068         Emit_116(Result, N, X86::MOVMSKPDrr, MVT::i32, N0, N1);
16069         return;
16070       }
16071     }
16072   }
16073   if ((Subtarget->hasSSE2())) {
16074     N0 = N.getOperand(0);
16075     if (isa<ConstantSDNode>(N0)) {
16076       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
16077 
16078       // Pattern: (intrinsic_wo_chain:i32 259:iPTR, VR128:v16i8:$src)
16079       // Emits: (PMOVMSKBrr:i32 VR128:v16i8:$src)
16080       // Pattern complexity = 5  cost = 1  size = 3
16081       if (CN0 == 259) {
16082         Emit_116(Result, N, X86::PMOVMSKBrr, MVT::i32, N0, N1);
16083         return;
16084       }
16085 
16086       // Pattern: (intrinsic_wo_chain:v2f64 237:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
16087       // Emits: (MOVLPDrr:v2f64 VR128:v16i8:$src1, VR128:v16i8:$src2)
16088       // Pattern complexity = 5  cost = 1  size = 3
16089       if (CN0 == 237) {
16090         Emit_124(Result, N, X86::MOVLPDrr, MVT::v2f64, N0, N1, N2);
16091         return;
16092       }
16093 
16094       // Pattern: (intrinsic_wo_chain:v2f64 297:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
16095       // Emits: (UNPCKHPDrr:v2f64 VR128:v16i8:$src1, VR128:v16i8:$src2)
16096       // Pattern complexity = 5  cost = 1  size = 3
16097       if (CN0 == 297) {
16098         Emit_124(Result, N, X86::UNPCKHPDrr, MVT::v2f64, N0, N1, N2);
16099         return;
16100       }
16101 
16102       // Pattern: (intrinsic_wo_chain:v2f64 298:iPTR, VR128:v2f64:$src1, VR128:v2f64:$src2)
16103       // Emits: (UNPCKLPDrr:v2f64 VR128:v16i8:$src1, VR128:v16i8:$src2)
16104       // Pattern complexity = 5  cost = 1  size = 3
16105       if (CN0 == 298) {
16106         Emit_124(Result, N, X86::UNPCKLPDrr, MVT::v2f64, N0, N1, N2);
16107         return;
16108       }
16109 
16110       // Pattern: (intrinsic_wo_chain:v2i64 278:iPTR, VR128:v2i64:$src1, VR128:v2i64:$src2)
16111       // Emits: (PUNPCKHQDQrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
16112       // Pattern complexity = 5  cost = 1  size = 3
16113       if (CN0 == 278) {
16114         Emit_124(Result, N, X86::PUNPCKHQDQrr, MVT::v2i64, N0, N1, N2);
16115         return;
16116       }
16117 
16118       // Pattern: (intrinsic_wo_chain:v2i64 279:iPTR, VR128:v2i64:$src1, VR128:v2i64:$src2)
16119       // Emits: (PUNPCKLQDQrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
16120       // Pattern complexity = 5  cost = 1
16121       if (CN0 == 279) {
16122         Emit_124(Result, N, X86::PUNPCKLQDQrr, MVT::v2i64, N0, N1, N2);
16123         return;
16124       }
16125     }
16126   }
16127   std::cerr << "Cannot yet select: ";
16128   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
16129   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
16130   std::cerr << '\n';
16131   abort();
16132 }
16133 
16134 void Select_load(SDOperand &Result, SDOperand N) {
16135   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
16136     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
16137     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
16138     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
16139     Result = Dummy;
16140     return;
16141   }
16142   SDOperand CPTmp0(0, 0);
16143   SDOperand CPTmp1(0, 0);
16144   SDOperand CPTmp2(0, 0);
16145   SDOperand CPTmp3(0, 0);
16146   SDOperand Chain(0, 0);
16147   SDOperand N1(0, 0);
16148   SDOperand Tmp0(0, 0);
16149   SDOperand Tmp1(0, 0);
16150   SDOperand Tmp2(0, 0);
16151   SDOperand Tmp3(0, 0);
16152   SDNode *ResNode = NULL;
16153 
16154   // Pattern: (load:f64 addr:iPTR:$src)
16155   // Emits: (FpLD64m:f64 addr:iPTR:$src)
16156   // Pattern complexity = 12  cost = 1  size = 0
16157   if ((!Subtarget->hasSSE2())) {
16158     Chain = N.getOperand(0);
16159     N1 = N.getOperand(1);
16160     if (N.Val->getValueType(0) == MVT::f64) {
16161       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16162       if (Match) {
16163         Emit_28(Result, N, X86::FpLD64m, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16164         return;
16165       }
16166     }
16167   }
16168   {
16169     Chain = N.getOperand(0);
16170     N1 = N.getOperand(1);
16171 
16172     // Pattern: (load:i8 addr:iPTR:$src)
16173     // Emits: (MOV8rm:i8 addr:iPTR:$src)
16174     // Pattern complexity = 12  cost = 1  size = 3
16175     if (N.Val->getValueType(0) == MVT::i8) {
16176       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16177       if (Match) {
16178         Emit_28(Result, N, X86::MOV8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16179         return;
16180       }
16181     }
16182 
16183     // Pattern: (load:i16 addr:iPTR:$src)
16184     // Emits: (MOV16rm:i16 addr:iPTR:$src)
16185     // Pattern complexity = 12  cost = 1  size = 3
16186     if (N.Val->getValueType(0) == MVT::i16) {
16187       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16188       if (Match) {
16189         Emit_28(Result, N, X86::MOV16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16190         return;
16191       }
16192     }
16193 
16194     // Pattern: (load:i32 addr:iPTR:$src)
16195     // Emits: (MOV32rm:i32 addr:iPTR:$src)
16196     // Pattern complexity = 12  cost = 1
16197     if (N.Val->getValueType(0) == MVT::i32) {
16198       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16199       if (Match) {
16200         Emit_28(Result, N, X86::MOV32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16201         return;
16202       }
16203     }
16204   }
16205 
16206   // Pattern: (load:f32 addr:iPTR:$src)
16207   // Emits: (MOVSSrm:f32 addr:iPTR:$src)
16208   // Pattern complexity = 12  cost = 1  size = 3
16209   if ((Subtarget->hasSSE1())) {
16210     Chain = N.getOperand(0);
16211     N1 = N.getOperand(1);
16212     if (N.Val->getValueType(0) == MVT::f32) {
16213       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16214       if (Match) {
16215         Emit_28(Result, N, X86::MOVSSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16216         return;
16217       }
16218     }
16219   }
16220 
16221   // Pattern: (load:f64 addr:iPTR:$src)
16222   // Emits: (MOVSDrm:f64 addr:iPTR:$src)
16223   // Pattern complexity = 12  cost = 1  size = 3
16224   if ((Subtarget->hasSSE2())) {
16225     Chain = N.getOperand(0);
16226     N1 = N.getOperand(1);
16227     if (N.Val->getValueType(0) == MVT::f64) {
16228       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16229       if (Match) {
16230         Emit_28(Result, N, X86::MOVSDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16231         return;
16232       }
16233     }
16234   }
16235 
16236   // Pattern: (load:v4f32 addr:iPTR:$src)
16237   // Emits: (MOVAPSrm:v4f32 addr:iPTR:$src)
16238   // Pattern complexity = 12  cost = 1  size = 3
16239   if ((Subtarget->hasSSE1())) {
16240     Chain = N.getOperand(0);
16241     N1 = N.getOperand(1);
16242     if (N.Val->getValueType(0) == MVT::v4f32) {
16243       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16244       if (Match) {
16245         Emit_28(Result, N, X86::MOVAPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16246         return;
16247       }
16248     }
16249   }
16250   if ((Subtarget->hasSSE2())) {
16251     Chain = N.getOperand(0);
16252     N1 = N.getOperand(1);
16253 
16254     // Pattern: (load:v2f64 addr:iPTR:$src)
16255     // Emits: (MOVAPDrm:v2f64 addr:iPTR:$src)
16256     // Pattern complexity = 12  cost = 1  size = 3
16257     if (N.Val->getValueType(0) == MVT::v2f64) {
16258       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16259       if (Match) {
16260         Emit_28(Result, N, X86::MOVAPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16261         return;
16262       }
16263     }
16264 
16265     // Pattern: (load:v2i64 addr:iPTR:$src)
16266     // Emits: (MOVDQArm:v2i64 addr:iPTR:$src)
16267     // Pattern complexity = 12  cost = 1
16268     if (N.Val->getValueType(0) == MVT::v2i64) {
16269       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16270       if (Match) {
16271         Emit_28(Result, N, X86::MOVDQArm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1);
16272         return;
16273       }
16274     }
16275   }
16276   std::cerr << "Cannot yet select: ";
16277   N.Val->dump(CurDAG);
16278   std::cerr << '\n';
16279   abort();
16280 }
16281 
16282 void Emit_125(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
16283   SDOperand Tmp0(0, 0);
16284   SDOperand Tmp1(0, 0);
16285   SDOperand Tmp2(0, 0);
16286   SDOperand Tmp3(0, 0);
16287   SDOperand Tmp4(0, 0);
16288   SDNode *ResNode = NULL;
16289   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
16290   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
16291   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
16292   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
16293   Select(Tmp0, CPTmp0);
16294   Select(Tmp1, CPTmp1);
16295   Select(Tmp2, CPTmp2);
16296   Select(Tmp3, CPTmp3);
16297   Tmp4 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
16298   Select(Chain0, Chain0);
16299   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
16300   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
16301   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
16302   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
16303   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16304   Chain0 = SDOperand(ResNode, 1);
16305   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16306   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16307   AddHandleReplacement(N0.Val, 1, ResNode, 1);
16308   Result = SDOperand(ResNode, N.ResNo);
16309   return;
16310 }
16311 void Emit_126(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
16312   SDOperand Tmp0(0, 0);
16313   SDOperand Tmp1(0, 0);
16314   SDOperand Tmp2(0, 0);
16315   SDOperand Tmp3(0, 0);
16316   SDOperand Tmp4(0, 0);
16317   SDNode *ResNode = NULL;
16318   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
16319   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
16320   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
16321   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
16322   Select(Tmp0, CPTmp0);
16323   Select(Tmp1, CPTmp1);
16324   Select(Tmp2, CPTmp2);
16325   Select(Tmp3, CPTmp3);
16326   Tmp4 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
16327   Select(Chain0, Chain0);
16328   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
16329   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
16330   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
16331   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
16332   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16333   Chain0 = SDOperand(ResNode, 1);
16334   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16335   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16336   AddHandleReplacement(N0.Val, 1, ResNode, 1);
16337   Result = SDOperand(ResNode, N.ResNo);
16338   return;
16339 }
16340 void Emit_127(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11) NOINLINE {
16341   SDOperand InFlag(0, 0);
16342   SDOperand Tmp0(0, 0);
16343   SDOperand Tmp1(0, 0);
16344   SDOperand Tmp2(0, 0);
16345   SDOperand Tmp3(0, 0);
16346   SDNode *ResNode = NULL;
16347   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
16348   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
16349   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
16350   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
16351   Select(Tmp0, CPTmp0);
16352   Select(Tmp1, CPTmp1);
16353   Select(Tmp2, CPTmp2);
16354   Select(Tmp3, CPTmp3);
16355   Select(Chain1, Chain1);
16356   Select(N0, N0);
16357   ResNode = CurDAG->getCopyToReg(Chain1, CurDAG->getRegister(X86::AL, MVT::i8), N0, InFlag).Val;
16358   Chain1 = SDOperand(ResNode, 0);
16359   InFlag = SDOperand(ResNode, 1);
16360   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
16361   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
16362   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
16363   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
16364   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
16365   Chain1 = SDOperand(ResNode, 1);
16366   InFlag = SDOperand(ResNode, 1);
16367   ResNode = CurDAG->getCopyFromReg(Chain1, X86::AL, MVT::i8, InFlag).Val;
16368   Chain1 = SDOperand(ResNode, 1);
16369   InFlag = SDOperand(ResNode, 2);
16370   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16371   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16372   AddHandleReplacement(N1.Val, 1, ResNode, 1);
16373   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16374   if (N.ResNo < 1)
16375     Result = SDOperand(ResNode, N.ResNo);
16376   else
16377     Result = SDOperand(ResNode, N.ResNo+1);
16378   return;
16379 }
16380 void Emit_128(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1) NOINLINE {
16381   SDOperand InFlag(0, 0);
16382   SDOperand Tmp0(0, 0);
16383   SDOperand Tmp1(0, 0);
16384   SDOperand Tmp2(0, 0);
16385   SDOperand Tmp3(0, 0);
16386   SDNode *ResNode = NULL;
16387   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
16388   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
16389   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
16390   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
16391   Select(Tmp0, CPTmp0);
16392   Select(Tmp1, CPTmp1);
16393   Select(Tmp2, CPTmp2);
16394   Select(Tmp3, CPTmp3);
16395   Select(Chain0, Chain0);
16396   Select(N1, N1);
16397   ResNode = CurDAG->getCopyToReg(Chain0, CurDAG->getRegister(X86::AL, MVT::i8), N1, InFlag).Val;
16398   Chain0 = SDOperand(ResNode, 0);
16399   InFlag = SDOperand(ResNode, 1);
16400   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
16401   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
16402   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
16403   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
16404   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain0, InFlag);
16405   Chain0 = SDOperand(ResNode, 1);
16406   InFlag = SDOperand(ResNode, 1);
16407   ResNode = CurDAG->getCopyFromReg(Chain0, X86::AL, MVT::i8, InFlag).Val;
16408   Chain0 = SDOperand(ResNode, 1);
16409   InFlag = SDOperand(ResNode, 2);
16410   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16411   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16412   AddHandleReplacement(N0.Val, 1, ResNode, 1);
16413   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16414   if (N.ResNo < 1)
16415     Result = SDOperand(ResNode, N.ResNo);
16416   else
16417     Result = SDOperand(ResNode, N.ResNo+1);
16418   return;
16419 }
16420 void Emit_129(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N0, SDOperand &N1) NOINLINE {
16421   SDOperand InFlag(0, 0);
16422   SDOperand Tmp0(0, 0);
16423   SDNode *ResNode = NULL;
16424   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
16425   Select(Tmp0, N1);
16426   Chain = CurDAG->getEntryNode();
16427   Select(N0, N0);
16428   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::AL, MVT::i8), N0, InFlag).Val;
16429   Chain = SDOperand(ResNode, 0);
16430   InFlag = SDOperand(ResNode, 1);
16431   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
16432   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, InFlag);
16433   InFlag = SDOperand(ResNode, 0);
16434   ResNode = CurDAG->getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).Val;
16435   Chain = SDOperand(ResNode, 1);
16436   InFlag = SDOperand(ResNode, 2);
16437   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16438   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16439   Result = SDOperand(ResNode, N.ResNo);
16440   return;
16441 }
16442 void Emit_130(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &Chain, SDOperand &N0, SDOperand &N1) NOINLINE {
16443   SDOperand InFlag(0, 0);
16444   SDOperand Tmp0(0, 0);
16445   SDNode *ResNode = NULL;
16446   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
16447   Select(Tmp0, N0);
16448   Chain = CurDAG->getEntryNode();
16449   Select(N1, N1);
16450   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::AL, MVT::i8), N1, InFlag).Val;
16451   Chain = SDOperand(ResNode, 0);
16452   InFlag = SDOperand(ResNode, 1);
16453   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
16454   ResNode = CurDAG->getTargetNode(Opc0, MVT::Flag, Tmp0, InFlag);
16455   InFlag = SDOperand(ResNode, 0);
16456   ResNode = CurDAG->getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).Val;
16457   Chain = SDOperand(ResNode, 1);
16458   InFlag = SDOperand(ResNode, 2);
16459   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16460   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16461   Result = SDOperand(ResNode, N.ResNo);
16462   return;
16463 }
16464 void Select_mul(SDOperand &Result, SDOperand N) {
16465   SDOperand CPTmp0(0, 0);
16466   SDOperand CPTmp1(0, 0);
16467   SDOperand CPTmp2(0, 0);
16468   SDOperand CPTmp3(0, 0);
16469   SDOperand Chain(0, 0);
16470   SDOperand Chain0(0, 0);
16471   SDOperand Chain00(0, 0);
16472   SDOperand Chain1(0, 0);
16473   SDOperand Chain10(0, 0);
16474   SDOperand InFlag(0, 0);
16475   SDOperand N0(0, 0);
16476   SDOperand N00(0, 0);
16477   SDOperand N001(0, 0);
16478   SDOperand N01(0, 0);
16479   SDOperand N1(0, 0);
16480   SDOperand N10(0, 0);
16481   SDOperand N101(0, 0);
16482   SDOperand N11(0, 0);
16483   SDOperand Tmp0(0, 0);
16484   SDOperand Tmp1(0, 0);
16485   SDOperand Tmp2(0, 0);
16486   SDOperand Tmp3(0, 0);
16487   SDOperand Tmp4(0, 0);
16488   SDNode *ResNode = NULL;
16489   {
16490     N0 = N.getOperand(0);
16491     if (N0.getOpcode() == ISD::LOAD &&
16492         InFlightSet.count(N0.Val) == 0 &&
16493         N0.hasOneUse() &&
16494         !CodeGenMap.count(N0.getValue(0)) &&
16495         !isNonImmUse(N.Val, N0.Val)) {
16496       Chain0 = N0.getOperand(0);
16497       N01 = N0.getOperand(1);
16498       N1 = N.getOperand(1);
16499       if (N1.getOpcode() == ISD::Constant) {
16500 
16501         // Pattern: (mul:i16 (load:i16 addr:iPTR:$src1), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
16502         // Emits: (IMUL16rmi8:i16 addr:iPTR:$src1, (imm:i16):$src2)
16503         // Pattern complexity = 17  cost = 1  size = 3
16504         if (Predicate_i16immSExt8(N1.Val) &&
16505             N.Val->getValueType(0) == MVT::i16) {
16506           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16507           if (Match) {
16508             Emit_125(Result, N, X86::IMUL16rmi8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16509             return;
16510           }
16511         }
16512 
16513         // Pattern: (mul:i32 (load:i32 addr:iPTR:$src1), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
16514         // Emits: (IMUL32rmi8:i32 addr:iPTR:$src1, (imm:i32):$src2)
16515         // Pattern complexity = 17  cost = 1  size = 3
16516         if (Predicate_i32immSExt8(N1.Val) &&
16517             N.Val->getValueType(0) == MVT::i32) {
16518           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16519           if (Match) {
16520             Emit_126(Result, N, X86::IMUL32rmi8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16521             return;
16522           }
16523         }
16524 
16525         // Pattern: (mul:i16 (load:i16 addr:iPTR:$src1), (imm:i16):$src2)
16526         // Emits: (IMUL16rmi:i16 addr:iPTR:$src1, (imm:i16):$src2)
16527         // Pattern complexity = 16  cost = 1  size = 3
16528         if (N.Val->getValueType(0) == MVT::i16) {
16529           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16530           if (Match) {
16531             Emit_125(Result, N, X86::IMUL16rmi, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16532             return;
16533           }
16534         }
16535 
16536         // Pattern: (mul:i32 (load:i32 addr:iPTR:$src1), (imm:i32):$src2)
16537         // Emits: (IMUL32rmi:i32 addr:iPTR:$src1, (imm:i32):$src2)
16538         // Pattern complexity = 16  cost = 1
16539         if (N.Val->getValueType(0) == MVT::i32) {
16540           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16541           if (Match) {
16542             Emit_126(Result, N, X86::IMUL32rmi, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16543             return;
16544           }
16545         }
16546       }
16547     }
16548   }
16549   if ((Subtarget->hasSSE2())) {
16550     N0 = N.getOperand(0);
16551 
16552     // Pattern: (mul:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
16553     // Emits: (PMULLWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
16554     // Pattern complexity = 16  cost = 1  size = 3
16555     {
16556       N1 = N.getOperand(1);
16557       if (N1.getOpcode() == ISD::BIT_CONVERT &&
16558           InFlightSet.count(N1.Val) == 0 &&
16559           N1.hasOneUse() &&
16560           !CodeGenMap.count(N1.getValue(0))) {
16561         N10 = N1.getOperand(0);
16562         if (N10.getOpcode() == ISD::LOAD &&
16563             InFlightSet.count(N10.Val) == 0 &&
16564             N10.hasOneUse() &&
16565             !CodeGenMap.count(N10.getValue(0))) {
16566           Chain10 = N10.getOperand(0);
16567           N101 = N10.getOperand(1);
16568           if (N.Val->getValueType(0) == MVT::v8i16 &&
16569               N10.Val->getValueType(0) == MVT::v2i64) {
16570             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
16571             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16572             if (!Match) {
16573               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
16574             }
16575             if (Match) {
16576               Emit_40(Result, N, X86::PMULLWrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
16577               return;
16578             }
16579           }
16580         }
16581       }
16582     }
16583 
16584     // Pattern: (mul:v8i16 (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)), VR128:v8i16:$src1)
16585     // Emits: (PMULLWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
16586     // Pattern complexity = 16  cost = 1
16587     if (N0.getOpcode() == ISD::BIT_CONVERT &&
16588         InFlightSet.count(N0.Val) == 0 &&
16589         N0.hasOneUse() &&
16590         !CodeGenMap.count(N0.getValue(0))) {
16591       N00 = N0.getOperand(0);
16592       if (N00.getOpcode() == ISD::LOAD &&
16593           InFlightSet.count(N00.Val) == 0 &&
16594           N00.hasOneUse() &&
16595           !CodeGenMap.count(N00.getValue(0))) {
16596         Chain00 = N00.getOperand(0);
16597         N001 = N00.getOperand(1);
16598         N1 = N.getOperand(1);
16599         if (N.Val->getValueType(0) == MVT::v8i16 &&
16600             N00.Val->getValueType(0) == MVT::v2i64) {
16601           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
16602           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16603           if (!Match) {
16604             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
16605           }
16606           if (Match) {
16607             Emit_41(Result, N, X86::PMULLWrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
16608             return;
16609           }
16610         }
16611       }
16612     }
16613   }
16614   {
16615     N0 = N.getOperand(0);
16616     {
16617       N1 = N.getOperand(1);
16618       if (N1.getOpcode() == ISD::LOAD &&
16619           InFlightSet.count(N1.Val) == 0 &&
16620           N1.hasOneUse() &&
16621           !CodeGenMap.count(N1.getValue(0)) &&
16622           !isNonImmUse(N.Val, N1.Val)) {
16623         Chain1 = N1.getOperand(0);
16624         N11 = N1.getOperand(1);
16625 
16626         // Pattern: (mul:i8 AL:i8, (load:i8 addr:iPTR:$src))
16627         // Emits: (MUL8m:void addr:iPTR:$src)
16628         // Pattern complexity = 14  cost = 1  size = 3
16629         if (N.Val->getValueType(0) == MVT::i8) {
16630           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16631           if (Match) {
16632             Emit_127(Result, N, X86::MUL8m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
16633             return;
16634           }
16635         }
16636 
16637         // Pattern: (mul:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
16638         // Emits: (IMUL16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
16639         // Pattern complexity = 14  cost = 1  size = 3
16640         if (N.Val->getValueType(0) == MVT::i16) {
16641           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
16642           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16643           if (!Match) {
16644             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
16645           }
16646           if (Match) {
16647             Emit_19(Result, N, X86::IMUL16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
16648             return;
16649           }
16650         }
16651 
16652         // Pattern: (mul:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
16653         // Emits: (IMUL32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
16654         // Pattern complexity = 14  cost = 1
16655         if (N.Val->getValueType(0) == MVT::i32) {
16656           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
16657           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16658           if (!Match) {
16659             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
16660           }
16661           if (Match) {
16662             Emit_19(Result, N, X86::IMUL32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
16663             return;
16664           }
16665         }
16666       }
16667     }
16668     if (N0.getOpcode() == ISD::LOAD &&
16669         InFlightSet.count(N0.Val) == 0 &&
16670         N0.hasOneUse() &&
16671         !CodeGenMap.count(N0.getValue(0)) &&
16672         !isNonImmUse(N.Val, N0.Val)) {
16673       Chain0 = N0.getOperand(0);
16674       N01 = N0.getOperand(1);
16675       N1 = N.getOperand(1);
16676 
16677       // Pattern: (mul:i8 (load:i8 addr:iPTR:$src), AL:i8)
16678       // Emits: (MUL8m:void addr:iPTR:$src)
16679       // Pattern complexity = 14  cost = 1  size = 3
16680       if (N.Val->getValueType(0) == MVT::i8) {
16681         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16682         if (Match) {
16683           Emit_128(Result, N, X86::MUL8m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16684           return;
16685         }
16686       }
16687 
16688       // Pattern: (mul:i16 (load:i16 addr:iPTR:$src2), GR16:i16:$src1)
16689       // Emits: (IMUL16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
16690       // Pattern complexity = 14  cost = 1  size = 3
16691       if (N.Val->getValueType(0) == MVT::i16) {
16692         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
16693         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16694         if (!Match) {
16695           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
16696         }
16697         if (Match) {
16698           Emit_20(Result, N, X86::IMUL16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16699           return;
16700         }
16701       }
16702 
16703       // Pattern: (mul:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
16704       // Emits: (IMUL32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
16705       // Pattern complexity = 14  cost = 1
16706       if (N.Val->getValueType(0) == MVT::i32) {
16707         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
16708         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16709         if (!Match) {
16710           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
16711         }
16712         if (Match) {
16713           Emit_20(Result, N, X86::IMUL32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
16714           return;
16715         }
16716       }
16717     }
16718   }
16719 
16720   // Pattern: leaaddr:i32:$src
16721   // Emits: (LEA32r:i32 leaaddr:i32:$src)
16722   // Pattern complexity = 10  cost = 1  size = 3
16723   if (N.Val->getValueType(0) == MVT::i32) {
16724     bool Match = SelectLEAAddr(N, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16725     if (Match) {
16726       Emit_42(Result, N, X86::LEA32r, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16727       return;
16728     }
16729   }
16730   {
16731     N0 = N.getOperand(0);
16732     N1 = N.getOperand(1);
16733     if (N1.getOpcode() == ISD::Constant) {
16734 
16735       // Pattern: (mul:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
16736       // Emits: (IMUL16rri8:i16 GR16:i16:$src1, (imm:i16):$src2)
16737       // Pattern complexity = 5  cost = 1  size = 3
16738       if (Predicate_i16immSExt8(N1.Val) &&
16739           N.Val->getValueType(0) == MVT::i16) {
16740         Emit_49(Result, N, X86::IMUL16rri8, MVT::i16, N0, N1);
16741         return;
16742       }
16743 
16744       // Pattern: (mul:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
16745       // Emits: (IMUL32rri8:i32 GR32:i32:$src1, (imm:i32):$src2)
16746       // Pattern complexity = 5  cost = 1  size = 3
16747       if (Predicate_i32immSExt8(N1.Val) &&
16748           N.Val->getValueType(0) == MVT::i32) {
16749         Emit_29(Result, N, X86::IMUL32rri8, MVT::i32, N0, N1);
16750         return;
16751       }
16752 
16753       // Pattern: (mul:i16 GR16:i16:$src1, (imm:i16):$src2)
16754       // Emits: (IMUL16rri:i16 GR16:i16:$src1, (imm:i16):$src2)
16755       // Pattern complexity = 4  cost = 1  size = 3
16756       if (N.Val->getValueType(0) == MVT::i16) {
16757         Emit_49(Result, N, X86::IMUL16rri, MVT::i16, N0, N1);
16758         return;
16759       }
16760 
16761       // Pattern: (mul:i32 GR32:i32:$src1, (imm:i32):$src2)
16762       // Emits: (IMUL32rri:i32 GR32:i32:$src1, (imm:i32):$src2)
16763       // Pattern complexity = 4  cost = 1
16764       if (N.Val->getValueType(0) == MVT::i32) {
16765         Emit_29(Result, N, X86::IMUL32rri, MVT::i32, N0, N1);
16766         return;
16767       }
16768     }
16769 
16770     // Pattern: (mul:i8 AL:i8, GR8:i8:$src)
16771     // Emits: (MUL8r:void GR8:i8:$src)
16772     // Pattern complexity = 2  cost = 1  size = 3
16773     if (N.Val->getValueType(0) == MVT::i8) {
16774       Emit_129(Result, N, X86::MUL8r, Chain, N0, N1);
16775       return;
16776     }
16777 
16778     // Pattern: (mul:i16 GR16:i16:$src1, GR16:i16:$src2)
16779     // Emits: (IMUL16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
16780     // Pattern complexity = 2  cost = 1  size = 3
16781     if (N.Val->getValueType(0) == MVT::i16) {
16782       Emit_21(Result, N, X86::IMUL16rr, MVT::i16, N0, N1);
16783       return;
16784     }
16785 
16786     // Pattern: (mul:i32 GR32:i32:$src1, GR32:i32:$src2)
16787     // Emits: (IMUL32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
16788     // Pattern complexity = 2  cost = 1
16789     if (N.Val->getValueType(0) == MVT::i32) {
16790       Emit_21(Result, N, X86::IMUL32rr, MVT::i32, N0, N1);
16791       return;
16792     }
16793   }
16794 
16795   // Pattern: (mul:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16796   // Emits: (PMULLWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16797   // Pattern complexity = 2  cost = 1  size = 3
16798   if ((Subtarget->hasSSE2())) {
16799     N0 = N.getOperand(0);
16800     N1 = N.getOperand(1);
16801     if (N.Val->getValueType(0) == MVT::v8i16) {
16802       Emit_21(Result, N, X86::PMULLWrr, MVT::v8i16, N0, N1);
16803       return;
16804     }
16805   }
16806 
16807   // Pattern: (mul:i8 GR8:i8:$src, AL:i8)
16808   // Emits: (MUL8r:void GR8:i8:$src)
16809   // Pattern complexity = 2  cost = 1
16810   N0 = N.getOperand(0);
16811   N1 = N.getOperand(1);
16812   if (N.Val->getValueType(0) == MVT::i8) {
16813     Emit_130(Result, N, X86::MUL8r, Chain, N0, N1);
16814     return;
16815   }
16816   std::cerr << "Cannot yet select: ";
16817   N.Val->dump(CurDAG);
16818   std::cerr << '\n';
16819   abort();
16820 }
16821 
16822 void Emit_131(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &Chain, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N110, SDOperand &N111) NOINLINE {
16823   SDOperand InFlag(0, 0);
16824   SDOperand Tmp1(0, 0);
16825   SDOperand Tmp2(0, 0);
16826   SDNode *ResNode = NULL;
16827   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
16828   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
16829   Select(Tmp1, N00);
16830   Select(Tmp2, N10);
16831   Chain = CurDAG->getEntryNode();
16832   Select(N01, N01);
16833   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N01, InFlag).Val;
16834   Chain = SDOperand(ResNode, 0);
16835   InFlag = SDOperand(ResNode, 1);
16836   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
16837   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
16838   if (N.Val->hasOneUse()) {
16839     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp2, InFlag);
16840   } else {
16841     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp2, InFlag);
16842     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16843     Result = SDOperand(ResNode, 0);
16844   }
16845   return;
16846 }
16847 void Emit_132(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &Chain, SDOperand &N0, SDOperand &N00, SDOperand &N01, SDOperand &N010, SDOperand &N011, SDOperand &N1, SDOperand &N10, SDOperand &N11) NOINLINE {
16848   SDOperand InFlag(0, 0);
16849   SDOperand Tmp1(0, 0);
16850   SDOperand Tmp2(0, 0);
16851   SDNode *ResNode = NULL;
16852   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
16853   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
16854   Select(Tmp1, N10);
16855   Select(Tmp2, N00);
16856   Chain = CurDAG->getEntryNode();
16857   Select(N011, N011);
16858   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N011, InFlag).Val;
16859   Chain = SDOperand(ResNode, 0);
16860   InFlag = SDOperand(ResNode, 1);
16861   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
16862   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
16863   if (N.Val->hasOneUse()) {
16864     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp2, InFlag);
16865   } else {
16866     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp2, InFlag);
16867     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16868     Result = SDOperand(ResNode, 0);
16869   }
16870   return;
16871 }
16872 void Select_or(SDOperand &Result, SDOperand N) {
16873   SDOperand CPTmp0(0, 0);
16874   SDOperand CPTmp1(0, 0);
16875   SDOperand CPTmp2(0, 0);
16876   SDOperand CPTmp3(0, 0);
16877   SDOperand Chain(0, 0);
16878   SDOperand Chain0(0, 0);
16879   SDOperand Chain00(0, 0);
16880   SDOperand Chain1(0, 0);
16881   SDOperand Chain10(0, 0);
16882   SDOperand InFlag(0, 0);
16883   SDOperand N0(0, 0);
16884   SDOperand N00(0, 0);
16885   SDOperand N001(0, 0);
16886   SDOperand N01(0, 0);
16887   SDOperand N010(0, 0);
16888   SDOperand N011(0, 0);
16889   SDOperand N1(0, 0);
16890   SDOperand N10(0, 0);
16891   SDOperand N101(0, 0);
16892   SDOperand N11(0, 0);
16893   SDOperand N110(0, 0);
16894   SDOperand N111(0, 0);
16895   SDOperand Tmp0(0, 0);
16896   SDOperand Tmp1(0, 0);
16897   SDOperand Tmp2(0, 0);
16898   SDOperand Tmp3(0, 0);
16899   SDOperand Tmp4(0, 0);
16900   SDNode *ResNode = NULL;
16901   if ((Subtarget->hasSSE2())) {
16902     N0 = N.getOperand(0);
16903     if (N0.getOpcode() == ISD::BIT_CONVERT) {
16904 
16905       // Pattern: (or:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)))
16906       // Emits: (ORPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
16907       // Pattern complexity = 18  cost = 1  size = 3
16908       {
16909         N00 = N0.getOperand(0);
16910         N1 = N.getOperand(1);
16911         if (N1.getOpcode() == ISD::BIT_CONVERT &&
16912             InFlightSet.count(N1.Val) == 0 &&
16913             N1.hasOneUse() &&
16914             !CodeGenMap.count(N1.getValue(0))) {
16915           N10 = N1.getOperand(0);
16916           if (N10.getOpcode() == ISD::LOAD &&
16917               InFlightSet.count(N10.Val) == 0 &&
16918               N10.hasOneUse() &&
16919               !CodeGenMap.count(N10.getValue(0))) {
16920             Chain10 = N10.getOperand(0);
16921             N101 = N10.getOperand(1);
16922             if (N.Val->getValueType(0) == MVT::v2i64 &&
16923                 N00.Val->getValueType(0) == MVT::v2f64 &&
16924                 N10.Val->getValueType(0) == MVT::v2f64) {
16925               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
16926               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16927               if (!Match) {
16928                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
16929               }
16930               if (Match) {
16931                 Emit_75(Result, N, X86::ORPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N1, N10, N101);
16932                 return;
16933               }
16934             }
16935           }
16936         }
16937       }
16938 
16939       // Pattern: (or:v2i64 (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
16940       // Emits: (ORPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
16941       // Pattern complexity = 18  cost = 1
16942       if (InFlightSet.count(N0.Val) == 0 &&
16943           N0.hasOneUse() &&
16944           !CodeGenMap.count(N0.getValue(0))) {
16945         N00 = N0.getOperand(0);
16946         if (N00.getOpcode() == ISD::LOAD &&
16947             InFlightSet.count(N00.Val) == 0 &&
16948             N00.hasOneUse() &&
16949             !CodeGenMap.count(N00.getValue(0))) {
16950           Chain00 = N00.getOperand(0);
16951           N001 = N00.getOperand(1);
16952           N1 = N.getOperand(1);
16953           if (N1.getOpcode() == ISD::BIT_CONVERT) {
16954             N10 = N1.getOperand(0);
16955             if (N.Val->getValueType(0) == MVT::v2i64 &&
16956                 N00.Val->getValueType(0) == MVT::v2f64 &&
16957                 N10.Val->getValueType(0) == MVT::v2f64) {
16958               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
16959               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16960               if (!Match) {
16961                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
16962               }
16963               if (Match) {
16964                 Emit_76(Result, N, X86::ORPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10);
16965                 return;
16966               }
16967             }
16968           }
16969         }
16970       }
16971     }
16972   }
16973   if ((Subtarget->hasSSE1())) {
16974     N0 = N.getOperand(0);
16975 
16976     // Pattern: (or:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)))
16977     // Emits: (ORPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
16978     // Pattern complexity = 16  cost = 1  size = 3
16979     {
16980       N1 = N.getOperand(1);
16981       if (N1.getOpcode() == ISD::BIT_CONVERT &&
16982           InFlightSet.count(N1.Val) == 0 &&
16983           N1.hasOneUse() &&
16984           !CodeGenMap.count(N1.getValue(0))) {
16985         N10 = N1.getOperand(0);
16986         if (N10.getOpcode() == ISD::LOAD &&
16987             InFlightSet.count(N10.Val) == 0 &&
16988             N10.hasOneUse() &&
16989             !CodeGenMap.count(N10.getValue(0))) {
16990           Chain10 = N10.getOperand(0);
16991           N101 = N10.getOperand(1);
16992           if (N.Val->getValueType(0) == MVT::v2i64 &&
16993               N10.Val->getValueType(0) == MVT::v4f32) {
16994             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
16995             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
16996             if (!Match) {
16997               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
16998             }
16999             if (Match) {
17000               Emit_40(Result, N, X86::ORPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
17001               return;
17002             }
17003           }
17004         }
17005       }
17006     }
17007 
17008     // Pattern: (or:v2i64 (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)), VR128:v2i64:$src1)
17009     // Emits: (ORPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
17010     // Pattern complexity = 16  cost = 1
17011     if (N0.getOpcode() == ISD::BIT_CONVERT &&
17012         InFlightSet.count(N0.Val) == 0 &&
17013         N0.hasOneUse() &&
17014         !CodeGenMap.count(N0.getValue(0))) {
17015       N00 = N0.getOperand(0);
17016       if (N00.getOpcode() == ISD::LOAD &&
17017           InFlightSet.count(N00.Val) == 0 &&
17018           N00.hasOneUse() &&
17019           !CodeGenMap.count(N00.getValue(0))) {
17020         Chain00 = N00.getOperand(0);
17021         N001 = N00.getOperand(1);
17022         N1 = N.getOperand(1);
17023         if (N.Val->getValueType(0) == MVT::v2i64 &&
17024             N00.Val->getValueType(0) == MVT::v4f32) {
17025           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
17026           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17027           if (!Match) {
17028             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
17029           }
17030           if (Match) {
17031             Emit_41(Result, N, X86::ORPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
17032             return;
17033           }
17034         }
17035       }
17036     }
17037   }
17038   {
17039     N0 = N.getOperand(0);
17040     N1 = N.getOperand(1);
17041     if (N1.getOpcode() == ISD::LOAD &&
17042         InFlightSet.count(N1.Val) == 0 &&
17043         N1.hasOneUse() &&
17044         !CodeGenMap.count(N1.getValue(0)) &&
17045         !isNonImmUse(N.Val, N1.Val)) {
17046       Chain1 = N1.getOperand(0);
17047       N11 = N1.getOperand(1);
17048 
17049       // Pattern: (or:i8 GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
17050       // Emits: (OR8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
17051       // Pattern complexity = 14  cost = 1  size = 3
17052       if (N.Val->getValueType(0) == MVT::i8) {
17053         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
17054         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17055         if (!Match) {
17056           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
17057         }
17058         if (Match) {
17059           Emit_19(Result, N, X86::OR8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
17060           return;
17061         }
17062       }
17063 
17064       // Pattern: (or:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
17065       // Emits: (OR16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
17066       // Pattern complexity = 14  cost = 1  size = 3
17067       if (N.Val->getValueType(0) == MVT::i16) {
17068         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
17069         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17070         if (!Match) {
17071           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
17072         }
17073         if (Match) {
17074           Emit_19(Result, N, X86::OR16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
17075           return;
17076         }
17077       }
17078 
17079       // Pattern: (or:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
17080       // Emits: (OR32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
17081       // Pattern complexity = 14  cost = 1
17082       if (N.Val->getValueType(0) == MVT::i32) {
17083         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
17084         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17085         if (!Match) {
17086           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
17087         }
17088         if (Match) {
17089           Emit_19(Result, N, X86::OR32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
17090           return;
17091         }
17092       }
17093     }
17094   }
17095 
17096   // Pattern: (or:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
17097   // Emits: (PORrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
17098   // Pattern complexity = 14  cost = 1  size = 3
17099   if ((Subtarget->hasSSE2())) {
17100     N0 = N.getOperand(0);
17101     N1 = N.getOperand(1);
17102     if (N1.getOpcode() == ISD::LOAD &&
17103         InFlightSet.count(N1.Val) == 0 &&
17104         N1.hasOneUse() &&
17105         !CodeGenMap.count(N1.getValue(0)) &&
17106         !isNonImmUse(N.Val, N1.Val)) {
17107       Chain1 = N1.getOperand(0);
17108       N11 = N1.getOperand(1);
17109       if (N.Val->getValueType(0) == MVT::v2i64) {
17110         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
17111         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17112         if (!Match) {
17113           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
17114         }
17115         if (Match) {
17116           Emit_19(Result, N, X86::PORrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
17117           return;
17118         }
17119       }
17120     }
17121   }
17122   {
17123     N0 = N.getOperand(0);
17124     if (N0.getOpcode() == ISD::LOAD &&
17125         InFlightSet.count(N0.Val) == 0 &&
17126         N0.hasOneUse() &&
17127         !CodeGenMap.count(N0.getValue(0)) &&
17128         !isNonImmUse(N.Val, N0.Val)) {
17129       Chain0 = N0.getOperand(0);
17130       N01 = N0.getOperand(1);
17131       N1 = N.getOperand(1);
17132 
17133       // Pattern: (or:i8 (load:i8 addr:iPTR:$src2), GR8:i8:$src1)
17134       // Emits: (OR8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
17135       // Pattern complexity = 14  cost = 1  size = 3
17136       if (N.Val->getValueType(0) == MVT::i8) {
17137         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
17138         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17139         if (!Match) {
17140           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
17141         }
17142         if (Match) {
17143           Emit_20(Result, N, X86::OR8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
17144           return;
17145         }
17146       }
17147 
17148       // Pattern: (or:i16 (load:i16 addr:iPTR:$src2), GR16:i16:$src1)
17149       // Emits: (OR16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
17150       // Pattern complexity = 14  cost = 1  size = 3
17151       if (N.Val->getValueType(0) == MVT::i16) {
17152         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
17153         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17154         if (!Match) {
17155           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
17156         }
17157         if (Match) {
17158           Emit_20(Result, N, X86::OR16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
17159           return;
17160         }
17161       }
17162 
17163       // Pattern: (or:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
17164       // Emits: (OR32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
17165       // Pattern complexity = 14  cost = 1
17166       if (N.Val->getValueType(0) == MVT::i32) {
17167         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
17168         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17169         if (!Match) {
17170           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
17171         }
17172         if (Match) {
17173           Emit_20(Result, N, X86::OR32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
17174           return;
17175         }
17176       }
17177     }
17178   }
17179 
17180   // Pattern: (or:v2i64 (load:v2i64 addr:iPTR:$src2), VR128:v2i64:$src1)
17181   // Emits: (PORrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
17182   // Pattern complexity = 14  cost = 1  size = 3
17183   if ((Subtarget->hasSSE2())) {
17184     N0 = N.getOperand(0);
17185     if (N0.getOpcode() == ISD::LOAD &&
17186         InFlightSet.count(N0.Val) == 0 &&
17187         N0.hasOneUse() &&
17188         !CodeGenMap.count(N0.getValue(0)) &&
17189         !isNonImmUse(N.Val, N0.Val)) {
17190       Chain0 = N0.getOperand(0);
17191       N01 = N0.getOperand(1);
17192       N1 = N.getOperand(1);
17193       if (N.Val->getValueType(0) == MVT::v2i64) {
17194         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
17195         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17196         if (!Match) {
17197           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
17198         }
17199         if (Match) {
17200           Emit_20(Result, N, X86::PORrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
17201           return;
17202         }
17203       }
17204     }
17205   }
17206   {
17207     N0 = N.getOperand(0);
17208 
17209     // Pattern: (or:i32 (srl:i32 GR32:i32:$src1, CL:i8:$amt), (shl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)))
17210     // Emits: (SHRD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
17211     // Pattern complexity = 11  cost = 1  size = 3
17212     if (N0.getOpcode() == ISD::SRL) {
17213       N00 = N0.getOperand(0);
17214       N01 = N0.getOperand(1);
17215       N1 = N.getOperand(1);
17216       if (N1.getOpcode() == ISD::SHL) {
17217         N10 = N1.getOperand(0);
17218         N11 = N1.getOperand(1);
17219         if (N11.getOpcode() == ISD::SUB) {
17220           N110 = N11.getOperand(0);
17221           if (isa<ConstantSDNode>(N110)) {
17222             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17223             if (CN0 == 32) {
17224               N111 = N11.getOperand(1);
17225               if (N01 == N111 &&
17226                   N.Val->getValueType(0) == MVT::i32 &&
17227                   N01.Val->getValueType(0) == MVT::i8 &&
17228                   N11.Val->getValueType(0) == MVT::i8) {
17229                 Emit_131(Result, N, X86::SHRD32rrCL, MVT::i32, Chain, N0, N00, N01, N1, N10, N11, N110, N111);
17230                 return;
17231               }
17232             }
17233           }
17234         }
17235       }
17236     }
17237 
17238     // Pattern: (or:i32 (shl:i32 GR32:i32:$src1, CL:i8:$amt), (srl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)))
17239     // Emits: (SHLD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
17240     // Pattern complexity = 11  cost = 1  size = 3
17241     if (N0.getOpcode() == ISD::SHL) {
17242       N00 = N0.getOperand(0);
17243       N01 = N0.getOperand(1);
17244       N1 = N.getOperand(1);
17245       if (N1.getOpcode() == ISD::SRL) {
17246         N10 = N1.getOperand(0);
17247         N11 = N1.getOperand(1);
17248         if (N11.getOpcode() == ISD::SUB) {
17249           N110 = N11.getOperand(0);
17250           if (isa<ConstantSDNode>(N110)) {
17251             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17252             if (CN0 == 32) {
17253               N111 = N11.getOperand(1);
17254               if (N01 == N111 &&
17255                   N.Val->getValueType(0) == MVT::i32 &&
17256                   N01.Val->getValueType(0) == MVT::i8 &&
17257                   N11.Val->getValueType(0) == MVT::i8) {
17258                 Emit_131(Result, N, X86::SHLD32rrCL, MVT::i32, Chain, N0, N00, N01, N1, N10, N11, N110, N111);
17259                 return;
17260               }
17261             }
17262           }
17263         }
17264       }
17265     }
17266 
17267     // Pattern: (or:i16 (srl:i16 GR16:i16:$src1, CL:i8:$amt), (shl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)))
17268     // Emits: (SHRD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
17269     // Pattern complexity = 11  cost = 1  size = 3
17270     if (N0.getOpcode() == ISD::SRL) {
17271       N00 = N0.getOperand(0);
17272       N01 = N0.getOperand(1);
17273       N1 = N.getOperand(1);
17274       if (N1.getOpcode() == ISD::SHL) {
17275         N10 = N1.getOperand(0);
17276         N11 = N1.getOperand(1);
17277         if (N11.getOpcode() == ISD::SUB) {
17278           N110 = N11.getOperand(0);
17279           if (isa<ConstantSDNode>(N110)) {
17280             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17281             if (CN0 == 16) {
17282               N111 = N11.getOperand(1);
17283               if (N01 == N111 &&
17284                   N.Val->getValueType(0) == MVT::i16 &&
17285                   N01.Val->getValueType(0) == MVT::i8 &&
17286                   N11.Val->getValueType(0) == MVT::i8) {
17287                 Emit_131(Result, N, X86::SHRD16rrCL, MVT::i16, Chain, N0, N00, N01, N1, N10, N11, N110, N111);
17288                 return;
17289               }
17290             }
17291           }
17292         }
17293       }
17294     }
17295     if (N0.getOpcode() == ISD::SHL) {
17296       N00 = N0.getOperand(0);
17297       N01 = N0.getOperand(1);
17298 
17299       // Pattern: (or:i16 (shl:i16 GR16:i16:$src1, CL:i8:$amt), (srl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)))
17300       // Emits: (SHLD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
17301       // Pattern complexity = 11  cost = 1  size = 3
17302       {
17303         N1 = N.getOperand(1);
17304         if (N1.getOpcode() == ISD::SRL) {
17305           N10 = N1.getOperand(0);
17306           N11 = N1.getOperand(1);
17307           if (N11.getOpcode() == ISD::SUB) {
17308             N110 = N11.getOperand(0);
17309             if (isa<ConstantSDNode>(N110)) {
17310               int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17311               if (CN0 == 16) {
17312                 N111 = N11.getOperand(1);
17313                 if (N01 == N111 &&
17314                     N.Val->getValueType(0) == MVT::i16 &&
17315                     N01.Val->getValueType(0) == MVT::i8 &&
17316                     N11.Val->getValueType(0) == MVT::i8) {
17317                   Emit_131(Result, N, X86::SHLD16rrCL, MVT::i16, Chain, N0, N00, N01, N1, N10, N11, N110, N111);
17318                   return;
17319                 }
17320               }
17321             }
17322           }
17323         }
17324       }
17325 
17326       // Pattern: (or:i32 (shl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (srl:i32 GR32:i32:$src1, CL:i8:$amt))
17327       // Emits: (SHRD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
17328       // Pattern complexity = 11  cost = 1
17329       if (N01.getOpcode() == ISD::SUB) {
17330         N010 = N01.getOperand(0);
17331         if (isa<ConstantSDNode>(N010)) {
17332           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17333           if (CN0 == 32) {
17334             N011 = N01.getOperand(1);
17335             N1 = N.getOperand(1);
17336             if (N1.getOpcode() == ISD::SRL) {
17337               N10 = N1.getOperand(0);
17338               N11 = N1.getOperand(1);
17339               if (N011 == N11 &&
17340                   N.Val->getValueType(0) == MVT::i32 &&
17341                   N01.Val->getValueType(0) == MVT::i8 &&
17342                   N11.Val->getValueType(0) == MVT::i8) {
17343                 Emit_132(Result, N, X86::SHRD32rrCL, MVT::i32, Chain, N0, N00, N01, N010, N011, N1, N10, N11);
17344                 return;
17345               }
17346             }
17347           }
17348         }
17349       }
17350     }
17351 
17352     // Pattern: (or:i32 (srl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (shl:i32 GR32:i32:$src1, CL:i8:$amt))
17353     // Emits: (SHLD32rrCL:i32 GR32:i32:$src1, GR32:i32:$src2)
17354     // Pattern complexity = 11  cost = 1  size = 3
17355     if (N0.getOpcode() == ISD::SRL) {
17356       N00 = N0.getOperand(0);
17357       N01 = N0.getOperand(1);
17358       if (N01.getOpcode() == ISD::SUB) {
17359         N010 = N01.getOperand(0);
17360         if (isa<ConstantSDNode>(N010)) {
17361           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17362           if (CN0 == 32) {
17363             N011 = N01.getOperand(1);
17364             N1 = N.getOperand(1);
17365             if (N1.getOpcode() == ISD::SHL) {
17366               N10 = N1.getOperand(0);
17367               N11 = N1.getOperand(1);
17368               if (N011 == N11 &&
17369                   N.Val->getValueType(0) == MVT::i32 &&
17370                   N01.Val->getValueType(0) == MVT::i8 &&
17371                   N11.Val->getValueType(0) == MVT::i8) {
17372                 Emit_132(Result, N, X86::SHLD32rrCL, MVT::i32, Chain, N0, N00, N01, N010, N011, N1, N10, N11);
17373                 return;
17374               }
17375             }
17376           }
17377         }
17378       }
17379     }
17380 
17381     // Pattern: (or:i16 (shl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (srl:i16 GR16:i16:$src1, CL:i8:$amt))
17382     // Emits: (SHRD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
17383     // Pattern complexity = 11  cost = 1  size = 3
17384     if (N0.getOpcode() == ISD::SHL) {
17385       N00 = N0.getOperand(0);
17386       N01 = N0.getOperand(1);
17387       if (N01.getOpcode() == ISD::SUB) {
17388         N010 = N01.getOperand(0);
17389         if (isa<ConstantSDNode>(N010)) {
17390           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17391           if (CN0 == 16) {
17392             N011 = N01.getOperand(1);
17393             N1 = N.getOperand(1);
17394             if (N1.getOpcode() == ISD::SRL) {
17395               N10 = N1.getOperand(0);
17396               N11 = N1.getOperand(1);
17397               if (N011 == N11 &&
17398                   N.Val->getValueType(0) == MVT::i16 &&
17399                   N01.Val->getValueType(0) == MVT::i8 &&
17400                   N11.Val->getValueType(0) == MVT::i8) {
17401                 Emit_132(Result, N, X86::SHRD16rrCL, MVT::i16, Chain, N0, N00, N01, N010, N011, N1, N10, N11);
17402                 return;
17403               }
17404             }
17405           }
17406         }
17407       }
17408     }
17409 
17410     // Pattern: (or:i16 (srl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (shl:i16 GR16:i16:$src1, CL:i8:$amt))
17411     // Emits: (SHLD16rrCL:i16 GR16:i16:$src1, GR16:i16:$src2)
17412     // Pattern complexity = 11  cost = 1
17413     if (N0.getOpcode() == ISD::SRL) {
17414       N00 = N0.getOperand(0);
17415       N01 = N0.getOperand(1);
17416       if (N01.getOpcode() == ISD::SUB) {
17417         N010 = N01.getOperand(0);
17418         if (isa<ConstantSDNode>(N010)) {
17419           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17420           if (CN0 == 16) {
17421             N011 = N01.getOperand(1);
17422             N1 = N.getOperand(1);
17423             if (N1.getOpcode() == ISD::SHL) {
17424               N10 = N1.getOperand(0);
17425               N11 = N1.getOperand(1);
17426               if (N011 == N11 &&
17427                   N.Val->getValueType(0) == MVT::i16 &&
17428                   N01.Val->getValueType(0) == MVT::i8 &&
17429                   N11.Val->getValueType(0) == MVT::i8) {
17430                 Emit_132(Result, N, X86::SHLD16rrCL, MVT::i16, Chain, N0, N00, N01, N010, N011, N1, N10, N11);
17431                 return;
17432               }
17433             }
17434           }
17435         }
17436       }
17437     }
17438   }
17439 
17440   // Pattern: leaaddr:i32:$src
17441   // Emits: (LEA32r:i32 leaaddr:i32:$src)
17442   // Pattern complexity = 10  cost = 1  size = 3
17443   if (N.Val->getValueType(0) == MVT::i32) {
17444     bool Match = SelectLEAAddr(N, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17445     if (Match) {
17446       Emit_42(Result, N, X86::LEA32r, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17447       return;
17448     }
17449   }
17450 
17451   // Pattern: (or:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
17452   // Emits: (ORPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
17453   // Pattern complexity = 6  cost = 1  size = 3
17454   if ((Subtarget->hasSSE2())) {
17455     N0 = N.getOperand(0);
17456     if (N0.getOpcode() == ISD::BIT_CONVERT) {
17457       N00 = N0.getOperand(0);
17458       N1 = N.getOperand(1);
17459       if (N1.getOpcode() == ISD::BIT_CONVERT) {
17460         N10 = N1.getOperand(0);
17461         if (N.Val->getValueType(0) == MVT::v2i64 &&
17462             N00.Val->getValueType(0) == MVT::v2f64 &&
17463             N10.Val->getValueType(0) == MVT::v2f64) {
17464           Emit_89(Result, N, X86::ORPDrr, MVT::v2i64, N0, N00, N1, N10);
17465           return;
17466         }
17467       }
17468     }
17469   }
17470   {
17471     N0 = N.getOperand(0);
17472     N1 = N.getOperand(1);
17473     if (N1.getOpcode() == ISD::Constant) {
17474 
17475       // Pattern: (or:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
17476       // Emits: (OR16ri8:i16 GR16:i16:$src1, (imm:i16):$src2)
17477       // Pattern complexity = 5  cost = 1  size = 3
17478       if (Predicate_i16immSExt8(N1.Val) &&
17479           N.Val->getValueType(0) == MVT::i16) {
17480         Emit_49(Result, N, X86::OR16ri8, MVT::i16, N0, N1);
17481         return;
17482       }
17483 
17484       // Pattern: (or:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
17485       // Emits: (OR32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
17486       // Pattern complexity = 5  cost = 1  size = 3
17487       if (Predicate_i32immSExt8(N1.Val) &&
17488           N.Val->getValueType(0) == MVT::i32) {
17489         Emit_29(Result, N, X86::OR32ri8, MVT::i32, N0, N1);
17490         return;
17491       }
17492 
17493       // Pattern: (or:i8 GR8:i8:$src1, (imm:i8):$src2)
17494       // Emits: (OR8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
17495       // Pattern complexity = 4  cost = 1  size = 3
17496       if (N.Val->getValueType(0) == MVT::i8) {
17497         Emit_50(Result, N, X86::OR8ri, MVT::i8, N0, N1);
17498         return;
17499       }
17500 
17501       // Pattern: (or:i16 GR16:i16:$src1, (imm:i16):$src2)
17502       // Emits: (OR16ri:i16 GR16:i16:$src1, (imm:i16):$src2)
17503       // Pattern complexity = 4  cost = 1  size = 3
17504       if (N.Val->getValueType(0) == MVT::i16) {
17505         Emit_49(Result, N, X86::OR16ri, MVT::i16, N0, N1);
17506         return;
17507       }
17508 
17509       // Pattern: (or:i32 GR32:i32:$src1, (imm:i32):$src2)
17510       // Emits: (OR32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
17511       // Pattern complexity = 4  cost = 1
17512       if (N.Val->getValueType(0) == MVT::i32) {
17513         Emit_29(Result, N, X86::OR32ri, MVT::i32, N0, N1);
17514         return;
17515       }
17516     }
17517 
17518     // Pattern: (or:i8 GR8:i8:$src1, GR8:i8:$src2)
17519     // Emits: (OR8rr:i8 GR8:i8:$src1, GR8:i8:$src2)
17520     // Pattern complexity = 2  cost = 1  size = 3
17521     if (N.Val->getValueType(0) == MVT::i8) {
17522       Emit_21(Result, N, X86::OR8rr, MVT::i8, N0, N1);
17523       return;
17524     }
17525 
17526     // Pattern: (or:i16 GR16:i16:$src1, GR16:i16:$src2)
17527     // Emits: (OR16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
17528     // Pattern complexity = 2  cost = 1  size = 3
17529     if (N.Val->getValueType(0) == MVT::i16) {
17530       Emit_21(Result, N, X86::OR16rr, MVT::i16, N0, N1);
17531       return;
17532     }
17533 
17534     // Pattern: (or:i32 GR32:i32:$src1, GR32:i32:$src2)
17535     // Emits: (OR32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
17536     // Pattern complexity = 2  cost = 1
17537     if (N.Val->getValueType(0) == MVT::i32) {
17538       Emit_21(Result, N, X86::OR32rr, MVT::i32, N0, N1);
17539       return;
17540     }
17541   }
17542 
17543   // Pattern: (or:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17544   // Emits: (ORPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17545   // Pattern complexity = 2  cost = 1  size = 3
17546   if ((Subtarget->hasSSE1())) {
17547     N0 = N.getOperand(0);
17548     N1 = N.getOperand(1);
17549     if (N.Val->getValueType(0) == MVT::v2i64) {
17550       Emit_21(Result, N, X86::ORPSrr, MVT::v2i64, N0, N1);
17551       return;
17552     }
17553   }
17554 
17555   // Pattern: (or:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17556   // Emits: (PORrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17557   // Pattern complexity = 2  cost = 1
17558   if ((Subtarget->hasSSE2())) {
17559     N0 = N.getOperand(0);
17560     N1 = N.getOperand(1);
17561     if (N.Val->getValueType(0) == MVT::v2i64) {
17562       Emit_21(Result, N, X86::PORrr, MVT::v2i64, N0, N1);
17563       return;
17564     }
17565   }
17566   std::cerr << "Cannot yet select: ";
17567   N.Val->dump(CurDAG);
17568   std::cerr << '\n';
17569   abort();
17570 }
17571 
17572 void Emit_133(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &Chain, SDOperand &N0, SDOperand &N1) NOINLINE {
17573   SDOperand InFlag(0, 0);
17574   SDOperand Tmp0(0, 0);
17575   SDNode *ResNode = NULL;
17576   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
17577   Select(Tmp0, N0);
17578   Chain = CurDAG->getEntryNode();
17579   Select(N1, N1);
17580   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17581   Chain = SDOperand(ResNode, 0);
17582   InFlag = SDOperand(ResNode, 1);
17583   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
17584   if (N.Val->hasOneUse()) {
17585     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, InFlag);
17586   } else {
17587     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, InFlag);
17588     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17589     Result = SDOperand(ResNode, 0);
17590   }
17591   return;
17592 }
17593 void Select_rotl(SDOperand &Result, SDOperand N) {
17594   SDOperand Chain(0, 0);
17595   SDOperand InFlag(0, 0);
17596   SDOperand N0(0, 0);
17597   SDOperand N1(0, 0);
17598   SDOperand Tmp0(0, 0);
17599   SDOperand Tmp1(0, 0);
17600   SDNode *ResNode = NULL;
17601   N0 = N.getOperand(0);
17602   N1 = N.getOperand(1);
17603   if (isa<ConstantSDNode>(N1)) {
17604     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
17605     if (CN0 == 1) {
17606 
17607       // Pattern: (rotl:i8 GR8:i8:$src1, 1:i8)
17608       // Emits: (ROL8r1:i8 GR8:i8:$src1)
17609       // Pattern complexity = 5  cost = 1  size = 3
17610       if (N.Val->getValueType(0) == MVT::i8 &&
17611           N1.Val->getValueType(0) == MVT::i8) {
17612         Emit_47(Result, N, X86::ROL8r1, MVT::i8, N0, N1);
17613         return;
17614       }
17615 
17616       // Pattern: (rotl:i16 GR16:i16:$src1, 1:i8)
17617       // Emits: (ROL16r1:i16 GR16:i16:$src1)
17618       // Pattern complexity = 5  cost = 1  size = 3
17619       if (N.Val->getValueType(0) == MVT::i16 &&
17620           N1.Val->getValueType(0) == MVT::i8) {
17621         Emit_47(Result, N, X86::ROL16r1, MVT::i16, N0, N1);
17622         return;
17623       }
17624 
17625       // Pattern: (rotl:i32 GR32:i32:$src1, 1:i8)
17626       // Emits: (ROL32r1:i32 GR32:i32:$src1)
17627       // Pattern complexity = 5  cost = 1
17628       if (N.Val->getValueType(0) == MVT::i32 &&
17629           N1.Val->getValueType(0) == MVT::i8) {
17630         Emit_47(Result, N, X86::ROL32r1, MVT::i32, N0, N1);
17631         return;
17632       }
17633     }
17634   }
17635   if (N1.getOpcode() == ISD::Constant) {
17636 
17637     // Pattern: (rotl:i8 GR8:i8:$src1, (imm:i8):$src2)
17638     // Emits: (ROL8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
17639     // Pattern complexity = 4  cost = 1  size = 3
17640     if (N.Val->getValueType(0) == MVT::i8 &&
17641         N1.Val->getValueType(0) == MVT::i8) {
17642       Emit_50(Result, N, X86::ROL8ri, MVT::i8, N0, N1);
17643       return;
17644     }
17645 
17646     // Pattern: (rotl:i16 GR16:i16:$src1, (imm:i8):$src2)
17647     // Emits: (ROL16ri:i16 GR16:i16:$src1, (imm:i8):$src2)
17648     // Pattern complexity = 4  cost = 1  size = 3
17649     if (N.Val->getValueType(0) == MVT::i16 &&
17650         N1.Val->getValueType(0) == MVT::i8) {
17651       Emit_50(Result, N, X86::ROL16ri, MVT::i16, N0, N1);
17652       return;
17653     }
17654 
17655     // Pattern: (rotl:i32 GR32:i32:$src1, (imm:i8):$src2)
17656     // Emits: (ROL32ri:i32 GR32:i32:$src1, (imm:i8):$src2)
17657     // Pattern complexity = 4  cost = 1
17658     if (N.Val->getValueType(0) == MVT::i32 &&
17659         N1.Val->getValueType(0) == MVT::i8) {
17660       Emit_50(Result, N, X86::ROL32ri, MVT::i32, N0, N1);
17661       return;
17662     }
17663   }
17664 
17665   // Pattern: (rotl:i8 GR8:i8:$src, CL:i8)
17666   // Emits: (ROL8rCL:i8 GR8:i8:$src)
17667   // Pattern complexity = 2  cost = 1  size = 3
17668   if (N.Val->getValueType(0) == MVT::i8 &&
17669       N1.Val->getValueType(0) == MVT::i8) {
17670     Emit_133(Result, N, X86::ROL8rCL, MVT::i8, Chain, N0, N1);
17671     return;
17672   }
17673 
17674   // Pattern: (rotl:i16 GR16:i16:$src, CL:i8)
17675   // Emits: (ROL16rCL:i16 GR16:i16:$src)
17676   // Pattern complexity = 2  cost = 1  size = 3
17677   if (N.Val->getValueType(0) == MVT::i16 &&
17678       N1.Val->getValueType(0) == MVT::i8) {
17679     Emit_133(Result, N, X86::ROL16rCL, MVT::i16, Chain, N0, N1);
17680     return;
17681   }
17682 
17683   // Pattern: (rotl:i32 GR32:i32:$src, CL:i8)
17684   // Emits: (ROL32rCL:i32 GR32:i32:$src)
17685   // Pattern complexity = 2  cost = 1
17686   if (N.Val->getValueType(0) == MVT::i32 &&
17687       N1.Val->getValueType(0) == MVT::i8) {
17688     Emit_133(Result, N, X86::ROL32rCL, MVT::i32, Chain, N0, N1);
17689     return;
17690   }
17691   std::cerr << "Cannot yet select: ";
17692   N.Val->dump(CurDAG);
17693   std::cerr << '\n';
17694   abort();
17695 }
17696 
17697 void Select_rotr(SDOperand &Result, SDOperand N) {
17698   SDOperand Chain(0, 0);
17699   SDOperand InFlag(0, 0);
17700   SDOperand N0(0, 0);
17701   SDOperand N1(0, 0);
17702   SDOperand Tmp0(0, 0);
17703   SDOperand Tmp1(0, 0);
17704   SDNode *ResNode = NULL;
17705   N0 = N.getOperand(0);
17706   N1 = N.getOperand(1);
17707   if (isa<ConstantSDNode>(N1)) {
17708     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
17709     if (CN0 == 1) {
17710 
17711       // Pattern: (rotr:i8 GR8:i8:$src1, 1:i8)
17712       // Emits: (ROR8r1:i8 GR8:i8:$src1)
17713       // Pattern complexity = 5  cost = 1  size = 3
17714       if (N.Val->getValueType(0) == MVT::i8 &&
17715           N1.Val->getValueType(0) == MVT::i8) {
17716         Emit_47(Result, N, X86::ROR8r1, MVT::i8, N0, N1);
17717         return;
17718       }
17719 
17720       // Pattern: (rotr:i16 GR16:i16:$src1, 1:i8)
17721       // Emits: (ROR16r1:i16 GR16:i16:$src1)
17722       // Pattern complexity = 5  cost = 1  size = 3
17723       if (N.Val->getValueType(0) == MVT::i16 &&
17724           N1.Val->getValueType(0) == MVT::i8) {
17725         Emit_47(Result, N, X86::ROR16r1, MVT::i16, N0, N1);
17726         return;
17727       }
17728 
17729       // Pattern: (rotr:i32 GR32:i32:$src1, 1:i8)
17730       // Emits: (ROR32r1:i32 GR32:i32:$src1)
17731       // Pattern complexity = 5  cost = 1
17732       if (N.Val->getValueType(0) == MVT::i32 &&
17733           N1.Val->getValueType(0) == MVT::i8) {
17734         Emit_47(Result, N, X86::ROR32r1, MVT::i32, N0, N1);
17735         return;
17736       }
17737     }
17738   }
17739   if (N1.getOpcode() == ISD::Constant) {
17740 
17741     // Pattern: (rotr:i8 GR8:i8:$src1, (imm:i8):$src2)
17742     // Emits: (ROR8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
17743     // Pattern complexity = 4  cost = 1  size = 3
17744     if (N.Val->getValueType(0) == MVT::i8 &&
17745         N1.Val->getValueType(0) == MVT::i8) {
17746       Emit_50(Result, N, X86::ROR8ri, MVT::i8, N0, N1);
17747       return;
17748     }
17749 
17750     // Pattern: (rotr:i16 GR16:i16:$src1, (imm:i8):$src2)
17751     // Emits: (ROR16ri:i16 GR16:i16:$src1, (imm:i8):$src2)
17752     // Pattern complexity = 4  cost = 1  size = 3
17753     if (N.Val->getValueType(0) == MVT::i16 &&
17754         N1.Val->getValueType(0) == MVT::i8) {
17755       Emit_50(Result, N, X86::ROR16ri, MVT::i16, N0, N1);
17756       return;
17757     }
17758 
17759     // Pattern: (rotr:i32 GR32:i32:$src1, (imm:i8):$src2)
17760     // Emits: (ROR32ri:i32 GR32:i32:$src1, (imm:i8):$src2)
17761     // Pattern complexity = 4  cost = 1
17762     if (N.Val->getValueType(0) == MVT::i32 &&
17763         N1.Val->getValueType(0) == MVT::i8) {
17764       Emit_50(Result, N, X86::ROR32ri, MVT::i32, N0, N1);
17765       return;
17766     }
17767   }
17768 
17769   // Pattern: (rotr:i8 GR8:i8:$src, CL:i8)
17770   // Emits: (ROR8rCL:i8 GR8:i8:$src)
17771   // Pattern complexity = 2  cost = 1  size = 3
17772   if (N.Val->getValueType(0) == MVT::i8 &&
17773       N1.Val->getValueType(0) == MVT::i8) {
17774     Emit_133(Result, N, X86::ROR8rCL, MVT::i8, Chain, N0, N1);
17775     return;
17776   }
17777 
17778   // Pattern: (rotr:i16 GR16:i16:$src, CL:i8)
17779   // Emits: (ROR16rCL:i16 GR16:i16:$src)
17780   // Pattern complexity = 2  cost = 1  size = 3
17781   if (N.Val->getValueType(0) == MVT::i16 &&
17782       N1.Val->getValueType(0) == MVT::i8) {
17783     Emit_133(Result, N, X86::ROR16rCL, MVT::i16, Chain, N0, N1);
17784     return;
17785   }
17786 
17787   // Pattern: (rotr:i32 GR32:i32:$src, CL:i8)
17788   // Emits: (ROR32rCL:i32 GR32:i32:$src)
17789   // Pattern complexity = 2  cost = 1
17790   if (N.Val->getValueType(0) == MVT::i32 &&
17791       N1.Val->getValueType(0) == MVT::i8) {
17792     Emit_133(Result, N, X86::ROR32rCL, MVT::i32, Chain, N0, N1);
17793     return;
17794   }
17795   std::cerr << "Cannot yet select: ";
17796   N.Val->dump(CurDAG);
17797   std::cerr << '\n';
17798   abort();
17799 }
17800 
17801 void Select_scalar_to_vector(SDOperand &Result, SDOperand N) {
17802   SDOperand CPTmp0(0, 0);
17803   SDOperand CPTmp1(0, 0);
17804   SDOperand CPTmp2(0, 0);
17805   SDOperand CPTmp3(0, 0);
17806   SDOperand Chain0(0, 0);
17807   SDOperand N0(0, 0);
17808   SDOperand N01(0, 0);
17809   SDOperand Tmp0(0, 0);
17810   SDOperand Tmp1(0, 0);
17811   SDOperand Tmp2(0, 0);
17812   SDOperand Tmp3(0, 0);
17813   SDNode *ResNode = NULL;
17814 
17815   // Pattern: (scalar_to_vector:v4f32 (load:f32 addr:iPTR:$src))
17816   // Emits: (MOVSS2PSrm:v4f32 addr:iPTR:$src)
17817   // Pattern complexity = 14  cost = 1  size = 3
17818   if ((Subtarget->hasSSE1())) {
17819     N0 = N.getOperand(0);
17820     if (N0.getOpcode() == ISD::LOAD &&
17821         InFlightSet.count(N0.Val) == 0 &&
17822         N0.hasOneUse() &&
17823         !CodeGenMap.count(N0.getValue(0))) {
17824       Chain0 = N0.getOperand(0);
17825       N01 = N0.getOperand(1);
17826       if (N.Val->getValueType(0) == MVT::v4f32 &&
17827           N0.Val->getValueType(0) == MVT::f32) {
17828         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17829         if (Match) {
17830           Emit_90(Result, N, X86::MOVSS2PSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
17831           return;
17832         }
17833       }
17834     }
17835   }
17836   if ((Subtarget->hasSSE2())) {
17837     N0 = N.getOperand(0);
17838     if (N0.getOpcode() == ISD::LOAD &&
17839         InFlightSet.count(N0.Val) == 0 &&
17840         N0.hasOneUse() &&
17841         !CodeGenMap.count(N0.getValue(0))) {
17842       Chain0 = N0.getOperand(0);
17843       N01 = N0.getOperand(1);
17844 
17845       // Pattern: (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src))
17846       // Emits: (MOVSD2PDrm:v2f64 addr:iPTR:$src)
17847       // Pattern complexity = 14  cost = 1  size = 3
17848       if (N.Val->getValueType(0) == MVT::v2f64 &&
17849           N0.Val->getValueType(0) == MVT::f64) {
17850         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17851         if (Match) {
17852           Emit_90(Result, N, X86::MOVSD2PDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
17853           return;
17854         }
17855       }
17856 
17857       // Pattern: (scalar_to_vector:v4i32 (load:i32 addr:iPTR:$src))
17858       // Emits: (MOVDI2PDIrm:v4i32 addr:iPTR:$src)
17859       // Pattern complexity = 14  cost = 1  size = 3
17860       if (N.Val->getValueType(0) == MVT::v4i32 &&
17861           N0.Val->getValueType(0) == MVT::i32) {
17862         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17863         if (Match) {
17864           Emit_90(Result, N, X86::MOVDI2PDIrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
17865           return;
17866         }
17867       }
17868 
17869       // Pattern: (scalar_to_vector:v2i64 (load:i64 addr:iPTR:$src))
17870       // Emits: (MOVQI2PQIrm:v2i64 addr:iPTR:$src)
17871       // Pattern complexity = 14  cost = 1
17872       if (N.Val->getValueType(0) == MVT::v2i64 &&
17873           N0.Val->getValueType(0) == MVT::i64) {
17874         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
17875         if (Match) {
17876           Emit_90(Result, N, X86::MOVQI2PQIrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
17877           return;
17878         }
17879       }
17880     }
17881   }
17882 
17883   // Pattern: (scalar_to_vector:v4f32 FR32:f32:$src)
17884   // Emits: (MOVSS2PSrr:v4f32 FR32:f32:$src)
17885   // Pattern complexity = 2  cost = 1  size = 3
17886   if ((Subtarget->hasSSE1())) {
17887     N0 = N.getOperand(0);
17888     if (N.Val->getValueType(0) == MVT::v4f32 &&
17889         N0.Val->getValueType(0) == MVT::f32) {
17890       Emit_36(Result, N, X86::MOVSS2PSrr, MVT::v4f32, N0);
17891       return;
17892     }
17893   }
17894   if ((Subtarget->hasSSE2())) {
17895     N0 = N.getOperand(0);
17896 
17897     // Pattern: (scalar_to_vector:v2f64 FR64:f64:$src)
17898     // Emits: (MOVSD2PDrr:v2f64 FR64:f64:$src)
17899     // Pattern complexity = 2  cost = 1  size = 3
17900     if (N.Val->getValueType(0) == MVT::v2f64 &&
17901         N0.Val->getValueType(0) == MVT::f64) {
17902       Emit_36(Result, N, X86::MOVSD2PDrr, MVT::v2f64, N0);
17903       return;
17904     }
17905 
17906     // Pattern: (scalar_to_vector:v4i32 GR32:i32:$src)
17907     // Emits: (MOVDI2PDIrr:v4i32 GR32:i32:$src)
17908     // Pattern complexity = 2  cost = 1  size = 3
17909     if (N.Val->getValueType(0) == MVT::v4i32 &&
17910         N0.Val->getValueType(0) == MVT::i32) {
17911       Emit_36(Result, N, X86::MOVDI2PDIrr, MVT::v4i32, N0);
17912       return;
17913     }
17914 
17915     // Pattern: (scalar_to_vector:v2i64 VR64:v8i8:$src)
17916     // Emits: (MOVQI2PQIrr:v2i64 VR64:v8i8:$src)
17917     // Pattern complexity = 2  cost = 1
17918     if (N.Val->getValueType(0) == MVT::v2i64 &&
17919         N0.Val->getValueType(0) == MVT::v8i8) {
17920       Emit_36(Result, N, X86::MOVQI2PQIrr, MVT::v2i64, N0);
17921       return;
17922     }
17923   }
17924   std::cerr << "Cannot yet select: ";
17925   N.Val->dump(CurDAG);
17926   std::cerr << '\n';
17927   abort();
17928 }
17929 
17930 void Select_sext(SDOperand &Result, SDOperand N) {
17931   SDOperand N0(0, 0);
17932   SDOperand Tmp0(0, 0);
17933   SDNode *ResNode = NULL;
17934   N0 = N.getOperand(0);
17935 
17936   // Pattern: (sext:i16 GR8:i8:$src)
17937   // Emits: (MOVSX16rr8:i16 GR8:i8:$src)
17938   // Pattern complexity = 2  cost = 1  size = 3
17939   if (N.Val->getValueType(0) == MVT::i16 &&
17940       N0.Val->getValueType(0) == MVT::i8) {
17941     Emit_36(Result, N, X86::MOVSX16rr8, MVT::i16, N0);
17942     return;
17943   }
17944   if (N.Val->getValueType(0) == MVT::i32) {
17945 
17946     // Pattern: (sext:i32 GR8:i8:$src)
17947     // Emits: (MOVSX32rr8:i32 GR8:i8:$src)
17948     // Pattern complexity = 2  cost = 1  size = 3
17949     if (N0.Val->getValueType(0) == MVT::i8) {
17950       Emit_36(Result, N, X86::MOVSX32rr8, MVT::i32, N0);
17951       return;
17952     }
17953 
17954     // Pattern: (sext:i32 GR16:i16:$src)
17955     // Emits: (MOVSX32rr16:i32 GR16:i16:$src)
17956     // Pattern complexity = 2  cost = 1
17957     if (N0.Val->getValueType(0) == MVT::i16) {
17958       Emit_36(Result, N, X86::MOVSX32rr16, MVT::i32, N0);
17959       return;
17960     }
17961   }
17962   std::cerr << "Cannot yet select: ";
17963   N.Val->dump(CurDAG);
17964   std::cerr << '\n';
17965   abort();
17966 }
17967 
17968 void Select_sextld(SDOperand &Result, SDOperand N) {
17969   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
17970     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
17971     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
17972     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
17973     Result = Dummy;
17974     return;
17975   }
17976   SDOperand CPTmp0(0, 0);
17977   SDOperand CPTmp1(0, 0);
17978   SDOperand CPTmp2(0, 0);
17979   SDOperand CPTmp3(0, 0);
17980   SDOperand Chain(0, 0);
17981   SDOperand N1(0, 0);
17982   SDOperand N2(0, 0);
17983   SDOperand N3(0, 0);
17984   SDOperand Tmp0(0, 0);
17985   SDOperand Tmp1(0, 0);
17986   SDOperand Tmp2(0, 0);
17987   SDOperand Tmp3(0, 0);
17988   SDNode *ResNode = NULL;
17989   Chain = N.getOperand(0);
17990   N1 = N.getOperand(1);
17991   N2 = N.getOperand(2);
17992   N3 = N.getOperand(3);
17993   if (cast<VTSDNode>(N3)->getVT() == MVT::i8) {
17994 
17995     // Pattern: (sextld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
17996     // Emits: (MOVSX16rm8:i16 addr:iPTR:$src)
17997     // Pattern complexity = 12  cost = 1  size = 3
17998     if (N.Val->getValueType(0) == MVT::i16) {
17999       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18000       if (Match) {
18001         Emit_98(Result, N, X86::MOVSX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
18002         return;
18003       }
18004     }
18005 
18006     // Pattern: (sextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
18007     // Emits: (MOVSX32rm8:i32 addr:iPTR:$src)
18008     // Pattern complexity = 12  cost = 1
18009     if (N.Val->getValueType(0) == MVT::i32) {
18010       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18011       if (Match) {
18012         Emit_98(Result, N, X86::MOVSX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
18013         return;
18014       }
18015     }
18016   }
18017 
18018   // Pattern: (sextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i16:Other)
18019   // Emits: (MOVSX32rm16:i32 addr:iPTR:$src)
18020   // Pattern complexity = 12  cost = 1  size = 3
18021   if (cast<VTSDNode>(N3)->getVT() == MVT::i16 &&
18022       N.Val->getValueType(0) == MVT::i32) {
18023     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18024     if (Match) {
18025       Emit_98(Result, N, X86::MOVSX32rm16, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
18026       return;
18027     }
18028   }
18029   if (cast<VTSDNode>(N3)->getVT() == MVT::i1) {
18030 
18031     // Pattern: (sextld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
18032     // Emits: (MOVSX16rm8:i16 addr:iPTR:$src)
18033     // Pattern complexity = 12  cost = 1  size = 3
18034     if (N.Val->getValueType(0) == MVT::i16) {
18035       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18036       if (Match) {
18037         Emit_98(Result, N, X86::MOVSX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
18038         return;
18039       }
18040     }
18041 
18042     // Pattern: (sextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
18043     // Emits: (MOVSX32rm8:i32 addr:iPTR:$src)
18044     // Pattern complexity = 12  cost = 1
18045     if (N.Val->getValueType(0) == MVT::i32) {
18046       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18047       if (Match) {
18048         Emit_98(Result, N, X86::MOVSX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
18049         return;
18050       }
18051     }
18052   }
18053   std::cerr << "Cannot yet select: ";
18054   N.Val->dump(CurDAG);
18055   std::cerr << '\n';
18056   abort();
18057 }
18058 
18059 void Emit_134(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
18060   SDOperand Tmp1(0, 0);
18061   SDNode *ResNode = NULL;
18062   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
18063   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
18064   Select(Tmp1, N0);
18065   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
18066   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
18067   if (N.Val->hasOneUse()) {
18068     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp1, Tmp1);
18069   } else {
18070     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp1, Tmp1);
18071     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18072     Result = SDOperand(ResNode, 0);
18073   }
18074   return;
18075 }
18076 void Select_shl(SDOperand &Result, SDOperand N) {
18077   SDOperand CPTmp0(0, 0);
18078   SDOperand CPTmp1(0, 0);
18079   SDOperand CPTmp2(0, 0);
18080   SDOperand CPTmp3(0, 0);
18081   SDOperand Chain(0, 0);
18082   SDOperand InFlag(0, 0);
18083   SDOperand N0(0, 0);
18084   SDOperand N1(0, 0);
18085   SDOperand Tmp0(0, 0);
18086   SDOperand Tmp1(0, 0);
18087   SDOperand Tmp2(0, 0);
18088   SDOperand Tmp3(0, 0);
18089   SDNode *ResNode = NULL;
18090 
18091   // Pattern: leaaddr:i32:$src
18092   // Emits: (LEA32r:i32 leaaddr:i32:$src)
18093   // Pattern complexity = 10  cost = 1  size = 3
18094   if (N.Val->getValueType(0) == MVT::i32) {
18095     bool Match = SelectLEAAddr(N, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18096     if (Match) {
18097       Emit_42(Result, N, X86::LEA32r, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18098       return;
18099     }
18100   }
18101   N0 = N.getOperand(0);
18102   N1 = N.getOperand(1);
18103   if (isa<ConstantSDNode>(N1)) {
18104     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
18105     if (CN0 == 1) {
18106 
18107       // Pattern: (shl:i8 GR8:i8:$src1, 1:i8)
18108       // Emits: (ADD8rr:i8 GR8:i8:$src1, GR8:i8:$src1)
18109       // Pattern complexity = 5  cost = 1  size = 3
18110       if (N.Val->getValueType(0) == MVT::i8 &&
18111           N1.Val->getValueType(0) == MVT::i8) {
18112         Emit_134(Result, N, X86::ADD8rr, MVT::i8, N0, N1);
18113         return;
18114       }
18115 
18116       // Pattern: (shl:i16 GR16:i16:$src1, 1:i8)
18117       // Emits: (ADD16rr:i16 GR16:i16:$src1, GR16:i16:$src1)
18118       // Pattern complexity = 5  cost = 1  size = 3
18119       if (N.Val->getValueType(0) == MVT::i16 &&
18120           N1.Val->getValueType(0) == MVT::i8) {
18121         Emit_134(Result, N, X86::ADD16rr, MVT::i16, N0, N1);
18122         return;
18123       }
18124 
18125       // Pattern: (shl:i32 GR32:i32:$src1, 1:i8)
18126       // Emits: (ADD32rr:i32 GR32:i32:$src1, GR32:i32:$src1)
18127       // Pattern complexity = 5  cost = 1
18128       if (N.Val->getValueType(0) == MVT::i32 &&
18129           N1.Val->getValueType(0) == MVT::i8) {
18130         Emit_134(Result, N, X86::ADD32rr, MVT::i32, N0, N1);
18131         return;
18132       }
18133     }
18134   }
18135   if (N1.getOpcode() == ISD::Constant) {
18136 
18137     // Pattern: (shl:i8 GR8:i8:$src1, (imm:i8):$src2)
18138     // Emits: (SHL8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
18139     // Pattern complexity = 4  cost = 1  size = 3
18140     if (N.Val->getValueType(0) == MVT::i8 &&
18141         N1.Val->getValueType(0) == MVT::i8) {
18142       Emit_50(Result, N, X86::SHL8ri, MVT::i8, N0, N1);
18143       return;
18144     }
18145 
18146     // Pattern: (shl:i16 GR16:i16:$src1, (imm:i8):$src2)
18147     // Emits: (SHL16ri:i16 GR16:i16:$src1, (imm:i8):$src2)
18148     // Pattern complexity = 4  cost = 1  size = 3
18149     if (N.Val->getValueType(0) == MVT::i16 &&
18150         N1.Val->getValueType(0) == MVT::i8) {
18151       Emit_50(Result, N, X86::SHL16ri, MVT::i16, N0, N1);
18152       return;
18153     }
18154 
18155     // Pattern: (shl:i32 GR32:i32:$src1, (imm:i8):$src2)
18156     // Emits: (SHL32ri:i32 GR32:i32:$src1, (imm:i8):$src2)
18157     // Pattern complexity = 4  cost = 1
18158     if (N.Val->getValueType(0) == MVT::i32 &&
18159         N1.Val->getValueType(0) == MVT::i8) {
18160       Emit_50(Result, N, X86::SHL32ri, MVT::i32, N0, N1);
18161       return;
18162     }
18163   }
18164 
18165   // Pattern: (shl:i8 GR8:i8:$src, CL:i8)
18166   // Emits: (SHL8rCL:i8 GR8:i8:$src)
18167   // Pattern complexity = 2  cost = 1  size = 3
18168   if (N.Val->getValueType(0) == MVT::i8 &&
18169       N1.Val->getValueType(0) == MVT::i8) {
18170     Emit_133(Result, N, X86::SHL8rCL, MVT::i8, Chain, N0, N1);
18171     return;
18172   }
18173 
18174   // Pattern: (shl:i16 GR16:i16:$src, CL:i8)
18175   // Emits: (SHL16rCL:i16 GR16:i16:$src)
18176   // Pattern complexity = 2  cost = 1  size = 3
18177   if (N.Val->getValueType(0) == MVT::i16 &&
18178       N1.Val->getValueType(0) == MVT::i8) {
18179     Emit_133(Result, N, X86::SHL16rCL, MVT::i16, Chain, N0, N1);
18180     return;
18181   }
18182 
18183   // Pattern: (shl:i32 GR32:i32:$src, CL:i8)
18184   // Emits: (SHL32rCL:i32 GR32:i32:$src)
18185   // Pattern complexity = 2  cost = 1
18186   if (N.Val->getValueType(0) == MVT::i32 &&
18187       N1.Val->getValueType(0) == MVT::i8) {
18188     Emit_133(Result, N, X86::SHL32rCL, MVT::i32, Chain, N0, N1);
18189     return;
18190   }
18191   std::cerr << "Cannot yet select: ";
18192   N.Val->dump(CurDAG);
18193   std::cerr << '\n';
18194   abort();
18195 }
18196 
18197 void Select_sint_to_fp(SDOperand &Result, SDOperand N) {
18198   SDOperand CPTmp0(0, 0);
18199   SDOperand CPTmp1(0, 0);
18200   SDOperand CPTmp2(0, 0);
18201   SDOperand CPTmp3(0, 0);
18202   SDOperand Chain0(0, 0);
18203   SDOperand N0(0, 0);
18204   SDOperand N01(0, 0);
18205   SDOperand Tmp0(0, 0);
18206   SDOperand Tmp1(0, 0);
18207   SDOperand Tmp2(0, 0);
18208   SDOperand Tmp3(0, 0);
18209   SDNode *ResNode = NULL;
18210 
18211   // Pattern: (sint_to_fp:f32 (load:i32 addr:iPTR:$src))
18212   // Emits: (CVTSI2SSrm:f32 addr:iPTR:$src)
18213   // Pattern complexity = 14  cost = 1  size = 3
18214   if ((Subtarget->hasSSE1())) {
18215     N0 = N.getOperand(0);
18216     if (N0.getOpcode() == ISD::LOAD &&
18217         InFlightSet.count(N0.Val) == 0 &&
18218         N0.hasOneUse() &&
18219         !CodeGenMap.count(N0.getValue(0))) {
18220       Chain0 = N0.getOperand(0);
18221       N01 = N0.getOperand(1);
18222       if (N.Val->getValueType(0) == MVT::f32 &&
18223           N0.Val->getValueType(0) == MVT::i32) {
18224         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18225         if (Match) {
18226           Emit_90(Result, N, X86::CVTSI2SSrm, MVT::f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
18227           return;
18228         }
18229       }
18230     }
18231   }
18232 
18233   // Pattern: (sint_to_fp:f64 (load:i32 addr:iPTR:$src))
18234   // Emits: (CVTSI2SDrm:f64 addr:iPTR:$src)
18235   // Pattern complexity = 14  cost = 1  size = 3
18236   if ((Subtarget->hasSSE2())) {
18237     N0 = N.getOperand(0);
18238     if (N0.getOpcode() == ISD::LOAD &&
18239         InFlightSet.count(N0.Val) == 0 &&
18240         N0.hasOneUse() &&
18241         !CodeGenMap.count(N0.getValue(0))) {
18242       Chain0 = N0.getOperand(0);
18243       N01 = N0.getOperand(1);
18244       if (N.Val->getValueType(0) == MVT::f64 &&
18245           N0.Val->getValueType(0) == MVT::i32) {
18246         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
18247         if (Match) {
18248           Emit_90(Result, N, X86::CVTSI2SDrm, MVT::f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01);
18249           return;
18250         }
18251       }
18252     }
18253   }
18254 
18255   // Pattern: (sint_to_fp:f32 GR32:i32:$src)
18256   // Emits: (CVTSI2SSrr:f32 GR32:i32:$src)
18257   // Pattern complexity = 2  cost = 1  size = 3
18258   if ((Subtarget->hasSSE1())) {
18259     N0 = N.getOperand(0);
18260     if (N.Val->getValueType(0) == MVT::f32 &&
18261         N0.Val->getValueType(0) == MVT::i32) {
18262       Emit_36(Result, N, X86::CVTSI2SSrr, MVT::f32, N0);
18263       return;
18264     }
18265   }
18266 
18267   // Pattern: (sint_to_fp:f64 GR32:i32:$src)
18268   // Emits: (CVTSI2SDrr:f64 GR32:i32:$src)
18269   // Pattern complexity = 2  cost = 1
18270   if ((Subtarget->hasSSE2())) {
18271     N0 = N.getOperand(0);
18272     if (N.Val->getValueType(0) == MVT::f64 &&
18273         N0.Val->getValueType(0) == MVT::i32) {
18274       Emit_36(Result, N, X86::CVTSI2SDrr, MVT::f64, N0);
18275       return;
18276     }
18277   }
18278   std::cerr << "Cannot yet select: ";
18279   N.Val->dump(CurDAG);
18280   std::cerr << '\n';
18281   abort();
18282 }
18283 
18284 void Select_sra(SDOperand &Result, SDOperand N) {
18285   SDOperand Chain(0, 0);
18286   SDOperand InFlag(0, 0);
18287   SDOperand N0(0, 0);
18288   SDOperand N1(0, 0);
18289   SDOperand Tmp0(0, 0);
18290   SDOperand Tmp1(0, 0);
18291   SDNode *ResNode = NULL;
18292   N0 = N.getOperand(0);
18293   N1 = N.getOperand(1);
18294   if (isa<ConstantSDNode>(N1)) {
18295     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
18296     if (CN0 == 1) {
18297 
18298       // Pattern: (sra:i8 GR8:i8:$src1, 1:i8)
18299       // Emits: (SAR8r1:i8 GR8:i8:$src1)
18300       // Pattern complexity = 5  cost = 1  size = 3
18301       if (N.Val->getValueType(0) == MVT::i8 &&
18302           N1.Val->getValueType(0) == MVT::i8) {
18303         Emit_47(Result, N, X86::SAR8r1, MVT::i8, N0, N1);
18304         return;
18305       }
18306 
18307       // Pattern: (sra:i16 GR16:i16:$src1, 1:i8)
18308       // Emits: (SAR16r1:i16 GR16:i16:$src1)
18309       // Pattern complexity = 5  cost = 1  size = 3
18310       if (N.Val->getValueType(0) == MVT::i16 &&
18311           N1.Val->getValueType(0) == MVT::i8) {
18312         Emit_47(Result, N, X86::SAR16r1, MVT::i16, N0, N1);
18313         return;
18314       }
18315 
18316       // Pattern: (sra:i32 GR32:i32:$src1, 1:i8)
18317       // Emits: (SAR32r1:i32 GR32:i32:$src1)
18318       // Pattern complexity = 5  cost = 1
18319       if (N.Val->getValueType(0) == MVT::i32 &&
18320           N1.Val->getValueType(0) == MVT::i8) {
18321         Emit_47(Result, N, X86::SAR32r1, MVT::i32, N0, N1);
18322         return;
18323       }
18324     }
18325   }
18326   if (N1.getOpcode() == ISD::Constant) {
18327 
18328     // Pattern: (sra:i8 GR8:i8:$src1, (imm:i8):$src2)
18329     // Emits: (SAR8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
18330     // Pattern complexity = 4  cost = 1  size = 3
18331     if (N.Val->getValueType(0) == MVT::i8 &&
18332         N1.Val->getValueType(0) == MVT::i8) {
18333       Emit_50(Result, N, X86::SAR8ri, MVT::i8, N0, N1);
18334       return;
18335     }
18336 
18337     // Pattern: (sra:i16 GR16:i16:$src1, (imm:i8):$src2)
18338     // Emits: (SAR16ri:i16 GR16:i16:$src1, (imm:i8):$src2)
18339     // Pattern complexity = 4  cost = 1  size = 3
18340     if (N.Val->getValueType(0) == MVT::i16 &&
18341         N1.Val->getValueType(0) == MVT::i8) {
18342       Emit_50(Result, N, X86::SAR16ri, MVT::i16, N0, N1);
18343       return;
18344     }
18345 
18346     // Pattern: (sra:i32 GR32:i32:$src1, (imm:i8):$src2)
18347     // Emits: (SAR32ri:i32 GR32:i32:$src1, (imm:i8):$src2)
18348     // Pattern complexity = 4  cost = 1
18349     if (N.Val->getValueType(0) == MVT::i32 &&
18350         N1.Val->getValueType(0) == MVT::i8) {
18351       Emit_50(Result, N, X86::SAR32ri, MVT::i32, N0, N1);
18352       return;
18353     }
18354   }
18355 
18356   // Pattern: (sra:i8 GR8:i8:$src, CL:i8)
18357   // Emits: (SAR8rCL:i8 GR8:i8:$src)
18358   // Pattern complexity = 2  cost = 1  size = 3
18359   if (N.Val->getValueType(0) == MVT::i8 &&
18360       N1.Val->getValueType(0) == MVT::i8) {
18361     Emit_133(Result, N, X86::SAR8rCL, MVT::i8, Chain, N0, N1);
18362     return;
18363   }
18364 
18365   // Pattern: (sra:i16 GR16:i16:$src, CL:i8)
18366   // Emits: (SAR16rCL:i16 GR16:i16:$src)
18367   // Pattern complexity = 2  cost = 1  size = 3
18368   if (N.Val->getValueType(0) == MVT::i16 &&
18369       N1.Val->getValueType(0) == MVT::i8) {
18370     Emit_133(Result, N, X86::SAR16rCL, MVT::i16, Chain, N0, N1);
18371     return;
18372   }
18373 
18374   // Pattern: (sra:i32 GR32:i32:$src, CL:i8)
18375   // Emits: (SAR32rCL:i32 GR32:i32:$src)
18376   // Pattern complexity = 2  cost = 1
18377   if (N.Val->getValueType(0) == MVT::i32 &&
18378       N1.Val->getValueType(0) == MVT::i8) {
18379     Emit_133(Result, N, X86::SAR32rCL, MVT::i32, Chain, N0, N1);
18380     return;
18381   }
18382   std::cerr << "Cannot yet select: ";
18383   N.Val->dump(CurDAG);
18384   std::cerr << '\n';
18385   abort();
18386 }
18387 
18388 void Select_srl(SDOperand &Result, SDOperand N) {
18389   SDOperand Chain(0, 0);
18390   SDOperand InFlag(0, 0);
18391   SDOperand N0(0, 0);
18392   SDOperand N1(0, 0);
18393   SDOperand Tmp0(0, 0);
18394   SDOperand Tmp1(0, 0);
18395   SDNode *ResNode = NULL;
18396   N0 = N.getOperand(0);
18397   N1 = N.getOperand(1);
18398   if (isa<ConstantSDNode>(N1)) {
18399     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
18400     if (CN0 == 1) {
18401 
18402       // Pattern: (srl:i8 GR8:i8:$src1, 1:i8)
18403       // Emits: (SHR8r1:i8 GR8:i8:$src1)
18404       // Pattern complexity = 5  cost = 1  size = 3
18405       if (N.Val->getValueType(0) == MVT::i8 &&
18406           N1.Val->getValueType(0) == MVT::i8) {
18407         Emit_47(Result, N, X86::SHR8r1, MVT::i8, N0, N1);
18408         return;
18409       }
18410 
18411       // Pattern: (srl:i16 GR16:i16:$src1, 1:i8)
18412       // Emits: (SHR16r1:i16 GR16:i16:$src1)
18413       // Pattern complexity = 5  cost = 1  size = 3
18414       if (N.Val->getValueType(0) == MVT::i16 &&
18415           N1.Val->getValueType(0) == MVT::i8) {
18416         Emit_47(Result, N, X86::SHR16r1, MVT::i16, N0, N1);
18417         return;
18418       }
18419 
18420       // Pattern: (srl:i32 GR32:i32:$src1, 1:i8)
18421       // Emits: (SHR32r1:i32 GR32:i32:$src1)
18422       // Pattern complexity = 5  cost = 1
18423       if (N.Val->getValueType(0) == MVT::i32 &&
18424           N1.Val->getValueType(0) == MVT::i8) {
18425         Emit_47(Result, N, X86::SHR32r1, MVT::i32, N0, N1);
18426         return;
18427       }
18428     }
18429   }
18430   if (N1.getOpcode() == ISD::Constant) {
18431 
18432     // Pattern: (srl:i8 GR8:i8:$src1, (imm:i8):$src2)
18433     // Emits: (SHR8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
18434     // Pattern complexity = 4  cost = 1  size = 3
18435     if (N.Val->getValueType(0) == MVT::i8 &&
18436         N1.Val->getValueType(0) == MVT::i8) {
18437       Emit_50(Result, N, X86::SHR8ri, MVT::i8, N0, N1);
18438       return;
18439     }
18440 
18441     // Pattern: (srl:i16 GR16:i16:$src1, (imm:i8):$src2)
18442     // Emits: (SHR16ri:i16 GR16:i16:$src1, (imm:i8):$src2)
18443     // Pattern complexity = 4  cost = 1  size = 3
18444     if (N.Val->getValueType(0) == MVT::i16 &&
18445         N1.Val->getValueType(0) == MVT::i8) {
18446       Emit_50(Result, N, X86::SHR16ri, MVT::i16, N0, N1);
18447       return;
18448     }
18449 
18450     // Pattern: (srl:i32 GR32:i32:$src1, (imm:i8):$src2)
18451     // Emits: (SHR32ri:i32 GR32:i32:$src1, (imm:i8):$src2)
18452     // Pattern complexity = 4  cost = 1
18453     if (N.Val->getValueType(0) == MVT::i32 &&
18454         N1.Val->getValueType(0) == MVT::i8) {
18455       Emit_50(Result, N, X86::SHR32ri, MVT::i32, N0, N1);
18456       return;
18457     }
18458   }
18459 
18460   // Pattern: (srl:i8 GR8:i8:$src, CL:i8)
18461   // Emits: (SHR8rCL:i8 GR8:i8:$src)
18462   // Pattern complexity = 2  cost = 1  size = 3
18463   if (N.Val->getValueType(0) == MVT::i8 &&
18464       N1.Val->getValueType(0) == MVT::i8) {
18465     Emit_133(Result, N, X86::SHR8rCL, MVT::i8, Chain, N0, N1);
18466     return;
18467   }
18468 
18469   // Pattern: (srl:i16 GR16:i16:$src, CL:i8)
18470   // Emits: (SHR16rCL:i16 GR16:i16:$src)
18471   // Pattern complexity = 2  cost = 1  size = 3
18472   if (N.Val->getValueType(0) == MVT::i16 &&
18473       N1.Val->getValueType(0) == MVT::i8) {
18474     Emit_133(Result, N, X86::SHR16rCL, MVT::i16, Chain, N0, N1);
18475     return;
18476   }
18477 
18478   // Pattern: (srl:i32 GR32:i32:$src, CL:i8)
18479   // Emits: (SHR32rCL:i32 GR32:i32:$src)
18480   // Pattern complexity = 2  cost = 1
18481   if (N.Val->getValueType(0) == MVT::i32 &&
18482       N1.Val->getValueType(0) == MVT::i8) {
18483     Emit_133(Result, N, X86::SHR32rCL, MVT::i32, Chain, N0, N1);
18484     return;
18485   }
18486   std::cerr << "Cannot yet select: ";
18487   N.Val->dump(CurDAG);
18488   std::cerr << '\n';
18489   abort();
18490 }
18491 
18492 void Emit_135(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &Chain100, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N1001, SDOperand &N101, SDOperand &N11, SDOperand &N110, SDOperand &N111, SDOperand &N1110, SDOperand &N1111, SDOperand &N2) NOINLINE {
18493   SDOperand InFlag(0, 0);
18494   SDOperand Tmp1(0, 0);
18495   SDOperand Tmp2(0, 0);
18496   SDOperand Tmp3(0, 0);
18497   SDOperand Tmp4(0, 0);
18498   SDOperand Tmp5(0, 0);
18499   SDNode *ResNode = NULL;
18500   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18501   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18502   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18503   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
18504   Select(Tmp1, CPTmp1);
18505   Select(Tmp2, CPTmp2);
18506   Select(Tmp3, CPTmp3);
18507   Select(Tmp4, CPTmp4);
18508   Select(Tmp5, N110);
18509   Select(Chain100, Chain100);
18510   Select(N101, N101);
18511   ResNode = CurDAG->getCopyToReg(Chain100, CurDAG->getRegister(X86::CL, MVT::i8), N101, InFlag).Val;
18512   Chain100 = SDOperand(ResNode, 0);
18513   InFlag = SDOperand(ResNode, 1);
18514   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
18515   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18516   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18517   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18518   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
18519   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain100, InFlag);
18520   Chain100 = SDOperand(ResNode, 0);
18521   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain100.Val, Chain100.ResNo);
18522   SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 0);
18523   AddHandleReplacement(N100.Val, 1, ResNode, 0);
18524   Result = SDOperand(ResNode, N.ResNo);
18525   return;
18526 }
18527 void Emit_136(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &Chain110, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N101, SDOperand &N1010, SDOperand &N1011, SDOperand &N11, SDOperand &N110, SDOperand &N1101, SDOperand &N111, SDOperand &N2) NOINLINE {
18528   SDOperand InFlag(0, 0);
18529   SDOperand Tmp1(0, 0);
18530   SDOperand Tmp2(0, 0);
18531   SDOperand Tmp3(0, 0);
18532   SDOperand Tmp4(0, 0);
18533   SDOperand Tmp5(0, 0);
18534   SDNode *ResNode = NULL;
18535   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18536   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18537   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18538   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
18539   Select(Tmp1, CPTmp1);
18540   Select(Tmp2, CPTmp2);
18541   Select(Tmp3, CPTmp3);
18542   Select(Tmp4, CPTmp4);
18543   Select(Tmp5, N100);
18544   Select(Chain110, Chain110);
18545   Select(N1011, N1011);
18546   ResNode = CurDAG->getCopyToReg(Chain110, CurDAG->getRegister(X86::CL, MVT::i8), N1011, InFlag).Val;
18547   Chain110 = SDOperand(ResNode, 0);
18548   InFlag = SDOperand(ResNode, 1);
18549   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
18550   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18551   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18552   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18553   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
18554   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain110, InFlag);
18555   Chain110 = SDOperand(ResNode, 0);
18556   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain110.Val, Chain110.ResNo);
18557   SelectionDAG::InsertISelMapEntry(CodeGenMap, N110.Val, 1, ResNode, 0);
18558   AddHandleReplacement(N110.Val, 1, ResNode, 0);
18559   Result = SDOperand(ResNode, N.ResNo);
18560   return;
18561 }
18562 void Emit_137(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &Chain11, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N111, SDOperand &N2) NOINLINE {
18563   SDOperand Tmp1(0, 0);
18564   SDOperand Tmp2(0, 0);
18565   SDOperand Tmp3(0, 0);
18566   SDOperand Tmp4(0, 0);
18567   SDNode *ResNode = NULL;
18568   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18569   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18570   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18571   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
18572   Select(Tmp1, CPTmp1);
18573   Select(Tmp2, CPTmp2);
18574   Select(Tmp3, CPTmp3);
18575   Select(Tmp4, CPTmp4);
18576   Select(Chain11, Chain11);
18577   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18578   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18579   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18580   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
18581   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
18582   Chain11 = SDOperand(ResNode, 0);
18583   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain11.Val, Chain11.ResNo);
18584   SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
18585   AddHandleReplacement(N11.Val, 1, ResNode, 0);
18586   Result = SDOperand(ResNode, N.ResNo);
18587   return;
18588 }
18589 void Emit_138(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18590   SDOperand Tmp0(0, 0);
18591   SDOperand Tmp1(0, 0);
18592   SDOperand Tmp2(0, 0);
18593   SDOperand Tmp3(0, 0);
18594   SDNode *ResNode = NULL;
18595   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18596   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18597   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18598   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18599   Select(Tmp0, CPTmp0);
18600   Select(Tmp1, CPTmp1);
18601   Select(Tmp2, CPTmp2);
18602   Select(Tmp3, CPTmp3);
18603   Select(Chain10, Chain10);
18604   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18605   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18606   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18607   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18608   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
18609   Chain10 = SDOperand(ResNode, 0);
18610   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18611   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18612   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18613   Result = SDOperand(ResNode, N.ResNo);
18614   return;
18615 }
18616 void Emit_139(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18617   SDOperand Tmp1(0, 0);
18618   SDOperand Tmp2(0, 0);
18619   SDOperand Tmp3(0, 0);
18620   SDOperand Tmp4(0, 0);
18621   SDNode *ResNode = NULL;
18622   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18623   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18624   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18625   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
18626   Select(Tmp1, CPTmp1);
18627   Select(Tmp2, CPTmp2);
18628   Select(Tmp3, CPTmp3);
18629   Select(Tmp4, CPTmp4);
18630   Select(Chain10, Chain10);
18631   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18632   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18633   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18634   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
18635   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
18636   Chain10 = SDOperand(ResNode, 0);
18637   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18638   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18639   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18640   Result = SDOperand(ResNode, N.ResNo);
18641   return;
18642 }
18643 void Emit_140(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18644   SDOperand Tmp0(0, 0);
18645   SDOperand Tmp1(0, 0);
18646   SDOperand Tmp2(0, 0);
18647   SDOperand Tmp3(0, 0);
18648   SDOperand Tmp4(0, 0);
18649   SDNode *ResNode = NULL;
18650   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18651   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18652   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18653   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18654   Select(Tmp0, CPTmp0);
18655   Select(Tmp1, CPTmp1);
18656   Select(Tmp2, CPTmp2);
18657   Select(Tmp3, CPTmp3);
18658   Tmp4 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N11)->getValue()), MVT::i16);
18659   Select(Chain10, Chain10);
18660   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18661   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18662   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18663   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18664   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
18665   Chain10 = SDOperand(ResNode, 0);
18666   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18667   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18668   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18669   Result = SDOperand(ResNode, N.ResNo);
18670   return;
18671 }
18672 void Emit_141(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18673   SDOperand Tmp0(0, 0);
18674   SDOperand Tmp1(0, 0);
18675   SDOperand Tmp2(0, 0);
18676   SDOperand Tmp3(0, 0);
18677   SDOperand Tmp4(0, 0);
18678   SDNode *ResNode = NULL;
18679   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18680   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18681   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18682   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18683   Select(Tmp0, CPTmp0);
18684   Select(Tmp1, CPTmp1);
18685   Select(Tmp2, CPTmp2);
18686   Select(Tmp3, CPTmp3);
18687   Tmp4 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N11)->getValue()), MVT::i32);
18688   Select(Chain10, Chain10);
18689   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18690   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18691   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18692   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18693   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
18694   Chain10 = SDOperand(ResNode, 0);
18695   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18696   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18697   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18698   Result = SDOperand(ResNode, N.ResNo);
18699   return;
18700 }
18701 void Emit_142(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18702   SDOperand InFlag(0, 0);
18703   SDOperand Tmp0(0, 0);
18704   SDOperand Tmp1(0, 0);
18705   SDOperand Tmp2(0, 0);
18706   SDOperand Tmp3(0, 0);
18707   SDOperand Tmp4(0, 0);
18708   SDNode *ResNode = NULL;
18709   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18710   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18711   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18712   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18713   Select(Tmp0, CPTmp0);
18714   Select(Tmp1, CPTmp1);
18715   Select(Tmp2, CPTmp2);
18716   Select(Tmp3, CPTmp3);
18717   Tmp4 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N11)->getValue()), MVT::i32);
18718   Select(Chain10, Chain10);
18719   Select(InFlag, N1.getOperand(2));
18720   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18721   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18722   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18723   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18724   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
18725   Chain10 = SDOperand(ResNode, 0);
18726   InFlag = SDOperand(ResNode, 1);
18727   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18728   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18729   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18730   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
18731   Result = SDOperand(ResNode, N.ResNo);
18732   return;
18733 }
18734 void Emit_143(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18735   SDOperand Tmp0(0, 0);
18736   SDOperand Tmp1(0, 0);
18737   SDOperand Tmp2(0, 0);
18738   SDOperand Tmp3(0, 0);
18739   SDOperand Tmp4(0, 0);
18740   SDNode *ResNode = NULL;
18741   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18742   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18743   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18744   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18745   Select(Tmp0, CPTmp0);
18746   Select(Tmp1, CPTmp1);
18747   Select(Tmp2, CPTmp2);
18748   Select(Tmp3, CPTmp3);
18749   Tmp4 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N11)->getValue()), MVT::i8);
18750   Select(Chain10, Chain10);
18751   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18752   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18753   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18754   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18755   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
18756   Chain10 = SDOperand(ResNode, 0);
18757   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18758   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18759   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18760   Result = SDOperand(ResNode, N.ResNo);
18761   return;
18762 }
18763 void Emit_144(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N12, SDOperand &N2) NOINLINE {
18764   SDOperand Tmp0(0, 0);
18765   SDOperand Tmp1(0, 0);
18766   SDOperand Tmp2(0, 0);
18767   SDOperand Tmp3(0, 0);
18768   SDOperand Tmp4(0, 0);
18769   SDOperand Tmp5(0, 0);
18770   SDNode *ResNode = NULL;
18771   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18772   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18773   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18774   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18775   Select(Tmp0, CPTmp0);
18776   Select(Tmp1, CPTmp1);
18777   Select(Tmp2, CPTmp2);
18778   Select(Tmp3, CPTmp3);
18779   Select(Tmp4, N11);
18780   Tmp5 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N12)->getValue()), MVT::i8);
18781   Select(Chain10, Chain10);
18782   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
18783   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18784   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18785   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18786   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18787   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain10);
18788   Chain10 = SDOperand(ResNode, 0);
18789   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18790   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18791   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18792   Result = SDOperand(ResNode, N.ResNo);
18793   return;
18794 }
18795 void Emit_145(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18796   SDOperand InFlag(0, 0);
18797   SDOperand Tmp0(0, 0);
18798   SDOperand Tmp1(0, 0);
18799   SDOperand Tmp2(0, 0);
18800   SDOperand Tmp3(0, 0);
18801   SDOperand Tmp4(0, 0);
18802   SDNode *ResNode = NULL;
18803   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18804   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18805   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18806   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18807   Select(Tmp0, CPTmp0);
18808   Select(Tmp1, CPTmp1);
18809   Select(Tmp2, CPTmp2);
18810   Select(Tmp3, CPTmp3);
18811   Tmp4 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N11)->getValue()), MVT::i8);
18812   Select(Chain10, Chain10);
18813   Select(InFlag, N1.getOperand(2));
18814   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18815   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18816   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18817   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18818   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
18819   Chain10 = SDOperand(ResNode, 0);
18820   InFlag = SDOperand(ResNode, 1);
18821   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18822   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18823   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18824   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
18825   Result = SDOperand(ResNode, N.ResNo);
18826   return;
18827 }
18828 void Emit_146(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18829   SDOperand Tmp0(0, 0);
18830   SDOperand Tmp1(0, 0);
18831   SDOperand Tmp2(0, 0);
18832   SDOperand Tmp3(0, 0);
18833   SDOperand Tmp4(0, 0);
18834   SDNode *ResNode = NULL;
18835   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18836   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18837   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18838   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18839   Select(Tmp0, CPTmp0);
18840   Select(Tmp1, CPTmp1);
18841   Select(Tmp2, CPTmp2);
18842   Select(Tmp3, CPTmp3);
18843   Select(Tmp4, N11);
18844   Select(Chain10, Chain10);
18845   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
18846   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18847   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18848   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18849   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18850   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
18851   Chain10 = SDOperand(ResNode, 0);
18852   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18853   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18854   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18855   Result = SDOperand(ResNode, N.ResNo);
18856   return;
18857 }
18858 void Emit_147(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18859   SDOperand InFlag(0, 0);
18860   SDOperand Tmp0(0, 0);
18861   SDOperand Tmp1(0, 0);
18862   SDOperand Tmp2(0, 0);
18863   SDOperand Tmp3(0, 0);
18864   SDNode *ResNode = NULL;
18865   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18866   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18867   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18868   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18869   Select(Tmp0, CPTmp0);
18870   Select(Tmp1, CPTmp1);
18871   Select(Tmp2, CPTmp2);
18872   Select(Tmp3, CPTmp3);
18873   Select(Chain10, Chain10);
18874   Select(N11, N11);
18875   ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
18876   Chain10 = SDOperand(ResNode, 0);
18877   InFlag = SDOperand(ResNode, 1);
18878   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18879   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18880   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18881   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18882   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
18883   Chain10 = SDOperand(ResNode, 0);
18884   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18885   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18886   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18887   Result = SDOperand(ResNode, N.ResNo);
18888   return;
18889 }
18890 void Emit_148(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N12, SDOperand &N2) NOINLINE {
18891   SDOperand InFlag(0, 0);
18892   SDOperand Tmp0(0, 0);
18893   SDOperand Tmp1(0, 0);
18894   SDOperand Tmp2(0, 0);
18895   SDOperand Tmp3(0, 0);
18896   SDOperand Tmp4(0, 0);
18897   SDNode *ResNode = NULL;
18898   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18899   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18900   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18901   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18902   Select(Tmp0, CPTmp0);
18903   Select(Tmp1, CPTmp1);
18904   Select(Tmp2, CPTmp2);
18905   Select(Tmp3, CPTmp3);
18906   Select(Tmp4, N11);
18907   Select(Chain10, Chain10);
18908   Select(N12, N12);
18909   ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N12, InFlag).Val;
18910   Chain10 = SDOperand(ResNode, 0);
18911   InFlag = SDOperand(ResNode, 1);
18912   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
18913   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18914   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18915   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18916   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18917   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
18918   Chain10 = SDOperand(ResNode, 0);
18919   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18920   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18921   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18922   Result = SDOperand(ResNode, N.ResNo);
18923   return;
18924 }
18925 void Emit_149(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain10, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N11, SDOperand &N2) NOINLINE {
18926   SDOperand InFlag(0, 0);
18927   SDOperand Tmp0(0, 0);
18928   SDOperand Tmp1(0, 0);
18929   SDOperand Tmp2(0, 0);
18930   SDOperand Tmp3(0, 0);
18931   SDOperand Tmp4(0, 0);
18932   SDNode *ResNode = NULL;
18933   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18934   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18935   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18936   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18937   Select(Tmp0, CPTmp0);
18938   Select(Tmp1, CPTmp1);
18939   Select(Tmp2, CPTmp2);
18940   Select(Tmp3, CPTmp3);
18941   Select(Tmp4, N11);
18942   Select(Chain10, Chain10);
18943   Select(InFlag, N1.getOperand(2));
18944   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
18945   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
18946   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18947   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18948   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18949   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
18950   Chain10 = SDOperand(ResNode, 0);
18951   InFlag = SDOperand(ResNode, 1);
18952   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain10.Val, Chain10.ResNo);
18953   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
18954   AddHandleReplacement(N10.Val, 1, ResNode, 0);
18955   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
18956   Result = SDOperand(ResNode, N.ResNo);
18957   return;
18958 }
18959 void Emit_150(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N1000, SDOperand &N101, SDOperand &N102, SDOperand &N11, SDOperand &N2) NOINLINE {
18960   SDOperand Tmp1(0, 0);
18961   SDOperand Tmp2(0, 0);
18962   SDOperand Tmp3(0, 0);
18963   SDOperand Tmp4(0, 0);
18964   SDOperand Tmp5(0, 0);
18965   SDNode *ResNode = NULL;
18966   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18967   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18968   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18969   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
18970   Select(Tmp1, CPTmp1);
18971   Select(Tmp2, CPTmp2);
18972   Select(Tmp3, CPTmp3);
18973   Select(Tmp4, CPTmp4);
18974   Select(Tmp5, N1000);
18975   Select(Chain, Chain);
18976   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1000.Val);
18977   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
18978   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
18979   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
18980   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
18981   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
18982   Chain = SDOperand(ResNode, 0);
18983   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
18984   Result = SDOperand(ResNode, N.ResNo);
18985   return;
18986 }
18987 void Emit_151(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain11, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N111, SDOperand &N2) NOINLINE {
18988   SDOperand Tmp0(0, 0);
18989   SDOperand Tmp1(0, 0);
18990   SDOperand Tmp2(0, 0);
18991   SDOperand Tmp3(0, 0);
18992   SDOperand Tmp4(0, 0);
18993   SDNode *ResNode = NULL;
18994   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
18995   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
18996   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
18997   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
18998   Select(Tmp0, CPTmp0);
18999   Select(Tmp1, CPTmp1);
19000   Select(Tmp2, CPTmp2);
19001   Select(Tmp3, CPTmp3);
19002   Select(Tmp4, N10);
19003   Select(Chain11, Chain11);
19004   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
19005   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19006   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19007   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19008   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19009   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
19010   Chain11 = SDOperand(ResNode, 0);
19011   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain11.Val, Chain11.ResNo);
19012   SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
19013   AddHandleReplacement(N11.Val, 1, ResNode, 0);
19014   Result = SDOperand(ResNode, N.ResNo);
19015   return;
19016 }
19017 void Emit_152(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &Chain11, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N111, SDOperand &N2) NOINLINE {
19018   SDOperand InFlag(0, 0);
19019   SDOperand Tmp0(0, 0);
19020   SDOperand Tmp1(0, 0);
19021   SDOperand Tmp2(0, 0);
19022   SDOperand Tmp3(0, 0);
19023   SDOperand Tmp4(0, 0);
19024   SDNode *ResNode = NULL;
19025   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
19026   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19027   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19028   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19029   Select(Tmp0, CPTmp0);
19030   Select(Tmp1, CPTmp1);
19031   Select(Tmp2, CPTmp2);
19032   Select(Tmp3, CPTmp3);
19033   Select(Tmp4, N10);
19034   Select(Chain11, Chain11);
19035   Select(InFlag, N1.getOperand(2));
19036   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
19037   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19038   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19039   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19040   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19041   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11, InFlag);
19042   Chain11 = SDOperand(ResNode, 0);
19043   InFlag = SDOperand(ResNode, 1);
19044   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain11.Val, Chain11.ResNo);
19045   SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
19046   AddHandleReplacement(N11.Val, 1, ResNode, 0);
19047   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
19048   Result = SDOperand(ResNode, N.ResNo);
19049   return;
19050 }
19051 void Emit_153(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N101, SDOperand &N102, SDOperand &N11, SDOperand &N2) NOINLINE {
19052   SDOperand Tmp1(0, 0);
19053   SDOperand Tmp2(0, 0);
19054   SDOperand Tmp3(0, 0);
19055   SDOperand Tmp4(0, 0);
19056   SDOperand Tmp5(0, 0);
19057   SDNode *ResNode = NULL;
19058   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19059   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19060   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19061   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
19062   Select(Tmp1, CPTmp1);
19063   Select(Tmp2, CPTmp2);
19064   Select(Tmp3, CPTmp3);
19065   Select(Tmp4, CPTmp4);
19066   Select(Tmp5, N100);
19067   Select(Chain, Chain);
19068   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19069   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19070   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19071   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19072   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
19073   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
19074   Chain = SDOperand(ResNode, 0);
19075   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19076   Result = SDOperand(ResNode, N.ResNo);
19077   return;
19078 }
19079 void Emit_154(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N11, SDOperand &N2) NOINLINE {
19080   SDOperand Tmp1(0, 0);
19081   SDOperand Tmp2(0, 0);
19082   SDOperand Tmp3(0, 0);
19083   SDOperand Tmp4(0, 0);
19084   SDOperand Tmp5(0, 0);
19085   SDNode *ResNode = NULL;
19086   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19087   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19088   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19089   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
19090   Select(Tmp1, CPTmp1);
19091   Select(Tmp2, CPTmp2);
19092   Select(Tmp3, CPTmp3);
19093   Select(Tmp4, CPTmp4);
19094   Select(Tmp5, N100);
19095   Select(Chain, Chain);
19096   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19097   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19098   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19099   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19100   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
19101   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
19102   Chain = SDOperand(ResNode, 0);
19103   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19104   Result = SDOperand(ResNode, N.ResNo);
19105   return;
19106 }
19107 void Emit_155(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N2) NOINLINE {
19108   SDOperand InFlag(0, 0);
19109   SDOperand Tmp1(0, 0);
19110   SDOperand Tmp2(0, 0);
19111   SDOperand Tmp3(0, 0);
19112   SDOperand Tmp4(0, 0);
19113   SDNode *ResNode = NULL;
19114   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19115   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19116   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19117   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
19118   Select(Tmp1, CPTmp1);
19119   Select(Tmp2, CPTmp2);
19120   Select(Tmp3, CPTmp3);
19121   Select(Tmp4, CPTmp4);
19122   Select(Chain, Chain);
19123   Select(InFlag, N1.getOperand(1));
19124   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19125   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19126   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19127   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
19128   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
19129   Chain = SDOperand(ResNode, 0);
19130   InFlag = SDOperand(ResNode, 1);
19131   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19132   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
19133   Result = SDOperand(ResNode, N.ResNo);
19134   return;
19135 }
19136 void Emit_156(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &CPTmp4, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N11, SDOperand &N2) NOINLINE {
19137   SDOperand Tmp1(0, 0);
19138   SDOperand Tmp2(0, 0);
19139   SDOperand Tmp3(0, 0);
19140   SDOperand Tmp4(0, 0);
19141   SDOperand Tmp5(0, 0);
19142   SDNode *ResNode = NULL;
19143   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19144   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19145   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19146   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp4.Val);
19147   Select(Tmp1, CPTmp1);
19148   Select(Tmp2, CPTmp2);
19149   Select(Tmp3, CPTmp3);
19150   Select(Tmp4, CPTmp4);
19151   Select(Tmp5, N10);
19152   Select(Chain, Chain);
19153   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
19154   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19155   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19156   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19157   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp4.Val);
19158   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
19159   Chain = SDOperand(ResNode, 0);
19160   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19161   Result = SDOperand(ResNode, N.ResNo);
19162   return;
19163 }
19164 void Emit_157(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N10, SDOperand &N2) NOINLINE {
19165   SDOperand Tmp0(0, 0);
19166   SDOperand Tmp1(0, 0);
19167   SDOperand Tmp2(0, 0);
19168   SDOperand Tmp3(0, 0);
19169   SDOperand Tmp4(0, 0);
19170   SDNode *ResNode = NULL;
19171   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
19172   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19173   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19174   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19175   Select(Tmp0, CPTmp0);
19176   Select(Tmp1, CPTmp1);
19177   Select(Tmp2, CPTmp2);
19178   Select(Tmp3, CPTmp3);
19179   Tmp4 = N10;
19180   Select(Chain, Chain);
19181   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19182   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19183   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19184   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19185   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
19186   Chain = SDOperand(ResNode, 0);
19187   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19188   Result = SDOperand(ResNode, N.ResNo);
19189   return;
19190 }
19191 void Emit_158(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
19192   SDOperand Tmp0(0, 0);
19193   SDOperand Tmp1(0, 0);
19194   SDOperand Tmp2(0, 0);
19195   SDOperand Tmp3(0, 0);
19196   SDOperand Tmp4(0, 0);
19197   SDNode *ResNode = NULL;
19198   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
19199   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19200   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19201   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19202   Select(Tmp0, CPTmp0);
19203   Select(Tmp1, CPTmp1);
19204   Select(Tmp2, CPTmp2);
19205   Select(Tmp3, CPTmp3);
19206   Tmp4 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N1)->getValue()), MVT::i8);
19207   Select(Chain, Chain);
19208   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19209   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19210   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19211   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19212   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
19213   Chain = SDOperand(ResNode, 0);
19214   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19215   Result = SDOperand(ResNode, N.ResNo);
19216   return;
19217 }
19218 void Emit_159(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
19219   SDOperand Tmp0(0, 0);
19220   SDOperand Tmp1(0, 0);
19221   SDOperand Tmp2(0, 0);
19222   SDOperand Tmp3(0, 0);
19223   SDOperand Tmp4(0, 0);
19224   SDNode *ResNode = NULL;
19225   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
19226   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19227   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19228   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19229   Select(Tmp0, CPTmp0);
19230   Select(Tmp1, CPTmp1);
19231   Select(Tmp2, CPTmp2);
19232   Select(Tmp3, CPTmp3);
19233   Tmp4 = CurDAG->getTargetConstant(((unsigned short) cast<ConstantSDNode>(N1)->getValue()), MVT::i16);
19234   Select(Chain, Chain);
19235   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19236   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19237   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19238   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19239   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
19240   Chain = SDOperand(ResNode, 0);
19241   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19242   Result = SDOperand(ResNode, N.ResNo);
19243   return;
19244 }
19245 void Emit_160(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2) NOINLINE {
19246   SDOperand Tmp0(0, 0);
19247   SDOperand Tmp1(0, 0);
19248   SDOperand Tmp2(0, 0);
19249   SDOperand Tmp3(0, 0);
19250   SDOperand Tmp4(0, 0);
19251   SDNode *ResNode = NULL;
19252   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
19253   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
19254   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
19255   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
19256   Select(Tmp0, CPTmp0);
19257   Select(Tmp1, CPTmp1);
19258   Select(Tmp2, CPTmp2);
19259   Select(Tmp3, CPTmp3);
19260   Tmp4 = CurDAG->getTargetConstant(((unsigned) cast<ConstantSDNode>(N1)->getValue()), MVT::i32);
19261   Select(Chain, Chain);
19262   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
19263   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
19264   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
19265   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
19266   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
19267   Chain = SDOperand(ResNode, 0);
19268   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
19269   Result = SDOperand(ResNode, N.ResNo);
19270   return;
19271 }
19272 void Select_store(SDOperand &Result, SDOperand N) {
19273   SDOperand CPTmp0(0, 0);
19274   SDOperand CPTmp1(0, 0);
19275   SDOperand CPTmp2(0, 0);
19276   SDOperand CPTmp3(0, 0);
19277   SDOperand CPTmp4(0, 0);
19278   SDOperand Chain(0, 0);
19279   SDOperand Chain10(0, 0);
19280   SDOperand Chain100(0, 0);
19281   SDOperand Chain11(0, 0);
19282   SDOperand Chain110(0, 0);
19283   SDOperand InFlag(0, 0);
19284   SDOperand N1(0, 0);
19285   SDOperand N10(0, 0);
19286   SDOperand N100(0, 0);
19287   SDOperand N1000(0, 0);
19288   SDOperand N1001(0, 0);
19289   SDOperand N101(0, 0);
19290   SDOperand N1010(0, 0);
19291   SDOperand N1011(0, 0);
19292   SDOperand N102(0, 0);
19293   SDOperand N11(0, 0);
19294   SDOperand N110(0, 0);
19295   SDOperand N1101(0, 0);
19296   SDOperand N111(0, 0);
19297   SDOperand N1110(0, 0);
19298   SDOperand N1111(0, 0);
19299   SDOperand N12(0, 0);
19300   SDOperand N2(0, 0);
19301   SDOperand Tmp0(0, 0);
19302   SDOperand Tmp1(0, 0);
19303   SDOperand Tmp2(0, 0);
19304   SDOperand Tmp3(0, 0);
19305   SDOperand Tmp4(0, 0);
19306   SDOperand Tmp5(0, 0);
19307   SDNode *ResNode = NULL;
19308   {
19309     Chain = N.getOperand(0);
19310     N1 = N.getOperand(1);
19311     if (N1.getOpcode() == ISD::OR &&
19312         InFlightSet.count(N1.Val) == 0 &&
19313         N1.hasOneUse() &&
19314         !CodeGenMap.count(N1.getValue(0))) {
19315       N10 = N1.getOperand(0);
19316 
19317       // Pattern: (store:void (or:i32 (srl:i32 (load:i32 addr:iPTR:$dst), CL:i8:$amt), (shl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt))), addr:iPTR:$dst)
19318       // Emits: (SHRD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
19319       // Pattern complexity = 35  cost = 1  size = 3
19320       if (N10.getOpcode() == ISD::SRL &&
19321           InFlightSet.count(N10.Val) == 0 &&
19322           N10.hasOneUse() &&
19323           !CodeGenMap.count(N10.getValue(0))) {
19324         N100 = N10.getOperand(0);
19325         if (N100.getOpcode() == ISD::LOAD &&
19326             InFlightSet.count(N100.Val) == 0 &&
19327             N100.hasOneUse() &&
19328             !CodeGenMap.count(N100.getValue(0)) &&
19329             !isNonImmUse(N10.Val, N100.Val) &&
19330             Chain.Val == N100.Val) {
19331           Chain100 = N100.getOperand(0);
19332           N1001 = N100.getOperand(1);
19333           N101 = N10.getOperand(1);
19334           N11 = N1.getOperand(1);
19335           if (N11.getOpcode() == ISD::SHL) {
19336             N110 = N11.getOperand(0);
19337             N111 = N11.getOperand(1);
19338             if (N111.getOpcode() == ISD::SUB) {
19339               N1110 = N111.getOperand(0);
19340               if (isa<ConstantSDNode>(N1110)) {
19341                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
19342                 if (CN0 == 32) {
19343                   N1111 = N111.getOperand(1);
19344                   if (N101 == N1111) {
19345                     N2 = N.getOperand(2);
19346                     if (N1001 == N2 &&
19347                         N1.Val->getValueType(0) == MVT::i32 &&
19348                         N101.Val->getValueType(0) == MVT::i8 &&
19349                         N111.Val->getValueType(0) == MVT::i8) {
19350                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
19351                       bool Match = SelectAddr(N1001, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19352                       if (!Match) {
19353                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
19354                       }
19355                       if (Match) {
19356                         Emit_135(Result, N, X86::SHRD32mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain100, N1, N10, N100, N1001, N101, N11, N110, N111, N1110, N1111, N2);
19357                         return;
19358                       }
19359                     }
19360                   }
19361                 }
19362               }
19363             }
19364           }
19365         }
19366       }
19367 
19368       // Pattern: (store:void (or:i32 (shl:i32 (load:i32 addr:iPTR:$dst), CL:i8:$amt), (srl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt))), addr:iPTR:$dst)
19369       // Emits: (SHLD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
19370       // Pattern complexity = 35  cost = 1  size = 3
19371       if (N10.getOpcode() == ISD::SHL &&
19372           InFlightSet.count(N10.Val) == 0 &&
19373           N10.hasOneUse() &&
19374           !CodeGenMap.count(N10.getValue(0))) {
19375         N100 = N10.getOperand(0);
19376         if (N100.getOpcode() == ISD::LOAD &&
19377             InFlightSet.count(N100.Val) == 0 &&
19378             N100.hasOneUse() &&
19379             !CodeGenMap.count(N100.getValue(0)) &&
19380             !isNonImmUse(N10.Val, N100.Val) &&
19381             Chain.Val == N100.Val) {
19382           Chain100 = N100.getOperand(0);
19383           N1001 = N100.getOperand(1);
19384           N101 = N10.getOperand(1);
19385           N11 = N1.getOperand(1);
19386           if (N11.getOpcode() == ISD::SRL) {
19387             N110 = N11.getOperand(0);
19388             N111 = N11.getOperand(1);
19389             if (N111.getOpcode() == ISD::SUB) {
19390               N1110 = N111.getOperand(0);
19391               if (isa<ConstantSDNode>(N1110)) {
19392                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
19393                 if (CN0 == 32) {
19394                   N1111 = N111.getOperand(1);
19395                   if (N101 == N1111) {
19396                     N2 = N.getOperand(2);
19397                     if (N1001 == N2 &&
19398                         N1.Val->getValueType(0) == MVT::i32 &&
19399                         N101.Val->getValueType(0) == MVT::i8 &&
19400                         N111.Val->getValueType(0) == MVT::i8) {
19401                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
19402                       bool Match = SelectAddr(N1001, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19403                       if (!Match) {
19404                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
19405                       }
19406                       if (Match) {
19407                         Emit_135(Result, N, X86::SHLD32mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain100, N1, N10, N100, N1001, N101, N11, N110, N111, N1110, N1111, N2);
19408                         return;
19409                       }
19410                     }
19411                   }
19412                 }
19413               }
19414             }
19415           }
19416         }
19417       }
19418 
19419       // Pattern: (store:void (or:i16 (srl:i16 (load:i16 addr:iPTR:$dst), CL:i8:$amt), (shl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt))), addr:iPTR:$dst)
19420       // Emits: (SHRD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
19421       // Pattern complexity = 35  cost = 1  size = 3
19422       if (N10.getOpcode() == ISD::SRL &&
19423           InFlightSet.count(N10.Val) == 0 &&
19424           N10.hasOneUse() &&
19425           !CodeGenMap.count(N10.getValue(0))) {
19426         N100 = N10.getOperand(0);
19427         if (N100.getOpcode() == ISD::LOAD &&
19428             InFlightSet.count(N100.Val) == 0 &&
19429             N100.hasOneUse() &&
19430             !CodeGenMap.count(N100.getValue(0)) &&
19431             !isNonImmUse(N10.Val, N100.Val) &&
19432             Chain.Val == N100.Val) {
19433           Chain100 = N100.getOperand(0);
19434           N1001 = N100.getOperand(1);
19435           N101 = N10.getOperand(1);
19436           N11 = N1.getOperand(1);
19437           if (N11.getOpcode() == ISD::SHL) {
19438             N110 = N11.getOperand(0);
19439             N111 = N11.getOperand(1);
19440             if (N111.getOpcode() == ISD::SUB) {
19441               N1110 = N111.getOperand(0);
19442               if (isa<ConstantSDNode>(N1110)) {
19443                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
19444                 if (CN0 == 16) {
19445                   N1111 = N111.getOperand(1);
19446                   if (N101 == N1111) {
19447                     N2 = N.getOperand(2);
19448                     if (N1001 == N2 &&
19449                         N1.Val->getValueType(0) == MVT::i16 &&
19450                         N101.Val->getValueType(0) == MVT::i8 &&
19451                         N111.Val->getValueType(0) == MVT::i8) {
19452                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
19453                       bool Match = SelectAddr(N1001, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19454                       if (!Match) {
19455                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
19456                       }
19457                       if (Match) {
19458                         Emit_135(Result, N, X86::SHRD16mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain100, N1, N10, N100, N1001, N101, N11, N110, N111, N1110, N1111, N2);
19459                         return;
19460                       }
19461                     }
19462                   }
19463                 }
19464               }
19465             }
19466           }
19467         }
19468       }
19469       if (N10.getOpcode() == ISD::SHL) {
19470 
19471         // Pattern: (store:void (or:i16 (shl:i16 (load:i16 addr:iPTR:$dst), CL:i8:$amt), (srl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt))), addr:iPTR:$dst)
19472         // Emits: (SHLD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
19473         // Pattern complexity = 35  cost = 1  size = 3
19474         if (InFlightSet.count(N10.Val) == 0 &&
19475             N10.hasOneUse() &&
19476             !CodeGenMap.count(N10.getValue(0))) {
19477           N100 = N10.getOperand(0);
19478           if (N100.getOpcode() == ISD::LOAD &&
19479               InFlightSet.count(N100.Val) == 0 &&
19480               N100.hasOneUse() &&
19481               !CodeGenMap.count(N100.getValue(0)) &&
19482               !isNonImmUse(N10.Val, N100.Val) &&
19483               Chain.Val == N100.Val) {
19484             Chain100 = N100.getOperand(0);
19485             N1001 = N100.getOperand(1);
19486             N101 = N10.getOperand(1);
19487             N11 = N1.getOperand(1);
19488             if (N11.getOpcode() == ISD::SRL) {
19489               N110 = N11.getOperand(0);
19490               N111 = N11.getOperand(1);
19491               if (N111.getOpcode() == ISD::SUB) {
19492                 N1110 = N111.getOperand(0);
19493                 if (isa<ConstantSDNode>(N1110)) {
19494                   int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
19495                   if (CN0 == 16) {
19496                     N1111 = N111.getOperand(1);
19497                     if (N101 == N1111) {
19498                       N2 = N.getOperand(2);
19499                       if (N1001 == N2 &&
19500                           N1.Val->getValueType(0) == MVT::i16 &&
19501                           N101.Val->getValueType(0) == MVT::i8 &&
19502                           N111.Val->getValueType(0) == MVT::i8) {
19503                         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N110.Val);
19504                         bool Match = SelectAddr(N1001, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19505                         if (!Match) {
19506                           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N110.Val);
19507                         }
19508                         if (Match) {
19509                           Emit_135(Result, N, X86::SHLD16mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain100, N1, N10, N100, N1001, N101, N11, N110, N111, N1110, N1111, N2);
19510                           return;
19511                         }
19512                       }
19513                     }
19514                   }
19515                 }
19516               }
19517             }
19518           }
19519         }
19520 
19521         // Pattern: (store:void (or:i32 (shl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (srl:i32 (load:i32 addr:iPTR:$dst), CL:i8:$amt)), addr:iPTR:$dst)
19522         // Emits: (SHRD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
19523         // Pattern complexity = 35  cost = 1
19524         N100 = N10.getOperand(0);
19525         N101 = N10.getOperand(1);
19526         if (N101.getOpcode() == ISD::SUB) {
19527           N1010 = N101.getOperand(0);
19528           if (isa<ConstantSDNode>(N1010)) {
19529             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19530             if (CN0 == 32) {
19531               N1011 = N101.getOperand(1);
19532               N11 = N1.getOperand(1);
19533               if (N11.getOpcode() == ISD::SRL &&
19534                   InFlightSet.count(N11.Val) == 0 &&
19535                   N11.hasOneUse() &&
19536                   !CodeGenMap.count(N11.getValue(0))) {
19537                 N110 = N11.getOperand(0);
19538                 if (N110.getOpcode() == ISD::LOAD &&
19539                     InFlightSet.count(N110.Val) == 0 &&
19540                     N110.hasOneUse() &&
19541                     !CodeGenMap.count(N110.getValue(0)) &&
19542                     !isNonImmUse(N11.Val, N110.Val) &&
19543                     Chain.Val == N110.Val) {
19544                   Chain110 = N110.getOperand(0);
19545                   N1101 = N110.getOperand(1);
19546                   N111 = N11.getOperand(1);
19547                   if (N1011 == N111) {
19548                     N2 = N.getOperand(2);
19549                     if (N1101 == N2 &&
19550                         N1.Val->getValueType(0) == MVT::i32 &&
19551                         N101.Val->getValueType(0) == MVT::i8 &&
19552                         N111.Val->getValueType(0) == MVT::i8) {
19553                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
19554                       bool Match = SelectAddr(N1101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19555                       if (!Match) {
19556                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19557                       }
19558                       if (Match) {
19559                         Emit_136(Result, N, X86::SHRD32mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain110, N1, N10, N100, N101, N1010, N1011, N11, N110, N1101, N111, N2);
19560                         return;
19561                       }
19562                     }
19563                   }
19564                 }
19565               }
19566             }
19567           }
19568         }
19569       }
19570 
19571       // Pattern: (store:void (or:i32 (srl:i32 GR32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (shl:i32 (load:i32 addr:iPTR:$dst), CL:i8:$amt)), addr:iPTR:$dst)
19572       // Emits: (SHLD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
19573       // Pattern complexity = 35  cost = 1  size = 3
19574       if (N10.getOpcode() == ISD::SRL) {
19575         N100 = N10.getOperand(0);
19576         N101 = N10.getOperand(1);
19577         if (N101.getOpcode() == ISD::SUB) {
19578           N1010 = N101.getOperand(0);
19579           if (isa<ConstantSDNode>(N1010)) {
19580             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19581             if (CN0 == 32) {
19582               N1011 = N101.getOperand(1);
19583               N11 = N1.getOperand(1);
19584               if (N11.getOpcode() == ISD::SHL &&
19585                   InFlightSet.count(N11.Val) == 0 &&
19586                   N11.hasOneUse() &&
19587                   !CodeGenMap.count(N11.getValue(0))) {
19588                 N110 = N11.getOperand(0);
19589                 if (N110.getOpcode() == ISD::LOAD &&
19590                     InFlightSet.count(N110.Val) == 0 &&
19591                     N110.hasOneUse() &&
19592                     !CodeGenMap.count(N110.getValue(0)) &&
19593                     !isNonImmUse(N11.Val, N110.Val) &&
19594                     Chain.Val == N110.Val) {
19595                   Chain110 = N110.getOperand(0);
19596                   N1101 = N110.getOperand(1);
19597                   N111 = N11.getOperand(1);
19598                   if (N1011 == N111) {
19599                     N2 = N.getOperand(2);
19600                     if (N1101 == N2 &&
19601                         N1.Val->getValueType(0) == MVT::i32 &&
19602                         N101.Val->getValueType(0) == MVT::i8 &&
19603                         N111.Val->getValueType(0) == MVT::i8) {
19604                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
19605                       bool Match = SelectAddr(N1101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19606                       if (!Match) {
19607                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19608                       }
19609                       if (Match) {
19610                         Emit_136(Result, N, X86::SHLD32mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain110, N1, N10, N100, N101, N1010, N1011, N11, N110, N1101, N111, N2);
19611                         return;
19612                       }
19613                     }
19614                   }
19615                 }
19616               }
19617             }
19618           }
19619         }
19620       }
19621 
19622       // Pattern: (store:void (or:i16 (shl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (srl:i16 (load:i16 addr:iPTR:$dst), CL:i8:$amt)), addr:iPTR:$dst)
19623       // Emits: (SHRD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
19624       // Pattern complexity = 35  cost = 1  size = 3
19625       if (N10.getOpcode() == ISD::SHL) {
19626         N100 = N10.getOperand(0);
19627         N101 = N10.getOperand(1);
19628         if (N101.getOpcode() == ISD::SUB) {
19629           N1010 = N101.getOperand(0);
19630           if (isa<ConstantSDNode>(N1010)) {
19631             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19632             if (CN0 == 16) {
19633               N1011 = N101.getOperand(1);
19634               N11 = N1.getOperand(1);
19635               if (N11.getOpcode() == ISD::SRL &&
19636                   InFlightSet.count(N11.Val) == 0 &&
19637                   N11.hasOneUse() &&
19638                   !CodeGenMap.count(N11.getValue(0))) {
19639                 N110 = N11.getOperand(0);
19640                 if (N110.getOpcode() == ISD::LOAD &&
19641                     InFlightSet.count(N110.Val) == 0 &&
19642                     N110.hasOneUse() &&
19643                     !CodeGenMap.count(N110.getValue(0)) &&
19644                     !isNonImmUse(N11.Val, N110.Val) &&
19645                     Chain.Val == N110.Val) {
19646                   Chain110 = N110.getOperand(0);
19647                   N1101 = N110.getOperand(1);
19648                   N111 = N11.getOperand(1);
19649                   if (N1011 == N111) {
19650                     N2 = N.getOperand(2);
19651                     if (N1101 == N2 &&
19652                         N1.Val->getValueType(0) == MVT::i16 &&
19653                         N101.Val->getValueType(0) == MVT::i8 &&
19654                         N111.Val->getValueType(0) == MVT::i8) {
19655                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
19656                       bool Match = SelectAddr(N1101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19657                       if (!Match) {
19658                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19659                       }
19660                       if (Match) {
19661                         Emit_136(Result, N, X86::SHRD16mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain110, N1, N10, N100, N101, N1010, N1011, N11, N110, N1101, N111, N2);
19662                         return;
19663                       }
19664                     }
19665                   }
19666                 }
19667               }
19668             }
19669           }
19670         }
19671       }
19672 
19673       // Pattern: (store:void (or:i16 (srl:i16 GR16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (shl:i16 (load:i16 addr:iPTR:$dst), CL:i8:$amt)), addr:iPTR:$dst)
19674       // Emits: (SHLD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
19675       // Pattern complexity = 35  cost = 1
19676       if (N10.getOpcode() == ISD::SRL) {
19677         N100 = N10.getOperand(0);
19678         N101 = N10.getOperand(1);
19679         if (N101.getOpcode() == ISD::SUB) {
19680           N1010 = N101.getOperand(0);
19681           if (isa<ConstantSDNode>(N1010)) {
19682             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19683             if (CN0 == 16) {
19684               N1011 = N101.getOperand(1);
19685               N11 = N1.getOperand(1);
19686               if (N11.getOpcode() == ISD::SHL &&
19687                   InFlightSet.count(N11.Val) == 0 &&
19688                   N11.hasOneUse() &&
19689                   !CodeGenMap.count(N11.getValue(0))) {
19690                 N110 = N11.getOperand(0);
19691                 if (N110.getOpcode() == ISD::LOAD &&
19692                     InFlightSet.count(N110.Val) == 0 &&
19693                     N110.hasOneUse() &&
19694                     !CodeGenMap.count(N110.getValue(0)) &&
19695                     !isNonImmUse(N11.Val, N110.Val) &&
19696                     Chain.Val == N110.Val) {
19697                   Chain110 = N110.getOperand(0);
19698                   N1101 = N110.getOperand(1);
19699                   N111 = N11.getOperand(1);
19700                   if (N1011 == N111) {
19701                     N2 = N.getOperand(2);
19702                     if (N1101 == N2 &&
19703                         N1.Val->getValueType(0) == MVT::i16 &&
19704                         N101.Val->getValueType(0) == MVT::i8 &&
19705                         N111.Val->getValueType(0) == MVT::i8) {
19706                       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
19707                       bool Match = SelectAddr(N1101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19708                       if (!Match) {
19709                         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
19710                       }
19711                       if (Match) {
19712                         Emit_136(Result, N, X86::SHLD16mrCL, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain110, N1, N10, N100, N101, N1010, N1011, N11, N110, N1101, N111, N2);
19713                         return;
19714                       }
19715                     }
19716                   }
19717                 }
19718               }
19719             }
19720           }
19721         }
19722       }
19723     }
19724     if (N1.getOpcode() == ISD::SUB &&
19725         InFlightSet.count(N1.Val) == 0 &&
19726         N1.hasOneUse() &&
19727         !CodeGenMap.count(N1.getValue(0))) {
19728       N10 = N1.getOperand(0);
19729       if (isa<ConstantSDNode>(N10)) {
19730         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
19731         if (CN0 == 0) {
19732           N11 = N1.getOperand(1);
19733           if (N11.getOpcode() == ISD::LOAD &&
19734               InFlightSet.count(N11.Val) == 0 &&
19735               N11.hasOneUse() &&
19736               !CodeGenMap.count(N11.getValue(0)) &&
19737               !isNonImmUse(N1.Val, N11.Val) &&
19738               Chain.Val == N11.Val) {
19739             Chain11 = N11.getOperand(0);
19740             N111 = N11.getOperand(1);
19741             N2 = N.getOperand(2);
19742             if (N111 == N2) {
19743 
19744               // Pattern: (store:void (sub:i8 0:i8, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
19745               // Emits: (NEG8m:void addr:iPTR:$dst)
19746               // Pattern complexity = 29  cost = 1  size = 2
19747               if (N1.Val->getValueType(0) == MVT::i8) {
19748                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19749                 if (Match) {
19750                   Emit_137(Result, N, X86::NEG8m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19751                   return;
19752                 }
19753               }
19754 
19755               // Pattern: (store:void (sub:i16 0:i16, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
19756               // Emits: (NEG16m:void addr:iPTR:$dst)
19757               // Pattern complexity = 29  cost = 1  size = 2
19758               if (N1.Val->getValueType(0) == MVT::i16) {
19759                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19760                 if (Match) {
19761                   Emit_137(Result, N, X86::NEG16m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19762                   return;
19763                 }
19764               }
19765 
19766               // Pattern: (store:void (sub:i32 0:i32, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
19767               // Emits: (NEG32m:void addr:iPTR:$dst)
19768               // Pattern complexity = 29  cost = 1
19769               if (N1.Val->getValueType(0) == MVT::i32) {
19770                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19771                 if (Match) {
19772                   Emit_137(Result, N, X86::NEG32m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19773                   return;
19774                 }
19775               }
19776             }
19777           }
19778         }
19779       }
19780     }
19781     if (N1.getOpcode() == ISD::XOR &&
19782         InFlightSet.count(N1.Val) == 0 &&
19783         N1.hasOneUse() &&
19784         !CodeGenMap.count(N1.getValue(0))) {
19785       N10 = N1.getOperand(0);
19786       if (N10.getOpcode() == ISD::LOAD &&
19787           InFlightSet.count(N10.Val) == 0 &&
19788           N10.hasOneUse() &&
19789           !CodeGenMap.count(N10.getValue(0)) &&
19790           !isNonImmUse(N1.Val, N10.Val) &&
19791           Chain.Val == N10.Val) {
19792         Chain10 = N10.getOperand(0);
19793         N101 = N10.getOperand(1);
19794         N11 = N1.getOperand(1);
19795         if (N11.getOpcode() == ISD::Constant &&
19796             Predicate_immAllOnes(N11.Val)) {
19797           N2 = N.getOperand(2);
19798           if (N101 == N2) {
19799 
19800             // Pattern: (store:void (xor:i8 (load:i8 addr:iPTR:$dst), (imm:i8)<<P:Predicate_immAllOnes>>), addr:iPTR:$dst)
19801             // Emits: (NOT8m:void addr:iPTR:$dst)
19802             // Pattern complexity = 29  cost = 1  size = 2
19803             if (N1.Val->getValueType(0) == MVT::i8) {
19804               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
19805               if (Match) {
19806                 Emit_138(Result, N, X86::NOT8m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
19807                 return;
19808               }
19809             }
19810 
19811             // Pattern: (store:void (xor:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_immAllOnes>>), addr:iPTR:$dst)
19812             // Emits: (NOT16m:void addr:iPTR:$dst)
19813             // Pattern complexity = 29  cost = 1  size = 2
19814             if (N1.Val->getValueType(0) == MVT::i16) {
19815               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
19816               if (Match) {
19817                 Emit_138(Result, N, X86::NOT16m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
19818                 return;
19819               }
19820             }
19821 
19822             // Pattern: (store:void (xor:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_immAllOnes>>), addr:iPTR:$dst)
19823             // Emits: (NOT32m:void addr:iPTR:$dst)
19824             // Pattern complexity = 29  cost = 1
19825             if (N1.Val->getValueType(0) == MVT::i32) {
19826               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
19827               if (Match) {
19828                 Emit_138(Result, N, X86::NOT32m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
19829                 return;
19830               }
19831             }
19832           }
19833         }
19834       }
19835     }
19836     if (N1.getOpcode() == ISD::ADD &&
19837         InFlightSet.count(N1.Val) == 0 &&
19838         N1.hasOneUse() &&
19839         !CodeGenMap.count(N1.getValue(0))) {
19840       N10 = N1.getOperand(0);
19841       if (N10.getOpcode() == ISD::LOAD &&
19842           InFlightSet.count(N10.Val) == 0 &&
19843           N10.hasOneUse() &&
19844           !CodeGenMap.count(N10.getValue(0)) &&
19845           !isNonImmUse(N1.Val, N10.Val) &&
19846           Chain.Val == N10.Val) {
19847         Chain10 = N10.getOperand(0);
19848         N101 = N10.getOperand(1);
19849         N11 = N1.getOperand(1);
19850         if (isa<ConstantSDNode>(N11)) {
19851           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
19852           if (CN0 == 1) {
19853             N2 = N.getOperand(2);
19854             if (N101 == N2) {
19855 
19856               // Pattern: (store:void (add:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
19857               // Emits: (INC8m:void addr:iPTR:$dst)
19858               // Pattern complexity = 29  cost = 1  size = 2
19859               if (N1.Val->getValueType(0) == MVT::i8) {
19860                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19861                 if (Match) {
19862                   Emit_139(Result, N, X86::INC8m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19863                   return;
19864                 }
19865               }
19866 
19867               // Pattern: (store:void (add:i16 (load:i16 addr:iPTR:$dst), 1:i16), addr:iPTR:$dst)
19868               // Emits: (INC16m:void addr:iPTR:$dst)
19869               // Pattern complexity = 29  cost = 1  size = 2
19870               if (N1.Val->getValueType(0) == MVT::i16) {
19871                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19872                 if (Match) {
19873                   Emit_139(Result, N, X86::INC16m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19874                   return;
19875                 }
19876               }
19877 
19878               // Pattern: (store:void (add:i32 (load:i32 addr:iPTR:$dst), 1:i32), addr:iPTR:$dst)
19879               // Emits: (INC32m:void addr:iPTR:$dst)
19880               // Pattern complexity = 29  cost = 1
19881               if (N1.Val->getValueType(0) == MVT::i32) {
19882                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19883                 if (Match) {
19884                   Emit_139(Result, N, X86::INC32m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19885                   return;
19886                 }
19887               }
19888             }
19889           }
19890           if (CN0 == -1) {
19891             N2 = N.getOperand(2);
19892             if (N101 == N2) {
19893 
19894               // Pattern: (store:void (add:i8 (load:i8 addr:iPTR:$dst), -1:i8), addr:iPTR:$dst)
19895               // Emits: (DEC8m:void addr:iPTR:$dst)
19896               // Pattern complexity = 29  cost = 1  size = 2
19897               if (N1.Val->getValueType(0) == MVT::i8) {
19898                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19899                 if (Match) {
19900                   Emit_139(Result, N, X86::DEC8m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19901                   return;
19902                 }
19903               }
19904 
19905               // Pattern: (store:void (add:i16 (load:i16 addr:iPTR:$dst), -1:i16), addr:iPTR:$dst)
19906               // Emits: (DEC16m:void addr:iPTR:$dst)
19907               // Pattern complexity = 29  cost = 1  size = 2
19908               if (N1.Val->getValueType(0) == MVT::i16) {
19909                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19910                 if (Match) {
19911                   Emit_139(Result, N, X86::DEC16m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19912                   return;
19913                 }
19914               }
19915 
19916               // Pattern: (store:void (add:i32 (load:i32 addr:iPTR:$dst), -1:i32), addr:iPTR:$dst)
19917               // Emits: (DEC32m:void addr:iPTR:$dst)
19918               // Pattern complexity = 29  cost = 1
19919               if (N1.Val->getValueType(0) == MVT::i32) {
19920                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19921                 if (Match) {
19922                   Emit_139(Result, N, X86::DEC32m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
19923                   return;
19924                 }
19925               }
19926             }
19927           }
19928         }
19929       }
19930       if (isa<ConstantSDNode>(N10)) {
19931         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
19932         if (CN0 == 1) {
19933           N11 = N1.getOperand(1);
19934           if (N11.getOpcode() == ISD::LOAD &&
19935               InFlightSet.count(N11.Val) == 0 &&
19936               N11.hasOneUse() &&
19937               !CodeGenMap.count(N11.getValue(0)) &&
19938               !isNonImmUse(N1.Val, N11.Val) &&
19939               Chain.Val == N11.Val) {
19940             Chain11 = N11.getOperand(0);
19941             N111 = N11.getOperand(1);
19942             N2 = N.getOperand(2);
19943             if (N111 == N2) {
19944 
19945               // Pattern: (store:void (add:i8 1:i8, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
19946               // Emits: (INC8m:void addr:iPTR:$dst)
19947               // Pattern complexity = 29  cost = 1  size = 2
19948               if (N1.Val->getValueType(0) == MVT::i8) {
19949                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19950                 if (Match) {
19951                   Emit_137(Result, N, X86::INC8m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19952                   return;
19953                 }
19954               }
19955 
19956               // Pattern: (store:void (add:i16 1:i16, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
19957               // Emits: (INC16m:void addr:iPTR:$dst)
19958               // Pattern complexity = 29  cost = 1  size = 2
19959               if (N1.Val->getValueType(0) == MVT::i16) {
19960                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19961                 if (Match) {
19962                   Emit_137(Result, N, X86::INC16m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19963                   return;
19964                 }
19965               }
19966 
19967               // Pattern: (store:void (add:i32 1:i32, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
19968               // Emits: (INC32m:void addr:iPTR:$dst)
19969               // Pattern complexity = 29  cost = 1
19970               if (N1.Val->getValueType(0) == MVT::i32) {
19971                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19972                 if (Match) {
19973                   Emit_137(Result, N, X86::INC32m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
19974                   return;
19975                 }
19976               }
19977             }
19978           }
19979         }
19980         if (CN0 == -1) {
19981           N11 = N1.getOperand(1);
19982           if (N11.getOpcode() == ISD::LOAD &&
19983               InFlightSet.count(N11.Val) == 0 &&
19984               N11.hasOneUse() &&
19985               !CodeGenMap.count(N11.getValue(0)) &&
19986               !isNonImmUse(N1.Val, N11.Val) &&
19987               Chain.Val == N11.Val) {
19988             Chain11 = N11.getOperand(0);
19989             N111 = N11.getOperand(1);
19990             N2 = N.getOperand(2);
19991             if (N111 == N2) {
19992 
19993               // Pattern: (store:void (add:i8 -1:i8, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
19994               // Emits: (DEC8m:void addr:iPTR:$dst)
19995               // Pattern complexity = 29  cost = 1  size = 2
19996               if (N1.Val->getValueType(0) == MVT::i8) {
19997                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
19998                 if (Match) {
19999                   Emit_137(Result, N, X86::DEC8m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
20000                   return;
20001                 }
20002               }
20003 
20004               // Pattern: (store:void (add:i16 -1:i16, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
20005               // Emits: (DEC16m:void addr:iPTR:$dst)
20006               // Pattern complexity = 29  cost = 1  size = 2
20007               if (N1.Val->getValueType(0) == MVT::i16) {
20008                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20009                 if (Match) {
20010                   Emit_137(Result, N, X86::DEC16m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
20011                   return;
20012                 }
20013               }
20014 
20015               // Pattern: (store:void (add:i32 -1:i32, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
20016               // Emits: (DEC32m:void addr:iPTR:$dst)
20017               // Pattern complexity = 29  cost = 1
20018               if (N1.Val->getValueType(0) == MVT::i32) {
20019                 bool Match = SelectAddr(N111, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20020                 if (Match) {
20021                   Emit_137(Result, N, X86::DEC32m, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain11, N1, N10, N11, N111, N2);
20022                   return;
20023                 }
20024               }
20025             }
20026           }
20027         }
20028       }
20029     }
20030     if (N1.getOpcode() == ISD::AND &&
20031         InFlightSet.count(N1.Val) == 0 &&
20032         N1.hasOneUse() &&
20033         !CodeGenMap.count(N1.getValue(0))) {
20034       N10 = N1.getOperand(0);
20035       if (N10.getOpcode() == ISD::LOAD &&
20036           InFlightSet.count(N10.Val) == 0 &&
20037           N10.hasOneUse() &&
20038           !CodeGenMap.count(N10.getValue(0)) &&
20039           !isNonImmUse(N1.Val, N10.Val) &&
20040           Chain.Val == N10.Val) {
20041         Chain10 = N10.getOperand(0);
20042         N101 = N10.getOperand(1);
20043         N11 = N1.getOperand(1);
20044         if (N11.getOpcode() == ISD::Constant) {
20045 
20046           // Pattern: (store:void (and:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:iPTR:$dst)
20047           // Emits: (AND16mi8:void addr:iPTR:$dst, (imm:i16):$src)
20048           // Pattern complexity = 29  cost = 1  size = 3
20049           if (Predicate_i16immSExt8(N11.Val)) {
20050             N2 = N.getOperand(2);
20051             if (N101 == N2 &&
20052                 N1.Val->getValueType(0) == MVT::i16) {
20053               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20054               if (Match) {
20055                 Emit_140(Result, N, X86::AND16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20056                 return;
20057               }
20058             }
20059           }
20060 
20061           // Pattern: (store:void (and:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:iPTR:$dst)
20062           // Emits: (AND32mi8:void addr:iPTR:$dst, (imm:i32):$src)
20063           // Pattern complexity = 29  cost = 1
20064           if (Predicate_i32immSExt8(N11.Val)) {
20065             N2 = N.getOperand(2);
20066             if (N101 == N2 &&
20067                 N1.Val->getValueType(0) == MVT::i32) {
20068               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20069               if (Match) {
20070                 Emit_141(Result, N, X86::AND32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20071                 return;
20072               }
20073             }
20074           }
20075         }
20076       }
20077     }
20078     if (N1.getOpcode() == ISD::OR &&
20079         InFlightSet.count(N1.Val) == 0 &&
20080         N1.hasOneUse() &&
20081         !CodeGenMap.count(N1.getValue(0))) {
20082       N10 = N1.getOperand(0);
20083       if (N10.getOpcode() == ISD::LOAD &&
20084           InFlightSet.count(N10.Val) == 0 &&
20085           N10.hasOneUse() &&
20086           !CodeGenMap.count(N10.getValue(0)) &&
20087           !isNonImmUse(N1.Val, N10.Val) &&
20088           Chain.Val == N10.Val) {
20089         Chain10 = N10.getOperand(0);
20090         N101 = N10.getOperand(1);
20091         N11 = N1.getOperand(1);
20092         if (N11.getOpcode() == ISD::Constant) {
20093 
20094           // Pattern: (store:void (or:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:iPTR:$dst)
20095           // Emits: (OR16mi8:void addr:iPTR:$dst, (imm:i16):$src)
20096           // Pattern complexity = 29  cost = 1  size = 3
20097           if (Predicate_i16immSExt8(N11.Val)) {
20098             N2 = N.getOperand(2);
20099             if (N101 == N2 &&
20100                 N1.Val->getValueType(0) == MVT::i16) {
20101               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20102               if (Match) {
20103                 Emit_140(Result, N, X86::OR16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20104                 return;
20105               }
20106             }
20107           }
20108 
20109           // Pattern: (store:void (or:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:iPTR:$dst)
20110           // Emits: (OR32mi8:void addr:iPTR:$dst, (imm:i32):$src)
20111           // Pattern complexity = 29  cost = 1
20112           if (Predicate_i32immSExt8(N11.Val)) {
20113             N2 = N.getOperand(2);
20114             if (N101 == N2 &&
20115                 N1.Val->getValueType(0) == MVT::i32) {
20116               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20117               if (Match) {
20118                 Emit_141(Result, N, X86::OR32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20119                 return;
20120               }
20121             }
20122           }
20123         }
20124       }
20125     }
20126     if (N1.getOpcode() == ISD::XOR &&
20127         InFlightSet.count(N1.Val) == 0 &&
20128         N1.hasOneUse() &&
20129         !CodeGenMap.count(N1.getValue(0))) {
20130       N10 = N1.getOperand(0);
20131       if (N10.getOpcode() == ISD::LOAD &&
20132           InFlightSet.count(N10.Val) == 0 &&
20133           N10.hasOneUse() &&
20134           !CodeGenMap.count(N10.getValue(0)) &&
20135           !isNonImmUse(N1.Val, N10.Val) &&
20136           Chain.Val == N10.Val) {
20137         Chain10 = N10.getOperand(0);
20138         N101 = N10.getOperand(1);
20139         N11 = N1.getOperand(1);
20140         if (N11.getOpcode() == ISD::Constant) {
20141 
20142           // Pattern: (store:void (xor:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:iPTR:$dst)
20143           // Emits: (XOR16mi8:void addr:iPTR:$dst, (imm:i16):$src)
20144           // Pattern complexity = 29  cost = 1  size = 3
20145           if (Predicate_i16immSExt8(N11.Val)) {
20146             N2 = N.getOperand(2);
20147             if (N101 == N2 &&
20148                 N1.Val->getValueType(0) == MVT::i16) {
20149               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20150               if (Match) {
20151                 Emit_140(Result, N, X86::XOR16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20152                 return;
20153               }
20154             }
20155           }
20156 
20157           // Pattern: (store:void (xor:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:iPTR:$dst)
20158           // Emits: (XOR32mi8:void addr:iPTR:$dst, (imm:i32):$src)
20159           // Pattern complexity = 29  cost = 1
20160           if (Predicate_i32immSExt8(N11.Val)) {
20161             N2 = N.getOperand(2);
20162             if (N101 == N2 &&
20163                 N1.Val->getValueType(0) == MVT::i32) {
20164               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20165               if (Match) {
20166                 Emit_141(Result, N, X86::XOR32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20167                 return;
20168               }
20169             }
20170           }
20171         }
20172       }
20173     }
20174     if (N1.getOpcode() == ISD::SHL &&
20175         InFlightSet.count(N1.Val) == 0 &&
20176         N1.hasOneUse() &&
20177         !CodeGenMap.count(N1.getValue(0))) {
20178       N10 = N1.getOperand(0);
20179       if (N10.getOpcode() == ISD::LOAD &&
20180           InFlightSet.count(N10.Val) == 0 &&
20181           N10.hasOneUse() &&
20182           !CodeGenMap.count(N10.getValue(0)) &&
20183           !isNonImmUse(N1.Val, N10.Val) &&
20184           Chain.Val == N10.Val) {
20185         Chain10 = N10.getOperand(0);
20186         N101 = N10.getOperand(1);
20187         N11 = N1.getOperand(1);
20188         if (isa<ConstantSDNode>(N11)) {
20189           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
20190           if (CN0 == 1) {
20191             N2 = N.getOperand(2);
20192             if (N101 == N2) {
20193 
20194               // Pattern: (store:void (shl:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20195               // Emits: (SHL8m1:void addr:iPTR:$dst)
20196               // Pattern complexity = 29  cost = 1  size = 3
20197               if (N1.Val->getValueType(0) == MVT::i8 &&
20198                   N11.Val->getValueType(0) == MVT::i8) {
20199                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20200                 if (Match) {
20201                   Emit_139(Result, N, X86::SHL8m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20202                   return;
20203                 }
20204               }
20205 
20206               // Pattern: (store:void (shl:i16 (load:i16 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20207               // Emits: (SHL16m1:void addr:iPTR:$dst)
20208               // Pattern complexity = 29  cost = 1  size = 3
20209               if (N1.Val->getValueType(0) == MVT::i16 &&
20210                   N11.Val->getValueType(0) == MVT::i8) {
20211                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20212                 if (Match) {
20213                   Emit_139(Result, N, X86::SHL16m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20214                   return;
20215                 }
20216               }
20217 
20218               // Pattern: (store:void (shl:i32 (load:i32 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20219               // Emits: (SHL32m1:void addr:iPTR:$dst)
20220               // Pattern complexity = 29  cost = 1
20221               if (N1.Val->getValueType(0) == MVT::i32 &&
20222                   N11.Val->getValueType(0) == MVT::i8) {
20223                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20224                 if (Match) {
20225                   Emit_139(Result, N, X86::SHL32m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20226                   return;
20227                 }
20228               }
20229             }
20230           }
20231         }
20232       }
20233     }
20234     if (N1.getOpcode() == ISD::SRL &&
20235         InFlightSet.count(N1.Val) == 0 &&
20236         N1.hasOneUse() &&
20237         !CodeGenMap.count(N1.getValue(0))) {
20238       N10 = N1.getOperand(0);
20239       if (N10.getOpcode() == ISD::LOAD &&
20240           InFlightSet.count(N10.Val) == 0 &&
20241           N10.hasOneUse() &&
20242           !CodeGenMap.count(N10.getValue(0)) &&
20243           !isNonImmUse(N1.Val, N10.Val) &&
20244           Chain.Val == N10.Val) {
20245         Chain10 = N10.getOperand(0);
20246         N101 = N10.getOperand(1);
20247         N11 = N1.getOperand(1);
20248         if (isa<ConstantSDNode>(N11)) {
20249           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
20250           if (CN0 == 1) {
20251             N2 = N.getOperand(2);
20252             if (N101 == N2) {
20253 
20254               // Pattern: (store:void (srl:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20255               // Emits: (SHR8m1:void addr:iPTR:$dst)
20256               // Pattern complexity = 29  cost = 1  size = 3
20257               if (N1.Val->getValueType(0) == MVT::i8 &&
20258                   N11.Val->getValueType(0) == MVT::i8) {
20259                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20260                 if (Match) {
20261                   Emit_139(Result, N, X86::SHR8m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20262                   return;
20263                 }
20264               }
20265 
20266               // Pattern: (store:void (srl:i16 (load:i16 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20267               // Emits: (SHR16m1:void addr:iPTR:$dst)
20268               // Pattern complexity = 29  cost = 1  size = 3
20269               if (N1.Val->getValueType(0) == MVT::i16 &&
20270                   N11.Val->getValueType(0) == MVT::i8) {
20271                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20272                 if (Match) {
20273                   Emit_139(Result, N, X86::SHR16m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20274                   return;
20275                 }
20276               }
20277 
20278               // Pattern: (store:void (srl:i32 (load:i32 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20279               // Emits: (SHR32m1:void addr:iPTR:$dst)
20280               // Pattern complexity = 29  cost = 1
20281               if (N1.Val->getValueType(0) == MVT::i32 &&
20282                   N11.Val->getValueType(0) == MVT::i8) {
20283                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20284                 if (Match) {
20285                   Emit_139(Result, N, X86::SHR32m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20286                   return;
20287                 }
20288               }
20289             }
20290           }
20291         }
20292       }
20293     }
20294     if (N1.getOpcode() == ISD::SRA &&
20295         InFlightSet.count(N1.Val) == 0 &&
20296         N1.hasOneUse() &&
20297         !CodeGenMap.count(N1.getValue(0))) {
20298       N10 = N1.getOperand(0);
20299       if (N10.getOpcode() == ISD::LOAD &&
20300           InFlightSet.count(N10.Val) == 0 &&
20301           N10.hasOneUse() &&
20302           !CodeGenMap.count(N10.getValue(0)) &&
20303           !isNonImmUse(N1.Val, N10.Val) &&
20304           Chain.Val == N10.Val) {
20305         Chain10 = N10.getOperand(0);
20306         N101 = N10.getOperand(1);
20307         N11 = N1.getOperand(1);
20308         if (isa<ConstantSDNode>(N11)) {
20309           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
20310           if (CN0 == 1) {
20311             N2 = N.getOperand(2);
20312             if (N101 == N2) {
20313 
20314               // Pattern: (store:void (sra:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20315               // Emits: (SAR8m1:void addr:iPTR:$dst)
20316               // Pattern complexity = 29  cost = 1  size = 3
20317               if (N1.Val->getValueType(0) == MVT::i8 &&
20318                   N11.Val->getValueType(0) == MVT::i8) {
20319                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20320                 if (Match) {
20321                   Emit_139(Result, N, X86::SAR8m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20322                   return;
20323                 }
20324               }
20325 
20326               // Pattern: (store:void (sra:i16 (load:i16 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20327               // Emits: (SAR16m1:void addr:iPTR:$dst)
20328               // Pattern complexity = 29  cost = 1  size = 3
20329               if (N1.Val->getValueType(0) == MVT::i16 &&
20330                   N11.Val->getValueType(0) == MVT::i8) {
20331                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20332                 if (Match) {
20333                   Emit_139(Result, N, X86::SAR16m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20334                   return;
20335                 }
20336               }
20337 
20338               // Pattern: (store:void (sra:i32 (load:i32 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20339               // Emits: (SAR32m1:void addr:iPTR:$dst)
20340               // Pattern complexity = 29  cost = 1
20341               if (N1.Val->getValueType(0) == MVT::i32 &&
20342                   N11.Val->getValueType(0) == MVT::i8) {
20343                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20344                 if (Match) {
20345                   Emit_139(Result, N, X86::SAR32m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20346                   return;
20347                 }
20348               }
20349             }
20350           }
20351         }
20352       }
20353     }
20354     if (N1.getOpcode() == ISD::ROTL &&
20355         InFlightSet.count(N1.Val) == 0 &&
20356         N1.hasOneUse() &&
20357         !CodeGenMap.count(N1.getValue(0))) {
20358       N10 = N1.getOperand(0);
20359       if (N10.getOpcode() == ISD::LOAD &&
20360           InFlightSet.count(N10.Val) == 0 &&
20361           N10.hasOneUse() &&
20362           !CodeGenMap.count(N10.getValue(0)) &&
20363           !isNonImmUse(N1.Val, N10.Val) &&
20364           Chain.Val == N10.Val) {
20365         Chain10 = N10.getOperand(0);
20366         N101 = N10.getOperand(1);
20367         N11 = N1.getOperand(1);
20368         if (isa<ConstantSDNode>(N11)) {
20369           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
20370           if (CN0 == 1) {
20371             N2 = N.getOperand(2);
20372             if (N101 == N2) {
20373 
20374               // Pattern: (store:void (rotl:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20375               // Emits: (ROL8m1:void addr:iPTR:$dst)
20376               // Pattern complexity = 29  cost = 1  size = 3
20377               if (N1.Val->getValueType(0) == MVT::i8 &&
20378                   N11.Val->getValueType(0) == MVT::i8) {
20379                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20380                 if (Match) {
20381                   Emit_139(Result, N, X86::ROL8m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20382                   return;
20383                 }
20384               }
20385 
20386               // Pattern: (store:void (rotl:i16 (load:i16 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20387               // Emits: (ROL16m1:void addr:iPTR:$dst)
20388               // Pattern complexity = 29  cost = 1  size = 3
20389               if (N1.Val->getValueType(0) == MVT::i16 &&
20390                   N11.Val->getValueType(0) == MVT::i8) {
20391                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20392                 if (Match) {
20393                   Emit_139(Result, N, X86::ROL16m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20394                   return;
20395                 }
20396               }
20397 
20398               // Pattern: (store:void (rotl:i32 (load:i32 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20399               // Emits: (ROL32m1:void addr:iPTR:$dst)
20400               // Pattern complexity = 29  cost = 1
20401               if (N1.Val->getValueType(0) == MVT::i32 &&
20402                   N11.Val->getValueType(0) == MVT::i8) {
20403                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20404                 if (Match) {
20405                   Emit_139(Result, N, X86::ROL32m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20406                   return;
20407                 }
20408               }
20409             }
20410           }
20411         }
20412       }
20413     }
20414     if (N1.getOpcode() == ISD::ROTR &&
20415         InFlightSet.count(N1.Val) == 0 &&
20416         N1.hasOneUse() &&
20417         !CodeGenMap.count(N1.getValue(0))) {
20418       N10 = N1.getOperand(0);
20419       if (N10.getOpcode() == ISD::LOAD &&
20420           InFlightSet.count(N10.Val) == 0 &&
20421           N10.hasOneUse() &&
20422           !CodeGenMap.count(N10.getValue(0)) &&
20423           !isNonImmUse(N1.Val, N10.Val) &&
20424           Chain.Val == N10.Val) {
20425         Chain10 = N10.getOperand(0);
20426         N101 = N10.getOperand(1);
20427         N11 = N1.getOperand(1);
20428         if (isa<ConstantSDNode>(N11)) {
20429           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
20430           if (CN0 == 1) {
20431             N2 = N.getOperand(2);
20432             if (N101 == N2) {
20433 
20434               // Pattern: (store:void (rotr:i8 (load:i8 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20435               // Emits: (ROR8m1:void addr:iPTR:$dst)
20436               // Pattern complexity = 29  cost = 1  size = 3
20437               if (N1.Val->getValueType(0) == MVT::i8 &&
20438                   N11.Val->getValueType(0) == MVT::i8) {
20439                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20440                 if (Match) {
20441                   Emit_139(Result, N, X86::ROR8m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20442                   return;
20443                 }
20444               }
20445 
20446               // Pattern: (store:void (rotr:i16 (load:i16 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20447               // Emits: (ROR16m1:void addr:iPTR:$dst)
20448               // Pattern complexity = 29  cost = 1  size = 3
20449               if (N1.Val->getValueType(0) == MVT::i16 &&
20450                   N11.Val->getValueType(0) == MVT::i8) {
20451                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20452                 if (Match) {
20453                   Emit_139(Result, N, X86::ROR16m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20454                   return;
20455                 }
20456               }
20457 
20458               // Pattern: (store:void (rotr:i32 (load:i32 addr:iPTR:$dst), 1:i8), addr:iPTR:$dst)
20459               // Emits: (ROR32m1:void addr:iPTR:$dst)
20460               // Pattern complexity = 29  cost = 1
20461               if (N1.Val->getValueType(0) == MVT::i32 &&
20462                   N11.Val->getValueType(0) == MVT::i8) {
20463                 bool Match = SelectAddr(N101, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
20464                 if (Match) {
20465                   Emit_139(Result, N, X86::ROR32m1, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, Chain10, N1, N10, N101, N11, N2);
20466                   return;
20467                 }
20468               }
20469             }
20470           }
20471         }
20472       }
20473     }
20474     if (N1.getOpcode() == ISD::ADD &&
20475         InFlightSet.count(N1.Val) == 0 &&
20476         N1.hasOneUse() &&
20477         !CodeGenMap.count(N1.getValue(0))) {
20478       N10 = N1.getOperand(0);
20479       if (N10.getOpcode() == ISD::LOAD &&
20480           InFlightSet.count(N10.Val) == 0 &&
20481           N10.hasOneUse() &&
20482           !CodeGenMap.count(N10.getValue(0)) &&
20483           !isNonImmUse(N1.Val, N10.Val) &&
20484           Chain.Val == N10.Val) {
20485         Chain10 = N10.getOperand(0);
20486         N101 = N10.getOperand(1);
20487         N11 = N1.getOperand(1);
20488         if (N11.getOpcode() == ISD::Constant) {
20489 
20490           // Pattern: (store:void (add:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2), addr:iPTR:$dst)
20491           // Emits: (ADD16mi8:void addr:iPTR:$dst, (imm:i16):$src2)
20492           // Pattern complexity = 29  cost = 1  size = 3
20493           if (Predicate_i16immSExt8(N11.Val)) {
20494             N2 = N.getOperand(2);
20495             if (N101 == N2 &&
20496                 N1.Val->getValueType(0) == MVT::i16) {
20497               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20498               if (Match) {
20499                 Emit_140(Result, N, X86::ADD16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20500                 return;
20501               }
20502             }
20503           }
20504 
20505           // Pattern: (store:void (add:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:iPTR:$dst)
20506           // Emits: (ADD32mi8:void addr:iPTR:$dst, (imm:i32):$src2)
20507           // Pattern complexity = 29  cost = 1
20508           if (Predicate_i32immSExt8(N11.Val)) {
20509             N2 = N.getOperand(2);
20510             if (N101 == N2 &&
20511                 N1.Val->getValueType(0) == MVT::i32) {
20512               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20513               if (Match) {
20514                 Emit_141(Result, N, X86::ADD32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20515                 return;
20516               }
20517             }
20518           }
20519         }
20520       }
20521     }
20522 
20523     // Pattern: (store:void (adde:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:iPTR:$dst)
20524     // Emits: (ADC32mi8:void addr:iPTR:$dst, (imm:i32):$src2)
20525     // Pattern complexity = 29  cost = 1  size = 3
20526     if (N1.getOpcode() == ISD::ADDE &&
20527         InFlightSet.count(N1.Val) == 0 &&
20528         N1.hasOneUse() &&
20529         !CodeGenMap.count(N1.getValue(0))) {
20530       N10 = N1.getOperand(0);
20531       if (N10.getOpcode() == ISD::LOAD &&
20532           InFlightSet.count(N10.Val) == 0 &&
20533           N10.hasOneUse() &&
20534           !CodeGenMap.count(N10.getValue(0)) &&
20535           N1.Val->isOnlyUse(N10.Val) &&
20536           !isNonImmUse(N1.Val, N10.Val) &&
20537           Chain.Val == N10.Val) {
20538         Chain10 = N10.getOperand(0);
20539         N101 = N10.getOperand(1);
20540         N11 = N1.getOperand(1);
20541         if (N11.getOpcode() == ISD::Constant &&
20542             Predicate_i32immSExt8(N11.Val)) {
20543           N2 = N.getOperand(2);
20544           if (N101 == N2 &&
20545               N1.Val->getValueType(0) == MVT::i32) {
20546             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20547             if (Match) {
20548               Emit_142(Result, N, X86::ADC32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20549               return;
20550             }
20551           }
20552         }
20553       }
20554     }
20555     if (N1.getOpcode() == ISD::SUB &&
20556         InFlightSet.count(N1.Val) == 0 &&
20557         N1.hasOneUse() &&
20558         !CodeGenMap.count(N1.getValue(0))) {
20559       N10 = N1.getOperand(0);
20560       if (N10.getOpcode() == ISD::LOAD &&
20561           InFlightSet.count(N10.Val) == 0 &&
20562           N10.hasOneUse() &&
20563           !CodeGenMap.count(N10.getValue(0)) &&
20564           !isNonImmUse(N1.Val, N10.Val) &&
20565           Chain.Val == N10.Val) {
20566         Chain10 = N10.getOperand(0);
20567         N101 = N10.getOperand(1);
20568         N11 = N1.getOperand(1);
20569         if (N11.getOpcode() == ISD::Constant) {
20570 
20571           // Pattern: (store:void (sub:i16 (load:i16 addr:iPTR:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2), addr:iPTR:$dst)
20572           // Emits: (SUB16mi8:void addr:iPTR:$dst, (imm:i16):$src2)
20573           // Pattern complexity = 29  cost = 1  size = 3
20574           if (Predicate_i16immSExt8(N11.Val)) {
20575             N2 = N.getOperand(2);
20576             if (N101 == N2 &&
20577                 N1.Val->getValueType(0) == MVT::i16) {
20578               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20579               if (Match) {
20580                 Emit_140(Result, N, X86::SUB16mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20581                 return;
20582               }
20583             }
20584           }
20585 
20586           // Pattern: (store:void (sub:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:iPTR:$dst)
20587           // Emits: (SUB32mi8:void addr:iPTR:$dst, (imm:i32):$src2)
20588           // Pattern complexity = 29  cost = 1
20589           if (Predicate_i32immSExt8(N11.Val)) {
20590             N2 = N.getOperand(2);
20591             if (N101 == N2 &&
20592                 N1.Val->getValueType(0) == MVT::i32) {
20593               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20594               if (Match) {
20595                 Emit_141(Result, N, X86::SUB32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20596                 return;
20597               }
20598             }
20599           }
20600         }
20601       }
20602     }
20603 
20604     // Pattern: (store:void (sube:i32 (load:i32 addr:iPTR:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:iPTR:$dst)
20605     // Emits: (SBB32mi8:void addr:iPTR:$dst, (imm:i32):$src2)
20606     // Pattern complexity = 29  cost = 1  size = 3
20607     if (N1.getOpcode() == ISD::SUBE &&
20608         InFlightSet.count(N1.Val) == 0 &&
20609         N1.hasOneUse() &&
20610         !CodeGenMap.count(N1.getValue(0))) {
20611       N10 = N1.getOperand(0);
20612       if (N10.getOpcode() == ISD::LOAD &&
20613           InFlightSet.count(N10.Val) == 0 &&
20614           N10.hasOneUse() &&
20615           !CodeGenMap.count(N10.getValue(0)) &&
20616           N1.Val->isOnlyUse(N10.Val) &&
20617           !isNonImmUse(N1.Val, N10.Val) &&
20618           Chain.Val == N10.Val) {
20619         Chain10 = N10.getOperand(0);
20620         N101 = N10.getOperand(1);
20621         N11 = N1.getOperand(1);
20622         if (N11.getOpcode() == ISD::Constant &&
20623             Predicate_i32immSExt8(N11.Val)) {
20624           N2 = N.getOperand(2);
20625           if (N101 == N2 &&
20626               N1.Val->getValueType(0) == MVT::i32) {
20627             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20628             if (Match) {
20629               Emit_142(Result, N, X86::SBB32mi8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20630               return;
20631             }
20632           }
20633         }
20634       }
20635     }
20636     if (N1.getOpcode() == ISD::AND &&
20637         InFlightSet.count(N1.Val) == 0 &&
20638         N1.hasOneUse() &&
20639         !CodeGenMap.count(N1.getValue(0))) {
20640       N10 = N1.getOperand(0);
20641       if (N10.getOpcode() == ISD::LOAD &&
20642           InFlightSet.count(N10.Val) == 0 &&
20643           N10.hasOneUse() &&
20644           !CodeGenMap.count(N10.getValue(0)) &&
20645           !isNonImmUse(N1.Val, N10.Val) &&
20646           Chain.Val == N10.Val) {
20647         Chain10 = N10.getOperand(0);
20648         N101 = N10.getOperand(1);
20649         N11 = N1.getOperand(1);
20650         if (N11.getOpcode() == ISD::Constant) {
20651           N2 = N.getOperand(2);
20652           if (N101 == N2) {
20653 
20654             // Pattern: (store:void (and:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20655             // Emits: (AND8mi:void addr:iPTR:$dst, (imm:i8):$src)
20656             // Pattern complexity = 28  cost = 1  size = 3
20657             if (N1.Val->getValueType(0) == MVT::i8) {
20658               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20659               if (Match) {
20660                 Emit_143(Result, N, X86::AND8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20661                 return;
20662               }
20663             }
20664 
20665             // Pattern: (store:void (and:i16 (load:i16 addr:iPTR:$dst), (imm:i16):$src), addr:iPTR:$dst)
20666             // Emits: (AND16mi:void addr:iPTR:$dst, (imm:i16):$src)
20667             // Pattern complexity = 28  cost = 1  size = 3
20668             if (N1.Val->getValueType(0) == MVT::i16) {
20669               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20670               if (Match) {
20671                 Emit_140(Result, N, X86::AND16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20672                 return;
20673               }
20674             }
20675 
20676             // Pattern: (store:void (and:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src), addr:iPTR:$dst)
20677             // Emits: (AND32mi:void addr:iPTR:$dst, (imm:i32):$src)
20678             // Pattern complexity = 28  cost = 1
20679             if (N1.Val->getValueType(0) == MVT::i32) {
20680               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20681               if (Match) {
20682                 Emit_141(Result, N, X86::AND32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20683                 return;
20684               }
20685             }
20686           }
20687         }
20688       }
20689     }
20690     if (N1.getOpcode() == ISD::OR &&
20691         InFlightSet.count(N1.Val) == 0 &&
20692         N1.hasOneUse() &&
20693         !CodeGenMap.count(N1.getValue(0))) {
20694       N10 = N1.getOperand(0);
20695       if (N10.getOpcode() == ISD::LOAD &&
20696           InFlightSet.count(N10.Val) == 0 &&
20697           N10.hasOneUse() &&
20698           !CodeGenMap.count(N10.getValue(0)) &&
20699           !isNonImmUse(N1.Val, N10.Val) &&
20700           Chain.Val == N10.Val) {
20701         Chain10 = N10.getOperand(0);
20702         N101 = N10.getOperand(1);
20703         N11 = N1.getOperand(1);
20704         if (N11.getOpcode() == ISD::Constant) {
20705           N2 = N.getOperand(2);
20706           if (N101 == N2) {
20707 
20708             // Pattern: (store:void (or:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20709             // Emits: (OR8mi:void addr:iPTR:$dst, (imm:i8):$src)
20710             // Pattern complexity = 28  cost = 1  size = 3
20711             if (N1.Val->getValueType(0) == MVT::i8) {
20712               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20713               if (Match) {
20714                 Emit_143(Result, N, X86::OR8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20715                 return;
20716               }
20717             }
20718 
20719             // Pattern: (store:void (or:i16 (load:i16 addr:iPTR:$dst), (imm:i16):$src), addr:iPTR:$dst)
20720             // Emits: (OR16mi:void addr:iPTR:$dst, (imm:i16):$src)
20721             // Pattern complexity = 28  cost = 1  size = 3
20722             if (N1.Val->getValueType(0) == MVT::i16) {
20723               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20724               if (Match) {
20725                 Emit_140(Result, N, X86::OR16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20726                 return;
20727               }
20728             }
20729 
20730             // Pattern: (store:void (or:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src), addr:iPTR:$dst)
20731             // Emits: (OR32mi:void addr:iPTR:$dst, (imm:i32):$src)
20732             // Pattern complexity = 28  cost = 1
20733             if (N1.Val->getValueType(0) == MVT::i32) {
20734               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20735               if (Match) {
20736                 Emit_141(Result, N, X86::OR32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20737                 return;
20738               }
20739             }
20740           }
20741         }
20742       }
20743     }
20744     if (N1.getOpcode() == ISD::XOR &&
20745         InFlightSet.count(N1.Val) == 0 &&
20746         N1.hasOneUse() &&
20747         !CodeGenMap.count(N1.getValue(0))) {
20748       N10 = N1.getOperand(0);
20749       if (N10.getOpcode() == ISD::LOAD &&
20750           InFlightSet.count(N10.Val) == 0 &&
20751           N10.hasOneUse() &&
20752           !CodeGenMap.count(N10.getValue(0)) &&
20753           !isNonImmUse(N1.Val, N10.Val) &&
20754           Chain.Val == N10.Val) {
20755         Chain10 = N10.getOperand(0);
20756         N101 = N10.getOperand(1);
20757         N11 = N1.getOperand(1);
20758         if (N11.getOpcode() == ISD::Constant) {
20759           N2 = N.getOperand(2);
20760           if (N101 == N2) {
20761 
20762             // Pattern: (store:void (xor:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20763             // Emits: (XOR8mi:void addr:iPTR:$dst, (imm:i8):$src)
20764             // Pattern complexity = 28  cost = 1  size = 3
20765             if (N1.Val->getValueType(0) == MVT::i8) {
20766               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20767               if (Match) {
20768                 Emit_143(Result, N, X86::XOR8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20769                 return;
20770               }
20771             }
20772 
20773             // Pattern: (store:void (xor:i16 (load:i16 addr:iPTR:$dst), (imm:i16):$src), addr:iPTR:$dst)
20774             // Emits: (XOR16mi:void addr:iPTR:$dst, (imm:i16):$src)
20775             // Pattern complexity = 28  cost = 1  size = 3
20776             if (N1.Val->getValueType(0) == MVT::i16) {
20777               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20778               if (Match) {
20779                 Emit_140(Result, N, X86::XOR16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20780                 return;
20781               }
20782             }
20783 
20784             // Pattern: (store:void (xor:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src), addr:iPTR:$dst)
20785             // Emits: (XOR32mi:void addr:iPTR:$dst, (imm:i32):$src)
20786             // Pattern complexity = 28  cost = 1
20787             if (N1.Val->getValueType(0) == MVT::i32) {
20788               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20789               if (Match) {
20790                 Emit_141(Result, N, X86::XOR32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20791                 return;
20792               }
20793             }
20794           }
20795         }
20796       }
20797     }
20798     if (N1.getOpcode() == ISD::SHL &&
20799         InFlightSet.count(N1.Val) == 0 &&
20800         N1.hasOneUse() &&
20801         !CodeGenMap.count(N1.getValue(0))) {
20802       N10 = N1.getOperand(0);
20803       if (N10.getOpcode() == ISD::LOAD &&
20804           InFlightSet.count(N10.Val) == 0 &&
20805           N10.hasOneUse() &&
20806           !CodeGenMap.count(N10.getValue(0)) &&
20807           !isNonImmUse(N1.Val, N10.Val) &&
20808           Chain.Val == N10.Val) {
20809         Chain10 = N10.getOperand(0);
20810         N101 = N10.getOperand(1);
20811         N11 = N1.getOperand(1);
20812         if (N11.getOpcode() == ISD::Constant) {
20813           N2 = N.getOperand(2);
20814           if (N101 == N2) {
20815 
20816             // Pattern: (store:void (shl:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20817             // Emits: (SHL8mi:void addr:iPTR:$dst, (imm:i8):$src)
20818             // Pattern complexity = 28  cost = 1  size = 3
20819             if (N1.Val->getValueType(0) == MVT::i8 &&
20820                 N11.Val->getValueType(0) == MVT::i8) {
20821               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20822               if (Match) {
20823                 Emit_143(Result, N, X86::SHL8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20824                 return;
20825               }
20826             }
20827 
20828             // Pattern: (store:void (shl:i16 (load:i16 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20829             // Emits: (SHL16mi:void addr:iPTR:$dst, (imm:i8):$src)
20830             // Pattern complexity = 28  cost = 1  size = 3
20831             if (N1.Val->getValueType(0) == MVT::i16 &&
20832                 N11.Val->getValueType(0) == MVT::i8) {
20833               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20834               if (Match) {
20835                 Emit_143(Result, N, X86::SHL16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20836                 return;
20837               }
20838             }
20839 
20840             // Pattern: (store:void (shl:i32 (load:i32 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20841             // Emits: (SHL32mi:void addr:iPTR:$dst, (imm:i8):$src)
20842             // Pattern complexity = 28  cost = 1
20843             if (N1.Val->getValueType(0) == MVT::i32 &&
20844                 N11.Val->getValueType(0) == MVT::i8) {
20845               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20846               if (Match) {
20847                 Emit_143(Result, N, X86::SHL32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20848                 return;
20849               }
20850             }
20851           }
20852         }
20853       }
20854     }
20855     if (N1.getOpcode() == ISD::SRL &&
20856         InFlightSet.count(N1.Val) == 0 &&
20857         N1.hasOneUse() &&
20858         !CodeGenMap.count(N1.getValue(0))) {
20859       N10 = N1.getOperand(0);
20860       if (N10.getOpcode() == ISD::LOAD &&
20861           InFlightSet.count(N10.Val) == 0 &&
20862           N10.hasOneUse() &&
20863           !CodeGenMap.count(N10.getValue(0)) &&
20864           !isNonImmUse(N1.Val, N10.Val) &&
20865           Chain.Val == N10.Val) {
20866         Chain10 = N10.getOperand(0);
20867         N101 = N10.getOperand(1);
20868         N11 = N1.getOperand(1);
20869         if (N11.getOpcode() == ISD::Constant) {
20870           N2 = N.getOperand(2);
20871           if (N101 == N2) {
20872 
20873             // Pattern: (store:void (srl:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20874             // Emits: (SHR8mi:void addr:iPTR:$dst, (imm:i8):$src)
20875             // Pattern complexity = 28  cost = 1  size = 3
20876             if (N1.Val->getValueType(0) == MVT::i8 &&
20877                 N11.Val->getValueType(0) == MVT::i8) {
20878               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20879               if (Match) {
20880                 Emit_143(Result, N, X86::SHR8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20881                 return;
20882               }
20883             }
20884 
20885             // Pattern: (store:void (srl:i16 (load:i16 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20886             // Emits: (SHR16mi:void addr:iPTR:$dst, (imm:i8):$src)
20887             // Pattern complexity = 28  cost = 1  size = 3
20888             if (N1.Val->getValueType(0) == MVT::i16 &&
20889                 N11.Val->getValueType(0) == MVT::i8) {
20890               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20891               if (Match) {
20892                 Emit_143(Result, N, X86::SHR16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20893                 return;
20894               }
20895             }
20896 
20897             // Pattern: (store:void (srl:i32 (load:i32 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20898             // Emits: (SHR32mi:void addr:iPTR:$dst, (imm:i8):$src)
20899             // Pattern complexity = 28  cost = 1
20900             if (N1.Val->getValueType(0) == MVT::i32 &&
20901                 N11.Val->getValueType(0) == MVT::i8) {
20902               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20903               if (Match) {
20904                 Emit_143(Result, N, X86::SHR32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20905                 return;
20906               }
20907             }
20908           }
20909         }
20910       }
20911     }
20912     if (N1.getOpcode() == ISD::SRA &&
20913         InFlightSet.count(N1.Val) == 0 &&
20914         N1.hasOneUse() &&
20915         !CodeGenMap.count(N1.getValue(0))) {
20916       N10 = N1.getOperand(0);
20917       if (N10.getOpcode() == ISD::LOAD &&
20918           InFlightSet.count(N10.Val) == 0 &&
20919           N10.hasOneUse() &&
20920           !CodeGenMap.count(N10.getValue(0)) &&
20921           !isNonImmUse(N1.Val, N10.Val) &&
20922           Chain.Val == N10.Val) {
20923         Chain10 = N10.getOperand(0);
20924         N101 = N10.getOperand(1);
20925         N11 = N1.getOperand(1);
20926         if (N11.getOpcode() == ISD::Constant) {
20927           N2 = N.getOperand(2);
20928           if (N101 == N2) {
20929 
20930             // Pattern: (store:void (sra:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20931             // Emits: (SAR8mi:void addr:iPTR:$dst, (imm:i8):$src)
20932             // Pattern complexity = 28  cost = 1  size = 3
20933             if (N1.Val->getValueType(0) == MVT::i8 &&
20934                 N11.Val->getValueType(0) == MVT::i8) {
20935               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20936               if (Match) {
20937                 Emit_143(Result, N, X86::SAR8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20938                 return;
20939               }
20940             }
20941 
20942             // Pattern: (store:void (sra:i16 (load:i16 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20943             // Emits: (SAR16mi:void addr:iPTR:$dst, (imm:i8):$src)
20944             // Pattern complexity = 28  cost = 1  size = 3
20945             if (N1.Val->getValueType(0) == MVT::i16 &&
20946                 N11.Val->getValueType(0) == MVT::i8) {
20947               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20948               if (Match) {
20949                 Emit_143(Result, N, X86::SAR16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20950                 return;
20951               }
20952             }
20953 
20954             // Pattern: (store:void (sra:i32 (load:i32 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20955             // Emits: (SAR32mi:void addr:iPTR:$dst, (imm:i8):$src)
20956             // Pattern complexity = 28  cost = 1
20957             if (N1.Val->getValueType(0) == MVT::i32 &&
20958                 N11.Val->getValueType(0) == MVT::i8) {
20959               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20960               if (Match) {
20961                 Emit_143(Result, N, X86::SAR32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20962                 return;
20963               }
20964             }
20965           }
20966         }
20967       }
20968     }
20969     if (N1.getOpcode() == ISD::ROTL &&
20970         InFlightSet.count(N1.Val) == 0 &&
20971         N1.hasOneUse() &&
20972         !CodeGenMap.count(N1.getValue(0))) {
20973       N10 = N1.getOperand(0);
20974       if (N10.getOpcode() == ISD::LOAD &&
20975           InFlightSet.count(N10.Val) == 0 &&
20976           N10.hasOneUse() &&
20977           !CodeGenMap.count(N10.getValue(0)) &&
20978           !isNonImmUse(N1.Val, N10.Val) &&
20979           Chain.Val == N10.Val) {
20980         Chain10 = N10.getOperand(0);
20981         N101 = N10.getOperand(1);
20982         N11 = N1.getOperand(1);
20983         if (N11.getOpcode() == ISD::Constant) {
20984           N2 = N.getOperand(2);
20985           if (N101 == N2) {
20986 
20987             // Pattern: (store:void (rotl:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
20988             // Emits: (ROL8mi:void addr:iPTR:$dst, (imm:i8):$src)
20989             // Pattern complexity = 28  cost = 1  size = 3
20990             if (N1.Val->getValueType(0) == MVT::i8 &&
20991                 N11.Val->getValueType(0) == MVT::i8) {
20992               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
20993               if (Match) {
20994                 Emit_143(Result, N, X86::ROL8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
20995                 return;
20996               }
20997             }
20998 
20999             // Pattern: (store:void (rotl:i16 (load:i16 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
21000             // Emits: (ROL16mi:void addr:iPTR:$dst, (imm:i8):$src)
21001             // Pattern complexity = 28  cost = 1  size = 3
21002             if (N1.Val->getValueType(0) == MVT::i16 &&
21003                 N11.Val->getValueType(0) == MVT::i8) {
21004               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21005               if (Match) {
21006                 Emit_143(Result, N, X86::ROL16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21007                 return;
21008               }
21009             }
21010 
21011             // Pattern: (store:void (rotl:i32 (load:i32 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
21012             // Emits: (ROL32mi:void addr:iPTR:$dst, (imm:i8):$src)
21013             // Pattern complexity = 28  cost = 1
21014             if (N1.Val->getValueType(0) == MVT::i32 &&
21015                 N11.Val->getValueType(0) == MVT::i8) {
21016               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21017               if (Match) {
21018                 Emit_143(Result, N, X86::ROL32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21019                 return;
21020               }
21021             }
21022           }
21023         }
21024       }
21025     }
21026     if (N1.getOpcode() == ISD::ROTR &&
21027         InFlightSet.count(N1.Val) == 0 &&
21028         N1.hasOneUse() &&
21029         !CodeGenMap.count(N1.getValue(0))) {
21030       N10 = N1.getOperand(0);
21031       if (N10.getOpcode() == ISD::LOAD &&
21032           InFlightSet.count(N10.Val) == 0 &&
21033           N10.hasOneUse() &&
21034           !CodeGenMap.count(N10.getValue(0)) &&
21035           !isNonImmUse(N1.Val, N10.Val) &&
21036           Chain.Val == N10.Val) {
21037         Chain10 = N10.getOperand(0);
21038         N101 = N10.getOperand(1);
21039         N11 = N1.getOperand(1);
21040         if (N11.getOpcode() == ISD::Constant) {
21041           N2 = N.getOperand(2);
21042           if (N101 == N2) {
21043 
21044             // Pattern: (store:void (rotr:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
21045             // Emits: (ROR8mi:void addr:iPTR:$dst, (imm:i8):$src)
21046             // Pattern complexity = 28  cost = 1  size = 3
21047             if (N1.Val->getValueType(0) == MVT::i8 &&
21048                 N11.Val->getValueType(0) == MVT::i8) {
21049               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21050               if (Match) {
21051                 Emit_143(Result, N, X86::ROR8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21052                 return;
21053               }
21054             }
21055 
21056             // Pattern: (store:void (rotr:i16 (load:i16 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
21057             // Emits: (ROR16mi:void addr:iPTR:$dst, (imm:i8):$src)
21058             // Pattern complexity = 28  cost = 1  size = 3
21059             if (N1.Val->getValueType(0) == MVT::i16 &&
21060                 N11.Val->getValueType(0) == MVT::i8) {
21061               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21062               if (Match) {
21063                 Emit_143(Result, N, X86::ROR16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21064                 return;
21065               }
21066             }
21067 
21068             // Pattern: (store:void (rotr:i32 (load:i32 addr:iPTR:$dst), (imm:i8):$src), addr:iPTR:$dst)
21069             // Emits: (ROR32mi:void addr:iPTR:$dst, (imm:i8):$src)
21070             // Pattern complexity = 28  cost = 1
21071             if (N1.Val->getValueType(0) == MVT::i32 &&
21072                 N11.Val->getValueType(0) == MVT::i8) {
21073               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21074               if (Match) {
21075                 Emit_143(Result, N, X86::ROR32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21076                 return;
21077               }
21078             }
21079           }
21080         }
21081       }
21082     }
21083 
21084     // Pattern: (store:void (X86shld:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2, (imm:i8):$src3), addr:iPTR:$dst)
21085     // Emits: (SHLD32mri8:void addr:iPTR:$dst, GR32:i32:$src2, (imm:i8):$src3)
21086     // Pattern complexity = 28  cost = 1  size = 3
21087     if (N1.getOpcode() == X86ISD::SHLD &&
21088         InFlightSet.count(N1.Val) == 0 &&
21089         N1.hasOneUse() &&
21090         !CodeGenMap.count(N1.getValue(0))) {
21091       N10 = N1.getOperand(0);
21092       if (N10.getOpcode() == ISD::LOAD &&
21093           InFlightSet.count(N10.Val) == 0 &&
21094           N10.hasOneUse() &&
21095           !CodeGenMap.count(N10.getValue(0)) &&
21096           !isNonImmUse(N1.Val, N10.Val) &&
21097           Chain.Val == N10.Val) {
21098         Chain10 = N10.getOperand(0);
21099         N101 = N10.getOperand(1);
21100         N11 = N1.getOperand(1);
21101         N12 = N1.getOperand(2);
21102         if (N12.getOpcode() == ISD::Constant) {
21103           N2 = N.getOperand(2);
21104           if (N101 == N2 &&
21105               N1.Val->getValueType(0) == MVT::i32 &&
21106               N12.Val->getValueType(0) == MVT::i8) {
21107             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21108             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21109             if (!Match) {
21110               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21111             }
21112             if (Match) {
21113               Emit_144(Result, N, X86::SHLD32mri8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21114               return;
21115             }
21116           }
21117         }
21118       }
21119     }
21120 
21121     // Pattern: (store:void (X86shrd:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2, (imm:i8):$src3), addr:iPTR:$dst)
21122     // Emits: (SHRD32mri8:void addr:iPTR:$dst, GR32:i32:$src2, (imm:i8):$src3)
21123     // Pattern complexity = 28  cost = 1  size = 3
21124     if (N1.getOpcode() == X86ISD::SHRD &&
21125         InFlightSet.count(N1.Val) == 0 &&
21126         N1.hasOneUse() &&
21127         !CodeGenMap.count(N1.getValue(0))) {
21128       N10 = N1.getOperand(0);
21129       if (N10.getOpcode() == ISD::LOAD &&
21130           InFlightSet.count(N10.Val) == 0 &&
21131           N10.hasOneUse() &&
21132           !CodeGenMap.count(N10.getValue(0)) &&
21133           !isNonImmUse(N1.Val, N10.Val) &&
21134           Chain.Val == N10.Val) {
21135         Chain10 = N10.getOperand(0);
21136         N101 = N10.getOperand(1);
21137         N11 = N1.getOperand(1);
21138         N12 = N1.getOperand(2);
21139         if (N12.getOpcode() == ISD::Constant) {
21140           N2 = N.getOperand(2);
21141           if (N101 == N2 &&
21142               N1.Val->getValueType(0) == MVT::i32 &&
21143               N12.Val->getValueType(0) == MVT::i8) {
21144             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21145             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21146             if (!Match) {
21147               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21148             }
21149             if (Match) {
21150               Emit_144(Result, N, X86::SHRD32mri8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21151               return;
21152             }
21153           }
21154         }
21155       }
21156     }
21157 
21158     // Pattern: (store:void (X86shld:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2, (imm:i8):$src3), addr:iPTR:$dst)
21159     // Emits: (SHLD16mri8:void addr:iPTR:$dst, GR16:i16:$src2, (imm:i8):$src3)
21160     // Pattern complexity = 28  cost = 1  size = 3
21161     if (N1.getOpcode() == X86ISD::SHLD &&
21162         InFlightSet.count(N1.Val) == 0 &&
21163         N1.hasOneUse() &&
21164         !CodeGenMap.count(N1.getValue(0))) {
21165       N10 = N1.getOperand(0);
21166       if (N10.getOpcode() == ISD::LOAD &&
21167           InFlightSet.count(N10.Val) == 0 &&
21168           N10.hasOneUse() &&
21169           !CodeGenMap.count(N10.getValue(0)) &&
21170           !isNonImmUse(N1.Val, N10.Val) &&
21171           Chain.Val == N10.Val) {
21172         Chain10 = N10.getOperand(0);
21173         N101 = N10.getOperand(1);
21174         N11 = N1.getOperand(1);
21175         N12 = N1.getOperand(2);
21176         if (N12.getOpcode() == ISD::Constant) {
21177           N2 = N.getOperand(2);
21178           if (N101 == N2 &&
21179               N1.Val->getValueType(0) == MVT::i16 &&
21180               N12.Val->getValueType(0) == MVT::i8) {
21181             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21182             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21183             if (!Match) {
21184               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21185             }
21186             if (Match) {
21187               Emit_144(Result, N, X86::SHLD16mri8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21188               return;
21189             }
21190           }
21191         }
21192       }
21193     }
21194 
21195     // Pattern: (store:void (X86shrd:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2, (imm:i8):$src3), addr:iPTR:$dst)
21196     // Emits: (SHRD16mri8:void addr:iPTR:$dst, GR16:i16:$src2, (imm:i8):$src3)
21197     // Pattern complexity = 28  cost = 1  size = 3
21198     if (N1.getOpcode() == X86ISD::SHRD &&
21199         InFlightSet.count(N1.Val) == 0 &&
21200         N1.hasOneUse() &&
21201         !CodeGenMap.count(N1.getValue(0))) {
21202       N10 = N1.getOperand(0);
21203       if (N10.getOpcode() == ISD::LOAD &&
21204           InFlightSet.count(N10.Val) == 0 &&
21205           N10.hasOneUse() &&
21206           !CodeGenMap.count(N10.getValue(0)) &&
21207           !isNonImmUse(N1.Val, N10.Val) &&
21208           Chain.Val == N10.Val) {
21209         Chain10 = N10.getOperand(0);
21210         N101 = N10.getOperand(1);
21211         N11 = N1.getOperand(1);
21212         N12 = N1.getOperand(2);
21213         if (N12.getOpcode() == ISD::Constant) {
21214           N2 = N.getOperand(2);
21215           if (N101 == N2 &&
21216               N1.Val->getValueType(0) == MVT::i16 &&
21217               N12.Val->getValueType(0) == MVT::i8) {
21218             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21219             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21220             if (!Match) {
21221               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21222             }
21223             if (Match) {
21224               Emit_144(Result, N, X86::SHRD16mri8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21225               return;
21226             }
21227           }
21228         }
21229       }
21230     }
21231     if (N1.getOpcode() == ISD::ADD &&
21232         InFlightSet.count(N1.Val) == 0 &&
21233         N1.hasOneUse() &&
21234         !CodeGenMap.count(N1.getValue(0))) {
21235       N10 = N1.getOperand(0);
21236       if (N10.getOpcode() == ISD::LOAD &&
21237           InFlightSet.count(N10.Val) == 0 &&
21238           N10.hasOneUse() &&
21239           !CodeGenMap.count(N10.getValue(0)) &&
21240           !isNonImmUse(N1.Val, N10.Val) &&
21241           Chain.Val == N10.Val) {
21242         Chain10 = N10.getOperand(0);
21243         N101 = N10.getOperand(1);
21244         N11 = N1.getOperand(1);
21245         if (N11.getOpcode() == ISD::Constant) {
21246           N2 = N.getOperand(2);
21247           if (N101 == N2) {
21248 
21249             // Pattern: (store:void (add:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src2), addr:iPTR:$dst)
21250             // Emits: (ADD8mi:void addr:iPTR:$dst, (imm:i8):$src2)
21251             // Pattern complexity = 28  cost = 1  size = 3
21252             if (N1.Val->getValueType(0) == MVT::i8) {
21253               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21254               if (Match) {
21255                 Emit_143(Result, N, X86::ADD8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21256                 return;
21257               }
21258             }
21259 
21260             // Pattern: (store:void (add:i16 (load:i16 addr:iPTR:$dst), (imm:i16):$src2), addr:iPTR:$dst)
21261             // Emits: (ADD16mi:void addr:iPTR:$dst, (imm:i16):$src2)
21262             // Pattern complexity = 28  cost = 1  size = 3
21263             if (N1.Val->getValueType(0) == MVT::i16) {
21264               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21265               if (Match) {
21266                 Emit_140(Result, N, X86::ADD16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21267                 return;
21268               }
21269             }
21270 
21271             // Pattern: (store:void (add:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src2), addr:iPTR:$dst)
21272             // Emits: (ADD32mi:void addr:iPTR:$dst, (imm:i32):$src2)
21273             // Pattern complexity = 28  cost = 1
21274             if (N1.Val->getValueType(0) == MVT::i32) {
21275               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21276               if (Match) {
21277                 Emit_141(Result, N, X86::ADD32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21278                 return;
21279               }
21280             }
21281           }
21282         }
21283       }
21284     }
21285 
21286     // Pattern: (store:void (adde:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src2), addr:iPTR:$dst)
21287     // Emits: (ADC32mi:void addr:iPTR:$dst, (imm:i32):$src2)
21288     // Pattern complexity = 28  cost = 1  size = 3
21289     if (N1.getOpcode() == ISD::ADDE &&
21290         InFlightSet.count(N1.Val) == 0 &&
21291         N1.hasOneUse() &&
21292         !CodeGenMap.count(N1.getValue(0))) {
21293       N10 = N1.getOperand(0);
21294       if (N10.getOpcode() == ISD::LOAD &&
21295           InFlightSet.count(N10.Val) == 0 &&
21296           N10.hasOneUse() &&
21297           !CodeGenMap.count(N10.getValue(0)) &&
21298           N1.Val->isOnlyUse(N10.Val) &&
21299           !isNonImmUse(N1.Val, N10.Val) &&
21300           Chain.Val == N10.Val) {
21301         Chain10 = N10.getOperand(0);
21302         N101 = N10.getOperand(1);
21303         N11 = N1.getOperand(1);
21304         if (N11.getOpcode() == ISD::Constant) {
21305           N2 = N.getOperand(2);
21306           if (N101 == N2 &&
21307               N1.Val->getValueType(0) == MVT::i32) {
21308             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21309             if (Match) {
21310               Emit_142(Result, N, X86::ADC32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21311               return;
21312             }
21313           }
21314         }
21315       }
21316     }
21317     if (N1.getOpcode() == ISD::SUB &&
21318         InFlightSet.count(N1.Val) == 0 &&
21319         N1.hasOneUse() &&
21320         !CodeGenMap.count(N1.getValue(0))) {
21321       N10 = N1.getOperand(0);
21322       if (N10.getOpcode() == ISD::LOAD &&
21323           InFlightSet.count(N10.Val) == 0 &&
21324           N10.hasOneUse() &&
21325           !CodeGenMap.count(N10.getValue(0)) &&
21326           !isNonImmUse(N1.Val, N10.Val) &&
21327           Chain.Val == N10.Val) {
21328         Chain10 = N10.getOperand(0);
21329         N101 = N10.getOperand(1);
21330         N11 = N1.getOperand(1);
21331         if (N11.getOpcode() == ISD::Constant) {
21332           N2 = N.getOperand(2);
21333           if (N101 == N2) {
21334 
21335             // Pattern: (store:void (sub:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src2), addr:iPTR:$dst)
21336             // Emits: (SUB8mi:void addr:iPTR:$dst, (imm:i8):$src2)
21337             // Pattern complexity = 28  cost = 1  size = 3
21338             if (N1.Val->getValueType(0) == MVT::i8) {
21339               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21340               if (Match) {
21341                 Emit_143(Result, N, X86::SUB8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21342                 return;
21343               }
21344             }
21345 
21346             // Pattern: (store:void (sub:i16 (load:i16 addr:iPTR:$dst), (imm:i16):$src2), addr:iPTR:$dst)
21347             // Emits: (SUB16mi:void addr:iPTR:$dst, (imm:i16):$src2)
21348             // Pattern complexity = 28  cost = 1  size = 3
21349             if (N1.Val->getValueType(0) == MVT::i16) {
21350               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21351               if (Match) {
21352                 Emit_140(Result, N, X86::SUB16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21353                 return;
21354               }
21355             }
21356 
21357             // Pattern: (store:void (sub:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src2), addr:iPTR:$dst)
21358             // Emits: (SUB32mi:void addr:iPTR:$dst, (imm:i32):$src2)
21359             // Pattern complexity = 28  cost = 1
21360             if (N1.Val->getValueType(0) == MVT::i32) {
21361               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21362               if (Match) {
21363                 Emit_141(Result, N, X86::SUB32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21364                 return;
21365               }
21366             }
21367           }
21368         }
21369       }
21370     }
21371     if (N1.getOpcode() == ISD::SUBE &&
21372         InFlightSet.count(N1.Val) == 0 &&
21373         N1.hasOneUse() &&
21374         !CodeGenMap.count(N1.getValue(0))) {
21375       N10 = N1.getOperand(0);
21376       if (N10.getOpcode() == ISD::LOAD &&
21377           InFlightSet.count(N10.Val) == 0 &&
21378           N10.hasOneUse() &&
21379           !CodeGenMap.count(N10.getValue(0)) &&
21380           N1.Val->isOnlyUse(N10.Val) &&
21381           !isNonImmUse(N1.Val, N10.Val) &&
21382           Chain.Val == N10.Val) {
21383         Chain10 = N10.getOperand(0);
21384         N101 = N10.getOperand(1);
21385         N11 = N1.getOperand(1);
21386         if (N11.getOpcode() == ISD::Constant) {
21387           N2 = N.getOperand(2);
21388           if (N101 == N2) {
21389 
21390             // Pattern: (store:void (sube:i8 (load:i8 addr:iPTR:$dst), (imm:i8):$src2), addr:iPTR:$dst)
21391             // Emits: (SBB8mi:void addr:iPTR:$dst, (imm:i8):$src2)
21392             // Pattern complexity = 28  cost = 1  size = 3
21393             if (N1.Val->getValueType(0) == MVT::i8) {
21394               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21395               if (Match) {
21396                 Emit_145(Result, N, X86::SBB8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21397                 return;
21398               }
21399             }
21400 
21401             // Pattern: (store:void (sube:i32 (load:i32 addr:iPTR:$dst), (imm:i32):$src2), addr:iPTR:$dst)
21402             // Emits: (SBB32mi:void addr:iPTR:$dst, (imm:i32):$src2)
21403             // Pattern complexity = 28  cost = 1
21404             if (N1.Val->getValueType(0) == MVT::i32) {
21405               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21406               if (Match) {
21407                 Emit_142(Result, N, X86::SBB32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21408                 return;
21409               }
21410             }
21411           }
21412         }
21413       }
21414     }
21415     if (N1.getOpcode() == ISD::AND &&
21416         InFlightSet.count(N1.Val) == 0 &&
21417         N1.hasOneUse() &&
21418         !CodeGenMap.count(N1.getValue(0))) {
21419       N10 = N1.getOperand(0);
21420       if (N10.getOpcode() == ISD::LOAD &&
21421           InFlightSet.count(N10.Val) == 0 &&
21422           N10.hasOneUse() &&
21423           !CodeGenMap.count(N10.getValue(0)) &&
21424           !isNonImmUse(N1.Val, N10.Val) &&
21425           Chain.Val == N10.Val) {
21426         Chain10 = N10.getOperand(0);
21427         N101 = N10.getOperand(1);
21428         N11 = N1.getOperand(1);
21429         N2 = N.getOperand(2);
21430         if (N101 == N2) {
21431 
21432           // Pattern: (store:void (and:i8 (load:i8 addr:iPTR:$dst), GR8:i8:$src), addr:iPTR:$dst)
21433           // Emits: (AND8mr:void addr:iPTR:$dst, GR8:i8:$src)
21434           // Pattern complexity = 26  cost = 1  size = 3
21435           if (N1.Val->getValueType(0) == MVT::i8) {
21436             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21437             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21438             if (!Match) {
21439               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21440             }
21441             if (Match) {
21442               Emit_146(Result, N, X86::AND8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21443               return;
21444             }
21445           }
21446 
21447           // Pattern: (store:void (and:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src), addr:iPTR:$dst)
21448           // Emits: (AND16mr:void addr:iPTR:$dst, GR16:i16:$src)
21449           // Pattern complexity = 26  cost = 1  size = 3
21450           if (N1.Val->getValueType(0) == MVT::i16) {
21451             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21452             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21453             if (!Match) {
21454               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21455             }
21456             if (Match) {
21457               Emit_146(Result, N, X86::AND16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21458               return;
21459             }
21460           }
21461 
21462           // Pattern: (store:void (and:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src), addr:iPTR:$dst)
21463           // Emits: (AND32mr:void addr:iPTR:$dst, GR32:i32:$src)
21464           // Pattern complexity = 26  cost = 1
21465           if (N1.Val->getValueType(0) == MVT::i32) {
21466             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21467             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21468             if (!Match) {
21469               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21470             }
21471             if (Match) {
21472               Emit_146(Result, N, X86::AND32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21473               return;
21474             }
21475           }
21476         }
21477       }
21478     }
21479     if (N1.getOpcode() == ISD::OR &&
21480         InFlightSet.count(N1.Val) == 0 &&
21481         N1.hasOneUse() &&
21482         !CodeGenMap.count(N1.getValue(0))) {
21483       N10 = N1.getOperand(0);
21484       if (N10.getOpcode() == ISD::LOAD &&
21485           InFlightSet.count(N10.Val) == 0 &&
21486           N10.hasOneUse() &&
21487           !CodeGenMap.count(N10.getValue(0)) &&
21488           !isNonImmUse(N1.Val, N10.Val) &&
21489           Chain.Val == N10.Val) {
21490         Chain10 = N10.getOperand(0);
21491         N101 = N10.getOperand(1);
21492         N11 = N1.getOperand(1);
21493         N2 = N.getOperand(2);
21494         if (N101 == N2) {
21495 
21496           // Pattern: (store:void (or:i8 (load:i8 addr:iPTR:$dst), GR8:i8:$src), addr:iPTR:$dst)
21497           // Emits: (OR8mr:void addr:iPTR:$dst, GR8:i8:$src)
21498           // Pattern complexity = 26  cost = 1  size = 3
21499           if (N1.Val->getValueType(0) == MVT::i8) {
21500             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21501             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21502             if (!Match) {
21503               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21504             }
21505             if (Match) {
21506               Emit_146(Result, N, X86::OR8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21507               return;
21508             }
21509           }
21510 
21511           // Pattern: (store:void (or:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src), addr:iPTR:$dst)
21512           // Emits: (OR16mr:void addr:iPTR:$dst, GR16:i16:$src)
21513           // Pattern complexity = 26  cost = 1  size = 3
21514           if (N1.Val->getValueType(0) == MVT::i16) {
21515             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21516             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21517             if (!Match) {
21518               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21519             }
21520             if (Match) {
21521               Emit_146(Result, N, X86::OR16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21522               return;
21523             }
21524           }
21525 
21526           // Pattern: (store:void (or:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src), addr:iPTR:$dst)
21527           // Emits: (OR32mr:void addr:iPTR:$dst, GR32:i32:$src)
21528           // Pattern complexity = 26  cost = 1
21529           if (N1.Val->getValueType(0) == MVT::i32) {
21530             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21531             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21532             if (!Match) {
21533               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21534             }
21535             if (Match) {
21536               Emit_146(Result, N, X86::OR32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21537               return;
21538             }
21539           }
21540         }
21541       }
21542     }
21543     if (N1.getOpcode() == ISD::XOR &&
21544         InFlightSet.count(N1.Val) == 0 &&
21545         N1.hasOneUse() &&
21546         !CodeGenMap.count(N1.getValue(0))) {
21547       N10 = N1.getOperand(0);
21548       if (N10.getOpcode() == ISD::LOAD &&
21549           InFlightSet.count(N10.Val) == 0 &&
21550           N10.hasOneUse() &&
21551           !CodeGenMap.count(N10.getValue(0)) &&
21552           !isNonImmUse(N1.Val, N10.Val) &&
21553           Chain.Val == N10.Val) {
21554         Chain10 = N10.getOperand(0);
21555         N101 = N10.getOperand(1);
21556         N11 = N1.getOperand(1);
21557         N2 = N.getOperand(2);
21558         if (N101 == N2) {
21559 
21560           // Pattern: (store:void (xor:i8 (load:i8 addr:iPTR:$dst), GR8:i8:$src), addr:iPTR:$dst)
21561           // Emits: (XOR8mr:void addr:iPTR:$dst, GR8:i8:$src)
21562           // Pattern complexity = 26  cost = 1  size = 3
21563           if (N1.Val->getValueType(0) == MVT::i8) {
21564             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21565             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21566             if (!Match) {
21567               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21568             }
21569             if (Match) {
21570               Emit_146(Result, N, X86::XOR8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21571               return;
21572             }
21573           }
21574 
21575           // Pattern: (store:void (xor:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src), addr:iPTR:$dst)
21576           // Emits: (XOR16mr:void addr:iPTR:$dst, GR16:i16:$src)
21577           // Pattern complexity = 26  cost = 1  size = 3
21578           if (N1.Val->getValueType(0) == MVT::i16) {
21579             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21580             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21581             if (!Match) {
21582               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21583             }
21584             if (Match) {
21585               Emit_146(Result, N, X86::XOR16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21586               return;
21587             }
21588           }
21589 
21590           // Pattern: (store:void (xor:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src), addr:iPTR:$dst)
21591           // Emits: (XOR32mr:void addr:iPTR:$dst, GR32:i32:$src)
21592           // Pattern complexity = 26  cost = 1
21593           if (N1.Val->getValueType(0) == MVT::i32) {
21594             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21595             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21596             if (!Match) {
21597               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21598             }
21599             if (Match) {
21600               Emit_146(Result, N, X86::XOR32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21601               return;
21602             }
21603           }
21604         }
21605       }
21606     }
21607     if (N1.getOpcode() == ISD::SHL &&
21608         InFlightSet.count(N1.Val) == 0 &&
21609         N1.hasOneUse() &&
21610         !CodeGenMap.count(N1.getValue(0))) {
21611       N10 = N1.getOperand(0);
21612       if (N10.getOpcode() == ISD::LOAD &&
21613           InFlightSet.count(N10.Val) == 0 &&
21614           N10.hasOneUse() &&
21615           !CodeGenMap.count(N10.getValue(0)) &&
21616           !isNonImmUse(N1.Val, N10.Val) &&
21617           Chain.Val == N10.Val) {
21618         Chain10 = N10.getOperand(0);
21619         N101 = N10.getOperand(1);
21620         N11 = N1.getOperand(1);
21621         N2 = N.getOperand(2);
21622         if (N101 == N2) {
21623 
21624           // Pattern: (store:void (shl:i8 (load:i8 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21625           // Emits: (SHL8mCL:void addr:iPTR:$dst)
21626           // Pattern complexity = 26  cost = 1  size = 3
21627           if (N1.Val->getValueType(0) == MVT::i8 &&
21628               N11.Val->getValueType(0) == MVT::i8) {
21629             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21630             if (Match) {
21631               Emit_147(Result, N, X86::SHL8mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21632               return;
21633             }
21634           }
21635 
21636           // Pattern: (store:void (shl:i16 (load:i16 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21637           // Emits: (SHL16mCL:void addr:iPTR:$dst)
21638           // Pattern complexity = 26  cost = 1  size = 3
21639           if (N1.Val->getValueType(0) == MVT::i16 &&
21640               N11.Val->getValueType(0) == MVT::i8) {
21641             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21642             if (Match) {
21643               Emit_147(Result, N, X86::SHL16mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21644               return;
21645             }
21646           }
21647 
21648           // Pattern: (store:void (shl:i32 (load:i32 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21649           // Emits: (SHL32mCL:void addr:iPTR:$dst)
21650           // Pattern complexity = 26  cost = 1
21651           if (N1.Val->getValueType(0) == MVT::i32 &&
21652               N11.Val->getValueType(0) == MVT::i8) {
21653             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21654             if (Match) {
21655               Emit_147(Result, N, X86::SHL32mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21656               return;
21657             }
21658           }
21659         }
21660       }
21661     }
21662     if (N1.getOpcode() == ISD::SRL &&
21663         InFlightSet.count(N1.Val) == 0 &&
21664         N1.hasOneUse() &&
21665         !CodeGenMap.count(N1.getValue(0))) {
21666       N10 = N1.getOperand(0);
21667       if (N10.getOpcode() == ISD::LOAD &&
21668           InFlightSet.count(N10.Val) == 0 &&
21669           N10.hasOneUse() &&
21670           !CodeGenMap.count(N10.getValue(0)) &&
21671           !isNonImmUse(N1.Val, N10.Val) &&
21672           Chain.Val == N10.Val) {
21673         Chain10 = N10.getOperand(0);
21674         N101 = N10.getOperand(1);
21675         N11 = N1.getOperand(1);
21676         N2 = N.getOperand(2);
21677         if (N101 == N2) {
21678 
21679           // Pattern: (store:void (srl:i8 (load:i8 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21680           // Emits: (SHR8mCL:void addr:iPTR:$dst)
21681           // Pattern complexity = 26  cost = 1  size = 3
21682           if (N1.Val->getValueType(0) == MVT::i8 &&
21683               N11.Val->getValueType(0) == MVT::i8) {
21684             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21685             if (Match) {
21686               Emit_147(Result, N, X86::SHR8mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21687               return;
21688             }
21689           }
21690 
21691           // Pattern: (store:void (srl:i16 (load:i16 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21692           // Emits: (SHR16mCL:void addr:iPTR:$dst)
21693           // Pattern complexity = 26  cost = 1  size = 3
21694           if (N1.Val->getValueType(0) == MVT::i16 &&
21695               N11.Val->getValueType(0) == MVT::i8) {
21696             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21697             if (Match) {
21698               Emit_147(Result, N, X86::SHR16mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21699               return;
21700             }
21701           }
21702 
21703           // Pattern: (store:void (srl:i32 (load:i32 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21704           // Emits: (SHR32mCL:void addr:iPTR:$dst)
21705           // Pattern complexity = 26  cost = 1
21706           if (N1.Val->getValueType(0) == MVT::i32 &&
21707               N11.Val->getValueType(0) == MVT::i8) {
21708             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21709             if (Match) {
21710               Emit_147(Result, N, X86::SHR32mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21711               return;
21712             }
21713           }
21714         }
21715       }
21716     }
21717     if (N1.getOpcode() == ISD::SRA &&
21718         InFlightSet.count(N1.Val) == 0 &&
21719         N1.hasOneUse() &&
21720         !CodeGenMap.count(N1.getValue(0))) {
21721       N10 = N1.getOperand(0);
21722       if (N10.getOpcode() == ISD::LOAD &&
21723           InFlightSet.count(N10.Val) == 0 &&
21724           N10.hasOneUse() &&
21725           !CodeGenMap.count(N10.getValue(0)) &&
21726           !isNonImmUse(N1.Val, N10.Val) &&
21727           Chain.Val == N10.Val) {
21728         Chain10 = N10.getOperand(0);
21729         N101 = N10.getOperand(1);
21730         N11 = N1.getOperand(1);
21731         N2 = N.getOperand(2);
21732         if (N101 == N2) {
21733 
21734           // Pattern: (store:void (sra:i8 (load:i8 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21735           // Emits: (SAR8mCL:void addr:iPTR:$dst)
21736           // Pattern complexity = 26  cost = 1  size = 3
21737           if (N1.Val->getValueType(0) == MVT::i8 &&
21738               N11.Val->getValueType(0) == MVT::i8) {
21739             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21740             if (Match) {
21741               Emit_147(Result, N, X86::SAR8mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21742               return;
21743             }
21744           }
21745 
21746           // Pattern: (store:void (sra:i16 (load:i16 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21747           // Emits: (SAR16mCL:void addr:iPTR:$dst)
21748           // Pattern complexity = 26  cost = 1  size = 3
21749           if (N1.Val->getValueType(0) == MVT::i16 &&
21750               N11.Val->getValueType(0) == MVT::i8) {
21751             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21752             if (Match) {
21753               Emit_147(Result, N, X86::SAR16mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21754               return;
21755             }
21756           }
21757 
21758           // Pattern: (store:void (sra:i32 (load:i32 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21759           // Emits: (SAR32mCL:void addr:iPTR:$dst)
21760           // Pattern complexity = 26  cost = 1
21761           if (N1.Val->getValueType(0) == MVT::i32 &&
21762               N11.Val->getValueType(0) == MVT::i8) {
21763             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21764             if (Match) {
21765               Emit_147(Result, N, X86::SAR32mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21766               return;
21767             }
21768           }
21769         }
21770       }
21771     }
21772     if (N1.getOpcode() == ISD::ROTL &&
21773         InFlightSet.count(N1.Val) == 0 &&
21774         N1.hasOneUse() &&
21775         !CodeGenMap.count(N1.getValue(0))) {
21776       N10 = N1.getOperand(0);
21777       if (N10.getOpcode() == ISD::LOAD &&
21778           InFlightSet.count(N10.Val) == 0 &&
21779           N10.hasOneUse() &&
21780           !CodeGenMap.count(N10.getValue(0)) &&
21781           !isNonImmUse(N1.Val, N10.Val) &&
21782           Chain.Val == N10.Val) {
21783         Chain10 = N10.getOperand(0);
21784         N101 = N10.getOperand(1);
21785         N11 = N1.getOperand(1);
21786         N2 = N.getOperand(2);
21787         if (N101 == N2) {
21788 
21789           // Pattern: (store:void (rotl:i8 (load:i8 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21790           // Emits: (ROL8mCL:void addr:iPTR:$dst)
21791           // Pattern complexity = 26  cost = 1  size = 3
21792           if (N1.Val->getValueType(0) == MVT::i8 &&
21793               N11.Val->getValueType(0) == MVT::i8) {
21794             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21795             if (Match) {
21796               Emit_147(Result, N, X86::ROL8mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21797               return;
21798             }
21799           }
21800 
21801           // Pattern: (store:void (rotl:i16 (load:i16 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21802           // Emits: (ROL16mCL:void addr:iPTR:$dst)
21803           // Pattern complexity = 26  cost = 1  size = 3
21804           if (N1.Val->getValueType(0) == MVT::i16 &&
21805               N11.Val->getValueType(0) == MVT::i8) {
21806             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21807             if (Match) {
21808               Emit_147(Result, N, X86::ROL16mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21809               return;
21810             }
21811           }
21812 
21813           // Pattern: (store:void (rotl:i32 (load:i32 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21814           // Emits: (ROL32mCL:void addr:iPTR:$dst)
21815           // Pattern complexity = 26  cost = 1
21816           if (N1.Val->getValueType(0) == MVT::i32 &&
21817               N11.Val->getValueType(0) == MVT::i8) {
21818             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21819             if (Match) {
21820               Emit_147(Result, N, X86::ROL32mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21821               return;
21822             }
21823           }
21824         }
21825       }
21826     }
21827     if (N1.getOpcode() == ISD::ROTR &&
21828         InFlightSet.count(N1.Val) == 0 &&
21829         N1.hasOneUse() &&
21830         !CodeGenMap.count(N1.getValue(0))) {
21831       N10 = N1.getOperand(0);
21832       if (N10.getOpcode() == ISD::LOAD &&
21833           InFlightSet.count(N10.Val) == 0 &&
21834           N10.hasOneUse() &&
21835           !CodeGenMap.count(N10.getValue(0)) &&
21836           !isNonImmUse(N1.Val, N10.Val) &&
21837           Chain.Val == N10.Val) {
21838         Chain10 = N10.getOperand(0);
21839         N101 = N10.getOperand(1);
21840         N11 = N1.getOperand(1);
21841         N2 = N.getOperand(2);
21842         if (N101 == N2) {
21843 
21844           // Pattern: (store:void (rotr:i8 (load:i8 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21845           // Emits: (ROR8mCL:void addr:iPTR:$dst)
21846           // Pattern complexity = 26  cost = 1  size = 3
21847           if (N1.Val->getValueType(0) == MVT::i8 &&
21848               N11.Val->getValueType(0) == MVT::i8) {
21849             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21850             if (Match) {
21851               Emit_147(Result, N, X86::ROR8mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21852               return;
21853             }
21854           }
21855 
21856           // Pattern: (store:void (rotr:i16 (load:i16 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21857           // Emits: (ROR16mCL:void addr:iPTR:$dst)
21858           // Pattern complexity = 26  cost = 1  size = 3
21859           if (N1.Val->getValueType(0) == MVT::i16 &&
21860               N11.Val->getValueType(0) == MVT::i8) {
21861             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21862             if (Match) {
21863               Emit_147(Result, N, X86::ROR16mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21864               return;
21865             }
21866           }
21867 
21868           // Pattern: (store:void (rotr:i32 (load:i32 addr:iPTR:$dst), CL:i8), addr:iPTR:$dst)
21869           // Emits: (ROR32mCL:void addr:iPTR:$dst)
21870           // Pattern complexity = 26  cost = 1
21871           if (N1.Val->getValueType(0) == MVT::i32 &&
21872               N11.Val->getValueType(0) == MVT::i8) {
21873             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21874             if (Match) {
21875               Emit_147(Result, N, X86::ROR32mCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
21876               return;
21877             }
21878           }
21879         }
21880       }
21881     }
21882 
21883     // Pattern: (store:void (X86shld:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2, CL:i8), addr:iPTR:$dst)
21884     // Emits: (SHLD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
21885     // Pattern complexity = 26  cost = 1  size = 3
21886     if (N1.getOpcode() == X86ISD::SHLD &&
21887         InFlightSet.count(N1.Val) == 0 &&
21888         N1.hasOneUse() &&
21889         !CodeGenMap.count(N1.getValue(0))) {
21890       N10 = N1.getOperand(0);
21891       if (N10.getOpcode() == ISD::LOAD &&
21892           InFlightSet.count(N10.Val) == 0 &&
21893           N10.hasOneUse() &&
21894           !CodeGenMap.count(N10.getValue(0)) &&
21895           !isNonImmUse(N1.Val, N10.Val) &&
21896           Chain.Val == N10.Val) {
21897         Chain10 = N10.getOperand(0);
21898         N101 = N10.getOperand(1);
21899         N11 = N1.getOperand(1);
21900         N12 = N1.getOperand(2);
21901         N2 = N.getOperand(2);
21902         if (N101 == N2 &&
21903             N1.Val->getValueType(0) == MVT::i32 &&
21904             N12.Val->getValueType(0) == MVT::i8) {
21905           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21906           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21907           if (!Match) {
21908             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21909           }
21910           if (Match) {
21911             Emit_148(Result, N, X86::SHLD32mrCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21912             return;
21913           }
21914         }
21915       }
21916     }
21917 
21918     // Pattern: (store:void (X86shrd:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2, CL:i8), addr:iPTR:$dst)
21919     // Emits: (SHRD32mrCL:void addr:iPTR:$dst, GR32:i32:$src2)
21920     // Pattern complexity = 26  cost = 1  size = 3
21921     if (N1.getOpcode() == X86ISD::SHRD &&
21922         InFlightSet.count(N1.Val) == 0 &&
21923         N1.hasOneUse() &&
21924         !CodeGenMap.count(N1.getValue(0))) {
21925       N10 = N1.getOperand(0);
21926       if (N10.getOpcode() == ISD::LOAD &&
21927           InFlightSet.count(N10.Val) == 0 &&
21928           N10.hasOneUse() &&
21929           !CodeGenMap.count(N10.getValue(0)) &&
21930           !isNonImmUse(N1.Val, N10.Val) &&
21931           Chain.Val == N10.Val) {
21932         Chain10 = N10.getOperand(0);
21933         N101 = N10.getOperand(1);
21934         N11 = N1.getOperand(1);
21935         N12 = N1.getOperand(2);
21936         N2 = N.getOperand(2);
21937         if (N101 == N2 &&
21938             N1.Val->getValueType(0) == MVT::i32 &&
21939             N12.Val->getValueType(0) == MVT::i8) {
21940           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21941           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21942           if (!Match) {
21943             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21944           }
21945           if (Match) {
21946             Emit_148(Result, N, X86::SHRD32mrCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21947             return;
21948           }
21949         }
21950       }
21951     }
21952 
21953     // Pattern: (store:void (X86shld:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2, CL:i8), addr:iPTR:$dst)
21954     // Emits: (SHLD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
21955     // Pattern complexity = 26  cost = 1  size = 3
21956     if (N1.getOpcode() == X86ISD::SHLD &&
21957         InFlightSet.count(N1.Val) == 0 &&
21958         N1.hasOneUse() &&
21959         !CodeGenMap.count(N1.getValue(0))) {
21960       N10 = N1.getOperand(0);
21961       if (N10.getOpcode() == ISD::LOAD &&
21962           InFlightSet.count(N10.Val) == 0 &&
21963           N10.hasOneUse() &&
21964           !CodeGenMap.count(N10.getValue(0)) &&
21965           !isNonImmUse(N1.Val, N10.Val) &&
21966           Chain.Val == N10.Val) {
21967         Chain10 = N10.getOperand(0);
21968         N101 = N10.getOperand(1);
21969         N11 = N1.getOperand(1);
21970         N12 = N1.getOperand(2);
21971         N2 = N.getOperand(2);
21972         if (N101 == N2 &&
21973             N1.Val->getValueType(0) == MVT::i16 &&
21974             N12.Val->getValueType(0) == MVT::i8) {
21975           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
21976           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
21977           if (!Match) {
21978             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
21979           }
21980           if (Match) {
21981             Emit_148(Result, N, X86::SHLD16mrCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
21982             return;
21983           }
21984         }
21985       }
21986     }
21987 
21988     // Pattern: (store:void (X86shrd:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2, CL:i8), addr:iPTR:$dst)
21989     // Emits: (SHRD16mrCL:void addr:iPTR:$dst, GR16:i16:$src2)
21990     // Pattern complexity = 26  cost = 1  size = 3
21991     if (N1.getOpcode() == X86ISD::SHRD &&
21992         InFlightSet.count(N1.Val) == 0 &&
21993         N1.hasOneUse() &&
21994         !CodeGenMap.count(N1.getValue(0))) {
21995       N10 = N1.getOperand(0);
21996       if (N10.getOpcode() == ISD::LOAD &&
21997           InFlightSet.count(N10.Val) == 0 &&
21998           N10.hasOneUse() &&
21999           !CodeGenMap.count(N10.getValue(0)) &&
22000           !isNonImmUse(N1.Val, N10.Val) &&
22001           Chain.Val == N10.Val) {
22002         Chain10 = N10.getOperand(0);
22003         N101 = N10.getOperand(1);
22004         N11 = N1.getOperand(1);
22005         N12 = N1.getOperand(2);
22006         N2 = N.getOperand(2);
22007         if (N101 == N2 &&
22008             N1.Val->getValueType(0) == MVT::i16 &&
22009             N12.Val->getValueType(0) == MVT::i8) {
22010           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22011           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22012           if (!Match) {
22013             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22014           }
22015           if (Match) {
22016             Emit_148(Result, N, X86::SHRD16mrCL, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N12, N2);
22017             return;
22018           }
22019         }
22020       }
22021     }
22022     if (N1.getOpcode() == ISD::ADD &&
22023         InFlightSet.count(N1.Val) == 0 &&
22024         N1.hasOneUse() &&
22025         !CodeGenMap.count(N1.getValue(0))) {
22026       N10 = N1.getOperand(0);
22027       if (N10.getOpcode() == ISD::LOAD &&
22028           InFlightSet.count(N10.Val) == 0 &&
22029           N10.hasOneUse() &&
22030           !CodeGenMap.count(N10.getValue(0)) &&
22031           !isNonImmUse(N1.Val, N10.Val) &&
22032           Chain.Val == N10.Val) {
22033         Chain10 = N10.getOperand(0);
22034         N101 = N10.getOperand(1);
22035         N11 = N1.getOperand(1);
22036         N2 = N.getOperand(2);
22037         if (N101 == N2) {
22038 
22039           // Pattern: (store:void (add:i8 (load:i8 addr:iPTR:$dst), GR8:i8:$src2), addr:iPTR:$dst)
22040           // Emits: (ADD8mr:void addr:iPTR:$dst, GR8:i8:$src2)
22041           // Pattern complexity = 26  cost = 1  size = 3
22042           if (N1.Val->getValueType(0) == MVT::i8) {
22043             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22044             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22045             if (!Match) {
22046               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22047             }
22048             if (Match) {
22049               Emit_146(Result, N, X86::ADD8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22050               return;
22051             }
22052           }
22053 
22054           // Pattern: (store:void (add:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2), addr:iPTR:$dst)
22055           // Emits: (ADD16mr:void addr:iPTR:$dst, GR16:i16:$src2)
22056           // Pattern complexity = 26  cost = 1  size = 3
22057           if (N1.Val->getValueType(0) == MVT::i16) {
22058             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22059             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22060             if (!Match) {
22061               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22062             }
22063             if (Match) {
22064               Emit_146(Result, N, X86::ADD16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22065               return;
22066             }
22067           }
22068 
22069           // Pattern: (store:void (add:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2), addr:iPTR:$dst)
22070           // Emits: (ADD32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22071           // Pattern complexity = 26  cost = 1
22072           if (N1.Val->getValueType(0) == MVT::i32) {
22073             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22074             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22075             if (!Match) {
22076               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22077             }
22078             if (Match) {
22079               Emit_146(Result, N, X86::ADD32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22080               return;
22081             }
22082           }
22083         }
22084       }
22085     }
22086 
22087     // Pattern: (store:void (adde:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2), addr:iPTR:$dst)
22088     // Emits: (ADC32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22089     // Pattern complexity = 26  cost = 1  size = 3
22090     if (N1.getOpcode() == ISD::ADDE &&
22091         InFlightSet.count(N1.Val) == 0 &&
22092         N1.hasOneUse() &&
22093         !CodeGenMap.count(N1.getValue(0))) {
22094       N10 = N1.getOperand(0);
22095       if (N10.getOpcode() == ISD::LOAD &&
22096           InFlightSet.count(N10.Val) == 0 &&
22097           N10.hasOneUse() &&
22098           !CodeGenMap.count(N10.getValue(0)) &&
22099           N1.Val->isOnlyUse(N10.Val) &&
22100           !isNonImmUse(N1.Val, N10.Val) &&
22101           Chain.Val == N10.Val) {
22102         Chain10 = N10.getOperand(0);
22103         N101 = N10.getOperand(1);
22104         N11 = N1.getOperand(1);
22105         N2 = N.getOperand(2);
22106         if (N101 == N2 &&
22107             N1.Val->getValueType(0) == MVT::i32) {
22108           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22109           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22110           if (!Match) {
22111             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22112           }
22113           if (Match) {
22114             Emit_149(Result, N, X86::ADC32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22115             return;
22116           }
22117         }
22118       }
22119     }
22120     if (N1.getOpcode() == ISD::SUB &&
22121         InFlightSet.count(N1.Val) == 0 &&
22122         N1.hasOneUse() &&
22123         !CodeGenMap.count(N1.getValue(0))) {
22124       N10 = N1.getOperand(0);
22125       if (N10.getOpcode() == ISD::LOAD &&
22126           InFlightSet.count(N10.Val) == 0 &&
22127           N10.hasOneUse() &&
22128           !CodeGenMap.count(N10.getValue(0)) &&
22129           !isNonImmUse(N1.Val, N10.Val) &&
22130           Chain.Val == N10.Val) {
22131         Chain10 = N10.getOperand(0);
22132         N101 = N10.getOperand(1);
22133         N11 = N1.getOperand(1);
22134         N2 = N.getOperand(2);
22135         if (N101 == N2) {
22136 
22137           // Pattern: (store:void (sub:i8 (load:i8 addr:iPTR:$dst), GR8:i8:$src2), addr:iPTR:$dst)
22138           // Emits: (SUB8mr:void addr:iPTR:$dst, GR8:i8:$src2)
22139           // Pattern complexity = 26  cost = 1  size = 3
22140           if (N1.Val->getValueType(0) == MVT::i8) {
22141             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22142             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22143             if (!Match) {
22144               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22145             }
22146             if (Match) {
22147               Emit_146(Result, N, X86::SUB8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22148               return;
22149             }
22150           }
22151 
22152           // Pattern: (store:void (sub:i16 (load:i16 addr:iPTR:$dst), GR16:i16:$src2), addr:iPTR:$dst)
22153           // Emits: (SUB16mr:void addr:iPTR:$dst, GR16:i16:$src2)
22154           // Pattern complexity = 26  cost = 1  size = 3
22155           if (N1.Val->getValueType(0) == MVT::i16) {
22156             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22157             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22158             if (!Match) {
22159               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22160             }
22161             if (Match) {
22162               Emit_146(Result, N, X86::SUB16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22163               return;
22164             }
22165           }
22166 
22167           // Pattern: (store:void (sub:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2), addr:iPTR:$dst)
22168           // Emits: (SUB32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22169           // Pattern complexity = 26  cost = 1
22170           if (N1.Val->getValueType(0) == MVT::i32) {
22171             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22172             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22173             if (!Match) {
22174               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22175             }
22176             if (Match) {
22177               Emit_146(Result, N, X86::SUB32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22178               return;
22179             }
22180           }
22181         }
22182       }
22183     }
22184 
22185     // Pattern: (store:void (sube:i32 (load:i32 addr:iPTR:$dst), GR32:i32:$src2), addr:iPTR:$dst)
22186     // Emits: (SBB32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22187     // Pattern complexity = 26  cost = 1
22188     if (N1.getOpcode() == ISD::SUBE &&
22189         InFlightSet.count(N1.Val) == 0 &&
22190         N1.hasOneUse() &&
22191         !CodeGenMap.count(N1.getValue(0))) {
22192       N10 = N1.getOperand(0);
22193       if (N10.getOpcode() == ISD::LOAD &&
22194           InFlightSet.count(N10.Val) == 0 &&
22195           N10.hasOneUse() &&
22196           !CodeGenMap.count(N10.getValue(0)) &&
22197           N1.Val->isOnlyUse(N10.Val) &&
22198           !isNonImmUse(N1.Val, N10.Val) &&
22199           Chain.Val == N10.Val) {
22200         Chain10 = N10.getOperand(0);
22201         N101 = N10.getOperand(1);
22202         N11 = N1.getOperand(1);
22203         N2 = N.getOperand(2);
22204         if (N101 == N2 &&
22205             N1.Val->getValueType(0) == MVT::i32) {
22206           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N11.Val);
22207           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22208           if (!Match) {
22209             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N11.Val);
22210           }
22211           if (Match) {
22212             Emit_149(Result, N, X86::SBB32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain10, N1, N10, N101, N11, N2);
22213             return;
22214           }
22215         }
22216       }
22217     }
22218   }
22219 
22220   // Pattern: (store:void (vector_extract:f64 (vector_shuffle:v2f64 (bitconvert:v2f64 VR128:v4f32:$src), (undef:v2f64), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>), 0:iPTR), addr:iPTR:$dst)
22221   // Emits: (MOVHPSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
22222   // Pattern complexity = 26  cost = 1  size = 3
22223   if ((Subtarget->hasSSE1())) {
22224     Chain = N.getOperand(0);
22225     N1 = N.getOperand(1);
22226     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22227       N10 = N1.getOperand(0);
22228       if (N10.getOpcode() == ISD::VECTOR_SHUFFLE) {
22229         N100 = N10.getOperand(0);
22230         if (N100.getOpcode() == ISD::BIT_CONVERT) {
22231           N1000 = N100.getOperand(0);
22232           N101 = N10.getOperand(1);
22233           if (N101.getOpcode() == ISD::UNDEF) {
22234             N102 = N10.getOperand(2);
22235             if (N102.getOpcode() == ISD::BUILD_VECTOR &&
22236                 Predicate_UNPCKH_shuffle_mask(N102.Val)) {
22237               N11 = N1.getOperand(1);
22238               if (isa<ConstantSDNode>(N11)) {
22239                 int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22240                 if (CN0 == 0) {
22241                   N2 = N.getOperand(2);
22242                   if (N1.Val->getValueType(0) == MVT::f64 &&
22243                       N10.Val->getValueType(0) == MVT::v2f64 &&
22244                       N1000.Val->getValueType(0) == MVT::v4f32) {
22245                     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1000.Val);
22246                     bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22247                     if (!Match) {
22248                       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1000.Val);
22249                     }
22250                     if (Match) {
22251                       Emit_150(Result, N, X86::MOVHPSmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N100, N1000, N101, N102, N11, N2);
22252                       return;
22253                     }
22254                   }
22255                 }
22256               }
22257             }
22258           }
22259         }
22260       }
22261     }
22262   }
22263   {
22264     Chain = N.getOperand(0);
22265     N1 = N.getOperand(1);
22266     if (N1.getOpcode() == ISD::AND &&
22267         InFlightSet.count(N1.Val) == 0 &&
22268         N1.hasOneUse() &&
22269         !CodeGenMap.count(N1.getValue(0))) {
22270       N10 = N1.getOperand(0);
22271       N11 = N1.getOperand(1);
22272       if (N11.getOpcode() == ISD::LOAD &&
22273           InFlightSet.count(N11.Val) == 0 &&
22274           N11.hasOneUse() &&
22275           !CodeGenMap.count(N11.getValue(0)) &&
22276           !isNonImmUse(N1.Val, N11.Val) &&
22277           Chain.Val == N11.Val) {
22278         Chain11 = N11.getOperand(0);
22279         N111 = N11.getOperand(1);
22280         N2 = N.getOperand(2);
22281         if (N111 == N2) {
22282 
22283           // Pattern: (store:void (and:i8 GR8:i8:$src, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
22284           // Emits: (AND8mr:void addr:iPTR:$dst, GR8:i8:$src)
22285           // Pattern complexity = 26  cost = 1  size = 3
22286           if (N1.Val->getValueType(0) == MVT::i8) {
22287             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22288             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22289             if (!Match) {
22290               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22291             }
22292             if (Match) {
22293               Emit_151(Result, N, X86::AND8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22294               return;
22295             }
22296           }
22297 
22298           // Pattern: (store:void (and:i16 GR16:i16:$src, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
22299           // Emits: (AND16mr:void addr:iPTR:$dst, GR16:i16:$src)
22300           // Pattern complexity = 26  cost = 1  size = 3
22301           if (N1.Val->getValueType(0) == MVT::i16) {
22302             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22303             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22304             if (!Match) {
22305               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22306             }
22307             if (Match) {
22308               Emit_151(Result, N, X86::AND16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22309               return;
22310             }
22311           }
22312 
22313           // Pattern: (store:void (and:i32 GR32:i32:$src, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
22314           // Emits: (AND32mr:void addr:iPTR:$dst, GR32:i32:$src)
22315           // Pattern complexity = 26  cost = 1
22316           if (N1.Val->getValueType(0) == MVT::i32) {
22317             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22318             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22319             if (!Match) {
22320               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22321             }
22322             if (Match) {
22323               Emit_151(Result, N, X86::AND32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22324               return;
22325             }
22326           }
22327         }
22328       }
22329     }
22330     if (N1.getOpcode() == ISD::OR &&
22331         InFlightSet.count(N1.Val) == 0 &&
22332         N1.hasOneUse() &&
22333         !CodeGenMap.count(N1.getValue(0))) {
22334       N10 = N1.getOperand(0);
22335       N11 = N1.getOperand(1);
22336       if (N11.getOpcode() == ISD::LOAD &&
22337           InFlightSet.count(N11.Val) == 0 &&
22338           N11.hasOneUse() &&
22339           !CodeGenMap.count(N11.getValue(0)) &&
22340           !isNonImmUse(N1.Val, N11.Val) &&
22341           Chain.Val == N11.Val) {
22342         Chain11 = N11.getOperand(0);
22343         N111 = N11.getOperand(1);
22344         N2 = N.getOperand(2);
22345         if (N111 == N2) {
22346 
22347           // Pattern: (store:void (or:i8 GR8:i8:$src, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
22348           // Emits: (OR8mr:void addr:iPTR:$dst, GR8:i8:$src)
22349           // Pattern complexity = 26  cost = 1  size = 3
22350           if (N1.Val->getValueType(0) == MVT::i8) {
22351             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22352             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22353             if (!Match) {
22354               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22355             }
22356             if (Match) {
22357               Emit_151(Result, N, X86::OR8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22358               return;
22359             }
22360           }
22361 
22362           // Pattern: (store:void (or:i16 GR16:i16:$src, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
22363           // Emits: (OR16mr:void addr:iPTR:$dst, GR16:i16:$src)
22364           // Pattern complexity = 26  cost = 1  size = 3
22365           if (N1.Val->getValueType(0) == MVT::i16) {
22366             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22367             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22368             if (!Match) {
22369               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22370             }
22371             if (Match) {
22372               Emit_151(Result, N, X86::OR16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22373               return;
22374             }
22375           }
22376 
22377           // Pattern: (store:void (or:i32 GR32:i32:$src, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
22378           // Emits: (OR32mr:void addr:iPTR:$dst, GR32:i32:$src)
22379           // Pattern complexity = 26  cost = 1
22380           if (N1.Val->getValueType(0) == MVT::i32) {
22381             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22382             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22383             if (!Match) {
22384               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22385             }
22386             if (Match) {
22387               Emit_151(Result, N, X86::OR32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22388               return;
22389             }
22390           }
22391         }
22392       }
22393     }
22394     if (N1.getOpcode() == ISD::XOR &&
22395         InFlightSet.count(N1.Val) == 0 &&
22396         N1.hasOneUse() &&
22397         !CodeGenMap.count(N1.getValue(0))) {
22398       N10 = N1.getOperand(0);
22399       N11 = N1.getOperand(1);
22400       if (N11.getOpcode() == ISD::LOAD &&
22401           InFlightSet.count(N11.Val) == 0 &&
22402           N11.hasOneUse() &&
22403           !CodeGenMap.count(N11.getValue(0)) &&
22404           !isNonImmUse(N1.Val, N11.Val) &&
22405           Chain.Val == N11.Val) {
22406         Chain11 = N11.getOperand(0);
22407         N111 = N11.getOperand(1);
22408         N2 = N.getOperand(2);
22409         if (N111 == N2) {
22410 
22411           // Pattern: (store:void (xor:i8 GR8:i8:$src, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
22412           // Emits: (XOR8mr:void addr:iPTR:$dst, GR8:i8:$src)
22413           // Pattern complexity = 26  cost = 1  size = 3
22414           if (N1.Val->getValueType(0) == MVT::i8) {
22415             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22416             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22417             if (!Match) {
22418               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22419             }
22420             if (Match) {
22421               Emit_151(Result, N, X86::XOR8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22422               return;
22423             }
22424           }
22425 
22426           // Pattern: (store:void (xor:i16 GR16:i16:$src, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
22427           // Emits: (XOR16mr:void addr:iPTR:$dst, GR16:i16:$src)
22428           // Pattern complexity = 26  cost = 1  size = 3
22429           if (N1.Val->getValueType(0) == MVT::i16) {
22430             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22431             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22432             if (!Match) {
22433               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22434             }
22435             if (Match) {
22436               Emit_151(Result, N, X86::XOR16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22437               return;
22438             }
22439           }
22440 
22441           // Pattern: (store:void (xor:i32 GR32:i32:$src, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
22442           // Emits: (XOR32mr:void addr:iPTR:$dst, GR32:i32:$src)
22443           // Pattern complexity = 26  cost = 1
22444           if (N1.Val->getValueType(0) == MVT::i32) {
22445             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22446             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22447             if (!Match) {
22448               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22449             }
22450             if (Match) {
22451               Emit_151(Result, N, X86::XOR32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22452               return;
22453             }
22454           }
22455         }
22456       }
22457     }
22458     if (N1.getOpcode() == ISD::ADD &&
22459         InFlightSet.count(N1.Val) == 0 &&
22460         N1.hasOneUse() &&
22461         !CodeGenMap.count(N1.getValue(0))) {
22462       N10 = N1.getOperand(0);
22463       N11 = N1.getOperand(1);
22464       if (N11.getOpcode() == ISD::LOAD &&
22465           InFlightSet.count(N11.Val) == 0 &&
22466           N11.hasOneUse() &&
22467           !CodeGenMap.count(N11.getValue(0)) &&
22468           !isNonImmUse(N1.Val, N11.Val) &&
22469           Chain.Val == N11.Val) {
22470         Chain11 = N11.getOperand(0);
22471         N111 = N11.getOperand(1);
22472         N2 = N.getOperand(2);
22473         if (N111 == N2) {
22474 
22475           // Pattern: (store:void (add:i8 GR8:i8:$src2, (load:i8 addr:iPTR:$dst)), addr:iPTR:$dst)
22476           // Emits: (ADD8mr:void addr:iPTR:$dst, GR8:i8:$src2)
22477           // Pattern complexity = 26  cost = 1  size = 3
22478           if (N1.Val->getValueType(0) == MVT::i8) {
22479             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22480             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22481             if (!Match) {
22482               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22483             }
22484             if (Match) {
22485               Emit_151(Result, N, X86::ADD8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22486               return;
22487             }
22488           }
22489 
22490           // Pattern: (store:void (add:i16 GR16:i16:$src2, (load:i16 addr:iPTR:$dst)), addr:iPTR:$dst)
22491           // Emits: (ADD16mr:void addr:iPTR:$dst, GR16:i16:$src2)
22492           // Pattern complexity = 26  cost = 1  size = 3
22493           if (N1.Val->getValueType(0) == MVT::i16) {
22494             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22495             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22496             if (!Match) {
22497               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22498             }
22499             if (Match) {
22500               Emit_151(Result, N, X86::ADD16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22501               return;
22502             }
22503           }
22504 
22505           // Pattern: (store:void (add:i32 GR32:i32:$src2, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
22506           // Emits: (ADD32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22507           // Pattern complexity = 26  cost = 1
22508           if (N1.Val->getValueType(0) == MVT::i32) {
22509             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22510             bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22511             if (!Match) {
22512               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22513             }
22514             if (Match) {
22515               Emit_151(Result, N, X86::ADD32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22516               return;
22517             }
22518           }
22519         }
22520       }
22521     }
22522 
22523     // Pattern: (store:void (adde:i32 GR32:i32:$src2, (load:i32 addr:iPTR:$dst)), addr:iPTR:$dst)
22524     // Emits: (ADC32mr:void addr:iPTR:$dst, GR32:i32:$src2)
22525     // Pattern complexity = 26  cost = 1
22526     if (N1.getOpcode() == ISD::ADDE &&
22527         InFlightSet.count(N1.Val) == 0 &&
22528         N1.hasOneUse() &&
22529         !CodeGenMap.count(N1.getValue(0))) {
22530       N10 = N1.getOperand(0);
22531       N11 = N1.getOperand(1);
22532       if (N11.getOpcode() == ISD::LOAD &&
22533           InFlightSet.count(N11.Val) == 0 &&
22534           N11.hasOneUse() &&
22535           !CodeGenMap.count(N11.getValue(0)) &&
22536           N1.Val->isOnlyUse(N11.Val) &&
22537           !isNonImmUse(N1.Val, N11.Val) &&
22538           Chain.Val == N11.Val) {
22539         Chain11 = N11.getOperand(0);
22540         N111 = N11.getOperand(1);
22541         N2 = N.getOperand(2);
22542         if (N111 == N2 &&
22543             N1.Val->getValueType(0) == MVT::i32) {
22544           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22545           bool Match = SelectAddr(N111, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22546           if (!Match) {
22547             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22548           }
22549           if (Match) {
22550             Emit_152(Result, N, X86::ADC32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, Chain11, N1, N10, N11, N111, N2);
22551             return;
22552           }
22553         }
22554       }
22555     }
22556   }
22557 
22558   // Pattern: (store:void (vector_extract:f64 (vector_shuffle:v2f64 VR128:v2f64:$src, (undef:v2f64), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>), 0:iPTR), addr:iPTR:$dst)
22559   // Emits: (MOVHPDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
22560   // Pattern complexity = 24  cost = 1  size = 3
22561   if ((Subtarget->hasSSE2())) {
22562     Chain = N.getOperand(0);
22563     N1 = N.getOperand(1);
22564     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22565       N10 = N1.getOperand(0);
22566       if (N10.getOpcode() == ISD::VECTOR_SHUFFLE) {
22567         N100 = N10.getOperand(0);
22568         N101 = N10.getOperand(1);
22569         if (N101.getOpcode() == ISD::UNDEF) {
22570           N102 = N10.getOperand(2);
22571           if (N102.getOpcode() == ISD::BUILD_VECTOR &&
22572               Predicate_UNPCKH_shuffle_mask(N102.Val)) {
22573             N11 = N1.getOperand(1);
22574             if (isa<ConstantSDNode>(N11)) {
22575               int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22576               if (CN0 == 0) {
22577                 N2 = N.getOperand(2);
22578                 if (N1.Val->getValueType(0) == MVT::f64 &&
22579                     N10.Val->getValueType(0) == MVT::v2f64) {
22580                   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
22581                   bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22582                   if (!Match) {
22583                     SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
22584                   }
22585                   if (Match) {
22586                     Emit_153(Result, N, X86::MOVHPDmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N100, N101, N102, N11, N2);
22587                     return;
22588                   }
22589                 }
22590               }
22591             }
22592           }
22593         }
22594       }
22595     }
22596   }
22597 
22598   // Pattern: (store:void (vector_extract:f64 (bitconvert:v2f64 VR128:v4f32:$src), 0:iPTR), addr:iPTR:$dst)
22599   // Emits: (MOVLPSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
22600   // Pattern complexity = 19  cost = 1  size = 3
22601   if ((Subtarget->hasSSE1())) {
22602     Chain = N.getOperand(0);
22603     N1 = N.getOperand(1);
22604     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22605       N10 = N1.getOperand(0);
22606       if (N10.getOpcode() == ISD::BIT_CONVERT) {
22607         N100 = N10.getOperand(0);
22608         N11 = N1.getOperand(1);
22609         if (isa<ConstantSDNode>(N11)) {
22610           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22611           if (CN0 == 0) {
22612             N2 = N.getOperand(2);
22613             if (N1.Val->getValueType(0) == MVT::f64 &&
22614                 N10.Val->getValueType(0) == MVT::v2f64 &&
22615                 N100.Val->getValueType(0) == MVT::v4f32) {
22616               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N100.Val);
22617               bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22618               if (!Match) {
22619                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N100.Val);
22620               }
22621               if (Match) {
22622                 Emit_154(Result, N, X86::MOVLPSmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N100, N11, N2);
22623                 return;
22624               }
22625             }
22626           }
22627         }
22628       }
22629     }
22630   }
22631   {
22632     Chain = N.getOperand(0);
22633     N1 = N.getOperand(1);
22634     if (N1.getOpcode() == X86ISD::SETCC &&
22635         N1.hasOneUse() &&
22636         !CodeGenMap.count(N1.getValue(0))) {
22637       N10 = N1.getOperand(0);
22638       if (isa<ConstantSDNode>(N10)) {
22639         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
22640 
22641         // Pattern: (store:void (X86setcc:i8 4:i8), addr:iPTR:$dst)
22642         // Emits: (SETEm:void addr:iPTR:$dst)
22643         // Pattern complexity = 17  cost = 1  size = 3
22644         if (CN0 == 4) {
22645           N2 = N.getOperand(2);
22646           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22647           if (Match) {
22648             Emit_155(Result, N, X86::SETEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22649             return;
22650           }
22651         }
22652 
22653         // Pattern: (store:void (X86setcc:i8 9:i8), addr:iPTR:$dst)
22654         // Emits: (SETNEm:void addr:iPTR:$dst)
22655         // Pattern complexity = 17  cost = 1  size = 3
22656         if (CN0 == 9) {
22657           N2 = N.getOperand(2);
22658           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22659           if (Match) {
22660             Emit_155(Result, N, X86::SETNEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22661             return;
22662           }
22663         }
22664 
22665         // Pattern: (store:void (X86setcc:i8 7:i8), addr:iPTR:$dst)
22666         // Emits: (SETLm:void addr:iPTR:$dst)
22667         // Pattern complexity = 17  cost = 1  size = 3
22668         if (CN0 == 7) {
22669           N2 = N.getOperand(2);
22670           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22671           if (Match) {
22672             Emit_155(Result, N, X86::SETLm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22673             return;
22674           }
22675         }
22676 
22677         // Pattern: (store:void (X86setcc:i8 6:i8), addr:iPTR:$dst)
22678         // Emits: (SETGEm:void addr:iPTR:$dst)
22679         // Pattern complexity = 17  cost = 1  size = 3
22680         if (CN0 == 6) {
22681           N2 = N.getOperand(2);
22682           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22683           if (Match) {
22684             Emit_155(Result, N, X86::SETGEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22685             return;
22686           }
22687         }
22688 
22689         // Pattern: (store:void (X86setcc:i8 8:i8), addr:iPTR:$dst)
22690         // Emits: (SETLEm:void addr:iPTR:$dst)
22691         // Pattern complexity = 17  cost = 1  size = 3
22692         if (CN0 == 8) {
22693           N2 = N.getOperand(2);
22694           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22695           if (Match) {
22696             Emit_155(Result, N, X86::SETLEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22697             return;
22698           }
22699         }
22700 
22701         // Pattern: (store:void (X86setcc:i8 5:i8), addr:iPTR:$dst)
22702         // Emits: (SETGm:void addr:iPTR:$dst)
22703         // Pattern complexity = 17  cost = 1  size = 3
22704         if (CN0 == 5) {
22705           N2 = N.getOperand(2);
22706           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22707           if (Match) {
22708             Emit_155(Result, N, X86::SETGm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22709             return;
22710           }
22711         }
22712 
22713         // Pattern: (store:void (X86setcc:i8 2:i8), addr:iPTR:$dst)
22714         // Emits: (SETBm:void addr:iPTR:$dst)
22715         // Pattern complexity = 17  cost = 1  size = 3
22716         if (CN0 == 2) {
22717           N2 = N.getOperand(2);
22718           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22719           if (Match) {
22720             Emit_155(Result, N, X86::SETBm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22721             return;
22722           }
22723         }
22724 
22725         // Pattern: (store:void (X86setcc:i8 1:i8), addr:iPTR:$dst)
22726         // Emits: (SETAEm:void addr:iPTR:$dst)
22727         // Pattern complexity = 17  cost = 1  size = 3
22728         if (CN0 == 1) {
22729           N2 = N.getOperand(2);
22730           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22731           if (Match) {
22732             Emit_155(Result, N, X86::SETAEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22733             return;
22734           }
22735         }
22736 
22737         // Pattern: (store:void (X86setcc:i8 3:i8), addr:iPTR:$dst)
22738         // Emits: (SETBEm:void addr:iPTR:$dst)
22739         // Pattern complexity = 17  cost = 1  size = 3
22740         if (CN0 == 3) {
22741           N2 = N.getOperand(2);
22742           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22743           if (Match) {
22744             Emit_155(Result, N, X86::SETBEm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22745             return;
22746           }
22747         }
22748 
22749         // Pattern: (store:void (X86setcc:i8 0:i8), addr:iPTR:$dst)
22750         // Emits: (SETAm:void addr:iPTR:$dst)
22751         // Pattern complexity = 17  cost = 1  size = 3
22752         if (CN0 == 0) {
22753           N2 = N.getOperand(2);
22754           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22755           if (Match) {
22756             Emit_155(Result, N, X86::SETAm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22757             return;
22758           }
22759         }
22760 
22761         // Pattern: (store:void (X86setcc:i8 15:i8), addr:iPTR:$dst)
22762         // Emits: (SETSm:void addr:iPTR:$dst)
22763         // Pattern complexity = 17  cost = 1  size = 3
22764         if (CN0 == 15) {
22765           N2 = N.getOperand(2);
22766           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22767           if (Match) {
22768             Emit_155(Result, N, X86::SETSm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22769             return;
22770           }
22771         }
22772 
22773         // Pattern: (store:void (X86setcc:i8 12:i8), addr:iPTR:$dst)
22774         // Emits: (SETNSm:void addr:iPTR:$dst)
22775         // Pattern complexity = 17  cost = 1  size = 3
22776         if (CN0 == 12) {
22777           N2 = N.getOperand(2);
22778           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22779           if (Match) {
22780             Emit_155(Result, N, X86::SETNSm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22781             return;
22782           }
22783         }
22784 
22785         // Pattern: (store:void (X86setcc:i8 14:i8), addr:iPTR:$dst)
22786         // Emits: (SETPm:void addr:iPTR:$dst)
22787         // Pattern complexity = 17  cost = 1  size = 3
22788         if (CN0 == 14) {
22789           N2 = N.getOperand(2);
22790           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22791           if (Match) {
22792             Emit_155(Result, N, X86::SETPm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22793             return;
22794           }
22795         }
22796 
22797         // Pattern: (store:void (X86setcc:i8 11:i8), addr:iPTR:$dst)
22798         // Emits: (SETNPm:void addr:iPTR:$dst)
22799         // Pattern complexity = 17  cost = 1
22800         if (CN0 == 11) {
22801           N2 = N.getOperand(2);
22802           bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22803           if (Match) {
22804             Emit_155(Result, N, X86::SETNPm, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N2);
22805             return;
22806           }
22807         }
22808       }
22809     }
22810   }
22811 
22812   // Pattern: (store:void (vector_extract:f64 VR128:v2f64:$src, 0:iPTR), addr:iPTR:$dst)
22813   // Emits: (MOVLPDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
22814   // Pattern complexity = 17  cost = 1  size = 3
22815   if ((Subtarget->hasSSE2())) {
22816     Chain = N.getOperand(0);
22817     N1 = N.getOperand(1);
22818     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22819       N10 = N1.getOperand(0);
22820       N11 = N1.getOperand(1);
22821       if (isa<ConstantSDNode>(N11)) {
22822         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22823         if (CN0 == 0) {
22824           N2 = N.getOperand(2);
22825           if (N1.Val->getValueType(0) == MVT::f64 &&
22826               N10.Val->getValueType(0) == MVT::v2f64) {
22827             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22828             bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22829             if (!Match) {
22830               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22831             }
22832             if (Match) {
22833               Emit_156(Result, N, X86::MOVLPDmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N11, N2);
22834               return;
22835             }
22836           }
22837         }
22838       }
22839     }
22840   }
22841 
22842   // Pattern: (store:void (vector_extract:f32 VR128:v4f32:$src, 0:iPTR), addr:iPTR:$dst)
22843   // Emits: (MOVPS2SSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
22844   // Pattern complexity = 17  cost = 1  size = 3
22845   if ((Subtarget->hasSSE1())) {
22846     Chain = N.getOperand(0);
22847     N1 = N.getOperand(1);
22848     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22849       N10 = N1.getOperand(0);
22850       N11 = N1.getOperand(1);
22851       if (isa<ConstantSDNode>(N11)) {
22852         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22853         if (CN0 == 0) {
22854           N2 = N.getOperand(2);
22855           if (N1.Val->getValueType(0) == MVT::f32 &&
22856               N10.Val->getValueType(0) == MVT::v4f32) {
22857             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22858             bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22859             if (!Match) {
22860               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22861             }
22862             if (Match) {
22863               Emit_156(Result, N, X86::MOVPS2SSmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N11, N2);
22864               return;
22865             }
22866           }
22867         }
22868       }
22869     }
22870   }
22871   if ((Subtarget->hasSSE2())) {
22872     Chain = N.getOperand(0);
22873     N1 = N.getOperand(1);
22874     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22875       N10 = N1.getOperand(0);
22876       N11 = N1.getOperand(1);
22877       if (isa<ConstantSDNode>(N11)) {
22878         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22879         if (CN0 == 0) {
22880           N2 = N.getOperand(2);
22881 
22882           // Pattern: (store:void (vector_extract:f64 VR128:v2f64:$src, 0:iPTR), addr:iPTR:$dst)
22883           // Emits: (MOVPD2SDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
22884           // Pattern complexity = 17  cost = 1  size = 3
22885           if (N1.Val->getValueType(0) == MVT::f64 &&
22886               N10.Val->getValueType(0) == MVT::v2f64) {
22887             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22888             bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22889             if (!Match) {
22890               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22891             }
22892             if (Match) {
22893               Emit_156(Result, N, X86::MOVPD2SDmr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N11, N2);
22894               return;
22895             }
22896           }
22897 
22898           // Pattern: (store:void (vector_extract:i32 VR128:v4i32:$src, 0:iPTR), addr:iPTR:$dst)
22899           // Emits: (MOVPDI2DImr:void addr:iPTR:$dst, VR128:v4i32:$src)
22900           // Pattern complexity = 17  cost = 1
22901           if (N1.Val->getValueType(0) == MVT::i32 &&
22902               N10.Val->getValueType(0) == MVT::v4i32) {
22903             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
22904             bool Match = SelectAddr(N2, CPTmp1, CPTmp2, CPTmp3, CPTmp4);
22905             if (!Match) {
22906               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
22907             }
22908             if (Match) {
22909               Emit_156(Result, N, X86::MOVPDI2DImr, CPTmp1, CPTmp2, CPTmp3, CPTmp4, Chain, N1, N10, N11, N2);
22910               return;
22911             }
22912           }
22913         }
22914       }
22915     }
22916   }
22917   {
22918     Chain = N.getOperand(0);
22919     N1 = N.getOperand(1);
22920     if (N1.getOpcode() == X86ISD::Wrapper) {
22921       N10 = N1.getOperand(0);
22922 
22923       // Pattern: (store:void (X86Wrapper:i32 (tglobaladdr:i32):$src), addr:iPTR:$dst)
22924       // Emits: (MOV32mi:void addr:iPTR:$dst, (tglobaladdr:i32):$src)
22925       // Pattern complexity = 16  cost = 1  size = 3
22926       if (N10.getOpcode() == ISD::TargetGlobalAddress) {
22927         N2 = N.getOperand(2);
22928         if (N1.Val->getValueType(0) == MVT::i32) {
22929           bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22930           if (Match) {
22931             Emit_157(Result, N, X86::MOV32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N10, N2);
22932             return;
22933           }
22934         }
22935       }
22936 
22937       // Pattern: (store:void (X86Wrapper:i32 (texternalsym:i32):$src), addr:iPTR:$dst)
22938       // Emits: (MOV32mi:void addr:iPTR:$dst, (texternalsym:i32):$src)
22939       // Pattern complexity = 16  cost = 1
22940       if (N10.getOpcode() == ISD::TargetExternalSymbol) {
22941         N2 = N.getOperand(2);
22942         if (N1.Val->getValueType(0) == MVT::i32) {
22943           bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22944           if (Match) {
22945             Emit_157(Result, N, X86::MOV32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N10, N2);
22946             return;
22947           }
22948         }
22949       }
22950     }
22951     if (N1.getOpcode() == ISD::Constant) {
22952       N2 = N.getOperand(2);
22953 
22954       // Pattern: (store:void (imm:i8):$src, addr:iPTR:$dst)
22955       // Emits: (MOV8mi:void addr:iPTR:$dst, (imm:i8):$src)
22956       // Pattern complexity = 14  cost = 1  size = 3
22957       if (N1.Val->getValueType(0) == MVT::i8) {
22958         bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22959         if (Match) {
22960           Emit_158(Result, N, X86::MOV8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
22961           return;
22962         }
22963       }
22964 
22965       // Pattern: (store:void (imm:i16):$src, addr:iPTR:$dst)
22966       // Emits: (MOV16mi:void addr:iPTR:$dst, (imm:i16):$src)
22967       // Pattern complexity = 14  cost = 1  size = 3
22968       if (N1.Val->getValueType(0) == MVT::i16) {
22969         bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22970         if (Match) {
22971           Emit_159(Result, N, X86::MOV16mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
22972           return;
22973         }
22974       }
22975 
22976       // Pattern: (store:void (imm:i32):$src, addr:iPTR:$dst)
22977       // Emits: (MOV32mi:void addr:iPTR:$dst, (imm:i32):$src)
22978       // Pattern complexity = 14  cost = 1
22979       if (N1.Val->getValueType(0) == MVT::i32) {
22980         bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22981         if (Match) {
22982           Emit_160(Result, N, X86::MOV32mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
22983           return;
22984         }
22985       }
22986     }
22987   }
22988 
22989   // Pattern: (store:void RFP:f64:$src, addr:iPTR:$op)
22990   // Emits: (FpST64m:void addr:iPTR:$op, RFP:f64:$src)
22991   // Pattern complexity = 12  cost = 1  size = 0
22992   if ((!Subtarget->hasSSE2())) {
22993     Chain = N.getOperand(0);
22994     N1 = N.getOperand(1);
22995     N2 = N.getOperand(2);
22996     if (N1.Val->getValueType(0) == MVT::f64) {
22997       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
22998       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
22999       if (!Match) {
23000         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23001       }
23002       if (Match) {
23003         Emit_24(Result, N, X86::FpST64m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23004         return;
23005       }
23006     }
23007   }
23008   {
23009     Chain = N.getOperand(0);
23010     N1 = N.getOperand(1);
23011     N2 = N.getOperand(2);
23012 
23013     // Pattern: (store:void GR8:i8:$src, addr:iPTR:$dst)
23014     // Emits: (MOV8mr:void addr:iPTR:$dst, GR8:i8:$src)
23015     // Pattern complexity = 12  cost = 1  size = 3
23016     if (N1.Val->getValueType(0) == MVT::i8) {
23017       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23018       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23019       if (!Match) {
23020         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23021       }
23022       if (Match) {
23023         Emit_24(Result, N, X86::MOV8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23024         return;
23025       }
23026     }
23027 
23028     // Pattern: (store:void GR16:i16:$src, addr:iPTR:$dst)
23029     // Emits: (MOV16mr:void addr:iPTR:$dst, GR16:i16:$src)
23030     // Pattern complexity = 12  cost = 1  size = 3
23031     if (N1.Val->getValueType(0) == MVT::i16) {
23032       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23033       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23034       if (!Match) {
23035         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23036       }
23037       if (Match) {
23038         Emit_24(Result, N, X86::MOV16mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23039         return;
23040       }
23041     }
23042 
23043     // Pattern: (store:void GR32:i32:$src, addr:iPTR:$dst)
23044     // Emits: (MOV32mr:void addr:iPTR:$dst, GR32:i32:$src)
23045     // Pattern complexity = 12  cost = 1
23046     if (N1.Val->getValueType(0) == MVT::i32) {
23047       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23048       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23049       if (!Match) {
23050         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23051       }
23052       if (Match) {
23053         Emit_24(Result, N, X86::MOV32mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23054         return;
23055       }
23056     }
23057   }
23058 
23059   // Pattern: (store:void FR32:f32:$src, addr:iPTR:$dst)
23060   // Emits: (MOVSSmr:void addr:iPTR:$dst, FR32:f32:$src)
23061   // Pattern complexity = 12  cost = 1  size = 3
23062   if ((Subtarget->hasSSE1())) {
23063     Chain = N.getOperand(0);
23064     N1 = N.getOperand(1);
23065     N2 = N.getOperand(2);
23066     if (N1.Val->getValueType(0) == MVT::f32) {
23067       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23068       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23069       if (!Match) {
23070         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23071       }
23072       if (Match) {
23073         Emit_24(Result, N, X86::MOVSSmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23074         return;
23075       }
23076     }
23077   }
23078 
23079   // Pattern: (store:void FR64:f64:$src, addr:iPTR:$dst)
23080   // Emits: (MOVSDmr:void addr:iPTR:$dst, FR64:f64:$src)
23081   // Pattern complexity = 12  cost = 1  size = 3
23082   if ((Subtarget->hasSSE2())) {
23083     Chain = N.getOperand(0);
23084     N1 = N.getOperand(1);
23085     N2 = N.getOperand(2);
23086     if (N1.Val->getValueType(0) == MVT::f64) {
23087       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23088       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23089       if (!Match) {
23090         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23091       }
23092       if (Match) {
23093         Emit_24(Result, N, X86::MOVSDmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23094         return;
23095       }
23096     }
23097   }
23098 
23099   // Pattern: (store:void VR128:v4f32:$src, addr:iPTR:$dst)
23100   // Emits: (MOVAPSmr:void addr:iPTR:$dst, VR128:v4f32:$src)
23101   // Pattern complexity = 12  cost = 1  size = 3
23102   if ((Subtarget->hasSSE1())) {
23103     Chain = N.getOperand(0);
23104     N1 = N.getOperand(1);
23105     N2 = N.getOperand(2);
23106     if (N1.Val->getValueType(0) == MVT::v4f32) {
23107       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23108       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23109       if (!Match) {
23110         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23111       }
23112       if (Match) {
23113         Emit_24(Result, N, X86::MOVAPSmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23114         return;
23115       }
23116     }
23117   }
23118   if ((Subtarget->hasSSE2())) {
23119     Chain = N.getOperand(0);
23120     N1 = N.getOperand(1);
23121     N2 = N.getOperand(2);
23122 
23123     // Pattern: (store:void VR128:v2f64:$src, addr:iPTR:$dst)
23124     // Emits: (MOVAPDmr:void addr:iPTR:$dst, VR128:v2f64:$src)
23125     // Pattern complexity = 12  cost = 1  size = 3
23126     if (N1.Val->getValueType(0) == MVT::v2f64) {
23127       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23128       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23129       if (!Match) {
23130         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23131       }
23132       if (Match) {
23133         Emit_24(Result, N, X86::MOVAPDmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23134         return;
23135       }
23136     }
23137 
23138     // Pattern: (store:void VR128:v2i64:$src, addr:iPTR:$dst)
23139     // Emits: (MOVDQAmr:void addr:iPTR:$dst, VR128:v2i64:$src)
23140     // Pattern complexity = 12  cost = 1  size = 3
23141     if (N1.Val->getValueType(0) == MVT::v2i64) {
23142       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23143       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23144       if (!Match) {
23145         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23146       }
23147       if (Match) {
23148         Emit_24(Result, N, X86::MOVDQAmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23149         return;
23150       }
23151     }
23152 
23153     // Pattern: (store:void VR128:v16i8:$src, addr:iPTR:$dst)
23154     // Emits: (MOVDQAmr:void addr:iPTR:$dst, VR128:v16i8:$src)
23155     // Pattern complexity = 12  cost = 1  size = 3
23156     if (N1.Val->getValueType(0) == MVT::v16i8) {
23157       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23158       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23159       if (!Match) {
23160         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23161       }
23162       if (Match) {
23163         Emit_24(Result, N, X86::MOVDQAmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23164         return;
23165       }
23166     }
23167 
23168     // Pattern: (store:void VR128:v8i16:$src, addr:iPTR:$dst)
23169     // Emits: (MOVDQAmr:void addr:iPTR:$dst, VR128:v16i8:$src)
23170     // Pattern complexity = 12  cost = 1  size = 3
23171     if (N1.Val->getValueType(0) == MVT::v8i16) {
23172       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23173       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23174       if (!Match) {
23175         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23176       }
23177       if (Match) {
23178         Emit_24(Result, N, X86::MOVDQAmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23179         return;
23180       }
23181     }
23182 
23183     // Pattern: (store:void VR128:v4i32:$src, addr:iPTR:$dst)
23184     // Emits: (MOVDQAmr:void addr:iPTR:$dst, VR128:v16i8:$src)
23185     // Pattern complexity = 12  cost = 1
23186     if (N1.Val->getValueType(0) == MVT::v4i32) {
23187       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23188       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23189       if (!Match) {
23190         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23191       }
23192       if (Match) {
23193         Emit_24(Result, N, X86::MOVDQAmr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2);
23194         return;
23195       }
23196     }
23197   }
23198   std::cerr << "Cannot yet select: ";
23199   N.Val->dump(CurDAG);
23200   std::cerr << '\n';
23201   abort();
23202 }
23203 
23204 void Select_sub(SDOperand &Result, SDOperand N) {
23205   SDOperand CPTmp0(0, 0);
23206   SDOperand CPTmp1(0, 0);
23207   SDOperand CPTmp2(0, 0);
23208   SDOperand CPTmp3(0, 0);
23209   SDOperand Chain1(0, 0);
23210   SDOperand Chain10(0, 0);
23211   SDOperand N0(0, 0);
23212   SDOperand N1(0, 0);
23213   SDOperand N10(0, 0);
23214   SDOperand N101(0, 0);
23215   SDOperand N11(0, 0);
23216   SDOperand Tmp0(0, 0);
23217   SDOperand Tmp1(0, 0);
23218   SDOperand Tmp2(0, 0);
23219   SDOperand Tmp3(0, 0);
23220   SDOperand Tmp4(0, 0);
23221   SDNode *ResNode = NULL;
23222   if ((Subtarget->hasSSE2())) {
23223     N0 = N.getOperand(0);
23224     N1 = N.getOperand(1);
23225     if (N1.getOpcode() == ISD::BIT_CONVERT &&
23226         InFlightSet.count(N1.Val) == 0 &&
23227         N1.hasOneUse() &&
23228         !CodeGenMap.count(N1.getValue(0))) {
23229       N10 = N1.getOperand(0);
23230       if (N10.getOpcode() == ISD::LOAD &&
23231           InFlightSet.count(N10.Val) == 0 &&
23232           N10.hasOneUse() &&
23233           !CodeGenMap.count(N10.getValue(0))) {
23234         Chain10 = N10.getOperand(0);
23235         N101 = N10.getOperand(1);
23236 
23237         // Pattern: (sub:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)))
23238         // Emits: (PSUBBrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
23239         // Pattern complexity = 16  cost = 1  size = 3
23240         if (N.Val->getValueType(0) == MVT::v16i8 &&
23241             N10.Val->getValueType(0) == MVT::v2i64) {
23242           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23243           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23244           if (!Match) {
23245             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23246           }
23247           if (Match) {
23248             Emit_40(Result, N, X86::PSUBBrm, MVT::v16i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
23249             return;
23250           }
23251         }
23252 
23253         // Pattern: (sub:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)))
23254         // Emits: (PSUBWrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
23255         // Pattern complexity = 16  cost = 1  size = 3
23256         if (N.Val->getValueType(0) == MVT::v8i16 &&
23257             N10.Val->getValueType(0) == MVT::v2i64) {
23258           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23259           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23260           if (!Match) {
23261             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23262           }
23263           if (Match) {
23264             Emit_40(Result, N, X86::PSUBWrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
23265             return;
23266           }
23267         }
23268 
23269         // Pattern: (sub:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)))
23270         // Emits: (PSUBDrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
23271         // Pattern complexity = 16  cost = 1
23272         if (N.Val->getValueType(0) == MVT::v4i32 &&
23273             N10.Val->getValueType(0) == MVT::v2i64) {
23274           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23275           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23276           if (!Match) {
23277             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23278           }
23279           if (Match) {
23280             Emit_40(Result, N, X86::PSUBDrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
23281             return;
23282           }
23283         }
23284       }
23285     }
23286   }
23287   {
23288     N0 = N.getOperand(0);
23289     N1 = N.getOperand(1);
23290     if (N1.getOpcode() == ISD::LOAD &&
23291         InFlightSet.count(N1.Val) == 0 &&
23292         N1.hasOneUse() &&
23293         !CodeGenMap.count(N1.getValue(0)) &&
23294         !isNonImmUse(N.Val, N1.Val)) {
23295       Chain1 = N1.getOperand(0);
23296       N11 = N1.getOperand(1);
23297 
23298       // Pattern: (sub:i8 GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
23299       // Emits: (SUB8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
23300       // Pattern complexity = 14  cost = 1  size = 3
23301       if (N.Val->getValueType(0) == MVT::i8) {
23302         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23303         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23304         if (!Match) {
23305           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23306         }
23307         if (Match) {
23308           Emit_19(Result, N, X86::SUB8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23309           return;
23310         }
23311       }
23312 
23313       // Pattern: (sub:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
23314       // Emits: (SUB16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
23315       // Pattern complexity = 14  cost = 1  size = 3
23316       if (N.Val->getValueType(0) == MVT::i16) {
23317         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23318         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23319         if (!Match) {
23320           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23321         }
23322         if (Match) {
23323           Emit_19(Result, N, X86::SUB16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23324           return;
23325         }
23326       }
23327 
23328       // Pattern: (sub:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
23329       // Emits: (SUB32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
23330       // Pattern complexity = 14  cost = 1
23331       if (N.Val->getValueType(0) == MVT::i32) {
23332         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23333         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23334         if (!Match) {
23335           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23336         }
23337         if (Match) {
23338           Emit_19(Result, N, X86::SUB32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23339           return;
23340         }
23341       }
23342     }
23343   }
23344 
23345   // Pattern: (sub:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
23346   // Emits: (PSUBQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
23347   // Pattern complexity = 14  cost = 1  size = 3
23348   if ((Subtarget->hasSSE2())) {
23349     N0 = N.getOperand(0);
23350     N1 = N.getOperand(1);
23351     if (N1.getOpcode() == ISD::LOAD &&
23352         InFlightSet.count(N1.Val) == 0 &&
23353         N1.hasOneUse() &&
23354         !CodeGenMap.count(N1.getValue(0)) &&
23355         !isNonImmUse(N.Val, N1.Val)) {
23356       Chain1 = N1.getOperand(0);
23357       N11 = N1.getOperand(1);
23358       if (N.Val->getValueType(0) == MVT::v2i64) {
23359         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23360         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23361         if (!Match) {
23362           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23363         }
23364         if (Match) {
23365           Emit_19(Result, N, X86::PSUBQrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23366           return;
23367         }
23368       }
23369     }
23370   }
23371   {
23372     N0 = N.getOperand(0);
23373     if (isa<ConstantSDNode>(N0)) {
23374       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
23375       if (CN0 == 0) {
23376         N1 = N.getOperand(1);
23377 
23378         // Pattern: (sub:i8 0:i8, GR8:i8:$src)
23379         // Emits: (NEG8r:i8 GR8:i8:$src)
23380         // Pattern complexity = 5  cost = 1  size = 2
23381         if (N.Val->getValueType(0) == MVT::i8) {
23382           Emit_48(Result, N, X86::NEG8r, MVT::i8, N0, N1);
23383           return;
23384         }
23385 
23386         // Pattern: (sub:i16 0:i16, GR16:i16:$src)
23387         // Emits: (NEG16r:i16 GR16:i16:$src)
23388         // Pattern complexity = 5  cost = 1  size = 2
23389         if (N.Val->getValueType(0) == MVT::i16) {
23390           Emit_48(Result, N, X86::NEG16r, MVT::i16, N0, N1);
23391           return;
23392         }
23393 
23394         // Pattern: (sub:i32 0:i32, GR32:i32:$src)
23395         // Emits: (NEG32r:i32 GR32:i32:$src)
23396         // Pattern complexity = 5  cost = 1
23397         if (N.Val->getValueType(0) == MVT::i32) {
23398           Emit_48(Result, N, X86::NEG32r, MVT::i32, N0, N1);
23399           return;
23400         }
23401       }
23402     }
23403     N1 = N.getOperand(1);
23404     if (N1.getOpcode() == ISD::Constant) {
23405 
23406       // Pattern: (sub:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
23407       // Emits: (SUB16ri8:i16 GR16:i16:$src1, (imm:i16):$src2)
23408       // Pattern complexity = 5  cost = 1  size = 3
23409       if (Predicate_i16immSExt8(N1.Val) &&
23410           N.Val->getValueType(0) == MVT::i16) {
23411         Emit_49(Result, N, X86::SUB16ri8, MVT::i16, N0, N1);
23412         return;
23413       }
23414 
23415       // Pattern: (sub:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
23416       // Emits: (SUB32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
23417       // Pattern complexity = 5  cost = 1  size = 3
23418       if (Predicate_i32immSExt8(N1.Val) &&
23419           N.Val->getValueType(0) == MVT::i32) {
23420         Emit_29(Result, N, X86::SUB32ri8, MVT::i32, N0, N1);
23421         return;
23422       }
23423 
23424       // Pattern: (sub:i8 GR8:i8:$src1, (imm:i8):$src2)
23425       // Emits: (SUB8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
23426       // Pattern complexity = 4  cost = 1  size = 3
23427       if (N.Val->getValueType(0) == MVT::i8) {
23428         Emit_50(Result, N, X86::SUB8ri, MVT::i8, N0, N1);
23429         return;
23430       }
23431 
23432       // Pattern: (sub:i16 GR16:i16:$src1, (imm:i16):$src2)
23433       // Emits: (SUB16ri:i16 GR16:i16:$src1, (imm:i16):$src2)
23434       // Pattern complexity = 4  cost = 1  size = 3
23435       if (N.Val->getValueType(0) == MVT::i16) {
23436         Emit_49(Result, N, X86::SUB16ri, MVT::i16, N0, N1);
23437         return;
23438       }
23439 
23440       // Pattern: (sub:i32 GR32:i32:$src1, (imm:i32):$src2)
23441       // Emits: (SUB32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
23442       // Pattern complexity = 4  cost = 1
23443       if (N.Val->getValueType(0) == MVT::i32) {
23444         Emit_29(Result, N, X86::SUB32ri, MVT::i32, N0, N1);
23445         return;
23446       }
23447     }
23448 
23449     // Pattern: (sub:i8 GR8:i8:$src1, GR8:i8:$src2)
23450     // Emits: (SUB8rr:i8 GR8:i8:$src1, GR8:i8:$src2)
23451     // Pattern complexity = 2  cost = 1  size = 3
23452     if (N.Val->getValueType(0) == MVT::i8) {
23453       Emit_21(Result, N, X86::SUB8rr, MVT::i8, N0, N1);
23454       return;
23455     }
23456 
23457     // Pattern: (sub:i16 GR16:i16:$src1, GR16:i16:$src2)
23458     // Emits: (SUB16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
23459     // Pattern complexity = 2  cost = 1  size = 3
23460     if (N.Val->getValueType(0) == MVT::i16) {
23461       Emit_21(Result, N, X86::SUB16rr, MVT::i16, N0, N1);
23462       return;
23463     }
23464 
23465     // Pattern: (sub:i32 GR32:i32:$src1, GR32:i32:$src2)
23466     // Emits: (SUB32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
23467     // Pattern complexity = 2  cost = 1
23468     if (N.Val->getValueType(0) == MVT::i32) {
23469       Emit_21(Result, N, X86::SUB32rr, MVT::i32, N0, N1);
23470       return;
23471     }
23472   }
23473   if ((Subtarget->hasSSE2())) {
23474     N0 = N.getOperand(0);
23475     N1 = N.getOperand(1);
23476 
23477     // Pattern: (sub:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
23478     // Emits: (PSUBBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
23479     // Pattern complexity = 2  cost = 1  size = 3
23480     if (N.Val->getValueType(0) == MVT::v16i8) {
23481       Emit_21(Result, N, X86::PSUBBrr, MVT::v16i8, N0, N1);
23482       return;
23483     }
23484 
23485     // Pattern: (sub:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
23486     // Emits: (PSUBWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
23487     // Pattern complexity = 2  cost = 1  size = 3
23488     if (N.Val->getValueType(0) == MVT::v8i16) {
23489       Emit_21(Result, N, X86::PSUBWrr, MVT::v8i16, N0, N1);
23490       return;
23491     }
23492 
23493     // Pattern: (sub:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
23494     // Emits: (PSUBDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
23495     // Pattern complexity = 2  cost = 1  size = 3
23496     if (N.Val->getValueType(0) == MVT::v4i32) {
23497       Emit_21(Result, N, X86::PSUBDrr, MVT::v4i32, N0, N1);
23498       return;
23499     }
23500 
23501     // Pattern: (sub:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
23502     // Emits: (PSUBQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
23503     // Pattern complexity = 2  cost = 1
23504     if (N.Val->getValueType(0) == MVT::v2i64) {
23505       Emit_21(Result, N, X86::PSUBQrr, MVT::v2i64, N0, N1);
23506       return;
23507     }
23508   }
23509   std::cerr << "Cannot yet select: ";
23510   N.Val->dump(CurDAG);
23511   std::cerr << '\n';
23512   abort();
23513 }
23514 
23515 void Select_subc(SDOperand &Result, SDOperand N) {
23516   SDOperand CPTmp0(0, 0);
23517   SDOperand CPTmp1(0, 0);
23518   SDOperand CPTmp2(0, 0);
23519   SDOperand CPTmp3(0, 0);
23520   SDOperand Chain1(0, 0);
23521   SDOperand InFlag(0, 0);
23522   SDOperand N0(0, 0);
23523   SDOperand N1(0, 0);
23524   SDOperand N11(0, 0);
23525   SDOperand Tmp0(0, 0);
23526   SDOperand Tmp1(0, 0);
23527   SDOperand Tmp2(0, 0);
23528   SDOperand Tmp3(0, 0);
23529   SDOperand Tmp4(0, 0);
23530   SDNode *ResNode = NULL;
23531   N0 = N.getOperand(0);
23532   N1 = N.getOperand(1);
23533 
23534   // Pattern: (subc:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
23535   // Emits: (SUB32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
23536   // Pattern complexity = 14  cost = 1  size = 3
23537   if (N1.getOpcode() == ISD::LOAD &&
23538       InFlightSet.count(N1.Val) == 0 &&
23539       N1.hasOneUse() &&
23540       !CodeGenMap.count(N1.getValue(0)) &&
23541       N.Val->isOnlyUse(N1.Val) &&
23542       !isNonImmUse(N.Val, N1.Val)) {
23543     Chain1 = N1.getOperand(0);
23544     N11 = N1.getOperand(1);
23545     if (N.Val->getValueType(0) == MVT::i32) {
23546       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23547       bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23548       if (!Match) {
23549         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23550       }
23551       if (Match) {
23552         Emit_51(Result, N, X86::SUB32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23553         return;
23554       }
23555     }
23556   }
23557   if (N1.getOpcode() == ISD::Constant) {
23558 
23559     // Pattern: (subc:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
23560     // Emits: (SUB32ri8:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
23561     // Pattern complexity = 5  cost = 1  size = 3
23562     if (Predicate_i32immSExt8(N1.Val) &&
23563         N.Val->getValueType(0) == MVT::i32) {
23564       Emit_53(Result, N, X86::SUB32ri8, MVT::i32, N0, N1);
23565       return;
23566     }
23567 
23568     // Pattern: (subc:i32 GR32:i32:$src1, (imm:i32):$src2)
23569     // Emits: (SUB32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
23570     // Pattern complexity = 4  cost = 1
23571     if (N.Val->getValueType(0) == MVT::i32) {
23572       Emit_53(Result, N, X86::SUB32ri, MVT::i32, N0, N1);
23573       return;
23574     }
23575   }
23576 
23577   // Pattern: (subc:i32 GR32:i32:$src1, GR32:i32:$src2)
23578   // Emits: (SUB32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
23579   // Pattern complexity = 2  cost = 1
23580   if (N.Val->getValueType(0) == MVT::i32) {
23581     Emit_54(Result, N, X86::SUB32rr, MVT::i32, N0, N1);
23582     return;
23583   }
23584   std::cerr << "Cannot yet select: ";
23585   N.Val->dump(CurDAG);
23586   std::cerr << '\n';
23587   abort();
23588 }
23589 
23590 void Select_sube(SDOperand &Result, SDOperand N) {
23591   SDOperand CPTmp0(0, 0);
23592   SDOperand CPTmp1(0, 0);
23593   SDOperand CPTmp2(0, 0);
23594   SDOperand CPTmp3(0, 0);
23595   SDOperand Chain1(0, 0);
23596   SDOperand InFlag(0, 0);
23597   SDOperand N0(0, 0);
23598   SDOperand N1(0, 0);
23599   SDOperand N11(0, 0);
23600   SDOperand Tmp0(0, 0);
23601   SDOperand Tmp1(0, 0);
23602   SDOperand Tmp2(0, 0);
23603   SDOperand Tmp3(0, 0);
23604   SDOperand Tmp4(0, 0);
23605   SDNode *ResNode = NULL;
23606   N0 = N.getOperand(0);
23607   N1 = N.getOperand(1);
23608 
23609   // Pattern: (sube:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
23610   // Emits: (SBB32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
23611   // Pattern complexity = 14  cost = 1  size = 3
23612   if (N1.getOpcode() == ISD::LOAD &&
23613       InFlightSet.count(N1.Val) == 0 &&
23614       N1.hasOneUse() &&
23615       !CodeGenMap.count(N1.getValue(0)) &&
23616       N.Val->isOnlyUse(N1.Val) &&
23617       !isNonImmUse(N.Val, N1.Val)) {
23618     Chain1 = N1.getOperand(0);
23619     N11 = N1.getOperand(1);
23620     if (N.Val->getValueType(0) == MVT::i32) {
23621       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
23622       bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23623       if (!Match) {
23624         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
23625       }
23626       if (Match) {
23627         Emit_55(Result, N, X86::SBB32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
23628         return;
23629       }
23630     }
23631   }
23632   if (N1.getOpcode() == ISD::Constant) {
23633 
23634     // Pattern: (sube:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
23635     // Emits: (SBB32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
23636     // Pattern complexity = 5  cost = 1  size = 3
23637     if (Predicate_i32immSExt8(N1.Val) &&
23638         N.Val->getValueType(0) == MVT::i32) {
23639       Emit_57(Result, N, X86::SBB32ri8, MVT::i32, N0, N1);
23640       return;
23641     }
23642 
23643     // Pattern: (sube:i32 GR32:i32:$src1, (imm:i32):$src2)
23644     // Emits: (SBB32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
23645     // Pattern complexity = 4  cost = 1
23646     if (N.Val->getValueType(0) == MVT::i32) {
23647       Emit_57(Result, N, X86::SBB32ri, MVT::i32, N0, N1);
23648       return;
23649     }
23650   }
23651 
23652   // Pattern: (sube:i32 GR32:i32:$src1, GR32:i32:$src2)
23653   // Emits: (SBB32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
23654   // Pattern complexity = 2  cost = 1
23655   if (N.Val->getValueType(0) == MVT::i32) {
23656     Emit_58(Result, N, X86::SBB32rr, MVT::i32, N0, N1);
23657     return;
23658   }
23659   std::cerr << "Cannot yet select: ";
23660   N.Val->dump(CurDAG);
23661   std::cerr << '\n';
23662   abort();
23663 }
23664 
23665 void Select_trunc(SDOperand &Result, SDOperand N) {
23666   SDOperand N0(0, 0);
23667   SDOperand Tmp0(0, 0);
23668   SDNode *ResNode = NULL;
23669   N0 = N.getOperand(0);
23670   if (N.Val->getValueType(0) == MVT::i16 &&
23671       N0.Val->getValueType(0) == MVT::i32) {
23672     Emit_36(Result, N, X86::TRUNC_GR32_GR16, MVT::i16, N0);
23673     return;
23674   }
23675   std::cerr << "Cannot yet select: ";
23676   N.Val->dump(CurDAG);
23677   std::cerr << '\n';
23678   abort();
23679 }
23680 
23681 void Emit_161(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3, SDOperand &N4) NOINLINE {
23682   SDOperand Tmp0(0, 0);
23683   SDOperand Tmp1(0, 0);
23684   SDOperand Tmp2(0, 0);
23685   SDOperand Tmp3(0, 0);
23686   SDOperand Tmp4(0, 0);
23687   SDNode *ResNode = NULL;
23688   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
23689   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
23690   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
23691   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
23692   Select(Tmp0, CPTmp0);
23693   Select(Tmp1, CPTmp1);
23694   Select(Tmp2, CPTmp2);
23695   Select(Tmp3, CPTmp3);
23696   Tmp4 = CurDAG->getTargetConstant(((unsigned char) cast<ConstantSDNode>(N1)->getValue()), MVT::i8);
23697   Select(Chain, Chain);
23698   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
23699   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
23700   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
23701   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
23702   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23703   Chain = SDOperand(ResNode, 0);
23704   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
23705   Result = SDOperand(ResNode, N.ResNo);
23706   return;
23707 }
23708 void Emit_162(SDOperand &Result, SDOperand &N, unsigned Opc0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain, SDOperand &N1, SDOperand &N2, SDOperand &N3, SDOperand &N4) NOINLINE {
23709   SDOperand Tmp0(0, 0);
23710   SDOperand Tmp1(0, 0);
23711   SDOperand Tmp2(0, 0);
23712   SDOperand Tmp3(0, 0);
23713   SDOperand Tmp4(0, 0);
23714   SDNode *ResNode = NULL;
23715   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
23716   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
23717   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
23718   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
23719   Select(Tmp0, CPTmp0);
23720   Select(Tmp1, CPTmp1);
23721   Select(Tmp2, CPTmp2);
23722   Select(Tmp3, CPTmp3);
23723   Select(Tmp4, N1);
23724   Select(Chain, Chain);
23725   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23726   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
23727   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
23728   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
23729   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
23730   ResNode = CurDAG->getTargetNode(Opc0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23731   Chain = SDOperand(ResNode, 0);
23732   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Chain.Val, Chain.ResNo);
23733   Result = SDOperand(ResNode, N.ResNo);
23734   return;
23735 }
23736 void Select_truncst(SDOperand &Result, SDOperand N) {
23737   SDOperand CPTmp0(0, 0);
23738   SDOperand CPTmp1(0, 0);
23739   SDOperand CPTmp2(0, 0);
23740   SDOperand CPTmp3(0, 0);
23741   SDOperand Chain(0, 0);
23742   SDOperand N1(0, 0);
23743   SDOperand N2(0, 0);
23744   SDOperand N3(0, 0);
23745   SDOperand N4(0, 0);
23746   SDOperand Tmp0(0, 0);
23747   SDOperand Tmp1(0, 0);
23748   SDOperand Tmp2(0, 0);
23749   SDOperand Tmp3(0, 0);
23750   SDOperand Tmp4(0, 0);
23751   SDNode *ResNode = NULL;
23752 
23753   // Pattern: (truncst:void (imm:i8):$src, addr:iPTR:$dst, srcvalue:Other:$dummy, i1:Other)
23754   // Emits: (MOV8mi:void addr:iPTR:$dst, (imm:i8):$src)
23755   // Pattern complexity = 14  cost = 1  size = 3
23756   {
23757     Chain = N.getOperand(0);
23758     N1 = N.getOperand(1);
23759     if (N1.getOpcode() == ISD::Constant) {
23760       N2 = N.getOperand(2);
23761       N3 = N.getOperand(3);
23762       N4 = N.getOperand(4);
23763       if (cast<VTSDNode>(N4)->getVT() == MVT::i1 &&
23764           N1.Val->getValueType(0) == MVT::i8) {
23765         bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23766         if (Match) {
23767           Emit_161(Result, N, X86::MOV8mi, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3, N4);
23768           return;
23769         }
23770       }
23771     }
23772   }
23773 
23774   // Pattern: (truncst:void RFP:f64:$src, addr:iPTR:$op, srcvalue:Other:$dummy, f32:Other)
23775   // Emits: (FpST32m:void addr:iPTR:$op, RFP:f64:$src)
23776   // Pattern complexity = 12  cost = 1  size = 0
23777   if ((!Subtarget->hasSSE2())) {
23778     Chain = N.getOperand(0);
23779     N1 = N.getOperand(1);
23780     N2 = N.getOperand(2);
23781     N3 = N.getOperand(3);
23782     N4 = N.getOperand(4);
23783     if (cast<VTSDNode>(N4)->getVT() == MVT::f32 &&
23784         N1.Val->getValueType(0) == MVT::f64) {
23785       SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23786       bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23787       if (!Match) {
23788         SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23789       }
23790       if (Match) {
23791         Emit_162(Result, N, X86::FpST32m, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3, N4);
23792         return;
23793       }
23794     }
23795   }
23796 
23797   // Pattern: (truncst:void GR8:i8:$src, addr:iPTR:$dst, srcvalue:Other:$dummy, i1:Other)
23798   // Emits: (MOV8mr:void addr:iPTR:$dst, GR8:i8:$src)
23799   // Pattern complexity = 12  cost = 1
23800   Chain = N.getOperand(0);
23801   N1 = N.getOperand(1);
23802   N2 = N.getOperand(2);
23803   N3 = N.getOperand(3);
23804   N4 = N.getOperand(4);
23805   if (cast<VTSDNode>(N4)->getVT() == MVT::i1 &&
23806       N1.Val->getValueType(0) == MVT::i8) {
23807     SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
23808     bool Match = SelectAddr(N2, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
23809     if (!Match) {
23810       SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
23811     }
23812     if (Match) {
23813       Emit_162(Result, N, X86::MOV8mr, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3, N4);
23814       return;
23815     }
23816   }
23817   std::cerr << "Cannot yet select: ";
23818   N.Val->dump(CurDAG);
23819   std::cerr << '\n';
23820   abort();
23821 }
23822 
23823 void Select_undef(SDOperand &Result, SDOperand N) {
23824   SDNode *ResNode = NULL;
23825 
23826   // Pattern: (undef:i8)
23827   // Emits: (IMPLICIT_DEF_GR8:i8)
23828   // Pattern complexity = 2  cost = 1  size = 3
23829   if (N.Val->getValueType(0) == MVT::i8) {
23830     Emit_96(Result, N, X86::IMPLICIT_DEF_GR8, MVT::i8);
23831     return;
23832   }
23833 
23834   // Pattern: (undef:i16)
23835   // Emits: (IMPLICIT_DEF_GR16:i16)
23836   // Pattern complexity = 2  cost = 1  size = 3
23837   if (N.Val->getValueType(0) == MVT::i16) {
23838     Emit_96(Result, N, X86::IMPLICIT_DEF_GR16, MVT::i16);
23839     return;
23840   }
23841 
23842   // Pattern: (undef:i32)
23843   // Emits: (IMPLICIT_DEF_GR32:i32)
23844   // Pattern complexity = 2  cost = 1  size = 3
23845   if (N.Val->getValueType(0) == MVT::i32) {
23846     Emit_96(Result, N, X86::IMPLICIT_DEF_GR32, MVT::i32);
23847     return;
23848   }
23849 
23850   // Pattern: (undef:v8i8)
23851   // Emits: (IMPLICIT_DEF_VR64:v8i8)
23852   // Pattern complexity = 2  cost = 1  size = 3
23853   if ((Subtarget->hasMMX()) &&
23854       N.Val->getValueType(0) == MVT::v8i8) {
23855     Emit_96(Result, N, X86::IMPLICIT_DEF_VR64, MVT::v8i8);
23856     return;
23857   }
23858   if ((Subtarget->hasSSE2())) {
23859 
23860     // Pattern: (undef:f32)
23861     // Emits: (IMPLICIT_DEF_FR32:f32)
23862     // Pattern complexity = 2  cost = 1  size = 3
23863     if (N.Val->getValueType(0) == MVT::f32) {
23864       Emit_96(Result, N, X86::IMPLICIT_DEF_FR32, MVT::f32);
23865       return;
23866     }
23867 
23868     // Pattern: (undef:f64)
23869     // Emits: (IMPLICIT_DEF_FR64:f64)
23870     // Pattern complexity = 2  cost = 1
23871     if (N.Val->getValueType(0) == MVT::f64) {
23872       Emit_96(Result, N, X86::IMPLICIT_DEF_FR64, MVT::f64);
23873       return;
23874     }
23875   }
23876 
23877   // Pattern: (undef:v4f32)
23878   // Emits: (IMPLICIT_DEF_VR128:v4f32)
23879   // Pattern complexity = 2  cost = 1  size = 3
23880   if ((Subtarget->hasSSE1()) &&
23881       N.Val->getValueType(0) == MVT::v4f32) {
23882     Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v4f32);
23883     return;
23884   }
23885   if ((Subtarget->hasMMX())) {
23886 
23887     // Pattern: (undef:v4i16)
23888     // Emits: (IMPLICIT_DEF_VR64:v4i16)
23889     // Pattern complexity = 2  cost = 1  size = 3
23890     if (N.Val->getValueType(0) == MVT::v4i16) {
23891       Emit_96(Result, N, X86::IMPLICIT_DEF_VR64, MVT::v4i16);
23892       return;
23893     }
23894 
23895     // Pattern: (undef:v2i32)
23896     // Emits: (IMPLICIT_DEF_VR64:v2i32)
23897     // Pattern complexity = 2  cost = 1
23898     if (N.Val->getValueType(0) == MVT::v2i32) {
23899       Emit_96(Result, N, X86::IMPLICIT_DEF_VR64, MVT::v2i32);
23900       return;
23901     }
23902   }
23903   if ((Subtarget->hasSSE2())) {
23904 
23905     // Pattern: (undef:v2f64)
23906     // Emits: (IMPLICIT_DEF_VR128:v2f64)
23907     // Pattern complexity = 2  cost = 1  size = 3
23908     if (N.Val->getValueType(0) == MVT::v2f64) {
23909       Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v2f64);
23910       return;
23911     }
23912 
23913     // Pattern: (undef:v16i8)
23914     // Emits: (IMPLICIT_DEF_VR128:v16i8)
23915     // Pattern complexity = 2  cost = 1  size = 3
23916     if (N.Val->getValueType(0) == MVT::v16i8) {
23917       Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
23918       return;
23919     }
23920 
23921     // Pattern: (undef:v8i16)
23922     // Emits: (IMPLICIT_DEF_VR128:v8i16)
23923     // Pattern complexity = 2  cost = 1  size = 3
23924     if (N.Val->getValueType(0) == MVT::v8i16) {
23925       Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v8i16);
23926       return;
23927     }
23928 
23929     // Pattern: (undef:v4i32)
23930     // Emits: (IMPLICIT_DEF_VR128:v4i32)
23931     // Pattern complexity = 2  cost = 1  size = 3
23932     if (N.Val->getValueType(0) == MVT::v4i32) {
23933       Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v4i32);
23934       return;
23935     }
23936 
23937     // Pattern: (undef:v2i64)
23938     // Emits: (IMPLICIT_DEF_VR128:v2i64)
23939     // Pattern complexity = 2  cost = 1
23940     if (N.Val->getValueType(0) == MVT::v2i64) {
23941       Emit_96(Result, N, X86::IMPLICIT_DEF_VR128, MVT::v2i64);
23942       return;
23943     }
23944   }
23945   std::cerr << "Cannot yet select: ";
23946   N.Val->dump(CurDAG);
23947   std::cerr << '\n';
23948   abort();
23949 }
23950 
23951 void Select_vector_extract(SDOperand &Result, SDOperand N) {
23952   SDOperand N0(0, 0);
23953   SDOperand N1(0, 0);
23954   SDOperand Tmp1(0, 0);
23955   SDNode *ResNode = NULL;
23956 
23957   // Pattern: (vector_extract:f32 VR128:v4f32:$src, 0:iPTR)
23958   // Emits: (MOVPS2SSrr:f32 VR128:v4f32:$src)
23959   // Pattern complexity = 5  cost = 1  size = 3
23960   if ((Subtarget->hasSSE1())) {
23961     N0 = N.getOperand(0);
23962     N1 = N.getOperand(1);
23963     if (isa<ConstantSDNode>(N1)) {
23964       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
23965       if (CN0 == 0 &&
23966           N.Val->getValueType(0) == MVT::f32 &&
23967           N0.Val->getValueType(0) == MVT::v4f32) {
23968         Emit_47(Result, N, X86::MOVPS2SSrr, MVT::f32, N0, N1);
23969         return;
23970       }
23971     }
23972   }
23973   if ((Subtarget->hasSSE2())) {
23974     N0 = N.getOperand(0);
23975     N1 = N.getOperand(1);
23976     if (isa<ConstantSDNode>(N1)) {
23977       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
23978       if (CN0 == 0) {
23979 
23980         // Pattern: (vector_extract:f64 VR128:v2f64:$src, 0:iPTR)
23981         // Emits: (MOVPD2SDrr:f64 VR128:v2f64:$src)
23982         // Pattern complexity = 5  cost = 1  size = 3
23983         if (N.Val->getValueType(0) == MVT::f64 &&
23984             N0.Val->getValueType(0) == MVT::v2f64) {
23985           Emit_47(Result, N, X86::MOVPD2SDrr, MVT::f64, N0, N1);
23986           return;
23987         }
23988 
23989         // Pattern: (vector_extract:i32 VR128:v4i32:$src, 0:iPTR)
23990         // Emits: (MOVPDI2DIrr:i32 VR128:v4i32:$src)
23991         // Pattern complexity = 5  cost = 1
23992         if (N.Val->getValueType(0) == MVT::i32 &&
23993             N0.Val->getValueType(0) == MVT::v4i32) {
23994           Emit_47(Result, N, X86::MOVPDI2DIrr, MVT::i32, N0, N1);
23995           return;
23996         }
23997       }
23998     }
23999   }
24000   std::cerr << "Cannot yet select: ";
24001   N.Val->dump(CurDAG);
24002   std::cerr << '\n';
24003   abort();
24004 }
24005 
24006 void Emit_163(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N2) NOINLINE {
24007   SDOperand Tmp0(0, 0);
24008   SDOperand Tmp1(0, 0);
24009   SDOperand Tmp2(0, 0);
24010   SDOperand Tmp3(0, 0);
24011   SDNode *ResNode = NULL;
24012   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24013   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24014   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24015   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24016   Select(Tmp0, CPTmp0);
24017   Select(Tmp1, CPTmp1);
24018   Select(Tmp2, CPTmp2);
24019   Select(Tmp3, CPTmp3);
24020   Select(Chain10, Chain10);
24021   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24022   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24023   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24024   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24025   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24026   Chain10 = SDOperand(ResNode, 1);
24027   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24028   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24029   AddHandleReplacement(N10.Val, 1, ResNode, 1);
24030   Result = SDOperand(ResNode, N.ResNo);
24031   return;
24032 }
24033 void Emit_164(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain100, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N100, SDOperand &N1001, SDOperand &N2) NOINLINE {
24034   SDOperand Tmp0(0, 0);
24035   SDOperand Tmp1(0, 0);
24036   SDOperand Tmp2(0, 0);
24037   SDOperand Tmp3(0, 0);
24038   SDOperand Tmp4(0, 0);
24039   SDNode *ResNode = NULL;
24040   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24041   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24042   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24043   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24044   Select(Tmp0, CPTmp0);
24045   Select(Tmp1, CPTmp1);
24046   Select(Tmp2, CPTmp2);
24047   Select(Tmp3, CPTmp3);
24048   Select(Tmp4, N0);
24049   Select(Chain100, Chain100);
24050   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24051   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24052   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24053   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24054   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24055   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain100);
24056   Chain100 = SDOperand(ResNode, 1);
24057   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24058   SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 1);
24059   AddHandleReplacement(N100.Val, 1, ResNode, 1);
24060   Result = SDOperand(ResNode, N.ResNo);
24061   return;
24062 }
24063 void Emit_165(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N2) NOINLINE {
24064   SDOperand Tmp0(0, 0);
24065   SDOperand Tmp1(0, 0);
24066   SDOperand Tmp2(0, 0);
24067   SDOperand Tmp3(0, 0);
24068   SDNode *ResNode = NULL;
24069   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24070   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24071   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24072   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24073   Select(Tmp0, CPTmp0);
24074   Select(Tmp1, CPTmp1);
24075   Select(Tmp2, CPTmp2);
24076   Select(Tmp3, CPTmp3);
24077   Select(Chain00, Chain00);
24078   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24079   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24080   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24081   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24082   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
24083   Chain00 = SDOperand(ResNode, 1);
24084   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24085   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24086   AddHandleReplacement(N00.Val, 1, ResNode, 1);
24087   Result = SDOperand(ResNode, N.ResNo);
24088   return;
24089 }
24090 void Emit_166(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N2) NOINLINE {
24091   SDOperand Tmp0(0, 0);
24092   SDOperand Tmp1(0, 0);
24093   SDOperand Tmp2(0, 0);
24094   SDOperand Tmp3(0, 0);
24095   SDOperand Tmp4(0, 0);
24096   SDNode *ResNode = NULL;
24097   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24098   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24099   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24100   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24101   Select(Tmp0, CPTmp0);
24102   Select(Tmp1, CPTmp1);
24103   Select(Tmp2, CPTmp2);
24104   Select(Tmp3, CPTmp3);
24105   Select(Tmp4, N0);
24106   Select(Chain10, Chain10);
24107   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24108   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24109   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24110   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24111   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24112   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24113   Chain10 = SDOperand(ResNode, 1);
24114   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24115   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24116   AddHandleReplacement(N10.Val, 1, ResNode, 1);
24117   Result = SDOperand(ResNode, N.ResNo);
24118   return;
24119 }
24120 void Emit_167(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11, SDOperand &N2) NOINLINE {
24121   SDOperand Tmp0(0, 0);
24122   SDOperand Tmp1(0, 0);
24123   SDOperand Tmp2(0, 0);
24124   SDOperand Tmp3(0, 0);
24125   SDOperand Tmp4(0, 0);
24126   SDNode *ResNode = NULL;
24127   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24128   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24129   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24130   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24131   Select(Tmp0, CPTmp0);
24132   Select(Tmp1, CPTmp1);
24133   Select(Tmp2, CPTmp2);
24134   Select(Tmp3, CPTmp3);
24135   Select(Tmp4, N0);
24136   Select(Chain1, Chain1);
24137   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24138   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24139   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24140   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24141   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24142   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
24143   Chain1 = SDOperand(ResNode, 1);
24144   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24145   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
24146   AddHandleReplacement(N1.Val, 1, ResNode, 1);
24147   Result = SDOperand(ResNode, N.ResNo);
24148   return;
24149 }
24150 void Emit_168(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N2) NOINLINE {
24151   SDOperand Tmp0(0, 0);
24152   SDNode *ResNode = NULL;
24153   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
24154   Select(Tmp0, N10);
24155   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
24156   if (N.Val->hasOneUse()) {
24157     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
24158   } else {
24159     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
24160     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24161     Result = SDOperand(ResNode, 0);
24162   }
24163   return;
24164 }
24165 void Emit_169(SDOperand &Result, SDOperand &N, unsigned Opc0, unsigned Opc1, MVT::ValueType VT0, MVT::ValueType VT1, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N2) NOINLINE {
24166   SDOperand Tmp0(0, 0);
24167   SDOperand Tmp1(0, 0);
24168   SDNode *ResNode = NULL;
24169   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
24170   Tmp0 = SDOperand(CurDAG->getTargetNode(Opc0, VT0), 0);
24171   Select(Tmp1, N10);
24172   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
24173   if (N.Val->hasOneUse()) {
24174     Result = CurDAG->SelectNodeTo(N.Val, Opc1, VT1, Tmp0, Tmp1);
24175   } else {
24176     ResNode = CurDAG->getTargetNode(Opc1, VT1, Tmp0, Tmp1);
24177     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24178     Result = SDOperand(ResNode, 0);
24179   }
24180   return;
24181 }
24182 void Emit_170(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24183   SDOperand Tmp0(0, 0);
24184   SDNode *ResNode = NULL;
24185   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24186   Select(Tmp0, N0);
24187   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24188   if (N.Val->hasOneUse()) {
24189     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
24190   } else {
24191     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
24192     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24193     Result = SDOperand(ResNode, 0);
24194   }
24195   return;
24196 }
24197 void Emit_171(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24198   SDOperand Tmp0(0, 0);
24199   SDNode *ResNode = NULL;
24200   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24201   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24202   Select(Tmp0, N0);
24203   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24204   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24205   if (N.Val->hasOneUse()) {
24206     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp0);
24207   } else {
24208     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp0);
24209     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24210     Result = SDOperand(ResNode, 0);
24211   }
24212   return;
24213 }
24214 void Emit_172(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24215   SDOperand Tmp0(0, 0);
24216   SDOperand Tmp1(0, 0);
24217   SDNode *ResNode = NULL;
24218   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24219   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
24220   Select(Tmp0, N0);
24221   Select(Tmp1, N1);
24222   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24223   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
24224   if (N.Val->hasOneUse()) {
24225     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1);
24226   } else {
24227     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1);
24228     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24229     Result = SDOperand(ResNode, 0);
24230   }
24231   return;
24232 }
24233 void Emit_173(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N2) NOINLINE {
24234   SDOperand Tmp0(0, 0);
24235   SDOperand Tmp1(0, 0);
24236   SDOperand Tmp2(0, 0);
24237   SDOperand Tmp3(0, 0);
24238   SDOperand Tmp4(0, 0);
24239   SDOperand Tmp5(0, 0);
24240   SDNode *ResNode = NULL;
24241   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24242   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24243   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24244   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24245   Select(Tmp0, CPTmp0);
24246   Select(Tmp1, CPTmp1);
24247   Select(Tmp2, CPTmp2);
24248   Select(Tmp3, CPTmp3);
24249   Tmp4 = N2;
24250   Tmp5 = Transform_SHUFFLE_get_shuf_imm(Tmp4.Val);
24251   Select(Chain00, Chain00);
24252   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24253   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24254   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24255   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24256   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24257   Chain00 = SDOperand(ResNode, 1);
24258   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24259   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24260   AddHandleReplacement(N00.Val, 1, ResNode, 1);
24261   Result = SDOperand(ResNode, N.ResNo);
24262   return;
24263 }
24264 void Emit_174(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N2) NOINLINE {
24265   SDOperand Tmp0(0, 0);
24266   SDOperand Tmp1(0, 0);
24267   SDOperand Tmp2(0, 0);
24268   SDOperand Tmp3(0, 0);
24269   SDOperand Tmp4(0, 0);
24270   SDOperand Tmp5(0, 0);
24271   SDNode *ResNode = NULL;
24272   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24273   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24274   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24275   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24276   Select(Tmp0, CPTmp0);
24277   Select(Tmp1, CPTmp1);
24278   Select(Tmp2, CPTmp2);
24279   Select(Tmp3, CPTmp3);
24280   Tmp4 = N2;
24281   Tmp5 = Transform_SHUFFLE_get_pshufhw_imm(Tmp4.Val);
24282   Select(Chain00, Chain00);
24283   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24284   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24285   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24286   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24287   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24288   Chain00 = SDOperand(ResNode, 1);
24289   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24290   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24291   AddHandleReplacement(N00.Val, 1, ResNode, 1);
24292   Result = SDOperand(ResNode, N.ResNo);
24293   return;
24294 }
24295 void Emit_175(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain00, SDOperand &N0, SDOperand &N00, SDOperand &N001, SDOperand &N1, SDOperand &N2) NOINLINE {
24296   SDOperand Tmp0(0, 0);
24297   SDOperand Tmp1(0, 0);
24298   SDOperand Tmp2(0, 0);
24299   SDOperand Tmp3(0, 0);
24300   SDOperand Tmp4(0, 0);
24301   SDOperand Tmp5(0, 0);
24302   SDNode *ResNode = NULL;
24303   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24304   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24305   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24306   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24307   Select(Tmp0, CPTmp0);
24308   Select(Tmp1, CPTmp1);
24309   Select(Tmp2, CPTmp2);
24310   Select(Tmp3, CPTmp3);
24311   Tmp4 = N2;
24312   Tmp5 = Transform_SHUFFLE_get_pshuflw_imm(Tmp4.Val);
24313   Select(Chain00, Chain00);
24314   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24315   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24316   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24317   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24318   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24319   Chain00 = SDOperand(ResNode, 1);
24320   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24321   SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24322   AddHandleReplacement(N00.Val, 1, ResNode, 1);
24323   Result = SDOperand(ResNode, N.ResNo);
24324   return;
24325 }
24326 void Emit_176(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N2) NOINLINE {
24327   SDOperand Tmp0(0, 0);
24328   SDOperand Tmp1(0, 0);
24329   SDOperand Tmp2(0, 0);
24330   SDOperand Tmp3(0, 0);
24331   SDNode *ResNode = NULL;
24332   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24333   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24334   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24335   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24336   Select(Tmp0, CPTmp0);
24337   Select(Tmp1, CPTmp1);
24338   Select(Tmp2, CPTmp2);
24339   Select(Tmp3, CPTmp3);
24340   Select(Chain0, Chain0);
24341   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24342   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24343   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24344   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24345   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
24346   Chain0 = SDOperand(ResNode, 1);
24347   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24348   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
24349   AddHandleReplacement(N0.Val, 1, ResNode, 1);
24350   Result = SDOperand(ResNode, N.ResNo);
24351   return;
24352 }
24353 void Emit_177(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain0, SDOperand &N0, SDOperand &N01, SDOperand &N1, SDOperand &N2) NOINLINE {
24354   SDOperand Tmp0(0, 0);
24355   SDOperand Tmp1(0, 0);
24356   SDOperand Tmp2(0, 0);
24357   SDOperand Tmp3(0, 0);
24358   SDOperand Tmp4(0, 0);
24359   SDOperand Tmp5(0, 0);
24360   SDNode *ResNode = NULL;
24361   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24362   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24363   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24364   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24365   Select(Tmp0, CPTmp0);
24366   Select(Tmp1, CPTmp1);
24367   Select(Tmp2, CPTmp2);
24368   Select(Tmp3, CPTmp3);
24369   Tmp4 = N2;
24370   Tmp5 = Transform_SHUFFLE_get_shuf_imm(Tmp4.Val);
24371   Select(Chain0, Chain0);
24372   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24373   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24374   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24375   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24376   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain0);
24377   Chain0 = SDOperand(ResNode, 1);
24378   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24379   SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
24380   AddHandleReplacement(N0.Val, 1, ResNode, 1);
24381   Result = SDOperand(ResNode, N.ResNo);
24382   return;
24383 }
24384 void Emit_178(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain10, SDOperand &N0, SDOperand &N1, SDOperand &N10, SDOperand &N101, SDOperand &N2) NOINLINE {
24385   SDOperand Tmp0(0, 0);
24386   SDOperand Tmp1(0, 0);
24387   SDOperand Tmp2(0, 0);
24388   SDOperand Tmp3(0, 0);
24389   SDOperand Tmp4(0, 0);
24390   SDOperand Tmp5(0, 0);
24391   SDOperand Tmp6(0, 0);
24392   SDNode *ResNode = NULL;
24393   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24394   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24395   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24396   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24397   Select(Tmp0, CPTmp0);
24398   Select(Tmp1, CPTmp1);
24399   Select(Tmp2, CPTmp2);
24400   Select(Tmp3, CPTmp3);
24401   Select(Tmp4, N0);
24402   Tmp5 = N2;
24403   Tmp6 = Transform_SHUFFLE_get_shuf_imm(Tmp5.Val);
24404   Select(Chain10, Chain10);
24405   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24406   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24407   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24408   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24409   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24410   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp6, Chain10);
24411   Chain10 = SDOperand(ResNode, 1);
24412   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24413   SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24414   AddHandleReplacement(N10.Val, 1, ResNode, 1);
24415   Result = SDOperand(ResNode, N.ResNo);
24416   return;
24417 }
24418 void Emit_179(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &CPTmp0, SDOperand &CPTmp1, SDOperand &CPTmp2, SDOperand &CPTmp3, SDOperand &Chain1, SDOperand &N0, SDOperand &N1, SDOperand &N11, SDOperand &N2) NOINLINE {
24419   SDOperand Tmp0(0, 0);
24420   SDOperand Tmp1(0, 0);
24421   SDOperand Tmp2(0, 0);
24422   SDOperand Tmp3(0, 0);
24423   SDOperand Tmp4(0, 0);
24424   SDOperand Tmp5(0, 0);
24425   SDOperand Tmp6(0, 0);
24426   SDNode *ResNode = NULL;
24427   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp0.Val);
24428   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp1.Val);
24429   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp2.Val);
24430   SelectionDAG::InsertInFlightSetEntry(InFlightSet, CPTmp3.Val);
24431   Select(Tmp0, CPTmp0);
24432   Select(Tmp1, CPTmp1);
24433   Select(Tmp2, CPTmp2);
24434   Select(Tmp3, CPTmp3);
24435   Select(Tmp4, N0);
24436   Tmp5 = N2;
24437   Tmp6 = Transform_SHUFFLE_get_shuf_imm(Tmp5.Val);
24438   Select(Chain1, Chain1);
24439   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24440   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp0.Val);
24441   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp1.Val);
24442   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp2.Val);
24443   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, CPTmp3.Val);
24444   ResNode = CurDAG->getTargetNode(Opc0, VT0, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp6, Chain1);
24445   Chain1 = SDOperand(ResNode, 1);
24446   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24447   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
24448   AddHandleReplacement(N1.Val, 1, ResNode, 1);
24449   Result = SDOperand(ResNode, N.ResNo);
24450   return;
24451 }
24452 void Emit_180(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24453   SDOperand Tmp0(0, 0);
24454   SDOperand Tmp1(0, 0);
24455   SDOperand Tmp2(0, 0);
24456   SDNode *ResNode = NULL;
24457   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24458   Select(Tmp0, N0);
24459   Tmp1 = N2;
24460   Tmp2 = Transform_SHUFFLE_get_shuf_imm(Tmp1.Val);
24461   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24462   if (N.Val->hasOneUse()) {
24463     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp2);
24464   } else {
24465     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp2);
24466     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24467     Result = SDOperand(ResNode, 0);
24468   }
24469   return;
24470 }
24471 void Emit_181(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24472   SDOperand Tmp0(0, 0);
24473   SDOperand Tmp1(0, 0);
24474   SDOperand Tmp2(0, 0);
24475   SDNode *ResNode = NULL;
24476   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24477   Select(Tmp0, N0);
24478   Tmp1 = N2;
24479   Tmp2 = Transform_SHUFFLE_get_pshufhw_imm(Tmp1.Val);
24480   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24481   if (N.Val->hasOneUse()) {
24482     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp2);
24483   } else {
24484     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp2);
24485     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24486     Result = SDOperand(ResNode, 0);
24487   }
24488   return;
24489 }
24490 void Emit_182(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24491   SDOperand Tmp0(0, 0);
24492   SDOperand Tmp1(0, 0);
24493   SDOperand Tmp2(0, 0);
24494   SDNode *ResNode = NULL;
24495   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24496   Select(Tmp0, N0);
24497   Tmp1 = N2;
24498   Tmp2 = Transform_SHUFFLE_get_pshuflw_imm(Tmp1.Val);
24499   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24500   if (N.Val->hasOneUse()) {
24501     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp2);
24502   } else {
24503     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp2);
24504     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24505     Result = SDOperand(ResNode, 0);
24506   }
24507   return;
24508 }
24509 void Emit_183(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24510   SDOperand Tmp0(0, 0);
24511   SDOperand Tmp1(0, 0);
24512   SDOperand Tmp2(0, 0);
24513   SDNode *ResNode = NULL;
24514   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24515   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24516   Select(Tmp0, N0);
24517   Tmp1 = N2;
24518   Tmp2 = Transform_SHUFFLE_get_shuf_imm(Tmp1.Val);
24519   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24520   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24521   if (N.Val->hasOneUse()) {
24522     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp0, Tmp2);
24523   } else {
24524     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp0, Tmp2);
24525     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24526     Result = SDOperand(ResNode, 0);
24527   }
24528   return;
24529 }
24530 void Emit_184(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1, SDOperand &N2) NOINLINE {
24531   SDOperand Tmp0(0, 0);
24532   SDOperand Tmp1(0, 0);
24533   SDOperand Tmp2(0, 0);
24534   SDOperand Tmp3(0, 0);
24535   SDNode *ResNode = NULL;
24536   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24537   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
24538   Select(Tmp0, N0);
24539   Select(Tmp1, N1);
24540   Tmp2 = N2;
24541   Tmp3 = Transform_SHUFFLE_get_shuf_imm(Tmp2.Val);
24542   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24543   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
24544   if (N.Val->hasOneUse()) {
24545     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0, Tmp1, Tmp3);
24546   } else {
24547     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0, Tmp1, Tmp3);
24548     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24549     Result = SDOperand(ResNode, 0);
24550   }
24551   return;
24552 }
24553 void Select_vector_shuffle(SDOperand &Result, SDOperand N) {
24554   SDOperand CPTmp0(0, 0);
24555   SDOperand CPTmp1(0, 0);
24556   SDOperand CPTmp2(0, 0);
24557   SDOperand CPTmp3(0, 0);
24558   SDOperand Chain0(0, 0);
24559   SDOperand Chain00(0, 0);
24560   SDOperand Chain1(0, 0);
24561   SDOperand Chain10(0, 0);
24562   SDOperand Chain100(0, 0);
24563   SDOperand N0(0, 0);
24564   SDOperand N00(0, 0);
24565   SDOperand N001(0, 0);
24566   SDOperand N01(0, 0);
24567   SDOperand N1(0, 0);
24568   SDOperand N10(0, 0);
24569   SDOperand N100(0, 0);
24570   SDOperand N1001(0, 0);
24571   SDOperand N101(0, 0);
24572   SDOperand N11(0, 0);
24573   SDOperand N2(0, 0);
24574   SDOperand Tmp0(0, 0);
24575   SDOperand Tmp1(0, 0);
24576   SDOperand Tmp2(0, 0);
24577   SDOperand Tmp3(0, 0);
24578   SDOperand Tmp4(0, 0);
24579   SDOperand Tmp5(0, 0);
24580   SDOperand Tmp6(0, 0);
24581   SDNode *ResNode = NULL;
24582 
24583   // Pattern: (vector_shuffle:v4f32 (build_vector:v4f32)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v4f32 (load:f32 addr:iPTR:$src)), (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
24584   // Emits: (MOVZSS2PSrm:v4f32 addr:iPTR:$src)
24585   // Pattern complexity = 42  cost = 1  size = 3
24586   if ((Subtarget->hasSSE1())) {
24587     N0 = N.getOperand(0);
24588     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
24589         Predicate_immAllZerosV(N0.Val)) {
24590       N1 = N.getOperand(1);
24591       if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24592           InFlightSet.count(N1.Val) == 0 &&
24593           N1.hasOneUse() &&
24594           !CodeGenMap.count(N1.getValue(0))) {
24595         N10 = N1.getOperand(0);
24596         if (N10.getOpcode() == ISD::LOAD &&
24597             InFlightSet.count(N10.Val) == 0 &&
24598             N10.hasOneUse() &&
24599             !CodeGenMap.count(N10.getValue(0))) {
24600           Chain10 = N10.getOperand(0);
24601           N101 = N10.getOperand(1);
24602           N2 = N.getOperand(2);
24603           if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24604               Predicate_MOVL_shuffle_mask(N2.Val) &&
24605               N.Val->getValueType(0) == MVT::v4f32 &&
24606               N10.Val->getValueType(0) == MVT::f32) {
24607             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24608             if (Match) {
24609               Emit_163(Result, N, X86::MOVZSS2PSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24610               return;
24611             }
24612           }
24613         }
24614       }
24615     }
24616   }
24617   if ((Subtarget->hasSSE2())) {
24618     N0 = N.getOperand(0);
24619     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
24620         Predicate_immAllZerosV(N0.Val)) {
24621       N1 = N.getOperand(1);
24622       if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24623           InFlightSet.count(N1.Val) == 0 &&
24624           N1.hasOneUse() &&
24625           !CodeGenMap.count(N1.getValue(0))) {
24626         N10 = N1.getOperand(0);
24627         if (N10.getOpcode() == ISD::LOAD &&
24628             InFlightSet.count(N10.Val) == 0 &&
24629             N10.hasOneUse() &&
24630             !CodeGenMap.count(N10.getValue(0))) {
24631           Chain10 = N10.getOperand(0);
24632           N101 = N10.getOperand(1);
24633           N2 = N.getOperand(2);
24634           if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24635               Predicate_MOVL_shuffle_mask(N2.Val)) {
24636 
24637             // Pattern: (vector_shuffle:v2f64 (build_vector:v2f64)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src)), (build_vector:v2i32)<<P:Predicate_MOVL_shuffle_mask>>)
24638             // Emits: (MOVZSD2PDrm:v2f64 addr:iPTR:$src)
24639             // Pattern complexity = 42  cost = 1  size = 3
24640             if (N.Val->getValueType(0) == MVT::v2f64 &&
24641                 N10.Val->getValueType(0) == MVT::f64) {
24642               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24643               if (Match) {
24644                 Emit_163(Result, N, X86::MOVZSD2PDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24645                 return;
24646               }
24647             }
24648 
24649             // Pattern: (vector_shuffle:v4i32 (build_vector:v4i32)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v4i32 (load:i32 addr:iPTR:$src)), (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
24650             // Emits: (MOVZDI2PDIrm:v4i32 addr:iPTR:$src)
24651             // Pattern complexity = 42  cost = 1
24652             if (N.Val->getValueType(0) == MVT::v4i32 &&
24653                 N10.Val->getValueType(0) == MVT::i32) {
24654               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24655               if (Match) {
24656                 Emit_163(Result, N, X86::MOVZDI2PDIrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24657                 return;
24658               }
24659             }
24660           }
24661         }
24662       }
24663     }
24664   }
24665   if ((Subtarget->hasSSE1())) {
24666     N0 = N.getOperand(0);
24667     N1 = N.getOperand(1);
24668     if (N1.getOpcode() == ISD::BIT_CONVERT &&
24669         InFlightSet.count(N1.Val) == 0 &&
24670         N1.hasOneUse() &&
24671         !CodeGenMap.count(N1.getValue(0))) {
24672       N10 = N1.getOperand(0);
24673       if (N10.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24674           InFlightSet.count(N10.Val) == 0 &&
24675           N10.hasOneUse() &&
24676           !CodeGenMap.count(N10.getValue(0))) {
24677         N100 = N10.getOperand(0);
24678         if (N100.getOpcode() == ISD::LOAD &&
24679             InFlightSet.count(N100.Val) == 0 &&
24680             N100.hasOneUse() &&
24681             !CodeGenMap.count(N100.getValue(0))) {
24682           Chain100 = N100.getOperand(0);
24683           N1001 = N100.getOperand(1);
24684           N2 = N.getOperand(2);
24685           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24686 
24687             // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (bitconvert:v4f32 (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src2))), (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
24688             // Emits: (MOVLPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
24689             // Pattern complexity = 41  cost = 1  size = 3
24690             if (Predicate_MOVLP_shuffle_mask(N2.Val) &&
24691                 N.Val->getValueType(0) == MVT::v4f32 &&
24692                 N10.Val->getValueType(0) == MVT::v2f64 &&
24693                 N100.Val->getValueType(0) == MVT::f64) {
24694               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24695               bool Match = SelectAddr(N1001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24696               if (!Match) {
24697                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24698               }
24699               if (Match) {
24700                 Emit_164(Result, N, X86::MOVLPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain100, N0, N1, N10, N100, N1001, N2);
24701                 return;
24702               }
24703             }
24704 
24705             // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (bitconvert:v4f32 (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src2))), (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
24706             // Emits: (MOVHPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
24707             // Pattern complexity = 41  cost = 1
24708             if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
24709                 N.Val->getValueType(0) == MVT::v4f32 &&
24710                 N10.Val->getValueType(0) == MVT::v2f64 &&
24711                 N100.Val->getValueType(0) == MVT::f64) {
24712               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24713               bool Match = SelectAddr(N1001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24714               if (!Match) {
24715                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24716               }
24717               if (Match) {
24718                 Emit_164(Result, N, X86::MOVHPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain100, N0, N1, N10, N100, N1001, N2);
24719                 return;
24720               }
24721             }
24722           }
24723         }
24724       }
24725     }
24726   }
24727   if ((Subtarget->hasSSE3())) {
24728     N0 = N.getOperand(0);
24729     if (N0.getOpcode() == ISD::BIT_CONVERT &&
24730         InFlightSet.count(N0.Val) == 0 &&
24731         N0.hasOneUse() &&
24732         !CodeGenMap.count(N0.getValue(0))) {
24733       N00 = N0.getOperand(0);
24734       if (N00.getOpcode() == ISD::LOAD &&
24735           InFlightSet.count(N00.Val) == 0 &&
24736           N00.hasOneUse() &&
24737           !CodeGenMap.count(N00.getValue(0))) {
24738         Chain00 = N00.getOperand(0);
24739         N001 = N00.getOperand(1);
24740         N1 = N.getOperand(1);
24741         if (N1.getOpcode() == ISD::UNDEF) {
24742           N2 = N.getOperand(2);
24743           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24744 
24745             // Pattern: (vector_shuffle:v4i32 (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src)), (undef:v4i32), (build_vector:v4i16)<<P:Predicate_MOVSHDUP_shuffle_mask>>)
24746             // Emits: (MOVSHDUPrm:v4i32 addr:iPTR:$src)
24747             // Pattern complexity = 41  cost = 1  size = 3
24748             if (Predicate_MOVSHDUP_shuffle_mask(N2.Val) &&
24749                 N.Val->getValueType(0) == MVT::v4i32 &&
24750                 N00.Val->getValueType(0) == MVT::v2i64) {
24751               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24752               if (Match) {
24753                 Emit_165(Result, N, X86::MOVSHDUPrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
24754                 return;
24755               }
24756             }
24757 
24758             // Pattern: (vector_shuffle:v4i32 (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src)), (undef:v4i32), (build_vector:v4i16)<<P:Predicate_MOVSLDUP_shuffle_mask>>)
24759             // Emits: (MOVSLDUPrm:v4i32 addr:iPTR:$src)
24760             // Pattern complexity = 41  cost = 1
24761             if (Predicate_MOVSLDUP_shuffle_mask(N2.Val) &&
24762                 N.Val->getValueType(0) == MVT::v4i32 &&
24763                 N00.Val->getValueType(0) == MVT::v2i64) {
24764               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24765               if (Match) {
24766                 Emit_165(Result, N, X86::MOVSLDUPrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
24767                 return;
24768               }
24769             }
24770           }
24771         }
24772       }
24773     }
24774   }
24775   if ((Subtarget->hasSSE2())) {
24776     N0 = N.getOperand(0);
24777     N1 = N.getOperand(1);
24778     if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24779         InFlightSet.count(N1.Val) == 0 &&
24780         N1.hasOneUse() &&
24781         !CodeGenMap.count(N1.getValue(0))) {
24782       N10 = N1.getOperand(0);
24783       if (N10.getOpcode() == ISD::LOAD &&
24784           InFlightSet.count(N10.Val) == 0 &&
24785           N10.hasOneUse() &&
24786           !CodeGenMap.count(N10.getValue(0))) {
24787         Chain10 = N10.getOperand(0);
24788         N101 = N10.getOperand(1);
24789         N2 = N.getOperand(2);
24790         if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24791 
24792           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src2)), (build_vector:v2i32)<<P:Predicate_MOVLP_shuffle_mask>>)
24793           // Emits: (MOVLPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
24794           // Pattern complexity = 39  cost = 1  size = 3
24795           if (Predicate_MOVLP_shuffle_mask(N2.Val) &&
24796               N.Val->getValueType(0) == MVT::v2f64 &&
24797               N10.Val->getValueType(0) == MVT::f64) {
24798             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24799             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24800             if (!Match) {
24801               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24802             }
24803             if (Match) {
24804               Emit_166(Result, N, X86::MOVLPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24805               return;
24806             }
24807           }
24808 
24809           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src2)), (build_vector:v2i32)<<P:Predicate_MOVHP_shuffle_mask>>)
24810           // Emits: (MOVHPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
24811           // Pattern complexity = 39  cost = 1
24812           if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
24813               N.Val->getValueType(0) == MVT::v2f64 &&
24814               N10.Val->getValueType(0) == MVT::f64) {
24815             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24816             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24817             if (!Match) {
24818               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24819             }
24820             if (Match) {
24821               Emit_166(Result, N, X86::MOVHPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24822               return;
24823             }
24824           }
24825         }
24826       }
24827     }
24828 
24829     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
24830     // Emits: (MOVLPSrm:v4i32 VR128:v16i8:$src1, addr:iPTR:$src2)
24831     // Pattern complexity = 39  cost = 1
24832     if (N1.getOpcode() == ISD::BIT_CONVERT &&
24833         InFlightSet.count(N1.Val) == 0 &&
24834         N1.hasOneUse() &&
24835         !CodeGenMap.count(N1.getValue(0))) {
24836       N10 = N1.getOperand(0);
24837       if (N10.getOpcode() == ISD::LOAD &&
24838           InFlightSet.count(N10.Val) == 0 &&
24839           N10.hasOneUse() &&
24840           !CodeGenMap.count(N10.getValue(0))) {
24841         Chain10 = N10.getOperand(0);
24842         N101 = N10.getOperand(1);
24843         N2 = N.getOperand(2);
24844         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24845             Predicate_MOVLP_shuffle_mask(N2.Val) &&
24846             N.Val->getValueType(0) == MVT::v4i32 &&
24847             N10.Val->getValueType(0) == MVT::v2i64) {
24848           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24849           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24850           if (!Match) {
24851             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24852           }
24853           if (Match) {
24854             Emit_166(Result, N, X86::MOVLPSrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24855             return;
24856           }
24857         }
24858       }
24859     }
24860   }
24861   if ((Subtarget->hasSSE1())) {
24862     N0 = N.getOperand(0);
24863     N1 = N.getOperand(1);
24864 
24865     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
24866     // Emits: (MOVHPSrm:v4i32 VR128:v16i8:$src1, addr:iPTR:$src2)
24867     // Pattern complexity = 39  cost = 1  size = 3
24868     if (N1.getOpcode() == ISD::BIT_CONVERT &&
24869         InFlightSet.count(N1.Val) == 0 &&
24870         N1.hasOneUse() &&
24871         !CodeGenMap.count(N1.getValue(0))) {
24872       N10 = N1.getOperand(0);
24873       if (N10.getOpcode() == ISD::LOAD &&
24874           InFlightSet.count(N10.Val) == 0 &&
24875           N10.hasOneUse() &&
24876           !CodeGenMap.count(N10.getValue(0))) {
24877         Chain10 = N10.getOperand(0);
24878         N101 = N10.getOperand(1);
24879         N2 = N.getOperand(2);
24880         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24881             Predicate_MOVHP_shuffle_mask(N2.Val) &&
24882             N.Val->getValueType(0) == MVT::v4i32 &&
24883             N10.Val->getValueType(0) == MVT::v2i64) {
24884           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24885           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24886           if (!Match) {
24887             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24888           }
24889           if (Match) {
24890             Emit_166(Result, N, X86::MOVHPSrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
24891             return;
24892           }
24893         }
24894       }
24895     }
24896 
24897     // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2), (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
24898     // Emits: (MOVLPSrm:v4f32 VR128:v16i8:$src1, addr:iPTR:$src2)
24899     // Pattern complexity = 37  cost = 1
24900     if (N1.getOpcode() == ISD::LOAD &&
24901         InFlightSet.count(N1.Val) == 0 &&
24902         N1.hasOneUse() &&
24903         !CodeGenMap.count(N1.getValue(0)) &&
24904         !isNonImmUse(N.Val, N1.Val)) {
24905       Chain1 = N1.getOperand(0);
24906       N11 = N1.getOperand(1);
24907       N2 = N.getOperand(2);
24908       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24909           Predicate_MOVLP_shuffle_mask(N2.Val) &&
24910           N.Val->getValueType(0) == MVT::v4f32) {
24911         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24912         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24913         if (!Match) {
24914           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24915         }
24916         if (Match) {
24917           Emit_167(Result, N, X86::MOVLPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
24918           return;
24919         }
24920       }
24921     }
24922   }
24923 
24924   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_MOVLP_shuffle_mask>>)
24925   // Emits: (MOVLPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
24926   // Pattern complexity = 37  cost = 1  size = 3
24927   if ((Subtarget->hasSSE2())) {
24928     N0 = N.getOperand(0);
24929     N1 = N.getOperand(1);
24930     if (N1.getOpcode() == ISD::LOAD &&
24931         InFlightSet.count(N1.Val) == 0 &&
24932         N1.hasOneUse() &&
24933         !CodeGenMap.count(N1.getValue(0)) &&
24934         !isNonImmUse(N.Val, N1.Val)) {
24935       Chain1 = N1.getOperand(0);
24936       N11 = N1.getOperand(1);
24937       N2 = N.getOperand(2);
24938       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24939           Predicate_MOVLP_shuffle_mask(N2.Val) &&
24940           N.Val->getValueType(0) == MVT::v2f64) {
24941         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24942         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24943         if (!Match) {
24944           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24945         }
24946         if (Match) {
24947           Emit_167(Result, N, X86::MOVLPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
24948           return;
24949         }
24950       }
24951     }
24952   }
24953 
24954   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2), (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
24955   // Emits: (MOVHPSrm:v4f32 VR128:v16i8:$src1, addr:iPTR:$src2)
24956   // Pattern complexity = 37  cost = 1  size = 3
24957   if ((Subtarget->hasSSE1())) {
24958     N0 = N.getOperand(0);
24959     N1 = N.getOperand(1);
24960     if (N1.getOpcode() == ISD::LOAD &&
24961         InFlightSet.count(N1.Val) == 0 &&
24962         N1.hasOneUse() &&
24963         !CodeGenMap.count(N1.getValue(0)) &&
24964         !isNonImmUse(N.Val, N1.Val)) {
24965       Chain1 = N1.getOperand(0);
24966       N11 = N1.getOperand(1);
24967       N2 = N.getOperand(2);
24968       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24969           Predicate_MOVHP_shuffle_mask(N2.Val) &&
24970           N.Val->getValueType(0) == MVT::v4f32) {
24971         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
24972         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
24973         if (!Match) {
24974           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
24975         }
24976         if (Match) {
24977           Emit_167(Result, N, X86::MOVHPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
24978           return;
24979         }
24980       }
24981     }
24982   }
24983   if ((Subtarget->hasSSE2())) {
24984     N0 = N.getOperand(0);
24985     {
24986       N1 = N.getOperand(1);
24987       if (N1.getOpcode() == ISD::LOAD &&
24988           InFlightSet.count(N1.Val) == 0 &&
24989           N1.hasOneUse() &&
24990           !CodeGenMap.count(N1.getValue(0)) &&
24991           !isNonImmUse(N.Val, N1.Val)) {
24992         Chain1 = N1.getOperand(0);
24993         N11 = N1.getOperand(1);
24994         N2 = N.getOperand(2);
24995         if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24996 
24997           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_MOVHP_shuffle_mask>>)
24998           // Emits: (MOVHPDrm:v2f64 VR128:v16i8:$src1, addr:iPTR:$src2)
24999           // Pattern complexity = 37  cost = 1  size = 3
25000           if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
25001               N.Val->getValueType(0) == MVT::v2f64) {
25002             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25003             bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25004             if (!Match) {
25005               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25006             }
25007             if (Match) {
25008               Emit_167(Result, N, X86::MOVHPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25009               return;
25010             }
25011           }
25012           if (Predicate_MOVLP_shuffle_mask(N2.Val) &&
25013               N.Val->getValueType(0) == MVT::v2i64) {
25014             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25015             bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25016             if (!Match) {
25017               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25018             }
25019             if (Match) {
25020               Emit_167(Result, N, X86::MOVLPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25021               return;
25022             }
25023           }
25024         }
25025       }
25026     }
25027     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
25028         Predicate_immAllZerosV(N0.Val)) {
25029       N1 = N.getOperand(1);
25030 
25031       // Pattern: (vector_shuffle:v4i32 (build_vector:v4i32)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v4i32 GR32:i32:$src), (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
25032       // Emits: (MOVZDI2PDIrr:v4i32 GR32:i32:$src)
25033       // Pattern complexity = 30  cost = 1  size = 3
25034       if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25035         N10 = N1.getOperand(0);
25036         N2 = N.getOperand(2);
25037         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25038             Predicate_MOVL_shuffle_mask(N2.Val) &&
25039             N.Val->getValueType(0) == MVT::v4i32 &&
25040             N10.Val->getValueType(0) == MVT::i32) {
25041           Emit_168(Result, N, X86::MOVZDI2PDIrr, MVT::v4i32, N0, N1, N10, N2);
25042           return;
25043         }
25044       }
25045       if (N1.getOpcode() == X86ISD::S2VEC) {
25046         N10 = N1.getOperand(0);
25047         N2 = N.getOperand(2);
25048         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25049             Predicate_MOVL_shuffle_mask(N2.Val)) {
25050 
25051           // Pattern: (vector_shuffle:v8i16 (build_vector:v8i16)<<P:Predicate_immAllZerosV>>, (X86s2vec:v8i16 GR32:i32:$src), (build_vector:v8i8)<<P:Predicate_MOVL_shuffle_mask>>)
25052           // Emits: (MOVZDI2PDIrr:v8i16 GR32:i32:$src)
25053           // Pattern complexity = 30  cost = 1  size = 3
25054           if (N.Val->getValueType(0) == MVT::v8i16 &&
25055               N10.Val->getValueType(0) == MVT::i32) {
25056             Emit_168(Result, N, X86::MOVZDI2PDIrr, MVT::v8i16, N0, N1, N10, N2);
25057             return;
25058           }
25059 
25060           // Pattern: (vector_shuffle:v16i8 (build_vector:v16i8)<<P:Predicate_immAllZerosV>>, (X86s2vec:v16i8 GR32:i32:$src), (build_vector:v16i8)<<P:Predicate_MOVL_shuffle_mask>>)
25061           // Emits: (MOVZDI2PDIrr:v16i8 GR32:i32:$src)
25062           // Pattern complexity = 30  cost = 1
25063           if (N.Val->getValueType(0) == MVT::v16i8 &&
25064               N10.Val->getValueType(0) == MVT::i32) {
25065             Emit_168(Result, N, X86::MOVZDI2PDIrr, MVT::v16i8, N0, N1, N10, N2);
25066             return;
25067           }
25068         }
25069       }
25070       if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25071         N10 = N1.getOperand(0);
25072         N2 = N.getOperand(2);
25073         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25074             Predicate_MOVL_shuffle_mask(N2.Val)) {
25075 
25076           // Pattern: (vector_shuffle:v2f64 (build_vector:v2f64)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v2f64 FR64:f64:$src), (build_vector:v2i32)<<P:Predicate_MOVL_shuffle_mask>>)
25077           // Emits: (MOVLSD2PDrr:v2f64 (V_SET0:v16i8), FR64:f64:$src)
25078           // Pattern complexity = 30  cost = 2  size = 6
25079           if (N.Val->getValueType(0) == MVT::v2f64 &&
25080               N10.Val->getValueType(0) == MVT::f64) {
25081             Emit_169(Result, N, X86::V_SET0, X86::MOVLSD2PDrr, MVT::v16i8, MVT::v2f64, N0, N1, N10, N2);
25082             return;
25083           }
25084 
25085           // Pattern: (vector_shuffle:v4f32 (build_vector:v4f32)<<P:Predicate_immAllZerosV>>, (scalar_to_vector:v4f32 FR32:f32:$src), (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
25086           // Emits: (MOVLSS2PSrr:v4f32 (V_SET0:v16i8), FR32:f32:$src)
25087           // Pattern complexity = 30  cost = 2
25088           if (N.Val->getValueType(0) == MVT::v4f32 &&
25089               N10.Val->getValueType(0) == MVT::f32) {
25090             Emit_169(Result, N, X86::V_SET0, X86::MOVLSS2PSrr, MVT::v16i8, MVT::v4f32, N0, N1, N10, N2);
25091             return;
25092           }
25093         }
25094       }
25095     }
25096   }
25097 
25098   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25099   // Emits: (UNPCKHPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
25100   // Pattern complexity = 27  cost = 1  size = 3
25101   if ((Subtarget->hasSSE1())) {
25102     N0 = N.getOperand(0);
25103     N1 = N.getOperand(1);
25104     if (N1.getOpcode() == ISD::LOAD &&
25105         InFlightSet.count(N1.Val) == 0 &&
25106         N1.hasOneUse() &&
25107         !CodeGenMap.count(N1.getValue(0)) &&
25108         !isNonImmUse(N.Val, N1.Val)) {
25109       Chain1 = N1.getOperand(0);
25110       N11 = N1.getOperand(1);
25111       N2 = N.getOperand(2);
25112       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25113           Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25114           N.Val->getValueType(0) == MVT::v4f32) {
25115         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25116         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25117         if (!Match) {
25118           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25119         }
25120         if (Match) {
25121           Emit_167(Result, N, X86::UNPCKHPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25122           return;
25123         }
25124       }
25125     }
25126   }
25127 
25128   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25129   // Emits: (UNPCKHPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
25130   // Pattern complexity = 27  cost = 1  size = 3
25131   if ((Subtarget->hasSSE2())) {
25132     N0 = N.getOperand(0);
25133     N1 = N.getOperand(1);
25134     if (N1.getOpcode() == ISD::LOAD &&
25135         InFlightSet.count(N1.Val) == 0 &&
25136         N1.hasOneUse() &&
25137         !CodeGenMap.count(N1.getValue(0)) &&
25138         !isNonImmUse(N.Val, N1.Val)) {
25139       Chain1 = N1.getOperand(0);
25140       N11 = N1.getOperand(1);
25141       N2 = N.getOperand(2);
25142       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25143           Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25144           N.Val->getValueType(0) == MVT::v2f64) {
25145         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25146         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25147         if (!Match) {
25148           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25149         }
25150         if (Match) {
25151           Emit_167(Result, N, X86::UNPCKHPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25152           return;
25153         }
25154       }
25155     }
25156   }
25157 
25158   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2), (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25159   // Emits: (UNPCKLPSrm:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2)
25160   // Pattern complexity = 27  cost = 1  size = 3
25161   if ((Subtarget->hasSSE1())) {
25162     N0 = N.getOperand(0);
25163     N1 = N.getOperand(1);
25164     if (N1.getOpcode() == ISD::LOAD &&
25165         InFlightSet.count(N1.Val) == 0 &&
25166         N1.hasOneUse() &&
25167         !CodeGenMap.count(N1.getValue(0)) &&
25168         !isNonImmUse(N.Val, N1.Val)) {
25169       Chain1 = N1.getOperand(0);
25170       N11 = N1.getOperand(1);
25171       N2 = N.getOperand(2);
25172       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25173           Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25174           N.Val->getValueType(0) == MVT::v4f32) {
25175         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25176         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25177         if (!Match) {
25178           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25179         }
25180         if (Match) {
25181           Emit_167(Result, N, X86::UNPCKLPSrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25182           return;
25183         }
25184       }
25185     }
25186   }
25187 
25188   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25189   // Emits: (UNPCKLPDrm:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2)
25190   // Pattern complexity = 27  cost = 1  size = 3
25191   if ((Subtarget->hasSSE2())) {
25192     N0 = N.getOperand(0);
25193     N1 = N.getOperand(1);
25194     if (N1.getOpcode() == ISD::LOAD &&
25195         InFlightSet.count(N1.Val) == 0 &&
25196         N1.hasOneUse() &&
25197         !CodeGenMap.count(N1.getValue(0)) &&
25198         !isNonImmUse(N.Val, N1.Val)) {
25199       Chain1 = N1.getOperand(0);
25200       N11 = N1.getOperand(1);
25201       N2 = N.getOperand(2);
25202       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25203           Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25204           N.Val->getValueType(0) == MVT::v2f64) {
25205         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25206         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25207         if (!Match) {
25208           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25209         }
25210         if (Match) {
25211           Emit_167(Result, N, X86::UNPCKLPDrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25212           return;
25213         }
25214       }
25215     }
25216   }
25217   if ((Subtarget->hasSSE3())) {
25218     N0 = N.getOperand(0);
25219     N1 = N.getOperand(1);
25220     if (N1.getOpcode() == ISD::UNDEF) {
25221       N2 = N.getOperand(2);
25222       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25223 
25224         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_MOVSHDUP_shuffle_mask>>)
25225         // Emits: (MOVSHDUPrr:v4i32 VR128:v16i8:$src)
25226         // Pattern complexity = 27  cost = 1  size = 3
25227         if (Predicate_MOVSHDUP_shuffle_mask(N2.Val) &&
25228             N.Val->getValueType(0) == MVT::v4i32) {
25229           Emit_170(Result, N, X86::MOVSHDUPrr, MVT::v4i32, N0, N1, N2);
25230           return;
25231         }
25232 
25233         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_MOVSLDUP_shuffle_mask>>)
25234         // Emits: (MOVSLDUPrr:v4i32 VR128:v16i8:$src)
25235         // Pattern complexity = 27  cost = 1
25236         if (Predicate_MOVSLDUP_shuffle_mask(N2.Val) &&
25237             N.Val->getValueType(0) == MVT::v4i32) {
25238           Emit_170(Result, N, X86::MOVSLDUPrr, MVT::v4i32, N0, N1, N2);
25239           return;
25240         }
25241       }
25242     }
25243   }
25244   {
25245     N0 = N.getOperand(0);
25246     N1 = N.getOperand(1);
25247     if (N1.getOpcode() == ISD::UNDEF) {
25248       N2 = N.getOperand(2);
25249       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25250           Predicate_UNPCKH_shuffle_mask(N2.Val)) {
25251 
25252         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25253         // Emits: (MOVHLPSrr:v4f32 VR128:v16i8:$src1, VR128:v16i8:$src1)
25254         // Pattern complexity = 27  cost = 1  size = 3
25255         if (N.Val->getValueType(0) == MVT::v4f32) {
25256           Emit_171(Result, N, X86::MOVHLPSrr, MVT::v4f32, N0, N1, N2);
25257           return;
25258         }
25259 
25260         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25261         // Emits: (MOVHLPSrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src1)
25262         // Pattern complexity = 27  cost = 1
25263         if (N.Val->getValueType(0) == MVT::v4i32) {
25264           Emit_171(Result, N, X86::MOVHLPSrr, MVT::v4i32, N0, N1, N2);
25265           return;
25266         }
25267       }
25268     }
25269   }
25270   if ((Subtarget->hasSSE1())) {
25271     N0 = N.getOperand(0);
25272     N1 = N.getOperand(1);
25273     N2 = N.getOperand(2);
25274     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25275 
25276       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
25277       // Emits: (MOVLHPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25278       // Pattern complexity = 25  cost = 1  size = 3
25279       if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
25280           N.Val->getValueType(0) == MVT::v4f32) {
25281         Emit_172(Result, N, X86::MOVLHPSrr, MVT::v4f32, N0, N1, N2);
25282         return;
25283       }
25284 
25285       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVHLPS_shuffle_mask>>)
25286       // Emits: (MOVHLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25287       // Pattern complexity = 25  cost = 1  size = 3
25288       if (Predicate_MOVHLPS_shuffle_mask(N2.Val) &&
25289           N.Val->getValueType(0) == MVT::v4f32) {
25290         Emit_172(Result, N, X86::MOVHLPSrr, MVT::v4f32, N0, N1, N2);
25291         return;
25292       }
25293 
25294       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
25295       // Emits: (MOVLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25296       // Pattern complexity = 25  cost = 1
25297       if (Predicate_MOVL_shuffle_mask(N2.Val) &&
25298           N.Val->getValueType(0) == MVT::v4f32) {
25299         Emit_172(Result, N, X86::MOVLPSrr, MVT::v4f32, N0, N1, N2);
25300         return;
25301       }
25302     }
25303   }
25304 
25305   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_MOVL_shuffle_mask>>)
25306   // Emits: (MOVLPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25307   // Pattern complexity = 25  cost = 1  size = 3
25308   if ((Subtarget->hasSSE2())) {
25309     N0 = N.getOperand(0);
25310     N1 = N.getOperand(1);
25311     N2 = N.getOperand(2);
25312     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25313         Predicate_MOVL_shuffle_mask(N2.Val) &&
25314         N.Val->getValueType(0) == MVT::v2f64) {
25315       Emit_172(Result, N, X86::MOVLPDrr, MVT::v2f64, N0, N1, N2);
25316       return;
25317     }
25318   }
25319   {
25320     N0 = N.getOperand(0);
25321     N1 = N.getOperand(1);
25322     N2 = N.getOperand(2);
25323     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25324 
25325       // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
25326       // Emits: (MOVLHPSrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2)
25327       // Pattern complexity = 25  cost = 1  size = 3
25328       if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
25329           N.Val->getValueType(0) == MVT::v4i32) {
25330         Emit_172(Result, N, X86::MOVLHPSrr, MVT::v4i32, N0, N1, N2);
25331         return;
25332       }
25333 
25334       // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_MOVHLPS_shuffle_mask>>)
25335       // Emits: (MOVHLPSrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2)
25336       // Pattern complexity = 25  cost = 1
25337       if (Predicate_MOVHLPS_shuffle_mask(N2.Val) &&
25338           N.Val->getValueType(0) == MVT::v4i32) {
25339         Emit_172(Result, N, X86::MOVHLPSrr, MVT::v4i32, N0, N1, N2);
25340         return;
25341       }
25342     }
25343   }
25344   if ((Subtarget->hasSSE2())) {
25345     N0 = N.getOperand(0);
25346     N1 = N.getOperand(1);
25347     N2 = N.getOperand(2);
25348     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25349       if (Predicate_MOVL_shuffle_mask(N2.Val)) {
25350 
25351         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_MOVL_shuffle_mask>>)
25352         // Emits: (MOVLPSrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2)
25353         // Pattern complexity = 25  cost = 1  size = 3
25354         if (N.Val->getValueType(0) == MVT::v4i32) {
25355           Emit_172(Result, N, X86::MOVLPSrr, MVT::v4i32, N0, N1, N2);
25356           return;
25357         }
25358 
25359         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2, (build_vector:v2i32)<<P:Predicate_MOVL_shuffle_mask>>)
25360         // Emits: (MOVLPDrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
25361         // Pattern complexity = 25  cost = 1
25362         if (N.Val->getValueType(0) == MVT::v2i64) {
25363           Emit_172(Result, N, X86::MOVLPDrr, MVT::v2i64, N0, N1, N2);
25364           return;
25365         }
25366       }
25367       if (Predicate_MOVLP_shuffle_mask(N2.Val)) {
25368 
25369         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
25370         // Emits: (MOVLPDrr:v4f32 VR128:v16i8:$src1, VR128:v16i8:$src2)
25371         // Pattern complexity = 25  cost = 1  size = 3
25372         if (N.Val->getValueType(0) == MVT::v4f32) {
25373           Emit_172(Result, N, X86::MOVLPDrr, MVT::v4f32, N0, N1, N2);
25374           return;
25375         }
25376 
25377         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
25378         // Emits: (MOVLPDrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2)
25379         // Pattern complexity = 25  cost = 1
25380         if (N.Val->getValueType(0) == MVT::v4i32) {
25381           Emit_172(Result, N, X86::MOVLPDrr, MVT::v4i32, N0, N1, N2);
25382           return;
25383         }
25384       }
25385     }
25386   }
25387 
25388   // Pattern: (vector_shuffle:v2f64 (scalar_to_vector:v2f64 (load:f64 addr:iPTR:$src)), (undef:v2f64), (build_vector:v2i32)<<P:Predicate_SSE_splat_v2_mask>>)
25389   // Emits: (MOVDDUPrm:v2f64 addr:iPTR:$src)
25390   // Pattern complexity = 21  cost = 1  size = 3
25391   if ((Subtarget->hasSSE3())) {
25392     N0 = N.getOperand(0);
25393     if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
25394         InFlightSet.count(N0.Val) == 0 &&
25395         N0.hasOneUse() &&
25396         !CodeGenMap.count(N0.getValue(0))) {
25397       N00 = N0.getOperand(0);
25398       if (N00.getOpcode() == ISD::LOAD &&
25399           InFlightSet.count(N00.Val) == 0 &&
25400           N00.hasOneUse() &&
25401           !CodeGenMap.count(N00.getValue(0))) {
25402         Chain00 = N00.getOperand(0);
25403         N001 = N00.getOperand(1);
25404         N1 = N.getOperand(1);
25405         if (N1.getOpcode() == ISD::UNDEF) {
25406           N2 = N.getOperand(2);
25407           if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25408               Predicate_SSE_splat_v2_mask(N2.Val) &&
25409               N.Val->getValueType(0) == MVT::v2f64 &&
25410               N00.Val->getValueType(0) == MVT::f64) {
25411             bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25412             if (Match) {
25413               Emit_165(Result, N, X86::MOVDDUPrm, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
25414               return;
25415             }
25416           }
25417         }
25418       }
25419     }
25420   }
25421   if ((Subtarget->hasSSE2())) {
25422     N0 = N.getOperand(0);
25423     if (N0.getOpcode() == ISD::BIT_CONVERT &&
25424         InFlightSet.count(N0.Val) == 0 &&
25425         N0.hasOneUse() &&
25426         !CodeGenMap.count(N0.getValue(0))) {
25427       N00 = N0.getOperand(0);
25428       if (N00.getOpcode() == ISD::LOAD &&
25429           InFlightSet.count(N00.Val) == 0 &&
25430           N00.hasOneUse() &&
25431           !CodeGenMap.count(N00.getValue(0))) {
25432         Chain00 = N00.getOperand(0);
25433         N001 = N00.getOperand(1);
25434         N1 = N.getOperand(1);
25435         if (N1.getOpcode() == ISD::UNDEF) {
25436           N2 = N.getOperand(2);
25437           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25438 
25439             // Pattern: (vector_shuffle:v4i32 (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src1)), (undef:v4i32), (build_vector:v4i16)<<P:Predicate_PSHUFD_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src2)
25440             // Emits: (PSHUFDmi:v4i32 addr:iPTR:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:v4i16):$src2))
25441             // Pattern complexity = 21  cost = 1  size = 3
25442             if (Predicate_PSHUFD_shuffle_mask(N2.Val) &&
25443                 N.Val->getValueType(0) == MVT::v4i32 &&
25444                 N00.Val->getValueType(0) == MVT::v2i64) {
25445               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25446               if (Match) {
25447                 Emit_173(Result, N, X86::PSHUFDmi, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
25448                 return;
25449               }
25450             }
25451 
25452             // Pattern: (vector_shuffle:v8i16 (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src1)), (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFHW_shuffle_mask>><<X:SHUFFLE_get_pshufhw_imm>>:$src2)
25453             // Emits: (PSHUFHWmi:v8i16 addr:iPTR:$src1, (SHUFFLE_get_pshufhw_imm:i8 (build_vector:v8i8):$src2))
25454             // Pattern complexity = 21  cost = 1  size = 3
25455             if (Predicate_PSHUFHW_shuffle_mask(N2.Val) &&
25456                 N.Val->getValueType(0) == MVT::v8i16 &&
25457                 N00.Val->getValueType(0) == MVT::v2i64) {
25458               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25459               if (Match) {
25460                 Emit_174(Result, N, X86::PSHUFHWmi, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
25461                 return;
25462               }
25463             }
25464 
25465             // Pattern: (vector_shuffle:v8i16 (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src1)), (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFLW_shuffle_mask>><<X:SHUFFLE_get_pshuflw_imm>>:$src2)
25466             // Emits: (PSHUFLWmi:v8i16 addr:iPTR:$src1, (SHUFFLE_get_pshuflw_imm:i32 (build_vector:v8i8):$src2))
25467             // Pattern complexity = 21  cost = 1
25468             if (Predicate_PSHUFLW_shuffle_mask(N2.Val) &&
25469                 N.Val->getValueType(0) == MVT::v8i16 &&
25470                 N00.Val->getValueType(0) == MVT::v2i64) {
25471               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25472               if (Match) {
25473                 Emit_175(Result, N, X86::PSHUFLWmi, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N2);
25474                 return;
25475               }
25476             }
25477           }
25478         }
25479       }
25480     }
25481   }
25482   if ((Subtarget->hasSSE3())) {
25483     N0 = N.getOperand(0);
25484     if (N0.getOpcode() == ISD::LOAD &&
25485         InFlightSet.count(N0.Val) == 0 &&
25486         N0.hasOneUse() &&
25487         !CodeGenMap.count(N0.getValue(0)) &&
25488         !isNonImmUse(N.Val, N0.Val)) {
25489       Chain0 = N0.getOperand(0);
25490       N01 = N0.getOperand(1);
25491       N1 = N.getOperand(1);
25492       if (N1.getOpcode() == ISD::UNDEF) {
25493         N2 = N.getOperand(2);
25494         if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25495 
25496           // Pattern: (vector_shuffle:v4f32 (load:v4f32 addr:iPTR:$src), (undef:v4f32), (build_vector:v4i16)<<P:Predicate_MOVSHDUP_shuffle_mask>>)
25497           // Emits: (MOVSHDUPrm:v4f32 addr:iPTR:$src)
25498           // Pattern complexity = 19  cost = 1  size = 3
25499           if (Predicate_MOVSHDUP_shuffle_mask(N2.Val) &&
25500               N.Val->getValueType(0) == MVT::v4f32) {
25501             bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25502             if (Match) {
25503               Emit_176(Result, N, X86::MOVSHDUPrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N2);
25504               return;
25505             }
25506           }
25507 
25508           // Pattern: (vector_shuffle:v4f32 (load:v4f32 addr:iPTR:$src), (undef:v4f32), (build_vector:v4i16)<<P:Predicate_MOVSLDUP_shuffle_mask>>)
25509           // Emits: (MOVSLDUPrm:v4f32 addr:iPTR:$src)
25510           // Pattern complexity = 19  cost = 1
25511           if (Predicate_MOVSLDUP_shuffle_mask(N2.Val) &&
25512               N.Val->getValueType(0) == MVT::v4f32) {
25513             bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25514             if (Match) {
25515               Emit_176(Result, N, X86::MOVSLDUPrm, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N2);
25516               return;
25517             }
25518           }
25519         }
25520       }
25521     }
25522   }
25523   if ((Subtarget->hasSSE2())) {
25524     N0 = N.getOperand(0);
25525     {
25526       N1 = N.getOperand(1);
25527       if (N1.getOpcode() == ISD::BIT_CONVERT &&
25528           InFlightSet.count(N1.Val) == 0 &&
25529           N1.hasOneUse() &&
25530           !CodeGenMap.count(N1.getValue(0))) {
25531         N10 = N1.getOperand(0);
25532         if (N10.getOpcode() == ISD::LOAD &&
25533             InFlightSet.count(N10.Val) == 0 &&
25534             N10.hasOneUse() &&
25535             !CodeGenMap.count(N10.getValue(0))) {
25536           Chain10 = N10.getOperand(0);
25537           N101 = N10.getOperand(1);
25538           N2 = N.getOperand(2);
25539           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25540             if (Predicate_UNPCKL_shuffle_mask(N2.Val)) {
25541 
25542               // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)), (build_vector:v16i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
25543               // Emits: (PUNPCKLBWrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
25544               // Pattern complexity = 19  cost = 1  size = 3
25545               if (N.Val->getValueType(0) == MVT::v16i8 &&
25546                   N10.Val->getValueType(0) == MVT::v2i64) {
25547                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25548                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25549                 if (!Match) {
25550                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25551                 }
25552                 if (Match) {
25553                   Emit_166(Result, N, X86::PUNPCKLBWrm, MVT::v16i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25554                   return;
25555                 }
25556               }
25557 
25558               // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)), (build_vector:v8i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
25559               // Emits: (PUNPCKLWDrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
25560               // Pattern complexity = 19  cost = 1  size = 3
25561               if (N.Val->getValueType(0) == MVT::v8i16 &&
25562                   N10.Val->getValueType(0) == MVT::v2i64) {
25563                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25564                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25565                 if (!Match) {
25566                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25567                 }
25568                 if (Match) {
25569                   Emit_166(Result, N, X86::PUNPCKLWDrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25570                   return;
25571                 }
25572               }
25573 
25574               // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25575               // Emits: (PUNPCKLDQrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
25576               // Pattern complexity = 19  cost = 1
25577               if (N.Val->getValueType(0) == MVT::v4i32 &&
25578                   N10.Val->getValueType(0) == MVT::v2i64) {
25579                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25580                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25581                 if (!Match) {
25582                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25583                 }
25584                 if (Match) {
25585                   Emit_166(Result, N, X86::PUNPCKLDQrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25586                   return;
25587                 }
25588               }
25589             }
25590             if (Predicate_UNPCKH_shuffle_mask(N2.Val)) {
25591 
25592               // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:iPTR:$src2)), (build_vector:v16i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
25593               // Emits: (PUNPCKHBWrm:v16i8 VR128:v16i8:$src1, addr:iPTR:$src2)
25594               // Pattern complexity = 19  cost = 1  size = 3
25595               if (N.Val->getValueType(0) == MVT::v16i8 &&
25596                   N10.Val->getValueType(0) == MVT::v2i64) {
25597                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25598                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25599                 if (!Match) {
25600                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25601                 }
25602                 if (Match) {
25603                   Emit_166(Result, N, X86::PUNPCKHBWrm, MVT::v16i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25604                   return;
25605                 }
25606               }
25607 
25608               // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:iPTR:$src2)), (build_vector:v8i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
25609               // Emits: (PUNPCKHWDrm:v8i16 VR128:v8i16:$src1, addr:iPTR:$src2)
25610               // Pattern complexity = 19  cost = 1  size = 3
25611               if (N.Val->getValueType(0) == MVT::v8i16 &&
25612                   N10.Val->getValueType(0) == MVT::v2i64) {
25613                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25614                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25615                 if (!Match) {
25616                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25617                 }
25618                 if (Match) {
25619                   Emit_166(Result, N, X86::PUNPCKHWDrm, MVT::v8i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25620                   return;
25621                 }
25622               }
25623 
25624               // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25625               // Emits: (PUNPCKHDQrm:v4i32 VR128:v4i32:$src1, addr:iPTR:$src2)
25626               // Pattern complexity = 19  cost = 1
25627               if (N.Val->getValueType(0) == MVT::v4i32 &&
25628                   N10.Val->getValueType(0) == MVT::v2i64) {
25629                 SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25630                 bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25631                 if (!Match) {
25632                   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25633                 }
25634                 if (Match) {
25635                   Emit_166(Result, N, X86::PUNPCKHDQrm, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25636                   return;
25637                 }
25638               }
25639             }
25640           }
25641         }
25642       }
25643     }
25644 
25645     // Pattern: (vector_shuffle:v4f32 (load:v4f32 addr:iPTR:$src1), (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SHUFP_unary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25646     // Emits: (PSHUFDmi:v4f32 addr:iPTR:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:i8)<<P:Predicate_SHUFP_unary_shuffle_mask>>:$sm))
25647     // Pattern complexity = 19  cost = 1  size = 3
25648     if (N0.getOpcode() == ISD::LOAD &&
25649         InFlightSet.count(N0.Val) == 0 &&
25650         N0.hasOneUse() &&
25651         !CodeGenMap.count(N0.getValue(0)) &&
25652         !isNonImmUse(N.Val, N0.Val)) {
25653       Chain0 = N0.getOperand(0);
25654       N01 = N0.getOperand(1);
25655       N1 = N.getOperand(1);
25656       if (N1.getOpcode() == ISD::UNDEF) {
25657         N2 = N.getOperand(2);
25658         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25659             Predicate_SHUFP_unary_shuffle_mask(N2.Val) &&
25660             N.Val->getValueType(0) == MVT::v4f32) {
25661           bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25662           if (Match) {
25663             Emit_177(Result, N, X86::PSHUFDmi, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1, N2);
25664             return;
25665           }
25666         }
25667       }
25668     }
25669 
25670     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:iPTR:$src2)), (build_vector:v4i16)<<P:Predicate_PSHUFD_binary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25671     // Emits: (SHUFPSrmi:v4i32 VR128:v16i8:$src1, addr:iPTR:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_PSHUFD_binary_shuffle_mask>>:$sm))
25672     // Pattern complexity = 19  cost = 1
25673     N1 = N.getOperand(1);
25674     if (N1.getOpcode() == ISD::BIT_CONVERT &&
25675         InFlightSet.count(N1.Val) == 0 &&
25676         N1.hasOneUse() &&
25677         !CodeGenMap.count(N1.getValue(0))) {
25678       N10 = N1.getOperand(0);
25679       if (N10.getOpcode() == ISD::LOAD &&
25680           InFlightSet.count(N10.Val) == 0 &&
25681           N10.hasOneUse() &&
25682           !CodeGenMap.count(N10.getValue(0))) {
25683         Chain10 = N10.getOperand(0);
25684         N101 = N10.getOperand(1);
25685         N2 = N.getOperand(2);
25686         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25687             Predicate_PSHUFD_binary_shuffle_mask(N2.Val) &&
25688             N.Val->getValueType(0) == MVT::v4i32 &&
25689             N10.Val->getValueType(0) == MVT::v2i64) {
25690           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25691           bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25692           if (!Match) {
25693             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25694           }
25695           if (Match) {
25696             Emit_178(Result, N, X86::SHUFPSrmi, MVT::v4i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101, N2);
25697             return;
25698           }
25699         }
25700       }
25701     }
25702   }
25703 
25704   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:iPTR:$src2), (build_vector:v4i16)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
25705   // Emits: (SHUFPSrmi:v4f32 VR128:v4f32:$src1, addr:iPTR:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:v4i16):$src3))
25706   // Pattern complexity = 17  cost = 1  size = 3
25707   if ((Subtarget->hasSSE1())) {
25708     N0 = N.getOperand(0);
25709     N1 = N.getOperand(1);
25710     if (N1.getOpcode() == ISD::LOAD &&
25711         InFlightSet.count(N1.Val) == 0 &&
25712         N1.hasOneUse() &&
25713         !CodeGenMap.count(N1.getValue(0)) &&
25714         !isNonImmUse(N.Val, N1.Val)) {
25715       Chain1 = N1.getOperand(0);
25716       N11 = N1.getOperand(1);
25717       N2 = N.getOperand(2);
25718       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25719           Predicate_SHUFP_shuffle_mask(N2.Val) &&
25720           N.Val->getValueType(0) == MVT::v4f32) {
25721         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25722         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25723         if (!Match) {
25724           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25725         }
25726         if (Match) {
25727           Emit_179(Result, N, X86::SHUFPSrmi, MVT::v4f32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25728           return;
25729         }
25730       }
25731     }
25732   }
25733   if ((Subtarget->hasSSE2())) {
25734     N0 = N.getOperand(0);
25735     N1 = N.getOperand(1);
25736     if (N1.getOpcode() == ISD::LOAD &&
25737         InFlightSet.count(N1.Val) == 0 &&
25738         N1.hasOneUse() &&
25739         !CodeGenMap.count(N1.getValue(0)) &&
25740         !isNonImmUse(N.Val, N1.Val)) {
25741       Chain1 = N1.getOperand(0);
25742       N11 = N1.getOperand(1);
25743       N2 = N.getOperand(2);
25744       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25745 
25746         // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
25747         // Emits: (SHUFPDrmi:v2f64 VR128:v2f64:$src1, addr:iPTR:$src2, (SHUFFLE_get_shuf_imm:i8 (build_vector:v2i32):$src3))
25748         // Pattern complexity = 17  cost = 1  size = 3
25749         if (Predicate_SHUFP_shuffle_mask(N2.Val) &&
25750             N.Val->getValueType(0) == MVT::v2f64) {
25751           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25752           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25753           if (!Match) {
25754             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25755           }
25756           if (Match) {
25757             Emit_179(Result, N, X86::SHUFPDrmi, MVT::v2f64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25758             return;
25759           }
25760         }
25761 
25762         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25763         // Emits: (PUNPCKLQDQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
25764         // Pattern complexity = 17  cost = 1  size = 3
25765         if (Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25766             N.Val->getValueType(0) == MVT::v2i64) {
25767           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25768           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25769           if (!Match) {
25770             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25771           }
25772           if (Match) {
25773             Emit_167(Result, N, X86::PUNPCKLQDQrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25774             return;
25775           }
25776         }
25777 
25778         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25779         // Emits: (PUNPCKHQDQrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
25780         // Pattern complexity = 17  cost = 1
25781         if (Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25782             N.Val->getValueType(0) == MVT::v2i64) {
25783           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
25784           bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
25785           if (!Match) {
25786             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
25787           }
25788           if (Match) {
25789             Emit_167(Result, N, X86::PUNPCKHQDQrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11, N2);
25790             return;
25791           }
25792         }
25793       }
25794     }
25795     if (N1.getOpcode() == ISD::UNDEF) {
25796       N2 = N.getOperand(2);
25797       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25798         if (Predicate_SSE_splat_v2_mask(N2.Val)) {
25799 
25800           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src, (undef:v2f64), (build_vector:v2i32)<<P:Predicate_SSE_splat_v2_mask>>:$sm)
25801           // Emits: (UNPCKLPDrr:v2f64 VR128:v16i8:$src, VR128:v16i8:$src)
25802           // Pattern complexity = 17  cost = 1  size = 3
25803           if (N.Val->getValueType(0) == MVT::v2f64) {
25804             Emit_171(Result, N, X86::UNPCKLPDrr, MVT::v2f64, N0, N1, N2);
25805             return;
25806           }
25807 
25808           // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src, (undef:v2i64), (build_vector:v2i32)<<P:Predicate_SSE_splat_v2_mask>>:$sm)
25809           // Emits: (PUNPCKLQDQrr:v2i64 VR128:v16i8:$src, VR128:v16i8:$src)
25810           // Pattern complexity = 17  cost = 1
25811           if (N.Val->getValueType(0) == MVT::v2i64) {
25812             Emit_171(Result, N, X86::PUNPCKLQDQrr, MVT::v2i64, N0, N1, N2);
25813             return;
25814           }
25815         }
25816         if (Predicate_UNPCKL_v_undef_shuffle_mask(N2.Val)) {
25817 
25818           // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25819           // Emits: (UNPCKLPSrr:v4f32 VR128:v16i8:$src, VR128:v16i8:$src)
25820           // Pattern complexity = 17  cost = 1  size = 3
25821           if (N.Val->getValueType(0) == MVT::v4f32) {
25822             Emit_171(Result, N, X86::UNPCKLPSrr, MVT::v4f32, N0, N1, N2);
25823             return;
25824           }
25825 
25826           // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src, (undef:v16i8), (build_vector:v16i8)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25827           // Emits: (PUNPCKLBWrr:v16i8 VR128:v16i8:$src, VR128:v16i8:$src)
25828           // Pattern complexity = 17  cost = 1  size = 3
25829           if (N.Val->getValueType(0) == MVT::v16i8) {
25830             Emit_171(Result, N, X86::PUNPCKLBWrr, MVT::v16i8, N0, N1, N2);
25831             return;
25832           }
25833 
25834           // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25835           // Emits: (PUNPCKLWDrr:v8i16 VR128:v16i8:$src, VR128:v16i8:$src)
25836           // Pattern complexity = 17  cost = 1
25837           if (N.Val->getValueType(0) == MVT::v8i16) {
25838             Emit_171(Result, N, X86::PUNPCKLWDrr, MVT::v8i16, N0, N1, N2);
25839             return;
25840           }
25841         }
25842       }
25843     }
25844   }
25845   if ((Subtarget->hasSSE1())) {
25846     N0 = N.getOperand(0);
25847     N1 = N.getOperand(1);
25848 
25849     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25850     // Emits: (PUNPCKLDQrr:v4i32 VR128:v16i8:$src, VR128:v16i8:$src)
25851     // Pattern complexity = 17  cost = 1  size = 3
25852     if (N1.getOpcode() == ISD::UNDEF) {
25853       N2 = N.getOperand(2);
25854       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25855           Predicate_UNPCKL_v_undef_shuffle_mask(N2.Val) &&
25856           N.Val->getValueType(0) == MVT::v4i32) {
25857         Emit_171(Result, N, X86::PUNPCKLDQrr, MVT::v4i32, N0, N1, N2);
25858         return;
25859       }
25860     }
25861 
25862     // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25863     // Emits: (UNPCKHPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25864     // Pattern complexity = 15  cost = 1
25865     N2 = N.getOperand(2);
25866     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25867         Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25868         N.Val->getValueType(0) == MVT::v4f32) {
25869       Emit_172(Result, N, X86::UNPCKHPSrr, MVT::v4f32, N0, N1, N2);
25870       return;
25871     }
25872   }
25873 
25874   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25875   // Emits: (UNPCKHPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25876   // Pattern complexity = 15  cost = 1  size = 3
25877   if ((Subtarget->hasSSE2())) {
25878     N0 = N.getOperand(0);
25879     N1 = N.getOperand(1);
25880     N2 = N.getOperand(2);
25881     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25882         Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25883         N.Val->getValueType(0) == MVT::v2f64) {
25884       Emit_172(Result, N, X86::UNPCKHPDrr, MVT::v2f64, N0, N1, N2);
25885       return;
25886     }
25887   }
25888 
25889   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25890   // Emits: (UNPCKLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25891   // Pattern complexity = 15  cost = 1  size = 3
25892   if ((Subtarget->hasSSE1())) {
25893     N0 = N.getOperand(0);
25894     N1 = N.getOperand(1);
25895     N2 = N.getOperand(2);
25896     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25897         Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25898         N.Val->getValueType(0) == MVT::v4f32) {
25899       Emit_172(Result, N, X86::UNPCKLPSrr, MVT::v4f32, N0, N1, N2);
25900       return;
25901     }
25902   }
25903 
25904   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25905   // Emits: (UNPCKLPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25906   // Pattern complexity = 15  cost = 1  size = 3
25907   if ((Subtarget->hasSSE2())) {
25908     N0 = N.getOperand(0);
25909     N1 = N.getOperand(1);
25910     N2 = N.getOperand(2);
25911     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25912         Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25913         N.Val->getValueType(0) == MVT::v2f64) {
25914       Emit_172(Result, N, X86::UNPCKLPDrr, MVT::v2f64, N0, N1, N2);
25915       return;
25916     }
25917   }
25918   if ((Subtarget->hasSSE3())) {
25919     N0 = N.getOperand(0);
25920     N1 = N.getOperand(1);
25921     if (N1.getOpcode() == ISD::UNDEF) {
25922       N2 = N.getOperand(2);
25923       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25924 
25925         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_MOVSHDUP_shuffle_mask>>)
25926         // Emits: (MOVSHDUPrr:v4f32 VR128:v4f32:$src)
25927         // Pattern complexity = 7  cost = 1  size = 3
25928         if (Predicate_MOVSHDUP_shuffle_mask(N2.Val) &&
25929             N.Val->getValueType(0) == MVT::v4f32) {
25930           Emit_170(Result, N, X86::MOVSHDUPrr, MVT::v4f32, N0, N1, N2);
25931           return;
25932         }
25933 
25934         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_MOVSLDUP_shuffle_mask>>)
25935         // Emits: (MOVSLDUPrr:v4f32 VR128:v4f32:$src)
25936         // Pattern complexity = 7  cost = 1  size = 3
25937         if (Predicate_MOVSLDUP_shuffle_mask(N2.Val) &&
25938             N.Val->getValueType(0) == MVT::v4f32) {
25939           Emit_170(Result, N, X86::MOVSLDUPrr, MVT::v4f32, N0, N1, N2);
25940           return;
25941         }
25942 
25943         // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src, (undef:v2f64), (build_vector:v2i32)<<P:Predicate_SSE_splat_v2_mask>>)
25944         // Emits: (MOVDDUPrr:v2f64 VR128:v2f64:$src)
25945         // Pattern complexity = 7  cost = 1
25946         if (Predicate_SSE_splat_v2_mask(N2.Val) &&
25947             N.Val->getValueType(0) == MVT::v2f64) {
25948           Emit_170(Result, N, X86::MOVDDUPrr, MVT::v2f64, N0, N1, N2);
25949           return;
25950         }
25951       }
25952     }
25953   }
25954   if ((Subtarget->hasSSE2())) {
25955     N0 = N.getOperand(0);
25956     N1 = N.getOperand(1);
25957     if (N1.getOpcode() == ISD::UNDEF) {
25958       N2 = N.getOperand(2);
25959       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25960 
25961         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_PSHUFD_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src2)
25962         // Emits: (PSHUFDri:v4i32 VR128:v4i32:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:v4i16):$src2))
25963         // Pattern complexity = 7  cost = 1  size = 3
25964         if (Predicate_PSHUFD_shuffle_mask(N2.Val) &&
25965             N.Val->getValueType(0) == MVT::v4i32) {
25966           Emit_180(Result, N, X86::PSHUFDri, MVT::v4i32, N0, N1, N2);
25967           return;
25968         }
25969 
25970         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFHW_shuffle_mask>><<X:SHUFFLE_get_pshufhw_imm>>:$src2)
25971         // Emits: (PSHUFHWri:v8i16 VR128:v8i16:$src1, (SHUFFLE_get_pshufhw_imm:i8 (build_vector:v8i8):$src2))
25972         // Pattern complexity = 7  cost = 1  size = 3
25973         if (Predicate_PSHUFHW_shuffle_mask(N2.Val) &&
25974             N.Val->getValueType(0) == MVT::v8i16) {
25975           Emit_181(Result, N, X86::PSHUFHWri, MVT::v8i16, N0, N1, N2);
25976           return;
25977         }
25978 
25979         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFLW_shuffle_mask>><<X:SHUFFLE_get_pshuflw_imm>>:$src2)
25980         // Emits: (PSHUFLWri:v8i16 VR128:v8i16:$src1, (SHUFFLE_get_pshuflw_imm:i32 (build_vector:v8i8):$src2))
25981         // Pattern complexity = 7  cost = 1
25982         if (Predicate_PSHUFLW_shuffle_mask(N2.Val) &&
25983             N.Val->getValueType(0) == MVT::v8i16) {
25984           Emit_182(Result, N, X86::PSHUFLWri, MVT::v8i16, N0, N1, N2);
25985           return;
25986         }
25987       }
25988     }
25989   }
25990   if ((Subtarget->hasSSE1())) {
25991     N0 = N.getOperand(0);
25992     N1 = N.getOperand(1);
25993     if (N1.getOpcode() == ISD::UNDEF) {
25994       N2 = N.getOperand(2);
25995       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25996 
25997         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SSE_splat_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25998         // Emits: (SHUFPSrri:v4f32 VR128:v16i8:$src, VR128:v16i8:$src, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_SSE_splat_mask>>:$sm))
25999         // Pattern complexity = 7  cost = 1  size = 3
26000         if (Predicate_SSE_splat_mask(N2.Val) &&
26001             N.Val->getValueType(0) == MVT::v4f32) {
26002           Emit_183(Result, N, X86::SHUFPSrri, MVT::v4f32, N0, N1, N2);
26003           return;
26004         }
26005 
26006         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SHUFP_unary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
26007         // Emits: (SHUFPSrri:v4f32 VR128:v16i8:$src1, VR128:v16i8:$src1, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_SHUFP_unary_shuffle_mask>>:$sm))
26008         // Pattern complexity = 7  cost = 1
26009         if (Predicate_SHUFP_unary_shuffle_mask(N2.Val) &&
26010             N.Val->getValueType(0) == MVT::v4f32) {
26011           Emit_183(Result, N, X86::SHUFPSrri, MVT::v4f32, N0, N1, N2);
26012           return;
26013         }
26014       }
26015     }
26016 
26017     // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
26018     // Emits: (SHUFPSrri:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:v4i16):$src3))
26019     // Pattern complexity = 5  cost = 1
26020     N2 = N.getOperand(2);
26021     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
26022         Predicate_SHUFP_shuffle_mask(N2.Val) &&
26023         N.Val->getValueType(0) == MVT::v4f32) {
26024       Emit_184(Result, N, X86::SHUFPSrri, MVT::v4f32, N0, N1, N2);
26025       return;
26026     }
26027   }
26028   if ((Subtarget->hasSSE2())) {
26029     N0 = N.getOperand(0);
26030     N1 = N.getOperand(1);
26031     N2 = N.getOperand(2);
26032     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
26033 
26034       // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
26035       // Emits: (SHUFPDrri:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (SHUFFLE_get_shuf_imm:i8 (build_vector:v2i32):$src3))
26036       // Pattern complexity = 5  cost = 1  size = 3
26037       if (Predicate_SHUFP_shuffle_mask(N2.Val) &&
26038           N.Val->getValueType(0) == MVT::v2f64) {
26039         Emit_184(Result, N, X86::SHUFPDrri, MVT::v2f64, N0, N1, N2);
26040         return;
26041       }
26042       if (Predicate_UNPCKL_shuffle_mask(N2.Val)) {
26043 
26044         // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2, (build_vector:v16i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
26045         // Emits: (PUNPCKLBWrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
26046         // Pattern complexity = 5  cost = 1  size = 3
26047         if (N.Val->getValueType(0) == MVT::v16i8) {
26048           Emit_172(Result, N, X86::PUNPCKLBWrr, MVT::v16i8, N0, N1, N2);
26049           return;
26050         }
26051 
26052         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2, (build_vector:v8i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
26053         // Emits: (PUNPCKLWDrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
26054         // Pattern complexity = 5  cost = 1  size = 3
26055         if (N.Val->getValueType(0) == MVT::v8i16) {
26056           Emit_172(Result, N, X86::PUNPCKLWDrr, MVT::v8i16, N0, N1, N2);
26057           return;
26058         }
26059 
26060         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
26061         // Emits: (PUNPCKLDQrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
26062         // Pattern complexity = 5  cost = 1  size = 3
26063         if (N.Val->getValueType(0) == MVT::v4i32) {
26064           Emit_172(Result, N, X86::PUNPCKLDQrr, MVT::v4i32, N0, N1, N2);
26065           return;
26066         }
26067 
26068         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
26069         // Emits: (PUNPCKLQDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26070         // Pattern complexity = 5  cost = 1
26071         if (N.Val->getValueType(0) == MVT::v2i64) {
26072           Emit_172(Result, N, X86::PUNPCKLQDQrr, MVT::v2i64, N0, N1, N2);
26073           return;
26074         }
26075       }
26076       if (Predicate_UNPCKH_shuffle_mask(N2.Val)) {
26077 
26078         // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2, (build_vector:v16i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
26079         // Emits: (PUNPCKHBWrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
26080         // Pattern complexity = 5  cost = 1  size = 3
26081         if (N.Val->getValueType(0) == MVT::v16i8) {
26082           Emit_172(Result, N, X86::PUNPCKHBWrr, MVT::v16i8, N0, N1, N2);
26083           return;
26084         }
26085 
26086         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2, (build_vector:v8i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
26087         // Emits: (PUNPCKHWDrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
26088         // Pattern complexity = 5  cost = 1  size = 3
26089         if (N.Val->getValueType(0) == MVT::v8i16) {
26090           Emit_172(Result, N, X86::PUNPCKHWDrr, MVT::v8i16, N0, N1, N2);
26091           return;
26092         }
26093 
26094         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
26095         // Emits: (PUNPCKHDQrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
26096         // Pattern complexity = 5  cost = 1  size = 3
26097         if (N.Val->getValueType(0) == MVT::v4i32) {
26098           Emit_172(Result, N, X86::PUNPCKHDQrr, MVT::v4i32, N0, N1, N2);
26099           return;
26100         }
26101 
26102         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
26103         // Emits: (PUNPCKHQDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26104         // Pattern complexity = 5  cost = 1
26105         if (N.Val->getValueType(0) == MVT::v2i64) {
26106           Emit_172(Result, N, X86::PUNPCKHQDQrr, MVT::v2i64, N0, N1, N2);
26107           return;
26108         }
26109       }
26110 
26111       // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_PSHUFD_binary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
26112       // Emits: (SHUFPSrri:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_PSHUFD_binary_shuffle_mask>>:$sm))
26113       // Pattern complexity = 5  cost = 1
26114       if (Predicate_PSHUFD_binary_shuffle_mask(N2.Val) &&
26115           N.Val->getValueType(0) == MVT::v4i32) {
26116         Emit_184(Result, N, X86::SHUFPSrri, MVT::v4i32, N0, N1, N2);
26117         return;
26118       }
26119     }
26120   }
26121   std::cerr << "Cannot yet select: ";
26122   N.Val->dump(CurDAG);
26123   std::cerr << '\n';
26124   abort();
26125 }
26126 
26127 void Emit_185(SDOperand &Result, SDOperand &N, unsigned Opc0, MVT::ValueType VT0, SDOperand &N0, SDOperand &N1) NOINLINE {
26128   SDOperand Tmp0(0, 0);
26129   SDNode *ResNode = NULL;
26130   SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26131   Select(Tmp0, N0);
26132   SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26133   if (N.Val->hasOneUse()) {
26134     Result = CurDAG->SelectNodeTo(N.Val, Opc0, VT0, Tmp0);
26135   } else {
26136     ResNode = CurDAG->getTargetNode(Opc0, VT0, Tmp0);
26137     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26138     Result = SDOperand(ResNode, 0);
26139   }
26140   return;
26141 }
26142 void Select_xor(SDOperand &Result, SDOperand N) {
26143   SDOperand CPTmp0(0, 0);
26144   SDOperand CPTmp1(0, 0);
26145   SDOperand CPTmp2(0, 0);
26146   SDOperand CPTmp3(0, 0);
26147   SDOperand Chain0(0, 0);
26148   SDOperand Chain00(0, 0);
26149   SDOperand Chain1(0, 0);
26150   SDOperand Chain10(0, 0);
26151   SDOperand N0(0, 0);
26152   SDOperand N00(0, 0);
26153   SDOperand N001(0, 0);
26154   SDOperand N01(0, 0);
26155   SDOperand N1(0, 0);
26156   SDOperand N10(0, 0);
26157   SDOperand N101(0, 0);
26158   SDOperand N11(0, 0);
26159   SDOperand Tmp0(0, 0);
26160   SDOperand Tmp1(0, 0);
26161   SDOperand Tmp2(0, 0);
26162   SDOperand Tmp3(0, 0);
26163   SDOperand Tmp4(0, 0);
26164   SDNode *ResNode = NULL;
26165   if ((Subtarget->hasSSE2())) {
26166     N0 = N.getOperand(0);
26167     if (N0.getOpcode() == ISD::BIT_CONVERT) {
26168 
26169       // Pattern: (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)))
26170       // Emits: (XORPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
26171       // Pattern complexity = 18  cost = 1  size = 3
26172       {
26173         N00 = N0.getOperand(0);
26174         N1 = N.getOperand(1);
26175         if (N1.getOpcode() == ISD::BIT_CONVERT &&
26176             InFlightSet.count(N1.Val) == 0 &&
26177             N1.hasOneUse() &&
26178             !CodeGenMap.count(N1.getValue(0))) {
26179           N10 = N1.getOperand(0);
26180           if (N10.getOpcode() == ISD::LOAD &&
26181               InFlightSet.count(N10.Val) == 0 &&
26182               N10.hasOneUse() &&
26183               !CodeGenMap.count(N10.getValue(0))) {
26184             Chain10 = N10.getOperand(0);
26185             N101 = N10.getOperand(1);
26186             if (N.Val->getValueType(0) == MVT::v2i64 &&
26187                 N00.Val->getValueType(0) == MVT::v2f64 &&
26188                 N10.Val->getValueType(0) == MVT::v2f64) {
26189               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N00.Val);
26190               bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26191               if (!Match) {
26192                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N00.Val);
26193               }
26194               if (Match) {
26195                 Emit_75(Result, N, X86::XORPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N00, N1, N10, N101);
26196                 return;
26197               }
26198             }
26199           }
26200         }
26201       }
26202 
26203       // Pattern: (xor:v2i64 (bitconvert:v2i64 (load:v2f64 addr:iPTR:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
26204       // Emits: (XORPDrm:v2i64 VR128:v2f64:$src1, addr:iPTR:$src2)
26205       // Pattern complexity = 18  cost = 1
26206       if (InFlightSet.count(N0.Val) == 0 &&
26207           N0.hasOneUse() &&
26208           !CodeGenMap.count(N0.getValue(0))) {
26209         N00 = N0.getOperand(0);
26210         if (N00.getOpcode() == ISD::LOAD &&
26211             InFlightSet.count(N00.Val) == 0 &&
26212             N00.hasOneUse() &&
26213             !CodeGenMap.count(N00.getValue(0))) {
26214           Chain00 = N00.getOperand(0);
26215           N001 = N00.getOperand(1);
26216           N1 = N.getOperand(1);
26217           if (N1.getOpcode() == ISD::BIT_CONVERT) {
26218             N10 = N1.getOperand(0);
26219             if (N.Val->getValueType(0) == MVT::v2i64 &&
26220                 N00.Val->getValueType(0) == MVT::v2f64 &&
26221                 N10.Val->getValueType(0) == MVT::v2f64) {
26222               SelectionDAG::InsertInFlightSetEntry(InFlightSet, N10.Val);
26223               bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26224               if (!Match) {
26225                 SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N10.Val);
26226               }
26227               if (Match) {
26228                 Emit_76(Result, N, X86::XORPDrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1, N10);
26229                 return;
26230               }
26231             }
26232           }
26233         }
26234       }
26235     }
26236   }
26237   if ((Subtarget->hasSSE1())) {
26238     N0 = N.getOperand(0);
26239 
26240     // Pattern: (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)))
26241     // Emits: (XORPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
26242     // Pattern complexity = 16  cost = 1  size = 3
26243     {
26244       N1 = N.getOperand(1);
26245       if (N1.getOpcode() == ISD::BIT_CONVERT &&
26246           InFlightSet.count(N1.Val) == 0 &&
26247           N1.hasOneUse() &&
26248           !CodeGenMap.count(N1.getValue(0))) {
26249         N10 = N1.getOperand(0);
26250         if (N10.getOpcode() == ISD::LOAD &&
26251             InFlightSet.count(N10.Val) == 0 &&
26252             N10.hasOneUse() &&
26253             !CodeGenMap.count(N10.getValue(0))) {
26254           Chain10 = N10.getOperand(0);
26255           N101 = N10.getOperand(1);
26256           if (N.Val->getValueType(0) == MVT::v2i64 &&
26257               N10.Val->getValueType(0) == MVT::v4f32) {
26258             SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26259             bool Match = SelectAddr(N101, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26260             if (!Match) {
26261               SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26262             }
26263             if (Match) {
26264               Emit_40(Result, N, X86::XORPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain10, N0, N1, N10, N101);
26265               return;
26266             }
26267           }
26268         }
26269       }
26270     }
26271 
26272     // Pattern: (xor:v2i64 (bitconvert:v2i64 (load:v4f32 addr:iPTR:$src2)), VR128:v2i64:$src1)
26273     // Emits: (XORPSrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
26274     // Pattern complexity = 16  cost = 1
26275     if (N0.getOpcode() == ISD::BIT_CONVERT &&
26276         InFlightSet.count(N0.Val) == 0 &&
26277         N0.hasOneUse() &&
26278         !CodeGenMap.count(N0.getValue(0))) {
26279       N00 = N0.getOperand(0);
26280       if (N00.getOpcode() == ISD::LOAD &&
26281           InFlightSet.count(N00.Val) == 0 &&
26282           N00.hasOneUse() &&
26283           !CodeGenMap.count(N00.getValue(0))) {
26284         Chain00 = N00.getOperand(0);
26285         N001 = N00.getOperand(1);
26286         N1 = N.getOperand(1);
26287         if (N.Val->getValueType(0) == MVT::v2i64 &&
26288             N00.Val->getValueType(0) == MVT::v4f32) {
26289           SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
26290           bool Match = SelectAddr(N001, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26291           if (!Match) {
26292             SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
26293           }
26294           if (Match) {
26295             Emit_41(Result, N, X86::XORPSrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain00, N0, N00, N001, N1);
26296             return;
26297           }
26298         }
26299       }
26300     }
26301   }
26302   {
26303     N0 = N.getOperand(0);
26304     N1 = N.getOperand(1);
26305     if (N1.getOpcode() == ISD::LOAD &&
26306         InFlightSet.count(N1.Val) == 0 &&
26307         N1.hasOneUse() &&
26308         !CodeGenMap.count(N1.getValue(0)) &&
26309         !isNonImmUse(N.Val, N1.Val)) {
26310       Chain1 = N1.getOperand(0);
26311       N11 = N1.getOperand(1);
26312 
26313       // Pattern: (xor:i8 GR8:i8:$src1, (load:i8 addr:iPTR:$src2))
26314       // Emits: (XOR8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
26315       // Pattern complexity = 14  cost = 1  size = 3
26316       if (N.Val->getValueType(0) == MVT::i8) {
26317         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26318         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26319         if (!Match) {
26320           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26321         }
26322         if (Match) {
26323           Emit_19(Result, N, X86::XOR8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
26324           return;
26325         }
26326       }
26327 
26328       // Pattern: (xor:i16 GR16:i16:$src1, (load:i16 addr:iPTR:$src2))
26329       // Emits: (XOR16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
26330       // Pattern complexity = 14  cost = 1  size = 3
26331       if (N.Val->getValueType(0) == MVT::i16) {
26332         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26333         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26334         if (!Match) {
26335           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26336         }
26337         if (Match) {
26338           Emit_19(Result, N, X86::XOR16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
26339           return;
26340         }
26341       }
26342 
26343       // Pattern: (xor:i32 GR32:i32:$src1, (load:i32 addr:iPTR:$src2))
26344       // Emits: (XOR32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
26345       // Pattern complexity = 14  cost = 1
26346       if (N.Val->getValueType(0) == MVT::i32) {
26347         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26348         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26349         if (!Match) {
26350           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26351         }
26352         if (Match) {
26353           Emit_19(Result, N, X86::XOR32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
26354           return;
26355         }
26356       }
26357     }
26358   }
26359 
26360   // Pattern: (xor:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:iPTR:$src2))
26361   // Emits: (PXORrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
26362   // Pattern complexity = 14  cost = 1  size = 3
26363   if ((Subtarget->hasSSE2())) {
26364     N0 = N.getOperand(0);
26365     N1 = N.getOperand(1);
26366     if (N1.getOpcode() == ISD::LOAD &&
26367         InFlightSet.count(N1.Val) == 0 &&
26368         N1.hasOneUse() &&
26369         !CodeGenMap.count(N1.getValue(0)) &&
26370         !isNonImmUse(N.Val, N1.Val)) {
26371       Chain1 = N1.getOperand(0);
26372       N11 = N1.getOperand(1);
26373       if (N.Val->getValueType(0) == MVT::v2i64) {
26374         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N0.Val);
26375         bool Match = SelectAddr(N11, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26376         if (!Match) {
26377           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N0.Val);
26378         }
26379         if (Match) {
26380           Emit_19(Result, N, X86::PXORrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain1, N0, N1, N11);
26381           return;
26382         }
26383       }
26384     }
26385   }
26386   {
26387     N0 = N.getOperand(0);
26388     if (N0.getOpcode() == ISD::LOAD &&
26389         InFlightSet.count(N0.Val) == 0 &&
26390         N0.hasOneUse() &&
26391         !CodeGenMap.count(N0.getValue(0)) &&
26392         !isNonImmUse(N.Val, N0.Val)) {
26393       Chain0 = N0.getOperand(0);
26394       N01 = N0.getOperand(1);
26395       N1 = N.getOperand(1);
26396 
26397       // Pattern: (xor:i8 (load:i8 addr:iPTR:$src2), GR8:i8:$src1)
26398       // Emits: (XOR8rm:i8 GR8:i8:$src1, addr:iPTR:$src2)
26399       // Pattern complexity = 14  cost = 1  size = 3
26400       if (N.Val->getValueType(0) == MVT::i8) {
26401         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
26402         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26403         if (!Match) {
26404           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
26405         }
26406         if (Match) {
26407           Emit_20(Result, N, X86::XOR8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
26408           return;
26409         }
26410       }
26411 
26412       // Pattern: (xor:i16 (load:i16 addr:iPTR:$src2), GR16:i16:$src1)
26413       // Emits: (XOR16rm:i16 GR16:i16:$src1, addr:iPTR:$src2)
26414       // Pattern complexity = 14  cost = 1  size = 3
26415       if (N.Val->getValueType(0) == MVT::i16) {
26416         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
26417         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26418         if (!Match) {
26419           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
26420         }
26421         if (Match) {
26422           Emit_20(Result, N, X86::XOR16rm, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
26423           return;
26424         }
26425       }
26426 
26427       // Pattern: (xor:i32 (load:i32 addr:iPTR:$src2), GR32:i32:$src1)
26428       // Emits: (XOR32rm:i32 GR32:i32:$src1, addr:iPTR:$src2)
26429       // Pattern complexity = 14  cost = 1
26430       if (N.Val->getValueType(0) == MVT::i32) {
26431         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
26432         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26433         if (!Match) {
26434           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
26435         }
26436         if (Match) {
26437           Emit_20(Result, N, X86::XOR32rm, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
26438           return;
26439         }
26440       }
26441     }
26442   }
26443   if ((Subtarget->hasSSE2())) {
26444     N0 = N.getOperand(0);
26445 
26446     // Pattern: (xor:v2i64 (load:v2i64 addr:iPTR:$src2), VR128:v2i64:$src1)
26447     // Emits: (PXORrm:v2i64 VR128:v2i64:$src1, addr:iPTR:$src2)
26448     // Pattern complexity = 14  cost = 1  size = 3
26449     if (N0.getOpcode() == ISD::LOAD &&
26450         InFlightSet.count(N0.Val) == 0 &&
26451         N0.hasOneUse() &&
26452         !CodeGenMap.count(N0.getValue(0)) &&
26453         !isNonImmUse(N.Val, N0.Val)) {
26454       Chain0 = N0.getOperand(0);
26455       N01 = N0.getOperand(1);
26456       N1 = N.getOperand(1);
26457       if (N.Val->getValueType(0) == MVT::v2i64) {
26458         SelectionDAG::InsertInFlightSetEntry(InFlightSet, N1.Val);
26459         bool Match = SelectAddr(N01, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26460         if (!Match) {
26461           SelectionDAG::RemoveInFlightSetEntry(InFlightSet, N1.Val);
26462         }
26463         if (Match) {
26464           Emit_20(Result, N, X86::PXORrm, MVT::v2i64, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain0, N0, N01, N1);
26465           return;
26466         }
26467       }
26468     }
26469 
26470     // Pattern: (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
26471     // Emits: (XORPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
26472     // Pattern complexity = 6  cost = 1
26473     if (N0.getOpcode() == ISD::BIT_CONVERT) {
26474       N00 = N0.getOperand(0);
26475       N1 = N.getOperand(1);
26476       if (N1.getOpcode() == ISD::BIT_CONVERT) {
26477         N10 = N1.getOperand(0);
26478         if (N.Val->getValueType(0) == MVT::v2i64 &&
26479             N00.Val->getValueType(0) == MVT::v2f64 &&
26480             N10.Val->getValueType(0) == MVT::v2f64) {
26481           Emit_89(Result, N, X86::XORPDrr, MVT::v2i64, N0, N00, N1, N10);
26482           return;
26483         }
26484       }
26485     }
26486   }
26487   {
26488     N0 = N.getOperand(0);
26489     N1 = N.getOperand(1);
26490     if (N1.getOpcode() == ISD::Constant) {
26491       if (Predicate_immAllOnes(N1.Val)) {
26492 
26493         // Pattern: (xor:i8 GR8:i8:$src, (imm:i8)<<P:Predicate_immAllOnes>>)
26494         // Emits: (NOT8r:i8 GR8:i8:$src)
26495         // Pattern complexity = 5  cost = 1  size = 2
26496         if (N.Val->getValueType(0) == MVT::i8) {
26497           Emit_185(Result, N, X86::NOT8r, MVT::i8, N0, N1);
26498           return;
26499         }
26500 
26501         // Pattern: (xor:i16 GR16:i16:$src, (imm:i16)<<P:Predicate_immAllOnes>>)
26502         // Emits: (NOT16r:i16 GR16:i16:$src)
26503         // Pattern complexity = 5  cost = 1  size = 2
26504         if (N.Val->getValueType(0) == MVT::i16) {
26505           Emit_185(Result, N, X86::NOT16r, MVT::i16, N0, N1);
26506           return;
26507         }
26508 
26509         // Pattern: (xor:i32 GR32:i32:$src, (imm:i32)<<P:Predicate_immAllOnes>>)
26510         // Emits: (NOT32r:i32 GR32:i32:$src)
26511         // Pattern complexity = 5  cost = 1
26512         if (N.Val->getValueType(0) == MVT::i32) {
26513           Emit_185(Result, N, X86::NOT32r, MVT::i32, N0, N1);
26514           return;
26515         }
26516       }
26517 
26518       // Pattern: (xor:i16 GR16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
26519       // Emits: (XOR16ri8:i16 GR16:i16:$src1, (imm:i16):$src2)
26520       // Pattern complexity = 5  cost = 1  size = 3
26521       if (Predicate_i16immSExt8(N1.Val) &&
26522           N.Val->getValueType(0) == MVT::i16) {
26523         Emit_49(Result, N, X86::XOR16ri8, MVT::i16, N0, N1);
26524         return;
26525       }
26526 
26527       // Pattern: (xor:i32 GR32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
26528       // Emits: (XOR32ri8:i32 GR32:i32:$src1, (imm:i32):$src2)
26529       // Pattern complexity = 5  cost = 1  size = 3
26530       if (Predicate_i32immSExt8(N1.Val) &&
26531           N.Val->getValueType(0) == MVT::i32) {
26532         Emit_29(Result, N, X86::XOR32ri8, MVT::i32, N0, N1);
26533         return;
26534       }
26535 
26536       // Pattern: (xor:i8 GR8:i8:$src1, (imm:i8):$src2)
26537       // Emits: (XOR8ri:i8 GR8:i8:$src1, (imm:i8):$src2)
26538       // Pattern complexity = 4  cost = 1  size = 3
26539       if (N.Val->getValueType(0) == MVT::i8) {
26540         Emit_50(Result, N, X86::XOR8ri, MVT::i8, N0, N1);
26541         return;
26542       }
26543 
26544       // Pattern: (xor:i16 GR16:i16:$src1, (imm:i16):$src2)
26545       // Emits: (XOR16ri:i16 GR16:i16:$src1, (imm:i16):$src2)
26546       // Pattern complexity = 4  cost = 1  size = 3
26547       if (N.Val->getValueType(0) == MVT::i16) {
26548         Emit_49(Result, N, X86::XOR16ri, MVT::i16, N0, N1);
26549         return;
26550       }
26551 
26552       // Pattern: (xor:i32 GR32:i32:$src1, (imm:i32):$src2)
26553       // Emits: (XOR32ri:i32 GR32:i32:$src1, (imm:i32):$src2)
26554       // Pattern complexity = 4  cost = 1
26555       if (N.Val->getValueType(0) == MVT::i32) {
26556         Emit_29(Result, N, X86::XOR32ri, MVT::i32, N0, N1);
26557         return;
26558       }
26559     }
26560 
26561     // Pattern: (xor:i8 GR8:i8:$src1, GR8:i8:$src2)
26562     // Emits: (XOR8rr:i8 GR8:i8:$src1, GR8:i8:$src2)
26563     // Pattern complexity = 2  cost = 1  size = 3
26564     if (N.Val->getValueType(0) == MVT::i8) {
26565       Emit_21(Result, N, X86::XOR8rr, MVT::i8, N0, N1);
26566       return;
26567     }
26568 
26569     // Pattern: (xor:i16 GR16:i16:$src1, GR16:i16:$src2)
26570     // Emits: (XOR16rr:i16 GR16:i16:$src1, GR16:i16:$src2)
26571     // Pattern complexity = 2  cost = 1  size = 3
26572     if (N.Val->getValueType(0) == MVT::i16) {
26573       Emit_21(Result, N, X86::XOR16rr, MVT::i16, N0, N1);
26574       return;
26575     }
26576 
26577     // Pattern: (xor:i32 GR32:i32:$src1, GR32:i32:$src2)
26578     // Emits: (XOR32rr:i32 GR32:i32:$src1, GR32:i32:$src2)
26579     // Pattern complexity = 2  cost = 1
26580     if (N.Val->getValueType(0) == MVT::i32) {
26581       Emit_21(Result, N, X86::XOR32rr, MVT::i32, N0, N1);
26582       return;
26583     }
26584   }
26585 
26586   // Pattern: (xor:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26587   // Emits: (XORPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26588   // Pattern complexity = 2  cost = 1  size = 3
26589   if ((Subtarget->hasSSE1())) {
26590     N0 = N.getOperand(0);
26591     N1 = N.getOperand(1);
26592     if (N.Val->getValueType(0) == MVT::v2i64) {
26593       Emit_21(Result, N, X86::XORPSrr, MVT::v2i64, N0, N1);
26594       return;
26595     }
26596   }
26597 
26598   // Pattern: (xor:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26599   // Emits: (PXORrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26600   // Pattern complexity = 2  cost = 1
26601   if ((Subtarget->hasSSE2())) {
26602     N0 = N.getOperand(0);
26603     N1 = N.getOperand(1);
26604     if (N.Val->getValueType(0) == MVT::v2i64) {
26605       Emit_21(Result, N, X86::PXORrr, MVT::v2i64, N0, N1);
26606       return;
26607     }
26608   }
26609   std::cerr << "Cannot yet select: ";
26610   N.Val->dump(CurDAG);
26611   std::cerr << '\n';
26612   abort();
26613 }
26614 
26615 void Select_zext(SDOperand &Result, SDOperand N) {
26616   SDOperand N0(0, 0);
26617   SDOperand Tmp0(0, 0);
26618   SDNode *ResNode = NULL;
26619   N0 = N.getOperand(0);
26620 
26621   // Pattern: (zext:i16 GR8:i8:$src)
26622   // Emits: (MOVZX16rr8:i16 GR8:i8:$src)
26623   // Pattern complexity = 2  cost = 1  size = 3
26624   if (N.Val->getValueType(0) == MVT::i16 &&
26625       N0.Val->getValueType(0) == MVT::i8) {
26626     Emit_36(Result, N, X86::MOVZX16rr8, MVT::i16, N0);
26627     return;
26628   }
26629   if (N.Val->getValueType(0) == MVT::i32) {
26630 
26631     // Pattern: (zext:i32 GR8:i8:$src)
26632     // Emits: (MOVZX32rr8:i32 GR8:i8:$src)
26633     // Pattern complexity = 2  cost = 1  size = 3
26634     if (N0.Val->getValueType(0) == MVT::i8) {
26635       Emit_36(Result, N, X86::MOVZX32rr8, MVT::i32, N0);
26636       return;
26637     }
26638 
26639     // Pattern: (zext:i32 GR16:i16:$src)
26640     // Emits: (MOVZX32rr16:i32 GR16:i16:$src)
26641     // Pattern complexity = 2  cost = 1
26642     if (N0.Val->getValueType(0) == MVT::i16) {
26643       Emit_36(Result, N, X86::MOVZX32rr16, MVT::i32, N0);
26644       return;
26645     }
26646   }
26647   std::cerr << "Cannot yet select: ";
26648   N.Val->dump(CurDAG);
26649   std::cerr << '\n';
26650   abort();
26651 }
26652 
26653 void Select_zextld(SDOperand &Result, SDOperand N) {
26654   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
26655     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
26656     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
26657     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
26658     Result = Dummy;
26659     return;
26660   }
26661   SDOperand CPTmp0(0, 0);
26662   SDOperand CPTmp1(0, 0);
26663   SDOperand CPTmp2(0, 0);
26664   SDOperand CPTmp3(0, 0);
26665   SDOperand Chain(0, 0);
26666   SDOperand N1(0, 0);
26667   SDOperand N2(0, 0);
26668   SDOperand N3(0, 0);
26669   SDOperand Tmp0(0, 0);
26670   SDOperand Tmp1(0, 0);
26671   SDOperand Tmp2(0, 0);
26672   SDOperand Tmp3(0, 0);
26673   SDNode *ResNode = NULL;
26674   Chain = N.getOperand(0);
26675   N1 = N.getOperand(1);
26676   N2 = N.getOperand(2);
26677   N3 = N.getOperand(3);
26678   if (cast<VTSDNode>(N3)->getVT() == MVT::i8) {
26679 
26680     // Pattern: (zextld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
26681     // Emits: (MOVZX16rm8:i16 addr:iPTR:$src)
26682     // Pattern complexity = 12  cost = 1  size = 3
26683     if (N.Val->getValueType(0) == MVT::i16) {
26684       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26685       if (Match) {
26686         Emit_98(Result, N, X86::MOVZX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26687         return;
26688       }
26689     }
26690 
26691     // Pattern: (zextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i8:Other)
26692     // Emits: (MOVZX32rm8:i32 addr:iPTR:$src)
26693     // Pattern complexity = 12  cost = 1
26694     if (N.Val->getValueType(0) == MVT::i32) {
26695       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26696       if (Match) {
26697         Emit_98(Result, N, X86::MOVZX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26698         return;
26699       }
26700     }
26701   }
26702 
26703   // Pattern: (zextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i16:Other)
26704   // Emits: (MOVZX32rm16:i32 addr:iPTR:$src)
26705   // Pattern complexity = 12  cost = 1  size = 3
26706   if (cast<VTSDNode>(N3)->getVT() == MVT::i16 &&
26707       N.Val->getValueType(0) == MVT::i32) {
26708     bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26709     if (Match) {
26710       Emit_98(Result, N, X86::MOVZX32rm16, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26711       return;
26712     }
26713   }
26714   if (cast<VTSDNode>(N3)->getVT() == MVT::i1) {
26715 
26716     // Pattern: (zextld:i8 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
26717     // Emits: (MOV8rm:i8 addr:iPTR:$src)
26718     // Pattern complexity = 12  cost = 1  size = 3
26719     if (N.Val->getValueType(0) == MVT::i8) {
26720       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26721       if (Match) {
26722         Emit_98(Result, N, X86::MOV8rm, MVT::i8, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26723         return;
26724       }
26725     }
26726 
26727     // Pattern: (zextld:i16 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
26728     // Emits: (MOVZX16rm8:i16 addr:iPTR:$src)
26729     // Pattern complexity = 12  cost = 1  size = 3
26730     if (N.Val->getValueType(0) == MVT::i16) {
26731       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26732       if (Match) {
26733         Emit_98(Result, N, X86::MOVZX16rm8, MVT::i16, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26734         return;
26735       }
26736     }
26737 
26738     // Pattern: (zextld:i32 addr:iPTR:$src, srcvalue:Other:$dummy, i1:Other)
26739     // Emits: (MOVZX32rm8:i32 addr:iPTR:$src)
26740     // Pattern complexity = 12  cost = 1
26741     if (N.Val->getValueType(0) == MVT::i32) {
26742       bool Match = SelectAddr(N1, CPTmp0, CPTmp1, CPTmp2, CPTmp3);
26743       if (Match) {
26744         Emit_98(Result, N, X86::MOVZX32rm8, MVT::i32, CPTmp0, CPTmp1, CPTmp2, CPTmp3, Chain, N1, N2, N3);
26745         return;
26746       }
26747     }
26748   }
26749   std::cerr << "Cannot yet select: ";
26750   N.Val->dump(CurDAG);
26751   std::cerr << '\n';
26752   abort();
26753 }
26754 
26755 void Select_INLINEASM(SDOperand& Result, SDOperand N) {
26756   std::vector<SDOperand> Ops(N.Val->op_begin(), N.Val->op_end());
26757   Select(Ops[0], N.getOperand(0)); // Select the chain.
26758 
26759   // Select the flag operand.
26760   if (Ops.back().getValueType() == MVT::Flag)
26761     Select(Ops.back(), Ops.back());
26762   SelectInlineAsmMemoryOperands(Ops, *CurDAG);
26763   std::vector<MVT::ValueType> VTs;
26764   VTs.push_back(MVT::Other);
26765   VTs.push_back(MVT::Flag);
26766   SDOperand New = CurDAG->getNode(ISD::INLINEASM, VTs, Ops);
26767   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26768   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26769   Result = New.getValue(N.ResNo);
26770   return;
26771 }
26772 
26773 // The main instruction selector code.
26774 void SelectCode(SDOperand &Result, SDOperand N) {
26775   if (N.getOpcode() >= ISD::BUILTIN_OP_END &&
26776       N.getOpcode() < (ISD::BUILTIN_OP_END+X86::INSTRUCTION_LIST_END)) {
26777     Result = N;
26778     return;   // Already selected.
26779   }
26780 
26781   std::map<SDOperand, SDOperand>::iterator CGMI = CodeGenMap.find(N);
26782   if (CGMI != CodeGenMap.end()) {
26783     Result = CGMI->second;
26784     return;
26785   }
26786 
26787   switch (N.getOpcode()) {
26788   default: break;
26789   case ISD::EntryToken:       // These leaves remain the same.
26790   case ISD::BasicBlock:
26791   case ISD::Register:
26792   case ISD::HANDLENODE:
26793   case ISD::TargetConstant:
26794   case ISD::TargetConstantPool:
26795   case ISD::TargetFrameIndex:
26796   case ISD::TargetJumpTable:
26797   case ISD::TargetGlobalAddress: {
26798     Result = N;
26799     return;
26800   }
26801   case ISD::AssertSext:
26802   case ISD::AssertZext: {
26803     SDOperand Tmp0;
26804     Select(Tmp0, N.getOperand(0));
26805     if (!N.Val->hasOneUse())
26806       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Tmp0.Val, Tmp0.ResNo);
26807     Result = Tmp0;
26808     return;
26809   }
26810   case ISD::TokenFactor:
26811     if (N.getNumOperands() == 2) {
26812       SDOperand Op0, Op1;
26813       Select(Op0, N.getOperand(0));
26814       Select(Op1, N.getOperand(1));
26815       Result = 
26816           CurDAG->getNode(ISD::TokenFactor, MVT::Other, Op0, Op1);
26817       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Result.Val, Result.ResNo);
26818     } else {
26819       std::vector<SDOperand> Ops;
26820       for (unsigned i = 0, e = N.getNumOperands(); i != e; ++i) {
26821         SDOperand Val;
26822         Select(Val, N.getOperand(i));
26823         Ops.push_back(Val);
26824       }
26825       Result = 
26826           CurDAG->getNode(ISD::TokenFactor, MVT::Other, Ops);
26827       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Result.Val, Result.ResNo);
26828     }
26829     return;
26830   case ISD::CopyFromReg: {
26831     SDOperand Chain;
26832     Select(Chain, N.getOperand(0));
26833     unsigned Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
26834     MVT::ValueType VT = N.Val->getValueType(0);
26835     if (N.Val->getNumValues() == 2) {
26836       if (Chain == N.getOperand(0)) {
26837         Result = N; // No change
26838         return;
26839       }
26840       SDOperand New = CurDAG->getCopyFromReg(Chain, Reg, VT);
26841       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26842       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26843       Result = New.getValue(N.ResNo);
26844       return;
26845     } else {
26846       SDOperand Flag;
26847       if (N.getNumOperands() == 3) Select(Flag, N.getOperand(2));
26848       if (Chain == N.getOperand(0) &&
26849           (N.getNumOperands() == 2 || Flag == N.getOperand(2))) {
26850         Result = N; // No change
26851         return;
26852       }
26853       SDOperand New = CurDAG->getCopyFromReg(Chain, Reg, VT, Flag);
26854       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26855       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26856       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, New.Val, 2);
26857       Result = New.getValue(N.ResNo);
26858       return;
26859     }
26860   }
26861   case ISD::CopyToReg: {
26862     SDOperand Chain;
26863     Select(Chain, N.getOperand(0));
26864     unsigned Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
26865     SDOperand Val;
26866     Select(Val, N.getOperand(2));
26867     Result = N;
26868     if (N.Val->getNumValues() == 1) {
26869       if (Chain != N.getOperand(0) || Val != N.getOperand(2))
26870         Result = CurDAG->getCopyToReg(Chain, Reg, Val);
26871       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Result.Val, 0);
26872     } else {
26873       SDOperand Flag(0, 0);
26874       if (N.getNumOperands() == 4) Select(Flag, N.getOperand(3));
26875       if (Chain != N.getOperand(0) || Val != N.getOperand(2) ||
26876           (N.getNumOperands() == 4 && Flag != N.getOperand(3)))
26877         Result = CurDAG->getCopyToReg(Chain, Reg, Val, Flag);
26878       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Result.Val, 0);
26879       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Result.Val, 1);
26880       Result = Result.getValue(N.ResNo);
26881     }
26882     return;
26883   }
26884   case ISD::INLINEASM:  Select_INLINEASM(Result, N); return;
26885   case X86ISD::Wrapper:          Select_X86Wrapper(Result, N); return;
26886   case X86ISD::BRCOND:           Select_X86brcond(Result, N); return;
26887   case X86ISD::CALL:             Select_X86call(Result, N); return;
26888   case ISD::CALLSEQ_END:         Select_X86callseq_end(Result, N); return;
26889   case ISD::CALLSEQ_START:       Select_X86callseq_start(Result, N); return;
26890   case X86ISD::CMOV:             Select_X86cmov(Result, N); return;
26891   case X86ISD::CMP:              Select_X86cmp(Result, N); return;
26892   case X86ISD::COMI:             Select_X86comi(Result, N); return;
26893   case X86ISD::FAND:             Select_X86fand(Result, N); return;
26894   case X86ISD::FILD:             Select_X86fild(Result, N); return;
26895   case X86ISD::FILD_FLAG:        Select_X86fildflag(Result, N); return;
26896   case X86ISD::FLD:              Select_X86fld(Result, N); return;
26897   case X86ISD::FP_TO_INT16_IN_MEM: Select_X86fp_to_i16mem(Result, N); return;
26898   case X86ISD::FP_TO_INT32_IN_MEM: Select_X86fp_to_i32mem(Result, N); return;
26899   case X86ISD::FP_TO_INT64_IN_MEM: Select_X86fp_to_i64mem(Result, N); return;
26900   case X86ISD::FP_GET_RESULT:    Select_X86fpget(Result, N); return;
26901   case X86ISD::FP_SET_RESULT:    Select_X86fpset(Result, N); return;
26902   case X86ISD::FST:              Select_X86fst(Result, N); return;
26903   case X86ISD::FXOR:             Select_X86fxor(Result, N); return;
26904   case X86ISD::LOAD_PACK:        Select_X86loadp(Result, N); return;
26905   case X86ISD::LOAD_UA:          Select_X86loadu(Result, N); return;
26906   case X86ISD::PEXTRW:           Select_X86pextrw(Result, N); return;
26907   case X86ISD::PINSRW:           Select_X86pinsrw(Result, N); return;
26908   case X86ISD::RDTSC_DAG:        Select_X86rdtsc(Result, N); return;
26909   case X86ISD::REP_MOVS:         Select_X86rep_movs(Result, N); return;
26910   case X86ISD::REP_STOS:         Select_X86rep_stos(Result, N); return;
26911   case X86ISD::RET_FLAG:         Select_X86retflag(Result, N); return;
26912   case X86ISD::S2VEC:            Select_X86s2vec(Result, N); return;
26913   case X86ISD::SETCC:            Select_X86setcc(Result, N); return;
26914   case X86ISD::SHLD:             Select_X86shld(Result, N); return;
26915   case X86ISD::SHRD:             Select_X86shrd(Result, N); return;
26916   case X86ISD::TAILCALL:         Select_X86tailcall(Result, N); return;
26917   case X86ISD::TEST:             Select_X86test(Result, N); return;
26918   case X86ISD::UCOMI:            Select_X86ucomi(Result, N); return;
26919   case ISD::ADD:                 Select_add(Result, N); return;
26920   case ISD::ADDC:                Select_addc(Result, N); return;
26921   case ISD::ADDE:                Select_adde(Result, N); return;
26922   case ISD::AND:                 Select_and(Result, N); return;
26923   case ISD::ANY_EXTEND:          Select_anyext(Result, N); return;
26924   case ISD::BIT_CONVERT:         Select_bitconvert(Result, N); return;
26925   case ISD::BR:                  Select_br(Result, N); return;
26926   case ISD::BRIND:               Select_brind(Result, N); return;
26927   case ISD::BSWAP:               Select_bswap(Result, N); return;
26928   case ISD::BUILD_VECTOR:        Select_build_vector(Result, N); return;
26929   case ISD::DEBUG_LABEL:         Select_dwarf_label(Result, N); return;
26930   case ISD::DEBUG_LOC:           Select_dwarf_loc(Result, N); return;
26931   case ISD::EXTLOAD:             Select_extld(Result, N); return;
26932   case ISD::FABS:                Select_fabs(Result, N); return;
26933   case ISD::FADD:                Select_fadd(Result, N); return;
26934   case ISD::FCOS:                Select_fcos(Result, N); return;
26935   case ISD::FDIV:                Select_fdiv(Result, N); return;
26936   case ISD::FP_EXTEND:           Select_fextend(Result, N); return;
26937   case ISD::FMUL:                Select_fmul(Result, N); return;
26938   case ISD::FNEG:                Select_fneg(Result, N); return;
26939   case ISD::FP_TO_SINT:          Select_fp_to_sint(Result, N); return;
26940   case ISD::TargetConstantFP:    Select_fpimm(Result, N); return;
26941   case ISD::FrameIndex:          Select_frameindex(Result, N); return;
26942   case ISD::FP_ROUND:            Select_fround(Result, N); return;
26943   case ISD::FSIN:                Select_fsin(Result, N); return;
26944   case ISD::FSQRT:               Select_fsqrt(Result, N); return;
26945   case ISD::FSUB:                Select_fsub(Result, N); return;
26946   case ISD::Constant:            Select_imm(Result, N); return;
26947   case ISD::INTRINSIC_VOID:      Select_intrinsic_void(Result, N); return;
26948   case ISD::INTRINSIC_W_CHAIN:   Select_intrinsic_w_chain(Result, N); return;
26949   case ISD::INTRINSIC_WO_CHAIN:  Select_intrinsic_wo_chain(Result, N); return;
26950   case ISD::LOAD:                Select_load(Result, N); return;
26951   case ISD::MUL:                 Select_mul(Result, N); return;
26952   case ISD::OR:                  Select_or(Result, N); return;
26953   case ISD::ROTL:                Select_rotl(Result, N); return;
26954   case ISD::ROTR:                Select_rotr(Result, N); return;
26955   case ISD::SCALAR_TO_VECTOR:    Select_scalar_to_vector(Result, N); return;
26956   case ISD::SIGN_EXTEND:         Select_sext(Result, N); return;
26957   case ISD::SEXTLOAD:            Select_sextld(Result, N); return;
26958   case ISD::SHL:                 Select_shl(Result, N); return;
26959   case ISD::SINT_TO_FP:          Select_sint_to_fp(Result, N); return;
26960   case ISD::SRA:                 Select_sra(Result, N); return;
26961   case ISD::SRL:                 Select_srl(Result, N); return;
26962   case ISD::STORE:               Select_store(Result, N); return;
26963   case ISD::SUB:                 Select_sub(Result, N); return;
26964   case ISD::SUBC:                Select_subc(Result, N); return;
26965   case ISD::SUBE:                Select_sube(Result, N); return;
26966   case ISD::TRUNCATE:            Select_trunc(Result, N); return;
26967   case ISD::TRUNCSTORE:          Select_truncst(Result, N); return;
26968   case ISD::UNDEF:               Select_undef(Result, N); return;
26969   case ISD::EXTRACT_VECTOR_ELT:  Select_vector_extract(Result, N); return;
26970   case ISD::VECTOR_SHUFFLE:      Select_vector_shuffle(Result, N); return;
26971   case ISD::XOR:                 Select_xor(Result, N); return;
26972   case ISD::ZERO_EXTEND:         Select_zext(Result, N); return;
26973   case ISD::ZEXTLOAD:            Select_zextld(Result, N); return;
26974   } // end of big switch.
26975 
26976   std::cerr << "Cannot yet select: ";
26977   if (N.getOpcode() != ISD::INTRINSIC_W_CHAIN &&
26978       N.getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
26979       N.getOpcode() != ISD::INTRINSIC_VOID) {
26980     N.Val->dump(CurDAG);
26981   } else {
26982     unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
26983     std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
26984   }
26985   std::cerr << '\n';
26986   abort();
26987 }