LLVM API Documentation

X86GenDAGISel.inc

Go to the documentation of this file.
00001 //===- TableGen'erated file -------------------------------------*- C++ -*-===//
00002 //
00003 // DAG Instruction Selector for the X86 target
00004 //
00005 // Automatically generated file, do not edit!
00006 //
00007 //===----------------------------------------------------------------------===//
00008 
00009 // *** NOTE: This file is #included into the middle of the target
00010 // *** instruction selector class.  These functions are really methods.
00011 
00012 // Instance var to keep track of multiply used nodes that have 
00013 // already been selected.
00014 std::map<SDOperand, SDOperand> CodeGenMap;
00015 // Instance var to keep track of mapping of chain generating nodes
00016 // and their place handle nodes.
00017 std::map<SDOperand, SDOperand> HandleMap;
00018 // Instance var to keep track of mapping of place handle nodes
00019 // and their replacement nodes.
00020 std::map<SDOperand, SDOperand> ReplaceMap;
00021 
00022 static void findNonImmUse(SDNode* Use, SDNode* Def, bool &found, std::set<SDNode *> &Visited) {
00023   if (found || !Visited.insert(Use).second) return;
00024   for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
00025     SDNode *N = Use->getOperand(i).Val;
00026     if (N->getNodeDepth() >= Def->getNodeDepth()) {
00027       if (N != Def) {
00028         findNonImmUse(N, Def, found, Visited);
00029       } else {
00030         found = true;
00031         break;
00032       }
00033     }
00034   }
00035 }
00036 
00037 static bool isNonImmUse(SDNode* Use, SDNode* Def) {
00038   std::set<SDNode *> Visited;
00039   bool found = false;
00040   for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
00041     SDNode *N = Use->getOperand(i).Val;
00042     if (N != Def) {
00043       findNonImmUse(N, Def, found, Visited);
00044       if (found) break;
00045     }
00046   }
00047   return found;
00048 }
00049 
00050 // AddHandleReplacement - Note the pending replacement node for a
00051 // handle node in ReplaceMap.
00052 void AddHandleReplacement(SDNode *H, unsigned HNum, SDNode *R, unsigned RNum) {
00053   SDOperand N(H, HNum);
00054   std::map<SDOperand, SDOperand>::iterator HMI = HandleMap.find(N);
00055   if (HMI != HandleMap.end()) {
00056     ReplaceMap[HMI->second] = SDOperand(R, RNum);
00057     HandleMap.erase(N);
00058   }
00059 }
00060 
00061 // SelectDanglingHandles - Select replacements for all `dangling`
00062 // handles.Some handles do not yet have replacements because the
00063 // nodes they replacements have only dead readers.
00064 void SelectDanglingHandles() {
00065   for (std::map<SDOperand, SDOperand>::iterator I = HandleMap.begin(),
00066          E = HandleMap.end(); I != E; ++I) {
00067     SDOperand N = I->first;
00068     SDOperand R;
00069     Select(R, N.getValue(0));
00070     AddHandleReplacement(N.Val, N.ResNo, R.Val, R.ResNo);
00071   }
00072 }
00073 
00074 // ReplaceHandles - Replace all the handles with the real target
00075 // specific nodes.
00076 void ReplaceHandles() {
00077   for (std::map<SDOperand, SDOperand>::iterator I = ReplaceMap.begin(),
00078         E = ReplaceMap.end(); I != E; ++I) {
00079     SDOperand From = I->first;
00080     SDOperand To   = I->second;
00081     for (SDNode::use_iterator UI = From.Val->use_begin(), E = From.Val->use_end(); UI != E; ++UI) {
00082       SDNode *Use = *UI;
00083       std::vector<SDOperand> Ops;
00084       for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
00085         SDOperand O = Use->getOperand(i);
00086         if (O.Val == From.Val)
00087           Ops.push_back(To);
00088         else
00089           Ops.push_back(O);
00090       }
00091       SDOperand U = SDOperand(Use, 0);
00092       CurDAG->UpdateNodeOperands(U, Ops);
00093     }
00094   }
00095 }
00096 
00097 // UpdateFoldedChain - return a SDOperand of the new chain created
00098 // if the folding were to happen. This is called when, for example,
00099 // a load is folded into a store. If the store's chain is the load,
00100 // then the resulting node's input chain would be the load's input
00101 // chain. If the store's chain is a TokenFactor and the load's
00102 // output chain feeds into in, then the new chain is a TokenFactor
00103 // with the other operands along with the input chain of the load.
00104 SDOperand UpdateFoldedChain(SelectionDAG *DAG, SDNode *N, SDNode *Chain, SDNode* &OldTF) {
00105   OldTF = NULL;
00106   if (N == Chain) {
00107     return N->getOperand(0);
00108   } else if (Chain->getOpcode() == ISD::TokenFactor &&
00109              N->isOperand(Chain)) {
00110     SDOperand Ch = SDOperand(Chain, 0);
00111     std::map<SDOperand, SDOperand>::iterator CGMI = CodeGenMap.find(Ch);
00112     if (CGMI != CodeGenMap.end())
00113       return SDOperand(0, 0);
00114     OldTF = Chain;
00115     std::vector<SDOperand> Ops;
00116     for (unsigned i = 0; i < Chain->getNumOperands(); ++i) {
00117       SDOperand Op = Chain->getOperand(i);
00118       if (Op.Val == N)
00119         Ops.push_back(N->getOperand(0));
00120       else
00121         Ops.push_back(Op);
00122     }
00123     return DAG->getNode(ISD::TokenFactor, MVT::Other, Ops);
00124   }
00125   return SDOperand(0, 0);
00126 }
00127 
00128 // SelectRoot - Top level entry to DAG isel.
00129 SDOperand SelectRoot(SDOperand N) {
00130   SDOperand ResNode;
00131   Select(ResNode, N);
00132   SelectDanglingHandles();
00133   ReplaceHandles();
00134   ReplaceMap.clear();
00135   return ResNode;
00136 }
00137 
00138 // Node transformations.
00139 inline SDOperand Transform_SHUFFLE_get_shuf_imm(SDNode *N) {
00140 
00141   return getI8Imm(X86::getShuffleSHUFImmediate(N));
00142 
00143 }
00144 inline SDOperand Transform_SHUFFLE_get_pshuflw_imm(SDNode *N) {
00145 
00146   return getI8Imm(X86::getShufflePSHUFLWImmediate(N));
00147 
00148 }
00149 inline SDOperand Transform_SHUFFLE_get_pshufhw_imm(SDNode *N) {
00150 
00151   return getI8Imm(X86::getShufflePSHUFHWImmediate(N));
00152 
00153 }
00154 inline SDOperand Transform_PSxLDQ_imm(SDNode *inN) {
00155   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00156 
00157   // Transformation function: imm >> 3
00158   return getI32Imm(N->getValue() >> 3);
00159 
00160 }
00161 
00162 // Predicate functions.
00163 inline bool Predicate_MOVHLPS_shuffle_mask(SDNode *N) {
00164 
00165   return X86::isMOVHLPSMask(N);
00166 
00167 }
00168 inline bool Predicate_MOVHP_shuffle_mask(SDNode *N) {
00169 
00170   return X86::isMOVHPMask(N);
00171 
00172 }
00173 inline bool Predicate_MOVLHPS_shuffle_mask(SDNode *N) {
00174 
00175   return X86::isMOVLHPSMask(N);
00176 
00177 }
00178 inline bool Predicate_MOVLP_shuffle_mask(SDNode *N) {
00179 
00180   return X86::isMOVLPMask(N);
00181 
00182 }
00183 inline bool Predicate_MOVS_shuffle_mask(SDNode *N) {
00184 
00185   return X86::isMOVSMask(N);
00186 
00187 }
00188 inline bool Predicate_PSHUFD_binary_shuffle_mask(SDNode *N) {
00189 
00190   return X86::isSHUFPMask(N);
00191 
00192 }
00193 inline bool Predicate_PSHUFD_shuffle_mask(SDNode *N) {
00194 
00195   return X86::isPSHUFDMask(N);
00196 
00197 }
00198 inline bool Predicate_PSHUFHW_shuffle_mask(SDNode *N) {
00199 
00200   return X86::isPSHUFHWMask(N);
00201 
00202 }
00203 inline bool Predicate_PSHUFLW_shuffle_mask(SDNode *N) {
00204 
00205   return X86::isPSHUFLWMask(N);
00206 
00207 }
00208 inline bool Predicate_SHUFP_shuffle_mask(SDNode *N) {
00209 
00210   return X86::isSHUFPMask(N);
00211 
00212 }
00213 inline bool Predicate_SHUFP_unary_shuffle_mask(SDNode *N) {
00214 
00215   return X86::isPSHUFDMask(N);
00216 
00217 }
00218 inline bool Predicate_SSE_splat_mask(SDNode *N) {
00219 
00220   return X86::isSplatMask(N);
00221 
00222 }
00223 inline bool Predicate_UNPCKH_shuffle_mask(SDNode *N) {
00224 
00225   return X86::isUNPCKHMask(N);
00226 
00227 }
00228 inline bool Predicate_UNPCKL_shuffle_mask(SDNode *N) {
00229 
00230   return X86::isUNPCKLMask(N);
00231 
00232 }
00233 inline bool Predicate_UNPCKL_v_undef_shuffle_mask(SDNode *N) {
00234 
00235   return X86::isUNPCKL_v_undef_Mask(N);
00236 
00237 }
00238 inline bool Predicate_fp32imm0(SDNode *inN) {
00239   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00240 
00241   return N->isExactlyValue(+0.0);
00242 
00243 }
00244 inline bool Predicate_fp64imm0(SDNode *inN) {
00245   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00246 
00247   return N->isExactlyValue(+0.0);
00248 
00249 }
00250 inline bool Predicate_fp64imm1(SDNode *inN) {
00251   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00252 
00253   return N->isExactlyValue(+1.0);
00254 
00255 }
00256 inline bool Predicate_fp64immneg0(SDNode *inN) {
00257   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00258 
00259   return N->isExactlyValue(-0.0);
00260 
00261 }
00262 inline bool Predicate_fp64immneg1(SDNode *inN) {
00263   ConstantFPSDNode *N = cast<ConstantFPSDNode>(inN);
00264 
00265   return N->isExactlyValue(-1.0);
00266 
00267 }
00268 inline bool Predicate_i16immSExt8(SDNode *inN) {
00269   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00270 
00271   // i16immSExt8 predicate - True if the 16-bit immediate fits in a 8-bit
00272   // sign extended field.
00273   return (int)N->getValue() == (signed char)N->getValue();
00274 
00275 }
00276 inline bool Predicate_i16immZExt8(SDNode *inN) {
00277   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00278 
00279   // i16immZExt8 predicate - True if the 16-bit immediate fits in a 8-bit zero
00280   // extended field.
00281   return (unsigned)N->getValue() == (unsigned char)N->getValue();
00282 
00283 }
00284 inline bool Predicate_i32immSExt8(SDNode *inN) {
00285   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00286 
00287   // i32immSExt8 predicate - True if the 32-bit immediate fits in a 8-bit
00288   // sign extended field.
00289   return (int)N->getValue() == (signed char)N->getValue();
00290 
00291 }
00292 inline bool Predicate_immAllOnes(SDNode *inN) {
00293   ConstantSDNode *N = cast<ConstantSDNode>(inN);
00294  return N->isAllOnesValue(); 
00295 }
00296 inline bool Predicate_immAllOnesV(SDNode *N) {
00297 
00298   return ISD::isBuildVectorAllOnes(N);
00299 
00300 }
00301 inline bool Predicate_immAllZerosV(SDNode *N) {
00302 
00303   return ISD::isBuildVectorAllZeros(N);
00304 
00305 }
00306 inline bool Predicate_vtFP(SDNode *inN) {
00307   VTSDNode *N = cast<VTSDNode>(inN);
00308  return MVT::isFloatingPoint(N->getVT()); 
00309 }
00310 inline bool Predicate_vtInt(SDNode *inN) {
00311   VTSDNode *N = cast<VTSDNode>(inN);
00312  return MVT::isInteger(N->getVT()); 
00313 }
00314 
00315 
00316 void Select_X86Wrapper(SDOperand &Result, SDOperand N) {
00317   SDOperand N0(0, 0);
00318   SDOperand Tmp0(0, 0);
00319   SDNode *ResNode;
00320   N0 = N.getOperand(0);
00321 
00322   // Pattern: (X86Wrapper:i32 (tconstpool:i32):$dst)
00323   // Emits: (MOV32ri:i32 (tconstpool:i32):$dst)
00324   // Pattern complexity = 4  cost = 1
00325   if (N0.getOpcode() == ISD::TargetConstantPool) {
00326     Tmp0 = N0;
00327     if (N.Val->hasOneUse()) {
00328       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV32ri, MVT::i32, Tmp0);
00329     } else {
00330       ResNode = CurDAG->getTargetNode(X86::MOV32ri, MVT::i32, Tmp0);
00331       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
00332       Result = SDOperand(ResNode, 0);
00333     }
00334     return;
00335   }
00336 
00337   // Pattern: (X86Wrapper:i32 (tglobaladdr:i32):$dst)
00338   // Emits: (MOV32ri:i32 (tglobaladdr:i32):$dst)
00339   // Pattern complexity = 4  cost = 1
00340   if (N0.getOpcode() == ISD::TargetGlobalAddress) {
00341     Tmp0 = N0;
00342     if (N.Val->hasOneUse()) {
00343       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV32ri, MVT::i32, Tmp0);
00344     } else {
00345       ResNode = CurDAG->getTargetNode(X86::MOV32ri, MVT::i32, Tmp0);
00346       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
00347       Result = SDOperand(ResNode, 0);
00348     }
00349     return;
00350   }
00351 
00352   // Pattern: (X86Wrapper:i32 (texternalsym:i32):$dst)
00353   // Emits: (MOV32ri:i32 (texternalsym:i32):$dst)
00354   // Pattern complexity = 4  cost = 1
00355   if (N0.getOpcode() == ISD::TargetExternalSymbol) {
00356     Tmp0 = N0;
00357     if (N.Val->hasOneUse()) {
00358       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV32ri, MVT::i32, Tmp0);
00359     } else {
00360       ResNode = CurDAG->getTargetNode(X86::MOV32ri, MVT::i32, Tmp0);
00361       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
00362       Result = SDOperand(ResNode, 0);
00363     }
00364     return;
00365   }
00366   std::cerr << "Cannot yet select: ";
00367   N.Val->dump(CurDAG);
00368   std::cerr << '\n';
00369   abort();
00370 }
00371 
00372 void Select_X86brcond(SDOperand &Result, SDOperand N) {
00373   SDOperand Chain(0, 0);
00374   SDOperand InFlag(0, 0);
00375   SDOperand N1(0, 0);
00376   SDOperand N2(0, 0);
00377   SDOperand Tmp1(0, 0);
00378   SDNode *ResNode;
00379   Chain = N.getOperand(0);
00380   N1 = N.getOperand(1);
00381   if (N1.getOpcode() == ISD::BasicBlock) {
00382     N2 = N.getOperand(2);
00383     if (isa<ConstantSDNode>(N2)) {
00384       int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
00385 
00386       // Pattern: (X86brcond:void (bb:Other):$dst, 4:i8)
00387       // Emits: (JE:void (bb:Other):$dst)
00388       // Pattern complexity = 5  cost = 1
00389       if (CN0 == 4) {
00390         Select(Tmp1, N1);
00391         Select(Chain, Chain);
00392         Select(InFlag, N.getOperand(3));
00393         ResNode = CurDAG->getTargetNode(X86::JE, MVT::Other, Tmp1, Chain, InFlag);
00394         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00395         Result = SDOperand(ResNode, N.ResNo);
00396         return;
00397       }
00398 
00399       // Pattern: (X86brcond:void (bb:Other):$dst, 9:i8)
00400       // Emits: (JNE:void (bb:Other):$dst)
00401       // Pattern complexity = 5  cost = 1
00402       if (CN0 == 9) {
00403         Select(Tmp1, N1);
00404         Select(Chain, Chain);
00405         Select(InFlag, N.getOperand(3));
00406         ResNode = CurDAG->getTargetNode(X86::JNE, MVT::Other, Tmp1, Chain, InFlag);
00407         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00408         Result = SDOperand(ResNode, N.ResNo);
00409         return;
00410       }
00411 
00412       // Pattern: (X86brcond:void (bb:Other):$dst, 7:i8)
00413       // Emits: (JL:void (bb:Other):$dst)
00414       // Pattern complexity = 5  cost = 1
00415       if (CN0 == 7) {
00416         Select(Tmp1, N1);
00417         Select(Chain, Chain);
00418         Select(InFlag, N.getOperand(3));
00419         ResNode = CurDAG->getTargetNode(X86::JL, MVT::Other, Tmp1, Chain, InFlag);
00420         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00421         Result = SDOperand(ResNode, N.ResNo);
00422         return;
00423       }
00424 
00425       // Pattern: (X86brcond:void (bb:Other):$dst, 8:i8)
00426       // Emits: (JLE:void (bb:Other):$dst)
00427       // Pattern complexity = 5  cost = 1
00428       if (CN0 == 8) {
00429         Select(Tmp1, N1);
00430         Select(Chain, Chain);
00431         Select(InFlag, N.getOperand(3));
00432         ResNode = CurDAG->getTargetNode(X86::JLE, MVT::Other, Tmp1, Chain, InFlag);
00433         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00434         Result = SDOperand(ResNode, N.ResNo);
00435         return;
00436       }
00437 
00438       // Pattern: (X86brcond:void (bb:Other):$dst, 5:i8)
00439       // Emits: (JG:void (bb:Other):$dst)
00440       // Pattern complexity = 5  cost = 1
00441       if (CN0 == 5) {
00442         Select(Tmp1, N1);
00443         Select(Chain, Chain);
00444         Select(InFlag, N.getOperand(3));
00445         ResNode = CurDAG->getTargetNode(X86::JG, MVT::Other, Tmp1, Chain, InFlag);
00446         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00447         Result = SDOperand(ResNode, N.ResNo);
00448         return;
00449       }
00450 
00451       // Pattern: (X86brcond:void (bb:Other):$dst, 6:i8)
00452       // Emits: (JGE:void (bb:Other):$dst)
00453       // Pattern complexity = 5  cost = 1
00454       if (CN0 == 6) {
00455         Select(Tmp1, N1);
00456         Select(Chain, Chain);
00457         Select(InFlag, N.getOperand(3));
00458         ResNode = CurDAG->getTargetNode(X86::JGE, MVT::Other, Tmp1, Chain, InFlag);
00459         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00460         Result = SDOperand(ResNode, N.ResNo);
00461         return;
00462       }
00463 
00464       // Pattern: (X86brcond:void (bb:Other):$dst, 2:i8)
00465       // Emits: (JB:void (bb:Other):$dst)
00466       // Pattern complexity = 5  cost = 1
00467       if (CN0 == 2) {
00468         Select(Tmp1, N1);
00469         Select(Chain, Chain);
00470         Select(InFlag, N.getOperand(3));
00471         ResNode = CurDAG->getTargetNode(X86::JB, MVT::Other, Tmp1, Chain, InFlag);
00472         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00473         Result = SDOperand(ResNode, N.ResNo);
00474         return;
00475       }
00476 
00477       // Pattern: (X86brcond:void (bb:Other):$dst, 3:i8)
00478       // Emits: (JBE:void (bb:Other):$dst)
00479       // Pattern complexity = 5  cost = 1
00480       if (CN0 == 3) {
00481         Select(Tmp1, N1);
00482         Select(Chain, Chain);
00483         Select(InFlag, N.getOperand(3));
00484         ResNode = CurDAG->getTargetNode(X86::JBE, MVT::Other, Tmp1, Chain, InFlag);
00485         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00486         Result = SDOperand(ResNode, N.ResNo);
00487         return;
00488       }
00489 
00490       // Pattern: (X86brcond:void (bb:Other):$dst, 0:i8)
00491       // Emits: (JA:void (bb:Other):$dst)
00492       // Pattern complexity = 5  cost = 1
00493       if (CN0 == 0) {
00494         Select(Tmp1, N1);
00495         Select(Chain, Chain);
00496         Select(InFlag, N.getOperand(3));
00497         ResNode = CurDAG->getTargetNode(X86::JA, MVT::Other, Tmp1, Chain, InFlag);
00498         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00499         Result = SDOperand(ResNode, N.ResNo);
00500         return;
00501       }
00502 
00503       // Pattern: (X86brcond:void (bb:Other):$dst, 1:i8)
00504       // Emits: (JAE:void (bb:Other):$dst)
00505       // Pattern complexity = 5  cost = 1
00506       if (CN0 == 1) {
00507         Select(Tmp1, N1);
00508         Select(Chain, Chain);
00509         Select(InFlag, N.getOperand(3));
00510         ResNode = CurDAG->getTargetNode(X86::JAE, MVT::Other, Tmp1, Chain, InFlag);
00511         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00512         Result = SDOperand(ResNode, N.ResNo);
00513         return;
00514       }
00515 
00516       // Pattern: (X86brcond:void (bb:Other):$dst, 15:i8)
00517       // Emits: (JS:void (bb:Other):$dst)
00518       // Pattern complexity = 5  cost = 1
00519       if (CN0 == 15) {
00520         Select(Tmp1, N1);
00521         Select(Chain, Chain);
00522         Select(InFlag, N.getOperand(3));
00523         ResNode = CurDAG->getTargetNode(X86::JS, MVT::Other, Tmp1, Chain, InFlag);
00524         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00525         Result = SDOperand(ResNode, N.ResNo);
00526         return;
00527       }
00528 
00529       // Pattern: (X86brcond:void (bb:Other):$dst, 12:i8)
00530       // Emits: (JNS:void (bb:Other):$dst)
00531       // Pattern complexity = 5  cost = 1
00532       if (CN0 == 12) {
00533         Select(Tmp1, N1);
00534         Select(Chain, Chain);
00535         Select(InFlag, N.getOperand(3));
00536         ResNode = CurDAG->getTargetNode(X86::JNS, MVT::Other, Tmp1, Chain, InFlag);
00537         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00538         Result = SDOperand(ResNode, N.ResNo);
00539         return;
00540       }
00541 
00542       // Pattern: (X86brcond:void (bb:Other):$dst, 14:i8)
00543       // Emits: (JP:void (bb:Other):$dst)
00544       // Pattern complexity = 5  cost = 1
00545       if (CN0 == 14) {
00546         Select(Tmp1, N1);
00547         Select(Chain, Chain);
00548         Select(InFlag, N.getOperand(3));
00549         ResNode = CurDAG->getTargetNode(X86::JP, MVT::Other, Tmp1, Chain, InFlag);
00550         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00551         Result = SDOperand(ResNode, N.ResNo);
00552         return;
00553       }
00554 
00555       // Pattern: (X86brcond:void (bb:Other):$dst, 11:i8)
00556       // Emits: (JNP:void (bb:Other):$dst)
00557       // Pattern complexity = 5  cost = 1
00558       if (CN0 == 11) {
00559         Select(Tmp1, N1);
00560         Select(Chain, Chain);
00561         Select(InFlag, N.getOperand(3));
00562         ResNode = CurDAG->getTargetNode(X86::JNP, MVT::Other, Tmp1, Chain, InFlag);
00563         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00564         Result = SDOperand(ResNode, N.ResNo);
00565         return;
00566       }
00567 
00568       // Pattern: (X86brcond:void (bb:Other):$dst, 13:i8)
00569       // Emits: (JO:void (bb:Other):$dst)
00570       // Pattern complexity = 5  cost = 1
00571       if (CN0 == 13) {
00572         Select(Tmp1, N1);
00573         Select(Chain, Chain);
00574         Select(InFlag, N.getOperand(3));
00575         ResNode = CurDAG->getTargetNode(X86::JO, MVT::Other, Tmp1, Chain, InFlag);
00576         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00577         Result = SDOperand(ResNode, N.ResNo);
00578         return;
00579       }
00580 
00581       // Pattern: (X86brcond:void (bb:Other):$dst, 10:i8)
00582       // Emits: (JNO:void (bb:Other):$dst)
00583       // Pattern complexity = 5  cost = 1
00584       if (CN0 == 10) {
00585         Select(Tmp1, N1);
00586         Select(Chain, Chain);
00587         Select(InFlag, N.getOperand(3));
00588         ResNode = CurDAG->getTargetNode(X86::JNO, MVT::Other, Tmp1, Chain, InFlag);
00589         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00590         Result = SDOperand(ResNode, N.ResNo);
00591         return;
00592       }
00593     }
00594   }
00595   std::cerr << "Cannot yet select: ";
00596   N.Val->dump(CurDAG);
00597   std::cerr << '\n';
00598   abort();
00599 }
00600 
00601 void Select_X86call(SDOperand &Result, SDOperand N) {
00602   SDOperand Chain(0, 0);
00603   SDOperand Chain1(0, 0);
00604   SDOperand InFlag(0, 0);
00605   SDOperand N1(0, 0);
00606   SDOperand N11(0, 0);
00607   SDOperand Tmp0(0, 0);
00608   SDOperand Tmp1(0, 0);
00609   SDOperand Tmp2(0, 0);
00610   SDOperand Tmp3(0, 0);
00611   SDNode *OldTF;
00612   SDNode *ResNode;
00613   Chain = N.getOperand(0);
00614   N1 = N.getOperand(1);
00615 
00616   // Pattern: (X86call:void (load:i32 addr:i32:$dst))
00617   // Emits: (CALL32m:void addr:i32:$dst)
00618   // Pattern complexity = 14  cost = 1
00619   if (N1.getOpcode() == ISD::LOAD &&
00620       N1.hasOneUse() &&
00621       !CodeGenMap.count(N1.getValue(0)) &&
00622       N.Val->isOnlyUse(N1.Val) &&
00623       (N.getNumOperands() == 1 || !isNonImmUse(N.Val, N1.Val)) &&
00624       (Chain1 = UpdateFoldedChain(CurDAG, N1.Val, Chain.Val, OldTF)).Val) {
00625     N11 = N1.getOperand(1);
00626     bool HasOptInFlag = false;
00627     if (SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
00628       Select(Tmp0, Tmp0);
00629       Select(Tmp1, Tmp1);
00630       Select(Tmp2, Tmp2);
00631       Select(Tmp3, Tmp3);
00632       Select(Chain1, Chain1);
00633       if (N.getNumOperands() == 3) {
00634         Select(InFlag, N.getOperand(2));
00635         HasOptInFlag = true;
00636       }
00637       if (HasOptInFlag)
00638         ResNode = CurDAG->getTargetNode(X86::CALL32m, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
00639       else
00640         ResNode = CurDAG->getTargetNode(X86::CALL32m, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
00641       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain1.Val, 0);
00642       InFlag = SDOperand(ResNode, 1);
00643       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00644       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
00645       AddHandleReplacement(N1.Val, 1, ResNode, 0);
00646       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00647       Result = SDOperand(ResNode, N.ResNo);
00648       return;
00649     }
00650   }
00651 
00652   // Pattern: (X86call:void (tglobaladdr:i32):$dst)
00653   // Emits: (CALLpcrel32:void (tglobaladdr:i32):$dst)
00654   // Pattern complexity = 4  cost = 1
00655   if (N1.getOpcode() == ISD::TargetGlobalAddress) {
00656     bool HasOptInFlag = false;
00657     Tmp0 = N1;
00658     Select(Chain, Chain);
00659     if (N.getNumOperands() == 3) {
00660       Select(InFlag, N.getOperand(2));
00661       HasOptInFlag = true;
00662     }
00663     if (HasOptInFlag)
00664       ResNode = CurDAG->getTargetNode(X86::CALLpcrel32, MVT::Other, MVT::Flag, Tmp0, Chain, InFlag);
00665     else
00666       ResNode = CurDAG->getTargetNode(X86::CALLpcrel32, MVT::Other, MVT::Flag, Tmp0, Chain);
00667     InFlag = SDOperand(ResNode, 1);
00668     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00669     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00670     Result = SDOperand(ResNode, N.ResNo);
00671     return;
00672   }
00673 
00674   // Pattern: (X86call:void (texternalsym:i32):$dst)
00675   // Emits: (CALLpcrel32:void (texternalsym:i32):$dst)
00676   // Pattern complexity = 4  cost = 1
00677   if (N1.getOpcode() == ISD::TargetExternalSymbol) {
00678     bool HasOptInFlag = false;
00679     Tmp0 = N1;
00680     Select(Chain, Chain);
00681     if (N.getNumOperands() == 3) {
00682       Select(InFlag, N.getOperand(2));
00683       HasOptInFlag = true;
00684     }
00685     if (HasOptInFlag)
00686       ResNode = CurDAG->getTargetNode(X86::CALLpcrel32, MVT::Other, MVT::Flag, Tmp0, Chain, InFlag);
00687     else
00688       ResNode = CurDAG->getTargetNode(X86::CALLpcrel32, MVT::Other, MVT::Flag, Tmp0, Chain);
00689     InFlag = SDOperand(ResNode, 1);
00690     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00691     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00692     Result = SDOperand(ResNode, N.ResNo);
00693     return;
00694   }
00695 
00696   // Pattern: (X86call:void R32:i32:$dst)
00697   // Emits: (CALL32r:void R32:i32:$dst)
00698   // Pattern complexity = 2  cost = 1
00699   bool HasOptInFlag = false;
00700   Select(Tmp0, N1);
00701   Select(Chain, Chain);
00702   if (N.getNumOperands() == 3) {
00703     Select(InFlag, N.getOperand(2));
00704     HasOptInFlag = true;
00705   }
00706   if (HasOptInFlag)
00707     ResNode = CurDAG->getTargetNode(X86::CALL32r, MVT::Other, MVT::Flag, Tmp0, Chain, InFlag);
00708   else
00709     ResNode = CurDAG->getTargetNode(X86::CALL32r, MVT::Other, MVT::Flag, Tmp0, Chain);
00710   InFlag = SDOperand(ResNode, 1);
00711   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00712   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00713   Result = SDOperand(ResNode, N.ResNo);
00714   return;
00715 }
00716 
00717 void Select_X86callseq_end(SDOperand &Result, SDOperand N) {
00718   SDOperand Chain(0, 0);
00719   SDOperand InFlag(0, 0);
00720   SDOperand N1(0, 0);
00721   SDOperand N2(0, 0);
00722   SDOperand Tmp0(0, 0);
00723   SDOperand Tmp1(0, 0);
00724   SDNode *ResNode;
00725   Chain = N.getOperand(0);
00726   N1 = N.getOperand(1);
00727   if (N1.getOpcode() == ISD::Constant) {
00728     N2 = N.getOperand(2);
00729     if (N2.getOpcode() == ISD::Constant) {
00730       unsigned Tmp0C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
00731       Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i32);
00732       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
00733       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
00734       Select(Chain, Chain);
00735       Select(InFlag, N.getOperand(3));
00736       ResNode = CurDAG->getTargetNode(X86::ADJCALLSTACKUP, MVT::Other, MVT::Flag, Tmp0, Tmp1, Chain, InFlag);
00737       InFlag = SDOperand(ResNode, 1);
00738       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00739       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00740       Result = SDOperand(ResNode, N.ResNo);
00741       return;
00742     }
00743   }
00744   std::cerr << "Cannot yet select: ";
00745   N.Val->dump(CurDAG);
00746   std::cerr << '\n';
00747   abort();
00748 }
00749 
00750 void Select_X86callseq_start(SDOperand &Result, SDOperand N) {
00751   SDOperand Chain(0, 0);
00752   SDOperand N1(0, 0);
00753   SDOperand Tmp0(0, 0);
00754   SDNode *ResNode;
00755   Chain = N.getOperand(0);
00756   N1 = N.getOperand(1);
00757   if (N1.getOpcode() == ISD::Constant) {
00758     unsigned Tmp0C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
00759     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i32);
00760     Select(Chain, Chain);
00761     ResNode = CurDAG->getTargetNode(X86::ADJCALLSTACKDOWN, MVT::Other, Tmp0, Chain);
00762     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00763     Result = SDOperand(ResNode, N.ResNo);
00764     return;
00765   }
00766   std::cerr << "Cannot yet select: ";
00767   N.Val->dump(CurDAG);
00768   std::cerr << '\n';
00769   abort();
00770 }
00771 
00772 void Select_X86cmov(SDOperand &Result, SDOperand N) {
00773   SDOperand Chain1(0, 0);
00774   SDOperand InFlag(0, 0);
00775   SDOperand N0(0, 0);
00776   SDOperand N1(0, 0);
00777   SDOperand N11(0, 0);
00778   SDOperand N2(0, 0);
00779   SDOperand Tmp0(0, 0);
00780   SDOperand Tmp1(0, 0);
00781   SDOperand Tmp2(0, 0);
00782   SDOperand Tmp3(0, 0);
00783   SDOperand Tmp4(0, 0);
00784   SDOperand Tmp5(0, 0);
00785   SDNode *ResNode;
00786   {
00787     N0 = N.getOperand(0);
00788     N1 = N.getOperand(1);
00789     if (N1.getOpcode() == ISD::LOAD &&
00790         N1.hasOneUse() &&
00791         !CodeGenMap.count(N1.getValue(0)) &&
00792         N.Val->isOnlyUse(N1.Val) &&
00793         !isNonImmUse(N.Val, N1.Val)) {
00794       Chain1 = N1.getOperand(0);
00795       N11 = N1.getOperand(1);
00796       N2 = N.getOperand(2);
00797       if (isa<ConstantSDNode>(N2)) {
00798         int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
00799         if (CN0 == 2) {
00800 
00801           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 2:i8)
00802           // Emits: (CMOVB16rm:i16 R16:i16:$src1, addr:i32:$src2)
00803           // Pattern complexity = 17  cost = 1
00804           if (N.Val->getValueType(0) == MVT::i16 &&
00805               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00806             Select(Tmp1, Tmp1);
00807             Select(Tmp2, Tmp2);
00808             Select(Tmp3, Tmp3);
00809             Select(Tmp4, Tmp4);
00810             Select(Tmp5, N0);
00811             Select(Chain1, Chain1);
00812             Select(InFlag, N.getOperand(3));
00813             ResNode = CurDAG->getTargetNode(X86::CMOVB16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00814             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00815             InFlag = SDOperand(ResNode, 2);
00816             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00817             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00818             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00819             if (N.ResNo < 1)
00820               Result = SDOperand(ResNode, N.ResNo);
00821             else
00822               Result = SDOperand(ResNode, N.ResNo+1);
00823             return;
00824           }
00825 
00826           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 2:i8)
00827           // Emits: (CMOVB32rm:i32 R32:i32:$src1, addr:i32:$src2)
00828           // Pattern complexity = 17  cost = 1
00829           if (N.Val->getValueType(0) == MVT::i32 &&
00830               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00831             Select(Tmp1, Tmp1);
00832             Select(Tmp2, Tmp2);
00833             Select(Tmp3, Tmp3);
00834             Select(Tmp4, Tmp4);
00835             Select(Tmp5, N0);
00836             Select(Chain1, Chain1);
00837             Select(InFlag, N.getOperand(3));
00838             ResNode = CurDAG->getTargetNode(X86::CMOVB32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00839             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00840             InFlag = SDOperand(ResNode, 2);
00841             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00842             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00843             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00844             if (N.ResNo < 1)
00845               Result = SDOperand(ResNode, N.ResNo);
00846             else
00847               Result = SDOperand(ResNode, N.ResNo+1);
00848             return;
00849           }
00850         }
00851         if (CN0 == 1) {
00852 
00853           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 1:i8)
00854           // Emits: (CMOVAE16rm:i16 R16:i16:$src1, addr:i32:$src2)
00855           // Pattern complexity = 17  cost = 1
00856           if (N.Val->getValueType(0) == MVT::i16 &&
00857               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00858             Select(Tmp1, Tmp1);
00859             Select(Tmp2, Tmp2);
00860             Select(Tmp3, Tmp3);
00861             Select(Tmp4, Tmp4);
00862             Select(Tmp5, N0);
00863             Select(Chain1, Chain1);
00864             Select(InFlag, N.getOperand(3));
00865             ResNode = CurDAG->getTargetNode(X86::CMOVAE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00866             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00867             InFlag = SDOperand(ResNode, 2);
00868             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00869             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00870             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00871             if (N.ResNo < 1)
00872               Result = SDOperand(ResNode, N.ResNo);
00873             else
00874               Result = SDOperand(ResNode, N.ResNo+1);
00875             return;
00876           }
00877 
00878           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 1:i8)
00879           // Emits: (CMOVAE32rm:i32 R32:i32:$src1, addr:i32:$src2)
00880           // Pattern complexity = 17  cost = 1
00881           if (N.Val->getValueType(0) == MVT::i32 &&
00882               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00883             Select(Tmp1, Tmp1);
00884             Select(Tmp2, Tmp2);
00885             Select(Tmp3, Tmp3);
00886             Select(Tmp4, Tmp4);
00887             Select(Tmp5, N0);
00888             Select(Chain1, Chain1);
00889             Select(InFlag, N.getOperand(3));
00890             ResNode = CurDAG->getTargetNode(X86::CMOVAE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00891             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00892             InFlag = SDOperand(ResNode, 2);
00893             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00894             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00895             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00896             if (N.ResNo < 1)
00897               Result = SDOperand(ResNode, N.ResNo);
00898             else
00899               Result = SDOperand(ResNode, N.ResNo+1);
00900             return;
00901           }
00902         }
00903         if (CN0 == 4) {
00904 
00905           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 4:i8)
00906           // Emits: (CMOVE16rm:i16 R16:i16:$src1, addr:i32:$src2)
00907           // Pattern complexity = 17  cost = 1
00908           if (N.Val->getValueType(0) == MVT::i16 &&
00909               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00910             Select(Tmp1, Tmp1);
00911             Select(Tmp2, Tmp2);
00912             Select(Tmp3, Tmp3);
00913             Select(Tmp4, Tmp4);
00914             Select(Tmp5, N0);
00915             Select(Chain1, Chain1);
00916             Select(InFlag, N.getOperand(3));
00917             ResNode = CurDAG->getTargetNode(X86::CMOVE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00918             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00919             InFlag = SDOperand(ResNode, 2);
00920             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00921             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00922             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00923             if (N.ResNo < 1)
00924               Result = SDOperand(ResNode, N.ResNo);
00925             else
00926               Result = SDOperand(ResNode, N.ResNo+1);
00927             return;
00928           }
00929 
00930           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 4:i8)
00931           // Emits: (CMOVE32rm:i32 R32:i32:$src1, addr:i32:$src2)
00932           // Pattern complexity = 17  cost = 1
00933           if (N.Val->getValueType(0) == MVT::i32 &&
00934               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00935             Select(Tmp1, Tmp1);
00936             Select(Tmp2, Tmp2);
00937             Select(Tmp3, Tmp3);
00938             Select(Tmp4, Tmp4);
00939             Select(Tmp5, N0);
00940             Select(Chain1, Chain1);
00941             Select(InFlag, N.getOperand(3));
00942             ResNode = CurDAG->getTargetNode(X86::CMOVE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00943             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00944             InFlag = SDOperand(ResNode, 2);
00945             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00946             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00947             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00948             if (N.ResNo < 1)
00949               Result = SDOperand(ResNode, N.ResNo);
00950             else
00951               Result = SDOperand(ResNode, N.ResNo+1);
00952             return;
00953           }
00954         }
00955         if (CN0 == 9) {
00956 
00957           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 9:i8)
00958           // Emits: (CMOVNE16rm:i16 R16:i16:$src1, addr:i32:$src2)
00959           // Pattern complexity = 17  cost = 1
00960           if (N.Val->getValueType(0) == MVT::i16 &&
00961               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00962             Select(Tmp1, Tmp1);
00963             Select(Tmp2, Tmp2);
00964             Select(Tmp3, Tmp3);
00965             Select(Tmp4, Tmp4);
00966             Select(Tmp5, N0);
00967             Select(Chain1, Chain1);
00968             Select(InFlag, N.getOperand(3));
00969             ResNode = CurDAG->getTargetNode(X86::CMOVNE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00970             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00971             InFlag = SDOperand(ResNode, 2);
00972             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00973             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00974             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
00975             if (N.ResNo < 1)
00976               Result = SDOperand(ResNode, N.ResNo);
00977             else
00978               Result = SDOperand(ResNode, N.ResNo+1);
00979             return;
00980           }
00981 
00982           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 9:i8)
00983           // Emits: (CMOVNE32rm:i32 R32:i32:$src1, addr:i32:$src2)
00984           // Pattern complexity = 17  cost = 1
00985           if (N.Val->getValueType(0) == MVT::i32 &&
00986               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
00987             Select(Tmp1, Tmp1);
00988             Select(Tmp2, Tmp2);
00989             Select(Tmp3, Tmp3);
00990             Select(Tmp4, Tmp4);
00991             Select(Tmp5, N0);
00992             Select(Chain1, Chain1);
00993             Select(InFlag, N.getOperand(3));
00994             ResNode = CurDAG->getTargetNode(X86::CMOVNE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
00995             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
00996             InFlag = SDOperand(ResNode, 2);
00997             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
00998             AddHandleReplacement(N1.Val, 1, ResNode, 1);
00999             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01000             if (N.ResNo < 1)
01001               Result = SDOperand(ResNode, N.ResNo);
01002             else
01003               Result = SDOperand(ResNode, N.ResNo+1);
01004             return;
01005           }
01006         }
01007         if (CN0 == 3) {
01008 
01009           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 3:i8)
01010           // Emits: (CMOVBE16rm:i16 R16:i16:$src1, addr:i32:$src2)
01011           // Pattern complexity = 17  cost = 1
01012           if (N.Val->getValueType(0) == MVT::i16 &&
01013               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01014             Select(Tmp1, Tmp1);
01015             Select(Tmp2, Tmp2);
01016             Select(Tmp3, Tmp3);
01017             Select(Tmp4, Tmp4);
01018             Select(Tmp5, N0);
01019             Select(Chain1, Chain1);
01020             Select(InFlag, N.getOperand(3));
01021             ResNode = CurDAG->getTargetNode(X86::CMOVBE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01022             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01023             InFlag = SDOperand(ResNode, 2);
01024             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01025             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01026             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01027             if (N.ResNo < 1)
01028               Result = SDOperand(ResNode, N.ResNo);
01029             else
01030               Result = SDOperand(ResNode, N.ResNo+1);
01031             return;
01032           }
01033 
01034           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 3:i8)
01035           // Emits: (CMOVBE32rm:i32 R32:i32:$src1, addr:i32:$src2)
01036           // Pattern complexity = 17  cost = 1
01037           if (N.Val->getValueType(0) == MVT::i32 &&
01038               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01039             Select(Tmp1, Tmp1);
01040             Select(Tmp2, Tmp2);
01041             Select(Tmp3, Tmp3);
01042             Select(Tmp4, Tmp4);
01043             Select(Tmp5, N0);
01044             Select(Chain1, Chain1);
01045             Select(InFlag, N.getOperand(3));
01046             ResNode = CurDAG->getTargetNode(X86::CMOVBE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01047             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01048             InFlag = SDOperand(ResNode, 2);
01049             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01050             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01051             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01052             if (N.ResNo < 1)
01053               Result = SDOperand(ResNode, N.ResNo);
01054             else
01055               Result = SDOperand(ResNode, N.ResNo+1);
01056             return;
01057           }
01058         }
01059         if (CN0 == 0) {
01060 
01061           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 0:i8)
01062           // Emits: (CMOVA16rm:i16 R16:i16:$src1, addr:i32:$src2)
01063           // Pattern complexity = 17  cost = 1
01064           if (N.Val->getValueType(0) == MVT::i16 &&
01065               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01066             Select(Tmp1, Tmp1);
01067             Select(Tmp2, Tmp2);
01068             Select(Tmp3, Tmp3);
01069             Select(Tmp4, Tmp4);
01070             Select(Tmp5, N0);
01071             Select(Chain1, Chain1);
01072             Select(InFlag, N.getOperand(3));
01073             ResNode = CurDAG->getTargetNode(X86::CMOVA16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01074             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01075             InFlag = SDOperand(ResNode, 2);
01076             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01077             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01078             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01079             if (N.ResNo < 1)
01080               Result = SDOperand(ResNode, N.ResNo);
01081             else
01082               Result = SDOperand(ResNode, N.ResNo+1);
01083             return;
01084           }
01085 
01086           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 0:i8)
01087           // Emits: (CMOVA32rm:i32 R32:i32:$src1, addr:i32:$src2)
01088           // Pattern complexity = 17  cost = 1
01089           if (N.Val->getValueType(0) == MVT::i32 &&
01090               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01091             Select(Tmp1, Tmp1);
01092             Select(Tmp2, Tmp2);
01093             Select(Tmp3, Tmp3);
01094             Select(Tmp4, Tmp4);
01095             Select(Tmp5, N0);
01096             Select(Chain1, Chain1);
01097             Select(InFlag, N.getOperand(3));
01098             ResNode = CurDAG->getTargetNode(X86::CMOVA32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01099             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01100             InFlag = SDOperand(ResNode, 2);
01101             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01102             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01103             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01104             if (N.ResNo < 1)
01105               Result = SDOperand(ResNode, N.ResNo);
01106             else
01107               Result = SDOperand(ResNode, N.ResNo+1);
01108             return;
01109           }
01110         }
01111         if (CN0 == 7) {
01112 
01113           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 7:i8)
01114           // Emits: (CMOVL16rm:i16 R16:i16:$src1, addr:i32:$src2)
01115           // Pattern complexity = 17  cost = 1
01116           if (N.Val->getValueType(0) == MVT::i16 &&
01117               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01118             Select(Tmp1, Tmp1);
01119             Select(Tmp2, Tmp2);
01120             Select(Tmp3, Tmp3);
01121             Select(Tmp4, Tmp4);
01122             Select(Tmp5, N0);
01123             Select(Chain1, Chain1);
01124             Select(InFlag, N.getOperand(3));
01125             ResNode = CurDAG->getTargetNode(X86::CMOVL16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01126             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01127             InFlag = SDOperand(ResNode, 2);
01128             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01129             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01130             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01131             if (N.ResNo < 1)
01132               Result = SDOperand(ResNode, N.ResNo);
01133             else
01134               Result = SDOperand(ResNode, N.ResNo+1);
01135             return;
01136           }
01137 
01138           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 7:i8)
01139           // Emits: (CMOVL32rm:i32 R32:i32:$src1, addr:i32:$src2)
01140           // Pattern complexity = 17  cost = 1
01141           if (N.Val->getValueType(0) == MVT::i32 &&
01142               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01143             Select(Tmp1, Tmp1);
01144             Select(Tmp2, Tmp2);
01145             Select(Tmp3, Tmp3);
01146             Select(Tmp4, Tmp4);
01147             Select(Tmp5, N0);
01148             Select(Chain1, Chain1);
01149             Select(InFlag, N.getOperand(3));
01150             ResNode = CurDAG->getTargetNode(X86::CMOVL32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01151             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01152             InFlag = SDOperand(ResNode, 2);
01153             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01154             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01155             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01156             if (N.ResNo < 1)
01157               Result = SDOperand(ResNode, N.ResNo);
01158             else
01159               Result = SDOperand(ResNode, N.ResNo+1);
01160             return;
01161           }
01162         }
01163         if (CN0 == 6) {
01164 
01165           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 6:i8)
01166           // Emits: (CMOVGE16rm:i16 R16:i16:$src1, addr:i32:$src2)
01167           // Pattern complexity = 17  cost = 1
01168           if (N.Val->getValueType(0) == MVT::i16 &&
01169               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01170             Select(Tmp1, Tmp1);
01171             Select(Tmp2, Tmp2);
01172             Select(Tmp3, Tmp3);
01173             Select(Tmp4, Tmp4);
01174             Select(Tmp5, N0);
01175             Select(Chain1, Chain1);
01176             Select(InFlag, N.getOperand(3));
01177             ResNode = CurDAG->getTargetNode(X86::CMOVGE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01178             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01179             InFlag = SDOperand(ResNode, 2);
01180             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01181             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01182             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01183             if (N.ResNo < 1)
01184               Result = SDOperand(ResNode, N.ResNo);
01185             else
01186               Result = SDOperand(ResNode, N.ResNo+1);
01187             return;
01188           }
01189 
01190           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 6:i8)
01191           // Emits: (CMOVGE32rm:i32 R32:i32:$src1, addr:i32:$src2)
01192           // Pattern complexity = 17  cost = 1
01193           if (N.Val->getValueType(0) == MVT::i32 &&
01194               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01195             Select(Tmp1, Tmp1);
01196             Select(Tmp2, Tmp2);
01197             Select(Tmp3, Tmp3);
01198             Select(Tmp4, Tmp4);
01199             Select(Tmp5, N0);
01200             Select(Chain1, Chain1);
01201             Select(InFlag, N.getOperand(3));
01202             ResNode = CurDAG->getTargetNode(X86::CMOVGE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01203             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01204             InFlag = SDOperand(ResNode, 2);
01205             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01206             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01207             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01208             if (N.ResNo < 1)
01209               Result = SDOperand(ResNode, N.ResNo);
01210             else
01211               Result = SDOperand(ResNode, N.ResNo+1);
01212             return;
01213           }
01214         }
01215         if (CN0 == 8) {
01216 
01217           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 8:i8)
01218           // Emits: (CMOVLE16rm:i16 R16:i16:$src1, addr:i32:$src2)
01219           // Pattern complexity = 17  cost = 1
01220           if (N.Val->getValueType(0) == MVT::i16 &&
01221               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01222             Select(Tmp1, Tmp1);
01223             Select(Tmp2, Tmp2);
01224             Select(Tmp3, Tmp3);
01225             Select(Tmp4, Tmp4);
01226             Select(Tmp5, N0);
01227             Select(Chain1, Chain1);
01228             Select(InFlag, N.getOperand(3));
01229             ResNode = CurDAG->getTargetNode(X86::CMOVLE16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01230             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01231             InFlag = SDOperand(ResNode, 2);
01232             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01233             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01234             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01235             if (N.ResNo < 1)
01236               Result = SDOperand(ResNode, N.ResNo);
01237             else
01238               Result = SDOperand(ResNode, N.ResNo+1);
01239             return;
01240           }
01241 
01242           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 8:i8)
01243           // Emits: (CMOVLE32rm:i32 R32:i32:$src1, addr:i32:$src2)
01244           // Pattern complexity = 17  cost = 1
01245           if (N.Val->getValueType(0) == MVT::i32 &&
01246               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01247             Select(Tmp1, Tmp1);
01248             Select(Tmp2, Tmp2);
01249             Select(Tmp3, Tmp3);
01250             Select(Tmp4, Tmp4);
01251             Select(Tmp5, N0);
01252             Select(Chain1, Chain1);
01253             Select(InFlag, N.getOperand(3));
01254             ResNode = CurDAG->getTargetNode(X86::CMOVLE32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01255             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01256             InFlag = SDOperand(ResNode, 2);
01257             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01258             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01259             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01260             if (N.ResNo < 1)
01261               Result = SDOperand(ResNode, N.ResNo);
01262             else
01263               Result = SDOperand(ResNode, N.ResNo+1);
01264             return;
01265           }
01266         }
01267         if (CN0 == 5) {
01268 
01269           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 5:i8)
01270           // Emits: (CMOVG16rm:i16 R16:i16:$src1, addr:i32:$src2)
01271           // Pattern complexity = 17  cost = 1
01272           if (N.Val->getValueType(0) == MVT::i16 &&
01273               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01274             Select(Tmp1, Tmp1);
01275             Select(Tmp2, Tmp2);
01276             Select(Tmp3, Tmp3);
01277             Select(Tmp4, Tmp4);
01278             Select(Tmp5, N0);
01279             Select(Chain1, Chain1);
01280             Select(InFlag, N.getOperand(3));
01281             ResNode = CurDAG->getTargetNode(X86::CMOVG16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01282             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01283             InFlag = SDOperand(ResNode, 2);
01284             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01285             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01286             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01287             if (N.ResNo < 1)
01288               Result = SDOperand(ResNode, N.ResNo);
01289             else
01290               Result = SDOperand(ResNode, N.ResNo+1);
01291             return;
01292           }
01293 
01294           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 5:i8)
01295           // Emits: (CMOVG32rm:i32 R32:i32:$src1, addr:i32:$src2)
01296           // Pattern complexity = 17  cost = 1
01297           if (N.Val->getValueType(0) == MVT::i32 &&
01298               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01299             Select(Tmp1, Tmp1);
01300             Select(Tmp2, Tmp2);
01301             Select(Tmp3, Tmp3);
01302             Select(Tmp4, Tmp4);
01303             Select(Tmp5, N0);
01304             Select(Chain1, Chain1);
01305             Select(InFlag, N.getOperand(3));
01306             ResNode = CurDAG->getTargetNode(X86::CMOVG32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01307             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01308             InFlag = SDOperand(ResNode, 2);
01309             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01310             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01311             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01312             if (N.ResNo < 1)
01313               Result = SDOperand(ResNode, N.ResNo);
01314             else
01315               Result = SDOperand(ResNode, N.ResNo+1);
01316             return;
01317           }
01318         }
01319         if (CN0 == 15) {
01320 
01321           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 15:i8)
01322           // Emits: (CMOVS16rm:i16 R16:i16:$src1, addr:i32:$src2)
01323           // Pattern complexity = 17  cost = 1
01324           if (N.Val->getValueType(0) == MVT::i16 &&
01325               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01326             Select(Tmp1, Tmp1);
01327             Select(Tmp2, Tmp2);
01328             Select(Tmp3, Tmp3);
01329             Select(Tmp4, Tmp4);
01330             Select(Tmp5, N0);
01331             Select(Chain1, Chain1);
01332             Select(InFlag, N.getOperand(3));
01333             ResNode = CurDAG->getTargetNode(X86::CMOVS16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01334             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01335             InFlag = SDOperand(ResNode, 2);
01336             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01337             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01338             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01339             if (N.ResNo < 1)
01340               Result = SDOperand(ResNode, N.ResNo);
01341             else
01342               Result = SDOperand(ResNode, N.ResNo+1);
01343             return;
01344           }
01345 
01346           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 15:i8)
01347           // Emits: (CMOVS32rm:i32 R32:i32:$src1, addr:i32:$src2)
01348           // Pattern complexity = 17  cost = 1
01349           if (N.Val->getValueType(0) == MVT::i32 &&
01350               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01351             Select(Tmp1, Tmp1);
01352             Select(Tmp2, Tmp2);
01353             Select(Tmp3, Tmp3);
01354             Select(Tmp4, Tmp4);
01355             Select(Tmp5, N0);
01356             Select(Chain1, Chain1);
01357             Select(InFlag, N.getOperand(3));
01358             ResNode = CurDAG->getTargetNode(X86::CMOVS32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01359             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01360             InFlag = SDOperand(ResNode, 2);
01361             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01362             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01363             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01364             if (N.ResNo < 1)
01365               Result = SDOperand(ResNode, N.ResNo);
01366             else
01367               Result = SDOperand(ResNode, N.ResNo+1);
01368             return;
01369           }
01370         }
01371         if (CN0 == 12) {
01372 
01373           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 12:i8)
01374           // Emits: (CMOVNS16rm:i16 R16:i16:$src1, addr:i32:$src2)
01375           // Pattern complexity = 17  cost = 1
01376           if (N.Val->getValueType(0) == MVT::i16 &&
01377               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01378             Select(Tmp1, Tmp1);
01379             Select(Tmp2, Tmp2);
01380             Select(Tmp3, Tmp3);
01381             Select(Tmp4, Tmp4);
01382             Select(Tmp5, N0);
01383             Select(Chain1, Chain1);
01384             Select(InFlag, N.getOperand(3));
01385             ResNode = CurDAG->getTargetNode(X86::CMOVNS16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01386             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01387             InFlag = SDOperand(ResNode, 2);
01388             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01389             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01390             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01391             if (N.ResNo < 1)
01392               Result = SDOperand(ResNode, N.ResNo);
01393             else
01394               Result = SDOperand(ResNode, N.ResNo+1);
01395             return;
01396           }
01397 
01398           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 12:i8)
01399           // Emits: (CMOVNS32rm:i32 R32:i32:$src1, addr:i32:$src2)
01400           // Pattern complexity = 17  cost = 1
01401           if (N.Val->getValueType(0) == MVT::i32 &&
01402               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01403             Select(Tmp1, Tmp1);
01404             Select(Tmp2, Tmp2);
01405             Select(Tmp3, Tmp3);
01406             Select(Tmp4, Tmp4);
01407             Select(Tmp5, N0);
01408             Select(Chain1, Chain1);
01409             Select(InFlag, N.getOperand(3));
01410             ResNode = CurDAG->getTargetNode(X86::CMOVNS32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01411             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01412             InFlag = SDOperand(ResNode, 2);
01413             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01414             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01415             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01416             if (N.ResNo < 1)
01417               Result = SDOperand(ResNode, N.ResNo);
01418             else
01419               Result = SDOperand(ResNode, N.ResNo+1);
01420             return;
01421           }
01422         }
01423         if (CN0 == 14) {
01424 
01425           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 14:i8)
01426           // Emits: (CMOVP16rm:i16 R16:i16:$src1, addr:i32:$src2)
01427           // Pattern complexity = 17  cost = 1
01428           if (N.Val->getValueType(0) == MVT::i16 &&
01429               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01430             Select(Tmp1, Tmp1);
01431             Select(Tmp2, Tmp2);
01432             Select(Tmp3, Tmp3);
01433             Select(Tmp4, Tmp4);
01434             Select(Tmp5, N0);
01435             Select(Chain1, Chain1);
01436             Select(InFlag, N.getOperand(3));
01437             ResNode = CurDAG->getTargetNode(X86::CMOVP16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01438             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01439             InFlag = SDOperand(ResNode, 2);
01440             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01441             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01442             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01443             if (N.ResNo < 1)
01444               Result = SDOperand(ResNode, N.ResNo);
01445             else
01446               Result = SDOperand(ResNode, N.ResNo+1);
01447             return;
01448           }
01449 
01450           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 14:i8)
01451           // Emits: (CMOVP32rm:i32 R32:i32:$src1, addr:i32:$src2)
01452           // Pattern complexity = 17  cost = 1
01453           if (N.Val->getValueType(0) == MVT::i32 &&
01454               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01455             Select(Tmp1, Tmp1);
01456             Select(Tmp2, Tmp2);
01457             Select(Tmp3, Tmp3);
01458             Select(Tmp4, Tmp4);
01459             Select(Tmp5, N0);
01460             Select(Chain1, Chain1);
01461             Select(InFlag, N.getOperand(3));
01462             ResNode = CurDAG->getTargetNode(X86::CMOVP32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01463             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01464             InFlag = SDOperand(ResNode, 2);
01465             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01466             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01467             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01468             if (N.ResNo < 1)
01469               Result = SDOperand(ResNode, N.ResNo);
01470             else
01471               Result = SDOperand(ResNode, N.ResNo+1);
01472             return;
01473           }
01474         }
01475         if (CN0 == 11) {
01476 
01477           // Pattern: (X86cmov:i16 R16:i16:$src1, (load:i16 addr:i32:$src2), 11:i8)
01478           // Emits: (CMOVNP16rm:i16 R16:i16:$src1, addr:i32:$src2)
01479           // Pattern complexity = 17  cost = 1
01480           if (N.Val->getValueType(0) == MVT::i16 &&
01481               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01482             Select(Tmp1, Tmp1);
01483             Select(Tmp2, Tmp2);
01484             Select(Tmp3, Tmp3);
01485             Select(Tmp4, Tmp4);
01486             Select(Tmp5, N0);
01487             Select(Chain1, Chain1);
01488             Select(InFlag, N.getOperand(3));
01489             ResNode = CurDAG->getTargetNode(X86::CMOVNP16rm, MVT::i16, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01490             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01491             InFlag = SDOperand(ResNode, 2);
01492             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01493             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01494             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01495             if (N.ResNo < 1)
01496               Result = SDOperand(ResNode, N.ResNo);
01497             else
01498               Result = SDOperand(ResNode, N.ResNo+1);
01499             return;
01500           }
01501 
01502           // Pattern: (X86cmov:i32 R32:i32:$src1, (load:i32 addr:i32:$src2), 11:i8)
01503           // Emits: (CMOVNP32rm:i32 R32:i32:$src1, addr:i32:$src2)
01504           // Pattern complexity = 17  cost = 1
01505           if (N.Val->getValueType(0) == MVT::i32 &&
01506               SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
01507             Select(Tmp1, Tmp1);
01508             Select(Tmp2, Tmp2);
01509             Select(Tmp3, Tmp3);
01510             Select(Tmp4, Tmp4);
01511             Select(Tmp5, N0);
01512             Select(Chain1, Chain1);
01513             Select(InFlag, N.getOperand(3));
01514             ResNode = CurDAG->getTargetNode(X86::CMOVNP32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain1, InFlag);
01515             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01516             InFlag = SDOperand(ResNode, 2);
01517             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
01518             AddHandleReplacement(N1.Val, 1, ResNode, 1);
01519             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01520             if (N.ResNo < 1)
01521               Result = SDOperand(ResNode, N.ResNo);
01522             else
01523               Result = SDOperand(ResNode, N.ResNo+1);
01524             return;
01525           }
01526         }
01527       }
01528     }
01529     N2 = N.getOperand(2);
01530     if (isa<ConstantSDNode>(N2)) {
01531       int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
01532       if (CN0 == 2) {
01533 
01534         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 2:i8)
01535         // Emits: (CMOVB16rr:i16 R16:i16:$src1, R16:i16:$src2)
01536         // Pattern complexity = 5  cost = 1
01537         if (N.Val->getValueType(0) == MVT::i16) {
01538           Select(Tmp1, N0);
01539           Select(Tmp2, N1);
01540           Select(InFlag, N.getOperand(3));
01541           ResNode = CurDAG->getTargetNode(X86::CMOVB16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01542           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01543           InFlag = SDOperand(ResNode, 1);
01544           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01545           Result = SDOperand(ResNode, N.ResNo);
01546           return;
01547         }
01548 
01549         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 2:i8)
01550         // Emits: (CMOVB32rr:i32 R32:i32:$src1, R32:i32:$src2)
01551         // Pattern complexity = 5  cost = 1
01552         if (N.Val->getValueType(0) == MVT::i32) {
01553           Select(Tmp1, N0);
01554           Select(Tmp2, N1);
01555           Select(InFlag, N.getOperand(3));
01556           ResNode = CurDAG->getTargetNode(X86::CMOVB32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01557           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01558           InFlag = SDOperand(ResNode, 1);
01559           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01560           Result = SDOperand(ResNode, N.ResNo);
01561           return;
01562         }
01563       }
01564       if (CN0 == 1) {
01565 
01566         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 1:i8)
01567         // Emits: (CMOVAE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01568         // Pattern complexity = 5  cost = 1
01569         if (N.Val->getValueType(0) == MVT::i16) {
01570           Select(Tmp1, N0);
01571           Select(Tmp2, N1);
01572           Select(InFlag, N.getOperand(3));
01573           ResNode = CurDAG->getTargetNode(X86::CMOVAE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01574           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01575           InFlag = SDOperand(ResNode, 1);
01576           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01577           Result = SDOperand(ResNode, N.ResNo);
01578           return;
01579         }
01580 
01581         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 1:i8)
01582         // Emits: (CMOVAE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01583         // Pattern complexity = 5  cost = 1
01584         if (N.Val->getValueType(0) == MVT::i32) {
01585           Select(Tmp1, N0);
01586           Select(Tmp2, N1);
01587           Select(InFlag, N.getOperand(3));
01588           ResNode = CurDAG->getTargetNode(X86::CMOVAE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01589           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01590           InFlag = SDOperand(ResNode, 1);
01591           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01592           Result = SDOperand(ResNode, N.ResNo);
01593           return;
01594         }
01595       }
01596       if (CN0 == 4) {
01597 
01598         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 4:i8)
01599         // Emits: (CMOVE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01600         // Pattern complexity = 5  cost = 1
01601         if (N.Val->getValueType(0) == MVT::i16) {
01602           Select(Tmp1, N0);
01603           Select(Tmp2, N1);
01604           Select(InFlag, N.getOperand(3));
01605           ResNode = CurDAG->getTargetNode(X86::CMOVE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01606           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01607           InFlag = SDOperand(ResNode, 1);
01608           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01609           Result = SDOperand(ResNode, N.ResNo);
01610           return;
01611         }
01612 
01613         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 4:i8)
01614         // Emits: (CMOVE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01615         // Pattern complexity = 5  cost = 1
01616         if (N.Val->getValueType(0) == MVT::i32) {
01617           Select(Tmp1, N0);
01618           Select(Tmp2, N1);
01619           Select(InFlag, N.getOperand(3));
01620           ResNode = CurDAG->getTargetNode(X86::CMOVE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01621           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01622           InFlag = SDOperand(ResNode, 1);
01623           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01624           Result = SDOperand(ResNode, N.ResNo);
01625           return;
01626         }
01627       }
01628       if (CN0 == 9) {
01629 
01630         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 9:i8)
01631         // Emits: (CMOVNE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01632         // Pattern complexity = 5  cost = 1
01633         if (N.Val->getValueType(0) == MVT::i16) {
01634           Select(Tmp1, N0);
01635           Select(Tmp2, N1);
01636           Select(InFlag, N.getOperand(3));
01637           ResNode = CurDAG->getTargetNode(X86::CMOVNE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01638           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01639           InFlag = SDOperand(ResNode, 1);
01640           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01641           Result = SDOperand(ResNode, N.ResNo);
01642           return;
01643         }
01644 
01645         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 9:i8)
01646         // Emits: (CMOVNE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01647         // Pattern complexity = 5  cost = 1
01648         if (N.Val->getValueType(0) == MVT::i32) {
01649           Select(Tmp1, N0);
01650           Select(Tmp2, N1);
01651           Select(InFlag, N.getOperand(3));
01652           ResNode = CurDAG->getTargetNode(X86::CMOVNE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01653           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01654           InFlag = SDOperand(ResNode, 1);
01655           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01656           Result = SDOperand(ResNode, N.ResNo);
01657           return;
01658         }
01659       }
01660       if (CN0 == 3) {
01661 
01662         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 3:i8)
01663         // Emits: (CMOVBE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01664         // Pattern complexity = 5  cost = 1
01665         if (N.Val->getValueType(0) == MVT::i16) {
01666           Select(Tmp1, N0);
01667           Select(Tmp2, N1);
01668           Select(InFlag, N.getOperand(3));
01669           ResNode = CurDAG->getTargetNode(X86::CMOVBE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01670           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01671           InFlag = SDOperand(ResNode, 1);
01672           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01673           Result = SDOperand(ResNode, N.ResNo);
01674           return;
01675         }
01676 
01677         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 3:i8)
01678         // Emits: (CMOVBE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01679         // Pattern complexity = 5  cost = 1
01680         if (N.Val->getValueType(0) == MVT::i32) {
01681           Select(Tmp1, N0);
01682           Select(Tmp2, N1);
01683           Select(InFlag, N.getOperand(3));
01684           ResNode = CurDAG->getTargetNode(X86::CMOVBE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01685           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01686           InFlag = SDOperand(ResNode, 1);
01687           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01688           Result = SDOperand(ResNode, N.ResNo);
01689           return;
01690         }
01691       }
01692       if (CN0 == 0) {
01693 
01694         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 0:i8)
01695         // Emits: (CMOVA16rr:i16 R16:i16:$src1, R16:i16:$src2)
01696         // Pattern complexity = 5  cost = 1
01697         if (N.Val->getValueType(0) == MVT::i16) {
01698           Select(Tmp1, N0);
01699           Select(Tmp2, N1);
01700           Select(InFlag, N.getOperand(3));
01701           ResNode = CurDAG->getTargetNode(X86::CMOVA16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01702           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01703           InFlag = SDOperand(ResNode, 1);
01704           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01705           Result = SDOperand(ResNode, N.ResNo);
01706           return;
01707         }
01708 
01709         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 0:i8)
01710         // Emits: (CMOVA32rr:i32 R32:i32:$src1, R32:i32:$src2)
01711         // Pattern complexity = 5  cost = 1
01712         if (N.Val->getValueType(0) == MVT::i32) {
01713           Select(Tmp1, N0);
01714           Select(Tmp2, N1);
01715           Select(InFlag, N.getOperand(3));
01716           ResNode = CurDAG->getTargetNode(X86::CMOVA32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01717           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01718           InFlag = SDOperand(ResNode, 1);
01719           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01720           Result = SDOperand(ResNode, N.ResNo);
01721           return;
01722         }
01723       }
01724       if (CN0 == 7) {
01725 
01726         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 7:i8)
01727         // Emits: (CMOVL16rr:i16 R16:i16:$src1, R16:i16:$src2)
01728         // Pattern complexity = 5  cost = 1
01729         if (N.Val->getValueType(0) == MVT::i16) {
01730           Select(Tmp1, N0);
01731           Select(Tmp2, N1);
01732           Select(InFlag, N.getOperand(3));
01733           ResNode = CurDAG->getTargetNode(X86::CMOVL16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01734           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01735           InFlag = SDOperand(ResNode, 1);
01736           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01737           Result = SDOperand(ResNode, N.ResNo);
01738           return;
01739         }
01740 
01741         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 7:i8)
01742         // Emits: (CMOVL32rr:i32 R32:i32:$src1, R32:i32:$src2)
01743         // Pattern complexity = 5  cost = 1
01744         if (N.Val->getValueType(0) == MVT::i32) {
01745           Select(Tmp1, N0);
01746           Select(Tmp2, N1);
01747           Select(InFlag, N.getOperand(3));
01748           ResNode = CurDAG->getTargetNode(X86::CMOVL32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01749           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01750           InFlag = SDOperand(ResNode, 1);
01751           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01752           Result = SDOperand(ResNode, N.ResNo);
01753           return;
01754         }
01755       }
01756       if (CN0 == 6) {
01757 
01758         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 6:i8)
01759         // Emits: (CMOVGE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01760         // Pattern complexity = 5  cost = 1
01761         if (N.Val->getValueType(0) == MVT::i16) {
01762           Select(Tmp1, N0);
01763           Select(Tmp2, N1);
01764           Select(InFlag, N.getOperand(3));
01765           ResNode = CurDAG->getTargetNode(X86::CMOVGE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01766           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01767           InFlag = SDOperand(ResNode, 1);
01768           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01769           Result = SDOperand(ResNode, N.ResNo);
01770           return;
01771         }
01772 
01773         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 6:i8)
01774         // Emits: (CMOVGE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01775         // Pattern complexity = 5  cost = 1
01776         if (N.Val->getValueType(0) == MVT::i32) {
01777           Select(Tmp1, N0);
01778           Select(Tmp2, N1);
01779           Select(InFlag, N.getOperand(3));
01780           ResNode = CurDAG->getTargetNode(X86::CMOVGE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01781           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01782           InFlag = SDOperand(ResNode, 1);
01783           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01784           Result = SDOperand(ResNode, N.ResNo);
01785           return;
01786         }
01787       }
01788       if (CN0 == 8) {
01789 
01790         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 8:i8)
01791         // Emits: (CMOVLE16rr:i16 R16:i16:$src1, R16:i16:$src2)
01792         // Pattern complexity = 5  cost = 1
01793         if (N.Val->getValueType(0) == MVT::i16) {
01794           Select(Tmp1, N0);
01795           Select(Tmp2, N1);
01796           Select(InFlag, N.getOperand(3));
01797           ResNode = CurDAG->getTargetNode(X86::CMOVLE16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01798           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01799           InFlag = SDOperand(ResNode, 1);
01800           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01801           Result = SDOperand(ResNode, N.ResNo);
01802           return;
01803         }
01804 
01805         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 8:i8)
01806         // Emits: (CMOVLE32rr:i32 R32:i32:$src1, R32:i32:$src2)
01807         // Pattern complexity = 5  cost = 1
01808         if (N.Val->getValueType(0) == MVT::i32) {
01809           Select(Tmp1, N0);
01810           Select(Tmp2, N1);
01811           Select(InFlag, N.getOperand(3));
01812           ResNode = CurDAG->getTargetNode(X86::CMOVLE32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01813           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01814           InFlag = SDOperand(ResNode, 1);
01815           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01816           Result = SDOperand(ResNode, N.ResNo);
01817           return;
01818         }
01819       }
01820       if (CN0 == 5) {
01821 
01822         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 5:i8)
01823         // Emits: (CMOVG16rr:i16 R16:i16:$src1, R16:i16:$src2)
01824         // Pattern complexity = 5  cost = 1
01825         if (N.Val->getValueType(0) == MVT::i16) {
01826           Select(Tmp1, N0);
01827           Select(Tmp2, N1);
01828           Select(InFlag, N.getOperand(3));
01829           ResNode = CurDAG->getTargetNode(X86::CMOVG16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01830           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01831           InFlag = SDOperand(ResNode, 1);
01832           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01833           Result = SDOperand(ResNode, N.ResNo);
01834           return;
01835         }
01836 
01837         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 5:i8)
01838         // Emits: (CMOVG32rr:i32 R32:i32:$src1, R32:i32:$src2)
01839         // Pattern complexity = 5  cost = 1
01840         if (N.Val->getValueType(0) == MVT::i32) {
01841           Select(Tmp1, N0);
01842           Select(Tmp2, N1);
01843           Select(InFlag, N.getOperand(3));
01844           ResNode = CurDAG->getTargetNode(X86::CMOVG32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01845           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01846           InFlag = SDOperand(ResNode, 1);
01847           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01848           Result = SDOperand(ResNode, N.ResNo);
01849           return;
01850         }
01851       }
01852       if (CN0 == 15) {
01853 
01854         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 15:i8)
01855         // Emits: (CMOVS16rr:i16 R16:i16:$src1, R16:i16:$src2)
01856         // Pattern complexity = 5  cost = 1
01857         if (N.Val->getValueType(0) == MVT::i16) {
01858           Select(Tmp1, N0);
01859           Select(Tmp2, N1);
01860           Select(InFlag, N.getOperand(3));
01861           ResNode = CurDAG->getTargetNode(X86::CMOVS16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01862           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01863           InFlag = SDOperand(ResNode, 1);
01864           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01865           Result = SDOperand(ResNode, N.ResNo);
01866           return;
01867         }
01868 
01869         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 15:i8)
01870         // Emits: (CMOVS32rr:i32 R32:i32:$src1, R32:i32:$src2)
01871         // Pattern complexity = 5  cost = 1
01872         if (N.Val->getValueType(0) == MVT::i32) {
01873           Select(Tmp1, N0);
01874           Select(Tmp2, N1);
01875           Select(InFlag, N.getOperand(3));
01876           ResNode = CurDAG->getTargetNode(X86::CMOVS32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01877           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01878           InFlag = SDOperand(ResNode, 1);
01879           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01880           Result = SDOperand(ResNode, N.ResNo);
01881           return;
01882         }
01883       }
01884       if (CN0 == 12) {
01885 
01886         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 12:i8)
01887         // Emits: (CMOVNS16rr:i16 R16:i16:$src1, R16:i16:$src2)
01888         // Pattern complexity = 5  cost = 1
01889         if (N.Val->getValueType(0) == MVT::i16) {
01890           Select(Tmp1, N0);
01891           Select(Tmp2, N1);
01892           Select(InFlag, N.getOperand(3));
01893           ResNode = CurDAG->getTargetNode(X86::CMOVNS16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01894           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01895           InFlag = SDOperand(ResNode, 1);
01896           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01897           Result = SDOperand(ResNode, N.ResNo);
01898           return;
01899         }
01900 
01901         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 12:i8)
01902         // Emits: (CMOVNS32rr:i32 R32:i32:$src1, R32:i32:$src2)
01903         // Pattern complexity = 5  cost = 1
01904         if (N.Val->getValueType(0) == MVT::i32) {
01905           Select(Tmp1, N0);
01906           Select(Tmp2, N1);
01907           Select(InFlag, N.getOperand(3));
01908           ResNode = CurDAG->getTargetNode(X86::CMOVNS32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01909           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01910           InFlag = SDOperand(ResNode, 1);
01911           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01912           Result = SDOperand(ResNode, N.ResNo);
01913           return;
01914         }
01915       }
01916       if (CN0 == 14) {
01917 
01918         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 14:i8)
01919         // Emits: (CMOVP16rr:i16 R16:i16:$src1, R16:i16:$src2)
01920         // Pattern complexity = 5  cost = 1
01921         if (N.Val->getValueType(0) == MVT::i16) {
01922           Select(Tmp1, N0);
01923           Select(Tmp2, N1);
01924           Select(InFlag, N.getOperand(3));
01925           ResNode = CurDAG->getTargetNode(X86::CMOVP16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01926           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01927           InFlag = SDOperand(ResNode, 1);
01928           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01929           Result = SDOperand(ResNode, N.ResNo);
01930           return;
01931         }
01932 
01933         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 14:i8)
01934         // Emits: (CMOVP32rr:i32 R32:i32:$src1, R32:i32:$src2)
01935         // Pattern complexity = 5  cost = 1
01936         if (N.Val->getValueType(0) == MVT::i32) {
01937           Select(Tmp1, N0);
01938           Select(Tmp2, N1);
01939           Select(InFlag, N.getOperand(3));
01940           ResNode = CurDAG->getTargetNode(X86::CMOVP32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01941           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01942           InFlag = SDOperand(ResNode, 1);
01943           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01944           Result = SDOperand(ResNode, N.ResNo);
01945           return;
01946         }
01947       }
01948       if (CN0 == 11) {
01949 
01950         // Pattern: (X86cmov:i16 R16:i16:$src1, R16:i16:$src2, 11:i8)
01951         // Emits: (CMOVNP16rr:i16 R16:i16:$src1, R16:i16:$src2)
01952         // Pattern complexity = 5  cost = 1
01953         if (N.Val->getValueType(0) == MVT::i16) {
01954           Select(Tmp1, N0);
01955           Select(Tmp2, N1);
01956           Select(InFlag, N.getOperand(3));
01957           ResNode = CurDAG->getTargetNode(X86::CMOVNP16rr, MVT::i16, MVT::Flag, Tmp1, Tmp2, InFlag);
01958           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01959           InFlag = SDOperand(ResNode, 1);
01960           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01961           Result = SDOperand(ResNode, N.ResNo);
01962           return;
01963         }
01964 
01965         // Pattern: (X86cmov:i32 R32:i32:$src1, R32:i32:$src2, 11:i8)
01966         // Emits: (CMOVNP32rr:i32 R32:i32:$src1, R32:i32:$src2)
01967         // Pattern complexity = 5  cost = 1
01968         if (N.Val->getValueType(0) == MVT::i32) {
01969           Select(Tmp1, N0);
01970           Select(Tmp2, N1);
01971           Select(InFlag, N.getOperand(3));
01972           ResNode = CurDAG->getTargetNode(X86::CMOVNP32rr, MVT::i32, MVT::Flag, Tmp1, Tmp2, InFlag);
01973           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01974           InFlag = SDOperand(ResNode, 1);
01975           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
01976           Result = SDOperand(ResNode, N.ResNo);
01977           return;
01978         }
01979       }
01980     }
01981   }
01982   if ((!Subtarget->hasSSE2())) {
01983     N0 = N.getOperand(0);
01984     N1 = N.getOperand(1);
01985     N2 = N.getOperand(2);
01986     if (isa<ConstantSDNode>(N2)) {
01987       int64_t CN0 = cast<ConstantSDNode>(N2)->getSignExtended();
01988 
01989       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 2:i8)
01990       // Emits: (FpCMOVB:f64 RFP:f64:$src1, RFP:f64:$src2)
01991       // Pattern complexity = 5  cost = 1
01992       if (CN0 == 2 &&
01993           N.Val->getValueType(0) == MVT::f64) {
01994         Select(Tmp1, N0);
01995         Select(Tmp2, N1);
01996         Select(InFlag, N.getOperand(3));
01997         ResNode = CurDAG->getTargetNode(X86::FpCMOVB, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
01998         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
01999         InFlag = SDOperand(ResNode, 1);
02000         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02001         Result = SDOperand(ResNode, N.ResNo);
02002         return;
02003       }
02004 
02005       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 3:i8)
02006       // Emits: (FpCMOVBE:f64 RFP:f64:$src1, RFP:f64:$src2)
02007       // Pattern complexity = 5  cost = 1
02008       if (CN0 == 3 &&
02009           N.Val->getValueType(0) == MVT::f64) {
02010         Select(Tmp1, N0);
02011         Select(Tmp2, N1);
02012         Select(InFlag, N.getOperand(3));
02013         ResNode = CurDAG->getTargetNode(X86::FpCMOVBE, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02014         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02015         InFlag = SDOperand(ResNode, 1);
02016         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02017         Result = SDOperand(ResNode, N.ResNo);
02018         return;
02019       }
02020 
02021       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 4:i8)
02022       // Emits: (FpCMOVE:f64 RFP:f64:$src1, RFP:f64:$src2)
02023       // Pattern complexity = 5  cost = 1
02024       if (CN0 == 4 &&
02025           N.Val->getValueType(0) == MVT::f64) {
02026         Select(Tmp1, N0);
02027         Select(Tmp2, N1);
02028         Select(InFlag, N.getOperand(3));
02029         ResNode = CurDAG->getTargetNode(X86::FpCMOVE, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02030         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02031         InFlag = SDOperand(ResNode, 1);
02032         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02033         Result = SDOperand(ResNode, N.ResNo);
02034         return;
02035       }
02036 
02037       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 14:i8)
02038       // Emits: (FpCMOVP:f64 RFP:f64:$src1, RFP:f64:$src2)
02039       // Pattern complexity = 5  cost = 1
02040       if (CN0 == 14 &&
02041           N.Val->getValueType(0) == MVT::f64) {
02042         Select(Tmp1, N0);
02043         Select(Tmp2, N1);
02044         Select(InFlag, N.getOperand(3));
02045         ResNode = CurDAG->getTargetNode(X86::FpCMOVP, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02046         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02047         InFlag = SDOperand(ResNode, 1);
02048         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02049         Result = SDOperand(ResNode, N.ResNo);
02050         return;
02051       }
02052 
02053       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 1:i8)
02054       // Emits: (FpCMOVNB:f64 RFP:f64:$src1, RFP:f64:$src2)
02055       // Pattern complexity = 5  cost = 1
02056       if (CN0 == 1 &&
02057           N.Val->getValueType(0) == MVT::f64) {
02058         Select(Tmp1, N0);
02059         Select(Tmp2, N1);
02060         Select(InFlag, N.getOperand(3));
02061         ResNode = CurDAG->getTargetNode(X86::FpCMOVNB, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02062         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02063         InFlag = SDOperand(ResNode, 1);
02064         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02065         Result = SDOperand(ResNode, N.ResNo);
02066         return;
02067       }
02068 
02069       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 0:i8)
02070       // Emits: (FpCMOVNBE:f64 RFP:f64:$src1, RFP:f64:$src2)
02071       // Pattern complexity = 5  cost = 1
02072       if (CN0 == 0 &&
02073           N.Val->getValueType(0) == MVT::f64) {
02074         Select(Tmp1, N0);
02075         Select(Tmp2, N1);
02076         Select(InFlag, N.getOperand(3));
02077         ResNode = CurDAG->getTargetNode(X86::FpCMOVNBE, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02078         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02079         InFlag = SDOperand(ResNode, 1);
02080         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02081         Result = SDOperand(ResNode, N.ResNo);
02082         return;
02083       }
02084 
02085       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 9:i8)
02086       // Emits: (FpCMOVNE:f64 RFP:f64:$src1, RFP:f64:$src2)
02087       // Pattern complexity = 5  cost = 1
02088       if (CN0 == 9 &&
02089           N.Val->getValueType(0) == MVT::f64) {
02090         Select(Tmp1, N0);
02091         Select(Tmp2, N1);
02092         Select(InFlag, N.getOperand(3));
02093         ResNode = CurDAG->getTargetNode(X86::FpCMOVNE, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02094         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02095         InFlag = SDOperand(ResNode, 1);
02096         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02097         Result = SDOperand(ResNode, N.ResNo);
02098         return;
02099       }
02100 
02101       // Pattern: (X86cmov:f64 RFP:f64:$src1, RFP:f64:$src2, 11:i8)
02102       // Emits: (FpCMOVNP:f64 RFP:f64:$src1, RFP:f64:$src2)
02103       // Pattern complexity = 5  cost = 1
02104       if (CN0 == 11 &&
02105           N.Val->getValueType(0) == MVT::f64) {
02106         Select(Tmp1, N0);
02107         Select(Tmp2, N1);
02108         Select(InFlag, N.getOperand(3));
02109         ResNode = CurDAG->getTargetNode(X86::FpCMOVNP, MVT::f64, MVT::Flag, Tmp1, Tmp2, InFlag);
02110         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02111         InFlag = SDOperand(ResNode, 1);
02112         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02113         Result = SDOperand(ResNode, N.ResNo);
02114         return;
02115       }
02116     }
02117   }
02118   N0 = N.getOperand(0);
02119   N1 = N.getOperand(1);
02120   N2 = N.getOperand(2);
02121   if (N2.getOpcode() == ISD::Constant) {
02122 
02123     // Pattern: (X86cmov:f32 FR32:f32:$t, FR32:f32:$f, (imm:i8):$cond)
02124     // Emits: (CMOV_FR32:f32 FR32:f32:$t, FR32:f32:$f, (imm:i8):$cond)
02125     // Pattern complexity = 4  cost = 11
02126     if (N.Val->getValueType(0) == MVT::f32) {
02127       Select(Tmp0, N0);
02128       Select(Tmp1, N1);
02129       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
02130       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
02131       Select(InFlag, N.getOperand(3));
02132       ResNode = CurDAG->getTargetNode(X86::CMOV_FR32, MVT::f32, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
02133       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02134       InFlag = SDOperand(ResNode, 1);
02135       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02136       Result = SDOperand(ResNode, N.ResNo);
02137       return;
02138     }
02139 
02140     // Pattern: (X86cmov:f64 FR64:f64:$t, FR64:f64:$f, (imm:i8):$cond)
02141     // Emits: (CMOV_FR64:f64 FR64:f64:$t, FR64:f64:$f, (imm:i8):$cond)
02142     // Pattern complexity = 4  cost = 11
02143     if (N.Val->getValueType(0) == MVT::f64) {
02144       Select(Tmp0, N0);
02145       Select(Tmp1, N1);
02146       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
02147       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
02148       Select(InFlag, N.getOperand(3));
02149       ResNode = CurDAG->getTargetNode(X86::CMOV_FR64, MVT::f64, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
02150       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02151       InFlag = SDOperand(ResNode, 1);
02152       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02153       Result = SDOperand(ResNode, N.ResNo);
02154       return;
02155     }
02156 
02157     // Pattern: (X86cmov:v4f32 VR128:v4f32:$t, VR128:v4f32:$f, (imm:i8):$cond)
02158     // Emits: (CMOV_V4F32:v4f32 VR128:v4f32:$t, VR128:v4f32:$f, (imm:i8):$cond)
02159     // Pattern complexity = 4  cost = 11
02160     if (N.Val->getValueType(0) == MVT::v4f32) {
02161       Select(Tmp0, N0);
02162       Select(Tmp1, N1);
02163       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
02164       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
02165       Select(InFlag, N.getOperand(3));
02166       ResNode = CurDAG->getTargetNode(X86::CMOV_V4F32, MVT::v4f32, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
02167       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02168       InFlag = SDOperand(ResNode, 1);
02169       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02170       Result = SDOperand(ResNode, N.ResNo);
02171       return;
02172     }
02173 
02174     // Pattern: (X86cmov:v2f64 VR128:v2f64:$t, VR128:v2f64:$f, (imm:i8):$cond)
02175     // Emits: (CMOV_V2F64:v2f64 VR128:v2f64:$t, VR128:v2f64:$f, (imm:i8):$cond)
02176     // Pattern complexity = 4  cost = 11
02177     if (N.Val->getValueType(0) == MVT::v2f64) {
02178       Select(Tmp0, N0);
02179       Select(Tmp1, N1);
02180       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
02181       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
02182       Select(InFlag, N.getOperand(3));
02183       ResNode = CurDAG->getTargetNode(X86::CMOV_V2F64, MVT::v2f64, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
02184       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02185       InFlag = SDOperand(ResNode, 1);
02186       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02187       Result = SDOperand(ResNode, N.ResNo);
02188       return;
02189     }
02190 
02191     // Pattern: (X86cmov:v2i64 VR128:v2i64:$t, VR128:v2i64:$f, (imm:i8):$cond)
02192     // Emits: (CMOV_V2I64:v2i64 VR128:v2i64:$t, VR128:v2i64:$f, (imm:i8):$cond)
02193     // Pattern complexity = 4  cost = 11
02194     if (N.Val->getValueType(0) == MVT::v2i64) {
02195       Select(Tmp0, N0);
02196       Select(Tmp1, N1);
02197       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
02198       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
02199       Select(InFlag, N.getOperand(3));
02200       ResNode = CurDAG->getTargetNode(X86::CMOV_V2I64, MVT::v2i64, MVT::Flag, Tmp0, Tmp1, Tmp2, InFlag);
02201       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02202       InFlag = SDOperand(ResNode, 1);
02203       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
02204       Result = SDOperand(ResNode, N.ResNo);
02205       return;
02206     }
02207   }
02208   std::cerr << "Cannot yet select: ";
02209   N.Val->dump(CurDAG);
02210   std::cerr << '\n';
02211   abort();
02212 }
02213 
02214 void Select_X86cmp(SDOperand &Result, SDOperand N) {
02215   SDOperand Chain0(0, 0);
02216   SDOperand Chain1(0, 0);
02217   SDOperand InFlag(0, 0);
02218   SDOperand N0(0, 0);
02219   SDOperand N01(0, 0);
02220   SDOperand N1(0, 0);
02221   SDOperand N11(0, 0);
02222   SDOperand Tmp0(0, 0);
02223   SDOperand Tmp1(0, 0);
02224   SDOperand Tmp2(0, 0);
02225   SDOperand Tmp3(0, 0);
02226   SDOperand Tmp4(0, 0);
02227   SDNode *ResNode;
02228   {
02229     N0 = N.getOperand(0);
02230     if (N0.getOpcode() == ISD::LOAD &&
02231         N0.hasOneUse() &&
02232         !CodeGenMap.count(N0.getValue(0)) &&
02233         N.Val->isOnlyUse(N0.Val) &&
02234         !isNonImmUse(N.Val, N0.Val)) {
02235       Chain0 = N0.getOperand(0);
02236       N01 = N0.getOperand(1);
02237       N1 = N.getOperand(1);
02238       if (N1.getOpcode() == ISD::Constant) {
02239 
02240         // Pattern: (X86cmp:void (load:i16 addr:i32:$src1), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
02241         // Emits: (CMP16mi8:void addr:i32:$src1, (imm:i16):$src2)
02242         // Pattern complexity = 17  cost = 1
02243         if (Predicate_i16immSExt8(N1.Val) &&
02244             N0.Val->getValueType(0) == MVT::i16 &&
02245             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02246           Select(Tmp0, Tmp0);
02247           Select(Tmp1, Tmp1);
02248           Select(Tmp2, Tmp2);
02249           Select(Tmp3, Tmp3);
02250           unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
02251           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
02252           Select(Chain0, Chain0);
02253           ResNode = CurDAG->getTargetNode(X86::CMP16mi8, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02254           InFlag = SDOperand(ResNode, 1);
02255           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02256           AddHandleReplacement(N0.Val, 1, ResNode, 0);
02257           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02258           Result = SDOperand(ResNode, N.ResNo+1);
02259           return;
02260         }
02261 
02262         // Pattern: (X86cmp:void (load:i32 addr:i32:$src1), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
02263         // Emits: (CMP32mi8:void addr:i32:$src1, (imm:i32):$src2)
02264         // Pattern complexity = 17  cost = 1
02265         if (Predicate_i32immSExt8(N1.Val) &&
02266             N0.Val->getValueType(0) == MVT::i32 &&
02267             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02268           Select(Tmp0, Tmp0);
02269           Select(Tmp1, Tmp1);
02270           Select(Tmp2, Tmp2);
02271           Select(Tmp3, Tmp3);
02272           unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
02273           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
02274           Select(Chain0, Chain0);
02275           ResNode = CurDAG->getTargetNode(X86::CMP32mi8, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02276           InFlag = SDOperand(ResNode, 1);
02277           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02278           AddHandleReplacement(N0.Val, 1, ResNode, 0);
02279           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02280           Result = SDOperand(ResNode, N.ResNo+1);
02281           return;
02282         }
02283 
02284         // Pattern: (X86cmp:void (load:i8 addr:i32:$src1), (imm:i8):$src2)
02285         // Emits: (CMP8mi:void addr:i32:$src1, (imm:i8):$src2)
02286         // Pattern complexity = 16  cost = 1
02287         if (N0.Val->getValueType(0) == MVT::i8 &&
02288             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02289           Select(Tmp0, Tmp0);
02290           Select(Tmp1, Tmp1);
02291           Select(Tmp2, Tmp2);
02292           Select(Tmp3, Tmp3);
02293           unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
02294           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
02295           Select(Chain0, Chain0);
02296           ResNode = CurDAG->getTargetNode(X86::CMP8mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02297           InFlag = SDOperand(ResNode, 1);
02298           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02299           AddHandleReplacement(N0.Val, 1, ResNode, 0);
02300           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02301           Result = SDOperand(ResNode, N.ResNo+1);
02302           return;
02303         }
02304 
02305         // Pattern: (X86cmp:void (load:i16 addr:i32:$src1), (imm:i16):$src2)
02306         // Emits: (CMP16mi:void addr:i32:$src1, (imm:i16):$src2)
02307         // Pattern complexity = 16  cost = 1
02308         if (N0.Val->getValueType(0) == MVT::i16 &&
02309             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02310           Select(Tmp0, Tmp0);
02311           Select(Tmp1, Tmp1);
02312           Select(Tmp2, Tmp2);
02313           Select(Tmp3, Tmp3);
02314           unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
02315           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
02316           Select(Chain0, Chain0);
02317           ResNode = CurDAG->getTargetNode(X86::CMP16mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02318           InFlag = SDOperand(ResNode, 1);
02319           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02320           AddHandleReplacement(N0.Val, 1, ResNode, 0);
02321           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02322           Result = SDOperand(ResNode, N.ResNo+1);
02323           return;
02324         }
02325 
02326         // Pattern: (X86cmp:void (load:i32 addr:i32:$src1), (imm:i32):$src2)
02327         // Emits: (CMP32mi:void addr:i32:$src1, (imm:i32):$src2)
02328         // Pattern complexity = 16  cost = 1
02329         if (N0.Val->getValueType(0) == MVT::i32 &&
02330             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02331           Select(Tmp0, Tmp0);
02332           Select(Tmp1, Tmp1);
02333           Select(Tmp2, Tmp2);
02334           Select(Tmp3, Tmp3);
02335           unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
02336           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
02337           Select(Chain0, Chain0);
02338           ResNode = CurDAG->getTargetNode(X86::CMP32mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02339           InFlag = SDOperand(ResNode, 1);
02340           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02341           AddHandleReplacement(N0.Val, 1, ResNode, 0);
02342           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02343           Result = SDOperand(ResNode, N.ResNo+1);
02344           return;
02345         }
02346       }
02347 
02348       // Pattern: (X86cmp:void (load:i8 addr:i32:$src1), R8:i8:$src2)
02349       // Emits: (CMP8mr:void addr:i32:$src1, R8:i8:$src2)
02350       // Pattern complexity = 14  cost = 1
02351       if (N0.Val->getValueType(0) == MVT::i8 &&
02352           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02353         Select(Tmp0, Tmp0);
02354         Select(Tmp1, Tmp1);
02355         Select(Tmp2, Tmp2);
02356         Select(Tmp3, Tmp3);
02357         Select(Tmp4, N1);
02358         Select(Chain0, Chain0);
02359         ResNode = CurDAG->getTargetNode(X86::CMP8mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02360         InFlag = SDOperand(ResNode, 1);
02361         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02362         AddHandleReplacement(N0.Val, 1, ResNode, 0);
02363         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02364         Result = SDOperand(ResNode, N.ResNo+1);
02365         return;
02366       }
02367 
02368       // Pattern: (X86cmp:void (load:i16 addr:i32:$src1), R16:i16:$src2)
02369       // Emits: (CMP16mr:void addr:i32:$src1, R16:i16:$src2)
02370       // Pattern complexity = 14  cost = 1
02371       if (N0.Val->getValueType(0) == MVT::i16 &&
02372           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02373         Select(Tmp0, Tmp0);
02374         Select(Tmp1, Tmp1);
02375         Select(Tmp2, Tmp2);
02376         Select(Tmp3, Tmp3);
02377         Select(Tmp4, N1);
02378         Select(Chain0, Chain0);
02379         ResNode = CurDAG->getTargetNode(X86::CMP16mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02380         InFlag = SDOperand(ResNode, 1);
02381         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02382         AddHandleReplacement(N0.Val, 1, ResNode, 0);
02383         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02384         Result = SDOperand(ResNode, N.ResNo+1);
02385         return;
02386       }
02387 
02388       // Pattern: (X86cmp:void (load:i32 addr:i32:$src1), R32:i32:$src2)
02389       // Emits: (CMP32mr:void addr:i32:$src1, R32:i32:$src2)
02390       // Pattern complexity = 14  cost = 1
02391       if (N0.Val->getValueType(0) == MVT::i32 &&
02392           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02393         Select(Tmp0, Tmp0);
02394         Select(Tmp1, Tmp1);
02395         Select(Tmp2, Tmp2);
02396         Select(Tmp3, Tmp3);
02397         Select(Tmp4, N1);
02398         Select(Chain0, Chain0);
02399         ResNode = CurDAG->getTargetNode(X86::CMP32mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
02400         InFlag = SDOperand(ResNode, 1);
02401         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
02402         AddHandleReplacement(N0.Val, 1, ResNode, 0);
02403         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02404         Result = SDOperand(ResNode, N.ResNo+1);
02405         return;
02406       }
02407     }
02408     N1 = N.getOperand(1);
02409     if (N1.getOpcode() == ISD::LOAD &&
02410         N1.hasOneUse() &&
02411         !CodeGenMap.count(N1.getValue(0)) &&
02412         N.Val->isOnlyUse(N1.Val) &&
02413         !isNonImmUse(N.Val, N1.Val)) {
02414       Chain1 = N1.getOperand(0);
02415       N11 = N1.getOperand(1);
02416 
02417       // Pattern: (X86cmp:void R8:i8:$src1, (load:i8 addr:i32:$src2))
02418       // Emits: (CMP8rm:void R8:i8:$src1, addr:i32:$src2)
02419       // Pattern complexity = 14  cost = 1
02420       if (N0.Val->getValueType(0) == MVT::i8 &&
02421           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02422         Select(Tmp0, Tmp0);
02423         Select(Tmp1, Tmp1);
02424         Select(Tmp2, Tmp2);
02425         Select(Tmp3, Tmp3);
02426         Select(Tmp4, N0);
02427         Select(Chain1, Chain1);
02428         ResNode = CurDAG->getTargetNode(X86::CMP8rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02429         InFlag = SDOperand(ResNode, 1);
02430         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02431         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02432         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02433         Result = SDOperand(ResNode, N.ResNo+1);
02434         return;
02435       }
02436 
02437       // Pattern: (X86cmp:void R16:i16:$src1, (load:i16 addr:i32:$src2))
02438       // Emits: (CMP16rm:void R16:i16:$src1, addr:i32:$src2)
02439       // Pattern complexity = 14  cost = 1
02440       if (N0.Val->getValueType(0) == MVT::i16 &&
02441           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02442         Select(Tmp0, Tmp0);
02443         Select(Tmp1, Tmp1);
02444         Select(Tmp2, Tmp2);
02445         Select(Tmp3, Tmp3);
02446         Select(Tmp4, N0);
02447         Select(Chain1, Chain1);
02448         ResNode = CurDAG->getTargetNode(X86::CMP16rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02449         InFlag = SDOperand(ResNode, 1);
02450         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02451         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02452         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02453         Result = SDOperand(ResNode, N.ResNo+1);
02454         return;
02455       }
02456 
02457       // Pattern: (X86cmp:void R32:i32:$src1, (load:i32 addr:i32:$src2))
02458       // Emits: (CMP32rm:void R32:i32:$src1, addr:i32:$src2)
02459       // Pattern complexity = 14  cost = 1
02460       if (N0.Val->getValueType(0) == MVT::i32 &&
02461           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02462         Select(Tmp0, Tmp0);
02463         Select(Tmp1, Tmp1);
02464         Select(Tmp2, Tmp2);
02465         Select(Tmp3, Tmp3);
02466         Select(Tmp4, N0);
02467         Select(Chain1, Chain1);
02468         ResNode = CurDAG->getTargetNode(X86::CMP32rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02469         InFlag = SDOperand(ResNode, 1);
02470         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02471         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02472         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02473         Result = SDOperand(ResNode, N.ResNo+1);
02474         return;
02475       }
02476     }
02477   }
02478 
02479   // Pattern: (X86cmp:void FR32:f32:$src1, (load:f32 addr:i32:$src2))
02480   // Emits: (UCOMISSrm:void FR32:f32:$src1, addr:i32:$src2)
02481   // Pattern complexity = 14  cost = 1
02482   if ((Subtarget->hasSSE1())) {
02483     N0 = N.getOperand(0);
02484     N1 = N.getOperand(1);
02485     if (N1.getOpcode() == ISD::LOAD &&
02486         N1.hasOneUse() &&
02487         !CodeGenMap.count(N1.getValue(0)) &&
02488         N.Val->isOnlyUse(N1.Val) &&
02489         !isNonImmUse(N.Val, N1.Val)) {
02490       Chain1 = N1.getOperand(0);
02491       N11 = N1.getOperand(1);
02492       if (N0.Val->getValueType(0) == MVT::f32 &&
02493           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02494         Select(Tmp0, Tmp0);
02495         Select(Tmp1, Tmp1);
02496         Select(Tmp2, Tmp2);
02497         Select(Tmp3, Tmp3);
02498         Select(Tmp4, N0);
02499         Select(Chain1, Chain1);
02500         ResNode = CurDAG->getTargetNode(X86::UCOMISSrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02501         InFlag = SDOperand(ResNode, 1);
02502         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02503         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02504         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02505         Result = SDOperand(ResNode, N.ResNo+1);
02506         return;
02507       }
02508     }
02509   }
02510 
02511   // Pattern: (X86cmp:void FR64:f64:$src1, (load:f64 addr:i32:$src2))
02512   // Emits: (UCOMISDrm:void FR64:f64:$src1, addr:i32:$src2)
02513   // Pattern complexity = 14  cost = 1
02514   if ((Subtarget->hasSSE2())) {
02515     N0 = N.getOperand(0);
02516     N1 = N.getOperand(1);
02517     if (N1.getOpcode() == ISD::LOAD &&
02518         N1.hasOneUse() &&
02519         !CodeGenMap.count(N1.getValue(0)) &&
02520         N.Val->isOnlyUse(N1.Val) &&
02521         !isNonImmUse(N.Val, N1.Val)) {
02522       Chain1 = N1.getOperand(0);
02523       N11 = N1.getOperand(1);
02524       if (N0.Val->getValueType(0) == MVT::f64 &&
02525           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02526         Select(Tmp0, Tmp0);
02527         Select(Tmp1, Tmp1);
02528         Select(Tmp2, Tmp2);
02529         Select(Tmp3, Tmp3);
02530         Select(Tmp4, N0);
02531         Select(Chain1, Chain1);
02532         ResNode = CurDAG->getTargetNode(X86::UCOMISDrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02533         InFlag = SDOperand(ResNode, 1);
02534         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02535         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02536         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02537         Result = SDOperand(ResNode, N.ResNo+1);
02538         return;
02539       }
02540     }
02541   }
02542   {
02543     N0 = N.getOperand(0);
02544     N1 = N.getOperand(1);
02545     if (N1.getOpcode() == ISD::Constant) {
02546 
02547       // Pattern: (X86cmp:void R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
02548       // Emits: (CMP16ri8:void R16:i16:$src1, (imm:i16):$src2)
02549       // Pattern complexity = 5  cost = 1
02550       if (Predicate_i16immSExt8(N1.Val) &&
02551           N0.Val->getValueType(0) == MVT::i16) {
02552         Select(Tmp0, N0);
02553         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
02554         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
02555         ResNode = CurDAG->getTargetNode(X86::CMP16ri8, MVT::Flag, Tmp0, Tmp1);
02556         InFlag = SDOperand(ResNode, 0);
02557         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02558         Result = SDOperand(ResNode, N.ResNo);
02559         return;
02560       }
02561 
02562       // Pattern: (X86cmp:void R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
02563       // Emits: (CMP32ri8:void R32:i32:$src1, (imm:i32):$src2)
02564       // Pattern complexity = 5  cost = 1
02565       if (Predicate_i32immSExt8(N1.Val) &&
02566           N0.Val->getValueType(0) == MVT::i32) {
02567         Select(Tmp0, N0);
02568         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
02569         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
02570         ResNode = CurDAG->getTargetNode(X86::CMP32ri8, MVT::Flag, Tmp0, Tmp1);
02571         InFlag = SDOperand(ResNode, 0);
02572         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02573         Result = SDOperand(ResNode, N.ResNo);
02574         return;
02575       }
02576 
02577       // Pattern: (X86cmp:void R8:i8:$src1, (imm:i8):$src2)
02578       // Emits: (CMP8ri:void R8:i8:$src1, (imm:i8):$src2)
02579       // Pattern complexity = 4  cost = 1
02580       if (N0.Val->getValueType(0) == MVT::i8) {
02581         Select(Tmp0, N0);
02582         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
02583         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
02584         ResNode = CurDAG->getTargetNode(X86::CMP8ri, MVT::Flag, Tmp0, Tmp1);
02585         InFlag = SDOperand(ResNode, 0);
02586         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02587         Result = SDOperand(ResNode, N.ResNo);
02588         return;
02589       }
02590 
02591       // Pattern: (X86cmp:void R16:i16:$src1, (imm:i16):$src2)
02592       // Emits: (CMP16ri:void R16:i16:$src1, (imm:i16):$src2)
02593       // Pattern complexity = 4  cost = 1
02594       if (N0.Val->getValueType(0) == MVT::i16) {
02595         Select(Tmp0, N0);
02596         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
02597         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
02598         ResNode = CurDAG->getTargetNode(X86::CMP16ri, MVT::Flag, Tmp0, Tmp1);
02599         InFlag = SDOperand(ResNode, 0);
02600         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02601         Result = SDOperand(ResNode, N.ResNo);
02602         return;
02603       }
02604 
02605       // Pattern: (X86cmp:void R32:i32:$src1, (imm:i32):$src2)
02606       // Emits: (CMP32ri:void R32:i32:$src1, (imm:i32):$src2)
02607       // Pattern complexity = 4  cost = 1
02608       if (N0.Val->getValueType(0) == MVT::i32) {
02609         Select(Tmp0, N0);
02610         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
02611         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
02612         ResNode = CurDAG->getTargetNode(X86::CMP32ri, MVT::Flag, Tmp0, Tmp1);
02613         InFlag = SDOperand(ResNode, 0);
02614         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02615         Result = SDOperand(ResNode, N.ResNo);
02616         return;
02617       }
02618     }
02619 
02620     // Pattern: (X86cmp:void R8:i8:$src1, R8:i8:$src2)
02621     // Emits: (CMP8rr:void R8:i8:$src1, R8:i8:$src2)
02622     // Pattern complexity = 2  cost = 1
02623     if (N0.Val->getValueType(0) == MVT::i8) {
02624       Select(Tmp0, N0);
02625       Select(Tmp1, N1);
02626       ResNode = CurDAG->getTargetNode(X86::CMP8rr, MVT::Flag, Tmp0, Tmp1);
02627       InFlag = SDOperand(ResNode, 0);
02628       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02629       Result = SDOperand(ResNode, N.ResNo);
02630       return;
02631     }
02632 
02633     // Pattern: (X86cmp:void R16:i16:$src1, R16:i16:$src2)
02634     // Emits: (CMP16rr:void R16:i16:$src1, R16:i16:$src2)
02635     // Pattern complexity = 2  cost = 1
02636     if (N0.Val->getValueType(0) == MVT::i16) {
02637       Select(Tmp0, N0);
02638       Select(Tmp1, N1);
02639       ResNode = CurDAG->getTargetNode(X86::CMP16rr, MVT::Flag, Tmp0, Tmp1);
02640       InFlag = SDOperand(ResNode, 0);
02641       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02642       Result = SDOperand(ResNode, N.ResNo);
02643       return;
02644     }
02645 
02646     // Pattern: (X86cmp:void R32:i32:$src1, R32:i32:$src2)
02647     // Emits: (CMP32rr:void R32:i32:$src1, R32:i32:$src2)
02648     // Pattern complexity = 2  cost = 1
02649     if (N0.Val->getValueType(0) == MVT::i32) {
02650       Select(Tmp0, N0);
02651       Select(Tmp1, N1);
02652       ResNode = CurDAG->getTargetNode(X86::CMP32rr, MVT::Flag, Tmp0, Tmp1);
02653       InFlag = SDOperand(ResNode, 0);
02654       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02655       Result = SDOperand(ResNode, N.ResNo);
02656       return;
02657     }
02658   }
02659 
02660   // Pattern: (X86cmp:void RFP:f64:$lhs, RFP:f64:$rhs)
02661   // Emits: (FpUCOMIr:void RFP:f64:$lhs, RFP:f64:$rhs)
02662   // Pattern complexity = 2  cost = 1
02663   if ((!Subtarget->hasSSE2())) {
02664     N0 = N.getOperand(0);
02665     N1 = N.getOperand(1);
02666     if (N0.Val->getValueType(0) == MVT::f64) {
02667       Select(Tmp0, N0);
02668       Select(Tmp1, N1);
02669       ResNode = CurDAG->getTargetNode(X86::FpUCOMIr, MVT::Flag, Tmp0, Tmp1);
02670       InFlag = SDOperand(ResNode, 0);
02671       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02672       Result = SDOperand(ResNode, N.ResNo);
02673       return;
02674     }
02675   }
02676 
02677   // Pattern: (X86cmp:void FR32:f32:$src1, FR32:f32:$src2)
02678   // Emits: (UCOMISSrr:void FR32:f32:$src1, FR32:f32:$src2)
02679   // Pattern complexity = 2  cost = 1
02680   if ((Subtarget->hasSSE1())) {
02681     N0 = N.getOperand(0);
02682     N1 = N.getOperand(1);
02683     if (N0.Val->getValueType(0) == MVT::f32) {
02684       Select(Tmp0, N0);
02685       Select(Tmp1, N1);
02686       ResNode = CurDAG->getTargetNode(X86::UCOMISSrr, MVT::Flag, Tmp0, Tmp1);
02687       InFlag = SDOperand(ResNode, 0);
02688       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02689       Result = SDOperand(ResNode, N.ResNo);
02690       return;
02691     }
02692   }
02693 
02694   // Pattern: (X86cmp:void FR64:f64:$src1, FR64:f64:$src2)
02695   // Emits: (UCOMISDrr:void FR64:f64:$src1, FR64:f64:$src2)
02696   // Pattern complexity = 2  cost = 1
02697   if ((Subtarget->hasSSE2())) {
02698     N0 = N.getOperand(0);
02699     N1 = N.getOperand(1);
02700     if (N0.Val->getValueType(0) == MVT::f64) {
02701       Select(Tmp0, N0);
02702       Select(Tmp1, N1);
02703       ResNode = CurDAG->getTargetNode(X86::UCOMISDrr, MVT::Flag, Tmp0, Tmp1);
02704       InFlag = SDOperand(ResNode, 0);
02705       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02706       Result = SDOperand(ResNode, N.ResNo);
02707       return;
02708     }
02709   }
02710   std::cerr << "Cannot yet select: ";
02711   N.Val->dump(CurDAG);
02712   std::cerr << '\n';
02713   abort();
02714 }
02715 
02716 void Select_X86comi(SDOperand &Result, SDOperand N) {
02717   SDOperand Chain1(0, 0);
02718   SDOperand InFlag(0, 0);
02719   SDOperand N0(0, 0);
02720   SDOperand N1(0, 0);
02721   SDOperand N11(0, 0);
02722   SDOperand Tmp0(0, 0);
02723   SDOperand Tmp1(0, 0);
02724   SDOperand Tmp2(0, 0);
02725   SDOperand Tmp3(0, 0);
02726   SDOperand Tmp4(0, 0);
02727   SDNode *ResNode;
02728 
02729   // Pattern: (X86comi:void VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
02730   // Emits: (Int_COMISSrm:void VR128:v4f32:$src1, addr:i32:$src2)
02731   // Pattern complexity = 14  cost = 1
02732   if ((Subtarget->hasSSE1())) {
02733     N0 = N.getOperand(0);
02734     N1 = N.getOperand(1);
02735     if (N1.getOpcode() == ISD::LOAD &&
02736         N1.hasOneUse() &&
02737         !CodeGenMap.count(N1.getValue(0)) &&
02738         N.Val->isOnlyUse(N1.Val) &&
02739         !isNonImmUse(N.Val, N1.Val)) {
02740       Chain1 = N1.getOperand(0);
02741       N11 = N1.getOperand(1);
02742       if (N0.Val->getValueType(0) == MVT::v4f32 &&
02743           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02744         Select(Tmp0, Tmp0);
02745         Select(Tmp1, Tmp1);
02746         Select(Tmp2, Tmp2);
02747         Select(Tmp3, Tmp3);
02748         Select(Tmp4, N0);
02749         Select(Chain1, Chain1);
02750         ResNode = CurDAG->getTargetNode(X86::Int_COMISSrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02751         InFlag = SDOperand(ResNode, 1);
02752         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02753         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02754         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02755         Result = SDOperand(ResNode, N.ResNo+1);
02756         return;
02757       }
02758     }
02759   }
02760 
02761   // Pattern: (X86comi:void VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
02762   // Emits: (Int_COMISDrm:void VR128:v2f64:$src1, addr:i32:$src2)
02763   // Pattern complexity = 14  cost = 1
02764   if ((Subtarget->hasSSE2())) {
02765     N0 = N.getOperand(0);
02766     N1 = N.getOperand(1);
02767     if (N1.getOpcode() == ISD::LOAD &&
02768         N1.hasOneUse() &&
02769         !CodeGenMap.count(N1.getValue(0)) &&
02770         N.Val->isOnlyUse(N1.Val) &&
02771         !isNonImmUse(N.Val, N1.Val)) {
02772       Chain1 = N1.getOperand(0);
02773       N11 = N1.getOperand(1);
02774       if (N0.Val->getValueType(0) == MVT::v2f64 &&
02775           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02776         Select(Tmp0, Tmp0);
02777         Select(Tmp1, Tmp1);
02778         Select(Tmp2, Tmp2);
02779         Select(Tmp3, Tmp3);
02780         Select(Tmp4, N0);
02781         Select(Chain1, Chain1);
02782         ResNode = CurDAG->getTargetNode(X86::Int_COMISDrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02783         InFlag = SDOperand(ResNode, 1);
02784         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
02785         AddHandleReplacement(N1.Val, 1, ResNode, 0);
02786         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02787         Result = SDOperand(ResNode, N.ResNo+1);
02788         return;
02789       }
02790     }
02791   }
02792 
02793   // Pattern: (X86comi:void VR128:v4f32:$src1, VR128:v4f32:$src2)
02794   // Emits: (Int_COMISSrr:void VR128:v4f32:$src1, VR128:v4f32:$src2)
02795   // Pattern complexity = 2  cost = 1
02796   if ((Subtarget->hasSSE1())) {
02797     N0 = N.getOperand(0);
02798     N1 = N.getOperand(1);
02799     if (N0.Val->getValueType(0) == MVT::v4f32) {
02800       Select(Tmp0, N0);
02801       Select(Tmp1, N1);
02802       ResNode = CurDAG->getTargetNode(X86::Int_COMISSrr, MVT::Flag, Tmp0, Tmp1);
02803       InFlag = SDOperand(ResNode, 0);
02804       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02805       Result = SDOperand(ResNode, N.ResNo);
02806       return;
02807     }
02808   }
02809 
02810   // Pattern: (X86comi:void VR128:v2f64:$src1, VR128:v2f64:$src2)
02811   // Emits: (Int_COMISDrr:void VR128:v2f64:$src1, VR128:v2f64:$src2)
02812   // Pattern complexity = 2  cost = 1
02813   if ((Subtarget->hasSSE2())) {
02814     N0 = N.getOperand(0);
02815     N1 = N.getOperand(1);
02816     if (N0.Val->getValueType(0) == MVT::v2f64) {
02817       Select(Tmp0, N0);
02818       Select(Tmp1, N1);
02819       ResNode = CurDAG->getTargetNode(X86::Int_COMISDrr, MVT::Flag, Tmp0, Tmp1);
02820       InFlag = SDOperand(ResNode, 0);
02821       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
02822       Result = SDOperand(ResNode, N.ResNo);
02823       return;
02824     }
02825   }
02826   std::cerr << "Cannot yet select: ";
02827   N.Val->dump(CurDAG);
02828   std::cerr << '\n';
02829   abort();
02830 }
02831 
02832 void Select_X86fand(SDOperand &Result, SDOperand N) {
02833   SDOperand Chain0(0, 0);
02834   SDOperand Chain1(0, 0);
02835   SDOperand N0(0, 0);
02836   SDOperand N01(0, 0);
02837   SDOperand N1(0, 0);
02838   SDOperand N11(0, 0);
02839   SDOperand Tmp0(0, 0);
02840   SDOperand Tmp1(0, 0);
02841   SDOperand Tmp2(0, 0);
02842   SDOperand Tmp3(0, 0);
02843   SDOperand Tmp4(0, 0);
02844   SDNode *ResNode;
02845 
02846   // Pattern: (X86fand:f32 FR32:f32:$src1, (X86loadp:f32 addr:i32:$src2))
02847   // Emits: (FsANDPSrm:f32 FR32:f32:$src1, addr:i32:$src2)
02848   // Pattern complexity = 14  cost = 1
02849   if ((Subtarget->hasSSE1())) {
02850     N0 = N.getOperand(0);
02851     N1 = N.getOperand(1);
02852     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
02853         N1.hasOneUse() &&
02854         !CodeGenMap.count(N1.getValue(0)) &&
02855         !isNonImmUse(N.Val, N1.Val)) {
02856       Chain1 = N1.getOperand(0);
02857       N11 = N1.getOperand(1);
02858       if (N.Val->getValueType(0) == MVT::f32 &&
02859           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02860         Select(Tmp0, Tmp0);
02861         Select(Tmp1, Tmp1);
02862         Select(Tmp2, Tmp2);
02863         Select(Tmp3, Tmp3);
02864         Select(Tmp4, N0);
02865         Select(Chain1, Chain1);
02866         ResNode = CurDAG->getTargetNode(X86::FsANDPSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02867         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02868         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
02869         AddHandleReplacement(N1.Val, 1, ResNode, 1);
02870         Result = SDOperand(ResNode, N.ResNo);
02871         return;
02872       }
02873     }
02874   }
02875 
02876   // Pattern: (X86fand:f64 FR64:f64:$src1, (X86loadp:f64 addr:i32:$src2))
02877   // Emits: (FsANDPDrm:f64 FR64:f64:$src1, addr:i32:$src2)
02878   // Pattern complexity = 14  cost = 1
02879   if ((Subtarget->hasSSE2())) {
02880     N0 = N.getOperand(0);
02881     N1 = N.getOperand(1);
02882     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
02883         N1.hasOneUse() &&
02884         !CodeGenMap.count(N1.getValue(0)) &&
02885         !isNonImmUse(N.Val, N1.Val)) {
02886       Chain1 = N1.getOperand(0);
02887       N11 = N1.getOperand(1);
02888       if (N.Val->getValueType(0) == MVT::f64 &&
02889           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
02890         Select(Tmp0, Tmp0);
02891         Select(Tmp1, Tmp1);
02892         Select(Tmp2, Tmp2);
02893         Select(Tmp3, Tmp3);
02894         Select(Tmp4, N0);
02895         Select(Chain1, Chain1);
02896         ResNode = CurDAG->getTargetNode(X86::FsANDPDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
02897         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02898         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
02899         AddHandleReplacement(N1.Val, 1, ResNode, 1);
02900         Result = SDOperand(ResNode, N.ResNo);
02901         return;
02902       }
02903     }
02904   }
02905 
02906   // Pattern: (X86fand:f32 (X86loadp:f32 addr:i32:$src2), FR32:f32:$src1)
02907   // Emits: (FsANDPSrm:f32 FR32:f32:$src1, addr:i32:$src2)
02908   // Pattern complexity = 14  cost = 1
02909   if ((Subtarget->hasSSE1())) {
02910     N0 = N.getOperand(0);
02911     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
02912         N0.hasOneUse() &&
02913         !CodeGenMap.count(N0.getValue(0)) &&
02914         !isNonImmUse(N.Val, N0.Val)) {
02915       Chain0 = N0.getOperand(0);
02916       N01 = N0.getOperand(1);
02917       N1 = N.getOperand(1);
02918       if (N.Val->getValueType(0) == MVT::f32 &&
02919           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02920         Select(Tmp0, Tmp0);
02921         Select(Tmp1, Tmp1);
02922         Select(Tmp2, Tmp2);
02923         Select(Tmp3, Tmp3);
02924         Select(Tmp4, N1);
02925         Select(Chain0, Chain0);
02926         ResNode = CurDAG->getTargetNode(X86::FsANDPSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
02927         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02928         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
02929         AddHandleReplacement(N0.Val, 1, ResNode, 1);
02930         Result = SDOperand(ResNode, N.ResNo);
02931         return;
02932       }
02933     }
02934   }
02935 
02936   // Pattern: (X86fand:f64 (X86loadp:f64 addr:i32:$src2), FR64:f64:$src1)
02937   // Emits: (FsANDPDrm:f64 FR64:f64:$src1, addr:i32:$src2)
02938   // Pattern complexity = 14  cost = 1
02939   if ((Subtarget->hasSSE2())) {
02940     N0 = N.getOperand(0);
02941     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
02942         N0.hasOneUse() &&
02943         !CodeGenMap.count(N0.getValue(0)) &&
02944         !isNonImmUse(N.Val, N0.Val)) {
02945       Chain0 = N0.getOperand(0);
02946       N01 = N0.getOperand(1);
02947       N1 = N.getOperand(1);
02948       if (N.Val->getValueType(0) == MVT::f64 &&
02949           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
02950         Select(Tmp0, Tmp0);
02951         Select(Tmp1, Tmp1);
02952         Select(Tmp2, Tmp2);
02953         Select(Tmp3, Tmp3);
02954         Select(Tmp4, N1);
02955         Select(Chain0, Chain0);
02956         ResNode = CurDAG->getTargetNode(X86::FsANDPDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
02957         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
02958         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
02959         AddHandleReplacement(N0.Val, 1, ResNode, 1);
02960         Result = SDOperand(ResNode, N.ResNo);
02961         return;
02962       }
02963     }
02964   }
02965 
02966   // Pattern: (X86fand:f32 FR32:f32:$src1, FR32:f32:$src2)
02967   // Emits: (FsANDPSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
02968   // Pattern complexity = 2  cost = 1
02969   if ((Subtarget->hasSSE1())) {
02970     N0 = N.getOperand(0);
02971     N1 = N.getOperand(1);
02972     if (N.Val->getValueType(0) == MVT::f32) {
02973       Select(Tmp0, N0);
02974       Select(Tmp1, N1);
02975       if (N.Val->hasOneUse()) {
02976         Result = CurDAG->SelectNodeTo(N.Val, X86::FsANDPSrr, MVT::f32, Tmp0, Tmp1);
02977       } else {
02978         ResNode = CurDAG->getTargetNode(X86::FsANDPSrr, MVT::f32, Tmp0, Tmp1);
02979         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
02980         Result = SDOperand(ResNode, 0);
02981       }
02982       return;
02983     }
02984   }
02985 
02986   // Pattern: (X86fand:f64 FR64:f64:$src1, FR64:f64:$src2)
02987   // Emits: (FsANDPDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
02988   // Pattern complexity = 2  cost = 1
02989   if ((Subtarget->hasSSE2())) {
02990     N0 = N.getOperand(0);
02991     N1 = N.getOperand(1);
02992     if (N.Val->getValueType(0) == MVT::f64) {
02993       Select(Tmp0, N0);
02994       Select(Tmp1, N1);
02995       if (N.Val->hasOneUse()) {
02996         Result = CurDAG->SelectNodeTo(N.Val, X86::FsANDPDrr, MVT::f64, Tmp0, Tmp1);
02997       } else {
02998         ResNode = CurDAG->getTargetNode(X86::FsANDPDrr, MVT::f64, Tmp0, Tmp1);
02999         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03000         Result = SDOperand(ResNode, 0);
03001       }
03002       return;
03003     }
03004   }
03005   std::cerr << "Cannot yet select: ";
03006   N.Val->dump(CurDAG);
03007   std::cerr << '\n';
03008   abort();
03009 }
03010 
03011 void Select_X86fild(SDOperand &Result, SDOperand N) {
03012   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03013     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03014     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03015     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03016     Result = Dummy;
03017     return;
03018   }
03019   SDOperand Chain(0, 0);
03020   SDOperand N1(0, 0);
03021   SDOperand N2(0, 0);
03022   SDOperand Tmp0(0, 0);
03023   SDOperand Tmp1(0, 0);
03024   SDOperand Tmp2(0, 0);
03025   SDOperand Tmp3(0, 0);
03026   SDNode *ResNode;
03027   if ((!Subtarget->hasSSE2())) {
03028     Chain = N.getOperand(0);
03029     N1 = N.getOperand(1);
03030     N2 = N.getOperand(2);
03031 
03032     // Pattern: (X86fild:f64 addr:i32:$src, i16:Other)
03033     // Emits: (FpILD16m:f64 addr:i32:$src)
03034     // Pattern complexity = 12  cost = 1
03035     if (cast<VTSDNode>(N2)->getVT() == MVT::i16 &&
03036         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03037       Select(Tmp0, Tmp0);
03038       Select(Tmp1, Tmp1);
03039       Select(Tmp2, Tmp2);
03040       Select(Tmp3, Tmp3);
03041       Select(Chain, Chain);
03042       ResNode = CurDAG->getTargetNode(X86::FpILD16m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03043       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03044       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03045       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03046       Result = SDOperand(ResNode, N.ResNo);
03047       return;
03048     }
03049 
03050     // Pattern: (X86fild:f64 addr:i32:$src, i32:Other)
03051     // Emits: (FpILD32m:f64 addr:i32:$src)
03052     // Pattern complexity = 12  cost = 1
03053     if (cast<VTSDNode>(N2)->getVT() == MVT::i32 &&
03054         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03055       Select(Tmp0, Tmp0);
03056       Select(Tmp1, Tmp1);
03057       Select(Tmp2, Tmp2);
03058       Select(Tmp3, Tmp3);
03059       Select(Chain, Chain);
03060       ResNode = CurDAG->getTargetNode(X86::FpILD32m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03061       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03062       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03063       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03064       Result = SDOperand(ResNode, N.ResNo);
03065       return;
03066     }
03067 
03068     // Pattern: (X86fild:f64 addr:i32:$src, i64:Other)
03069     // Emits: (FpILD64m:f64 addr:i32:$src)
03070     // Pattern complexity = 12  cost = 1
03071     if (cast<VTSDNode>(N2)->getVT() == MVT::i64 &&
03072         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03073       Select(Tmp0, Tmp0);
03074       Select(Tmp1, Tmp1);
03075       Select(Tmp2, Tmp2);
03076       Select(Tmp3, Tmp3);
03077       Select(Chain, Chain);
03078       ResNode = CurDAG->getTargetNode(X86::FpILD64m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03079       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03080       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03081       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03082       Result = SDOperand(ResNode, N.ResNo);
03083       return;
03084     }
03085   }
03086   std::cerr << "Cannot yet select: ";
03087   N.Val->dump(CurDAG);
03088   std::cerr << '\n';
03089   abort();
03090 }
03091 
03092 void Select_X86fildflag(SDOperand &Result, SDOperand N) {
03093   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03094     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03095     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03096     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03097     Result = Dummy;
03098     return;
03099   }
03100   SDOperand Chain(0, 0);
03101   SDOperand InFlag(0, 0);
03102   SDOperand N1(0, 0);
03103   SDOperand N2(0, 0);
03104   SDOperand Tmp0(0, 0);
03105   SDOperand Tmp1(0, 0);
03106   SDOperand Tmp2(0, 0);
03107   SDOperand Tmp3(0, 0);
03108   SDNode *ResNode;
03109   Chain = N.getOperand(0);
03110   N1 = N.getOperand(1);
03111   N2 = N.getOperand(2);
03112   if (cast<VTSDNode>(N2)->getVT() == MVT::i64 &&
03113       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03114     Select(Tmp0, Tmp0);
03115     Select(Tmp1, Tmp1);
03116     Select(Tmp2, Tmp2);
03117     Select(Tmp3, Tmp3);
03118     Select(Chain, Chain);
03119     ResNode = CurDAG->getTargetNode(X86::FpILD64m, MVT::f64, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03120     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03121     InFlag = SDOperand(ResNode, 2);
03122     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03123     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03124     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, InFlag.Val, InFlag.ResNo);
03125     Result = SDOperand(ResNode, N.ResNo);
03126     return;
03127   }
03128   std::cerr << "Cannot yet select: ";
03129   N.Val->dump(CurDAG);
03130   std::cerr << '\n';
03131   abort();
03132 }
03133 
03134 void Select_X86fld(SDOperand &Result, SDOperand N) {
03135   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03136     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03137     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03138     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03139     Result = Dummy;
03140     return;
03141   }
03142   SDOperand Chain(0, 0);
03143   SDOperand N1(0, 0);
03144   SDOperand N2(0, 0);
03145   SDOperand Tmp0(0, 0);
03146   SDOperand Tmp1(0, 0);
03147   SDOperand Tmp2(0, 0);
03148   SDOperand Tmp3(0, 0);
03149   SDNode *ResNode;
03150   Chain = N.getOperand(0);
03151   N1 = N.getOperand(1);
03152   N2 = N.getOperand(2);
03153 
03154   // Pattern: (X86fld:f64 addr:i32:$src, f32:Other)
03155   // Emits: (FpLD32m:f64 addr:i32:$src)
03156   // Pattern complexity = 12  cost = 1
03157   if (cast<VTSDNode>(N2)->getVT() == MVT::f32 &&
03158       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03159     Select(Tmp0, Tmp0);
03160     Select(Tmp1, Tmp1);
03161     Select(Tmp2, Tmp2);
03162     Select(Tmp3, Tmp3);
03163     Select(Chain, Chain);
03164     ResNode = CurDAG->getTargetNode(X86::FpLD32m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03165     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03166     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03167     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03168     Result = SDOperand(ResNode, N.ResNo);
03169     return;
03170   }
03171 
03172   // Pattern: (X86fld:f64 addr:i32:$src, f64:Other)
03173   // Emits: (FpLD64m:f64 addr:i32:$src)
03174   // Pattern complexity = 12  cost = 1
03175   if (cast<VTSDNode>(N2)->getVT() == MVT::f64 &&
03176       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03177     Select(Tmp0, Tmp0);
03178     Select(Tmp1, Tmp1);
03179     Select(Tmp2, Tmp2);
03180     Select(Tmp3, Tmp3);
03181     Select(Chain, Chain);
03182     ResNode = CurDAG->getTargetNode(X86::FpLD64m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03183     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03184     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03185     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03186     Result = SDOperand(ResNode, N.ResNo);
03187     return;
03188   }
03189   std::cerr << "Cannot yet select: ";
03190   N.Val->dump(CurDAG);
03191   std::cerr << '\n';
03192   abort();
03193 }
03194 
03195 void Select_X86fp_to_i16mem(SDOperand &Result, SDOperand N) {
03196   SDOperand Chain(0, 0);
03197   SDOperand N1(0, 0);
03198   SDOperand N2(0, 0);
03199   SDOperand Tmp0(0, 0);
03200   SDOperand Tmp1(0, 0);
03201   SDOperand Tmp2(0, 0);
03202   SDOperand Tmp3(0, 0);
03203   SDOperand Tmp4(0, 0);
03204   SDNode *ResNode;
03205 
03206   // Pattern: (X86fp_to_i16mem:void RFP:f64:$src, addr:i32:$op)
03207   // Emits: (FpISTT16m:void addr:i32:$op, RFP:f64:$src)
03208   // Pattern complexity = 12  cost = 1
03209   if ((Subtarget->hasSSE3())) {
03210     Chain = N.getOperand(0);
03211     N1 = N.getOperand(1);
03212     N2 = N.getOperand(2);
03213     if (N1.Val->getValueType(0) == MVT::f64 &&
03214         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03215       Select(Tmp0, Tmp0);
03216       Select(Tmp1, Tmp1);
03217       Select(Tmp2, Tmp2);
03218       Select(Tmp3, Tmp3);
03219       Select(Tmp4, N1);
03220       Select(Chain, Chain);
03221       ResNode = CurDAG->getTargetNode(X86::FpISTT16m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03222       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03223       Result = SDOperand(ResNode, N.ResNo);
03224       return;
03225     }
03226   }
03227 
03228   // Pattern: (X86fp_to_i16mem:void RFP:f64:$src, addr:i32:$dst)
03229   // Emits: (FP_TO_INT16_IN_MEM:void addr:i32:$dst, RFP:f64:$src)
03230   // Pattern complexity = 12  cost = 11
03231   Chain = N.getOperand(0);
03232   N1 = N.getOperand(1);
03233   N2 = N.getOperand(2);
03234   if (N1.Val->getValueType(0) == MVT::f64 &&
03235       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03236     Select(Tmp0, Tmp0);
03237     Select(Tmp1, Tmp1);
03238     Select(Tmp2, Tmp2);
03239     Select(Tmp3, Tmp3);
03240     Select(Tmp4, N1);
03241     Select(Chain, Chain);
03242     ResNode = CurDAG->getTargetNode(X86::FP_TO_INT16_IN_MEM, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03243     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03244     Result = SDOperand(ResNode, N.ResNo);
03245     return;
03246   }
03247   std::cerr << "Cannot yet select: ";
03248   N.Val->dump(CurDAG);
03249   std::cerr << '\n';
03250   abort();
03251 }
03252 
03253 void Select_X86fp_to_i32mem(SDOperand &Result, SDOperand N) {
03254   SDOperand Chain(0, 0);
03255   SDOperand N1(0, 0);
03256   SDOperand N2(0, 0);
03257   SDOperand Tmp0(0, 0);
03258   SDOperand Tmp1(0, 0);
03259   SDOperand Tmp2(0, 0);
03260   SDOperand Tmp3(0, 0);
03261   SDOperand Tmp4(0, 0);
03262   SDNode *ResNode;
03263 
03264   // Pattern: (X86fp_to_i32mem:void RFP:f64:$src, addr:i32:$op)
03265   // Emits: (FpISTT32m:void addr:i32:$op, RFP:f64:$src)
03266   // Pattern complexity = 12  cost = 1
03267   if ((Subtarget->hasSSE3())) {
03268     Chain = N.getOperand(0);
03269     N1 = N.getOperand(1);
03270     N2 = N.getOperand(2);
03271     if (N1.Val->getValueType(0) == MVT::f64 &&
03272         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03273       Select(Tmp0, Tmp0);
03274       Select(Tmp1, Tmp1);
03275       Select(Tmp2, Tmp2);
03276       Select(Tmp3, Tmp3);
03277       Select(Tmp4, N1);
03278       Select(Chain, Chain);
03279       ResNode = CurDAG->getTargetNode(X86::FpISTT32m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03280       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03281       Result = SDOperand(ResNode, N.ResNo);
03282       return;
03283     }
03284   }
03285 
03286   // Pattern: (X86fp_to_i32mem:void RFP:f64:$src, addr:i32:$dst)
03287   // Emits: (FP_TO_INT32_IN_MEM:void addr:i32:$dst, RFP:f64:$src)
03288   // Pattern complexity = 12  cost = 11
03289   Chain = N.getOperand(0);
03290   N1 = N.getOperand(1);
03291   N2 = N.getOperand(2);
03292   if (N1.Val->getValueType(0) == MVT::f64 &&
03293       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03294     Select(Tmp0, Tmp0);
03295     Select(Tmp1, Tmp1);
03296     Select(Tmp2, Tmp2);
03297     Select(Tmp3, Tmp3);
03298     Select(Tmp4, N1);
03299     Select(Chain, Chain);
03300     ResNode = CurDAG->getTargetNode(X86::FP_TO_INT32_IN_MEM, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03301     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03302     Result = SDOperand(ResNode, N.ResNo);
03303     return;
03304   }
03305   std::cerr << "Cannot yet select: ";
03306   N.Val->dump(CurDAG);
03307   std::cerr << '\n';
03308   abort();
03309 }
03310 
03311 void Select_X86fp_to_i64mem(SDOperand &Result, SDOperand N) {
03312   SDOperand Chain(0, 0);
03313   SDOperand N1(0, 0);
03314   SDOperand N2(0, 0);
03315   SDOperand Tmp0(0, 0);
03316   SDOperand Tmp1(0, 0);
03317   SDOperand Tmp2(0, 0);
03318   SDOperand Tmp3(0, 0);
03319   SDOperand Tmp4(0, 0);
03320   SDNode *ResNode;
03321 
03322   // Pattern: (X86fp_to_i64mem:void RFP:f64:$src, addr:i32:$op)
03323   // Emits: (FpISTT64m:void addr:i32:$op, RFP:f64:$src)
03324   // Pattern complexity = 12  cost = 1
03325   if ((Subtarget->hasSSE3())) {
03326     Chain = N.getOperand(0);
03327     N1 = N.getOperand(1);
03328     N2 = N.getOperand(2);
03329     if (N1.Val->getValueType(0) == MVT::f64 &&
03330         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03331       Select(Tmp0, Tmp0);
03332       Select(Tmp1, Tmp1);
03333       Select(Tmp2, Tmp2);
03334       Select(Tmp3, Tmp3);
03335       Select(Tmp4, N1);
03336       Select(Chain, Chain);
03337       ResNode = CurDAG->getTargetNode(X86::FpISTT64m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03338       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03339       Result = SDOperand(ResNode, N.ResNo);
03340       return;
03341     }
03342   }
03343 
03344   // Pattern: (X86fp_to_i64mem:void RFP:f64:$src, addr:i32:$dst)
03345   // Emits: (FP_TO_INT64_IN_MEM:void addr:i32:$dst, RFP:f64:$src)
03346   // Pattern complexity = 12  cost = 11
03347   Chain = N.getOperand(0);
03348   N1 = N.getOperand(1);
03349   N2 = N.getOperand(2);
03350   if (N1.Val->getValueType(0) == MVT::f64 &&
03351       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03352     Select(Tmp0, Tmp0);
03353     Select(Tmp1, Tmp1);
03354     Select(Tmp2, Tmp2);
03355     Select(Tmp3, Tmp3);
03356     Select(Tmp4, N1);
03357     Select(Chain, Chain);
03358     ResNode = CurDAG->getTargetNode(X86::FP_TO_INT64_IN_MEM, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
03359     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03360     Result = SDOperand(ResNode, N.ResNo);
03361     return;
03362   }
03363   std::cerr << "Cannot yet select: ";
03364   N.Val->dump(CurDAG);
03365   std::cerr << '\n';
03366   abort();
03367 }
03368 
03369 void Select_X86fpget(SDOperand &Result, SDOperand N) {
03370   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03371     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03372     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03373     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03374     Result = Dummy;
03375     return;
03376   }
03377   SDOperand Chain(0, 0);
03378   SDOperand InFlag(0, 0);
03379   SDNode *ResNode;
03380   Chain = N.getOperand(0);
03381   Select(Chain, Chain);
03382   Select(InFlag, N.getOperand(1));
03383   ResNode = CurDAG->getTargetNode(X86::FpGETRESULT, MVT::f64, MVT::Other, MVT::Flag, Chain, InFlag);
03384   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03385   InFlag = SDOperand(ResNode, 2);
03386   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03387   if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03388   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, InFlag.Val, InFlag.ResNo);
03389   Result = SDOperand(ResNode, N.ResNo);
03390   return;
03391 }
03392 
03393 void Select_X86fpset(SDOperand &Result, SDOperand N) {
03394   SDOperand Chain(0, 0);
03395   SDOperand InFlag(0, 0);
03396   SDOperand N1(0, 0);
03397   SDOperand Tmp0(0, 0);
03398   SDNode *ResNode;
03399   Chain = N.getOperand(0);
03400   N1 = N.getOperand(1);
03401   if (N1.Val->getValueType(0) == MVT::f64) {
03402     Select(Tmp0, N1);
03403     Select(Chain, Chain);
03404     ResNode = CurDAG->getTargetNode(X86::FpSETRESULT, MVT::Other, MVT::Flag, Tmp0, Chain);
03405     InFlag = SDOperand(ResNode, 1);
03406     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03407     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03408     Result = SDOperand(ResNode, N.ResNo);
03409     return;
03410   }
03411   std::cerr << "Cannot yet select: ";
03412   N.Val->dump(CurDAG);
03413   std::cerr << '\n';
03414   abort();
03415 }
03416 
03417 void Select_X86fst(SDOperand &Result, SDOperand N) {
03418   SDOperand Chain(0, 0);
03419   SDOperand InFlag(0, 0);
03420   SDOperand N1(0, 0);
03421   SDOperand N2(0, 0);
03422   SDOperand N3(0, 0);
03423   SDOperand Tmp0(0, 0);
03424   SDOperand Tmp1(0, 0);
03425   SDOperand Tmp2(0, 0);
03426   SDOperand Tmp3(0, 0);
03427   SDOperand Tmp4(0, 0);
03428   SDNode *ResNode;
03429   Chain = N.getOperand(0);
03430   N1 = N.getOperand(1);
03431   N2 = N.getOperand(2);
03432   N3 = N.getOperand(3);
03433 
03434   // Pattern: (X86fst:void RFP:f64:$src, addr:i32:$op, f32:Other)
03435   // Emits: (FpST32m:void addr:i32:$op, RFP:f64:$src)
03436   // Pattern complexity = 12  cost = 1
03437   if (cast<VTSDNode>(N3)->getVT() == MVT::f32 &&
03438       N1.Val->getValueType(0) == MVT::f64 &&
03439       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03440     Select(Tmp0, Tmp0);
03441     Select(Tmp1, Tmp1);
03442     Select(Tmp2, Tmp2);
03443     Select(Tmp3, Tmp3);
03444     Select(Tmp4, N1);
03445     Select(Chain, Chain);
03446     Select(InFlag, N.getOperand(4));
03447     ResNode = CurDAG->getTargetNode(X86::FpST32m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
03448     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03449     Result = SDOperand(ResNode, N.ResNo);
03450     return;
03451   }
03452 
03453   // Pattern: (X86fst:void RFP:f64:$src, addr:i32:$op, f64:Other)
03454   // Emits: (FpST64m:void addr:i32:$op, RFP:f64:$src)
03455   // Pattern complexity = 12  cost = 1
03456   if (cast<VTSDNode>(N3)->getVT() == MVT::f64 &&
03457       N1.Val->getValueType(0) == MVT::f64 &&
03458       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
03459     Select(Tmp0, Tmp0);
03460     Select(Tmp1, Tmp1);
03461     Select(Tmp2, Tmp2);
03462     Select(Tmp3, Tmp3);
03463     Select(Tmp4, N1);
03464     Select(Chain, Chain);
03465     Select(InFlag, N.getOperand(4));
03466     ResNode = CurDAG->getTargetNode(X86::FpST64m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
03467     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03468     Result = SDOperand(ResNode, N.ResNo);
03469     return;
03470   }
03471   std::cerr << "Cannot yet select: ";
03472   N.Val->dump(CurDAG);
03473   std::cerr << '\n';
03474   abort();
03475 }
03476 
03477 void Select_X86fxor(SDOperand &Result, SDOperand N) {
03478   SDOperand Chain0(0, 0);
03479   SDOperand Chain1(0, 0);
03480   SDOperand N0(0, 0);
03481   SDOperand N01(0, 0);
03482   SDOperand N1(0, 0);
03483   SDOperand N11(0, 0);
03484   SDOperand Tmp0(0, 0);
03485   SDOperand Tmp1(0, 0);
03486   SDOperand Tmp2(0, 0);
03487   SDOperand Tmp3(0, 0);
03488   SDOperand Tmp4(0, 0);
03489   SDNode *ResNode;
03490 
03491   // Pattern: (X86fxor:f32 FR32:f32:$src1, (X86loadp:f32 addr:i32:$src2))
03492   // Emits: (FsXORPSrm:f32 FR32:f32:$src1, addr:i32:$src2)
03493   // Pattern complexity = 14  cost = 1
03494   if ((Subtarget->hasSSE1())) {
03495     N0 = N.getOperand(0);
03496     N1 = N.getOperand(1);
03497     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
03498         N1.hasOneUse() &&
03499         !CodeGenMap.count(N1.getValue(0)) &&
03500         !isNonImmUse(N.Val, N1.Val)) {
03501       Chain1 = N1.getOperand(0);
03502       N11 = N1.getOperand(1);
03503       if (N.Val->getValueType(0) == MVT::f32 &&
03504           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
03505         Select(Tmp0, Tmp0);
03506         Select(Tmp1, Tmp1);
03507         Select(Tmp2, Tmp2);
03508         Select(Tmp3, Tmp3);
03509         Select(Tmp4, N0);
03510         Select(Chain1, Chain1);
03511         ResNode = CurDAG->getTargetNode(X86::FsXORPSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
03512         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03513         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
03514         AddHandleReplacement(N1.Val, 1, ResNode, 1);
03515         Result = SDOperand(ResNode, N.ResNo);
03516         return;
03517       }
03518     }
03519   }
03520 
03521   // Pattern: (X86fxor:f64 FR64:f64:$src1, (X86loadp:f64 addr:i32:$src2))
03522   // Emits: (FsXORPDrm:f64 FR64:f64:$src1, addr:i32:$src2)
03523   // Pattern complexity = 14  cost = 1
03524   if ((Subtarget->hasSSE2())) {
03525     N0 = N.getOperand(0);
03526     N1 = N.getOperand(1);
03527     if (N1.getOpcode() == X86ISD::LOAD_PACK &&
03528         N1.hasOneUse() &&
03529         !CodeGenMap.count(N1.getValue(0)) &&
03530         !isNonImmUse(N.Val, N1.Val)) {
03531       Chain1 = N1.getOperand(0);
03532       N11 = N1.getOperand(1);
03533       if (N.Val->getValueType(0) == MVT::f64 &&
03534           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
03535         Select(Tmp0, Tmp0);
03536         Select(Tmp1, Tmp1);
03537         Select(Tmp2, Tmp2);
03538         Select(Tmp3, Tmp3);
03539         Select(Tmp4, N0);
03540         Select(Chain1, Chain1);
03541         ResNode = CurDAG->getTargetNode(X86::FsXORPDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
03542         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03543         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
03544         AddHandleReplacement(N1.Val, 1, ResNode, 1);
03545         Result = SDOperand(ResNode, N.ResNo);
03546         return;
03547       }
03548     }
03549   }
03550 
03551   // Pattern: (X86fxor:f32 (X86loadp:f32 addr:i32:$src2), FR32:f32:$src1)
03552   // Emits: (FsXORPSrm:f32 FR32:f32:$src1, addr:i32:$src2)
03553   // Pattern complexity = 14  cost = 1
03554   if ((Subtarget->hasSSE1())) {
03555     N0 = N.getOperand(0);
03556     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
03557         N0.hasOneUse() &&
03558         !CodeGenMap.count(N0.getValue(0)) &&
03559         !isNonImmUse(N.Val, N0.Val)) {
03560       Chain0 = N0.getOperand(0);
03561       N01 = N0.getOperand(1);
03562       N1 = N.getOperand(1);
03563       if (N.Val->getValueType(0) == MVT::f32 &&
03564           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
03565         Select(Tmp0, Tmp0);
03566         Select(Tmp1, Tmp1);
03567         Select(Tmp2, Tmp2);
03568         Select(Tmp3, Tmp3);
03569         Select(Tmp4, N1);
03570         Select(Chain0, Chain0);
03571         ResNode = CurDAG->getTargetNode(X86::FsXORPSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
03572         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03573         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
03574         AddHandleReplacement(N0.Val, 1, ResNode, 1);
03575         Result = SDOperand(ResNode, N.ResNo);
03576         return;
03577       }
03578     }
03579   }
03580 
03581   // Pattern: (X86fxor:f64 (X86loadp:f64 addr:i32:$src2), FR64:f64:$src1)
03582   // Emits: (FsXORPDrm:f64 FR64:f64:$src1, addr:i32:$src2)
03583   // Pattern complexity = 14  cost = 1
03584   if ((Subtarget->hasSSE2())) {
03585     N0 = N.getOperand(0);
03586     if (N0.getOpcode() == X86ISD::LOAD_PACK &&
03587         N0.hasOneUse() &&
03588         !CodeGenMap.count(N0.getValue(0)) &&
03589         !isNonImmUse(N.Val, N0.Val)) {
03590       Chain0 = N0.getOperand(0);
03591       N01 = N0.getOperand(1);
03592       N1 = N.getOperand(1);
03593       if (N.Val->getValueType(0) == MVT::f64 &&
03594           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
03595         Select(Tmp0, Tmp0);
03596         Select(Tmp1, Tmp1);
03597         Select(Tmp2, Tmp2);
03598         Select(Tmp3, Tmp3);
03599         Select(Tmp4, N1);
03600         Select(Chain0, Chain0);
03601         ResNode = CurDAG->getTargetNode(X86::FsXORPDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
03602         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03603         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
03604         AddHandleReplacement(N0.Val, 1, ResNode, 1);
03605         Result = SDOperand(ResNode, N.ResNo);
03606         return;
03607       }
03608     }
03609   }
03610 
03611   // Pattern: (X86fxor:f32 FR32:f32:$src1, FR32:f32:$src2)
03612   // Emits: (FsXORPSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
03613   // Pattern complexity = 2  cost = 1
03614   if ((Subtarget->hasSSE1())) {
03615     N0 = N.getOperand(0);
03616     N1 = N.getOperand(1);
03617     if (N.Val->getValueType(0) == MVT::f32) {
03618       Select(Tmp0, N0);
03619       Select(Tmp1, N1);
03620       if (N.Val->hasOneUse()) {
03621         Result = CurDAG->SelectNodeTo(N.Val, X86::FsXORPSrr, MVT::f32, Tmp0, Tmp1);
03622       } else {
03623         ResNode = CurDAG->getTargetNode(X86::FsXORPSrr, MVT::f32, Tmp0, Tmp1);
03624         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03625         Result = SDOperand(ResNode, 0);
03626       }
03627       return;
03628     }
03629   }
03630 
03631   // Pattern: (X86fxor:f64 FR64:f64:$src1, FR64:f64:$src2)
03632   // Emits: (FsXORPDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
03633   // Pattern complexity = 2  cost = 1
03634   if ((Subtarget->hasSSE2())) {
03635     N0 = N.getOperand(0);
03636     N1 = N.getOperand(1);
03637     if (N.Val->getValueType(0) == MVT::f64) {
03638       Select(Tmp0, N0);
03639       Select(Tmp1, N1);
03640       if (N.Val->hasOneUse()) {
03641         Result = CurDAG->SelectNodeTo(N.Val, X86::FsXORPDrr, MVT::f64, Tmp0, Tmp1);
03642       } else {
03643         ResNode = CurDAG->getTargetNode(X86::FsXORPDrr, MVT::f64, Tmp0, Tmp1);
03644         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03645         Result = SDOperand(ResNode, 0);
03646       }
03647       return;
03648     }
03649   }
03650   std::cerr << "Cannot yet select: ";
03651   N.Val->dump(CurDAG);
03652   std::cerr << '\n';
03653   abort();
03654 }
03655 
03656 void Select_X86loadp(SDOperand &Result, SDOperand N) {
03657   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
03658     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
03659     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
03660     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
03661     Result = Dummy;
03662     return;
03663   }
03664   SDOperand Chain(0, 0);
03665   SDOperand N1(0, 0);
03666   SDOperand Tmp0(0, 0);
03667   SDOperand Tmp1(0, 0);
03668   SDOperand Tmp2(0, 0);
03669   SDOperand Tmp3(0, 0);
03670   SDNode *ResNode;
03671 
03672   // Pattern: (X86loadp:f32 addr:i32:$src)
03673   // Emits: (FsMOVAPSrm:f32 addr:i32:$src)
03674   // Pattern complexity = 12  cost = 1
03675   if ((Subtarget->hasSSE1())) {
03676     Chain = N.getOperand(0);
03677     N1 = N.getOperand(1);
03678     if (N.Val->getValueType(0) == MVT::f32 &&
03679         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03680       Select(Tmp0, Tmp0);
03681       Select(Tmp1, Tmp1);
03682       Select(Tmp2, Tmp2);
03683       Select(Tmp3, Tmp3);
03684       Select(Chain, Chain);
03685       ResNode = CurDAG->getTargetNode(X86::FsMOVAPSrm, MVT::f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03686       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03687       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03688       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03689       Result = SDOperand(ResNode, N.ResNo);
03690       return;
03691     }
03692   }
03693 
03694   // Pattern: (X86loadp:f64 addr:i32:$src)
03695   // Emits: (FsMOVAPDrm:f64 addr:i32:$src)
03696   // Pattern complexity = 12  cost = 1
03697   if ((Subtarget->hasSSE2())) {
03698     Chain = N.getOperand(0);
03699     N1 = N.getOperand(1);
03700     if (N.Val->getValueType(0) == MVT::f64 &&
03701         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
03702       Select(Tmp0, Tmp0);
03703       Select(Tmp1, Tmp1);
03704       Select(Tmp2, Tmp2);
03705       Select(Tmp3, Tmp3);
03706       Select(Chain, Chain);
03707       ResNode = CurDAG->getTargetNode(X86::FsMOVAPDrm, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
03708       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03709       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
03710       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
03711       Result = SDOperand(ResNode, N.ResNo);
03712       return;
03713     }
03714   }
03715   std::cerr << "Cannot yet select: ";
03716   N.Val->dump(CurDAG);
03717   std::cerr << '\n';
03718   abort();
03719 }
03720 
03721 void Select_X86pextrw(SDOperand &Result, SDOperand N) {
03722   SDOperand Chain00(0, 0);
03723   SDOperand N0(0, 0);
03724   SDOperand N00(0, 0);
03725   SDOperand N001(0, 0);
03726   SDOperand N1(0, 0);
03727   SDOperand Tmp0(0, 0);
03728   SDOperand Tmp1(0, 0);
03729   SDOperand Tmp2(0, 0);
03730   SDOperand Tmp3(0, 0);
03731   SDOperand Tmp4(0, 0);
03732   SDNode *ResNode;
03733   if ((Subtarget->hasSSE2())) {
03734     N0 = N.getOperand(0);
03735 
03736     // Pattern: (X86pextrw:i32 (bitconvert:v8i16 (load:v2i64 addr:i32:$src1)), (imm:i32):$src2)
03737     // Emits: (PEXTRWm:i32 addr:i32:$src1, (imm:i32):$src2)
03738     // Pattern complexity = 18  cost = 1
03739     if (N0.getOpcode() == ISD::BIT_CONVERT &&
03740         N0.hasOneUse() &&
03741         !CodeGenMap.count(N0.getValue(0))) {
03742       N00 = N0.getOperand(0);
03743       if (N00.getOpcode() == ISD::LOAD &&
03744           N00.hasOneUse() &&
03745           !CodeGenMap.count(N00.getValue(0))) {
03746         Chain00 = N00.getOperand(0);
03747         N001 = N00.getOperand(1);
03748         N1 = N.getOperand(1);
03749         if (N1.getOpcode() == ISD::Constant &&
03750             N.Val->getValueType(0) == MVT::i32 &&
03751             N0.Val->getValueType(0) == MVT::v8i16 &&
03752             N00.Val->getValueType(0) == MVT::v2i64 &&
03753             N1.Val->getValueType(0) == MVT::i32 &&
03754             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
03755           Select(Tmp0, Tmp0);
03756           Select(Tmp1, Tmp1);
03757           Select(Tmp2, Tmp2);
03758           Select(Tmp3, Tmp3);
03759           unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
03760           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
03761           Select(Chain00, Chain00);
03762           ResNode = CurDAG->getTargetNode(X86::PEXTRWm, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain00);
03763           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03764           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
03765           AddHandleReplacement(N00.Val, 1, ResNode, 1);
03766           Result = SDOperand(ResNode, N.ResNo);
03767           return;
03768         }
03769       }
03770     }
03771 
03772     // Pattern: (X86pextrw:i32 VR128:v8i16:$src1, (imm:i32):$src2)
03773     // Emits: (PEXTRWr:i32 VR128:v8i16:$src1, (imm:i32):$src2)
03774     // Pattern complexity = 4  cost = 1
03775     N1 = N.getOperand(1);
03776     if (N1.getOpcode() == ISD::Constant &&
03777         N.Val->getValueType(0) == MVT::i32 &&
03778         N0.Val->getValueType(0) == MVT::v8i16 &&
03779         N1.Val->getValueType(0) == MVT::i32) {
03780       Select(Tmp0, N0);
03781       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
03782       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
03783       if (N.Val->hasOneUse()) {
03784         Result = CurDAG->SelectNodeTo(N.Val, X86::PEXTRWr, MVT::i32, Tmp0, Tmp1);
03785       } else {
03786         ResNode = CurDAG->getTargetNode(X86::PEXTRWr, MVT::i32, Tmp0, Tmp1);
03787         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03788         Result = SDOperand(ResNode, 0);
03789       }
03790       return;
03791     }
03792   }
03793   std::cerr << "Cannot yet select: ";
03794   N.Val->dump(CurDAG);
03795   std::cerr << '\n';
03796   abort();
03797 }
03798 
03799 void Select_X86pinsrw(SDOperand &Result, SDOperand N) {
03800   SDOperand Chain10(0, 0);
03801   SDOperand N0(0, 0);
03802   SDOperand N1(0, 0);
03803   SDOperand N10(0, 0);
03804   SDOperand N101(0, 0);
03805   SDOperand N2(0, 0);
03806   SDOperand Tmp0(0, 0);
03807   SDOperand Tmp1(0, 0);
03808   SDOperand Tmp2(0, 0);
03809   SDOperand Tmp3(0, 0);
03810   SDOperand Tmp4(0, 0);
03811   SDOperand Tmp5(0, 0);
03812   SDNode *ResNode;
03813   if ((Subtarget->hasSSE2())) {
03814     N0 = N.getOperand(0);
03815     N1 = N.getOperand(1);
03816 
03817     // Pattern: (X86pinsrw:v8i16 VR128:v8i16:$src1, (anyext:i32 (load:i16 addr:i32:$src2)), (imm:i32):$src3)
03818     // Emits: (PINSRWm:v8i16 VR128:v8i16:$src1, addr:i32:$src2, (imm:i32):$src3)
03819     // Pattern complexity = 18  cost = 1
03820     if (N1.getOpcode() == ISD::ANY_EXTEND &&
03821         N1.hasOneUse() &&
03822         !CodeGenMap.count(N1.getValue(0))) {
03823       N10 = N1.getOperand(0);
03824       if (N10.getOpcode() == ISD::LOAD &&
03825           N10.hasOneUse() &&
03826           !CodeGenMap.count(N10.getValue(0))) {
03827         Chain10 = N10.getOperand(0);
03828         N101 = N10.getOperand(1);
03829         N2 = N.getOperand(2);
03830         if (N2.getOpcode() == ISD::Constant &&
03831             N.Val->getValueType(0) == MVT::v8i16 &&
03832             N0.Val->getValueType(0) == MVT::v8i16 &&
03833             N1.Val->getValueType(0) == MVT::i32 &&
03834             N10.Val->getValueType(0) == MVT::i16 &&
03835             N2.Val->getValueType(0) == MVT::i32 &&
03836             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
03837           Select(Tmp0, Tmp0);
03838           Select(Tmp1, Tmp1);
03839           Select(Tmp2, Tmp2);
03840           Select(Tmp3, Tmp3);
03841           Select(Tmp4, N0);
03842           unsigned Tmp5C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
03843           Tmp5 = CurDAG->getTargetConstant(Tmp5C, MVT::i32);
03844           Select(Chain10, Chain10);
03845           ResNode = CurDAG->getTargetNode(X86::PINSRWm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain10);
03846           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03847           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
03848           AddHandleReplacement(N10.Val, 1, ResNode, 1);
03849           Result = SDOperand(ResNode, N.ResNo);
03850           return;
03851         }
03852       }
03853     }
03854 
03855     // Pattern: (X86pinsrw:v8i16 VR128:v8i16:$src1, R32:i32:$src2, (imm:i32):$src3)
03856     // Emits: (PINSRWr:v8i16 VR128:v8i16:$src1, R32:i32:$src2, (imm:i32):$src3)
03857     // Pattern complexity = 4  cost = 1
03858     N2 = N.getOperand(2);
03859     if (N2.getOpcode() == ISD::Constant &&
03860         N.Val->getValueType(0) == MVT::v8i16 &&
03861         N0.Val->getValueType(0) == MVT::v8i16 &&
03862         N1.Val->getValueType(0) == MVT::i32 &&
03863         N2.Val->getValueType(0) == MVT::i32) {
03864       Select(Tmp0, N0);
03865       Select(Tmp1, N1);
03866       unsigned Tmp2C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
03867       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i32);
03868       if (N.Val->hasOneUse()) {
03869         Result = CurDAG->SelectNodeTo(N.Val, X86::PINSRWr, MVT::v8i16, Tmp0, Tmp1, Tmp2);
03870       } else {
03871         ResNode = CurDAG->getTargetNode(X86::PINSRWr, MVT::v8i16, Tmp0, Tmp1, Tmp2);
03872         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
03873         Result = SDOperand(ResNode, 0);
03874       }
03875       return;
03876     }
03877   }
03878   std::cerr << "Cannot yet select: ";
03879   N.Val->dump(CurDAG);
03880   std::cerr << '\n';
03881   abort();
03882 }
03883 
03884 void Select_X86rdtsc(SDOperand &Result, SDOperand N) {
03885   SDOperand Chain(0, 0);
03886   SDOperand InFlag(0, 0);
03887   SDNode *ResNode;
03888   Chain = N.getOperand(0);
03889   Select(Chain, Chain);
03890   ResNode = CurDAG->getTargetNode(X86::RDTSC, MVT::Other, MVT::Flag, Chain);
03891   InFlag = SDOperand(ResNode, 1);
03892   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03893   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03894   Result = SDOperand(ResNode, N.ResNo);
03895   return;
03896 }
03897 
03898 void Select_X86rep_movs(SDOperand &Result, SDOperand N) {
03899   SDOperand Chain(0, 0);
03900   SDOperand InFlag(0, 0);
03901   SDOperand N1(0, 0);
03902   SDNode *ResNode;
03903   Chain = N.getOperand(0);
03904   N1 = N.getOperand(1);
03905 
03906   // Pattern: (X86rep_movs:void i8:Other)
03907   // Emits: (REP_MOVSB:void)
03908   // Pattern complexity = 2  cost = 1
03909   if (cast<VTSDNode>(N1)->getVT() == MVT::i8) {
03910     Select(Chain, Chain);
03911     Select(InFlag, N.getOperand(2));
03912     ResNode = CurDAG->getTargetNode(X86::REP_MOVSB, MVT::Other, MVT::Flag, Chain, InFlag);
03913     InFlag = SDOperand(ResNode, 1);
03914     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03915     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03916     Result = SDOperand(ResNode, N.ResNo);
03917     return;
03918   }
03919 
03920   // Pattern: (X86rep_movs:void i16:Other)
03921   // Emits: (REP_MOVSW:void)
03922   // Pattern complexity = 2  cost = 1
03923   if (cast<VTSDNode>(N1)->getVT() == MVT::i16) {
03924     Select(Chain, Chain);
03925     Select(InFlag, N.getOperand(2));
03926     ResNode = CurDAG->getTargetNode(X86::REP_MOVSW, MVT::Other, MVT::Flag, Chain, InFlag);
03927     InFlag = SDOperand(ResNode, 1);
03928     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03929     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03930     Result = SDOperand(ResNode, N.ResNo);
03931     return;
03932   }
03933 
03934   // Pattern: (X86rep_movs:void i32:Other)
03935   // Emits: (REP_MOVSD:void)
03936   // Pattern complexity = 2  cost = 1
03937   if (cast<VTSDNode>(N1)->getVT() == MVT::i32) {
03938     Select(Chain, Chain);
03939     Select(InFlag, N.getOperand(2));
03940     ResNode = CurDAG->getTargetNode(X86::REP_MOVSD, MVT::Other, MVT::Flag, Chain, InFlag);
03941     InFlag = SDOperand(ResNode, 1);
03942     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03943     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03944     Result = SDOperand(ResNode, N.ResNo);
03945     return;
03946   }
03947   std::cerr << "Cannot yet select: ";
03948   N.Val->dump(CurDAG);
03949   std::cerr << '\n';
03950   abort();
03951 }
03952 
03953 void Select_X86rep_stos(SDOperand &Result, SDOperand N) {
03954   SDOperand Chain(0, 0);
03955   SDOperand InFlag(0, 0);
03956   SDOperand N1(0, 0);
03957   SDNode *ResNode;
03958   Chain = N.getOperand(0);
03959   N1 = N.getOperand(1);
03960 
03961   // Pattern: (X86rep_stos:void i8:Other)
03962   // Emits: (REP_STOSB:void)
03963   // Pattern complexity = 2  cost = 1
03964   if (cast<VTSDNode>(N1)->getVT() == MVT::i8) {
03965     Select(Chain, Chain);
03966     Select(InFlag, N.getOperand(2));
03967     ResNode = CurDAG->getTargetNode(X86::REP_STOSB, MVT::Other, MVT::Flag, Chain, InFlag);
03968     InFlag = SDOperand(ResNode, 1);
03969     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03970     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03971     Result = SDOperand(ResNode, N.ResNo);
03972     return;
03973   }
03974 
03975   // Pattern: (X86rep_stos:void i16:Other)
03976   // Emits: (REP_STOSW:void)
03977   // Pattern complexity = 2  cost = 1
03978   if (cast<VTSDNode>(N1)->getVT() == MVT::i16) {
03979     Select(Chain, Chain);
03980     Select(InFlag, N.getOperand(2));
03981     ResNode = CurDAG->getTargetNode(X86::REP_STOSW, MVT::Other, MVT::Flag, Chain, InFlag);
03982     InFlag = SDOperand(ResNode, 1);
03983     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03984     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03985     Result = SDOperand(ResNode, N.ResNo);
03986     return;
03987   }
03988 
03989   // Pattern: (X86rep_stos:void i32:Other)
03990   // Emits: (REP_STOSD:void)
03991   // Pattern complexity = 2  cost = 1
03992   if (cast<VTSDNode>(N1)->getVT() == MVT::i32) {
03993     Select(Chain, Chain);
03994     Select(InFlag, N.getOperand(2));
03995     ResNode = CurDAG->getTargetNode(X86::REP_STOSD, MVT::Other, MVT::Flag, Chain, InFlag);
03996     InFlag = SDOperand(ResNode, 1);
03997     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
03998     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
03999     Result = SDOperand(ResNode, N.ResNo);
04000     return;
04001   }
04002   std::cerr << "Cannot yet select: ";
04003   N.Val->dump(CurDAG);
04004   std::cerr << '\n';
04005   abort();
04006 }
04007 
04008 void Select_X86retflag(SDOperand &Result, SDOperand N) {
04009   SDOperand Chain(0, 0);
04010   SDOperand InFlag(0, 0);
04011   SDOperand N1(0, 0);
04012   SDOperand Tmp0(0, 0);
04013   SDNode *ResNode;
04014   Chain = N.getOperand(0);
04015   N1 = N.getOperand(1);
04016 
04017   // Pattern: (X86retflag:void 0:i16)
04018   // Emits: (RET:void)
04019   // Pattern complexity = 5  cost = 1
04020   if (isa<ConstantSDNode>(N1)) {
04021     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
04022     if (CN0 == 0) {
04023       bool HasOptInFlag = false;
04024       Select(Chain, Chain);
04025       if (N.getNumOperands() == 3) {
04026         Select(InFlag, N.getOperand(2));
04027         HasOptInFlag = true;
04028       }
04029       if (HasOptInFlag)
04030         ResNode = CurDAG->getTargetNode(X86::RET, MVT::Other, Chain, InFlag);
04031       else
04032         ResNode = CurDAG->getTargetNode(X86::RET, MVT::Other, Chain);
04033       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04034       Result = SDOperand(ResNode, N.ResNo);
04035       return;
04036     }
04037   }
04038 
04039   // Pattern: (X86retflag:void (imm:i16):$amt)
04040   // Emits: (RETI:void (imm:i16):$amt)
04041   // Pattern complexity = 4  cost = 1
04042   if (N1.getOpcode() == ISD::Constant) {
04043     bool HasOptInFlag = false;
04044     unsigned short Tmp0C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
04045     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i16);
04046     Select(Chain, Chain);
04047     if (N.getNumOperands() == 3) {
04048       Select(InFlag, N.getOperand(2));
04049       HasOptInFlag = true;
04050     }
04051     if (HasOptInFlag)
04052       ResNode = CurDAG->getTargetNode(X86::RETI, MVT::Other, Tmp0, Chain, InFlag);
04053     else
04054       ResNode = CurDAG->getTargetNode(X86::RETI, MVT::Other, Tmp0, Chain);
04055     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04056     Result = SDOperand(ResNode, N.ResNo);
04057     return;
04058   }
04059   std::cerr << "Cannot yet select: ";
04060   N.Val->dump(CurDAG);
04061   std::cerr << '\n';
04062   abort();
04063 }
04064 
04065 void Select_X86s2vec(SDOperand &Result, SDOperand N) {
04066   SDOperand N0(0, 0);
04067   SDOperand Tmp0(0, 0);
04068   SDNode *ResNode;
04069   if ((Subtarget->hasSSE2())) {
04070     N0 = N.getOperand(0);
04071 
04072     // Pattern: (X86s2vec:v8i16 R32:i32:$src)
04073     // Emits: (MOVDI2PDIrr:v16i8 R32:i32:$src)
04074     // Pattern complexity = 2  cost = 1
04075     if (N.Val->getValueType(0) == MVT::v8i16 &&
04076         N0.Val->getValueType(0) == MVT::i32) {
04077       Select(Tmp0, N0);
04078       if (N.Val->hasOneUse()) {
04079         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVDI2PDIrr, MVT::v16i8, Tmp0);
04080       } else {
04081         ResNode = CurDAG->getTargetNode(X86::MOVDI2PDIrr, MVT::v16i8, Tmp0);
04082         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04083         Result = SDOperand(ResNode, 0);
04084       }
04085       return;
04086     }
04087 
04088     // Pattern: (X86s2vec:v16i8 R32:i32:$src)
04089     // Emits: (MOVDI2PDIrr:v16i8 R32:i32:$src)
04090     // Pattern complexity = 2  cost = 1
04091     if (N.Val->getValueType(0) == MVT::v16i8 &&
04092         N0.Val->getValueType(0) == MVT::i32) {
04093       Select(Tmp0, N0);
04094       if (N.Val->hasOneUse()) {
04095         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVDI2PDIrr, MVT::v16i8, Tmp0);
04096       } else {
04097         ResNode = CurDAG->getTargetNode(X86::MOVDI2PDIrr, MVT::v16i8, Tmp0);
04098         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04099         Result = SDOperand(ResNode, 0);
04100       }
04101       return;
04102     }
04103   }
04104   std::cerr << "Cannot yet select: ";
04105   N.Val->dump(CurDAG);
04106   std::cerr << '\n';
04107   abort();
04108 }
04109 
04110 void Select_X86setcc(SDOperand &Result, SDOperand N) {
04111   SDOperand InFlag(0, 0);
04112   SDOperand N0(0, 0);
04113   SDNode *ResNode;
04114   N0 = N.getOperand(0);
04115   if (isa<ConstantSDNode>(N0)) {
04116     int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
04117 
04118     // Pattern: (X86setcc:i8 4:i8)
04119     // Emits: (SETEr:i8)
04120     // Pattern complexity = 5  cost = 1
04121     if (CN0 == 4) {
04122       Select(InFlag, N.getOperand(1));
04123       ResNode = CurDAG->getTargetNode(X86::SETEr, MVT::i8, MVT::Flag, InFlag);
04124       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04125       InFlag = SDOperand(ResNode, 1);
04126       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04127       Result = SDOperand(ResNode, N.ResNo);
04128       return;
04129     }
04130 
04131     // Pattern: (X86setcc:i8 9:i8)
04132     // Emits: (SETNEr:i8)
04133     // Pattern complexity = 5  cost = 1
04134     if (CN0 == 9) {
04135       Select(InFlag, N.getOperand(1));
04136       ResNode = CurDAG->getTargetNode(X86::SETNEr, MVT::i8, MVT::Flag, InFlag);
04137       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04138       InFlag = SDOperand(ResNode, 1);
04139       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04140       Result = SDOperand(ResNode, N.ResNo);
04141       return;
04142     }
04143 
04144     // Pattern: (X86setcc:i8 7:i8)
04145     // Emits: (SETLr:i8)
04146     // Pattern complexity = 5  cost = 1
04147     if (CN0 == 7) {
04148       Select(InFlag, N.getOperand(1));
04149       ResNode = CurDAG->getTargetNode(X86::SETLr, MVT::i8, MVT::Flag, InFlag);
04150       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04151       InFlag = SDOperand(ResNode, 1);
04152       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04153       Result = SDOperand(ResNode, N.ResNo);
04154       return;
04155     }
04156 
04157     // Pattern: (X86setcc:i8 6:i8)
04158     // Emits: (SETGEr:i8)
04159     // Pattern complexity = 5  cost = 1
04160     if (CN0 == 6) {
04161       Select(InFlag, N.getOperand(1));
04162       ResNode = CurDAG->getTargetNode(X86::SETGEr, MVT::i8, MVT::Flag, InFlag);
04163       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04164       InFlag = SDOperand(ResNode, 1);
04165       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04166       Result = SDOperand(ResNode, N.ResNo);
04167       return;
04168     }
04169 
04170     // Pattern: (X86setcc:i8 8:i8)
04171     // Emits: (SETLEr:i8)
04172     // Pattern complexity = 5  cost = 1
04173     if (CN0 == 8) {
04174       Select(InFlag, N.getOperand(1));
04175       ResNode = CurDAG->getTargetNode(X86::SETLEr, MVT::i8, MVT::Flag, InFlag);
04176       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04177       InFlag = SDOperand(ResNode, 1);
04178       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04179       Result = SDOperand(ResNode, N.ResNo);
04180       return;
04181     }
04182 
04183     // Pattern: (X86setcc:i8 5:i8)
04184     // Emits: (SETGr:i8)
04185     // Pattern complexity = 5  cost = 1
04186     if (CN0 == 5) {
04187       Select(InFlag, N.getOperand(1));
04188       ResNode = CurDAG->getTargetNode(X86::SETGr, MVT::i8, MVT::Flag, InFlag);
04189       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04190       InFlag = SDOperand(ResNode, 1);
04191       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04192       Result = SDOperand(ResNode, N.ResNo);
04193       return;
04194     }
04195 
04196     // Pattern: (X86setcc:i8 2:i8)
04197     // Emits: (SETBr:i8)
04198     // Pattern complexity = 5  cost = 1
04199     if (CN0 == 2) {
04200       Select(InFlag, N.getOperand(1));
04201       ResNode = CurDAG->getTargetNode(X86::SETBr, MVT::i8, MVT::Flag, InFlag);
04202       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04203       InFlag = SDOperand(ResNode, 1);
04204       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04205       Result = SDOperand(ResNode, N.ResNo);
04206       return;
04207     }
04208 
04209     // Pattern: (X86setcc:i8 1:i8)
04210     // Emits: (SETAEr:i8)
04211     // Pattern complexity = 5  cost = 1
04212     if (CN0 == 1) {
04213       Select(InFlag, N.getOperand(1));
04214       ResNode = CurDAG->getTargetNode(X86::SETAEr, MVT::i8, MVT::Flag, InFlag);
04215       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04216       InFlag = SDOperand(ResNode, 1);
04217       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04218       Result = SDOperand(ResNode, N.ResNo);
04219       return;
04220     }
04221 
04222     // Pattern: (X86setcc:i8 3:i8)
04223     // Emits: (SETBEr:i8)
04224     // Pattern complexity = 5  cost = 1
04225     if (CN0 == 3) {
04226       Select(InFlag, N.getOperand(1));
04227       ResNode = CurDAG->getTargetNode(X86::SETBEr, MVT::i8, MVT::Flag, InFlag);
04228       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04229       InFlag = SDOperand(ResNode, 1);
04230       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04231       Result = SDOperand(ResNode, N.ResNo);
04232       return;
04233     }
04234 
04235     // Pattern: (X86setcc:i8 0:i8)
04236     // Emits: (SETAr:i8)
04237     // Pattern complexity = 5  cost = 1
04238     if (CN0 == 0) {
04239       Select(InFlag, N.getOperand(1));
04240       ResNode = CurDAG->getTargetNode(X86::SETAr, MVT::i8, MVT::Flag, InFlag);
04241       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04242       InFlag = SDOperand(ResNode, 1);
04243       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04244       Result = SDOperand(ResNode, N.ResNo);
04245       return;
04246     }
04247 
04248     // Pattern: (X86setcc:i8 15:i8)
04249     // Emits: (SETSr:i8)
04250     // Pattern complexity = 5  cost = 1
04251     if (CN0 == 15) {
04252       Select(InFlag, N.getOperand(1));
04253       ResNode = CurDAG->getTargetNode(X86::SETSr, MVT::i8, MVT::Flag, InFlag);
04254       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04255       InFlag = SDOperand(ResNode, 1);
04256       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04257       Result = SDOperand(ResNode, N.ResNo);
04258       return;
04259     }
04260 
04261     // Pattern: (X86setcc:i8 12:i8)
04262     // Emits: (SETNSr:i8)
04263     // Pattern complexity = 5  cost = 1
04264     if (CN0 == 12) {
04265       Select(InFlag, N.getOperand(1));
04266       ResNode = CurDAG->getTargetNode(X86::SETNSr, MVT::i8, MVT::Flag, InFlag);
04267       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04268       InFlag = SDOperand(ResNode, 1);
04269       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04270       Result = SDOperand(ResNode, N.ResNo);
04271       return;
04272     }
04273 
04274     // Pattern: (X86setcc:i8 14:i8)
04275     // Emits: (SETPr:i8)
04276     // Pattern complexity = 5  cost = 1
04277     if (CN0 == 14) {
04278       Select(InFlag, N.getOperand(1));
04279       ResNode = CurDAG->getTargetNode(X86::SETPr, MVT::i8, MVT::Flag, InFlag);
04280       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04281       InFlag = SDOperand(ResNode, 1);
04282       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04283       Result = SDOperand(ResNode, N.ResNo);
04284       return;
04285     }
04286 
04287     // Pattern: (X86setcc:i8 11:i8)
04288     // Emits: (SETNPr:i8)
04289     // Pattern complexity = 5  cost = 1
04290     if (CN0 == 11) {
04291       Select(InFlag, N.getOperand(1));
04292       ResNode = CurDAG->getTargetNode(X86::SETNPr, MVT::i8, MVT::Flag, InFlag);
04293       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04294       InFlag = SDOperand(ResNode, 1);
04295       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
04296       Result = SDOperand(ResNode, N.ResNo);
04297       return;
04298     }
04299   }
04300   std::cerr << "Cannot yet select: ";
04301   N.Val->dump(CurDAG);
04302   std::cerr << '\n';
04303   abort();
04304 }
04305 
04306 void Select_X86shld(SDOperand &Result, SDOperand N) {
04307   SDOperand Chain(0, 0);
04308   SDOperand InFlag(0, 0);
04309   SDOperand N0(0, 0);
04310   SDOperand N1(0, 0);
04311   SDOperand N2(0, 0);
04312   SDOperand Tmp0(0, 0);
04313   SDOperand Tmp1(0, 0);
04314   SDOperand Tmp2(0, 0);
04315   SDNode *ResNode;
04316   N0 = N.getOperand(0);
04317   N1 = N.getOperand(1);
04318   N2 = N.getOperand(2);
04319   if (N2.getOpcode() == ISD::Constant) {
04320 
04321     // Pattern: (X86shld:i32 R32:i32:$src1, R32:i32:$src2, (imm:i8):$src3)
04322     // Emits: (SHLD32rri8:i32 R32:i32:$src1, R32:i32:$src2, (imm:i8):$src3)
04323     // Pattern complexity = 4  cost = 1
04324     if (N.Val->getValueType(0) == MVT::i32 &&
04325         N2.Val->getValueType(0) == MVT::i8) {
04326       Select(Tmp0, N0);
04327       Select(Tmp1, N1);
04328       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
04329       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
04330       if (N.Val->hasOneUse()) {
04331         Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD32rri8, MVT::i32, Tmp0, Tmp1, Tmp2);
04332       } else {
04333         ResNode = CurDAG->getTargetNode(X86::SHLD32rri8, MVT::i32, Tmp0, Tmp1, Tmp2);
04334         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04335         Result = SDOperand(ResNode, 0);
04336       }
04337       return;
04338     }
04339 
04340     // Pattern: (X86shld:i16 R16:i16:$src1, R16:i16:$src2, (imm:i8):$src3)
04341     // Emits: (SHLD16rri8:i16 R16:i16:$src1, R16:i16:$src2, (imm:i8):$src3)
04342     // Pattern complexity = 4  cost = 1
04343     if (N.Val->getValueType(0) == MVT::i16 &&
04344         N2.Val->getValueType(0) == MVT::i8) {
04345       Select(Tmp0, N0);
04346       Select(Tmp1, N1);
04347       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
04348       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
04349       if (N.Val->hasOneUse()) {
04350         Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD16rri8, MVT::i16, Tmp0, Tmp1, Tmp2);
04351       } else {
04352         ResNode = CurDAG->getTargetNode(X86::SHLD16rri8, MVT::i16, Tmp0, Tmp1, Tmp2);
04353         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04354         Result = SDOperand(ResNode, 0);
04355       }
04356       return;
04357     }
04358   }
04359 
04360   // Pattern: (X86shld:i32 R32:i32:$src1, R32:i32:$src2, CL:i8)
04361   // Emits: (SHLD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
04362   // Pattern complexity = 2  cost = 1
04363   if (N.Val->getValueType(0) == MVT::i32 &&
04364       N2.Val->getValueType(0) == MVT::i8) {
04365     Select(Tmp0, N0);
04366     Select(Tmp1, N1);
04367     Chain = CurDAG->getEntryNode();
04368     Select(N2, N2);
04369     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N2, InFlag).Val;
04370     Chain = SDOperand(ResNode, 0);
04371     InFlag = SDOperand(ResNode, 1);
04372     if (N.Val->hasOneUse()) {
04373       Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD32rrCL, MVT::i32, Tmp0, Tmp1, InFlag);
04374     } else {
04375       ResNode = CurDAG->getTargetNode(X86::SHLD32rrCL, MVT::i32, Tmp0, Tmp1, InFlag);
04376       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04377       Result = SDOperand(ResNode, 0);
04378     }
04379     return;
04380   }
04381 
04382   // Pattern: (X86shld:i16 R16:i16:$src1, R16:i16:$src2, CL:i8)
04383   // Emits: (SHLD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
04384   // Pattern complexity = 2  cost = 1
04385   if (N.Val->getValueType(0) == MVT::i16 &&
04386       N2.Val->getValueType(0) == MVT::i8) {
04387     Select(Tmp0, N0);
04388     Select(Tmp1, N1);
04389     Chain = CurDAG->getEntryNode();
04390     Select(N2, N2);
04391     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N2, InFlag).Val;
04392     Chain = SDOperand(ResNode, 0);
04393     InFlag = SDOperand(ResNode, 1);
04394     if (N.Val->hasOneUse()) {
04395       Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD16rrCL, MVT::i16, Tmp0, Tmp1, InFlag);
04396     } else {
04397       ResNode = CurDAG->getTargetNode(X86::SHLD16rrCL, MVT::i16, Tmp0, Tmp1, InFlag);
04398       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04399       Result = SDOperand(ResNode, 0);
04400     }
04401     return;
04402   }
04403   std::cerr << "Cannot yet select: ";
04404   N.Val->dump(CurDAG);
04405   std::cerr << '\n';
04406   abort();
04407 }
04408 
04409 void Select_X86shrd(SDOperand &Result, SDOperand N) {
04410   SDOperand Chain(0, 0);
04411   SDOperand InFlag(0, 0);
04412   SDOperand N0(0, 0);
04413   SDOperand N1(0, 0);
04414   SDOperand N2(0, 0);
04415   SDOperand Tmp0(0, 0);
04416   SDOperand Tmp1(0, 0);
04417   SDOperand Tmp2(0, 0);
04418   SDNode *ResNode;
04419   N0 = N.getOperand(0);
04420   N1 = N.getOperand(1);
04421   N2 = N.getOperand(2);
04422   if (N2.getOpcode() == ISD::Constant) {
04423 
04424     // Pattern: (X86shrd:i32 R32:i32:$src1, R32:i32:$src2, (imm:i8):$src3)
04425     // Emits: (SHRD32rri8:i32 R32:i32:$src1, R32:i32:$src2, (imm:i8):$src3)
04426     // Pattern complexity = 4  cost = 1
04427     if (N.Val->getValueType(0) == MVT::i32 &&
04428         N2.Val->getValueType(0) == MVT::i8) {
04429       Select(Tmp0, N0);
04430       Select(Tmp1, N1);
04431       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
04432       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
04433       if (N.Val->hasOneUse()) {
04434         Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD32rri8, MVT::i32, Tmp0, Tmp1, Tmp2);
04435       } else {
04436         ResNode = CurDAG->getTargetNode(X86::SHRD32rri8, MVT::i32, Tmp0, Tmp1, Tmp2);
04437         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04438         Result = SDOperand(ResNode, 0);
04439       }
04440       return;
04441     }
04442 
04443     // Pattern: (X86shrd:i16 R16:i16:$src1, R16:i16:$src2, (imm:i8):$src3)
04444     // Emits: (SHRD16rri8:i16 R16:i16:$src1, R16:i16:$src2, (imm:i8):$src3)
04445     // Pattern complexity = 4  cost = 1
04446     if (N.Val->getValueType(0) == MVT::i16 &&
04447         N2.Val->getValueType(0) == MVT::i8) {
04448       Select(Tmp0, N0);
04449       Select(Tmp1, N1);
04450       unsigned char Tmp2C = (unsigned char)cast<ConstantSDNode>(N2)->getValue();
04451       Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i8);
04452       if (N.Val->hasOneUse()) {
04453         Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD16rri8, MVT::i16, Tmp0, Tmp1, Tmp2);
04454       } else {
04455         ResNode = CurDAG->getTargetNode(X86::SHRD16rri8, MVT::i16, Tmp0, Tmp1, Tmp2);
04456         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04457         Result = SDOperand(ResNode, 0);
04458       }
04459       return;
04460     }
04461   }
04462 
04463   // Pattern: (X86shrd:i32 R32:i32:$src1, R32:i32:$src2, CL:i8)
04464   // Emits: (SHRD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
04465   // Pattern complexity = 2  cost = 1
04466   if (N.Val->getValueType(0) == MVT::i32 &&
04467       N2.Val->getValueType(0) == MVT::i8) {
04468     Select(Tmp0, N0);
04469     Select(Tmp1, N1);
04470     Chain = CurDAG->getEntryNode();
04471     Select(N2, N2);
04472     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N2, InFlag).Val;
04473     Chain = SDOperand(ResNode, 0);
04474     InFlag = SDOperand(ResNode, 1);
04475     if (N.Val->hasOneUse()) {
04476       Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD32rrCL, MVT::i32, Tmp0, Tmp1, InFlag);
04477     } else {
04478       ResNode = CurDAG->getTargetNode(X86::SHRD32rrCL, MVT::i32, Tmp0, Tmp1, InFlag);
04479       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04480       Result = SDOperand(ResNode, 0);
04481     }
04482     return;
04483   }
04484 
04485   // Pattern: (X86shrd:i16 R16:i16:$src1, R16:i16:$src2, CL:i8)
04486   // Emits: (SHRD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
04487   // Pattern complexity = 2  cost = 1
04488   if (N.Val->getValueType(0) == MVT::i16 &&
04489       N2.Val->getValueType(0) == MVT::i8) {
04490     Select(Tmp0, N0);
04491     Select(Tmp1, N1);
04492     Chain = CurDAG->getEntryNode();
04493     Select(N2, N2);
04494     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N2, InFlag).Val;
04495     Chain = SDOperand(ResNode, 0);
04496     InFlag = SDOperand(ResNode, 1);
04497     if (N.Val->hasOneUse()) {
04498       Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD16rrCL, MVT::i16, Tmp0, Tmp1, InFlag);
04499     } else {
04500       ResNode = CurDAG->getTargetNode(X86::SHRD16rrCL, MVT::i16, Tmp0, Tmp1, InFlag);
04501       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
04502       Result = SDOperand(ResNode, 0);
04503     }
04504     return;
04505   }
04506   std::cerr << "Cannot yet select: ";
04507   N.Val->dump(CurDAG);
04508   std::cerr << '\n';
04509   abort();
04510 }
04511 
04512 void Select_X86test(SDOperand &Result, SDOperand N) {
04513   SDOperand Chain0(0, 0);
04514   SDOperand Chain1(0, 0);
04515   SDOperand InFlag(0, 0);
04516   SDOperand N0(0, 0);
04517   SDOperand N01(0, 0);
04518   SDOperand N1(0, 0);
04519   SDOperand N11(0, 0);
04520   SDOperand Tmp0(0, 0);
04521   SDOperand Tmp1(0, 0);
04522   SDOperand Tmp2(0, 0);
04523   SDOperand Tmp3(0, 0);
04524   SDOperand Tmp4(0, 0);
04525   SDNode *ResNode;
04526   N0 = N.getOperand(0);
04527   if (N0.getOpcode() == ISD::LOAD &&
04528       N0.hasOneUse() &&
04529       !CodeGenMap.count(N0.getValue(0)) &&
04530       N.Val->isOnlyUse(N0.Val) &&
04531       !isNonImmUse(N.Val, N0.Val)) {
04532     Chain0 = N0.getOperand(0);
04533     N01 = N0.getOperand(1);
04534     N1 = N.getOperand(1);
04535     if (N1.getOpcode() == ISD::Constant) {
04536 
04537       // Pattern: (X86test:void (load:i8 addr:i32:$src1), (imm:i8):$src2)
04538       // Emits: (TEST8mi:void addr:i32:$src1, (imm:i8):$src2)
04539       // Pattern complexity = 16  cost = 1
04540       if (N0.Val->getValueType(0) == MVT::i8 &&
04541           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04542         Select(Tmp0, Tmp0);
04543         Select(Tmp1, Tmp1);
04544         Select(Tmp2, Tmp2);
04545         Select(Tmp3, Tmp3);
04546         unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
04547         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
04548         Select(Chain0, Chain0);
04549         ResNode = CurDAG->getTargetNode(X86::TEST8mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04550         InFlag = SDOperand(ResNode, 1);
04551         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04552         AddHandleReplacement(N0.Val, 1, ResNode, 0);
04553         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04554         Result = SDOperand(ResNode, N.ResNo+1);
04555         return;
04556       }
04557 
04558       // Pattern: (X86test:void (load:i16 addr:i32:$src1), (imm:i16):$src2)
04559       // Emits: (TEST16mi:void addr:i32:$src1, (imm:i16):$src2)
04560       // Pattern complexity = 16  cost = 1
04561       if (N0.Val->getValueType(0) == MVT::i16 &&
04562           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04563         Select(Tmp0, Tmp0);
04564         Select(Tmp1, Tmp1);
04565         Select(Tmp2, Tmp2);
04566         Select(Tmp3, Tmp3);
04567         unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
04568         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
04569         Select(Chain0, Chain0);
04570         ResNode = CurDAG->getTargetNode(X86::TEST16mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04571         InFlag = SDOperand(ResNode, 1);
04572         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04573         AddHandleReplacement(N0.Val, 1, ResNode, 0);
04574         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04575         Result = SDOperand(ResNode, N.ResNo+1);
04576         return;
04577       }
04578 
04579       // Pattern: (X86test:void (load:i32 addr:i32:$src1), (imm:i32):$src2)
04580       // Emits: (TEST32mi:void addr:i32:$src1, (imm:i32):$src2)
04581       // Pattern complexity = 16  cost = 1
04582       if (N0.Val->getValueType(0) == MVT::i32 &&
04583           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04584         Select(Tmp0, Tmp0);
04585         Select(Tmp1, Tmp1);
04586         Select(Tmp2, Tmp2);
04587         Select(Tmp3, Tmp3);
04588         unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
04589         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
04590         Select(Chain0, Chain0);
04591         ResNode = CurDAG->getTargetNode(X86::TEST32mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04592         InFlag = SDOperand(ResNode, 1);
04593         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04594         AddHandleReplacement(N0.Val, 1, ResNode, 0);
04595         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04596         Result = SDOperand(ResNode, N.ResNo+1);
04597         return;
04598       }
04599     }
04600 
04601     // Pattern: (X86test:void (load:i8 addr:i32:$src1), R8:i8:$src2)
04602     // Emits: (TEST8mr:void addr:i32:$src1, R8:i8:$src2)
04603     // Pattern complexity = 14  cost = 1
04604     if (N0.Val->getValueType(0) == MVT::i8 &&
04605         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04606       Select(Tmp0, Tmp0);
04607       Select(Tmp1, Tmp1);
04608       Select(Tmp2, Tmp2);
04609       Select(Tmp3, Tmp3);
04610       Select(Tmp4, N1);
04611       Select(Chain0, Chain0);
04612       ResNode = CurDAG->getTargetNode(X86::TEST8mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04613       InFlag = SDOperand(ResNode, 1);
04614       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04615       AddHandleReplacement(N0.Val, 1, ResNode, 0);
04616       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04617       Result = SDOperand(ResNode, N.ResNo+1);
04618       return;
04619     }
04620 
04621     // Pattern: (X86test:void (load:i16 addr:i32:$src1), R16:i16:$src2)
04622     // Emits: (TEST16mr:void addr:i32:$src1, R16:i16:$src2)
04623     // Pattern complexity = 14  cost = 1
04624     if (N0.Val->getValueType(0) == MVT::i16 &&
04625         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04626       Select(Tmp0, Tmp0);
04627       Select(Tmp1, Tmp1);
04628       Select(Tmp2, Tmp2);
04629       Select(Tmp3, Tmp3);
04630       Select(Tmp4, N1);
04631       Select(Chain0, Chain0);
04632       ResNode = CurDAG->getTargetNode(X86::TEST16mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04633       InFlag = SDOperand(ResNode, 1);
04634       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04635       AddHandleReplacement(N0.Val, 1, ResNode, 0);
04636       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04637       Result = SDOperand(ResNode, N.ResNo+1);
04638       return;
04639     }
04640 
04641     // Pattern: (X86test:void (load:i32 addr:i32:$src1), R32:i32:$src2)
04642     // Emits: (TEST32mr:void addr:i32:$src1, R32:i32:$src2)
04643     // Pattern complexity = 14  cost = 1
04644     if (N0.Val->getValueType(0) == MVT::i32 &&
04645         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04646       Select(Tmp0, Tmp0);
04647       Select(Tmp1, Tmp1);
04648       Select(Tmp2, Tmp2);
04649       Select(Tmp3, Tmp3);
04650       Select(Tmp4, N1);
04651       Select(Chain0, Chain0);
04652       ResNode = CurDAG->getTargetNode(X86::TEST32mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
04653       InFlag = SDOperand(ResNode, 1);
04654       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 0);
04655       AddHandleReplacement(N0.Val, 1, ResNode, 0);
04656       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04657       Result = SDOperand(ResNode, N.ResNo+1);
04658       return;
04659     }
04660   }
04661   N1 = N.getOperand(1);
04662   if (N1.getOpcode() == ISD::LOAD &&
04663       N1.hasOneUse() &&
04664       !CodeGenMap.count(N1.getValue(0)) &&
04665       N.Val->isOnlyUse(N1.Val) &&
04666       !isNonImmUse(N.Val, N1.Val)) {
04667     Chain1 = N1.getOperand(0);
04668     N11 = N1.getOperand(1);
04669 
04670     // Pattern: (X86test:void R8:i8:$src1, (load:i8 addr:i32:$src2))
04671     // Emits: (TEST8rm:void R8:i8:$src1, addr:i32:$src2)
04672     // Pattern complexity = 14  cost = 1
04673     if (N0.Val->getValueType(0) == MVT::i8 &&
04674         SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
04675       Select(Tmp0, Tmp0);
04676       Select(Tmp1, Tmp1);
04677       Select(Tmp2, Tmp2);
04678       Select(Tmp3, Tmp3);
04679       Select(Tmp4, N0);
04680       Select(Chain1, Chain1);
04681       ResNode = CurDAG->getTargetNode(X86::TEST8rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
04682       InFlag = SDOperand(ResNode, 1);
04683       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
04684       AddHandleReplacement(N1.Val, 1, ResNode, 0);
04685       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04686       Result = SDOperand(ResNode, N.ResNo+1);
04687       return;
04688     }
04689 
04690     // Pattern: (X86test:void R16:i16:$src1, (load:i16 addr:i32:$src2))
04691     // Emits: (TEST16rm:void R16:i16:$src1, addr:i32:$src2)
04692     // Pattern complexity = 14  cost = 1
04693     if (N0.Val->getValueType(0) == MVT::i16 &&
04694         SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
04695       Select(Tmp0, Tmp0);
04696       Select(Tmp1, Tmp1);
04697       Select(Tmp2, Tmp2);
04698       Select(Tmp3, Tmp3);
04699       Select(Tmp4, N0);
04700       Select(Chain1, Chain1);
04701       ResNode = CurDAG->getTargetNode(X86::TEST16rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
04702       InFlag = SDOperand(ResNode, 1);
04703       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
04704       AddHandleReplacement(N1.Val, 1, ResNode, 0);
04705       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04706       Result = SDOperand(ResNode, N.ResNo+1);
04707       return;
04708     }
04709 
04710     // Pattern: (X86test:void R32:i32:$src1, (load:i32 addr:i32:$src2))
04711     // Emits: (TEST32rm:void R32:i32:$src1, addr:i32:$src2)
04712     // Pattern complexity = 14  cost = 1
04713     if (N0.Val->getValueType(0) == MVT::i32 &&
04714         SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
04715       Select(Tmp0, Tmp0);
04716       Select(Tmp1, Tmp1);
04717       Select(Tmp2, Tmp2);
04718       Select(Tmp3, Tmp3);
04719       Select(Tmp4, N0);
04720       Select(Chain1, Chain1);
04721       ResNode = CurDAG->getTargetNode(X86::TEST32rm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
04722       InFlag = SDOperand(ResNode, 1);
04723       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
04724       AddHandleReplacement(N1.Val, 1, ResNode, 0);
04725       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04726       Result = SDOperand(ResNode, N.ResNo+1);
04727       return;
04728     }
04729   }
04730   if (N1.getOpcode() == ISD::Constant) {
04731 
04732     // Pattern: (X86test:void R8:i8:$src1, (imm:i8):$src2)
04733     // Emits: (TEST8ri:void R8:i8:$src1, (imm:i8):$src2)
04734     // Pattern complexity = 4  cost = 1
04735     if (N0.Val->getValueType(0) == MVT::i8) {
04736       Select(Tmp0, N0);
04737       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
04738       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
04739       ResNode = CurDAG->getTargetNode(X86::TEST8ri, MVT::Flag, Tmp0, Tmp1);
04740       InFlag = SDOperand(ResNode, 0);
04741       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04742       Result = SDOperand(ResNode, N.ResNo);
04743       return;
04744     }
04745 
04746     // Pattern: (X86test:void R16:i16:$src1, (imm:i16):$src2)
04747     // Emits: (TEST16ri:void R16:i16:$src1, (imm:i16):$src2)
04748     // Pattern complexity = 4  cost = 1
04749     if (N0.Val->getValueType(0) == MVT::i16) {
04750       Select(Tmp0, N0);
04751       unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
04752       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
04753       ResNode = CurDAG->getTargetNode(X86::TEST16ri, MVT::Flag, Tmp0, Tmp1);
04754       InFlag = SDOperand(ResNode, 0);
04755       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04756       Result = SDOperand(ResNode, N.ResNo);
04757       return;
04758     }
04759 
04760     // Pattern: (X86test:void R32:i32:$src1, (imm:i32):$src2)
04761     // Emits: (TEST32ri:void R32:i32:$src1, (imm:i32):$src2)
04762     // Pattern complexity = 4  cost = 1
04763     if (N0.Val->getValueType(0) == MVT::i32) {
04764       Select(Tmp0, N0);
04765       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
04766       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
04767       ResNode = CurDAG->getTargetNode(X86::TEST32ri, MVT::Flag, Tmp0, Tmp1);
04768       InFlag = SDOperand(ResNode, 0);
04769       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04770       Result = SDOperand(ResNode, N.ResNo);
04771       return;
04772     }
04773   }
04774 
04775   // Pattern: (X86test:void R8:i8:$src1, R8:i8:$src2)
04776   // Emits: (TEST8rr:void R8:i8:$src1, R8:i8:$src2)
04777   // Pattern complexity = 2  cost = 1
04778   if (N0.Val->getValueType(0) == MVT::i8) {
04779     Select(Tmp0, N0);
04780     Select(Tmp1, N1);
04781     ResNode = CurDAG->getTargetNode(X86::TEST8rr, MVT::Flag, Tmp0, Tmp1);
04782     InFlag = SDOperand(ResNode, 0);
04783     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04784     Result = SDOperand(ResNode, N.ResNo);
04785     return;
04786   }
04787 
04788   // Pattern: (X86test:void R16:i16:$src1, R16:i16:$src2)
04789   // Emits: (TEST16rr:void R16:i16:$src1, R16:i16:$src2)
04790   // Pattern complexity = 2  cost = 1
04791   if (N0.Val->getValueType(0) == MVT::i16) {
04792     Select(Tmp0, N0);
04793     Select(Tmp1, N1);
04794     ResNode = CurDAG->getTargetNode(X86::TEST16rr, MVT::Flag, Tmp0, Tmp1);
04795     InFlag = SDOperand(ResNode, 0);
04796     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04797     Result = SDOperand(ResNode, N.ResNo);
04798     return;
04799   }
04800 
04801   // Pattern: (X86test:void R32:i32:$src1, R32:i32:$src2)
04802   // Emits: (TEST32rr:void R32:i32:$src1, R32:i32:$src2)
04803   // Pattern complexity = 2  cost = 1
04804   if (N0.Val->getValueType(0) == MVT::i32) {
04805     Select(Tmp0, N0);
04806     Select(Tmp1, N1);
04807     ResNode = CurDAG->getTargetNode(X86::TEST32rr, MVT::Flag, Tmp0, Tmp1);
04808     InFlag = SDOperand(ResNode, 0);
04809     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04810     Result = SDOperand(ResNode, N.ResNo);
04811     return;
04812   }
04813   std::cerr << "Cannot yet select: ";
04814   N.Val->dump(CurDAG);
04815   std::cerr << '\n';
04816   abort();
04817 }
04818 
04819 void Select_X86ucomi(SDOperand &Result, SDOperand N) {
04820   SDOperand Chain1(0, 0);
04821   SDOperand InFlag(0, 0);
04822   SDOperand N0(0, 0);
04823   SDOperand N1(0, 0);
04824   SDOperand N11(0, 0);
04825   SDOperand Tmp0(0, 0);
04826   SDOperand Tmp1(0, 0);
04827   SDOperand Tmp2(0, 0);
04828   SDOperand Tmp3(0, 0);
04829   SDOperand Tmp4(0, 0);
04830   SDNode *ResNode;
04831 
04832   // Pattern: (X86ucomi:void VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
04833   // Emits: (Int_UCOMISSrm:void VR128:v4f32:$src1, addr:i32:$src2)
04834   // Pattern complexity = 14  cost = 1
04835   if ((Subtarget->hasSSE1())) {
04836     N0 = N.getOperand(0);
04837     N1 = N.getOperand(1);
04838     if (N1.getOpcode() == ISD::LOAD &&
04839         N1.hasOneUse() &&
04840         !CodeGenMap.count(N1.getValue(0)) &&
04841         N.Val->isOnlyUse(N1.Val) &&
04842         !isNonImmUse(N.Val, N1.Val)) {
04843       Chain1 = N1.getOperand(0);
04844       N11 = N1.getOperand(1);
04845       if (N0.Val->getValueType(0) == MVT::v4f32 &&
04846           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
04847         Select(Tmp0, Tmp0);
04848         Select(Tmp1, Tmp1);
04849         Select(Tmp2, Tmp2);
04850         Select(Tmp3, Tmp3);
04851         Select(Tmp4, N0);
04852         Select(Chain1, Chain1);
04853         ResNode = CurDAG->getTargetNode(X86::Int_UCOMISSrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
04854         InFlag = SDOperand(ResNode, 1);
04855         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
04856         AddHandleReplacement(N1.Val, 1, ResNode, 0);
04857         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04858         Result = SDOperand(ResNode, N.ResNo+1);
04859         return;
04860       }
04861     }
04862   }
04863 
04864   // Pattern: (X86ucomi:void VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
04865   // Emits: (Int_UCOMISDrm:void VR128:v2f64:$src1, addr:i32:$src2)
04866   // Pattern complexity = 14  cost = 1
04867   if ((Subtarget->hasSSE2())) {
04868     N0 = N.getOperand(0);
04869     N1 = N.getOperand(1);
04870     if (N1.getOpcode() == ISD::LOAD &&
04871         N1.hasOneUse() &&
04872         !CodeGenMap.count(N1.getValue(0)) &&
04873         N.Val->isOnlyUse(N1.Val) &&
04874         !isNonImmUse(N.Val, N1.Val)) {
04875       Chain1 = N1.getOperand(0);
04876       N11 = N1.getOperand(1);
04877       if (N0.Val->getValueType(0) == MVT::v2f64 &&
04878           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
04879         Select(Tmp0, Tmp0);
04880         Select(Tmp1, Tmp1);
04881         Select(Tmp2, Tmp2);
04882         Select(Tmp3, Tmp3);
04883         Select(Tmp4, N0);
04884         Select(Chain1, Chain1);
04885         ResNode = CurDAG->getTargetNode(X86::Int_UCOMISDrm, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
04886         InFlag = SDOperand(ResNode, 1);
04887         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 0);
04888         AddHandleReplacement(N1.Val, 1, ResNode, 0);
04889         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04890         Result = SDOperand(ResNode, N.ResNo+1);
04891         return;
04892       }
04893     }
04894   }
04895 
04896   // Pattern: (X86ucomi:void VR128:v4f32:$src1, VR128:v4f32:$src2)
04897   // Emits: (Int_UCOMISSrr:void VR128:v4f32:$src1, VR128:v4f32:$src2)
04898   // Pattern complexity = 2  cost = 1
04899   if ((Subtarget->hasSSE1())) {
04900     N0 = N.getOperand(0);
04901     N1 = N.getOperand(1);
04902     if (N0.Val->getValueType(0) == MVT::v4f32) {
04903       Select(Tmp0, N0);
04904       Select(Tmp1, N1);
04905       ResNode = CurDAG->getTargetNode(X86::Int_UCOMISSrr, MVT::Flag, Tmp0, Tmp1);
04906       InFlag = SDOperand(ResNode, 0);
04907       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04908       Result = SDOperand(ResNode, N.ResNo);
04909       return;
04910     }
04911   }
04912 
04913   // Pattern: (X86ucomi:void VR128:v2f64:$src1, VR128:v2f64:$src2)
04914   // Emits: (Int_UCOMISDrr:void VR128:v2f64:$src1, VR128:v2f64:$src2)
04915   // Pattern complexity = 2  cost = 1
04916   if ((Subtarget->hasSSE2())) {
04917     N0 = N.getOperand(0);
04918     N1 = N.getOperand(1);
04919     if (N0.Val->getValueType(0) == MVT::v2f64) {
04920       Select(Tmp0, N0);
04921       Select(Tmp1, N1);
04922       ResNode = CurDAG->getTargetNode(X86::Int_UCOMISDrr, MVT::Flag, Tmp0, Tmp1);
04923       InFlag = SDOperand(ResNode, 0);
04924       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, InFlag.Val, InFlag.ResNo);
04925       Result = SDOperand(ResNode, N.ResNo);
04926       return;
04927     }
04928   }
04929   std::cerr << "Cannot yet select: ";
04930   N.Val->dump(CurDAG);
04931   std::cerr << '\n';
04932   abort();
04933 }
04934 
04935 void Select_X86zexts2vec(SDOperand &Result, SDOperand N) {
04936   SDOperand Chain0(0, 0);
04937   SDOperand N0(0, 0);
04938   SDOperand N01(0, 0);
04939   SDOperand Tmp0(0, 0);
04940   SDOperand Tmp1(0, 0);
04941   SDOperand Tmp2(0, 0);
04942   SDOperand Tmp3(0, 0);
04943   SDNode *ResNode;
04944 
04945   // Pattern: (X86zexts2vec:v4f32 (load:f32 addr:i32:$src))
04946   // Emits: (MOVZSS2PSrm:v4f32 addr:i32:$src)
04947   // Pattern complexity = 14  cost = 1
04948   if ((Subtarget->hasSSE1())) {
04949     N0 = N.getOperand(0);
04950     if (N0.getOpcode() == ISD::LOAD &&
04951         N0.hasOneUse() &&
04952         !CodeGenMap.count(N0.getValue(0))) {
04953       Chain0 = N0.getOperand(0);
04954       N01 = N0.getOperand(1);
04955       if (N.Val->getValueType(0) == MVT::v4f32 &&
04956           N0.Val->getValueType(0) == MVT::f32 &&
04957           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04958         Select(Tmp0, Tmp0);
04959         Select(Tmp1, Tmp1);
04960         Select(Tmp2, Tmp2);
04961         Select(Tmp3, Tmp3);
04962         Select(Chain0, Chain0);
04963         ResNode = CurDAG->getTargetNode(X86::MOVZSS2PSrm, MVT::v4f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
04964         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04965         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
04966         AddHandleReplacement(N0.Val, 1, ResNode, 1);
04967         Result = SDOperand(ResNode, N.ResNo);
04968         return;
04969       }
04970     }
04971   }
04972   if ((Subtarget->hasSSE2())) {
04973     N0 = N.getOperand(0);
04974     if (N0.getOpcode() == ISD::LOAD &&
04975         N0.hasOneUse() &&
04976         !CodeGenMap.count(N0.getValue(0))) {
04977       Chain0 = N0.getOperand(0);
04978       N01 = N0.getOperand(1);
04979 
04980       // Pattern: (X86zexts2vec:v2f64 (load:f64 addr:i32:$src))
04981       // Emits: (MOVZSD2PDrm:v2f64 addr:i32:$src)
04982       // Pattern complexity = 14  cost = 1
04983       if (N.Val->getValueType(0) == MVT::v2f64 &&
04984           N0.Val->getValueType(0) == MVT::f64 &&
04985           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
04986         Select(Tmp0, Tmp0);
04987         Select(Tmp1, Tmp1);
04988         Select(Tmp2, Tmp2);
04989         Select(Tmp3, Tmp3);
04990         Select(Chain0, Chain0);
04991         ResNode = CurDAG->getTargetNode(X86::MOVZSD2PDrm, MVT::v2f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
04992         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
04993         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
04994         AddHandleReplacement(N0.Val, 1, ResNode, 1);
04995         Result = SDOperand(ResNode, N.ResNo);
04996         return;
04997       }
04998 
04999       // Pattern: (X86zexts2vec:v4i32 (load:i32 addr:i32:$src))
05000       // Emits: (MOVZDI2PDIrm:v4i32 addr:i32:$src)
05001       // Pattern complexity = 14  cost = 1
05002       if (N.Val->getValueType(0) == MVT::v4i32 &&
05003           N0.Val->getValueType(0) == MVT::i32 &&
05004           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05005         Select(Tmp0, Tmp0);
05006         Select(Tmp1, Tmp1);
05007         Select(Tmp2, Tmp2);
05008         Select(Tmp3, Tmp3);
05009         Select(Chain0, Chain0);
05010         ResNode = CurDAG->getTargetNode(X86::MOVZDI2PDIrm, MVT::v4i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05011         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05012         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05013         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05014         Result = SDOperand(ResNode, N.ResNo);
05015         return;
05016       }
05017     }
05018 
05019     // Pattern: (X86zexts2vec:v2f64 FR64:f64:$src)
05020     // Emits: (MOVLSD2PDrr:v16i8 (V_SET0_PD:v16i8), FR64:f64:$src)
05021     // Pattern complexity = 2  cost = 2
05022     if (N.Val->getValueType(0) == MVT::v2f64 &&
05023         N0.Val->getValueType(0) == MVT::f64) {
05024       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::V_SET0_PD, MVT::v16i8), 0);
05025       Select(Tmp1, N0);
05026       if (N.Val->hasOneUse()) {
05027         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLSD2PDrr, MVT::v16i8, Tmp0, Tmp1);
05028       } else {
05029         ResNode = CurDAG->getTargetNode(X86::MOVLSD2PDrr, MVT::v16i8, Tmp0, Tmp1);
05030         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05031         Result = SDOperand(ResNode, 0);
05032       }
05033       return;
05034     }
05035 
05036     // Pattern: (X86zexts2vec:v4f32 FR32:f32:$src)
05037     // Emits: (MOVLSS2PSrr:v16i8 (V_SET0_PS:v16i8), FR32:f32:$src)
05038     // Pattern complexity = 2  cost = 2
05039     if (N.Val->getValueType(0) == MVT::v4f32 &&
05040         N0.Val->getValueType(0) == MVT::f32) {
05041       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::V_SET0_PS, MVT::v16i8), 0);
05042       Select(Tmp1, N0);
05043       if (N.Val->hasOneUse()) {
05044         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLSS2PSrr, MVT::v16i8, Tmp0, Tmp1);
05045       } else {
05046         ResNode = CurDAG->getTargetNode(X86::MOVLSS2PSrr, MVT::v16i8, Tmp0, Tmp1);
05047         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05048         Result = SDOperand(ResNode, 0);
05049       }
05050       return;
05051     }
05052 
05053     // Pattern: (X86zexts2vec:v4i32 R32:i32:$src)
05054     // Emits: (MOVLDI2PDIrr:v16i8 (V_SET0_PI:v16i8), R32:i32:$src)
05055     // Pattern complexity = 2  cost = 2
05056     if (N.Val->getValueType(0) == MVT::v4i32 &&
05057         N0.Val->getValueType(0) == MVT::i32) {
05058       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v16i8), 0);
05059       Select(Tmp1, N0);
05060       if (N.Val->hasOneUse()) {
05061         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp1);
05062       } else {
05063         ResNode = CurDAG->getTargetNode(X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp1);
05064         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05065         Result = SDOperand(ResNode, 0);
05066       }
05067       return;
05068     }
05069 
05070     // Pattern: (X86zexts2vec:v8i16 R16:i16:$src)
05071     // Emits: (MOVLDI2PDIrr:v16i8 (V_SET0_PI:v16i8), (MOVZX32rr16:i32 R16:i16:$src))
05072     // Pattern complexity = 2  cost = 3
05073     if (N.Val->getValueType(0) == MVT::v8i16 &&
05074         N0.Val->getValueType(0) == MVT::i16) {
05075       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v16i8), 0);
05076       Select(Tmp1, N0);
05077       Tmp2 = SDOperand(CurDAG->getTargetNode(X86::MOVZX32rr16, MVT::i32, Tmp1), 0);
05078       if (N.Val->hasOneUse()) {
05079         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp2);
05080       } else {
05081         ResNode = CurDAG->getTargetNode(X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp2);
05082         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05083         Result = SDOperand(ResNode, 0);
05084       }
05085       return;
05086     }
05087 
05088     // Pattern: (X86zexts2vec:v16i8 R8:i8:$src)
05089     // Emits: (MOVLDI2PDIrr:v16i8 (V_SET0_PI:v16i8), (MOVZX32rr8:i32 R8:i8:$src))
05090     // Pattern complexity = 2  cost = 3
05091     if (N.Val->getValueType(0) == MVT::v16i8 &&
05092         N0.Val->getValueType(0) == MVT::i8) {
05093       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v16i8), 0);
05094       Select(Tmp1, N0);
05095       Tmp2 = SDOperand(CurDAG->getTargetNode(X86::MOVZX32rr8, MVT::i32, Tmp1), 0);
05096       if (N.Val->hasOneUse()) {
05097         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp2);
05098       } else {
05099         ResNode = CurDAG->getTargetNode(X86::MOVLDI2PDIrr, MVT::v16i8, Tmp0, Tmp2);
05100         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05101         Result = SDOperand(ResNode, 0);
05102       }
05103       return;
05104     }
05105   }
05106   std::cerr << "Cannot yet select: ";
05107   N.Val->dump(CurDAG);
05108   std::cerr << '\n';
05109   abort();
05110 }
05111 
05112 void Select_add(SDOperand &Result, SDOperand N) {
05113   SDOperand Chain0(0, 0);
05114   SDOperand Chain1(0, 0);
05115   SDOperand N0(0, 0);
05116   SDOperand N00(0, 0);
05117   SDOperand N01(0, 0);
05118   SDOperand N1(0, 0);
05119   SDOperand N10(0, 0);
05120   SDOperand N11(0, 0);
05121   SDOperand Tmp0(0, 0);
05122   SDOperand Tmp1(0, 0);
05123   SDOperand Tmp2(0, 0);
05124   SDOperand Tmp3(0, 0);
05125   SDOperand Tmp4(0, 0);
05126   SDNode *ResNode;
05127   {
05128     N0 = N.getOperand(0);
05129     N1 = N.getOperand(1);
05130     if (N1.getOpcode() == ISD::LOAD &&
05131         N1.hasOneUse() &&
05132         !CodeGenMap.count(N1.getValue(0)) &&
05133         !isNonImmUse(N.Val, N1.Val)) {
05134       Chain1 = N1.getOperand(0);
05135       N11 = N1.getOperand(1);
05136 
05137       // Pattern: (add:i8 R8:i8:$src1, (load:i8 addr:i32:$src2))
05138       // Emits: (ADD8rm:i8 R8:i8:$src1, addr:i32:$src2)
05139       // Pattern complexity = 14  cost = 1
05140       if (N.Val->getValueType(0) == MVT::i8 &&
05141           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05142         Select(Tmp0, Tmp0);
05143         Select(Tmp1, Tmp1);
05144         Select(Tmp2, Tmp2);
05145         Select(Tmp3, Tmp3);
05146         Select(Tmp4, N0);
05147         Select(Chain1, Chain1);
05148         ResNode = CurDAG->getTargetNode(X86::ADD8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05149         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05150         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05151         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05152         Result = SDOperand(ResNode, N.ResNo);
05153         return;
05154       }
05155 
05156       // Pattern: (add:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
05157       // Emits: (ADD16rm:i16 R16:i16:$src1, addr:i32:$src2)
05158       // Pattern complexity = 14  cost = 1
05159       if (N.Val->getValueType(0) == MVT::i16 &&
05160           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05161         Select(Tmp0, Tmp0);
05162         Select(Tmp1, Tmp1);
05163         Select(Tmp2, Tmp2);
05164         Select(Tmp3, Tmp3);
05165         Select(Tmp4, N0);
05166         Select(Chain1, Chain1);
05167         ResNode = CurDAG->getTargetNode(X86::ADD16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05168         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05169         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05170         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05171         Result = SDOperand(ResNode, N.ResNo);
05172         return;
05173       }
05174 
05175       // Pattern: (add:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
05176       // Emits: (ADD32rm:i32 R32:i32:$src1, addr:i32:$src2)
05177       // Pattern complexity = 14  cost = 1
05178       if (N.Val->getValueType(0) == MVT::i32 &&
05179           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05180         Select(Tmp0, Tmp0);
05181         Select(Tmp1, Tmp1);
05182         Select(Tmp2, Tmp2);
05183         Select(Tmp3, Tmp3);
05184         Select(Tmp4, N0);
05185         Select(Chain1, Chain1);
05186         ResNode = CurDAG->getTargetNode(X86::ADD32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05187         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05188         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05189         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05190         Result = SDOperand(ResNode, N.ResNo);
05191         return;
05192       }
05193     }
05194   }
05195   if ((Subtarget->hasSSE2())) {
05196     N0 = N.getOperand(0);
05197     N1 = N.getOperand(1);
05198     if (N1.getOpcode() == ISD::LOAD &&
05199         N1.hasOneUse() &&
05200         !CodeGenMap.count(N1.getValue(0)) &&
05201         !isNonImmUse(N.Val, N1.Val)) {
05202       Chain1 = N1.getOperand(0);
05203       N11 = N1.getOperand(1);
05204 
05205       // Pattern: (add:v16i8 VR128:v16i8:$src1, (load:v16i8 addr:i32:$src2))
05206       // Emits: (PADDBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
05207       // Pattern complexity = 14  cost = 1
05208       if (N.Val->getValueType(0) == MVT::v16i8 &&
05209           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05210         Select(Tmp0, Tmp0);
05211         Select(Tmp1, Tmp1);
05212         Select(Tmp2, Tmp2);
05213         Select(Tmp3, Tmp3);
05214         Select(Tmp4, N0);
05215         Select(Chain1, Chain1);
05216         ResNode = CurDAG->getTargetNode(X86::PADDBrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05217         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05218         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05219         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05220         Result = SDOperand(ResNode, N.ResNo);
05221         return;
05222       }
05223 
05224       // Pattern: (add:v8i16 VR128:v8i16:$src1, (load:v8i16 addr:i32:$src2))
05225       // Emits: (PADDWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
05226       // Pattern complexity = 14  cost = 1
05227       if (N.Val->getValueType(0) == MVT::v8i16 &&
05228           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05229         Select(Tmp0, Tmp0);
05230         Select(Tmp1, Tmp1);
05231         Select(Tmp2, Tmp2);
05232         Select(Tmp3, Tmp3);
05233         Select(Tmp4, N0);
05234         Select(Chain1, Chain1);
05235         ResNode = CurDAG->getTargetNode(X86::PADDWrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05236         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05237         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05238         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05239         Result = SDOperand(ResNode, N.ResNo);
05240         return;
05241       }
05242 
05243       // Pattern: (add:v4i32 VR128:v4i32:$src1, (load:v4i32 addr:i32:$src2))
05244       // Emits: (PADDDrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
05245       // Pattern complexity = 14  cost = 1
05246       if (N.Val->getValueType(0) == MVT::v4i32 &&
05247           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05248         Select(Tmp0, Tmp0);
05249         Select(Tmp1, Tmp1);
05250         Select(Tmp2, Tmp2);
05251         Select(Tmp3, Tmp3);
05252         Select(Tmp4, N0);
05253         Select(Chain1, Chain1);
05254         ResNode = CurDAG->getTargetNode(X86::PADDDrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05255         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05256         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05257         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05258         Result = SDOperand(ResNode, N.ResNo);
05259         return;
05260       }
05261 
05262       // Pattern: (add:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2))
05263       // Emits: (PADDQrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
05264       // Pattern complexity = 14  cost = 1
05265       if (N.Val->getValueType(0) == MVT::v2i64 &&
05266           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
05267         Select(Tmp0, Tmp0);
05268         Select(Tmp1, Tmp1);
05269         Select(Tmp2, Tmp2);
05270         Select(Tmp3, Tmp3);
05271         Select(Tmp4, N0);
05272         Select(Chain1, Chain1);
05273         ResNode = CurDAG->getTargetNode(X86::PADDQrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
05274         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05275         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
05276         AddHandleReplacement(N1.Val, 1, ResNode, 1);
05277         Result = SDOperand(ResNode, N.ResNo);
05278         return;
05279       }
05280     }
05281   }
05282   {
05283     N0 = N.getOperand(0);
05284     if (N0.getOpcode() == ISD::LOAD &&
05285         N0.hasOneUse() &&
05286         !CodeGenMap.count(N0.getValue(0)) &&
05287         !isNonImmUse(N.Val, N0.Val)) {
05288       Chain0 = N0.getOperand(0);
05289       N01 = N0.getOperand(1);
05290       N1 = N.getOperand(1);
05291 
05292       // Pattern: (add:i8 (load:i8 addr:i32:$src2), R8:i8:$src1)
05293       // Emits: (ADD8rm:i8 R8:i8:$src1, addr:i32:$src2)
05294       // Pattern complexity = 14  cost = 1
05295       if (N.Val->getValueType(0) == MVT::i8 &&
05296           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05297         Select(Tmp0, Tmp0);
05298         Select(Tmp1, Tmp1);
05299         Select(Tmp2, Tmp2);
05300         Select(Tmp3, Tmp3);
05301         Select(Tmp4, N1);
05302         Select(Chain0, Chain0);
05303         ResNode = CurDAG->getTargetNode(X86::ADD8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05304         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05305         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05306         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05307         Result = SDOperand(ResNode, N.ResNo);
05308         return;
05309       }
05310 
05311       // Pattern: (add:i16 (load:i16 addr:i32:$src2), R16:i16:$src1)
05312       // Emits: (ADD16rm:i16 R16:i16:$src1, addr:i32:$src2)
05313       // Pattern complexity = 14  cost = 1
05314       if (N.Val->getValueType(0) == MVT::i16 &&
05315           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05316         Select(Tmp0, Tmp0);
05317         Select(Tmp1, Tmp1);
05318         Select(Tmp2, Tmp2);
05319         Select(Tmp3, Tmp3);
05320         Select(Tmp4, N1);
05321         Select(Chain0, Chain0);
05322         ResNode = CurDAG->getTargetNode(X86::ADD16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05323         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05324         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05325         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05326         Result = SDOperand(ResNode, N.ResNo);
05327         return;
05328       }
05329 
05330       // Pattern: (add:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
05331       // Emits: (ADD32rm:i32 R32:i32:$src1, addr:i32:$src2)
05332       // Pattern complexity = 14  cost = 1
05333       if (N.Val->getValueType(0) == MVT::i32 &&
05334           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05335         Select(Tmp0, Tmp0);
05336         Select(Tmp1, Tmp1);
05337         Select(Tmp2, Tmp2);
05338         Select(Tmp3, Tmp3);
05339         Select(Tmp4, N1);
05340         Select(Chain0, Chain0);
05341         ResNode = CurDAG->getTargetNode(X86::ADD32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05342         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05343         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05344         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05345         Result = SDOperand(ResNode, N.ResNo);
05346         return;
05347       }
05348     }
05349   }
05350   if ((Subtarget->hasSSE2())) {
05351     N0 = N.getOperand(0);
05352     if (N0.getOpcode() == ISD::LOAD &&
05353         N0.hasOneUse() &&
05354         !CodeGenMap.count(N0.getValue(0)) &&
05355         !isNonImmUse(N.Val, N0.Val)) {
05356       Chain0 = N0.getOperand(0);
05357       N01 = N0.getOperand(1);
05358       N1 = N.getOperand(1);
05359 
05360       // Pattern: (add:v16i8 (load:v16i8 addr:i32:$src2), VR128:v16i8:$src1)
05361       // Emits: (PADDBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
05362       // Pattern complexity = 14  cost = 1
05363       if (N.Val->getValueType(0) == MVT::v16i8 &&
05364           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05365         Select(Tmp0, Tmp0);
05366         Select(Tmp1, Tmp1);
05367         Select(Tmp2, Tmp2);
05368         Select(Tmp3, Tmp3);
05369         Select(Tmp4, N1);
05370         Select(Chain0, Chain0);
05371         ResNode = CurDAG->getTargetNode(X86::PADDBrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05372         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05373         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05374         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05375         Result = SDOperand(ResNode, N.ResNo);
05376         return;
05377       }
05378 
05379       // Pattern: (add:v8i16 (load:v8i16 addr:i32:$src2), VR128:v8i16:$src1)
05380       // Emits: (PADDWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
05381       // Pattern complexity = 14  cost = 1
05382       if (N.Val->getValueType(0) == MVT::v8i16 &&
05383           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05384         Select(Tmp0, Tmp0);
05385         Select(Tmp1, Tmp1);
05386         Select(Tmp2, Tmp2);
05387         Select(Tmp3, Tmp3);
05388         Select(Tmp4, N1);
05389         Select(Chain0, Chain0);
05390         ResNode = CurDAG->getTargetNode(X86::PADDWrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05391         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05392         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05393         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05394         Result = SDOperand(ResNode, N.ResNo);
05395         return;
05396       }
05397 
05398       // Pattern: (add:v4i32 (load:v4i32 addr:i32:$src2), VR128:v4i32:$src1)
05399       // Emits: (PADDDrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
05400       // Pattern complexity = 14  cost = 1
05401       if (N.Val->getValueType(0) == MVT::v4i32 &&
05402           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05403         Select(Tmp0, Tmp0);
05404         Select(Tmp1, Tmp1);
05405         Select(Tmp2, Tmp2);
05406         Select(Tmp3, Tmp3);
05407         Select(Tmp4, N1);
05408         Select(Chain0, Chain0);
05409         ResNode = CurDAG->getTargetNode(X86::PADDDrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05410         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05411         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05412         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05413         Result = SDOperand(ResNode, N.ResNo);
05414         return;
05415       }
05416 
05417       // Pattern: (add:v2i64 (load:v2i64 addr:i32:$src2), VR128:v2i64:$src1)
05418       // Emits: (PADDQrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
05419       // Pattern complexity = 14  cost = 1
05420       if (N.Val->getValueType(0) == MVT::v2i64 &&
05421           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
05422         Select(Tmp0, Tmp0);
05423         Select(Tmp1, Tmp1);
05424         Select(Tmp2, Tmp2);
05425         Select(Tmp3, Tmp3);
05426         Select(Tmp4, N1);
05427         Select(Chain0, Chain0);
05428         ResNode = CurDAG->getTargetNode(X86::PADDQrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
05429         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
05430         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
05431         AddHandleReplacement(N0.Val, 1, ResNode, 1);
05432         Result = SDOperand(ResNode, N.ResNo);
05433         return;
05434       }
05435     }
05436   }
05437 
05438   // Pattern: leaaddr:i32:$src
05439   // Emits: (LEA32r:i32 leaaddr:i32:$src)
05440   // Pattern complexity = 10  cost = 1
05441   if (N.Val->getValueType(0) == MVT::i32 &&
05442       SelectLEAAddr(N, Tmp0, Tmp1, Tmp2, Tmp3)) {
05443     Select(Tmp0, Tmp0);
05444     Select(Tmp1, Tmp1);
05445     Select(Tmp2, Tmp2);
05446     Select(Tmp3, Tmp3);
05447     if (N.Val->hasOneUse()) {
05448       Result = CurDAG->SelectNodeTo(N.Val, X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
05449     } else {
05450       ResNode = CurDAG->getTargetNode(X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
05451       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05452       Result = SDOperand(ResNode, 0);
05453     }
05454     return;
05455   }
05456   {
05457     N0 = N.getOperand(0);
05458     {
05459       N1 = N.getOperand(1);
05460       if (N1.getOpcode() == X86ISD::Wrapper) {
05461         N10 = N1.getOperand(0);
05462 
05463         // Pattern: (add:i32 R32:i32:$src1, (X86Wrapper:i32 (tconstpool:i32):$src2))
05464         // Emits: (ADD32ri:i32 R32:i32:$src1, (tconstpool:i32):$src2)
05465         // Pattern complexity = 6  cost = 1
05466         if (N10.getOpcode() == ISD::TargetConstantPool) {
05467           Select(Tmp0, N0);
05468           Tmp1 = N10;
05469           if (N.Val->hasOneUse()) {
05470             Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05471           } else {
05472             ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05473             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05474             Result = SDOperand(ResNode, 0);
05475           }
05476           return;
05477         }
05478 
05479         // Pattern: (add:i32 R32:i32:$src1, (X86Wrapper:i32 (tglobaladdr:i32):$src2))
05480         // Emits: (ADD32ri:i32 R32:i32:$src1, (tglobaladdr:i32):$src2)
05481         // Pattern complexity = 6  cost = 1
05482         if (N10.getOpcode() == ISD::TargetGlobalAddress) {
05483           Select(Tmp0, N0);
05484           Tmp1 = N10;
05485           if (N.Val->hasOneUse()) {
05486             Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05487           } else {
05488             ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05489             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05490             Result = SDOperand(ResNode, 0);
05491           }
05492           return;
05493         }
05494 
05495         // Pattern: (add:i32 R32:i32:$src1, (X86Wrapper:i32 (texternalsym:i32):$src2))
05496         // Emits: (ADD32ri:i32 R32:i32:$src1, (texternalsym:i32):$src2)
05497         // Pattern complexity = 6  cost = 1
05498         if (N10.getOpcode() == ISD::TargetExternalSymbol) {
05499           Select(Tmp0, N0);
05500           Tmp1 = N10;
05501           if (N.Val->hasOneUse()) {
05502             Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05503           } else {
05504             ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05505             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05506             Result = SDOperand(ResNode, 0);
05507           }
05508           return;
05509         }
05510       }
05511     }
05512     if (N0.getOpcode() == X86ISD::Wrapper) {
05513       N00 = N0.getOperand(0);
05514 
05515       // Pattern: (add:i32 (X86Wrapper:i32 (tconstpool:i32):$src2), R32:i32:$src1)
05516       // Emits: (ADD32ri:i32 R32:i32:$src1, (tconstpool:i32):$src2)
05517       // Pattern complexity = 6  cost = 1
05518       if (N00.getOpcode() == ISD::TargetConstantPool) {
05519         N1 = N.getOperand(1);
05520         Select(Tmp0, N1);
05521         Tmp1 = N00;
05522         if (N.Val->hasOneUse()) {
05523           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05524         } else {
05525           ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05526           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05527           Result = SDOperand(ResNode, 0);
05528         }
05529         return;
05530       }
05531 
05532       // Pattern: (add:i32 (X86Wrapper:i32 (tglobaladdr:i32):$src2), R32:i32:$src1)
05533       // Emits: (ADD32ri:i32 R32:i32:$src1, (tglobaladdr:i32):$src2)
05534       // Pattern complexity = 6  cost = 1
05535       if (N00.getOpcode() == ISD::TargetGlobalAddress) {
05536         N1 = N.getOperand(1);
05537         Select(Tmp0, N1);
05538         Tmp1 = N00;
05539         if (N.Val->hasOneUse()) {
05540           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05541         } else {
05542           ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05543           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05544           Result = SDOperand(ResNode, 0);
05545         }
05546         return;
05547       }
05548 
05549       // Pattern: (add:i32 (X86Wrapper:i32 (texternalsym:i32):$src2), R32:i32:$src1)
05550       // Emits: (ADD32ri:i32 R32:i32:$src1, (texternalsym:i32):$src2)
05551       // Pattern complexity = 6  cost = 1
05552       if (N00.getOpcode() == ISD::TargetExternalSymbol) {
05553         N1 = N.getOperand(1);
05554         Select(Tmp0, N1);
05555         Tmp1 = N00;
05556         if (N.Val->hasOneUse()) {
05557           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05558         } else {
05559           ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05560           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05561           Result = SDOperand(ResNode, 0);
05562         }
05563         return;
05564       }
05565     }
05566     {
05567       N1 = N.getOperand(1);
05568       if (isa<ConstantSDNode>(N1)) {
05569         int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
05570         if (CN0 == 1) {
05571 
05572           // Pattern: (add:i8 R8:i8:$src, 1:i8)
05573           // Emits: (INC8r:i8 R8:i8:$src)
05574           // Pattern complexity = 5  cost = 1
05575           if (N.Val->getValueType(0) == MVT::i8) {
05576             Select(Tmp1, N0);
05577             if (N.Val->hasOneUse()) {
05578               Result = CurDAG->SelectNodeTo(N.Val, X86::INC8r, MVT::i8, Tmp1);
05579             } else {
05580               ResNode = CurDAG->getTargetNode(X86::INC8r, MVT::i8, Tmp1);
05581               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05582               Result = SDOperand(ResNode, 0);
05583             }
05584             return;
05585           }
05586 
05587           // Pattern: (add:i16 R16:i16:$src, 1:i16)
05588           // Emits: (INC16r:i16 R16:i16:$src)
05589           // Pattern complexity = 5  cost = 1
05590           if (N.Val->getValueType(0) == MVT::i16) {
05591             Select(Tmp1, N0);
05592             if (N.Val->hasOneUse()) {
05593               Result = CurDAG->SelectNodeTo(N.Val, X86::INC16r, MVT::i16, Tmp1);
05594             } else {
05595               ResNode = CurDAG->getTargetNode(X86::INC16r, MVT::i16, Tmp1);
05596               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05597               Result = SDOperand(ResNode, 0);
05598             }
05599             return;
05600           }
05601 
05602           // Pattern: (add:i32 R32:i32:$src, 1:i32)
05603           // Emits: (INC32r:i32 R32:i32:$src)
05604           // Pattern complexity = 5  cost = 1
05605           if (N.Val->getValueType(0) == MVT::i32) {
05606             Select(Tmp1, N0);
05607             if (N.Val->hasOneUse()) {
05608               Result = CurDAG->SelectNodeTo(N.Val, X86::INC32r, MVT::i32, Tmp1);
05609             } else {
05610               ResNode = CurDAG->getTargetNode(X86::INC32r, MVT::i32, Tmp1);
05611               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05612               Result = SDOperand(ResNode, 0);
05613             }
05614             return;
05615           }
05616         }
05617         if (CN0 == -1) {
05618 
05619           // Pattern: (add:i8 R8:i8:$src, -1:i8)
05620           // Emits: (DEC8r:i8 R8:i8:$src)
05621           // Pattern complexity = 5  cost = 1
05622           if (N.Val->getValueType(0) == MVT::i8) {
05623             Select(Tmp1, N0);
05624             if (N.Val->hasOneUse()) {
05625               Result = CurDAG->SelectNodeTo(N.Val, X86::DEC8r, MVT::i8, Tmp1);
05626             } else {
05627               ResNode = CurDAG->getTargetNode(X86::DEC8r, MVT::i8, Tmp1);
05628               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05629               Result = SDOperand(ResNode, 0);
05630             }
05631             return;
05632           }
05633 
05634           // Pattern: (add:i16 R16:i16:$src, -1:i16)
05635           // Emits: (DEC16r:i16 R16:i16:$src)
05636           // Pattern complexity = 5  cost = 1
05637           if (N.Val->getValueType(0) == MVT::i16) {
05638             Select(Tmp1, N0);
05639             if (N.Val->hasOneUse()) {
05640               Result = CurDAG->SelectNodeTo(N.Val, X86::DEC16r, MVT::i16, Tmp1);
05641             } else {
05642               ResNode = CurDAG->getTargetNode(X86::DEC16r, MVT::i16, Tmp1);
05643               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05644               Result = SDOperand(ResNode, 0);
05645             }
05646             return;
05647           }
05648 
05649           // Pattern: (add:i32 R32:i32:$src, -1:i32)
05650           // Emits: (DEC32r:i32 R32:i32:$src)
05651           // Pattern complexity = 5  cost = 1
05652           if (N.Val->getValueType(0) == MVT::i32) {
05653             Select(Tmp1, N0);
05654             if (N.Val->hasOneUse()) {
05655               Result = CurDAG->SelectNodeTo(N.Val, X86::DEC32r, MVT::i32, Tmp1);
05656             } else {
05657               ResNode = CurDAG->getTargetNode(X86::DEC32r, MVT::i32, Tmp1);
05658               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05659               Result = SDOperand(ResNode, 0);
05660             }
05661             return;
05662           }
05663         }
05664       }
05665       if (N1.getOpcode() == ISD::Constant) {
05666 
05667         // Pattern: (add:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
05668         // Emits: (ADD16ri8:i16 R16:i16:$src1, (imm:i16):$src2)
05669         // Pattern complexity = 5  cost = 1
05670         if (Predicate_i16immSExt8(N1.Val) &&
05671             N.Val->getValueType(0) == MVT::i16) {
05672           Select(Tmp0, N0);
05673           unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
05674           Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
05675           if (N.Val->hasOneUse()) {
05676             Result = CurDAG->SelectNodeTo(N.Val, X86::ADD16ri8, MVT::i16, Tmp0, Tmp1);
05677           } else {
05678             ResNode = CurDAG->getTargetNode(X86::ADD16ri8, MVT::i16, Tmp0, Tmp1);
05679             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05680             Result = SDOperand(ResNode, 0);
05681           }
05682           return;
05683         }
05684 
05685         // Pattern: (add:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
05686         // Emits: (ADD32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
05687         // Pattern complexity = 5  cost = 1
05688         if (Predicate_i32immSExt8(N1.Val) &&
05689             N.Val->getValueType(0) == MVT::i32) {
05690           Select(Tmp0, N0);
05691           unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
05692           Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
05693           if (N.Val->hasOneUse()) {
05694             Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri8, MVT::i32, Tmp0, Tmp1);
05695           } else {
05696             ResNode = CurDAG->getTargetNode(X86::ADD32ri8, MVT::i32, Tmp0, Tmp1);
05697             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05698             Result = SDOperand(ResNode, 0);
05699           }
05700           return;
05701         }
05702       }
05703     }
05704     if (isa<ConstantSDNode>(N0)) {
05705       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
05706       if (CN0 == 1) {
05707         N1 = N.getOperand(1);
05708 
05709         // Pattern: (add:i8 1:i8, R8:i8:$src)
05710         // Emits: (INC8r:i8 R8:i8:$src)
05711         // Pattern complexity = 5  cost = 1
05712         if (N.Val->getValueType(0) == MVT::i8) {
05713           Select(Tmp1, N1);
05714           if (N.Val->hasOneUse()) {
05715             Result = CurDAG->SelectNodeTo(N.Val, X86::INC8r, MVT::i8, Tmp1);
05716           } else {
05717             ResNode = CurDAG->getTargetNode(X86::INC8r, MVT::i8, Tmp1);
05718             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05719             Result = SDOperand(ResNode, 0);
05720           }
05721           return;
05722         }
05723 
05724         // Pattern: (add:i16 1:i16, R16:i16:$src)
05725         // Emits: (INC16r:i16 R16:i16:$src)
05726         // Pattern complexity = 5  cost = 1
05727         if (N.Val->getValueType(0) == MVT::i16) {
05728           Select(Tmp1, N1);
05729           if (N.Val->hasOneUse()) {
05730             Result = CurDAG->SelectNodeTo(N.Val, X86::INC16r, MVT::i16, Tmp1);
05731           } else {
05732             ResNode = CurDAG->getTargetNode(X86::INC16r, MVT::i16, Tmp1);
05733             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05734             Result = SDOperand(ResNode, 0);
05735           }
05736           return;
05737         }
05738 
05739         // Pattern: (add:i32 1:i32, R32:i32:$src)
05740         // Emits: (INC32r:i32 R32:i32:$src)
05741         // Pattern complexity = 5  cost = 1
05742         if (N.Val->getValueType(0) == MVT::i32) {
05743           Select(Tmp1, N1);
05744           if (N.Val->hasOneUse()) {
05745             Result = CurDAG->SelectNodeTo(N.Val, X86::INC32r, MVT::i32, Tmp1);
05746           } else {
05747             ResNode = CurDAG->getTargetNode(X86::INC32r, MVT::i32, Tmp1);
05748             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05749             Result = SDOperand(ResNode, 0);
05750           }
05751           return;
05752         }
05753       }
05754       if (CN0 == -1) {
05755         N1 = N.getOperand(1);
05756 
05757         // Pattern: (add:i8 -1:i8, R8:i8:$src)
05758         // Emits: (DEC8r:i8 R8:i8:$src)
05759         // Pattern complexity = 5  cost = 1
05760         if (N.Val->getValueType(0) == MVT::i8) {
05761           Select(Tmp1, N1);
05762           if (N.Val->hasOneUse()) {
05763             Result = CurDAG->SelectNodeTo(N.Val, X86::DEC8r, MVT::i8, Tmp1);
05764           } else {
05765             ResNode = CurDAG->getTargetNode(X86::DEC8r, MVT::i8, Tmp1);
05766             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05767             Result = SDOperand(ResNode, 0);
05768           }
05769           return;
05770         }
05771 
05772         // Pattern: (add:i16 -1:i16, R16:i16:$src)
05773         // Emits: (DEC16r:i16 R16:i16:$src)
05774         // Pattern complexity = 5  cost = 1
05775         if (N.Val->getValueType(0) == MVT::i16) {
05776           Select(Tmp1, N1);
05777           if (N.Val->hasOneUse()) {
05778             Result = CurDAG->SelectNodeTo(N.Val, X86::DEC16r, MVT::i16, Tmp1);
05779           } else {
05780             ResNode = CurDAG->getTargetNode(X86::DEC16r, MVT::i16, Tmp1);
05781             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05782             Result = SDOperand(ResNode, 0);
05783           }
05784           return;
05785         }
05786 
05787         // Pattern: (add:i32 -1:i32, R32:i32:$src)
05788         // Emits: (DEC32r:i32 R32:i32:$src)
05789         // Pattern complexity = 5  cost = 1
05790         if (N.Val->getValueType(0) == MVT::i32) {
05791           Select(Tmp1, N1);
05792           if (N.Val->hasOneUse()) {
05793             Result = CurDAG->SelectNodeTo(N.Val, X86::DEC32r, MVT::i32, Tmp1);
05794           } else {
05795             ResNode = CurDAG->getTargetNode(X86::DEC32r, MVT::i32, Tmp1);
05796             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05797             Result = SDOperand(ResNode, 0);
05798           }
05799           return;
05800         }
05801       }
05802     }
05803     N1 = N.getOperand(1);
05804     if (N1.getOpcode() == ISD::Constant) {
05805 
05806       // Pattern: (add:i8 R8:i8:$src1, (imm:i8):$src2)
05807       // Emits: (ADD8ri:i8 R8:i8:$src1, (imm:i8):$src2)
05808       // Pattern complexity = 4  cost = 1
05809       if (N.Val->getValueType(0) == MVT::i8) {
05810         Select(Tmp0, N0);
05811         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
05812         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
05813         if (N.Val->hasOneUse()) {
05814           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD8ri, MVT::i8, Tmp0, Tmp1);
05815         } else {
05816           ResNode = CurDAG->getTargetNode(X86::ADD8ri, MVT::i8, Tmp0, Tmp1);
05817           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05818           Result = SDOperand(ResNode, 0);
05819         }
05820         return;
05821       }
05822 
05823       // Pattern: (add:i16 R16:i16:$src1, (imm:i16):$src2)
05824       // Emits: (ADD16ri:i16 R16:i16:$src1, (imm:i16):$src2)
05825       // Pattern complexity = 4  cost = 1
05826       if (N.Val->getValueType(0) == MVT::i16) {
05827         Select(Tmp0, N0);
05828         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
05829         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
05830         if (N.Val->hasOneUse()) {
05831           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD16ri, MVT::i16, Tmp0, Tmp1);
05832         } else {
05833           ResNode = CurDAG->getTargetNode(X86::ADD16ri, MVT::i16, Tmp0, Tmp1);
05834           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05835           Result = SDOperand(ResNode, 0);
05836         }
05837         return;
05838       }
05839 
05840       // Pattern: (add:i32 R32:i32:$src1, (imm:i32):$src2)
05841       // Emits: (ADD32ri:i32 R32:i32:$src1, (imm:i32):$src2)
05842       // Pattern complexity = 4  cost = 1
05843       if (N.Val->getValueType(0) == MVT::i32) {
05844         Select(Tmp0, N0);
05845         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
05846         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
05847         if (N.Val->hasOneUse()) {
05848           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05849         } else {
05850           ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, Tmp0, Tmp1);
05851           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05852           Result = SDOperand(ResNode, 0);
05853         }
05854         return;
05855       }
05856     }
05857 
05858     // Pattern: (add:i8 R8:i8:$src1, R8:i8:$src2)
05859     // Emits: (ADD8rr:i8 R8:i8:$src1, R8:i8:$src2)
05860     // Pattern complexity = 2  cost = 1
05861     if (N.Val->getValueType(0) == MVT::i8) {
05862       Select(Tmp0, N0);
05863       Select(Tmp1, N1);
05864       if (N.Val->hasOneUse()) {
05865         Result = CurDAG->SelectNodeTo(N.Val, X86::ADD8rr, MVT::i8, Tmp0, Tmp1);
05866       } else {
05867         ResNode = CurDAG->getTargetNode(X86::ADD8rr, MVT::i8, Tmp0, Tmp1);
05868         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05869         Result = SDOperand(ResNode, 0);
05870       }
05871       return;
05872     }
05873 
05874     // Pattern: (add:i16 R16:i16:$src1, R16:i16:$src2)
05875     // Emits: (ADD16rr:i16 R16:i16:$src1, R16:i16:$src2)
05876     // Pattern complexity = 2  cost = 1
05877     if (N.Val->getValueType(0) == MVT::i16) {
05878       Select(Tmp0, N0);
05879       Select(Tmp1, N1);
05880       if (N.Val->hasOneUse()) {
05881         Result = CurDAG->SelectNodeTo(N.Val, X86::ADD16rr, MVT::i16, Tmp0, Tmp1);
05882       } else {
05883         ResNode = CurDAG->getTargetNode(X86::ADD16rr, MVT::i16, Tmp0, Tmp1);
05884         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05885         Result = SDOperand(ResNode, 0);
05886       }
05887       return;
05888     }
05889 
05890     // Pattern: (add:i32 R32:i32:$src1, R32:i32:$src2)
05891     // Emits: (ADD32rr:i32 R32:i32:$src1, R32:i32:$src2)
05892     // Pattern complexity = 2  cost = 1
05893     if (N.Val->getValueType(0) == MVT::i32) {
05894       Select(Tmp0, N0);
05895       Select(Tmp1, N1);
05896       if (N.Val->hasOneUse()) {
05897         Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32rr, MVT::i32, Tmp0, Tmp1);
05898       } else {
05899         ResNode = CurDAG->getTargetNode(X86::ADD32rr, MVT::i32, Tmp0, Tmp1);
05900         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05901         Result = SDOperand(ResNode, 0);
05902       }
05903       return;
05904     }
05905   }
05906   if ((Subtarget->hasSSE2())) {
05907     N0 = N.getOperand(0);
05908     N1 = N.getOperand(1);
05909 
05910     // Pattern: (add:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
05911     // Emits: (PADDBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
05912     // Pattern complexity = 2  cost = 1
05913     if (N.Val->getValueType(0) == MVT::v16i8) {
05914       Select(Tmp0, N0);
05915       Select(Tmp1, N1);
05916       if (N.Val->hasOneUse()) {
05917         Result = CurDAG->SelectNodeTo(N.Val, X86::PADDBrr, MVT::v16i8, Tmp0, Tmp1);
05918       } else {
05919         ResNode = CurDAG->getTargetNode(X86::PADDBrr, MVT::v16i8, Tmp0, Tmp1);
05920         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05921         Result = SDOperand(ResNode, 0);
05922       }
05923       return;
05924     }
05925 
05926     // Pattern: (add:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
05927     // Emits: (PADDWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
05928     // Pattern complexity = 2  cost = 1
05929     if (N.Val->getValueType(0) == MVT::v8i16) {
05930       Select(Tmp0, N0);
05931       Select(Tmp1, N1);
05932       if (N.Val->hasOneUse()) {
05933         Result = CurDAG->SelectNodeTo(N.Val, X86::PADDWrr, MVT::v8i16, Tmp0, Tmp1);
05934       } else {
05935         ResNode = CurDAG->getTargetNode(X86::PADDWrr, MVT::v8i16, Tmp0, Tmp1);
05936         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05937         Result = SDOperand(ResNode, 0);
05938       }
05939       return;
05940     }
05941 
05942     // Pattern: (add:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
05943     // Emits: (PADDDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
05944     // Pattern complexity = 2  cost = 1
05945     if (N.Val->getValueType(0) == MVT::v4i32) {
05946       Select(Tmp0, N0);
05947       Select(Tmp1, N1);
05948       if (N.Val->hasOneUse()) {
05949         Result = CurDAG->SelectNodeTo(N.Val, X86::PADDDrr, MVT::v4i32, Tmp0, Tmp1);
05950       } else {
05951         ResNode = CurDAG->getTargetNode(X86::PADDDrr, MVT::v4i32, Tmp0, Tmp1);
05952         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05953         Result = SDOperand(ResNode, 0);
05954       }
05955       return;
05956     }
05957 
05958     // Pattern: (add:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
05959     // Emits: (PADDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
05960     // Pattern complexity = 2  cost = 1
05961     if (N.Val->getValueType(0) == MVT::v2i64) {
05962       Select(Tmp0, N0);
05963       Select(Tmp1, N1);
05964       if (N.Val->hasOneUse()) {
05965         Result = CurDAG->SelectNodeTo(N.Val, X86::PADDQrr, MVT::v2i64, Tmp0, Tmp1);
05966       } else {
05967         ResNode = CurDAG->getTargetNode(X86::PADDQrr, MVT::v2i64, Tmp0, Tmp1);
05968         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
05969         Result = SDOperand(ResNode, 0);
05970       }
05971       return;
05972     }
05973   }
05974   std::cerr << "Cannot yet select: ";
05975   N.Val->dump(CurDAG);
05976   std::cerr << '\n';
05977   abort();
05978 }
05979 
05980 void Select_addc(SDOperand &Result, SDOperand N) {
05981   SDOperand Chain0(0, 0);
05982   SDOperand Chain1(0, 0);
05983   SDOperand InFlag(0, 0);
05984   SDOperand N0(0, 0);
05985   SDOperand N01(0, 0);
05986   SDOperand N1(0, 0);
05987   SDOperand N11(0, 0);
05988   SDOperand Tmp0(0, 0);
05989   SDOperand Tmp1(0, 0);
05990   SDOperand Tmp2(0, 0);
05991   SDOperand Tmp3(0, 0);
05992   SDOperand Tmp4(0, 0);
05993   SDNode *ResNode;
05994   N0 = N.getOperand(0);
05995 
05996   // Pattern: (addc:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
05997   // Emits: (ADD32rm:i32 R32:i32:$src1, addr:i32:$src2)
05998   // Pattern complexity = 14  cost = 1
05999   {
06000     N1 = N.getOperand(1);
06001     if (N1.getOpcode() == ISD::LOAD &&
06002         N1.hasOneUse() &&
06003         !CodeGenMap.count(N1.getValue(0)) &&
06004         N.Val->isOnlyUse(N1.Val) &&
06005         !isNonImmUse(N.Val, N1.Val)) {
06006       Chain1 = N1.getOperand(0);
06007       N11 = N1.getOperand(1);
06008       if (N.Val->getValueType(0) == MVT::i32 &&
06009           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06010         Select(Tmp0, Tmp0);
06011         Select(Tmp1, Tmp1);
06012         Select(Tmp2, Tmp2);
06013         Select(Tmp3, Tmp3);
06014         Select(Tmp4, N0);
06015         Select(Chain1, Chain1);
06016         ResNode = CurDAG->getTargetNode(X86::ADD32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06017         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06018         InFlag = SDOperand(ResNode, 2);
06019         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06020         AddHandleReplacement(N1.Val, 1, ResNode, 1);
06021         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06022         if (N.ResNo < 1)
06023           Result = SDOperand(ResNode, N.ResNo);
06024         else
06025           Result = SDOperand(ResNode, N.ResNo+1);
06026         return;
06027       }
06028     }
06029   }
06030 
06031   // Pattern: (addc:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
06032   // Emits: (ADD32rm:i32 R32:i32:$src1, addr:i32:$src2)
06033   // Pattern complexity = 14  cost = 1
06034   if (N0.getOpcode() == ISD::LOAD &&
06035       N0.hasOneUse() &&
06036       !CodeGenMap.count(N0.getValue(0)) &&
06037       N.Val->isOnlyUse(N0.Val) &&
06038       !isNonImmUse(N.Val, N0.Val)) {
06039     Chain0 = N0.getOperand(0);
06040     N01 = N0.getOperand(1);
06041     N1 = N.getOperand(1);
06042     if (N.Val->getValueType(0) == MVT::i32 &&
06043         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06044       Select(Tmp0, Tmp0);
06045       Select(Tmp1, Tmp1);
06046       Select(Tmp2, Tmp2);
06047       Select(Tmp3, Tmp3);
06048       Select(Tmp4, N1);
06049       Select(Chain0, Chain0);
06050       ResNode = CurDAG->getTargetNode(X86::ADD32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06051       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06052       InFlag = SDOperand(ResNode, 2);
06053       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06054       AddHandleReplacement(N0.Val, 1, ResNode, 1);
06055       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06056       if (N.ResNo < 1)
06057         Result = SDOperand(ResNode, N.ResNo);
06058       else
06059         Result = SDOperand(ResNode, N.ResNo+1);
06060       return;
06061     }
06062   }
06063   N1 = N.getOperand(1);
06064   if (N1.getOpcode() == ISD::Constant) {
06065 
06066     // Pattern: (addc:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
06067     // Emits: (ADD32ri8:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
06068     // Pattern complexity = 5  cost = 1
06069     if (Predicate_i32immSExt8(N1.Val) &&
06070         N.Val->getValueType(0) == MVT::i32) {
06071       Select(Tmp0, N0);
06072       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
06073       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
06074       ResNode = CurDAG->getTargetNode(X86::ADD32ri8, MVT::i32, MVT::Flag, Tmp0, Tmp1);
06075       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06076       InFlag = SDOperand(ResNode, 1);
06077       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06078       Result = SDOperand(ResNode, N.ResNo);
06079       return;
06080     }
06081 
06082     // Pattern: (addc:i32 R32:i32:$src1, (imm:i32):$src2)
06083     // Emits: (ADD32ri:i32 R32:i32:$src1, (imm:i32):$src2)
06084     // Pattern complexity = 4  cost = 1
06085     if (N.Val->getValueType(0) == MVT::i32) {
06086       Select(Tmp0, N0);
06087       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
06088       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
06089       ResNode = CurDAG->getTargetNode(X86::ADD32ri, MVT::i32, MVT::Flag, Tmp0, Tmp1);
06090       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06091       InFlag = SDOperand(ResNode, 1);
06092       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06093       Result = SDOperand(ResNode, N.ResNo);
06094       return;
06095     }
06096   }
06097 
06098   // Pattern: (addc:i32 R32:i32:$src1, R32:i32:$src2)
06099   // Emits: (ADD32rr:i32 R32:i32:$src1, R32:i32:$src2)
06100   // Pattern complexity = 2  cost = 1
06101   if (N.Val->getValueType(0) == MVT::i32) {
06102     Select(Tmp0, N0);
06103     Select(Tmp1, N1);
06104     ResNode = CurDAG->getTargetNode(X86::ADD32rr, MVT::i32, MVT::Flag, Tmp0, Tmp1);
06105     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06106     InFlag = SDOperand(ResNode, 1);
06107     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06108     Result = SDOperand(ResNode, N.ResNo);
06109     return;
06110   }
06111   std::cerr << "Cannot yet select: ";
06112   N.Val->dump(CurDAG);
06113   std::cerr << '\n';
06114   abort();
06115 }
06116 
06117 void Select_adde(SDOperand &Result, SDOperand N) {
06118   SDOperand Chain0(0, 0);
06119   SDOperand Chain1(0, 0);
06120   SDOperand InFlag(0, 0);
06121   SDOperand N0(0, 0);
06122   SDOperand N01(0, 0);
06123   SDOperand N1(0, 0);
06124   SDOperand N11(0, 0);
06125   SDOperand Tmp0(0, 0);
06126   SDOperand Tmp1(0, 0);
06127   SDOperand Tmp2(0, 0);
06128   SDOperand Tmp3(0, 0);
06129   SDOperand Tmp4(0, 0);
06130   SDNode *ResNode;
06131   N0 = N.getOperand(0);
06132 
06133   // Pattern: (adde:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
06134   // Emits: (ADC32rm:i32 R32:i32:$src1, addr:i32:$src2)
06135   // Pattern complexity = 14  cost = 1
06136   {
06137     N1 = N.getOperand(1);
06138     if (N1.getOpcode() == ISD::LOAD &&
06139         N1.hasOneUse() &&
06140         !CodeGenMap.count(N1.getValue(0)) &&
06141         N.Val->isOnlyUse(N1.Val) &&
06142         !isNonImmUse(N.Val, N1.Val)) {
06143       Chain1 = N1.getOperand(0);
06144       N11 = N1.getOperand(1);
06145       if (N.Val->getValueType(0) == MVT::i32 &&
06146           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06147         Select(Tmp0, Tmp0);
06148         Select(Tmp1, Tmp1);
06149         Select(Tmp2, Tmp2);
06150         Select(Tmp3, Tmp3);
06151         Select(Tmp4, N0);
06152         Select(Chain1, Chain1);
06153         Select(InFlag, N.getOperand(2));
06154         ResNode = CurDAG->getTargetNode(X86::ADC32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
06155         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06156         InFlag = SDOperand(ResNode, 2);
06157         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06158         AddHandleReplacement(N1.Val, 1, ResNode, 1);
06159         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06160         if (N.ResNo < 1)
06161           Result = SDOperand(ResNode, N.ResNo);
06162         else
06163           Result = SDOperand(ResNode, N.ResNo+1);
06164         return;
06165       }
06166     }
06167   }
06168 
06169   // Pattern: (adde:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
06170   // Emits: (ADC32rm:i32 R32:i32:$src1, addr:i32:$src2)
06171   // Pattern complexity = 14  cost = 1
06172   if (N0.getOpcode() == ISD::LOAD &&
06173       N0.hasOneUse() &&
06174       !CodeGenMap.count(N0.getValue(0)) &&
06175       N.Val->isOnlyUse(N0.Val) &&
06176       !isNonImmUse(N.Val, N0.Val)) {
06177     Chain0 = N0.getOperand(0);
06178     N01 = N0.getOperand(1);
06179     N1 = N.getOperand(1);
06180     if (N.Val->getValueType(0) == MVT::i32 &&
06181         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06182       Select(Tmp0, Tmp0);
06183       Select(Tmp1, Tmp1);
06184       Select(Tmp2, Tmp2);
06185       Select(Tmp3, Tmp3);
06186       Select(Tmp4, N1);
06187       Select(Chain0, Chain0);
06188       Select(InFlag, N.getOperand(2));
06189       ResNode = CurDAG->getTargetNode(X86::ADC32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0, InFlag);
06190       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06191       InFlag = SDOperand(ResNode, 2);
06192       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06193       AddHandleReplacement(N0.Val, 1, ResNode, 1);
06194       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06195       if (N.ResNo < 1)
06196         Result = SDOperand(ResNode, N.ResNo);
06197       else
06198         Result = SDOperand(ResNode, N.ResNo+1);
06199       return;
06200     }
06201   }
06202   N1 = N.getOperand(1);
06203   if (N1.getOpcode() == ISD::Constant) {
06204 
06205     // Pattern: (adde:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
06206     // Emits: (ADC32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
06207     // Pattern complexity = 5  cost = 1
06208     if (Predicate_i32immSExt8(N1.Val) &&
06209         N.Val->getValueType(0) == MVT::i32) {
06210       Select(Tmp0, N0);
06211       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
06212       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
06213       Select(InFlag, N.getOperand(2));
06214       ResNode = CurDAG->getTargetNode(X86::ADC32ri8, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
06215       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06216       InFlag = SDOperand(ResNode, 1);
06217       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06218       Result = SDOperand(ResNode, N.ResNo);
06219       return;
06220     }
06221 
06222     // Pattern: (adde:i32 R32:i32:$src1, (imm:i32):$src2)
06223     // Emits: (ADC32ri:i32 R32:i32:$src1, (imm:i32):$src2)
06224     // Pattern complexity = 4  cost = 1
06225     if (N.Val->getValueType(0) == MVT::i32) {
06226       Select(Tmp0, N0);
06227       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
06228       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
06229       Select(InFlag, N.getOperand(2));
06230       ResNode = CurDAG->getTargetNode(X86::ADC32ri, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
06231       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06232       InFlag = SDOperand(ResNode, 1);
06233       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06234       Result = SDOperand(ResNode, N.ResNo);
06235       return;
06236     }
06237   }
06238 
06239   // Pattern: (adde:i32 R32:i32:$src1, R32:i32:$src2)
06240   // Emits: (ADC32rr:i32 R32:i32:$src1, R32:i32:$src2)
06241   // Pattern complexity = 2  cost = 1
06242   if (N.Val->getValueType(0) == MVT::i32) {
06243     Select(Tmp0, N0);
06244     Select(Tmp1, N1);
06245     Select(InFlag, N.getOperand(2));
06246     ResNode = CurDAG->getTargetNode(X86::ADC32rr, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
06247     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06248     InFlag = SDOperand(ResNode, 1);
06249     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
06250     Result = SDOperand(ResNode, N.ResNo);
06251     return;
06252   }
06253   std::cerr << "Cannot yet select: ";
06254   N.Val->dump(CurDAG);
06255   std::cerr << '\n';
06256   abort();
06257 }
06258 
06259 void Select_and(SDOperand &Result, SDOperand N) {
06260   SDOperand Chain0(0, 0);
06261   SDOperand Chain00(0, 0);
06262   SDOperand Chain1(0, 0);
06263   SDOperand Chain10(0, 0);
06264   SDOperand N0(0, 0);
06265   SDOperand N00(0, 0);
06266   SDOperand N000(0, 0);
06267   SDOperand N001(0, 0);
06268   SDOperand N01(0, 0);
06269   SDOperand N010(0, 0);
06270   SDOperand N1(0, 0);
06271   SDOperand N10(0, 0);
06272   SDOperand N100(0, 0);
06273   SDOperand N101(0, 0);
06274   SDOperand N11(0, 0);
06275   SDOperand N110(0, 0);
06276   SDOperand Tmp0(0, 0);
06277   SDOperand Tmp1(0, 0);
06278   SDOperand Tmp2(0, 0);
06279   SDOperand Tmp3(0, 0);
06280   SDOperand Tmp4(0, 0);
06281   SDNode *ResNode;
06282 
06283   // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)))
06284   // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
06285   // Pattern complexity = 23  cost = 1
06286   if ((Subtarget->hasSSE1())) {
06287     N0 = N.getOperand(0);
06288     if (N0.getOpcode() == ISD::XOR) {
06289       N00 = N0.getOperand(0);
06290       N01 = N0.getOperand(1);
06291       if (N01.getOpcode() == ISD::BIT_CONVERT) {
06292         N010 = N01.getOperand(0);
06293         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
06294             Predicate_immAllOnesV(N010.Val)) {
06295           N1 = N.getOperand(1);
06296           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06297               N1.hasOneUse() &&
06298               !CodeGenMap.count(N1.getValue(0))) {
06299             N10 = N1.getOperand(0);
06300             if (N10.getOpcode() == ISD::LOAD &&
06301                 N10.hasOneUse() &&
06302                 !CodeGenMap.count(N10.getValue(0))) {
06303               Chain10 = N10.getOperand(0);
06304               N101 = N10.getOperand(1);
06305               if (N.Val->getValueType(0) == MVT::v2i64 &&
06306                   N010.Val->getValueType(0) == MVT::v4i32 &&
06307                   N10.Val->getValueType(0) == MVT::v4f32 &&
06308                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
06309                 Select(Tmp0, Tmp0);
06310                 Select(Tmp1, Tmp1);
06311                 Select(Tmp2, Tmp2);
06312                 Select(Tmp3, Tmp3);
06313                 Select(Tmp4, N00);
06314                 Select(Chain10, Chain10);
06315                 ResNode = CurDAG->getTargetNode(X86::ANDNPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
06316                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06317                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
06318                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
06319                 Result = SDOperand(ResNode, N.ResNo);
06320                 return;
06321               }
06322             }
06323           }
06324         }
06325       }
06326     }
06327   }
06328 
06329   // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)))
06330   // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
06331   // Pattern complexity = 23  cost = 1
06332   if ((Subtarget->hasSSE2())) {
06333     N0 = N.getOperand(0);
06334     if (N0.getOpcode() == ISD::XOR) {
06335       N00 = N0.getOperand(0);
06336       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06337         N000 = N00.getOperand(0);
06338         N01 = N0.getOperand(1);
06339         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
06340             Predicate_immAllOnesV(N01.Val)) {
06341           N1 = N.getOperand(1);
06342           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06343               N1.hasOneUse() &&
06344               !CodeGenMap.count(N1.getValue(0))) {
06345             N10 = N1.getOperand(0);
06346             if (N10.getOpcode() == ISD::LOAD &&
06347                 N10.hasOneUse() &&
06348                 !CodeGenMap.count(N10.getValue(0))) {
06349               Chain10 = N10.getOperand(0);
06350               N101 = N10.getOperand(1);
06351               if (N.Val->getValueType(0) == MVT::v2i64 &&
06352                   N000.Val->getValueType(0) == MVT::v2f64 &&
06353                   N10.Val->getValueType(0) == MVT::v2f64 &&
06354                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
06355                 Select(Tmp0, Tmp0);
06356                 Select(Tmp1, Tmp1);
06357                 Select(Tmp2, Tmp2);
06358                 Select(Tmp3, Tmp3);
06359                 Select(Tmp4, N000);
06360                 Select(Chain10, Chain10);
06361                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
06362                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06363                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
06364                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
06365                 Result = SDOperand(ResNode, N.ResNo);
06366                 return;
06367               }
06368             }
06369           }
06370         }
06371       }
06372     }
06373   }
06374   if ((Subtarget->hasSSE1())) {
06375     N0 = N.getOperand(0);
06376 
06377     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)))
06378     // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
06379     // Pattern complexity = 23  cost = 1
06380     if (N0.getOpcode() == ISD::XOR) {
06381       N00 = N0.getOperand(0);
06382       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06383         N000 = N00.getOperand(0);
06384         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06385             Predicate_immAllOnesV(N000.Val)) {
06386           N01 = N0.getOperand(1);
06387           N1 = N.getOperand(1);
06388           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06389               N1.hasOneUse() &&
06390               !CodeGenMap.count(N1.getValue(0))) {
06391             N10 = N1.getOperand(0);
06392             if (N10.getOpcode() == ISD::LOAD &&
06393                 N10.hasOneUse() &&
06394                 !CodeGenMap.count(N10.getValue(0))) {
06395               Chain10 = N10.getOperand(0);
06396               N101 = N10.getOperand(1);
06397               if (N.Val->getValueType(0) == MVT::v2i64 &&
06398                   N000.Val->getValueType(0) == MVT::v4i32 &&
06399                   N10.Val->getValueType(0) == MVT::v4f32 &&
06400                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
06401                 Select(Tmp0, Tmp0);
06402                 Select(Tmp1, Tmp1);
06403                 Select(Tmp2, Tmp2);
06404                 Select(Tmp3, Tmp3);
06405                 Select(Tmp4, N01);
06406                 Select(Chain10, Chain10);
06407                 ResNode = CurDAG->getTargetNode(X86::ANDNPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
06408                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06409                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
06410                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
06411                 Result = SDOperand(ResNode, N.ResNo);
06412                 return;
06413               }
06414             }
06415           }
06416         }
06417       }
06418     }
06419     if (N0.getOpcode() == ISD::BIT_CONVERT &&
06420         N0.hasOneUse() &&
06421         !CodeGenMap.count(N0.getValue(0))) {
06422       N00 = N0.getOperand(0);
06423       if (N00.getOpcode() == ISD::LOAD &&
06424           N00.hasOneUse() &&
06425           !CodeGenMap.count(N00.getValue(0))) {
06426         Chain00 = N00.getOperand(0);
06427         N001 = N00.getOperand(1);
06428         N1 = N.getOperand(1);
06429         if (N1.getOpcode() == ISD::XOR) {
06430           N10 = N1.getOperand(0);
06431 
06432           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
06433           // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
06434           // Pattern complexity = 23  cost = 1
06435           {
06436             N11 = N1.getOperand(1);
06437             if (N11.getOpcode() == ISD::BIT_CONVERT) {
06438               N110 = N11.getOperand(0);
06439               if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06440                   Predicate_immAllOnesV(N110.Val) &&
06441                   N.Val->getValueType(0) == MVT::v2i64 &&
06442                   N00.Val->getValueType(0) == MVT::v4f32 &&
06443                   N110.Val->getValueType(0) == MVT::v4i32 &&
06444                   SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
06445                 Select(Tmp0, Tmp0);
06446                 Select(Tmp1, Tmp1);
06447                 Select(Tmp2, Tmp2);
06448                 Select(Tmp3, Tmp3);
06449                 Select(Tmp4, N10);
06450                 Select(Chain00, Chain00);
06451                 ResNode = CurDAG->getTargetNode(X86::ANDNPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
06452                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06453                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
06454                 AddHandleReplacement(N00.Val, 1, ResNode, 1);
06455                 Result = SDOperand(ResNode, N.ResNo);
06456                 return;
06457               }
06458             }
06459           }
06460 
06461           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)), (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
06462           // Emits: (ANDNPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
06463           // Pattern complexity = 23  cost = 1
06464           if (N10.getOpcode() == ISD::BIT_CONVERT) {
06465             N100 = N10.getOperand(0);
06466             if (N100.getOpcode() == ISD::BUILD_VECTOR &&
06467                 Predicate_immAllOnesV(N100.Val)) {
06468               N11 = N1.getOperand(1);
06469               if (N.Val->getValueType(0) == MVT::v2i64 &&
06470                   N00.Val->getValueType(0) == MVT::v4f32 &&
06471                   N100.Val->getValueType(0) == MVT::v4i32 &&
06472                   SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
06473                 Select(Tmp0, Tmp0);
06474                 Select(Tmp1, Tmp1);
06475                 Select(Tmp2, Tmp2);
06476                 Select(Tmp3, Tmp3);
06477                 Select(Tmp4, N11);
06478                 Select(Chain00, Chain00);
06479                 ResNode = CurDAG->getTargetNode(X86::ANDNPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
06480                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06481                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
06482                 AddHandleReplacement(N00.Val, 1, ResNode, 1);
06483                 Result = SDOperand(ResNode, N.ResNo);
06484                 return;
06485               }
06486             }
06487           }
06488         }
06489       }
06490     }
06491   }
06492   if ((Subtarget->hasSSE2())) {
06493     N0 = N.getOperand(0);
06494 
06495     // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)), (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)))
06496     // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
06497     // Pattern complexity = 23  cost = 1
06498     if (N0.getOpcode() == ISD::XOR) {
06499       N00 = N0.getOperand(0);
06500       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
06501           Predicate_immAllOnesV(N00.Val)) {
06502         N01 = N0.getOperand(1);
06503         if (N01.getOpcode() == ISD::BIT_CONVERT) {
06504           N010 = N01.getOperand(0);
06505           N1 = N.getOperand(1);
06506           if (N1.getOpcode() == ISD::BIT_CONVERT &&
06507               N1.hasOneUse() &&
06508               !CodeGenMap.count(N1.getValue(0))) {
06509             N10 = N1.getOperand(0);
06510             if (N10.getOpcode() == ISD::LOAD &&
06511                 N10.hasOneUse() &&
06512                 !CodeGenMap.count(N10.getValue(0))) {
06513               Chain10 = N10.getOperand(0);
06514               N101 = N10.getOperand(1);
06515               if (N.Val->getValueType(0) == MVT::v2i64 &&
06516                   N010.Val->getValueType(0) == MVT::v2f64 &&
06517                   N10.Val->getValueType(0) == MVT::v2f64 &&
06518                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
06519                 Select(Tmp0, Tmp0);
06520                 Select(Tmp1, Tmp1);
06521                 Select(Tmp2, Tmp2);
06522                 Select(Tmp3, Tmp3);
06523                 Select(Tmp4, N010);
06524                 Select(Chain10, Chain10);
06525                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
06526                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06527                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
06528                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
06529                 Result = SDOperand(ResNode, N.ResNo);
06530                 return;
06531               }
06532             }
06533           }
06534         }
06535       }
06536     }
06537     if (N0.getOpcode() == ISD::BIT_CONVERT &&
06538         N0.hasOneUse() &&
06539         !CodeGenMap.count(N0.getValue(0))) {
06540       N00 = N0.getOperand(0);
06541       if (N00.getOpcode() == ISD::LOAD &&
06542           N00.hasOneUse() &&
06543           !CodeGenMap.count(N00.getValue(0))) {
06544         Chain00 = N00.getOperand(0);
06545         N001 = N00.getOperand(1);
06546         N1 = N.getOperand(1);
06547         if (N1.getOpcode() == ISD::XOR) {
06548           N10 = N1.getOperand(0);
06549 
06550           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)), (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
06551           // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
06552           // Pattern complexity = 23  cost = 1
06553           if (N10.getOpcode() == ISD::BIT_CONVERT) {
06554             N100 = N10.getOperand(0);
06555             N11 = N1.getOperand(1);
06556             if (N11.getOpcode() == ISD::BUILD_VECTOR &&
06557                 Predicate_immAllOnesV(N11.Val) &&
06558                 N.Val->getValueType(0) == MVT::v2i64 &&
06559                 N00.Val->getValueType(0) == MVT::v2f64 &&
06560                 N100.Val->getValueType(0) == MVT::v2f64 &&
06561                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
06562               Select(Tmp0, Tmp0);
06563               Select(Tmp1, Tmp1);
06564               Select(Tmp2, Tmp2);
06565               Select(Tmp3, Tmp3);
06566               Select(Tmp4, N100);
06567               Select(Chain00, Chain00);
06568               ResNode = CurDAG->getTargetNode(X86::ANDNPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
06569               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06570               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
06571               AddHandleReplacement(N00.Val, 1, ResNode, 1);
06572               Result = SDOperand(ResNode, N.ResNo);
06573               return;
06574             }
06575           }
06576 
06577           // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)))
06578           // Emits: (ANDNPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
06579           // Pattern complexity = 23  cost = 1
06580           if (N10.getOpcode() == ISD::BUILD_VECTOR &&
06581               Predicate_immAllOnesV(N10.Val)) {
06582             N11 = N1.getOperand(1);
06583             if (N11.getOpcode() == ISD::BIT_CONVERT) {
06584               N110 = N11.getOperand(0);
06585               if (N.Val->getValueType(0) == MVT::v2i64 &&
06586                   N00.Val->getValueType(0) == MVT::v2f64 &&
06587                   N110.Val->getValueType(0) == MVT::v2f64 &&
06588                   SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
06589                 Select(Tmp0, Tmp0);
06590                 Select(Tmp1, Tmp1);
06591                 Select(Tmp2, Tmp2);
06592                 Select(Tmp3, Tmp3);
06593                 Select(Tmp4, N110);
06594                 Select(Chain00, Chain00);
06595                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
06596                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06597                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
06598                 AddHandleReplacement(N00.Val, 1, ResNode, 1);
06599                 Result = SDOperand(ResNode, N.ResNo);
06600                 return;
06601               }
06602             }
06603           }
06604         }
06605       }
06606     }
06607     if (N0.getOpcode() == ISD::XOR) {
06608       N00 = N0.getOperand(0);
06609       {
06610         N01 = N0.getOperand(1);
06611         if (N01.getOpcode() == ISD::BIT_CONVERT) {
06612           N010 = N01.getOperand(0);
06613           if (N010.getOpcode() == ISD::BUILD_VECTOR &&
06614               Predicate_immAllOnesV(N010.Val)) {
06615             N1 = N.getOperand(1);
06616             if (N1.getOpcode() == ISD::LOAD &&
06617                 N1.hasOneUse() &&
06618                 !CodeGenMap.count(N1.getValue(0)) &&
06619                 !isNonImmUse(N.Val, N1.Val)) {
06620               Chain1 = N1.getOperand(0);
06621               N11 = N1.getOperand(1);
06622               if (N.Val->getValueType(0) == MVT::v2i64) {
06623 
06624                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:i32:$src2))
06625                 // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06626                 // Pattern complexity = 21  cost = 1
06627                 if (N010.Val->getValueType(0) == MVT::v4i32 &&
06628                     SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06629                   Select(Tmp0, Tmp0);
06630                   Select(Tmp1, Tmp1);
06631                   Select(Tmp2, Tmp2);
06632                   Select(Tmp3, Tmp3);
06633                   Select(Tmp4, N00);
06634                   Select(Chain1, Chain1);
06635                   ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06636                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06637                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06638                   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06639                   Result = SDOperand(ResNode, N.ResNo);
06640                   return;
06641                 }
06642 
06643                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:i32:$src2))
06644                 // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06645                 // Pattern complexity = 21  cost = 1
06646                 if (N010.Val->getValueType(0) == MVT::v8i16 &&
06647                     SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06648                   Select(Tmp0, Tmp0);
06649                   Select(Tmp1, Tmp1);
06650                   Select(Tmp2, Tmp2);
06651                   Select(Tmp3, Tmp3);
06652                   Select(Tmp4, N00);
06653                   Select(Chain1, Chain1);
06654                   ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06655                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06656                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06657                   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06658                   Result = SDOperand(ResNode, N.ResNo);
06659                   return;
06660                 }
06661 
06662                 // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)), (load:v2i64 addr:i32:$src2))
06663                 // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06664                 // Pattern complexity = 21  cost = 1
06665                 if (N010.Val->getValueType(0) == MVT::v16i8 &&
06666                     SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06667                   Select(Tmp0, Tmp0);
06668                   Select(Tmp1, Tmp1);
06669                   Select(Tmp2, Tmp2);
06670                   Select(Tmp3, Tmp3);
06671                   Select(Tmp4, N00);
06672                   Select(Chain1, Chain1);
06673                   ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06674                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06675                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06676                   AddHandleReplacement(N1.Val, 1, ResNode, 1);
06677                   Result = SDOperand(ResNode, N.ResNo);
06678                   return;
06679                 }
06680               }
06681             }
06682           }
06683         }
06684       }
06685 
06686       // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:i32:$src2))
06687       // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06688       // Pattern complexity = 21  cost = 1
06689       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06690         N000 = N00.getOperand(0);
06691         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06692             Predicate_immAllOnesV(N000.Val)) {
06693           N01 = N0.getOperand(1);
06694           N1 = N.getOperand(1);
06695           if (N1.getOpcode() == ISD::LOAD &&
06696               N1.hasOneUse() &&
06697               !CodeGenMap.count(N1.getValue(0)) &&
06698               !isNonImmUse(N.Val, N1.Val)) {
06699             Chain1 = N1.getOperand(0);
06700             N11 = N1.getOperand(1);
06701             if (N.Val->getValueType(0) == MVT::v2i64 &&
06702                 N000.Val->getValueType(0) == MVT::v4i32 &&
06703                 SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06704               Select(Tmp0, Tmp0);
06705               Select(Tmp1, Tmp1);
06706               Select(Tmp2, Tmp2);
06707               Select(Tmp3, Tmp3);
06708               Select(Tmp4, N01);
06709               Select(Chain1, Chain1);
06710               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06711               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06712               SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06713               AddHandleReplacement(N1.Val, 1, ResNode, 1);
06714               Result = SDOperand(ResNode, N.ResNo);
06715               return;
06716             }
06717           }
06718         }
06719       }
06720     }
06721     if (N0.getOpcode() == ISD::LOAD &&
06722         N0.hasOneUse() &&
06723         !CodeGenMap.count(N0.getValue(0)) &&
06724         !isNonImmUse(N.Val, N0.Val)) {
06725       Chain0 = N0.getOperand(0);
06726       N01 = N0.getOperand(1);
06727       N1 = N.getOperand(1);
06728       if (N1.getOpcode() == ISD::XOR) {
06729         N10 = N1.getOperand(0);
06730 
06731         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
06732         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06733         // Pattern complexity = 21  cost = 1
06734         {
06735           N11 = N1.getOperand(1);
06736           if (N11.getOpcode() == ISD::BIT_CONVERT) {
06737             N110 = N11.getOperand(0);
06738             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06739                 Predicate_immAllOnesV(N110.Val) &&
06740                 N.Val->getValueType(0) == MVT::v2i64 &&
06741                 N110.Val->getValueType(0) == MVT::v4i32 &&
06742                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06743               Select(Tmp0, Tmp0);
06744               Select(Tmp1, Tmp1);
06745               Select(Tmp2, Tmp2);
06746               Select(Tmp3, Tmp3);
06747               Select(Tmp4, N10);
06748               Select(Chain0, Chain0);
06749               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06750               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06751               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06752               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06753               Result = SDOperand(ResNode, N.ResNo);
06754               return;
06755             }
06756           }
06757         }
06758 
06759         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
06760         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06761         // Pattern complexity = 21  cost = 1
06762         if (N10.getOpcode() == ISD::BIT_CONVERT) {
06763           N100 = N10.getOperand(0);
06764           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
06765               Predicate_immAllOnesV(N100.Val)) {
06766             N11 = N1.getOperand(1);
06767             if (N.Val->getValueType(0) == MVT::v2i64 &&
06768                 N100.Val->getValueType(0) == MVT::v4i32 &&
06769                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06770               Select(Tmp0, Tmp0);
06771               Select(Tmp1, Tmp1);
06772               Select(Tmp2, Tmp2);
06773               Select(Tmp3, Tmp3);
06774               Select(Tmp4, N11);
06775               Select(Chain0, Chain0);
06776               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06777               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06778               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06779               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06780               Result = SDOperand(ResNode, N.ResNo);
06781               return;
06782             }
06783           }
06784         }
06785       }
06786     }
06787 
06788     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:i32:$src2))
06789     // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06790     // Pattern complexity = 21  cost = 1
06791     if (N0.getOpcode() == ISD::XOR) {
06792       N00 = N0.getOperand(0);
06793       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06794         N000 = N00.getOperand(0);
06795         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06796             Predicate_immAllOnesV(N000.Val)) {
06797           N01 = N0.getOperand(1);
06798           N1 = N.getOperand(1);
06799           if (N1.getOpcode() == ISD::LOAD &&
06800               N1.hasOneUse() &&
06801               !CodeGenMap.count(N1.getValue(0)) &&
06802               !isNonImmUse(N.Val, N1.Val)) {
06803             Chain1 = N1.getOperand(0);
06804             N11 = N1.getOperand(1);
06805             if (N.Val->getValueType(0) == MVT::v2i64 &&
06806                 N000.Val->getValueType(0) == MVT::v8i16 &&
06807                 SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06808               Select(Tmp0, Tmp0);
06809               Select(Tmp1, Tmp1);
06810               Select(Tmp2, Tmp2);
06811               Select(Tmp3, Tmp3);
06812               Select(Tmp4, N01);
06813               Select(Chain1, Chain1);
06814               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06815               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06816               SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06817               AddHandleReplacement(N1.Val, 1, ResNode, 1);
06818               Result = SDOperand(ResNode, N.ResNo);
06819               return;
06820             }
06821           }
06822         }
06823       }
06824     }
06825     if (N0.getOpcode() == ISD::LOAD &&
06826         N0.hasOneUse() &&
06827         !CodeGenMap.count(N0.getValue(0)) &&
06828         !isNonImmUse(N.Val, N0.Val)) {
06829       Chain0 = N0.getOperand(0);
06830       N01 = N0.getOperand(1);
06831       N1 = N.getOperand(1);
06832       if (N1.getOpcode() == ISD::XOR) {
06833         N10 = N1.getOperand(0);
06834 
06835         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)))
06836         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06837         // Pattern complexity = 21  cost = 1
06838         {
06839           N11 = N1.getOperand(1);
06840           if (N11.getOpcode() == ISD::BIT_CONVERT) {
06841             N110 = N11.getOperand(0);
06842             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06843                 Predicate_immAllOnesV(N110.Val) &&
06844                 N.Val->getValueType(0) == MVT::v2i64 &&
06845                 N110.Val->getValueType(0) == MVT::v8i16 &&
06846                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06847               Select(Tmp0, Tmp0);
06848               Select(Tmp1, Tmp1);
06849               Select(Tmp2, Tmp2);
06850               Select(Tmp3, Tmp3);
06851               Select(Tmp4, N10);
06852               Select(Chain0, Chain0);
06853               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06854               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06855               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06856               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06857               Result = SDOperand(ResNode, N.ResNo);
06858               return;
06859             }
06860           }
06861         }
06862 
06863         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
06864         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06865         // Pattern complexity = 21  cost = 1
06866         if (N10.getOpcode() == ISD::BIT_CONVERT) {
06867           N100 = N10.getOperand(0);
06868           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
06869               Predicate_immAllOnesV(N100.Val)) {
06870             N11 = N1.getOperand(1);
06871             if (N.Val->getValueType(0) == MVT::v2i64 &&
06872                 N100.Val->getValueType(0) == MVT::v8i16 &&
06873                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06874               Select(Tmp0, Tmp0);
06875               Select(Tmp1, Tmp1);
06876               Select(Tmp2, Tmp2);
06877               Select(Tmp3, Tmp3);
06878               Select(Tmp4, N11);
06879               Select(Chain0, Chain0);
06880               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06881               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06882               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06883               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06884               Result = SDOperand(ResNode, N.ResNo);
06885               return;
06886             }
06887           }
06888         }
06889       }
06890     }
06891 
06892     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), (load:v2i64 addr:i32:$src2))
06893     // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06894     // Pattern complexity = 21  cost = 1
06895     if (N0.getOpcode() == ISD::XOR) {
06896       N00 = N0.getOperand(0);
06897       if (N00.getOpcode() == ISD::BIT_CONVERT) {
06898         N000 = N00.getOperand(0);
06899         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
06900             Predicate_immAllOnesV(N000.Val)) {
06901           N01 = N0.getOperand(1);
06902           N1 = N.getOperand(1);
06903           if (N1.getOpcode() == ISD::LOAD &&
06904               N1.hasOneUse() &&
06905               !CodeGenMap.count(N1.getValue(0)) &&
06906               !isNonImmUse(N.Val, N1.Val)) {
06907             Chain1 = N1.getOperand(0);
06908             N11 = N1.getOperand(1);
06909             if (N.Val->getValueType(0) == MVT::v2i64 &&
06910                 N000.Val->getValueType(0) == MVT::v16i8 &&
06911                 SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
06912               Select(Tmp0, Tmp0);
06913               Select(Tmp1, Tmp1);
06914               Select(Tmp2, Tmp2);
06915               Select(Tmp3, Tmp3);
06916               Select(Tmp4, N01);
06917               Select(Chain1, Chain1);
06918               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
06919               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06920               SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
06921               AddHandleReplacement(N1.Val, 1, ResNode, 1);
06922               Result = SDOperand(ResNode, N.ResNo);
06923               return;
06924             }
06925           }
06926         }
06927       }
06928     }
06929     if (N0.getOpcode() == ISD::LOAD &&
06930         N0.hasOneUse() &&
06931         !CodeGenMap.count(N0.getValue(0)) &&
06932         !isNonImmUse(N.Val, N0.Val)) {
06933       Chain0 = N0.getOperand(0);
06934       N01 = N0.getOperand(1);
06935       N1 = N.getOperand(1);
06936       if (N1.getOpcode() == ISD::XOR) {
06937         N10 = N1.getOperand(0);
06938 
06939         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)))
06940         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06941         // Pattern complexity = 21  cost = 1
06942         {
06943           N11 = N1.getOperand(1);
06944           if (N11.getOpcode() == ISD::BIT_CONVERT) {
06945             N110 = N11.getOperand(0);
06946             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
06947                 Predicate_immAllOnesV(N110.Val) &&
06948                 N.Val->getValueType(0) == MVT::v2i64 &&
06949                 N110.Val->getValueType(0) == MVT::v16i8 &&
06950                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06951               Select(Tmp0, Tmp0);
06952               Select(Tmp1, Tmp1);
06953               Select(Tmp2, Tmp2);
06954               Select(Tmp3, Tmp3);
06955               Select(Tmp4, N10);
06956               Select(Chain0, Chain0);
06957               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06958               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06959               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06960               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06961               Result = SDOperand(ResNode, N.ResNo);
06962               return;
06963             }
06964           }
06965         }
06966 
06967         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
06968         // Emits: (PANDNrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
06969         // Pattern complexity = 21  cost = 1
06970         if (N10.getOpcode() == ISD::BIT_CONVERT) {
06971           N100 = N10.getOperand(0);
06972           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
06973               Predicate_immAllOnesV(N100.Val)) {
06974             N11 = N1.getOperand(1);
06975             if (N.Val->getValueType(0) == MVT::v2i64 &&
06976                 N100.Val->getValueType(0) == MVT::v16i8 &&
06977                 SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
06978               Select(Tmp0, Tmp0);
06979               Select(Tmp1, Tmp1);
06980               Select(Tmp2, Tmp2);
06981               Select(Tmp3, Tmp3);
06982               Select(Tmp4, N11);
06983               Select(Chain0, Chain0);
06984               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
06985               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
06986               SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
06987               AddHandleReplacement(N0.Val, 1, ResNode, 1);
06988               Result = SDOperand(ResNode, N.ResNo);
06989               return;
06990             }
06991           }
06992         }
06993       }
06994     }
06995     if (N0.getOpcode() == ISD::XOR) {
06996       N00 = N0.getOperand(0);
06997 
06998       // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (load:v2i64 addr:i32:$src2))
06999       // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07000       // Pattern complexity = 19  cost = 1
07001       {
07002         N01 = N0.getOperand(1);
07003         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07004             Predicate_immAllOnesV(N01.Val)) {
07005           N1 = N.getOperand(1);
07006           if (N1.getOpcode() == ISD::LOAD &&
07007               N1.hasOneUse() &&
07008               !CodeGenMap.count(N1.getValue(0)) &&
07009               !isNonImmUse(N.Val, N1.Val)) {
07010             Chain1 = N1.getOperand(0);
07011             N11 = N1.getOperand(1);
07012             if (N.Val->getValueType(0) == MVT::v2i64 &&
07013                 SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07014               Select(Tmp0, Tmp0);
07015               Select(Tmp1, Tmp1);
07016               Select(Tmp2, Tmp2);
07017               Select(Tmp3, Tmp3);
07018               Select(Tmp4, N00);
07019               Select(Chain1, Chain1);
07020               ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07021               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07022               SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07023               AddHandleReplacement(N1.Val, 1, ResNode, 1);
07024               Result = SDOperand(ResNode, N.ResNo);
07025               return;
07026             }
07027           }
07028         }
07029       }
07030 
07031       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1), (load:v2i64 addr:i32:$src2))
07032       // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07033       // Pattern complexity = 19  cost = 1
07034       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07035           Predicate_immAllOnesV(N00.Val)) {
07036         N01 = N0.getOperand(1);
07037         N1 = N.getOperand(1);
07038         if (N1.getOpcode() == ISD::LOAD &&
07039             N1.hasOneUse() &&
07040             !CodeGenMap.count(N1.getValue(0)) &&
07041             !isNonImmUse(N.Val, N1.Val)) {
07042           Chain1 = N1.getOperand(0);
07043           N11 = N1.getOperand(1);
07044           if (N.Val->getValueType(0) == MVT::v2i64 &&
07045               SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07046             Select(Tmp0, Tmp0);
07047             Select(Tmp1, Tmp1);
07048             Select(Tmp2, Tmp2);
07049             Select(Tmp3, Tmp3);
07050             Select(Tmp4, N01);
07051             Select(Chain1, Chain1);
07052             ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07053             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07054             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07055             AddHandleReplacement(N1.Val, 1, ResNode, 1);
07056             Result = SDOperand(ResNode, N.ResNo);
07057             return;
07058           }
07059         }
07060       }
07061     }
07062     if (N0.getOpcode() == ISD::LOAD &&
07063         N0.hasOneUse() &&
07064         !CodeGenMap.count(N0.getValue(0)) &&
07065         !isNonImmUse(N.Val, N0.Val)) {
07066       Chain0 = N0.getOperand(0);
07067       N01 = N0.getOperand(1);
07068       N1 = N.getOperand(1);
07069       if (N1.getOpcode() == ISD::XOR) {
07070         N10 = N1.getOperand(0);
07071 
07072         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07073         // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07074         // Pattern complexity = 19  cost = 1
07075         {
07076           N11 = N1.getOperand(1);
07077           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
07078               Predicate_immAllOnesV(N11.Val) &&
07079               N.Val->getValueType(0) == MVT::v2i64 &&
07080               SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07081             Select(Tmp0, Tmp0);
07082             Select(Tmp1, Tmp1);
07083             Select(Tmp2, Tmp2);
07084             Select(Tmp3, Tmp3);
07085             Select(Tmp4, N10);
07086             Select(Chain0, Chain0);
07087             ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07088             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07089             SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07090             AddHandleReplacement(N0.Val, 1, ResNode, 1);
07091             Result = SDOperand(ResNode, N.ResNo);
07092             return;
07093           }
07094         }
07095 
07096         // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1))
07097         // Emits: (PANDNrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07098         // Pattern complexity = 19  cost = 1
07099         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
07100             Predicate_immAllOnesV(N10.Val)) {
07101           N11 = N1.getOperand(1);
07102           if (N.Val->getValueType(0) == MVT::v2i64 &&
07103               SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07104             Select(Tmp0, Tmp0);
07105             Select(Tmp1, Tmp1);
07106             Select(Tmp2, Tmp2);
07107             Select(Tmp3, Tmp3);
07108             Select(Tmp4, N11);
07109             Select(Chain0, Chain0);
07110             ResNode = CurDAG->getTargetNode(X86::PANDNrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07111             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07112             SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07113             AddHandleReplacement(N0.Val, 1, ResNode, 1);
07114             Result = SDOperand(ResNode, N.ResNo);
07115             return;
07116           }
07117         }
07118       }
07119     }
07120     if (N0.getOpcode() == ISD::BIT_CONVERT) {
07121 
07122       // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)))
07123       // Emits: (ANDPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
07124       // Pattern complexity = 18  cost = 1
07125       {
07126         N00 = N0.getOperand(0);
07127         N1 = N.getOperand(1);
07128         if (N1.getOpcode() == ISD::BIT_CONVERT &&
07129             N1.hasOneUse() &&
07130             !CodeGenMap.count(N1.getValue(0))) {
07131           N10 = N1.getOperand(0);
07132           if (N10.getOpcode() == ISD::LOAD &&
07133               N10.hasOneUse() &&
07134               !CodeGenMap.count(N10.getValue(0))) {
07135             Chain10 = N10.getOperand(0);
07136             N101 = N10.getOperand(1);
07137             if (N.Val->getValueType(0) == MVT::v2i64 &&
07138                 N00.Val->getValueType(0) == MVT::v2f64 &&
07139                 N10.Val->getValueType(0) == MVT::v2f64 &&
07140                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
07141               Select(Tmp0, Tmp0);
07142               Select(Tmp1, Tmp1);
07143               Select(Tmp2, Tmp2);
07144               Select(Tmp3, Tmp3);
07145               Select(Tmp4, N00);
07146               Select(Chain10, Chain10);
07147               ResNode = CurDAG->getTargetNode(X86::ANDPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
07148               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07149               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
07150               AddHandleReplacement(N10.Val, 1, ResNode, 1);
07151               Result = SDOperand(ResNode, N.ResNo);
07152               return;
07153             }
07154           }
07155         }
07156       }
07157 
07158       // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
07159       // Emits: (ANDPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
07160       // Pattern complexity = 18  cost = 1
07161       if (N0.hasOneUse() &&
07162           !CodeGenMap.count(N0.getValue(0))) {
07163         N00 = N0.getOperand(0);
07164         if (N00.getOpcode() == ISD::LOAD &&
07165             N00.hasOneUse() &&
07166             !CodeGenMap.count(N00.getValue(0))) {
07167           Chain00 = N00.getOperand(0);
07168           N001 = N00.getOperand(1);
07169           N1 = N.getOperand(1);
07170           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07171             N10 = N1.getOperand(0);
07172             if (N.Val->getValueType(0) == MVT::v2i64 &&
07173                 N00.Val->getValueType(0) == MVT::v2f64 &&
07174                 N10.Val->getValueType(0) == MVT::v2f64 &&
07175                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
07176               Select(Tmp0, Tmp0);
07177               Select(Tmp1, Tmp1);
07178               Select(Tmp2, Tmp2);
07179               Select(Tmp3, Tmp3);
07180               Select(Tmp4, N10);
07181               Select(Chain00, Chain00);
07182               ResNode = CurDAG->getTargetNode(X86::ANDPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
07183               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07184               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
07185               AddHandleReplacement(N00.Val, 1, ResNode, 1);
07186               Result = SDOperand(ResNode, N.ResNo);
07187               return;
07188             }
07189           }
07190         }
07191       }
07192     }
07193   }
07194   if ((Subtarget->hasSSE1())) {
07195     N0 = N.getOperand(0);
07196 
07197     // Pattern: (and:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)))
07198     // Emits: (ANDPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07199     // Pattern complexity = 16  cost = 1
07200     {
07201       N1 = N.getOperand(1);
07202       if (N1.getOpcode() == ISD::BIT_CONVERT &&
07203           N1.hasOneUse() &&
07204           !CodeGenMap.count(N1.getValue(0))) {
07205         N10 = N1.getOperand(0);
07206         if (N10.getOpcode() == ISD::LOAD &&
07207             N10.hasOneUse() &&
07208             !CodeGenMap.count(N10.getValue(0))) {
07209           Chain10 = N10.getOperand(0);
07210           N101 = N10.getOperand(1);
07211           if (N.Val->getValueType(0) == MVT::v2i64 &&
07212               N10.Val->getValueType(0) == MVT::v4f32 &&
07213               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
07214             Select(Tmp0, Tmp0);
07215             Select(Tmp1, Tmp1);
07216             Select(Tmp2, Tmp2);
07217             Select(Tmp3, Tmp3);
07218             Select(Tmp4, N0);
07219             Select(Chain10, Chain10);
07220             ResNode = CurDAG->getTargetNode(X86::ANDPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
07221             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07222             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
07223             AddHandleReplacement(N10.Val, 1, ResNode, 1);
07224             Result = SDOperand(ResNode, N.ResNo);
07225             return;
07226           }
07227         }
07228       }
07229     }
07230 
07231     // Pattern: (and:v2i64 (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)), VR128:v2i64:$src1)
07232     // Emits: (ANDPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07233     // Pattern complexity = 16  cost = 1
07234     if (N0.getOpcode() == ISD::BIT_CONVERT &&
07235         N0.hasOneUse() &&
07236         !CodeGenMap.count(N0.getValue(0))) {
07237       N00 = N0.getOperand(0);
07238       if (N00.getOpcode() == ISD::LOAD &&
07239           N00.hasOneUse() &&
07240           !CodeGenMap.count(N00.getValue(0))) {
07241         Chain00 = N00.getOperand(0);
07242         N001 = N00.getOperand(1);
07243         N1 = N.getOperand(1);
07244         if (N.Val->getValueType(0) == MVT::v2i64 &&
07245             N00.Val->getValueType(0) == MVT::v4f32 &&
07246             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
07247           Select(Tmp0, Tmp0);
07248           Select(Tmp1, Tmp1);
07249           Select(Tmp2, Tmp2);
07250           Select(Tmp3, Tmp3);
07251           Select(Tmp4, N1);
07252           Select(Chain00, Chain00);
07253           ResNode = CurDAG->getTargetNode(X86::ANDPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
07254           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07255           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
07256           AddHandleReplacement(N00.Val, 1, ResNode, 1);
07257           Result = SDOperand(ResNode, N.ResNo);
07258           return;
07259         }
07260       }
07261     }
07262   }
07263   {
07264     N0 = N.getOperand(0);
07265     N1 = N.getOperand(1);
07266     if (N1.getOpcode() == ISD::LOAD &&
07267         N1.hasOneUse() &&
07268         !CodeGenMap.count(N1.getValue(0)) &&
07269         !isNonImmUse(N.Val, N1.Val)) {
07270       Chain1 = N1.getOperand(0);
07271       N11 = N1.getOperand(1);
07272 
07273       // Pattern: (and:i8 R8:i8:$src1, (load:i8 addr:i32:$src2))
07274       // Emits: (AND8rm:i8 R8:i8:$src1, addr:i32:$src2)
07275       // Pattern complexity = 14  cost = 1
07276       if (N.Val->getValueType(0) == MVT::i8 &&
07277           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07278         Select(Tmp0, Tmp0);
07279         Select(Tmp1, Tmp1);
07280         Select(Tmp2, Tmp2);
07281         Select(Tmp3, Tmp3);
07282         Select(Tmp4, N0);
07283         Select(Chain1, Chain1);
07284         ResNode = CurDAG->getTargetNode(X86::AND8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07285         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07286         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07287         AddHandleReplacement(N1.Val, 1, ResNode, 1);
07288         Result = SDOperand(ResNode, N.ResNo);
07289         return;
07290       }
07291 
07292       // Pattern: (and:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
07293       // Emits: (AND16rm:i16 R16:i16:$src1, addr:i32:$src2)
07294       // Pattern complexity = 14  cost = 1
07295       if (N.Val->getValueType(0) == MVT::i16 &&
07296           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07297         Select(Tmp0, Tmp0);
07298         Select(Tmp1, Tmp1);
07299         Select(Tmp2, Tmp2);
07300         Select(Tmp3, Tmp3);
07301         Select(Tmp4, N0);
07302         Select(Chain1, Chain1);
07303         ResNode = CurDAG->getTargetNode(X86::AND16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07304         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07305         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07306         AddHandleReplacement(N1.Val, 1, ResNode, 1);
07307         Result = SDOperand(ResNode, N.ResNo);
07308         return;
07309       }
07310 
07311       // Pattern: (and:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
07312       // Emits: (AND32rm:i32 R32:i32:$src1, addr:i32:$src2)
07313       // Pattern complexity = 14  cost = 1
07314       if (N.Val->getValueType(0) == MVT::i32 &&
07315           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07316         Select(Tmp0, Tmp0);
07317         Select(Tmp1, Tmp1);
07318         Select(Tmp2, Tmp2);
07319         Select(Tmp3, Tmp3);
07320         Select(Tmp4, N0);
07321         Select(Chain1, Chain1);
07322         ResNode = CurDAG->getTargetNode(X86::AND32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07323         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07324         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07325         AddHandleReplacement(N1.Val, 1, ResNode, 1);
07326         Result = SDOperand(ResNode, N.ResNo);
07327         return;
07328       }
07329     }
07330   }
07331 
07332   // Pattern: (and:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2))
07333   // Emits: (PANDrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07334   // Pattern complexity = 14  cost = 1
07335   if ((Subtarget->hasSSE2())) {
07336     N0 = N.getOperand(0);
07337     N1 = N.getOperand(1);
07338     if (N1.getOpcode() == ISD::LOAD &&
07339         N1.hasOneUse() &&
07340         !CodeGenMap.count(N1.getValue(0)) &&
07341         !isNonImmUse(N.Val, N1.Val)) {
07342       Chain1 = N1.getOperand(0);
07343       N11 = N1.getOperand(1);
07344       if (N.Val->getValueType(0) == MVT::v2i64 &&
07345           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
07346         Select(Tmp0, Tmp0);
07347         Select(Tmp1, Tmp1);
07348         Select(Tmp2, Tmp2);
07349         Select(Tmp3, Tmp3);
07350         Select(Tmp4, N0);
07351         Select(Chain1, Chain1);
07352         ResNode = CurDAG->getTargetNode(X86::PANDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
07353         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07354         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
07355         AddHandleReplacement(N1.Val, 1, ResNode, 1);
07356         Result = SDOperand(ResNode, N.ResNo);
07357         return;
07358       }
07359     }
07360   }
07361   {
07362     N0 = N.getOperand(0);
07363     if (N0.getOpcode() == ISD::LOAD &&
07364         N0.hasOneUse() &&
07365         !CodeGenMap.count(N0.getValue(0)) &&
07366         !isNonImmUse(N.Val, N0.Val)) {
07367       Chain0 = N0.getOperand(0);
07368       N01 = N0.getOperand(1);
07369       N1 = N.getOperand(1);
07370 
07371       // Pattern: (and:i8 (load:i8 addr:i32:$src2), R8:i8:$src1)
07372       // Emits: (AND8rm:i8 R8:i8:$src1, addr:i32:$src2)
07373       // Pattern complexity = 14  cost = 1
07374       if (N.Val->getValueType(0) == MVT::i8 &&
07375           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07376         Select(Tmp0, Tmp0);
07377         Select(Tmp1, Tmp1);
07378         Select(Tmp2, Tmp2);
07379         Select(Tmp3, Tmp3);
07380         Select(Tmp4, N1);
07381         Select(Chain0, Chain0);
07382         ResNode = CurDAG->getTargetNode(X86::AND8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07383         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07384         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07385         AddHandleReplacement(N0.Val, 1, ResNode, 1);
07386         Result = SDOperand(ResNode, N.ResNo);
07387         return;
07388       }
07389 
07390       // Pattern: (and:i16 (load:i16 addr:i32:$src2), R16:i16:$src1)
07391       // Emits: (AND16rm:i16 R16:i16:$src1, addr:i32:$src2)
07392       // Pattern complexity = 14  cost = 1
07393       if (N.Val->getValueType(0) == MVT::i16 &&
07394           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07395         Select(Tmp0, Tmp0);
07396         Select(Tmp1, Tmp1);
07397         Select(Tmp2, Tmp2);
07398         Select(Tmp3, Tmp3);
07399         Select(Tmp4, N1);
07400         Select(Chain0, Chain0);
07401         ResNode = CurDAG->getTargetNode(X86::AND16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07402         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07403         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07404         AddHandleReplacement(N0.Val, 1, ResNode, 1);
07405         Result = SDOperand(ResNode, N.ResNo);
07406         return;
07407       }
07408 
07409       // Pattern: (and:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
07410       // Emits: (AND32rm:i32 R32:i32:$src1, addr:i32:$src2)
07411       // Pattern complexity = 14  cost = 1
07412       if (N.Val->getValueType(0) == MVT::i32 &&
07413           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07414         Select(Tmp0, Tmp0);
07415         Select(Tmp1, Tmp1);
07416         Select(Tmp2, Tmp2);
07417         Select(Tmp3, Tmp3);
07418         Select(Tmp4, N1);
07419         Select(Chain0, Chain0);
07420         ResNode = CurDAG->getTargetNode(X86::AND32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07421         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07422         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07423         AddHandleReplacement(N0.Val, 1, ResNode, 1);
07424         Result = SDOperand(ResNode, N.ResNo);
07425         return;
07426       }
07427     }
07428   }
07429   if ((Subtarget->hasSSE2())) {
07430     N0 = N.getOperand(0);
07431 
07432     // Pattern: (and:v2i64 (load:v2i64 addr:i32:$src2), VR128:v2i64:$src1)
07433     // Emits: (PANDrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
07434     // Pattern complexity = 14  cost = 1
07435     if (N0.getOpcode() == ISD::LOAD &&
07436         N0.hasOneUse() &&
07437         !CodeGenMap.count(N0.getValue(0)) &&
07438         !isNonImmUse(N.Val, N0.Val)) {
07439       Chain0 = N0.getOperand(0);
07440       N01 = N0.getOperand(1);
07441       N1 = N.getOperand(1);
07442       if (N.Val->getValueType(0) == MVT::v2i64 &&
07443           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
07444         Select(Tmp0, Tmp0);
07445         Select(Tmp1, Tmp1);
07446         Select(Tmp2, Tmp2);
07447         Select(Tmp3, Tmp3);
07448         Select(Tmp4, N1);
07449         Select(Chain0, Chain0);
07450         ResNode = CurDAG->getTargetNode(X86::PANDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
07451         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
07452         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
07453         AddHandleReplacement(N0.Val, 1, ResNode, 1);
07454         Result = SDOperand(ResNode, N.ResNo);
07455         return;
07456       }
07457     }
07458     if (N0.getOpcode() == ISD::XOR) {
07459       N00 = N0.getOperand(0);
07460 
07461       // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), (bitconvert:v2i64 VR128:v2f64:$src2))
07462       // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07463       // Pattern complexity = 11  cost = 1
07464       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07465         N000 = N00.getOperand(0);
07466         N01 = N0.getOperand(1);
07467         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07468             Predicate_immAllOnesV(N01.Val)) {
07469           N1 = N.getOperand(1);
07470           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07471             N10 = N1.getOperand(0);
07472             if (N.Val->getValueType(0) == MVT::v2i64 &&
07473                 N000.Val->getValueType(0) == MVT::v2f64 &&
07474                 N10.Val->getValueType(0) == MVT::v2f64) {
07475               Select(Tmp0, N000);
07476               Select(Tmp1, N10);
07477               if (N.Val->hasOneUse()) {
07478                 Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07479               } else {
07480                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07481                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07482                 Result = SDOperand(ResNode, 0);
07483               }
07484               return;
07485             }
07486           }
07487         }
07488       }
07489 
07490       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)), (bitconvert:v2i64 VR128:v2f64:$src2))
07491       // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07492       // Pattern complexity = 11  cost = 1
07493       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07494           Predicate_immAllOnesV(N00.Val)) {
07495         N01 = N0.getOperand(1);
07496         if (N01.getOpcode() == ISD::BIT_CONVERT) {
07497           N010 = N01.getOperand(0);
07498           N1 = N.getOperand(1);
07499           if (N1.getOpcode() == ISD::BIT_CONVERT) {
07500             N10 = N1.getOperand(0);
07501             if (N.Val->getValueType(0) == MVT::v2i64 &&
07502                 N010.Val->getValueType(0) == MVT::v2f64 &&
07503                 N10.Val->getValueType(0) == MVT::v2f64) {
07504               Select(Tmp0, N010);
07505               Select(Tmp1, N10);
07506               if (N.Val->hasOneUse()) {
07507                 Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07508               } else {
07509                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07510                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07511                 Result = SDOperand(ResNode, 0);
07512               }
07513               return;
07514             }
07515           }
07516         }
07517       }
07518     }
07519     if (N0.getOpcode() == ISD::BIT_CONVERT) {
07520       N00 = N0.getOperand(0);
07521       N1 = N.getOperand(1);
07522       if (N1.getOpcode() == ISD::XOR) {
07523         N10 = N1.getOperand(0);
07524 
07525         // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src2), (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07526         // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07527         // Pattern complexity = 11  cost = 1
07528         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07529           N100 = N10.getOperand(0);
07530           N11 = N1.getOperand(1);
07531           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
07532               Predicate_immAllOnesV(N11.Val) &&
07533               N.Val->getValueType(0) == MVT::v2i64 &&
07534               N00.Val->getValueType(0) == MVT::v2f64 &&
07535               N100.Val->getValueType(0) == MVT::v2f64) {
07536             Select(Tmp0, N100);
07537             Select(Tmp1, N00);
07538             if (N.Val->hasOneUse()) {
07539               Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07540             } else {
07541               ResNode = CurDAG->getTargetNode(X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07542               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07543               Result = SDOperand(ResNode, 0);
07544             }
07545             return;
07546           }
07547         }
07548 
07549         // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src2), (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, (bitconvert:v2i64 VR128:v2f64:$src1)))
07550         // Emits: (ANDNPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
07551         // Pattern complexity = 11  cost = 1
07552         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
07553             Predicate_immAllOnesV(N10.Val)) {
07554           N11 = N1.getOperand(1);
07555           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07556             N110 = N11.getOperand(0);
07557             if (N.Val->getValueType(0) == MVT::v2i64 &&
07558                 N00.Val->getValueType(0) == MVT::v2f64 &&
07559                 N110.Val->getValueType(0) == MVT::v2f64) {
07560               Select(Tmp0, N110);
07561               Select(Tmp1, N00);
07562               if (N.Val->hasOneUse()) {
07563                 Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07564               } else {
07565                 ResNode = CurDAG->getTargetNode(X86::ANDNPDrr, MVT::v2i64, Tmp0, Tmp1);
07566                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07567                 Result = SDOperand(ResNode, 0);
07568               }
07569               return;
07570             }
07571           }
07572         }
07573       }
07574     }
07575   }
07576 
07577   // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07578   // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07579   // Pattern complexity = 9  cost = 1
07580   if ((Subtarget->hasSSE1())) {
07581     N0 = N.getOperand(0);
07582     if (N0.getOpcode() == ISD::XOR) {
07583       N00 = N0.getOperand(0);
07584       N01 = N0.getOperand(1);
07585       if (N01.getOpcode() == ISD::BIT_CONVERT) {
07586         N010 = N01.getOperand(0);
07587         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
07588             Predicate_immAllOnesV(N010.Val)) {
07589           N1 = N.getOperand(1);
07590           if (N.Val->getValueType(0) == MVT::v2i64 &&
07591               N010.Val->getValueType(0) == MVT::v4i32) {
07592             Select(Tmp0, N00);
07593             Select(Tmp1, N1);
07594             if (N.Val->hasOneUse()) {
07595               Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07596             } else {
07597               ResNode = CurDAG->getTargetNode(X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07598               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07599               Result = SDOperand(ResNode, 0);
07600             }
07601             return;
07602           }
07603         }
07604       }
07605     }
07606   }
07607   if ((Subtarget->hasSSE2())) {
07608     N0 = N.getOperand(0);
07609     if (N0.getOpcode() == ISD::XOR) {
07610       N00 = N0.getOperand(0);
07611       N01 = N0.getOperand(1);
07612       if (N01.getOpcode() == ISD::BIT_CONVERT) {
07613         N010 = N01.getOperand(0);
07614         if (N010.getOpcode() == ISD::BUILD_VECTOR &&
07615             Predicate_immAllOnesV(N010.Val)) {
07616           N1 = N.getOperand(1);
07617           if (N.Val->getValueType(0) == MVT::v2i64) {
07618 
07619             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07620             // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07621             // Pattern complexity = 9  cost = 1
07622             if (N010.Val->getValueType(0) == MVT::v4i32) {
07623               Select(Tmp0, N00);
07624               Select(Tmp1, N1);
07625               if (N.Val->hasOneUse()) {
07626                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07627               } else {
07628                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07629                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07630                 Result = SDOperand(ResNode, 0);
07631               }
07632               return;
07633             }
07634 
07635             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07636             // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07637             // Pattern complexity = 9  cost = 1
07638             if (N010.Val->getValueType(0) == MVT::v8i16) {
07639               Select(Tmp0, N00);
07640               Select(Tmp1, N1);
07641               if (N.Val->hasOneUse()) {
07642                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07643               } else {
07644                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07645                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07646                 Result = SDOperand(ResNode, 0);
07647               }
07648               return;
07649             }
07650 
07651             // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)), VR128:v2i64:$src2)
07652             // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07653             // Pattern complexity = 9  cost = 1
07654             if (N010.Val->getValueType(0) == MVT::v16i8) {
07655               Select(Tmp0, N00);
07656               Select(Tmp1, N1);
07657               if (N.Val->hasOneUse()) {
07658                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07659               } else {
07660                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07661                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07662                 Result = SDOperand(ResNode, 0);
07663               }
07664               return;
07665             }
07666           }
07667         }
07668       }
07669     }
07670   }
07671   if ((Subtarget->hasSSE1())) {
07672     N0 = N.getOperand(0);
07673 
07674     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07675     // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07676     // Pattern complexity = 9  cost = 1
07677     if (N0.getOpcode() == ISD::XOR) {
07678       N00 = N0.getOperand(0);
07679       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07680         N000 = N00.getOperand(0);
07681         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07682             Predicate_immAllOnesV(N000.Val)) {
07683           N01 = N0.getOperand(1);
07684           N1 = N.getOperand(1);
07685           if (N.Val->getValueType(0) == MVT::v2i64 &&
07686               N000.Val->getValueType(0) == MVT::v4i32) {
07687             Select(Tmp0, N01);
07688             Select(Tmp1, N1);
07689             if (N.Val->hasOneUse()) {
07690               Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07691             } else {
07692               ResNode = CurDAG->getTargetNode(X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07693               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07694               Result = SDOperand(ResNode, 0);
07695             }
07696             return;
07697           }
07698         }
07699       }
07700     }
07701     N1 = N.getOperand(1);
07702     if (N1.getOpcode() == ISD::XOR) {
07703       N10 = N1.getOperand(0);
07704 
07705       // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>)))
07706       // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07707       // Pattern complexity = 9  cost = 1
07708       {
07709         N11 = N1.getOperand(1);
07710         if (N11.getOpcode() == ISD::BIT_CONVERT) {
07711           N110 = N11.getOperand(0);
07712           if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07713               Predicate_immAllOnesV(N110.Val) &&
07714               N.Val->getValueType(0) == MVT::v2i64 &&
07715               N110.Val->getValueType(0) == MVT::v4i32) {
07716             Select(Tmp0, N10);
07717             Select(Tmp1, N0);
07718             if (N.Val->hasOneUse()) {
07719               Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07720             } else {
07721               ResNode = CurDAG->getTargetNode(X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07722               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07723               Result = SDOperand(ResNode, 0);
07724             }
07725             return;
07726           }
07727         }
07728       }
07729 
07730       // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v4i32)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07731       // Emits: (ANDNPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07732       // Pattern complexity = 9  cost = 1
07733       if (N10.getOpcode() == ISD::BIT_CONVERT) {
07734         N100 = N10.getOperand(0);
07735         if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07736             Predicate_immAllOnesV(N100.Val)) {
07737           N11 = N1.getOperand(1);
07738           if (N.Val->getValueType(0) == MVT::v2i64 &&
07739               N100.Val->getValueType(0) == MVT::v4i32) {
07740             Select(Tmp0, N11);
07741             Select(Tmp1, N0);
07742             if (N.Val->hasOneUse()) {
07743               Result = CurDAG->SelectNodeTo(N.Val, X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07744             } else {
07745               ResNode = CurDAG->getTargetNode(X86::ANDNPSrr, MVT::v2i64, Tmp0, Tmp1);
07746               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07747               Result = SDOperand(ResNode, 0);
07748             }
07749             return;
07750           }
07751         }
07752       }
07753     }
07754   }
07755   if ((Subtarget->hasSSE2())) {
07756     N0 = N.getOperand(0);
07757 
07758     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07759     // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07760     // Pattern complexity = 9  cost = 1
07761     if (N0.getOpcode() == ISD::XOR) {
07762       N00 = N0.getOperand(0);
07763       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07764         N000 = N00.getOperand(0);
07765         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07766             Predicate_immAllOnesV(N000.Val)) {
07767           N01 = N0.getOperand(1);
07768           N1 = N.getOperand(1);
07769           if (N.Val->getValueType(0) == MVT::v2i64 &&
07770               N000.Val->getValueType(0) == MVT::v8i16) {
07771             Select(Tmp0, N01);
07772             Select(Tmp1, N1);
07773             if (N.Val->hasOneUse()) {
07774               Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07775             } else {
07776               ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07777               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07778               Result = SDOperand(ResNode, 0);
07779             }
07780             return;
07781           }
07782         }
07783       }
07784     }
07785     {
07786       N1 = N.getOperand(1);
07787       if (N1.getOpcode() == ISD::XOR) {
07788         N10 = N1.getOperand(0);
07789 
07790         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>)))
07791         // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07792         // Pattern complexity = 9  cost = 1
07793         {
07794           N11 = N1.getOperand(1);
07795           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07796             N110 = N11.getOperand(0);
07797             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07798                 Predicate_immAllOnesV(N110.Val) &&
07799                 N.Val->getValueType(0) == MVT::v2i64 &&
07800                 N110.Val->getValueType(0) == MVT::v8i16) {
07801               Select(Tmp0, N10);
07802               Select(Tmp1, N0);
07803               if (N.Val->hasOneUse()) {
07804                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07805               } else {
07806                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07807                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07808                 Result = SDOperand(ResNode, 0);
07809               }
07810               return;
07811             }
07812           }
07813         }
07814 
07815         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v8i16)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07816         // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07817         // Pattern complexity = 9  cost = 1
07818         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07819           N100 = N10.getOperand(0);
07820           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07821               Predicate_immAllOnesV(N100.Val)) {
07822             N11 = N1.getOperand(1);
07823             if (N.Val->getValueType(0) == MVT::v2i64 &&
07824                 N100.Val->getValueType(0) == MVT::v8i16) {
07825               Select(Tmp0, N11);
07826               Select(Tmp1, N0);
07827               if (N.Val->hasOneUse()) {
07828                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07829               } else {
07830                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07831                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07832                 Result = SDOperand(ResNode, 0);
07833               }
07834               return;
07835             }
07836           }
07837         }
07838       }
07839     }
07840 
07841     // Pattern: (and:v2i64 (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1), VR128:v2i64:$src2)
07842     // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07843     // Pattern complexity = 9  cost = 1
07844     if (N0.getOpcode() == ISD::XOR) {
07845       N00 = N0.getOperand(0);
07846       if (N00.getOpcode() == ISD::BIT_CONVERT) {
07847         N000 = N00.getOperand(0);
07848         if (N000.getOpcode() == ISD::BUILD_VECTOR &&
07849             Predicate_immAllOnesV(N000.Val)) {
07850           N01 = N0.getOperand(1);
07851           N1 = N.getOperand(1);
07852           if (N.Val->getValueType(0) == MVT::v2i64 &&
07853               N000.Val->getValueType(0) == MVT::v16i8) {
07854             Select(Tmp0, N01);
07855             Select(Tmp1, N1);
07856             if (N.Val->hasOneUse()) {
07857               Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07858             } else {
07859               ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07860               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07861               Result = SDOperand(ResNode, 0);
07862             }
07863             return;
07864           }
07865         }
07866       }
07867     }
07868     {
07869       N1 = N.getOperand(1);
07870       if (N1.getOpcode() == ISD::XOR) {
07871         N10 = N1.getOperand(0);
07872 
07873         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>)))
07874         // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07875         // Pattern complexity = 9  cost = 1
07876         {
07877           N11 = N1.getOperand(1);
07878           if (N11.getOpcode() == ISD::BIT_CONVERT) {
07879             N110 = N11.getOperand(0);
07880             if (N110.getOpcode() == ISD::BUILD_VECTOR &&
07881                 Predicate_immAllOnesV(N110.Val) &&
07882                 N.Val->getValueType(0) == MVT::v2i64 &&
07883                 N110.Val->getValueType(0) == MVT::v16i8) {
07884               Select(Tmp0, N10);
07885               Select(Tmp1, N0);
07886               if (N.Val->hasOneUse()) {
07887                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07888               } else {
07889                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07890                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07891                 Result = SDOperand(ResNode, 0);
07892               }
07893               return;
07894             }
07895           }
07896         }
07897 
07898         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (bitconvert:v2i64 (build_vector:v16i8)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src1))
07899         // Emits: (PANDNrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
07900         // Pattern complexity = 9  cost = 1
07901         if (N10.getOpcode() == ISD::BIT_CONVERT) {
07902           N100 = N10.getOperand(0);
07903           if (N100.getOpcode() == ISD::BUILD_VECTOR &&
07904               Predicate_immAllOnesV(N100.Val)) {
07905             N11 = N1.getOperand(1);
07906             if (N.Val->getValueType(0) == MVT::v2i64 &&
07907                 N100.Val->getValueType(0) == MVT::v16i8) {
07908               Select(Tmp0, N11);
07909               Select(Tmp1, N0);
07910               if (N.Val->hasOneUse()) {
07911                 Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07912               } else {
07913                 ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v16i8, Tmp0, Tmp1);
07914                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07915                 Result = SDOperand(ResNode, 0);
07916               }
07917               return;
07918             }
07919           }
07920         }
07921       }
07922     }
07923     if (N0.getOpcode() == ISD::XOR) {
07924       N00 = N0.getOperand(0);
07925 
07926       // Pattern: (and:v2i64 (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>), VR128:v2i64:$src2)
07927       // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07928       // Pattern complexity = 7  cost = 1
07929       {
07930         N01 = N0.getOperand(1);
07931         if (N01.getOpcode() == ISD::BUILD_VECTOR &&
07932             Predicate_immAllOnesV(N01.Val)) {
07933           N1 = N.getOperand(1);
07934           if (N.Val->getValueType(0) == MVT::v2i64) {
07935             Select(Tmp0, N00);
07936             Select(Tmp1, N1);
07937             if (N.Val->hasOneUse()) {
07938               Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07939             } else {
07940               ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07941               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07942               Result = SDOperand(ResNode, 0);
07943             }
07944             return;
07945           }
07946         }
07947       }
07948 
07949       // Pattern: (and:v2i64 (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1), VR128:v2i64:$src2)
07950       // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07951       // Pattern complexity = 7  cost = 1
07952       if (N00.getOpcode() == ISD::BUILD_VECTOR &&
07953           Predicate_immAllOnesV(N00.Val)) {
07954         N01 = N0.getOperand(1);
07955         N1 = N.getOperand(1);
07956         if (N.Val->getValueType(0) == MVT::v2i64) {
07957           Select(Tmp0, N01);
07958           Select(Tmp1, N1);
07959           if (N.Val->hasOneUse()) {
07960             Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07961           } else {
07962             ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07963             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07964             Result = SDOperand(ResNode, 0);
07965           }
07966           return;
07967         }
07968       }
07969     }
07970     {
07971       N1 = N.getOperand(1);
07972       if (N1.getOpcode() == ISD::XOR) {
07973         N10 = N1.getOperand(0);
07974 
07975         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 VR128:v2i64:$src1, (build_vector:v2i64)<<P:Predicate_immAllOnesV>>))
07976         // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07977         // Pattern complexity = 7  cost = 1
07978         {
07979           N11 = N1.getOperand(1);
07980           if (N11.getOpcode() == ISD::BUILD_VECTOR &&
07981               Predicate_immAllOnesV(N11.Val) &&
07982               N.Val->getValueType(0) == MVT::v2i64) {
07983             Select(Tmp0, N10);
07984             Select(Tmp1, N0);
07985             if (N.Val->hasOneUse()) {
07986               Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07987             } else {
07988               ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
07989               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
07990               Result = SDOperand(ResNode, 0);
07991             }
07992             return;
07993           }
07994         }
07995 
07996         // Pattern: (and:v2i64 VR128:v2i64:$src2, (xor:v2i64 (build_vector:v2i64)<<P:Predicate_immAllOnesV>>, VR128:v2i64:$src1))
07997         // Emits: (PANDNrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
07998         // Pattern complexity = 7  cost = 1
07999         if (N10.getOpcode() == ISD::BUILD_VECTOR &&
08000             Predicate_immAllOnesV(N10.Val)) {
08001           N11 = N1.getOperand(1);
08002           if (N.Val->getValueType(0) == MVT::v2i64) {
08003             Select(Tmp0, N11);
08004             Select(Tmp1, N0);
08005             if (N.Val->hasOneUse()) {
08006               Result = CurDAG->SelectNodeTo(N.Val, X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
08007             } else {
08008               ResNode = CurDAG->getTargetNode(X86::PANDNrr, MVT::v2i64, Tmp0, Tmp1);
08009               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08010               Result = SDOperand(ResNode, 0);
08011             }
08012             return;
08013           }
08014         }
08015       }
08016     }
08017 
08018     // Pattern: (and:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
08019     // Emits: (ANDPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
08020     // Pattern complexity = 6  cost = 1
08021     if (N0.getOpcode() == ISD::BIT_CONVERT) {
08022       N00 = N0.getOperand(0);
08023       N1 = N.getOperand(1);
08024       if (N1.getOpcode() == ISD::BIT_CONVERT) {
08025         N10 = N1.getOperand(0);
08026         if (N.Val->getValueType(0) == MVT::v2i64 &&
08027             N00.Val->getValueType(0) == MVT::v2f64 &&
08028             N10.Val->getValueType(0) == MVT::v2f64) {
08029           Select(Tmp0, N00);
08030           Select(Tmp1, N10);
08031           if (N.Val->hasOneUse()) {
08032             Result = CurDAG->SelectNodeTo(N.Val, X86::ANDPDrr, MVT::v2i64, Tmp0, Tmp1);
08033           } else {
08034             ResNode = CurDAG->getTargetNode(X86::ANDPDrr, MVT::v2i64, Tmp0, Tmp1);
08035             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08036             Result = SDOperand(ResNode, 0);
08037           }
08038           return;
08039         }
08040       }
08041     }
08042   }
08043   {
08044     N0 = N.getOperand(0);
08045     N1 = N.getOperand(1);
08046     if (N1.getOpcode() == ISD::Constant) {
08047 
08048       // Pattern: (and:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
08049       // Emits: (AND16ri8:i16 R16:i16:$src1, (imm:i16):$src2)
08050       // Pattern complexity = 5  cost = 1
08051       if (Predicate_i16immSExt8(N1.Val) &&
08052           N.Val->getValueType(0) == MVT::i16) {
08053         Select(Tmp0, N0);
08054         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
08055         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
08056         if (N.Val->hasOneUse()) {
08057           Result = CurDAG->SelectNodeTo(N.Val, X86::AND16ri8, MVT::i16, Tmp0, Tmp1);
08058         } else {
08059           ResNode = CurDAG->getTargetNode(X86::AND16ri8, MVT::i16, Tmp0, Tmp1);
08060           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08061           Result = SDOperand(ResNode, 0);
08062         }
08063         return;
08064       }
08065 
08066       // Pattern: (and:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
08067       // Emits: (AND32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
08068       // Pattern complexity = 5  cost = 1
08069       if (Predicate_i32immSExt8(N1.Val) &&
08070           N.Val->getValueType(0) == MVT::i32) {
08071         Select(Tmp0, N0);
08072         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
08073         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
08074         if (N.Val->hasOneUse()) {
08075           Result = CurDAG->SelectNodeTo(N.Val, X86::AND32ri8, MVT::i32, Tmp0, Tmp1);
08076         } else {
08077           ResNode = CurDAG->getTargetNode(X86::AND32ri8, MVT::i32, Tmp0, Tmp1);
08078           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08079           Result = SDOperand(ResNode, 0);
08080         }
08081         return;
08082       }
08083 
08084       // Pattern: (and:i8 R8:i8:$src1, (imm:i8):$src2)
08085       // Emits: (AND8ri:i8 R8:i8:$src1, (imm:i8):$src2)
08086       // Pattern complexity = 4  cost = 1
08087       if (N.Val->getValueType(0) == MVT::i8) {
08088         Select(Tmp0, N0);
08089         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
08090         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
08091         if (N.Val->hasOneUse()) {
08092           Result = CurDAG->SelectNodeTo(N.Val, X86::AND8ri, MVT::i8, Tmp0, Tmp1);
08093         } else {
08094           ResNode = CurDAG->getTargetNode(X86::AND8ri, MVT::i8, Tmp0, Tmp1);
08095           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08096           Result = SDOperand(ResNode, 0);
08097         }
08098         return;
08099       }
08100 
08101       // Pattern: (and:i16 R16:i16:$src1, (imm:i16):$src2)
08102       // Emits: (AND16ri:i16 R16:i16:$src1, (imm:i16):$src2)
08103       // Pattern complexity = 4  cost = 1
08104       if (N.Val->getValueType(0) == MVT::i16) {
08105         Select(Tmp0, N0);
08106         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
08107         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
08108         if (N.Val->hasOneUse()) {
08109           Result = CurDAG->SelectNodeTo(N.Val, X86::AND16ri, MVT::i16, Tmp0, Tmp1);
08110         } else {
08111           ResNode = CurDAG->getTargetNode(X86::AND16ri, MVT::i16, Tmp0, Tmp1);
08112           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08113           Result = SDOperand(ResNode, 0);
08114         }
08115         return;
08116       }
08117 
08118       // Pattern: (and:i32 R32:i32:$src1, (imm:i32):$src2)
08119       // Emits: (AND32ri:i32 R32:i32:$src1, (imm:i32):$src2)
08120       // Pattern complexity = 4  cost = 1
08121       if (N.Val->getValueType(0) == MVT::i32) {
08122         Select(Tmp0, N0);
08123         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
08124         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
08125         if (N.Val->hasOneUse()) {
08126           Result = CurDAG->SelectNodeTo(N.Val, X86::AND32ri, MVT::i32, Tmp0, Tmp1);
08127         } else {
08128           ResNode = CurDAG->getTargetNode(X86::AND32ri, MVT::i32, Tmp0, Tmp1);
08129           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08130           Result = SDOperand(ResNode, 0);
08131         }
08132         return;
08133       }
08134     }
08135 
08136     // Pattern: (and:i8 R8:i8:$src1, R8:i8:$src2)
08137     // Emits: (AND8rr:i8 R8:i8:$src1, R8:i8:$src2)
08138     // Pattern complexity = 2  cost = 1
08139     if (N.Val->getValueType(0) == MVT::i8) {
08140       Select(Tmp0, N0);
08141       Select(Tmp1, N1);
08142       if (N.Val->hasOneUse()) {
08143         Result = CurDAG->SelectNodeTo(N.Val, X86::AND8rr, MVT::i8, Tmp0, Tmp1);
08144       } else {
08145         ResNode = CurDAG->getTargetNode(X86::AND8rr, MVT::i8, Tmp0, Tmp1);
08146         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08147         Result = SDOperand(ResNode, 0);
08148       }
08149       return;
08150     }
08151 
08152     // Pattern: (and:i16 R16:i16:$src1, R16:i16:$src2)
08153     // Emits: (AND16rr:i16 R16:i16:$src1, R16:i16:$src2)
08154     // Pattern complexity = 2  cost = 1
08155     if (N.Val->getValueType(0) == MVT::i16) {
08156       Select(Tmp0, N0);
08157       Select(Tmp1, N1);
08158       if (N.Val->hasOneUse()) {
08159         Result = CurDAG->SelectNodeTo(N.Val, X86::AND16rr, MVT::i16, Tmp0, Tmp1);
08160       } else {
08161         ResNode = CurDAG->getTargetNode(X86::AND16rr, MVT::i16, Tmp0, Tmp1);
08162         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08163         Result = SDOperand(ResNode, 0);
08164       }
08165       return;
08166     }
08167 
08168     // Pattern: (and:i32 R32:i32:$src1, R32:i32:$src2)
08169     // Emits: (AND32rr:i32 R32:i32:$src1, R32:i32:$src2)
08170     // Pattern complexity = 2  cost = 1
08171     if (N.Val->getValueType(0) == MVT::i32) {
08172       Select(Tmp0, N0);
08173       Select(Tmp1, N1);
08174       if (N.Val->hasOneUse()) {
08175         Result = CurDAG->SelectNodeTo(N.Val, X86::AND32rr, MVT::i32, Tmp0, Tmp1);
08176       } else {
08177         ResNode = CurDAG->getTargetNode(X86::AND32rr, MVT::i32, Tmp0, Tmp1);
08178         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08179         Result = SDOperand(ResNode, 0);
08180       }
08181       return;
08182     }
08183   }
08184 
08185   // Pattern: (and:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08186   // Emits: (ANDPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08187   // Pattern complexity = 2  cost = 1
08188   if ((Subtarget->hasSSE1())) {
08189     N0 = N.getOperand(0);
08190     N1 = N.getOperand(1);
08191     if (N.Val->getValueType(0) == MVT::v2i64) {
08192       Select(Tmp0, N0);
08193       Select(Tmp1, N1);
08194       if (N.Val->hasOneUse()) {
08195         Result = CurDAG->SelectNodeTo(N.Val, X86::ANDPSrr, MVT::v2i64, Tmp0, Tmp1);
08196       } else {
08197         ResNode = CurDAG->getTargetNode(X86::ANDPSrr, MVT::v2i64, Tmp0, Tmp1);
08198         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08199         Result = SDOperand(ResNode, 0);
08200       }
08201       return;
08202     }
08203   }
08204 
08205   // Pattern: (and:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08206   // Emits: (PANDrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
08207   // Pattern complexity = 2  cost = 1
08208   if ((Subtarget->hasSSE2())) {
08209     N0 = N.getOperand(0);
08210     N1 = N.getOperand(1);
08211     if (N.Val->getValueType(0) == MVT::v2i64) {
08212       Select(Tmp0, N0);
08213       Select(Tmp1, N1);
08214       if (N.Val->hasOneUse()) {
08215         Result = CurDAG->SelectNodeTo(N.Val, X86::PANDrr, MVT::v2i64, Tmp0, Tmp1);
08216       } else {
08217         ResNode = CurDAG->getTargetNode(X86::PANDrr, MVT::v2i64, Tmp0, Tmp1);
08218         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08219         Result = SDOperand(ResNode, 0);
08220       }
08221       return;
08222     }
08223   }
08224   std::cerr << "Cannot yet select: ";
08225   N.Val->dump(CurDAG);
08226   std::cerr << '\n';
08227   abort();
08228 }
08229 
08230 void Select_anyext(SDOperand &Result, SDOperand N) {
08231   SDOperand Chain0(0, 0);
08232   SDOperand N0(0, 0);
08233   SDOperand N01(0, 0);
08234   SDOperand Tmp0(0, 0);
08235   SDOperand Tmp1(0, 0);
08236   SDOperand Tmp2(0, 0);
08237   SDOperand Tmp3(0, 0);
08238   SDNode *ResNode;
08239   N0 = N.getOperand(0);
08240   if (N0.getOpcode() == ISD::LOAD &&
08241       N0.hasOneUse() &&
08242       !CodeGenMap.count(N0.getValue(0))) {
08243     Chain0 = N0.getOperand(0);
08244     N01 = N0.getOperand(1);
08245 
08246     // Pattern: (anyext:i16 (load:i8 addr:i32:$src))
08247     // Emits: (MOVZX16rm8:i16 addr:i32:$src)
08248     // Pattern complexity = 14  cost = 1
08249     if (N.Val->getValueType(0) == MVT::i16 &&
08250         N0.Val->getValueType(0) == MVT::i8 &&
08251         SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
08252       Select(Tmp0, Tmp0);
08253       Select(Tmp1, Tmp1);
08254       Select(Tmp2, Tmp2);
08255       Select(Tmp3, Tmp3);
08256       Select(Chain0, Chain0);
08257       ResNode = CurDAG->getTargetNode(X86::MOVZX16rm8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
08258       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08259       SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
08260       AddHandleReplacement(N0.Val, 1, ResNode, 1);
08261       Result = SDOperand(ResNode, N.ResNo);
08262       return;
08263     }
08264     if (N.Val->getValueType(0) == MVT::i32) {
08265 
08266       // Pattern: (anyext:i32 (load:i8 addr:i32:$src))
08267       // Emits: (MOVZX32rm8:i32 addr:i32:$src)
08268       // Pattern complexity = 14  cost = 1
08269       if (N0.Val->getValueType(0) == MVT::i8 &&
08270           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
08271         Select(Tmp0, Tmp0);
08272         Select(Tmp1, Tmp1);
08273         Select(Tmp2, Tmp2);
08274         Select(Tmp3, Tmp3);
08275         Select(Chain0, Chain0);
08276         ResNode = CurDAG->getTargetNode(X86::MOVZX32rm8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
08277         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08278         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
08279         AddHandleReplacement(N0.Val, 1, ResNode, 1);
08280         Result = SDOperand(ResNode, N.ResNo);
08281         return;
08282       }
08283 
08284       // Pattern: (anyext:i32 (load:i16 addr:i32:$src))
08285       // Emits: (MOVZX32rm16:i32 addr:i32:$src)
08286       // Pattern complexity = 14  cost = 1
08287       if (N0.Val->getValueType(0) == MVT::i16 &&
08288           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
08289         Select(Tmp0, Tmp0);
08290         Select(Tmp1, Tmp1);
08291         Select(Tmp2, Tmp2);
08292         Select(Tmp3, Tmp3);
08293         Select(Chain0, Chain0);
08294         ResNode = CurDAG->getTargetNode(X86::MOVZX32rm16, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
08295         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08296         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
08297         AddHandleReplacement(N0.Val, 1, ResNode, 1);
08298         Result = SDOperand(ResNode, N.ResNo);
08299         return;
08300       }
08301     }
08302   }
08303 
08304   // Pattern: (anyext:i16 R8:i8:$src)
08305   // Emits: (MOVZX16rr8:i16 R8:i8:$src)
08306   // Pattern complexity = 2  cost = 1
08307   if (N.Val->getValueType(0) == MVT::i16 &&
08308       N0.Val->getValueType(0) == MVT::i8) {
08309     Select(Tmp0, N0);
08310     if (N.Val->hasOneUse()) {
08311       Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX16rr8, MVT::i16, Tmp0);
08312     } else {
08313       ResNode = CurDAG->getTargetNode(X86::MOVZX16rr8, MVT::i16, Tmp0);
08314       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08315       Result = SDOperand(ResNode, 0);
08316     }
08317     return;
08318   }
08319   if (N.Val->getValueType(0) == MVT::i32) {
08320 
08321     // Pattern: (anyext:i32 R8:i8:$src)
08322     // Emits: (MOVZX32rr8:i32 R8:i8:$src)
08323     // Pattern complexity = 2  cost = 1
08324     if (N0.Val->getValueType(0) == MVT::i8) {
08325       Select(Tmp0, N0);
08326       if (N.Val->hasOneUse()) {
08327         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX32rr8, MVT::i32, Tmp0);
08328       } else {
08329         ResNode = CurDAG->getTargetNode(X86::MOVZX32rr8, MVT::i32, Tmp0);
08330         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08331         Result = SDOperand(ResNode, 0);
08332       }
08333       return;
08334     }
08335 
08336     // Pattern: (anyext:i32 R16:i16:$src)
08337     // Emits: (MOVZX32rr16:i32 R16:i16:$src)
08338     // Pattern complexity = 2  cost = 1
08339     if (N0.Val->getValueType(0) == MVT::i16) {
08340       Select(Tmp0, N0);
08341       if (N.Val->hasOneUse()) {
08342         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX32rr16, MVT::i32, Tmp0);
08343       } else {
08344         ResNode = CurDAG->getTargetNode(X86::MOVZX32rr16, MVT::i32, Tmp0);
08345         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08346         Result = SDOperand(ResNode, 0);
08347       }
08348       return;
08349     }
08350   }
08351   std::cerr << "Cannot yet select: ";
08352   N.Val->dump(CurDAG);
08353   std::cerr << '\n';
08354   abort();
08355 }
08356 
08357 void Select_bitconvert(SDOperand &Result, SDOperand N) {
08358   SDOperand Chain00(0, 0);
08359   SDOperand N0(0, 0);
08360   SDOperand N00(0, 0);
08361   SDOperand N001(0, 0);
08362   SDOperand Tmp0(0, 0);
08363   SDOperand Tmp1(0, 0);
08364   SDOperand Tmp2(0, 0);
08365   SDOperand Tmp3(0, 0);
08366   SDNode *ResNode;
08367   if ((Subtarget->hasSSE2())) {
08368     N0 = N.getOperand(0);
08369 
08370     // Pattern: (bitconvert:v2i64 (X86zexts2vec:v2f64 (load:f64 addr:i32:$src)))
08371     // Emits: (MOVZQI2PQIrm:v2i64 addr:i32:$src)
08372     // Pattern complexity = 16  cost = 1
08373     if (N0.getOpcode() == X86ISD::ZEXT_S2VEC &&
08374         N0.hasOneUse() &&
08375         !CodeGenMap.count(N0.getValue(0))) {
08376       N00 = N0.getOperand(0);
08377       if (N00.getOpcode() == ISD::LOAD &&
08378           N00.hasOneUse() &&
08379           !CodeGenMap.count(N00.getValue(0))) {
08380         Chain00 = N00.getOperand(0);
08381         N001 = N00.getOperand(1);
08382         if (N.Val->getValueType(0) == MVT::v2i64 &&
08383             N0.Val->getValueType(0) == MVT::v2f64 &&
08384             N00.Val->getValueType(0) == MVT::f64 &&
08385             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
08386           Select(Tmp0, Tmp0);
08387           Select(Tmp1, Tmp1);
08388           Select(Tmp2, Tmp2);
08389           Select(Tmp3, Tmp3);
08390           Select(Chain00, Chain00);
08391           ResNode = CurDAG->getTargetNode(X86::MOVZQI2PQIrm, MVT::v2i64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
08392           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08393           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
08394           AddHandleReplacement(N00.Val, 1, ResNode, 1);
08395           Result = SDOperand(ResNode, N.ResNo);
08396           return;
08397         }
08398       }
08399     }
08400     if (N.Val->getValueType(0) == MVT::v2i64) {
08401 
08402       // Pattern: (bitconvert:v2i64 VR128:v4i32:$src)
08403       // Emits: VR128:v2i64:$src
08404       // Pattern complexity = 2  cost = 0
08405       if (N0.Val->getValueType(0) == MVT::v4i32) {
08406         Select(Tmp0, N0);
08407         Result = Tmp0;
08408         return;
08409       }
08410 
08411       // Pattern: (bitconvert:v2i64 VR128:v8i16:$src)
08412       // Emits: VR128:v2i64:$src
08413       // Pattern complexity = 2  cost = 0
08414       if (N0.Val->getValueType(0) == MVT::v8i16) {
08415         Select(Tmp0, N0);
08416         Result = Tmp0;
08417         return;
08418       }
08419 
08420       // Pattern: (bitconvert:v2i64 VR128:v16i8:$src)
08421       // Emits: VR128:v2i64:$src
08422       // Pattern complexity = 2  cost = 0
08423       if (N0.Val->getValueType(0) == MVT::v16i8) {
08424         Select(Tmp0, N0);
08425         Result = Tmp0;
08426         return;
08427       }
08428 
08429       // Pattern: (bitconvert:v2i64 VR128:v2f64:$src)
08430       // Emits: VR128:v2i64:$src
08431       // Pattern complexity = 2  cost = 0
08432       if (N0.Val->getValueType(0) == MVT::v2f64) {
08433         Select(Tmp0, N0);
08434         Result = Tmp0;
08435         return;
08436       }
08437 
08438       // Pattern: (bitconvert:v2i64 VR128:v4f32:$src)
08439       // Emits: VR128:v2i64:$src
08440       // Pattern complexity = 2  cost = 0
08441       if (N0.Val->getValueType(0) == MVT::v4f32) {
08442         Select(Tmp0, N0);
08443         Result = Tmp0;
08444         return;
08445       }
08446     }
08447     if (N.Val->getValueType(0) == MVT::v4i32) {
08448 
08449       // Pattern: (bitconvert:v4i32 VR128:v2i64:$src)
08450       // Emits: VR128:v4i32:$src
08451       // Pattern complexity = 2  cost = 0
08452       if (N0.Val->getValueType(0) == MVT::v2i64) {
08453         Select(Tmp0, N0);
08454         Result = Tmp0;
08455         return;
08456       }
08457 
08458       // Pattern: (bitconvert:v4i32 VR128:v8i16:$src)
08459       // Emits: VR128:v4i32:$src
08460       // Pattern complexity = 2  cost = 0
08461       if (N0.Val->getValueType(0) == MVT::v8i16) {
08462         Select(Tmp0, N0);
08463         Result = Tmp0;
08464         return;
08465       }
08466 
08467       // Pattern: (bitconvert:v4i32 VR128:v16i8:$src)
08468       // Emits: VR128:v4i32:$src
08469       // Pattern complexity = 2  cost = 0
08470       if (N0.Val->getValueType(0) == MVT::v16i8) {
08471         Select(Tmp0, N0);
08472         Result = Tmp0;
08473         return;
08474       }
08475 
08476       // Pattern: (bitconvert:v4i32 VR128:v2f64:$src)
08477       // Emits: VR128:v4i32:$src
08478       // Pattern complexity = 2  cost = 0
08479       if (N0.Val->getValueType(0) == MVT::v2f64) {
08480         Select(Tmp0, N0);
08481         Result = Tmp0;
08482         return;
08483       }
08484 
08485       // Pattern: (bitconvert:v4i32 VR128:v4f32:$src)
08486       // Emits: VR128:v4i32:$src
08487       // Pattern complexity = 2  cost = 0
08488       if (N0.Val->getValueType(0) == MVT::v4f32) {
08489         Select(Tmp0, N0);
08490         Result = Tmp0;
08491         return;
08492       }
08493     }
08494     if (N.Val->getValueType(0) == MVT::v8i16) {
08495 
08496       // Pattern: (bitconvert:v8i16 VR128:v2i64:$src)
08497       // Emits: VR128:v4i32:$src
08498       // Pattern complexity = 2  cost = 0
08499       if (N0.Val->getValueType(0) == MVT::v2i64) {
08500         Select(Tmp0, N0);
08501         Result = Tmp0;
08502         return;
08503       }
08504 
08505       // Pattern: (bitconvert:v8i16 VR128:v4i32:$src)
08506       // Emits: VR128:v4i32:$src
08507       // Pattern complexity = 2  cost = 0
08508       if (N0.Val->getValueType(0) == MVT::v4i32) {
08509         Select(Tmp0, N0);
08510         Result = Tmp0;
08511         return;
08512       }
08513 
08514       // Pattern: (bitconvert:v8i16 VR128:v16i8:$src)
08515       // Emits: VR128:v4i32:$src
08516       // Pattern complexity = 2  cost = 0
08517       if (N0.Val->getValueType(0) == MVT::v16i8) {
08518         Select(Tmp0, N0);
08519         Result = Tmp0;
08520         return;
08521       }
08522 
08523       // Pattern: (bitconvert:v8i16 VR128:v2f64:$src)
08524       // Emits: VR128:v8i16:$src
08525       // Pattern complexity = 2  cost = 0
08526       if (N0.Val->getValueType(0) == MVT::v2f64) {
08527         Select(Tmp0, N0);
08528         Result = Tmp0;
08529         return;
08530       }
08531 
08532       // Pattern: (bitconvert:v8i16 VR128:v4f32:$src)
08533       // Emits: VR128:v8i16:$src
08534       // Pattern complexity = 2  cost = 0
08535       if (N0.Val->getValueType(0) == MVT::v4f32) {
08536         Select(Tmp0, N0);
08537         Result = Tmp0;
08538         return;
08539       }
08540     }
08541     if (N.Val->getValueType(0) == MVT::v16i8) {
08542 
08543       // Pattern: (bitconvert:v16i8 VR128:v2i64:$src)
08544       // Emits: VR128:v4i32:$src
08545       // Pattern complexity = 2  cost = 0
08546       if (N0.Val->getValueType(0) == MVT::v2i64) {
08547         Select(Tmp0, N0);
08548         Result = Tmp0;
08549         return;
08550       }
08551 
08552       // Pattern: (bitconvert:v16i8 VR128:v4i32:$src)
08553       // Emits: VR128:v4i32:$src
08554       // Pattern complexity = 2  cost = 0
08555       if (N0.Val->getValueType(0) == MVT::v4i32) {
08556         Select(Tmp0, N0);
08557         Result = Tmp0;
08558         return;
08559       }
08560 
08561       // Pattern: (bitconvert:v16i8 VR128:v8i16:$src)
08562       // Emits: VR128:v4i32:$src
08563       // Pattern complexity = 2  cost = 0
08564       if (N0.Val->getValueType(0) == MVT::v8i16) {
08565         Select(Tmp0, N0);
08566         Result = Tmp0;
08567         return;
08568       }
08569 
08570       // Pattern: (bitconvert:v16i8 VR128:v2f64:$src)
08571       // Emits: VR128:v16i8:$src
08572       // Pattern complexity = 2  cost = 0
08573       if (N0.Val->getValueType(0) == MVT::v2f64) {
08574         Select(Tmp0, N0);
08575         Result = Tmp0;
08576         return;
08577       }
08578 
08579       // Pattern: (bitconvert:v16i8 VR128:v4f32:$src)
08580       // Emits: VR128:v16i8:$src
08581       // Pattern complexity = 2  cost = 0
08582       if (N0.Val->getValueType(0) == MVT::v4f32) {
08583         Select(Tmp0, N0);
08584         Result = Tmp0;
08585         return;
08586       }
08587     }
08588     if (N.Val->getValueType(0) == MVT::v4f32) {
08589 
08590       // Pattern: (bitconvert:v4f32 VR128:v2i64:$src)
08591       // Emits: VR128:v4f32:$src
08592       // Pattern complexity = 2  cost = 0
08593       if (N0.Val->getValueType(0) == MVT::v2i64) {
08594         Select(Tmp0, N0);
08595         Result = Tmp0;
08596         return;
08597       }
08598 
08599       // Pattern: (bitconvert:v4f32 VR128:v4i32:$src)
08600       // Emits: VR128:v4f32:$src
08601       // Pattern complexity = 2  cost = 0
08602       if (N0.Val->getValueType(0) == MVT::v4i32) {
08603         Select(Tmp0, N0);
08604         Result = Tmp0;
08605         return;
08606       }
08607 
08608       // Pattern: (bitconvert:v4f32 VR128:v8i16:$src)
08609       // Emits: VR128:v4f32:$src
08610       // Pattern complexity = 2  cost = 0
08611       if (N0.Val->getValueType(0) == MVT::v8i16) {
08612         Select(Tmp0, N0);
08613         Result = Tmp0;
08614         return;
08615       }
08616 
08617       // Pattern: (bitconvert:v4f32 VR128:v16i8:$src)
08618       // Emits: VR128:v4f32:$src
08619       // Pattern complexity = 2  cost = 0
08620       if (N0.Val->getValueType(0) == MVT::v16i8) {
08621         Select(Tmp0, N0);
08622         Result = Tmp0;
08623         return;
08624       }
08625 
08626       // Pattern: (bitconvert:v4f32 VR128:v2f64:$src)
08627       // Emits: VR128:v4f32:$src
08628       // Pattern complexity = 2  cost = 0
08629       if (N0.Val->getValueType(0) == MVT::v2f64) {
08630         Select(Tmp0, N0);
08631         Result = Tmp0;
08632         return;
08633       }
08634     }
08635     if (N.Val->getValueType(0) == MVT::v2f64) {
08636 
08637       // Pattern: (bitconvert:v2f64 VR128:v2i64:$src)
08638       // Emits: VR128:v2f64:$src
08639       // Pattern complexity = 2  cost = 0
08640       if (N0.Val->getValueType(0) == MVT::v2i64) {
08641         Select(Tmp0, N0);
08642         Result = Tmp0;
08643         return;
08644       }
08645 
08646       // Pattern: (bitconvert:v2f64 VR128:v4i32:$src)
08647       // Emits: VR128:v2f64:$src
08648       // Pattern complexity = 2  cost = 0
08649       if (N0.Val->getValueType(0) == MVT::v4i32) {
08650         Select(Tmp0, N0);
08651         Result = Tmp0;
08652         return;
08653       }
08654 
08655       // Pattern: (bitconvert:v2f64 VR128:v8i16:$src)
08656       // Emits: VR128:v2f64:$src
08657       // Pattern complexity = 2  cost = 0
08658       if (N0.Val->getValueType(0) == MVT::v8i16) {
08659         Select(Tmp0, N0);
08660         Result = Tmp0;
08661         return;
08662       }
08663 
08664       // Pattern: (bitconvert:v2f64 VR128:v16i8:$src)
08665       // Emits: VR128:v2f64:$src
08666       // Pattern complexity = 2  cost = 0
08667       if (N0.Val->getValueType(0) == MVT::v16i8) {
08668         Select(Tmp0, N0);
08669         Result = Tmp0;
08670         return;
08671       }
08672 
08673       // Pattern: (bitconvert:v2f64 VR128:v4f32:$src)
08674       // Emits: VR128:v2f64:$src
08675       // Pattern complexity = 2  cost = 0
08676       if (N0.Val->getValueType(0) == MVT::v4f32) {
08677         Select(Tmp0, N0);
08678         Result = Tmp0;
08679         return;
08680       }
08681     }
08682   }
08683   std::cerr << "Cannot yet select: ";
08684   N.Val->dump(CurDAG);
08685   std::cerr << '\n';
08686   abort();
08687 }
08688 
08689 void Select_br(SDOperand &Result, SDOperand N) {
08690   SDOperand Chain(0, 0);
08691   SDOperand N1(0, 0);
08692   SDOperand Tmp0(0, 0);
08693   SDNode *ResNode;
08694   Chain = N.getOperand(0);
08695   N1 = N.getOperand(1);
08696   if (N1.getOpcode() == ISD::BasicBlock) {
08697     Select(Tmp0, N1);
08698     Select(Chain, Chain);
08699     ResNode = CurDAG->getTargetNode(X86::JMP, MVT::Other, Tmp0, Chain);
08700     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08701     Result = SDOperand(ResNode, N.ResNo);
08702     return;
08703   }
08704   std::cerr << "Cannot yet select: ";
08705   N.Val->dump(CurDAG);
08706   std::cerr << '\n';
08707   abort();
08708 }
08709 
08710 void Select_bswap(SDOperand &Result, SDOperand N) {
08711   SDOperand N0(0, 0);
08712   SDOperand Tmp0(0, 0);
08713   SDNode *ResNode;
08714   N0 = N.getOperand(0);
08715   if (N.Val->getValueType(0) == MVT::i32) {
08716     Select(Tmp0, N0);
08717     if (N.Val->hasOneUse()) {
08718       Result = CurDAG->SelectNodeTo(N.Val, X86::BSWAP32r, MVT::i32, Tmp0);
08719     } else {
08720       ResNode = CurDAG->getTargetNode(X86::BSWAP32r, MVT::i32, Tmp0);
08721       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08722       Result = SDOperand(ResNode, 0);
08723     }
08724     return;
08725   }
08726   std::cerr << "Cannot yet select: ";
08727   N.Val->dump(CurDAG);
08728   std::cerr << '\n';
08729   abort();
08730 }
08731 
08732 void Select_build_vector(SDOperand &Result, SDOperand N) {
08733   SDNode *ResNode;
08734 
08735   // Pattern: (build_vector:v2i64)<<P:Predicate_immAllZerosV>>
08736   // Emits: (V_SET0_PI:v2i64)
08737   // Pattern complexity = 3  cost = 1
08738   if ((Subtarget->hasSSE2()) &&
08739       Predicate_immAllZerosV(N.Val) &&
08740       N.Val->getValueType(0) == MVT::v2i64) {
08741     if (N.Val->hasOneUse()) {
08742       Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PI, MVT::v2i64);
08743     } else {
08744       ResNode = CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v2i64);
08745       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08746       Result = SDOperand(ResNode, 0);
08747     }
08748     return;
08749   }
08750 
08751   // Pattern: (build_vector:v4f32)<<P:Predicate_immAllZerosV>>
08752   // Emits: (V_SET0_PS:v4f32)
08753   // Pattern complexity = 3  cost = 1
08754   if ((Subtarget->hasSSE1()) &&
08755       Predicate_immAllZerosV(N.Val) &&
08756       N.Val->getValueType(0) == MVT::v4f32) {
08757     if (N.Val->hasOneUse()) {
08758       Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PS, MVT::v4f32);
08759     } else {
08760       ResNode = CurDAG->getTargetNode(X86::V_SET0_PS, MVT::v4f32);
08761       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08762       Result = SDOperand(ResNode, 0);
08763     }
08764     return;
08765   }
08766   if ((Subtarget->hasSSE2())) {
08767 
08768     // Pattern: (build_vector:v2f64)<<P:Predicate_immAllZerosV>>
08769     // Emits: (V_SET0_PD:v2f64)
08770     // Pattern complexity = 3  cost = 1
08771     if (Predicate_immAllZerosV(N.Val) &&
08772         N.Val->getValueType(0) == MVT::v2f64) {
08773       if (N.Val->hasOneUse()) {
08774         Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PD, MVT::v2f64);
08775       } else {
08776         ResNode = CurDAG->getTargetNode(X86::V_SET0_PD, MVT::v2f64);
08777         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08778         Result = SDOperand(ResNode, 0);
08779       }
08780       return;
08781     }
08782 
08783     // Pattern: (build_vector:v2f64)<<P:Predicate_immAllOnesV>>
08784     // Emits: (V_SETALLONES:v2f64)
08785     // Pattern complexity = 3  cost = 1
08786     if (Predicate_immAllOnesV(N.Val) &&
08787         N.Val->getValueType(0) == MVT::v2f64) {
08788       if (N.Val->hasOneUse()) {
08789         Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v2f64);
08790       } else {
08791         ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v2f64);
08792         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08793         Result = SDOperand(ResNode, 0);
08794       }
08795       return;
08796     }
08797     if (Predicate_immAllZerosV(N.Val)) {
08798 
08799       // Pattern: (build_vector:v16i8)<<P:Predicate_immAllZerosV>>
08800       // Emits: (V_SET0_PI:v16i8)
08801       // Pattern complexity = 3  cost = 1
08802       if (N.Val->getValueType(0) == MVT::v16i8) {
08803         if (N.Val->hasOneUse()) {
08804           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PI, MVT::v16i8);
08805         } else {
08806           ResNode = CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v16i8);
08807           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08808           Result = SDOperand(ResNode, 0);
08809         }
08810         return;
08811       }
08812 
08813       // Pattern: (build_vector:v8i16)<<P:Predicate_immAllZerosV>>
08814       // Emits: (V_SET0_PI:v8i16)
08815       // Pattern complexity = 3  cost = 1
08816       if (N.Val->getValueType(0) == MVT::v8i16) {
08817         if (N.Val->hasOneUse()) {
08818           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PI, MVT::v8i16);
08819         } else {
08820           ResNode = CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v8i16);
08821           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08822           Result = SDOperand(ResNode, 0);
08823         }
08824         return;
08825       }
08826 
08827       // Pattern: (build_vector:v4i32)<<P:Predicate_immAllZerosV>>
08828       // Emits: (V_SET0_PI:v4i32)
08829       // Pattern complexity = 3  cost = 1
08830       if (N.Val->getValueType(0) == MVT::v4i32) {
08831         if (N.Val->hasOneUse()) {
08832           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SET0_PI, MVT::v4i32);
08833         } else {
08834           ResNode = CurDAG->getTargetNode(X86::V_SET0_PI, MVT::v4i32);
08835           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08836           Result = SDOperand(ResNode, 0);
08837         }
08838         return;
08839       }
08840     }
08841     if (Predicate_immAllOnesV(N.Val)) {
08842 
08843       // Pattern: (build_vector:v16i8)<<P:Predicate_immAllOnesV>>
08844       // Emits: (V_SETALLONES:v16i8)
08845       // Pattern complexity = 3  cost = 1
08846       if (N.Val->getValueType(0) == MVT::v16i8) {
08847         if (N.Val->hasOneUse()) {
08848           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v16i8);
08849         } else {
08850           ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v16i8);
08851           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08852           Result = SDOperand(ResNode, 0);
08853         }
08854         return;
08855       }
08856 
08857       // Pattern: (build_vector:v8i16)<<P:Predicate_immAllOnesV>>
08858       // Emits: (V_SETALLONES:v8i16)
08859       // Pattern complexity = 3  cost = 1
08860       if (N.Val->getValueType(0) == MVT::v8i16) {
08861         if (N.Val->hasOneUse()) {
08862           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v8i16);
08863         } else {
08864           ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v8i16);
08865           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08866           Result = SDOperand(ResNode, 0);
08867         }
08868         return;
08869       }
08870 
08871       // Pattern: (build_vector:v4i32)<<P:Predicate_immAllOnesV>>
08872       // Emits: (V_SETALLONES:v4i32)
08873       // Pattern complexity = 3  cost = 1
08874       if (N.Val->getValueType(0) == MVT::v4i32) {
08875         if (N.Val->hasOneUse()) {
08876           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v4i32);
08877         } else {
08878           ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v4i32);
08879           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08880           Result = SDOperand(ResNode, 0);
08881         }
08882         return;
08883       }
08884 
08885       // Pattern: (build_vector:v2i64)<<P:Predicate_immAllOnesV>>
08886       // Emits: (V_SETALLONES:v2i64)
08887       // Pattern complexity = 3  cost = 1
08888       if (N.Val->getValueType(0) == MVT::v2i64) {
08889         if (N.Val->hasOneUse()) {
08890           Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v2i64);
08891         } else {
08892           ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v2i64);
08893           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08894           Result = SDOperand(ResNode, 0);
08895         }
08896         return;
08897       }
08898     }
08899   }
08900 
08901   // Pattern: (build_vector:v4f32)<<P:Predicate_immAllOnesV>>
08902   // Emits: (V_SETALLONES:v4f32)
08903   // Pattern complexity = 3  cost = 1
08904   if ((Subtarget->hasSSE1()) &&
08905       Predicate_immAllOnesV(N.Val) &&
08906       N.Val->getValueType(0) == MVT::v4f32) {
08907     if (N.Val->hasOneUse()) {
08908       Result = CurDAG->SelectNodeTo(N.Val, X86::V_SETALLONES, MVT::v4f32);
08909     } else {
08910       ResNode = CurDAG->getTargetNode(X86::V_SETALLONES, MVT::v4f32);
08911       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
08912       Result = SDOperand(ResNode, 0);
08913     }
08914     return;
08915   }
08916   std::cerr << "Cannot yet select: ";
08917   N.Val->dump(CurDAG);
08918   std::cerr << '\n';
08919   abort();
08920 }
08921 
08922 void Select_dwarf_label(SDOperand &Result, SDOperand N) {
08923   SDOperand Chain(0, 0);
08924   SDOperand N1(0, 0);
08925   SDOperand Tmp0(0, 0);
08926   SDNode *ResNode;
08927   Chain = N.getOperand(0);
08928   N1 = N.getOperand(1);
08929   if (N1.getOpcode() == ISD::Constant &&
08930       N1.Val->getValueType(0) == MVT::i32) {
08931     unsigned Tmp0C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
08932     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i32);
08933     Select(Chain, Chain);
08934     ResNode = CurDAG->getTargetNode(X86::DWARF_LABEL, MVT::Other, Tmp0, Chain);
08935     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08936     Result = SDOperand(ResNode, N.ResNo);
08937     return;
08938   }
08939   std::cerr << "Cannot yet select: ";
08940   N.Val->dump(CurDAG);
08941   std::cerr << '\n';
08942   abort();
08943 }
08944 
08945 void Select_dwarf_loc(SDOperand &Result, SDOperand N) {
08946   SDOperand Chain(0, 0);
08947   SDOperand N1(0, 0);
08948   SDOperand N2(0, 0);
08949   SDOperand N3(0, 0);
08950   SDOperand Tmp0(0, 0);
08951   SDOperand Tmp1(0, 0);
08952   SDOperand Tmp2(0, 0);
08953   SDNode *ResNode;
08954   Chain = N.getOperand(0);
08955   N1 = N.getOperand(1);
08956   if (N1.getOpcode() == ISD::Constant) {
08957     N2 = N.getOperand(2);
08958     if (N2.getOpcode() == ISD::Constant) {
08959       N3 = N.getOperand(3);
08960       if (N3.getOpcode() == ISD::Constant &&
08961           N1.Val->getValueType(0) == MVT::i32 &&
08962           N2.Val->getValueType(0) == MVT::i32 &&
08963           N3.Val->getValueType(0) == MVT::i32) {
08964         unsigned Tmp0C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
08965         Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i32);
08966         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
08967         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
08968         unsigned Tmp2C = (unsigned)cast<ConstantSDNode>(N3)->getValue();
08969         Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i32);
08970         Select(Chain, Chain);
08971         ResNode = CurDAG->getTargetNode(X86::DWARF_LOC, MVT::Other, Tmp0, Tmp1, Tmp2, Chain);
08972         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
08973         Result = SDOperand(ResNode, N.ResNo);
08974         return;
08975       }
08976     }
08977   }
08978   std::cerr << "Cannot yet select: ";
08979   N.Val->dump(CurDAG);
08980   std::cerr << '\n';
08981   abort();
08982 }
08983 
08984 void Select_extld(SDOperand &Result, SDOperand N) {
08985   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
08986     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
08987     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
08988     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
08989     Result = Dummy;
08990     return;
08991   }
08992   SDOperand Chain(0, 0);
08993   SDOperand N1(0, 0);
08994   SDOperand N2(0, 0);
08995   SDOperand N3(0, 0);
08996   SDOperand Tmp0(0, 0);
08997   SDOperand Tmp1(0, 0);
08998   SDOperand Tmp2(0, 0);
08999   SDOperand Tmp3(0, 0);
09000   SDNode *ResNode;
09001 
09002   // Pattern: (extld:f64 addr:i32:$src, srcvalue:Other:$dummy, f32:Other)
09003   // Emits: (FpLD32m:f64 addr:i32:$src)
09004   // Pattern complexity = 12  cost = 1
09005   if ((!Subtarget->hasSSE2())) {
09006     Chain = N.getOperand(0);
09007     N1 = N.getOperand(1);
09008     N2 = N.getOperand(2);
09009     N3 = N.getOperand(3);
09010     if (cast<VTSDNode>(N3)->getVT() == MVT::f32 &&
09011         N.Val->getValueType(0) == MVT::f64 &&
09012         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
09013       Select(Tmp0, Tmp0);
09014       Select(Tmp1, Tmp1);
09015       Select(Tmp2, Tmp2);
09016       Select(Tmp3, Tmp3);
09017       Select(Chain, Chain);
09018       ResNode = CurDAG->getTargetNode(X86::FpLD32m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
09019       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09020       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
09021       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
09022       Result = SDOperand(ResNode, N.ResNo);
09023       return;
09024     }
09025   }
09026 
09027   // Pattern: (extld:i8 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
09028   // Emits: (MOV8rm:i8 addr:i32:$src)
09029   // Pattern complexity = 12  cost = 1
09030   Chain = N.getOperand(0);
09031   N1 = N.getOperand(1);
09032   N2 = N.getOperand(2);
09033   N3 = N.getOperand(3);
09034   if (cast<VTSDNode>(N3)->getVT() == MVT::i1 &&
09035       N.Val->getValueType(0) == MVT::i8 &&
09036       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
09037     Select(Tmp0, Tmp0);
09038     Select(Tmp1, Tmp1);
09039     Select(Tmp2, Tmp2);
09040     Select(Tmp3, Tmp3);
09041     Select(Chain, Chain);
09042     ResNode = CurDAG->getTargetNode(X86::MOV8rm, MVT::i8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
09043     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09044     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
09045     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
09046     Result = SDOperand(ResNode, N.ResNo);
09047     return;
09048   }
09049   std::cerr << "Cannot yet select: ";
09050   N.Val->dump(CurDAG);
09051   std::cerr << '\n';
09052   abort();
09053 }
09054 
09055 void Select_fabs(SDOperand &Result, SDOperand N) {
09056   SDOperand N0(0, 0);
09057   SDOperand Tmp0(0, 0);
09058   SDNode *ResNode;
09059   if ((!Subtarget->hasSSE2())) {
09060     N0 = N.getOperand(0);
09061     if (N.Val->getValueType(0) == MVT::f64) {
09062       Select(Tmp0, N0);
09063       if (N.Val->hasOneUse()) {
09064         Result = CurDAG->SelectNodeTo(N.Val, X86::FpABS, MVT::f64, Tmp0);
09065       } else {
09066         ResNode = CurDAG->getTargetNode(X86::FpABS, MVT::f64, Tmp0);
09067         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09068         Result = SDOperand(ResNode, 0);
09069       }
09070       return;
09071     }
09072   }
09073   std::cerr << "Cannot yet select: ";
09074   N.Val->dump(CurDAG);
09075   std::cerr << '\n';
09076   abort();
09077 }
09078 
09079 void Select_fadd(SDOperand &Result, SDOperand N) {
09080   SDOperand Chain0(0, 0);
09081   SDOperand Chain1(0, 0);
09082   SDOperand N0(0, 0);
09083   SDOperand N01(0, 0);
09084   SDOperand N02(0, 0);
09085   SDOperand N03(0, 0);
09086   SDOperand N1(0, 0);
09087   SDOperand N11(0, 0);
09088   SDOperand N12(0, 0);
09089   SDOperand N13(0, 0);
09090   SDOperand Tmp0(0, 0);
09091   SDOperand Tmp1(0, 0);
09092   SDOperand Tmp2(0, 0);
09093   SDOperand Tmp3(0, 0);
09094   SDOperand Tmp4(0, 0);
09095   SDNode *ResNode;
09096   if ((!Subtarget->hasSSE2())) {
09097     N0 = N.getOperand(0);
09098     N1 = N.getOperand(1);
09099 
09100     // Pattern: (fadd:f64 RFP:f64:$src1, (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other))
09101     // Emits: (FpADD32m:f64 RFP:f64:$src1, addr:i32:$src2)
09102     // Pattern complexity = 14  cost = 1
09103     if (N1.getOpcode() == ISD::EXTLOAD &&
09104         N1.hasOneUse() &&
09105         !CodeGenMap.count(N1.getValue(0)) &&
09106         !isNonImmUse(N.Val, N1.Val)) {
09107       Chain1 = N1.getOperand(0);
09108       N11 = N1.getOperand(1);
09109       N12 = N1.getOperand(2);
09110       N13 = N1.getOperand(3);
09111       if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
09112           N.Val->getValueType(0) == MVT::f64 &&
09113           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09114         Select(Tmp0, Tmp0);
09115         Select(Tmp1, Tmp1);
09116         Select(Tmp2, Tmp2);
09117         Select(Tmp3, Tmp3);
09118         Select(Tmp4, N0);
09119         Select(Chain1, Chain1);
09120         ResNode = CurDAG->getTargetNode(X86::FpADD32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09121         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09122         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09123         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09124         Result = SDOperand(ResNode, N.ResNo);
09125         return;
09126       }
09127     }
09128 
09129     // Pattern: (fadd:f64 RFP:f64:$src1, (load:f64 addr:i32:$src2))
09130     // Emits: (FpADD64m:f64 RFP:f64:$src1, addr:i32:$src2)
09131     // Pattern complexity = 14  cost = 1
09132     if (N1.getOpcode() == ISD::LOAD &&
09133         N1.hasOneUse() &&
09134         !CodeGenMap.count(N1.getValue(0)) &&
09135         !isNonImmUse(N.Val, N1.Val)) {
09136       Chain1 = N1.getOperand(0);
09137       N11 = N1.getOperand(1);
09138       if (N.Val->getValueType(0) == MVT::f64 &&
09139           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09140         Select(Tmp0, Tmp0);
09141         Select(Tmp1, Tmp1);
09142         Select(Tmp2, Tmp2);
09143         Select(Tmp3, Tmp3);
09144         Select(Tmp4, N0);
09145         Select(Chain1, Chain1);
09146         ResNode = CurDAG->getTargetNode(X86::FpADD64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09147         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09148         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09149         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09150         Result = SDOperand(ResNode, N.ResNo);
09151         return;
09152       }
09153     }
09154     if (N1.getOpcode() == X86ISD::FILD &&
09155         N1.hasOneUse() &&
09156         !CodeGenMap.count(N1.getValue(0)) &&
09157         !isNonImmUse(N.Val, N1.Val)) {
09158       Chain1 = N1.getOperand(0);
09159       N11 = N1.getOperand(1);
09160       N12 = N1.getOperand(2);
09161 
09162       // Pattern: (fadd:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i16:Other))
09163       // Emits: (FpIADD16m:f64 RFP:f64:$src1, addr:i32:$src2)
09164       // Pattern complexity = 14  cost = 1
09165       if (cast<VTSDNode>(N12)->getVT() == MVT::i16 &&
09166           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09167         Select(Tmp0, Tmp0);
09168         Select(Tmp1, Tmp1);
09169         Select(Tmp2, Tmp2);
09170         Select(Tmp3, Tmp3);
09171         Select(Tmp4, N0);
09172         Select(Chain1, Chain1);
09173         ResNode = CurDAG->getTargetNode(X86::FpIADD16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09174         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09175         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09176         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09177         Result = SDOperand(ResNode, N.ResNo);
09178         return;
09179       }
09180 
09181       // Pattern: (fadd:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i32:Other))
09182       // Emits: (FpIADD32m:f64 RFP:f64:$src1, addr:i32:$src2)
09183       // Pattern complexity = 14  cost = 1
09184       if (cast<VTSDNode>(N12)->getVT() == MVT::i32 &&
09185           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09186         Select(Tmp0, Tmp0);
09187         Select(Tmp1, Tmp1);
09188         Select(Tmp2, Tmp2);
09189         Select(Tmp3, Tmp3);
09190         Select(Tmp4, N0);
09191         Select(Chain1, Chain1);
09192         ResNode = CurDAG->getTargetNode(X86::FpIADD32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09193         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09194         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09195         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09196         Result = SDOperand(ResNode, N.ResNo);
09197         return;
09198       }
09199     }
09200   }
09201 
09202   // Pattern: (fadd:f32 FR32:f32:$src1, (load:f32 addr:i32:$src2))
09203   // Emits: (ADDSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
09204   // Pattern complexity = 14  cost = 1
09205   if ((Subtarget->hasSSE1())) {
09206     N0 = N.getOperand(0);
09207     N1 = N.getOperand(1);
09208     if (N1.getOpcode() == ISD::LOAD &&
09209         N1.hasOneUse() &&
09210         !CodeGenMap.count(N1.getValue(0)) &&
09211         !isNonImmUse(N.Val, N1.Val)) {
09212       Chain1 = N1.getOperand(0);
09213       N11 = N1.getOperand(1);
09214       if (N.Val->getValueType(0) == MVT::f32 &&
09215           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09216         Select(Tmp0, Tmp0);
09217         Select(Tmp1, Tmp1);
09218         Select(Tmp2, Tmp2);
09219         Select(Tmp3, Tmp3);
09220         Select(Tmp4, N0);
09221         Select(Chain1, Chain1);
09222         ResNode = CurDAG->getTargetNode(X86::ADDSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09223         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09224         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09225         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09226         Result = SDOperand(ResNode, N.ResNo);
09227         return;
09228       }
09229     }
09230   }
09231 
09232   // Pattern: (fadd:f64 FR64:f64:$src1, (load:f64 addr:i32:$src2))
09233   // Emits: (ADDSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
09234   // Pattern complexity = 14  cost = 1
09235   if ((Subtarget->hasSSE2())) {
09236     N0 = N.getOperand(0);
09237     N1 = N.getOperand(1);
09238     if (N1.getOpcode() == ISD::LOAD &&
09239         N1.hasOneUse() &&
09240         !CodeGenMap.count(N1.getValue(0)) &&
09241         !isNonImmUse(N.Val, N1.Val)) {
09242       Chain1 = N1.getOperand(0);
09243       N11 = N1.getOperand(1);
09244       if (N.Val->getValueType(0) == MVT::f64 &&
09245           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09246         Select(Tmp0, Tmp0);
09247         Select(Tmp1, Tmp1);
09248         Select(Tmp2, Tmp2);
09249         Select(Tmp3, Tmp3);
09250         Select(Tmp4, N0);
09251         Select(Chain1, Chain1);
09252         ResNode = CurDAG->getTargetNode(X86::ADDSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09253         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09254         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09255         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09256         Result = SDOperand(ResNode, N.ResNo);
09257         return;
09258       }
09259     }
09260   }
09261 
09262   // Pattern: (fadd:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
09263   // Emits: (ADDPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
09264   // Pattern complexity = 14  cost = 1
09265   if ((Subtarget->hasSSE1())) {
09266     N0 = N.getOperand(0);
09267     N1 = N.getOperand(1);
09268     if (N1.getOpcode() == ISD::LOAD &&
09269         N1.hasOneUse() &&
09270         !CodeGenMap.count(N1.getValue(0)) &&
09271         !isNonImmUse(N.Val, N1.Val)) {
09272       Chain1 = N1.getOperand(0);
09273       N11 = N1.getOperand(1);
09274       if (N.Val->getValueType(0) == MVT::v4f32 &&
09275           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09276         Select(Tmp0, Tmp0);
09277         Select(Tmp1, Tmp1);
09278         Select(Tmp2, Tmp2);
09279         Select(Tmp3, Tmp3);
09280         Select(Tmp4, N0);
09281         Select(Chain1, Chain1);
09282         ResNode = CurDAG->getTargetNode(X86::ADDPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09283         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09284         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09285         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09286         Result = SDOperand(ResNode, N.ResNo);
09287         return;
09288       }
09289     }
09290   }
09291 
09292   // Pattern: (fadd:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
09293   // Emits: (ADDPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
09294   // Pattern complexity = 14  cost = 1
09295   if ((Subtarget->hasSSE2())) {
09296     N0 = N.getOperand(0);
09297     N1 = N.getOperand(1);
09298     if (N1.getOpcode() == ISD::LOAD &&
09299         N1.hasOneUse() &&
09300         !CodeGenMap.count(N1.getValue(0)) &&
09301         !isNonImmUse(N.Val, N1.Val)) {
09302       Chain1 = N1.getOperand(0);
09303       N11 = N1.getOperand(1);
09304       if (N.Val->getValueType(0) == MVT::v2f64 &&
09305           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09306         Select(Tmp0, Tmp0);
09307         Select(Tmp1, Tmp1);
09308         Select(Tmp2, Tmp2);
09309         Select(Tmp3, Tmp3);
09310         Select(Tmp4, N0);
09311         Select(Chain1, Chain1);
09312         ResNode = CurDAG->getTargetNode(X86::ADDPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09313         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09314         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09315         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09316         Result = SDOperand(ResNode, N.ResNo);
09317         return;
09318       }
09319     }
09320   }
09321   if ((!Subtarget->hasSSE2())) {
09322     N0 = N.getOperand(0);
09323 
09324     // Pattern: (fadd:f64 (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
09325     // Emits: (FpADD32m:f64 RFP:f64:$src1, addr:i32:$src2)
09326     // Pattern complexity = 14  cost = 1
09327     if (N0.getOpcode() == ISD::EXTLOAD &&
09328         N0.hasOneUse() &&
09329         !CodeGenMap.count(N0.getValue(0)) &&
09330         !isNonImmUse(N.Val, N0.Val)) {
09331       Chain0 = N0.getOperand(0);
09332       N01 = N0.getOperand(1);
09333       N02 = N0.getOperand(2);
09334       N03 = N0.getOperand(3);
09335       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
09336         N1 = N.getOperand(1);
09337         if (N.Val->getValueType(0) == MVT::f64 &&
09338             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09339           Select(Tmp0, Tmp0);
09340           Select(Tmp1, Tmp1);
09341           Select(Tmp2, Tmp2);
09342           Select(Tmp3, Tmp3);
09343           Select(Tmp4, N1);
09344           Select(Chain0, Chain0);
09345           ResNode = CurDAG->getTargetNode(X86::FpADD32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09346           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09347           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09348           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09349           Result = SDOperand(ResNode, N.ResNo);
09350           return;
09351         }
09352       }
09353     }
09354 
09355     // Pattern: (fadd:f64 (load:f64 addr:i32:$src2), RFP:f64:$src1)
09356     // Emits: (FpADD64m:f64 RFP:f64:$src1, addr:i32:$src2)
09357     // Pattern complexity = 14  cost = 1
09358     if (N0.getOpcode() == ISD::LOAD &&
09359         N0.hasOneUse() &&
09360         !CodeGenMap.count(N0.getValue(0)) &&
09361         !isNonImmUse(N.Val, N0.Val)) {
09362       Chain0 = N0.getOperand(0);
09363       N01 = N0.getOperand(1);
09364       N1 = N.getOperand(1);
09365       if (N.Val->getValueType(0) == MVT::f64 &&
09366           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09367         Select(Tmp0, Tmp0);
09368         Select(Tmp1, Tmp1);
09369         Select(Tmp2, Tmp2);
09370         Select(Tmp3, Tmp3);
09371         Select(Tmp4, N1);
09372         Select(Chain0, Chain0);
09373         ResNode = CurDAG->getTargetNode(X86::FpADD64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09374         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09375         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09376         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09377         Result = SDOperand(ResNode, N.ResNo);
09378         return;
09379       }
09380     }
09381     if (N0.getOpcode() == X86ISD::FILD &&
09382         N0.hasOneUse() &&
09383         !CodeGenMap.count(N0.getValue(0)) &&
09384         !isNonImmUse(N.Val, N0.Val)) {
09385       Chain0 = N0.getOperand(0);
09386       N01 = N0.getOperand(1);
09387       N02 = N0.getOperand(2);
09388 
09389       // Pattern: (fadd:f64 (X86fild:f64 addr:i32:$src2, i16:Other), RFP:f64:$src1)
09390       // Emits: (FpIADD16m:f64 RFP:f64:$src1, addr:i32:$src2)
09391       // Pattern complexity = 14  cost = 1
09392       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
09393         N1 = N.getOperand(1);
09394         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09395           Select(Tmp0, Tmp0);
09396           Select(Tmp1, Tmp1);
09397           Select(Tmp2, Tmp2);
09398           Select(Tmp3, Tmp3);
09399           Select(Tmp4, N1);
09400           Select(Chain0, Chain0);
09401           ResNode = CurDAG->getTargetNode(X86::FpIADD16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09402           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09403           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09404           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09405           Result = SDOperand(ResNode, N.ResNo);
09406           return;
09407         }
09408       }
09409 
09410       // Pattern: (fadd:f64 (X86fild:f64 addr:i32:$src2, i32:Other), RFP:f64:$src1)
09411       // Emits: (FpIADD32m:f64 RFP:f64:$src1, addr:i32:$src2)
09412       // Pattern complexity = 14  cost = 1
09413       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
09414         N1 = N.getOperand(1);
09415         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09416           Select(Tmp0, Tmp0);
09417           Select(Tmp1, Tmp1);
09418           Select(Tmp2, Tmp2);
09419           Select(Tmp3, Tmp3);
09420           Select(Tmp4, N1);
09421           Select(Chain0, Chain0);
09422           ResNode = CurDAG->getTargetNode(X86::FpIADD32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09423           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09424           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09425           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09426           Result = SDOperand(ResNode, N.ResNo);
09427           return;
09428         }
09429       }
09430     }
09431   }
09432 
09433   // Pattern: (fadd:f32 (load:f32 addr:i32:$src2), FR32:f32:$src1)
09434   // Emits: (ADDSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
09435   // Pattern complexity = 14  cost = 1
09436   if ((Subtarget->hasSSE1())) {
09437     N0 = N.getOperand(0);
09438     if (N0.getOpcode() == ISD::LOAD &&
09439         N0.hasOneUse() &&
09440         !CodeGenMap.count(N0.getValue(0)) &&
09441         !isNonImmUse(N.Val, N0.Val)) {
09442       Chain0 = N0.getOperand(0);
09443       N01 = N0.getOperand(1);
09444       N1 = N.getOperand(1);
09445       if (N.Val->getValueType(0) == MVT::f32 &&
09446           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09447         Select(Tmp0, Tmp0);
09448         Select(Tmp1, Tmp1);
09449         Select(Tmp2, Tmp2);
09450         Select(Tmp3, Tmp3);
09451         Select(Tmp4, N1);
09452         Select(Chain0, Chain0);
09453         ResNode = CurDAG->getTargetNode(X86::ADDSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09454         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09455         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09456         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09457         Result = SDOperand(ResNode, N.ResNo);
09458         return;
09459       }
09460     }
09461   }
09462 
09463   // Pattern: (fadd:f64 (load:f64 addr:i32:$src2), FR64:f64:$src1)
09464   // Emits: (ADDSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
09465   // Pattern complexity = 14  cost = 1
09466   if ((Subtarget->hasSSE2())) {
09467     N0 = N.getOperand(0);
09468     if (N0.getOpcode() == ISD::LOAD &&
09469         N0.hasOneUse() &&
09470         !CodeGenMap.count(N0.getValue(0)) &&
09471         !isNonImmUse(N.Val, N0.Val)) {
09472       Chain0 = N0.getOperand(0);
09473       N01 = N0.getOperand(1);
09474       N1 = N.getOperand(1);
09475       if (N.Val->getValueType(0) == MVT::f64 &&
09476           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09477         Select(Tmp0, Tmp0);
09478         Select(Tmp1, Tmp1);
09479         Select(Tmp2, Tmp2);
09480         Select(Tmp3, Tmp3);
09481         Select(Tmp4, N1);
09482         Select(Chain0, Chain0);
09483         ResNode = CurDAG->getTargetNode(X86::ADDSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09484         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09485         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09486         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09487         Result = SDOperand(ResNode, N.ResNo);
09488         return;
09489       }
09490     }
09491   }
09492 
09493   // Pattern: (fadd:v4f32 (load:v4f32 addr:i32:$src2), VR128:v4f32:$src1)
09494   // Emits: (ADDPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
09495   // Pattern complexity = 14  cost = 1
09496   if ((Subtarget->hasSSE1())) {
09497     N0 = N.getOperand(0);
09498     if (N0.getOpcode() == ISD::LOAD &&
09499         N0.hasOneUse() &&
09500         !CodeGenMap.count(N0.getValue(0)) &&
09501         !isNonImmUse(N.Val, N0.Val)) {
09502       Chain0 = N0.getOperand(0);
09503       N01 = N0.getOperand(1);
09504       N1 = N.getOperand(1);
09505       if (N.Val->getValueType(0) == MVT::v4f32 &&
09506           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09507         Select(Tmp0, Tmp0);
09508         Select(Tmp1, Tmp1);
09509         Select(Tmp2, Tmp2);
09510         Select(Tmp3, Tmp3);
09511         Select(Tmp4, N1);
09512         Select(Chain0, Chain0);
09513         ResNode = CurDAG->getTargetNode(X86::ADDPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09514         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09515         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09516         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09517         Result = SDOperand(ResNode, N.ResNo);
09518         return;
09519       }
09520     }
09521   }
09522 
09523   // Pattern: (fadd:v2f64 (load:v2f64 addr:i32:$src2), VR128:v2f64:$src1)
09524   // Emits: (ADDPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
09525   // Pattern complexity = 14  cost = 1
09526   if ((Subtarget->hasSSE2())) {
09527     N0 = N.getOperand(0);
09528     if (N0.getOpcode() == ISD::LOAD &&
09529         N0.hasOneUse() &&
09530         !CodeGenMap.count(N0.getValue(0)) &&
09531         !isNonImmUse(N.Val, N0.Val)) {
09532       Chain0 = N0.getOperand(0);
09533       N01 = N0.getOperand(1);
09534       N1 = N.getOperand(1);
09535       if (N.Val->getValueType(0) == MVT::v2f64 &&
09536           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09537         Select(Tmp0, Tmp0);
09538         Select(Tmp1, Tmp1);
09539         Select(Tmp2, Tmp2);
09540         Select(Tmp3, Tmp3);
09541         Select(Tmp4, N1);
09542         Select(Chain0, Chain0);
09543         ResNode = CurDAG->getTargetNode(X86::ADDPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09544         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09545         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09546         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09547         Result = SDOperand(ResNode, N.ResNo);
09548         return;
09549       }
09550     }
09551   }
09552 
09553   // Pattern: (fadd:f64 RFP:f64:$src1, RFP:f64:$src2)
09554   // Emits: (FpADD:f64 RFP:f64:$src1, RFP:f64:$src2)
09555   // Pattern complexity = 2  cost = 1
09556   if ((!Subtarget->hasSSE2())) {
09557     N0 = N.getOperand(0);
09558     N1 = N.getOperand(1);
09559     if (N.Val->getValueType(0) == MVT::f64) {
09560       Select(Tmp0, N0);
09561       Select(Tmp1, N1);
09562       if (N.Val->hasOneUse()) {
09563         Result = CurDAG->SelectNodeTo(N.Val, X86::FpADD, MVT::f64, Tmp0, Tmp1);
09564       } else {
09565         ResNode = CurDAG->getTargetNode(X86::FpADD, MVT::f64, Tmp0, Tmp1);
09566         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09567         Result = SDOperand(ResNode, 0);
09568       }
09569       return;
09570     }
09571   }
09572 
09573   // Pattern: (fadd:f32 FR32:f32:$src1, FR32:f32:$src2)
09574   // Emits: (ADDSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
09575   // Pattern complexity = 2  cost = 1
09576   if ((Subtarget->hasSSE1())) {
09577     N0 = N.getOperand(0);
09578     N1 = N.getOperand(1);
09579     if (N.Val->getValueType(0) == MVT::f32) {
09580       Select(Tmp0, N0);
09581       Select(Tmp1, N1);
09582       if (N.Val->hasOneUse()) {
09583         Result = CurDAG->SelectNodeTo(N.Val, X86::ADDSSrr, MVT::f32, Tmp0, Tmp1);
09584       } else {
09585         ResNode = CurDAG->getTargetNode(X86::ADDSSrr, MVT::f32, Tmp0, Tmp1);
09586         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09587         Result = SDOperand(ResNode, 0);
09588       }
09589       return;
09590     }
09591   }
09592 
09593   // Pattern: (fadd:f64 FR64:f64:$src1, FR64:f64:$src2)
09594   // Emits: (ADDSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
09595   // Pattern complexity = 2  cost = 1
09596   if ((Subtarget->hasSSE2())) {
09597     N0 = N.getOperand(0);
09598     N1 = N.getOperand(1);
09599     if (N.Val->getValueType(0) == MVT::f64) {
09600       Select(Tmp0, N0);
09601       Select(Tmp1, N1);
09602       if (N.Val->hasOneUse()) {
09603         Result = CurDAG->SelectNodeTo(N.Val, X86::ADDSDrr, MVT::f64, Tmp0, Tmp1);
09604       } else {
09605         ResNode = CurDAG->getTargetNode(X86::ADDSDrr, MVT::f64, Tmp0, Tmp1);
09606         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09607         Result = SDOperand(ResNode, 0);
09608       }
09609       return;
09610     }
09611   }
09612 
09613   // Pattern: (fadd:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
09614   // Emits: (ADDPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
09615   // Pattern complexity = 2  cost = 1
09616   if ((Subtarget->hasSSE1())) {
09617     N0 = N.getOperand(0);
09618     N1 = N.getOperand(1);
09619     if (N.Val->getValueType(0) == MVT::v4f32) {
09620       Select(Tmp0, N0);
09621       Select(Tmp1, N1);
09622       if (N.Val->hasOneUse()) {
09623         Result = CurDAG->SelectNodeTo(N.Val, X86::ADDPSrr, MVT::v4f32, Tmp0, Tmp1);
09624       } else {
09625         ResNode = CurDAG->getTargetNode(X86::ADDPSrr, MVT::v4f32, Tmp0, Tmp1);
09626         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09627         Result = SDOperand(ResNode, 0);
09628       }
09629       return;
09630     }
09631   }
09632 
09633   // Pattern: (fadd:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
09634   // Emits: (ADDPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
09635   // Pattern complexity = 2  cost = 1
09636   if ((Subtarget->hasSSE2())) {
09637     N0 = N.getOperand(0);
09638     N1 = N.getOperand(1);
09639     if (N.Val->getValueType(0) == MVT::v2f64) {
09640       Select(Tmp0, N0);
09641       Select(Tmp1, N1);
09642       if (N.Val->hasOneUse()) {
09643         Result = CurDAG->SelectNodeTo(N.Val, X86::ADDPDrr, MVT::v2f64, Tmp0, Tmp1);
09644       } else {
09645         ResNode = CurDAG->getTargetNode(X86::ADDPDrr, MVT::v2f64, Tmp0, Tmp1);
09646         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09647         Result = SDOperand(ResNode, 0);
09648       }
09649       return;
09650     }
09651   }
09652   std::cerr << "Cannot yet select: ";
09653   N.Val->dump(CurDAG);
09654   std::cerr << '\n';
09655   abort();
09656 }
09657 
09658 void Select_fcos(SDOperand &Result, SDOperand N) {
09659   SDOperand N0(0, 0);
09660   SDOperand Tmp0(0, 0);
09661   SDNode *ResNode;
09662   if ((!Subtarget->hasSSE2())) {
09663     N0 = N.getOperand(0);
09664     if (N.Val->getValueType(0) == MVT::f64) {
09665       Select(Tmp0, N0);
09666       if (N.Val->hasOneUse()) {
09667         Result = CurDAG->SelectNodeTo(N.Val, X86::FpCOS, MVT::f64, Tmp0);
09668       } else {
09669         ResNode = CurDAG->getTargetNode(X86::FpCOS, MVT::f64, Tmp0);
09670         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
09671         Result = SDOperand(ResNode, 0);
09672       }
09673       return;
09674     }
09675   }
09676   std::cerr << "Cannot yet select: ";
09677   N.Val->dump(CurDAG);
09678   std::cerr << '\n';
09679   abort();
09680 }
09681 
09682 void Select_fdiv(SDOperand &Result, SDOperand N) {
09683   SDOperand Chain0(0, 0);
09684   SDOperand Chain1(0, 0);
09685   SDOperand N0(0, 0);
09686   SDOperand N01(0, 0);
09687   SDOperand N02(0, 0);
09688   SDOperand N03(0, 0);
09689   SDOperand N1(0, 0);
09690   SDOperand N11(0, 0);
09691   SDOperand N12(0, 0);
09692   SDOperand N13(0, 0);
09693   SDOperand Tmp0(0, 0);
09694   SDOperand Tmp1(0, 0);
09695   SDOperand Tmp2(0, 0);
09696   SDOperand Tmp3(0, 0);
09697   SDOperand Tmp4(0, 0);
09698   SDNode *ResNode;
09699   if ((!Subtarget->hasSSE2())) {
09700     N0 = N.getOperand(0);
09701     {
09702       N1 = N.getOperand(1);
09703 
09704       // Pattern: (fdiv:f64 RFP:f64:$src1, (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other))
09705       // Emits: (FpDIV32m:f64 RFP:f64:$src1, addr:i32:$src2)
09706       // Pattern complexity = 14  cost = 1
09707       if (N1.getOpcode() == ISD::EXTLOAD &&
09708           N1.hasOneUse() &&
09709           !CodeGenMap.count(N1.getValue(0)) &&
09710           !isNonImmUse(N.Val, N1.Val)) {
09711         Chain1 = N1.getOperand(0);
09712         N11 = N1.getOperand(1);
09713         N12 = N1.getOperand(2);
09714         N13 = N1.getOperand(3);
09715         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
09716             N.Val->getValueType(0) == MVT::f64 &&
09717             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09718           Select(Tmp0, Tmp0);
09719           Select(Tmp1, Tmp1);
09720           Select(Tmp2, Tmp2);
09721           Select(Tmp3, Tmp3);
09722           Select(Tmp4, N0);
09723           Select(Chain1, Chain1);
09724           ResNode = CurDAG->getTargetNode(X86::FpDIV32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09725           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09726           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09727           AddHandleReplacement(N1.Val, 1, ResNode, 1);
09728           Result = SDOperand(ResNode, N.ResNo);
09729           return;
09730         }
09731       }
09732 
09733       // Pattern: (fdiv:f64 RFP:f64:$src1, (load:f64 addr:i32:$src2))
09734       // Emits: (FpDIV64m:f64 RFP:f64:$src1, addr:i32:$src2)
09735       // Pattern complexity = 14  cost = 1
09736       if (N1.getOpcode() == ISD::LOAD &&
09737           N1.hasOneUse() &&
09738           !CodeGenMap.count(N1.getValue(0)) &&
09739           !isNonImmUse(N.Val, N1.Val)) {
09740         Chain1 = N1.getOperand(0);
09741         N11 = N1.getOperand(1);
09742         if (N.Val->getValueType(0) == MVT::f64 &&
09743             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09744           Select(Tmp0, Tmp0);
09745           Select(Tmp1, Tmp1);
09746           Select(Tmp2, Tmp2);
09747           Select(Tmp3, Tmp3);
09748           Select(Tmp4, N0);
09749           Select(Chain1, Chain1);
09750           ResNode = CurDAG->getTargetNode(X86::FpDIV64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09751           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09752           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09753           AddHandleReplacement(N1.Val, 1, ResNode, 1);
09754           Result = SDOperand(ResNode, N.ResNo);
09755           return;
09756         }
09757       }
09758     }
09759 
09760     // Pattern: (fdiv:f64 (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
09761     // Emits: (FpDIVR32m:f64 RFP:f64:$src1, addr:i32:$src2)
09762     // Pattern complexity = 14  cost = 1
09763     if (N0.getOpcode() == ISD::EXTLOAD &&
09764         N0.hasOneUse() &&
09765         !CodeGenMap.count(N0.getValue(0)) &&
09766         !isNonImmUse(N.Val, N0.Val)) {
09767       Chain0 = N0.getOperand(0);
09768       N01 = N0.getOperand(1);
09769       N02 = N0.getOperand(2);
09770       N03 = N0.getOperand(3);
09771       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
09772         N1 = N.getOperand(1);
09773         if (N.Val->getValueType(0) == MVT::f64 &&
09774             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09775           Select(Tmp0, Tmp0);
09776           Select(Tmp1, Tmp1);
09777           Select(Tmp2, Tmp2);
09778           Select(Tmp3, Tmp3);
09779           Select(Tmp4, N1);
09780           Select(Chain0, Chain0);
09781           ResNode = CurDAG->getTargetNode(X86::FpDIVR32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09782           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09783           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09784           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09785           Result = SDOperand(ResNode, N.ResNo);
09786           return;
09787         }
09788       }
09789     }
09790 
09791     // Pattern: (fdiv:f64 (load:f64 addr:i32:$src2), RFP:f64:$src1)
09792     // Emits: (FpDIVR64m:f64 RFP:f64:$src1, addr:i32:$src2)
09793     // Pattern complexity = 14  cost = 1
09794     if (N0.getOpcode() == ISD::LOAD &&
09795         N0.hasOneUse() &&
09796         !CodeGenMap.count(N0.getValue(0)) &&
09797         !isNonImmUse(N.Val, N0.Val)) {
09798       Chain0 = N0.getOperand(0);
09799       N01 = N0.getOperand(1);
09800       N1 = N.getOperand(1);
09801       if (N.Val->getValueType(0) == MVT::f64 &&
09802           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09803         Select(Tmp0, Tmp0);
09804         Select(Tmp1, Tmp1);
09805         Select(Tmp2, Tmp2);
09806         Select(Tmp3, Tmp3);
09807         Select(Tmp4, N1);
09808         Select(Chain0, Chain0);
09809         ResNode = CurDAG->getTargetNode(X86::FpDIVR64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09810         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09811         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09812         AddHandleReplacement(N0.Val, 1, ResNode, 1);
09813         Result = SDOperand(ResNode, N.ResNo);
09814         return;
09815       }
09816     }
09817     {
09818       N1 = N.getOperand(1);
09819       if (N1.getOpcode() == X86ISD::FILD &&
09820           N1.hasOneUse() &&
09821           !CodeGenMap.count(N1.getValue(0)) &&
09822           !isNonImmUse(N.Val, N1.Val)) {
09823         Chain1 = N1.getOperand(0);
09824         N11 = N1.getOperand(1);
09825         N12 = N1.getOperand(2);
09826 
09827         // Pattern: (fdiv:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i16:Other))
09828         // Emits: (FpIDIV16m:f64 RFP:f64:$src1, addr:i32:$src2)
09829         // Pattern complexity = 14  cost = 1
09830         if (cast<VTSDNode>(N12)->getVT() == MVT::i16 &&
09831             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09832           Select(Tmp0, Tmp0);
09833           Select(Tmp1, Tmp1);
09834           Select(Tmp2, Tmp2);
09835           Select(Tmp3, Tmp3);
09836           Select(Tmp4, N0);
09837           Select(Chain1, Chain1);
09838           ResNode = CurDAG->getTargetNode(X86::FpIDIV16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09839           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09840           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09841           AddHandleReplacement(N1.Val, 1, ResNode, 1);
09842           Result = SDOperand(ResNode, N.ResNo);
09843           return;
09844         }
09845 
09846         // Pattern: (fdiv:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i32:Other))
09847         // Emits: (FpIDIV32m:f64 RFP:f64:$src1, addr:i32:$src2)
09848         // Pattern complexity = 14  cost = 1
09849         if (cast<VTSDNode>(N12)->getVT() == MVT::i32 &&
09850             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09851           Select(Tmp0, Tmp0);
09852           Select(Tmp1, Tmp1);
09853           Select(Tmp2, Tmp2);
09854           Select(Tmp3, Tmp3);
09855           Select(Tmp4, N0);
09856           Select(Chain1, Chain1);
09857           ResNode = CurDAG->getTargetNode(X86::FpIDIV32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09858           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09859           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09860           AddHandleReplacement(N1.Val, 1, ResNode, 1);
09861           Result = SDOperand(ResNode, N.ResNo);
09862           return;
09863         }
09864       }
09865     }
09866     if (N0.getOpcode() == X86ISD::FILD &&
09867         N0.hasOneUse() &&
09868         !CodeGenMap.count(N0.getValue(0)) &&
09869         !isNonImmUse(N.Val, N0.Val)) {
09870       Chain0 = N0.getOperand(0);
09871       N01 = N0.getOperand(1);
09872       N02 = N0.getOperand(2);
09873 
09874       // Pattern: (fdiv:f64 (X86fild:f64 addr:i32:$src2, i16:Other), RFP:f64:$src1)
09875       // Emits: (FpIDIVR16m:f64 RFP:f64:$src1, addr:i32:$src2)
09876       // Pattern complexity = 14  cost = 1
09877       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
09878         N1 = N.getOperand(1);
09879         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09880           Select(Tmp0, Tmp0);
09881           Select(Tmp1, Tmp1);
09882           Select(Tmp2, Tmp2);
09883           Select(Tmp3, Tmp3);
09884           Select(Tmp4, N1);
09885           Select(Chain0, Chain0);
09886           ResNode = CurDAG->getTargetNode(X86::FpIDIVR16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09887           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09888           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09889           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09890           Result = SDOperand(ResNode, N.ResNo);
09891           return;
09892         }
09893       }
09894 
09895       // Pattern: (fdiv:f64 (X86fild:f64 addr:i32:$src2, i32:Other), RFP:f64:$src1)
09896       // Emits: (FpIDIVR32m:f64 RFP:f64:$src1, addr:i32:$src2)
09897       // Pattern complexity = 14  cost = 1
09898       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
09899         N1 = N.getOperand(1);
09900         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
09901           Select(Tmp0, Tmp0);
09902           Select(Tmp1, Tmp1);
09903           Select(Tmp2, Tmp2);
09904           Select(Tmp3, Tmp3);
09905           Select(Tmp4, N1);
09906           Select(Chain0, Chain0);
09907           ResNode = CurDAG->getTargetNode(X86::FpIDIVR32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
09908           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09909           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
09910           AddHandleReplacement(N0.Val, 1, ResNode, 1);
09911           Result = SDOperand(ResNode, N.ResNo);
09912           return;
09913         }
09914       }
09915     }
09916   }
09917 
09918   // Pattern: (fdiv:f32 FR32:f32:$src1, (load:f32 addr:i32:$src2))
09919   // Emits: (DIVSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
09920   // Pattern complexity = 14  cost = 1
09921   if ((Subtarget->hasSSE1())) {
09922     N0 = N.getOperand(0);
09923     N1 = N.getOperand(1);
09924     if (N1.getOpcode() == ISD::LOAD &&
09925         N1.hasOneUse() &&
09926         !CodeGenMap.count(N1.getValue(0)) &&
09927         !isNonImmUse(N.Val, N1.Val)) {
09928       Chain1 = N1.getOperand(0);
09929       N11 = N1.getOperand(1);
09930       if (N.Val->getValueType(0) == MVT::f32 &&
09931           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09932         Select(Tmp0, Tmp0);
09933         Select(Tmp1, Tmp1);
09934         Select(Tmp2, Tmp2);
09935         Select(Tmp3, Tmp3);
09936         Select(Tmp4, N0);
09937         Select(Chain1, Chain1);
09938         ResNode = CurDAG->getTargetNode(X86::DIVSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09939         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09940         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09941         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09942         Result = SDOperand(ResNode, N.ResNo);
09943         return;
09944       }
09945     }
09946   }
09947 
09948   // Pattern: (fdiv:f64 FR64:f64:$src1, (load:f64 addr:i32:$src2))
09949   // Emits: (DIVSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
09950   // Pattern complexity = 14  cost = 1
09951   if ((Subtarget->hasSSE2())) {
09952     N0 = N.getOperand(0);
09953     N1 = N.getOperand(1);
09954     if (N1.getOpcode() == ISD::LOAD &&
09955         N1.hasOneUse() &&
09956         !CodeGenMap.count(N1.getValue(0)) &&
09957         !isNonImmUse(N.Val, N1.Val)) {
09958       Chain1 = N1.getOperand(0);
09959       N11 = N1.getOperand(1);
09960       if (N.Val->getValueType(0) == MVT::f64 &&
09961           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09962         Select(Tmp0, Tmp0);
09963         Select(Tmp1, Tmp1);
09964         Select(Tmp2, Tmp2);
09965         Select(Tmp3, Tmp3);
09966         Select(Tmp4, N0);
09967         Select(Chain1, Chain1);
09968         ResNode = CurDAG->getTargetNode(X86::DIVSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09969         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
09970         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
09971         AddHandleReplacement(N1.Val, 1, ResNode, 1);
09972         Result = SDOperand(ResNode, N.ResNo);
09973         return;
09974       }
09975     }
09976   }
09977 
09978   // Pattern: (fdiv:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
09979   // Emits: (DIVPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
09980   // Pattern complexity = 14  cost = 1
09981   if ((Subtarget->hasSSE1())) {
09982     N0 = N.getOperand(0);
09983     N1 = N.getOperand(1);
09984     if (N1.getOpcode() == ISD::LOAD &&
09985         N1.hasOneUse() &&
09986         !CodeGenMap.count(N1.getValue(0)) &&
09987         !isNonImmUse(N.Val, N1.Val)) {
09988       Chain1 = N1.getOperand(0);
09989       N11 = N1.getOperand(1);
09990       if (N.Val->getValueType(0) == MVT::v4f32 &&
09991           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
09992         Select(Tmp0, Tmp0);
09993         Select(Tmp1, Tmp1);
09994         Select(Tmp2, Tmp2);
09995         Select(Tmp3, Tmp3);
09996         Select(Tmp4, N0);
09997         Select(Chain1, Chain1);
09998         ResNode = CurDAG->getTargetNode(X86::DIVPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
09999         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10000         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10001         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10002         Result = SDOperand(ResNode, N.ResNo);
10003         return;
10004       }
10005     }
10006   }
10007 
10008   // Pattern: (fdiv:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
10009   // Emits: (DIVPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
10010   // Pattern complexity = 14  cost = 1
10011   if ((Subtarget->hasSSE2())) {
10012     N0 = N.getOperand(0);
10013     N1 = N.getOperand(1);
10014     if (N1.getOpcode() == ISD::LOAD &&
10015         N1.hasOneUse() &&
10016         !CodeGenMap.count(N1.getValue(0)) &&
10017         !isNonImmUse(N.Val, N1.Val)) {
10018       Chain1 = N1.getOperand(0);
10019       N11 = N1.getOperand(1);
10020       if (N.Val->getValueType(0) == MVT::v2f64 &&
10021           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10022         Select(Tmp0, Tmp0);
10023         Select(Tmp1, Tmp1);
10024         Select(Tmp2, Tmp2);
10025         Select(Tmp3, Tmp3);
10026         Select(Tmp4, N0);
10027         Select(Chain1, Chain1);
10028         ResNode = CurDAG->getTargetNode(X86::DIVPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10029         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10030         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10031         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10032         Result = SDOperand(ResNode, N.ResNo);
10033         return;
10034       }
10035     }
10036   }
10037 
10038   // Pattern: (fdiv:f64 RFP:f64:$src1, RFP:f64:$src2)
10039   // Emits: (FpDIV:f64 RFP:f64:$src1, RFP:f64:$src2)
10040   // Pattern complexity = 2  cost = 1
10041   if ((!Subtarget->hasSSE2())) {
10042     N0 = N.getOperand(0);
10043     N1 = N.getOperand(1);
10044     if (N.Val->getValueType(0) == MVT::f64) {
10045       Select(Tmp0, N0);
10046       Select(Tmp1, N1);
10047       if (N.Val->hasOneUse()) {
10048         Result = CurDAG->SelectNodeTo(N.Val, X86::FpDIV, MVT::f64, Tmp0, Tmp1);
10049       } else {
10050         ResNode = CurDAG->getTargetNode(X86::FpDIV, MVT::f64, Tmp0, Tmp1);
10051         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10052         Result = SDOperand(ResNode, 0);
10053       }
10054       return;
10055     }
10056   }
10057 
10058   // Pattern: (fdiv:f32 FR32:f32:$src1, FR32:f32:$src2)
10059   // Emits: (DIVSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
10060   // Pattern complexity = 2  cost = 1
10061   if ((Subtarget->hasSSE1())) {
10062     N0 = N.getOperand(0);
10063     N1 = N.getOperand(1);
10064     if (N.Val->getValueType(0) == MVT::f32) {
10065       Select(Tmp0, N0);
10066       Select(Tmp1, N1);
10067       if (N.Val->hasOneUse()) {
10068         Result = CurDAG->SelectNodeTo(N.Val, X86::DIVSSrr, MVT::f32, Tmp0, Tmp1);
10069       } else {
10070         ResNode = CurDAG->getTargetNode(X86::DIVSSrr, MVT::f32, Tmp0, Tmp1);
10071         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10072         Result = SDOperand(ResNode, 0);
10073       }
10074       return;
10075     }
10076   }
10077 
10078   // Pattern: (fdiv:f64 FR64:f64:$src1, FR64:f64:$src2)
10079   // Emits: (DIVSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
10080   // Pattern complexity = 2  cost = 1
10081   if ((Subtarget->hasSSE2())) {
10082     N0 = N.getOperand(0);
10083     N1 = N.getOperand(1);
10084     if (N.Val->getValueType(0) == MVT::f64) {
10085       Select(Tmp0, N0);
10086       Select(Tmp1, N1);
10087       if (N.Val->hasOneUse()) {
10088         Result = CurDAG->SelectNodeTo(N.Val, X86::DIVSDrr, MVT::f64, Tmp0, Tmp1);
10089       } else {
10090         ResNode = CurDAG->getTargetNode(X86::DIVSDrr, MVT::f64, Tmp0, Tmp1);
10091         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10092         Result = SDOperand(ResNode, 0);
10093       }
10094       return;
10095     }
10096   }
10097 
10098   // Pattern: (fdiv:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10099   // Emits: (DIVPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10100   // Pattern complexity = 2  cost = 1
10101   if ((Subtarget->hasSSE1())) {
10102     N0 = N.getOperand(0);
10103     N1 = N.getOperand(1);
10104     if (N.Val->getValueType(0) == MVT::v4f32) {
10105       Select(Tmp0, N0);
10106       Select(Tmp1, N1);
10107       if (N.Val->hasOneUse()) {
10108         Result = CurDAG->SelectNodeTo(N.Val, X86::DIVPSrr, MVT::v4f32, Tmp0, Tmp1);
10109       } else {
10110         ResNode = CurDAG->getTargetNode(X86::DIVPSrr, MVT::v4f32, Tmp0, Tmp1);
10111         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10112         Result = SDOperand(ResNode, 0);
10113       }
10114       return;
10115     }
10116   }
10117 
10118   // Pattern: (fdiv:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10119   // Emits: (DIVPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10120   // Pattern complexity = 2  cost = 1
10121   if ((Subtarget->hasSSE2())) {
10122     N0 = N.getOperand(0);
10123     N1 = N.getOperand(1);
10124     if (N.Val->getValueType(0) == MVT::v2f64) {
10125       Select(Tmp0, N0);
10126       Select(Tmp1, N1);
10127       if (N.Val->hasOneUse()) {
10128         Result = CurDAG->SelectNodeTo(N.Val, X86::DIVPDrr, MVT::v2f64, Tmp0, Tmp1);
10129       } else {
10130         ResNode = CurDAG->getTargetNode(X86::DIVPDrr, MVT::v2f64, Tmp0, Tmp1);
10131         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10132         Result = SDOperand(ResNode, 0);
10133       }
10134       return;
10135     }
10136   }
10137   std::cerr << "Cannot yet select: ";
10138   N.Val->dump(CurDAG);
10139   std::cerr << '\n';
10140   abort();
10141 }
10142 
10143 void Select_fextend(SDOperand &Result, SDOperand N) {
10144   SDOperand Chain0(0, 0);
10145   SDOperand N0(0, 0);
10146   SDOperand N01(0, 0);
10147   SDOperand Tmp0(0, 0);
10148   SDOperand Tmp1(0, 0);
10149   SDOperand Tmp2(0, 0);
10150   SDOperand Tmp3(0, 0);
10151   SDNode *ResNode;
10152   if ((Subtarget->hasSSE2())) {
10153     N0 = N.getOperand(0);
10154 
10155     // Pattern: (fextend:f64 (load:f32 addr:i32:$src))
10156     // Emits: (CVTSS2SDrm:f64 addr:i32:$src)
10157     // Pattern complexity = 14  cost = 1
10158     if (N0.getOpcode() == ISD::LOAD &&
10159         N0.hasOneUse() &&
10160         !CodeGenMap.count(N0.getValue(0))) {
10161       Chain0 = N0.getOperand(0);
10162       N01 = N0.getOperand(1);
10163       if (N.Val->getValueType(0) == MVT::f64 &&
10164           N0.Val->getValueType(0) == MVT::f32 &&
10165           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10166         Select(Tmp0, Tmp0);
10167         Select(Tmp1, Tmp1);
10168         Select(Tmp2, Tmp2);
10169         Select(Tmp3, Tmp3);
10170         Select(Chain0, Chain0);
10171         ResNode = CurDAG->getTargetNode(X86::CVTSS2SDrm, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10172         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10173         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10174         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10175         Result = SDOperand(ResNode, N.ResNo);
10176         return;
10177       }
10178     }
10179 
10180     // Pattern: (fextend:f64 FR32:f32:$src)
10181     // Emits: (CVTSS2SDrr:f64 FR32:f32:$src)
10182     // Pattern complexity = 2  cost = 1
10183     if (N.Val->getValueType(0) == MVT::f64 &&
10184         N0.Val->getValueType(0) == MVT::f32) {
10185       Select(Tmp0, N0);
10186       if (N.Val->hasOneUse()) {
10187         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSS2SDrr, MVT::f64, Tmp0);
10188       } else {
10189         ResNode = CurDAG->getTargetNode(X86::CVTSS2SDrr, MVT::f64, Tmp0);
10190         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10191         Result = SDOperand(ResNode, 0);
10192       }
10193       return;
10194     }
10195   }
10196   std::cerr << "Cannot yet select: ";
10197   N.Val->dump(CurDAG);
10198   std::cerr << '\n';
10199   abort();
10200 }
10201 
10202 void Select_fmul(SDOperand &Result, SDOperand N) {
10203   SDOperand Chain0(0, 0);
10204   SDOperand Chain1(0, 0);
10205   SDOperand N0(0, 0);
10206   SDOperand N01(0, 0);
10207   SDOperand N02(0, 0);
10208   SDOperand N03(0, 0);
10209   SDOperand N1(0, 0);
10210   SDOperand N11(0, 0);
10211   SDOperand N12(0, 0);
10212   SDOperand N13(0, 0);
10213   SDOperand Tmp0(0, 0);
10214   SDOperand Tmp1(0, 0);
10215   SDOperand Tmp2(0, 0);
10216   SDOperand Tmp3(0, 0);
10217   SDOperand Tmp4(0, 0);
10218   SDNode *ResNode;
10219   if ((!Subtarget->hasSSE2())) {
10220     N0 = N.getOperand(0);
10221     N1 = N.getOperand(1);
10222 
10223     // Pattern: (fmul:f64 RFP:f64:$src1, (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other))
10224     // Emits: (FpMUL32m:f64 RFP:f64:$src1, addr:i32:$src2)
10225     // Pattern complexity = 14  cost = 1
10226     if (N1.getOpcode() == ISD::EXTLOAD &&
10227         N1.hasOneUse() &&
10228         !CodeGenMap.count(N1.getValue(0)) &&
10229         !isNonImmUse(N.Val, N1.Val)) {
10230       Chain1 = N1.getOperand(0);
10231       N11 = N1.getOperand(1);
10232       N12 = N1.getOperand(2);
10233       N13 = N1.getOperand(3);
10234       if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
10235           N.Val->getValueType(0) == MVT::f64 &&
10236           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10237         Select(Tmp0, Tmp0);
10238         Select(Tmp1, Tmp1);
10239         Select(Tmp2, Tmp2);
10240         Select(Tmp3, Tmp3);
10241         Select(Tmp4, N0);
10242         Select(Chain1, Chain1);
10243         ResNode = CurDAG->getTargetNode(X86::FpMUL32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10244         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10245         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10246         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10247         Result = SDOperand(ResNode, N.ResNo);
10248         return;
10249       }
10250     }
10251 
10252     // Pattern: (fmul:f64 RFP:f64:$src1, (load:f64 addr:i32:$src2))
10253     // Emits: (FpMUL64m:f64 RFP:f64:$src1, addr:i32:$src2)
10254     // Pattern complexity = 14  cost = 1
10255     if (N1.getOpcode() == ISD::LOAD &&
10256         N1.hasOneUse() &&
10257         !CodeGenMap.count(N1.getValue(0)) &&
10258         !isNonImmUse(N.Val, N1.Val)) {
10259       Chain1 = N1.getOperand(0);
10260       N11 = N1.getOperand(1);
10261       if (N.Val->getValueType(0) == MVT::f64 &&
10262           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10263         Select(Tmp0, Tmp0);
10264         Select(Tmp1, Tmp1);
10265         Select(Tmp2, Tmp2);
10266         Select(Tmp3, Tmp3);
10267         Select(Tmp4, N0);
10268         Select(Chain1, Chain1);
10269         ResNode = CurDAG->getTargetNode(X86::FpMUL64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10270         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10271         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10272         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10273         Result = SDOperand(ResNode, N.ResNo);
10274         return;
10275       }
10276     }
10277     if (N1.getOpcode() == X86ISD::FILD &&
10278         N1.hasOneUse() &&
10279         !CodeGenMap.count(N1.getValue(0)) &&
10280         !isNonImmUse(N.Val, N1.Val)) {
10281       Chain1 = N1.getOperand(0);
10282       N11 = N1.getOperand(1);
10283       N12 = N1.getOperand(2);
10284 
10285       // Pattern: (fmul:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i16:Other))
10286       // Emits: (FpIMUL16m:f64 RFP:f64:$src1, addr:i32:$src2)
10287       // Pattern complexity = 14  cost = 1
10288       if (cast<VTSDNode>(N12)->getVT() == MVT::i16 &&
10289           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10290         Select(Tmp0, Tmp0);
10291         Select(Tmp1, Tmp1);
10292         Select(Tmp2, Tmp2);
10293         Select(Tmp3, Tmp3);
10294         Select(Tmp4, N0);
10295         Select(Chain1, Chain1);
10296         ResNode = CurDAG->getTargetNode(X86::FpIMUL16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10297         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10298         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10299         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10300         Result = SDOperand(ResNode, N.ResNo);
10301         return;
10302       }
10303 
10304       // Pattern: (fmul:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i32:Other))
10305       // Emits: (FpIMUL32m:f64 RFP:f64:$src1, addr:i32:$src2)
10306       // Pattern complexity = 14  cost = 1
10307       if (cast<VTSDNode>(N12)->getVT() == MVT::i32 &&
10308           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10309         Select(Tmp0, Tmp0);
10310         Select(Tmp1, Tmp1);
10311         Select(Tmp2, Tmp2);
10312         Select(Tmp3, Tmp3);
10313         Select(Tmp4, N0);
10314         Select(Chain1, Chain1);
10315         ResNode = CurDAG->getTargetNode(X86::FpIMUL32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10316         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10317         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10318         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10319         Result = SDOperand(ResNode, N.ResNo);
10320         return;
10321       }
10322     }
10323   }
10324 
10325   // Pattern: (fmul:f32 FR32:f32:$src1, (load:f32 addr:i32:$src2))
10326   // Emits: (MULSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
10327   // Pattern complexity = 14  cost = 1
10328   if ((Subtarget->hasSSE1())) {
10329     N0 = N.getOperand(0);
10330     N1 = N.getOperand(1);
10331     if (N1.getOpcode() == ISD::LOAD &&
10332         N1.hasOneUse() &&
10333         !CodeGenMap.count(N1.getValue(0)) &&
10334         !isNonImmUse(N.Val, N1.Val)) {
10335       Chain1 = N1.getOperand(0);
10336       N11 = N1.getOperand(1);
10337       if (N.Val->getValueType(0) == MVT::f32 &&
10338           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10339         Select(Tmp0, Tmp0);
10340         Select(Tmp1, Tmp1);
10341         Select(Tmp2, Tmp2);
10342         Select(Tmp3, Tmp3);
10343         Select(Tmp4, N0);
10344         Select(Chain1, Chain1);
10345         ResNode = CurDAG->getTargetNode(X86::MULSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10346         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10347         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10348         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10349         Result = SDOperand(ResNode, N.ResNo);
10350         return;
10351       }
10352     }
10353   }
10354 
10355   // Pattern: (fmul:f64 FR64:f64:$src1, (load:f64 addr:i32:$src2))
10356   // Emits: (MULSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
10357   // Pattern complexity = 14  cost = 1
10358   if ((Subtarget->hasSSE2())) {
10359     N0 = N.getOperand(0);
10360     N1 = N.getOperand(1);
10361     if (N1.getOpcode() == ISD::LOAD &&
10362         N1.hasOneUse() &&
10363         !CodeGenMap.count(N1.getValue(0)) &&
10364         !isNonImmUse(N.Val, N1.Val)) {
10365       Chain1 = N1.getOperand(0);
10366       N11 = N1.getOperand(1);
10367       if (N.Val->getValueType(0) == MVT::f64 &&
10368           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10369         Select(Tmp0, Tmp0);
10370         Select(Tmp1, Tmp1);
10371         Select(Tmp2, Tmp2);
10372         Select(Tmp3, Tmp3);
10373         Select(Tmp4, N0);
10374         Select(Chain1, Chain1);
10375         ResNode = CurDAG->getTargetNode(X86::MULSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10376         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10377         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10378         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10379         Result = SDOperand(ResNode, N.ResNo);
10380         return;
10381       }
10382     }
10383   }
10384 
10385   // Pattern: (fmul:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
10386   // Emits: (MULPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
10387   // Pattern complexity = 14  cost = 1
10388   if ((Subtarget->hasSSE1())) {
10389     N0 = N.getOperand(0);
10390     N1 = N.getOperand(1);
10391     if (N1.getOpcode() == ISD::LOAD &&
10392         N1.hasOneUse() &&
10393         !CodeGenMap.count(N1.getValue(0)) &&
10394         !isNonImmUse(N.Val, N1.Val)) {
10395       Chain1 = N1.getOperand(0);
10396       N11 = N1.getOperand(1);
10397       if (N.Val->getValueType(0) == MVT::v4f32 &&
10398           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10399         Select(Tmp0, Tmp0);
10400         Select(Tmp1, Tmp1);
10401         Select(Tmp2, Tmp2);
10402         Select(Tmp3, Tmp3);
10403         Select(Tmp4, N0);
10404         Select(Chain1, Chain1);
10405         ResNode = CurDAG->getTargetNode(X86::MULPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10406         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10407         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10408         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10409         Result = SDOperand(ResNode, N.ResNo);
10410         return;
10411       }
10412     }
10413   }
10414 
10415   // Pattern: (fmul:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
10416   // Emits: (MULPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
10417   // Pattern complexity = 14  cost = 1
10418   if ((Subtarget->hasSSE2())) {
10419     N0 = N.getOperand(0);
10420     N1 = N.getOperand(1);
10421     if (N1.getOpcode() == ISD::LOAD &&
10422         N1.hasOneUse() &&
10423         !CodeGenMap.count(N1.getValue(0)) &&
10424         !isNonImmUse(N.Val, N1.Val)) {
10425       Chain1 = N1.getOperand(0);
10426       N11 = N1.getOperand(1);
10427       if (N.Val->getValueType(0) == MVT::v2f64 &&
10428           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
10429         Select(Tmp0, Tmp0);
10430         Select(Tmp1, Tmp1);
10431         Select(Tmp2, Tmp2);
10432         Select(Tmp3, Tmp3);
10433         Select(Tmp4, N0);
10434         Select(Chain1, Chain1);
10435         ResNode = CurDAG->getTargetNode(X86::MULPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
10436         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10437         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
10438         AddHandleReplacement(N1.Val, 1, ResNode, 1);
10439         Result = SDOperand(ResNode, N.ResNo);
10440         return;
10441       }
10442     }
10443   }
10444   if ((!Subtarget->hasSSE2())) {
10445     N0 = N.getOperand(0);
10446 
10447     // Pattern: (fmul:f64 (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
10448     // Emits: (FpMUL32m:f64 RFP:f64:$src1, addr:i32:$src2)
10449     // Pattern complexity = 14  cost = 1
10450     if (N0.getOpcode() == ISD::EXTLOAD &&
10451         N0.hasOneUse() &&
10452         !CodeGenMap.count(N0.getValue(0)) &&
10453         !isNonImmUse(N.Val, N0.Val)) {
10454       Chain0 = N0.getOperand(0);
10455       N01 = N0.getOperand(1);
10456       N02 = N0.getOperand(2);
10457       N03 = N0.getOperand(3);
10458       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
10459         N1 = N.getOperand(1);
10460         if (N.Val->getValueType(0) == MVT::f64 &&
10461             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10462           Select(Tmp0, Tmp0);
10463           Select(Tmp1, Tmp1);
10464           Select(Tmp2, Tmp2);
10465           Select(Tmp3, Tmp3);
10466           Select(Tmp4, N1);
10467           Select(Chain0, Chain0);
10468           ResNode = CurDAG->getTargetNode(X86::FpMUL32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10469           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10470           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10471           AddHandleReplacement(N0.Val, 1, ResNode, 1);
10472           Result = SDOperand(ResNode, N.ResNo);
10473           return;
10474         }
10475       }
10476     }
10477 
10478     // Pattern: (fmul:f64 (load:f64 addr:i32:$src2), RFP:f64:$src1)
10479     // Emits: (FpMUL64m:f64 RFP:f64:$src1, addr:i32:$src2)
10480     // Pattern complexity = 14  cost = 1
10481     if (N0.getOpcode() == ISD::LOAD &&
10482         N0.hasOneUse() &&
10483         !CodeGenMap.count(N0.getValue(0)) &&
10484         !isNonImmUse(N.Val, N0.Val)) {
10485       Chain0 = N0.getOperand(0);
10486       N01 = N0.getOperand(1);
10487       N1 = N.getOperand(1);
10488       if (N.Val->getValueType(0) == MVT::f64 &&
10489           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10490         Select(Tmp0, Tmp0);
10491         Select(Tmp1, Tmp1);
10492         Select(Tmp2, Tmp2);
10493         Select(Tmp3, Tmp3);
10494         Select(Tmp4, N1);
10495         Select(Chain0, Chain0);
10496         ResNode = CurDAG->getTargetNode(X86::FpMUL64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10497         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10498         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10499         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10500         Result = SDOperand(ResNode, N.ResNo);
10501         return;
10502       }
10503     }
10504     if (N0.getOpcode() == X86ISD::FILD &&
10505         N0.hasOneUse() &&
10506         !CodeGenMap.count(N0.getValue(0)) &&
10507         !isNonImmUse(N.Val, N0.Val)) {
10508       Chain0 = N0.getOperand(0);
10509       N01 = N0.getOperand(1);
10510       N02 = N0.getOperand(2);
10511 
10512       // Pattern: (fmul:f64 (X86fild:f64 addr:i32:$src2, i16:Other), RFP:f64:$src1)
10513       // Emits: (FpIMUL16m:f64 RFP:f64:$src1, addr:i32:$src2)
10514       // Pattern complexity = 14  cost = 1
10515       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
10516         N1 = N.getOperand(1);
10517         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10518           Select(Tmp0, Tmp0);
10519           Select(Tmp1, Tmp1);
10520           Select(Tmp2, Tmp2);
10521           Select(Tmp3, Tmp3);
10522           Select(Tmp4, N1);
10523           Select(Chain0, Chain0);
10524           ResNode = CurDAG->getTargetNode(X86::FpIMUL16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10525           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10526           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10527           AddHandleReplacement(N0.Val, 1, ResNode, 1);
10528           Result = SDOperand(ResNode, N.ResNo);
10529           return;
10530         }
10531       }
10532 
10533       // Pattern: (fmul:f64 (X86fild:f64 addr:i32:$src2, i32:Other), RFP:f64:$src1)
10534       // Emits: (FpIMUL32m:f64 RFP:f64:$src1, addr:i32:$src2)
10535       // Pattern complexity = 14  cost = 1
10536       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
10537         N1 = N.getOperand(1);
10538         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10539           Select(Tmp0, Tmp0);
10540           Select(Tmp1, Tmp1);
10541           Select(Tmp2, Tmp2);
10542           Select(Tmp3, Tmp3);
10543           Select(Tmp4, N1);
10544           Select(Chain0, Chain0);
10545           ResNode = CurDAG->getTargetNode(X86::FpIMUL32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10546           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10547           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10548           AddHandleReplacement(N0.Val, 1, ResNode, 1);
10549           Result = SDOperand(ResNode, N.ResNo);
10550           return;
10551         }
10552       }
10553     }
10554   }
10555 
10556   // Pattern: (fmul:f32 (load:f32 addr:i32:$src2), FR32:f32:$src1)
10557   // Emits: (MULSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
10558   // Pattern complexity = 14  cost = 1
10559   if ((Subtarget->hasSSE1())) {
10560     N0 = N.getOperand(0);
10561     if (N0.getOpcode() == ISD::LOAD &&
10562         N0.hasOneUse() &&
10563         !CodeGenMap.count(N0.getValue(0)) &&
10564         !isNonImmUse(N.Val, N0.Val)) {
10565       Chain0 = N0.getOperand(0);
10566       N01 = N0.getOperand(1);
10567       N1 = N.getOperand(1);
10568       if (N.Val->getValueType(0) == MVT::f32 &&
10569           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10570         Select(Tmp0, Tmp0);
10571         Select(Tmp1, Tmp1);
10572         Select(Tmp2, Tmp2);
10573         Select(Tmp3, Tmp3);
10574         Select(Tmp4, N1);
10575         Select(Chain0, Chain0);
10576         ResNode = CurDAG->getTargetNode(X86::MULSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10577         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10578         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10579         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10580         Result = SDOperand(ResNode, N.ResNo);
10581         return;
10582       }
10583     }
10584   }
10585 
10586   // Pattern: (fmul:f64 (load:f64 addr:i32:$src2), FR64:f64:$src1)
10587   // Emits: (MULSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
10588   // Pattern complexity = 14  cost = 1
10589   if ((Subtarget->hasSSE2())) {
10590     N0 = N.getOperand(0);
10591     if (N0.getOpcode() == ISD::LOAD &&
10592         N0.hasOneUse() &&
10593         !CodeGenMap.count(N0.getValue(0)) &&
10594         !isNonImmUse(N.Val, N0.Val)) {
10595       Chain0 = N0.getOperand(0);
10596       N01 = N0.getOperand(1);
10597       N1 = N.getOperand(1);
10598       if (N.Val->getValueType(0) == MVT::f64 &&
10599           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10600         Select(Tmp0, Tmp0);
10601         Select(Tmp1, Tmp1);
10602         Select(Tmp2, Tmp2);
10603         Select(Tmp3, Tmp3);
10604         Select(Tmp4, N1);
10605         Select(Chain0, Chain0);
10606         ResNode = CurDAG->getTargetNode(X86::MULSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10607         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10608         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10609         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10610         Result = SDOperand(ResNode, N.ResNo);
10611         return;
10612       }
10613     }
10614   }
10615 
10616   // Pattern: (fmul:v4f32 (load:v4f32 addr:i32:$src2), VR128:v4f32:$src1)
10617   // Emits: (MULPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
10618   // Pattern complexity = 14  cost = 1
10619   if ((Subtarget->hasSSE1())) {
10620     N0 = N.getOperand(0);
10621     if (N0.getOpcode() == ISD::LOAD &&
10622         N0.hasOneUse() &&
10623         !CodeGenMap.count(N0.getValue(0)) &&
10624         !isNonImmUse(N.Val, N0.Val)) {
10625       Chain0 = N0.getOperand(0);
10626       N01 = N0.getOperand(1);
10627       N1 = N.getOperand(1);
10628       if (N.Val->getValueType(0) == MVT::v4f32 &&
10629           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10630         Select(Tmp0, Tmp0);
10631         Select(Tmp1, Tmp1);
10632         Select(Tmp2, Tmp2);
10633         Select(Tmp3, Tmp3);
10634         Select(Tmp4, N1);
10635         Select(Chain0, Chain0);
10636         ResNode = CurDAG->getTargetNode(X86::MULPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10637         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10638         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10639         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10640         Result = SDOperand(ResNode, N.ResNo);
10641         return;
10642       }
10643     }
10644   }
10645 
10646   // Pattern: (fmul:v2f64 (load:v2f64 addr:i32:$src2), VR128:v2f64:$src1)
10647   // Emits: (MULPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
10648   // Pattern complexity = 14  cost = 1
10649   if ((Subtarget->hasSSE2())) {
10650     N0 = N.getOperand(0);
10651     if (N0.getOpcode() == ISD::LOAD &&
10652         N0.hasOneUse() &&
10653         !CodeGenMap.count(N0.getValue(0)) &&
10654         !isNonImmUse(N.Val, N0.Val)) {
10655       Chain0 = N0.getOperand(0);
10656       N01 = N0.getOperand(1);
10657       N1 = N.getOperand(1);
10658       if (N.Val->getValueType(0) == MVT::v2f64 &&
10659           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10660         Select(Tmp0, Tmp0);
10661         Select(Tmp1, Tmp1);
10662         Select(Tmp2, Tmp2);
10663         Select(Tmp3, Tmp3);
10664         Select(Tmp4, N1);
10665         Select(Chain0, Chain0);
10666         ResNode = CurDAG->getTargetNode(X86::MULPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10667         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10668         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10669         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10670         Result = SDOperand(ResNode, N.ResNo);
10671         return;
10672       }
10673     }
10674   }
10675 
10676   // Pattern: (fmul:f64 RFP:f64:$src1, RFP:f64:$src2)
10677   // Emits: (FpMUL:f64 RFP:f64:$src1, RFP:f64:$src2)
10678   // Pattern complexity = 2  cost = 1
10679   if ((!Subtarget->hasSSE2())) {
10680     N0 = N.getOperand(0);
10681     N1 = N.getOperand(1);
10682     if (N.Val->getValueType(0) == MVT::f64) {
10683       Select(Tmp0, N0);
10684       Select(Tmp1, N1);
10685       if (N.Val->hasOneUse()) {
10686         Result = CurDAG->SelectNodeTo(N.Val, X86::FpMUL, MVT::f64, Tmp0, Tmp1);
10687       } else {
10688         ResNode = CurDAG->getTargetNode(X86::FpMUL, MVT::f64, Tmp0, Tmp1);
10689         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10690         Result = SDOperand(ResNode, 0);
10691       }
10692       return;
10693     }
10694   }
10695 
10696   // Pattern: (fmul:f32 FR32:f32:$src1, FR32:f32:$src2)
10697   // Emits: (MULSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
10698   // Pattern complexity = 2  cost = 1
10699   if ((Subtarget->hasSSE1())) {
10700     N0 = N.getOperand(0);
10701     N1 = N.getOperand(1);
10702     if (N.Val->getValueType(0) == MVT::f32) {
10703       Select(Tmp0, N0);
10704       Select(Tmp1, N1);
10705       if (N.Val->hasOneUse()) {
10706         Result = CurDAG->SelectNodeTo(N.Val, X86::MULSSrr, MVT::f32, Tmp0, Tmp1);
10707       } else {
10708         ResNode = CurDAG->getTargetNode(X86::MULSSrr, MVT::f32, Tmp0, Tmp1);
10709         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10710         Result = SDOperand(ResNode, 0);
10711       }
10712       return;
10713     }
10714   }
10715 
10716   // Pattern: (fmul:f64 FR64:f64:$src1, FR64:f64:$src2)
10717   // Emits: (MULSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
10718   // Pattern complexity = 2  cost = 1
10719   if ((Subtarget->hasSSE2())) {
10720     N0 = N.getOperand(0);
10721     N1 = N.getOperand(1);
10722     if (N.Val->getValueType(0) == MVT::f64) {
10723       Select(Tmp0, N0);
10724       Select(Tmp1, N1);
10725       if (N.Val->hasOneUse()) {
10726         Result = CurDAG->SelectNodeTo(N.Val, X86::MULSDrr, MVT::f64, Tmp0, Tmp1);
10727       } else {
10728         ResNode = CurDAG->getTargetNode(X86::MULSDrr, MVT::f64, Tmp0, Tmp1);
10729         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10730         Result = SDOperand(ResNode, 0);
10731       }
10732       return;
10733     }
10734   }
10735 
10736   // Pattern: (fmul:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10737   // Emits: (MULPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
10738   // Pattern complexity = 2  cost = 1
10739   if ((Subtarget->hasSSE1())) {
10740     N0 = N.getOperand(0);
10741     N1 = N.getOperand(1);
10742     if (N.Val->getValueType(0) == MVT::v4f32) {
10743       Select(Tmp0, N0);
10744       Select(Tmp1, N1);
10745       if (N.Val->hasOneUse()) {
10746         Result = CurDAG->SelectNodeTo(N.Val, X86::MULPSrr, MVT::v4f32, Tmp0, Tmp1);
10747       } else {
10748         ResNode = CurDAG->getTargetNode(X86::MULPSrr, MVT::v4f32, Tmp0, Tmp1);
10749         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10750         Result = SDOperand(ResNode, 0);
10751       }
10752       return;
10753     }
10754   }
10755 
10756   // Pattern: (fmul:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10757   // Emits: (MULPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
10758   // Pattern complexity = 2  cost = 1
10759   if ((Subtarget->hasSSE2())) {
10760     N0 = N.getOperand(0);
10761     N1 = N.getOperand(1);
10762     if (N.Val->getValueType(0) == MVT::v2f64) {
10763       Select(Tmp0, N0);
10764       Select(Tmp1, N1);
10765       if (N.Val->hasOneUse()) {
10766         Result = CurDAG->SelectNodeTo(N.Val, X86::MULPDrr, MVT::v2f64, Tmp0, Tmp1);
10767       } else {
10768         ResNode = CurDAG->getTargetNode(X86::MULPDrr, MVT::v2f64, Tmp0, Tmp1);
10769         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10770         Result = SDOperand(ResNode, 0);
10771       }
10772       return;
10773     }
10774   }
10775   std::cerr << "Cannot yet select: ";
10776   N.Val->dump(CurDAG);
10777   std::cerr << '\n';
10778   abort();
10779 }
10780 
10781 void Select_fneg(SDOperand &Result, SDOperand N) {
10782   SDOperand N0(0, 0);
10783   SDOperand Tmp0(0, 0);
10784   SDNode *ResNode;
10785   if ((!Subtarget->hasSSE2())) {
10786     N0 = N.getOperand(0);
10787     if (N.Val->getValueType(0) == MVT::f64) {
10788       Select(Tmp0, N0);
10789       if (N.Val->hasOneUse()) {
10790         Result = CurDAG->SelectNodeTo(N.Val, X86::FpCHS, MVT::f64, Tmp0);
10791       } else {
10792         ResNode = CurDAG->getTargetNode(X86::FpCHS, MVT::f64, Tmp0);
10793         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10794         Result = SDOperand(ResNode, 0);
10795       }
10796       return;
10797     }
10798   }
10799   std::cerr << "Cannot yet select: ";
10800   N.Val->dump(CurDAG);
10801   std::cerr << '\n';
10802   abort();
10803 }
10804 
10805 void Select_fp_to_sint(SDOperand &Result, SDOperand N) {
10806   SDOperand Chain0(0, 0);
10807   SDOperand N0(0, 0);
10808   SDOperand N01(0, 0);
10809   SDOperand Tmp0(0, 0);
10810   SDOperand Tmp1(0, 0);
10811   SDOperand Tmp2(0, 0);
10812   SDOperand Tmp3(0, 0);
10813   SDNode *ResNode;
10814 
10815   // Pattern: (fp_to_sint:i32 (load:f32 addr:i32:$src))
10816   // Emits: (CVTTSS2SIrm:i32 addr:i32:$src)
10817   // Pattern complexity = 14  cost = 1
10818   if ((Subtarget->hasSSE1())) {
10819     N0 = N.getOperand(0);
10820     if (N0.getOpcode() == ISD::LOAD &&
10821         N0.hasOneUse() &&
10822         !CodeGenMap.count(N0.getValue(0))) {
10823       Chain0 = N0.getOperand(0);
10824       N01 = N0.getOperand(1);
10825       if (N.Val->getValueType(0) == MVT::i32 &&
10826           N0.Val->getValueType(0) == MVT::f32 &&
10827           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10828         Select(Tmp0, Tmp0);
10829         Select(Tmp1, Tmp1);
10830         Select(Tmp2, Tmp2);
10831         Select(Tmp3, Tmp3);
10832         Select(Chain0, Chain0);
10833         ResNode = CurDAG->getTargetNode(X86::CVTTSS2SIrm, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10834         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10835         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10836         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10837         Result = SDOperand(ResNode, N.ResNo);
10838         return;
10839       }
10840     }
10841   }
10842 
10843   // Pattern: (fp_to_sint:i32 (load:f64 addr:i32:$src))
10844   // Emits: (CVTTSD2SIrm:i32 addr:i32:$src)
10845   // Pattern complexity = 14  cost = 1
10846   if ((Subtarget->hasSSE2())) {
10847     N0 = N.getOperand(0);
10848     if (N0.getOpcode() == ISD::LOAD &&
10849         N0.hasOneUse() &&
10850         !CodeGenMap.count(N0.getValue(0))) {
10851       Chain0 = N0.getOperand(0);
10852       N01 = N0.getOperand(1);
10853       if (N.Val->getValueType(0) == MVT::i32 &&
10854           N0.Val->getValueType(0) == MVT::f64 &&
10855           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
10856         Select(Tmp0, Tmp0);
10857         Select(Tmp1, Tmp1);
10858         Select(Tmp2, Tmp2);
10859         Select(Tmp3, Tmp3);
10860         Select(Chain0, Chain0);
10861         ResNode = CurDAG->getTargetNode(X86::CVTTSD2SIrm, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
10862         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
10863         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
10864         AddHandleReplacement(N0.Val, 1, ResNode, 1);
10865         Result = SDOperand(ResNode, N.ResNo);
10866         return;
10867       }
10868     }
10869   }
10870 
10871   // Pattern: (fp_to_sint:i32 FR32:f32:$src)
10872   // Emits: (CVTTSS2SIrr:i32 FR32:f32:$src)
10873   // Pattern complexity = 2  cost = 1
10874   if ((Subtarget->hasSSE1())) {
10875     N0 = N.getOperand(0);
10876     if (N.Val->getValueType(0) == MVT::i32 &&
10877         N0.Val->getValueType(0) == MVT::f32) {
10878       Select(Tmp0, N0);
10879       if (N.Val->hasOneUse()) {
10880         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTTSS2SIrr, MVT::i32, Tmp0);
10881       } else {
10882         ResNode = CurDAG->getTargetNode(X86::CVTTSS2SIrr, MVT::i32, Tmp0);
10883         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10884         Result = SDOperand(ResNode, 0);
10885       }
10886       return;
10887     }
10888   }
10889 
10890   // Pattern: (fp_to_sint:i32 FR64:f64:$src)
10891   // Emits: (CVTTSD2SIrr:i32 FR64:f64:$src)
10892   // Pattern complexity = 2  cost = 1
10893   if ((Subtarget->hasSSE2())) {
10894     N0 = N.getOperand(0);
10895     if (N.Val->getValueType(0) == MVT::i32 &&
10896         N0.Val->getValueType(0) == MVT::f64) {
10897       Select(Tmp0, N0);
10898       if (N.Val->hasOneUse()) {
10899         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTTSD2SIrr, MVT::i32, Tmp0);
10900       } else {
10901         ResNode = CurDAG->getTargetNode(X86::CVTTSD2SIrr, MVT::i32, Tmp0);
10902         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10903         Result = SDOperand(ResNode, 0);
10904       }
10905       return;
10906     }
10907   }
10908   std::cerr << "Cannot yet select: ";
10909   N.Val->dump(CurDAG);
10910   std::cerr << '\n';
10911   abort();
10912 }
10913 
10914 void Select_fpimm(SDOperand &Result, SDOperand N) {
10915   SDOperand Tmp0(0, 0);
10916   SDNode *ResNode;
10917   if ((!Subtarget->hasSSE2())) {
10918 
10919     // Pattern: (fpimm:f64)<<P:Predicate_fp64imm0>>
10920     // Emits: (FpLD0:f64)
10921     // Pattern complexity = 3  cost = 1
10922     if (Predicate_fp64imm0(N.Val) &&
10923         N.Val->getValueType(0) == MVT::f64) {
10924       if (N.Val->hasOneUse()) {
10925         Result = CurDAG->SelectNodeTo(N.Val, X86::FpLD0, MVT::f64);
10926       } else {
10927         ResNode = CurDAG->getTargetNode(X86::FpLD0, MVT::f64);
10928         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10929         Result = SDOperand(ResNode, 0);
10930       }
10931       return;
10932     }
10933 
10934     // Pattern: (fpimm:f64)<<P:Predicate_fp64imm1>>
10935     // Emits: (FpLD1:f64)
10936     // Pattern complexity = 3  cost = 1
10937     if (Predicate_fp64imm1(N.Val) &&
10938         N.Val->getValueType(0) == MVT::f64) {
10939       if (N.Val->hasOneUse()) {
10940         Result = CurDAG->SelectNodeTo(N.Val, X86::FpLD1, MVT::f64);
10941       } else {
10942         ResNode = CurDAG->getTargetNode(X86::FpLD1, MVT::f64);
10943         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10944         Result = SDOperand(ResNode, 0);
10945       }
10946       return;
10947     }
10948   }
10949 
10950   // Pattern: (fpimm:f32)<<P:Predicate_fp32imm0>>
10951   // Emits: (FsFLD0SS:f32)
10952   // Pattern complexity = 3  cost = 1
10953   if ((Subtarget->hasSSE1()) &&
10954       Predicate_fp32imm0(N.Val) &&
10955       N.Val->getValueType(0) == MVT::f32) {
10956     if (N.Val->hasOneUse()) {
10957       Result = CurDAG->SelectNodeTo(N.Val, X86::FsFLD0SS, MVT::f32);
10958     } else {
10959       ResNode = CurDAG->getTargetNode(X86::FsFLD0SS, MVT::f32);
10960       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10961       Result = SDOperand(ResNode, 0);
10962     }
10963     return;
10964   }
10965 
10966   // Pattern: (fpimm:f64)<<P:Predicate_fp64imm0>>
10967   // Emits: (FsFLD0SD:f64)
10968   // Pattern complexity = 3  cost = 1
10969   if ((Subtarget->hasSSE2()) &&
10970       Predicate_fp64imm0(N.Val) &&
10971       N.Val->getValueType(0) == MVT::f64) {
10972     if (N.Val->hasOneUse()) {
10973       Result = CurDAG->SelectNodeTo(N.Val, X86::FsFLD0SD, MVT::f64);
10974     } else {
10975       ResNode = CurDAG->getTargetNode(X86::FsFLD0SD, MVT::f64);
10976       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10977       Result = SDOperand(ResNode, 0);
10978     }
10979     return;
10980   }
10981   if ((!Subtarget->hasSSE2())) {
10982 
10983     // Pattern: (fpimm:f64)<<P:Predicate_fp64immneg0>>
10984     // Emits: (FpCHS:f64 (FpLD0:f64))
10985     // Pattern complexity = 3  cost = 2
10986     if (Predicate_fp64immneg0(N.Val) &&
10987         N.Val->getValueType(0) == MVT::f64) {
10988       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::FpLD0, MVT::f64), 0);
10989       if (N.Val->hasOneUse()) {
10990         Result = CurDAG->SelectNodeTo(N.Val, X86::FpCHS, MVT::f64, Tmp0);
10991       } else {
10992         ResNode = CurDAG->getTargetNode(X86::FpCHS, MVT::f64, Tmp0);
10993         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
10994         Result = SDOperand(ResNode, 0);
10995       }
10996       return;
10997     }
10998 
10999     // Pattern: (fpimm:f64)<<P:Predicate_fp64immneg1>>
11000     // Emits: (FpCHS:f64 (FpLD1:f64))
11001     // Pattern complexity = 3  cost = 2
11002     if (Predicate_fp64immneg1(N.Val) &&
11003         N.Val->getValueType(0) == MVT::f64) {
11004       Tmp0 = SDOperand(CurDAG->getTargetNode(X86::FpLD1, MVT::f64), 0);
11005       if (N.Val->hasOneUse()) {
11006         Result = CurDAG->SelectNodeTo(N.Val, X86::FpCHS, MVT::f64, Tmp0);
11007       } else {
11008         ResNode = CurDAG->getTargetNode(X86::FpCHS, MVT::f64, Tmp0);
11009         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11010         Result = SDOperand(ResNode, 0);
11011       }
11012       return;
11013     }
11014   }
11015   std::cerr << "Cannot yet select: ";
11016   N.Val->dump(CurDAG);
11017   std::cerr << '\n';
11018   abort();
11019 }
11020 
11021 void Select_frameindex(SDOperand &Result, SDOperand N) {
11022   SDOperand Tmp0(0, 0);
11023   SDOperand Tmp1(0, 0);
11024   SDOperand Tmp2(0, 0);
11025   SDOperand Tmp3(0, 0);
11026   SDNode *ResNode;
11027   if (N.Val->getValueType(0) == MVT::i32 &&
11028       SelectLEAAddr(N, Tmp0, Tmp1, Tmp2, Tmp3)) {
11029     Select(Tmp0, Tmp0);
11030     Select(Tmp1, Tmp1);
11031     Select(Tmp2, Tmp2);
11032     Select(Tmp3, Tmp3);
11033     if (N.Val->hasOneUse()) {
11034       Result = CurDAG->SelectNodeTo(N.Val, X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
11035     } else {
11036       ResNode = CurDAG->getTargetNode(X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
11037       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11038       Result = SDOperand(ResNode, 0);
11039     }
11040     return;
11041   }
11042   std::cerr << "Cannot yet select: ";
11043   N.Val->dump(CurDAG);
11044   std::cerr << '\n';
11045   abort();
11046 }
11047 
11048 void Select_fround(SDOperand &Result, SDOperand N) {
11049   SDOperand Chain0(0, 0);
11050   SDOperand N0(0, 0);
11051   SDOperand N01(0, 0);
11052   SDOperand Tmp0(0, 0);
11053   SDOperand Tmp1(0, 0);
11054   SDOperand Tmp2(0, 0);
11055   SDOperand Tmp3(0, 0);
11056   SDNode *ResNode;
11057   if ((Subtarget->hasSSE2())) {
11058     N0 = N.getOperand(0);
11059 
11060     // Pattern: (fround:f32 (load:f64 addr:i32:$src))
11061     // Emits: (CVTSD2SSrm:f32 addr:i32:$src)
11062     // Pattern complexity = 14  cost = 1
11063     if (N0.getOpcode() == ISD::LOAD &&
11064         N0.hasOneUse() &&
11065         !CodeGenMap.count(N0.getValue(0))) {
11066       Chain0 = N0.getOperand(0);
11067       N01 = N0.getOperand(1);
11068       if (N.Val->getValueType(0) == MVT::f32 &&
11069           N0.Val->getValueType(0) == MVT::f64 &&
11070           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11071         Select(Tmp0, Tmp0);
11072         Select(Tmp1, Tmp1);
11073         Select(Tmp2, Tmp2);
11074         Select(Tmp3, Tmp3);
11075         Select(Chain0, Chain0);
11076         ResNode = CurDAG->getTargetNode(X86::CVTSD2SSrm, MVT::f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11077         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11078         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11079         AddHandleReplacement(N0.Val, 1, ResNode, 1);
11080         Result = SDOperand(ResNode, N.ResNo);
11081         return;
11082       }
11083     }
11084 
11085     // Pattern: (fround:f32 FR64:f64:$src)
11086     // Emits: (CVTSD2SSrr:f32 FR64:f64:$src)
11087     // Pattern complexity = 2  cost = 1
11088     if (N.Val->getValueType(0) == MVT::f32 &&
11089         N0.Val->getValueType(0) == MVT::f64) {
11090       Select(Tmp0, N0);
11091       if (N.Val->hasOneUse()) {
11092         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSD2SSrr, MVT::f32, Tmp0);
11093       } else {
11094         ResNode = CurDAG->getTargetNode(X86::CVTSD2SSrr, MVT::f32, Tmp0);
11095         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11096         Result = SDOperand(ResNode, 0);
11097       }
11098       return;
11099     }
11100   }
11101   std::cerr << "Cannot yet select: ";
11102   N.Val->dump(CurDAG);
11103   std::cerr << '\n';
11104   abort();
11105 }
11106 
11107 void Select_fsin(SDOperand &Result, SDOperand N) {
11108   SDOperand N0(0, 0);
11109   SDOperand Tmp0(0, 0);
11110   SDNode *ResNode;
11111   if ((!Subtarget->hasSSE2())) {
11112     N0 = N.getOperand(0);
11113     if (N.Val->getValueType(0) == MVT::f64) {
11114       Select(Tmp0, N0);
11115       if (N.Val->hasOneUse()) {
11116         Result = CurDAG->SelectNodeTo(N.Val, X86::FpSIN, MVT::f64, Tmp0);
11117       } else {
11118         ResNode = CurDAG->getTargetNode(X86::FpSIN, MVT::f64, Tmp0);
11119         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11120         Result = SDOperand(ResNode, 0);
11121       }
11122       return;
11123     }
11124   }
11125   std::cerr << "Cannot yet select: ";
11126   N.Val->dump(CurDAG);
11127   std::cerr << '\n';
11128   abort();
11129 }
11130 
11131 void Select_fsqrt(SDOperand &Result, SDOperand N) {
11132   SDOperand Chain0(0, 0);
11133   SDOperand N0(0, 0);
11134   SDOperand N01(0, 0);
11135   SDOperand Tmp0(0, 0);
11136   SDOperand Tmp1(0, 0);
11137   SDOperand Tmp2(0, 0);
11138   SDOperand Tmp3(0, 0);
11139   SDNode *ResNode;
11140 
11141   // Pattern: (fsqrt:f32 (load:f32 addr:i32:$src))
11142   // Emits: (SQRTSSm:f32 addr:i32:$src)
11143   // Pattern complexity = 14  cost = 1
11144   if ((Subtarget->hasSSE1())) {
11145     N0 = N.getOperand(0);
11146     if (N0.getOpcode() == ISD::LOAD &&
11147         N0.hasOneUse() &&
11148         !CodeGenMap.count(N0.getValue(0))) {
11149       Chain0 = N0.getOperand(0);
11150       N01 = N0.getOperand(1);
11151       if (N.Val->getValueType(0) == MVT::f32 &&
11152           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11153         Select(Tmp0, Tmp0);
11154         Select(Tmp1, Tmp1);
11155         Select(Tmp2, Tmp2);
11156         Select(Tmp3, Tmp3);
11157         Select(Chain0, Chain0);
11158         ResNode = CurDAG->getTargetNode(X86::SQRTSSm, MVT::f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11159         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11160         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11161         AddHandleReplacement(N0.Val, 1, ResNode, 1);
11162         Result = SDOperand(ResNode, N.ResNo);
11163         return;
11164       }
11165     }
11166   }
11167 
11168   // Pattern: (fsqrt:f64 (load:f64 addr:i32:$src))
11169   // Emits: (SQRTSDm:f64 addr:i32:$src)
11170   // Pattern complexity = 14  cost = 1
11171   if ((Subtarget->hasSSE2())) {
11172     N0 = N.getOperand(0);
11173     if (N0.getOpcode() == ISD::LOAD &&
11174         N0.hasOneUse() &&
11175         !CodeGenMap.count(N0.getValue(0))) {
11176       Chain0 = N0.getOperand(0);
11177       N01 = N0.getOperand(1);
11178       if (N.Val->getValueType(0) == MVT::f64 &&
11179           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11180         Select(Tmp0, Tmp0);
11181         Select(Tmp1, Tmp1);
11182         Select(Tmp2, Tmp2);
11183         Select(Tmp3, Tmp3);
11184         Select(Chain0, Chain0);
11185         ResNode = CurDAG->getTargetNode(X86::SQRTSDm, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11186         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11187         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11188         AddHandleReplacement(N0.Val, 1, ResNode, 1);
11189         Result = SDOperand(ResNode, N.ResNo);
11190         return;
11191       }
11192     }
11193   }
11194 
11195   // Pattern: (fsqrt:f64 RFP:f64:$src)
11196   // Emits: (FpSQRT:f64 RFP:f64:$src)
11197   // Pattern complexity = 2  cost = 1
11198   if ((!Subtarget->hasSSE2())) {
11199     N0 = N.getOperand(0);
11200     if (N.Val->getValueType(0) == MVT::f64) {
11201       Select(Tmp0, N0);
11202       if (N.Val->hasOneUse()) {
11203         Result = CurDAG->SelectNodeTo(N.Val, X86::FpSQRT, MVT::f64, Tmp0);
11204       } else {
11205         ResNode = CurDAG->getTargetNode(X86::FpSQRT, MVT::f64, Tmp0);
11206         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11207         Result = SDOperand(ResNode, 0);
11208       }
11209       return;
11210     }
11211   }
11212 
11213   // Pattern: (fsqrt:f32 FR32:f32:$src)
11214   // Emits: (SQRTSSr:f32 FR32:f32:$src)
11215   // Pattern complexity = 2  cost = 1
11216   if ((Subtarget->hasSSE1())) {
11217     N0 = N.getOperand(0);
11218     if (N.Val->getValueType(0) == MVT::f32) {
11219       Select(Tmp0, N0);
11220       if (N.Val->hasOneUse()) {
11221         Result = CurDAG->SelectNodeTo(N.Val, X86::SQRTSSr, MVT::f32, Tmp0);
11222       } else {
11223         ResNode = CurDAG->getTargetNode(X86::SQRTSSr, MVT::f32, Tmp0);
11224         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11225         Result = SDOperand(ResNode, 0);
11226       }
11227       return;
11228     }
11229   }
11230 
11231   // Pattern: (fsqrt:f64 FR64:f64:$src)
11232   // Emits: (SQRTSDr:f64 FR64:f64:$src)
11233   // Pattern complexity = 2  cost = 1
11234   if ((Subtarget->hasSSE2())) {
11235     N0 = N.getOperand(0);
11236     if (N.Val->getValueType(0) == MVT::f64) {
11237       Select(Tmp0, N0);
11238       if (N.Val->hasOneUse()) {
11239         Result = CurDAG->SelectNodeTo(N.Val, X86::SQRTSDr, MVT::f64, Tmp0);
11240       } else {
11241         ResNode = CurDAG->getTargetNode(X86::SQRTSDr, MVT::f64, Tmp0);
11242         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11243         Result = SDOperand(ResNode, 0);
11244       }
11245       return;
11246     }
11247   }
11248   std::cerr << "Cannot yet select: ";
11249   N.Val->dump(CurDAG);
11250   std::cerr << '\n';
11251   abort();
11252 }
11253 
11254 void Select_fsub(SDOperand &Result, SDOperand N) {
11255   SDOperand Chain0(0, 0);
11256   SDOperand Chain1(0, 0);
11257   SDOperand N0(0, 0);
11258   SDOperand N01(0, 0);
11259   SDOperand N02(0, 0);
11260   SDOperand N03(0, 0);
11261   SDOperand N1(0, 0);
11262   SDOperand N11(0, 0);
11263   SDOperand N12(0, 0);
11264   SDOperand N13(0, 0);
11265   SDOperand Tmp0(0, 0);
11266   SDOperand Tmp1(0, 0);
11267   SDOperand Tmp2(0, 0);
11268   SDOperand Tmp3(0, 0);
11269   SDOperand Tmp4(0, 0);
11270   SDNode *ResNode;
11271   if ((!Subtarget->hasSSE2())) {
11272     N0 = N.getOperand(0);
11273     {
11274       N1 = N.getOperand(1);
11275 
11276       // Pattern: (fsub:f64 RFP:f64:$src1, (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other))
11277       // Emits: (FpSUB32m:f64 RFP:f64:$src1, addr:i32:$src2)
11278       // Pattern complexity = 14  cost = 1
11279       if (N1.getOpcode() == ISD::EXTLOAD &&
11280           N1.hasOneUse() &&
11281           !CodeGenMap.count(N1.getValue(0)) &&
11282           !isNonImmUse(N.Val, N1.Val)) {
11283         Chain1 = N1.getOperand(0);
11284         N11 = N1.getOperand(1);
11285         N12 = N1.getOperand(2);
11286         N13 = N1.getOperand(3);
11287         if (cast<VTSDNode>(N13)->getVT() == MVT::f32 &&
11288             N.Val->getValueType(0) == MVT::f64 &&
11289             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11290           Select(Tmp0, Tmp0);
11291           Select(Tmp1, Tmp1);
11292           Select(Tmp2, Tmp2);
11293           Select(Tmp3, Tmp3);
11294           Select(Tmp4, N0);
11295           Select(Chain1, Chain1);
11296           ResNode = CurDAG->getTargetNode(X86::FpSUB32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11297           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11298           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11299           AddHandleReplacement(N1.Val, 1, ResNode, 1);
11300           Result = SDOperand(ResNode, N.ResNo);
11301           return;
11302         }
11303       }
11304 
11305       // Pattern: (fsub:f64 RFP:f64:$src1, (load:f64 addr:i32:$src2))
11306       // Emits: (FpSUB64m:f64 RFP:f64:$src1, addr:i32:$src2)
11307       // Pattern complexity = 14  cost = 1
11308       if (N1.getOpcode() == ISD::LOAD &&
11309           N1.hasOneUse() &&
11310           !CodeGenMap.count(N1.getValue(0)) &&
11311           !isNonImmUse(N.Val, N1.Val)) {
11312         Chain1 = N1.getOperand(0);
11313         N11 = N1.getOperand(1);
11314         if (N.Val->getValueType(0) == MVT::f64 &&
11315             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11316           Select(Tmp0, Tmp0);
11317           Select(Tmp1, Tmp1);
11318           Select(Tmp2, Tmp2);
11319           Select(Tmp3, Tmp3);
11320           Select(Tmp4, N0);
11321           Select(Chain1, Chain1);
11322           ResNode = CurDAG->getTargetNode(X86::FpSUB64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11323           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11324           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11325           AddHandleReplacement(N1.Val, 1, ResNode, 1);
11326           Result = SDOperand(ResNode, N.ResNo);
11327           return;
11328         }
11329       }
11330     }
11331 
11332     // Pattern: (fsub:f64 (extld:f64 addr:i32:$src2, srcvalue:Other:$dummy, f32:Other), RFP:f64:$src1)
11333     // Emits: (FpSUBR32m:f64 RFP:f64:$src1, addr:i32:$src2)
11334     // Pattern complexity = 14  cost = 1
11335     if (N0.getOpcode() == ISD::EXTLOAD &&
11336         N0.hasOneUse() &&
11337         !CodeGenMap.count(N0.getValue(0)) &&
11338         !isNonImmUse(N.Val, N0.Val)) {
11339       Chain0 = N0.getOperand(0);
11340       N01 = N0.getOperand(1);
11341       N02 = N0.getOperand(2);
11342       N03 = N0.getOperand(3);
11343       if (cast<VTSDNode>(N03)->getVT() == MVT::f32) {
11344         N1 = N.getOperand(1);
11345         if (N.Val->getValueType(0) == MVT::f64 &&
11346             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11347           Select(Tmp0, Tmp0);
11348           Select(Tmp1, Tmp1);
11349           Select(Tmp2, Tmp2);
11350           Select(Tmp3, Tmp3);
11351           Select(Tmp4, N1);
11352           Select(Chain0, Chain0);
11353           ResNode = CurDAG->getTargetNode(X86::FpSUBR32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11354           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11355           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11356           AddHandleReplacement(N0.Val, 1, ResNode, 1);
11357           Result = SDOperand(ResNode, N.ResNo);
11358           return;
11359         }
11360       }
11361     }
11362 
11363     // Pattern: (fsub:f64 (load:f64 addr:i32:$src2), RFP:f64:$src1)
11364     // Emits: (FpSUBR64m:f64 RFP:f64:$src1, addr:i32:$src2)
11365     // Pattern complexity = 14  cost = 1
11366     if (N0.getOpcode() == ISD::LOAD &&
11367         N0.hasOneUse() &&
11368         !CodeGenMap.count(N0.getValue(0)) &&
11369         !isNonImmUse(N.Val, N0.Val)) {
11370       Chain0 = N0.getOperand(0);
11371       N01 = N0.getOperand(1);
11372       N1 = N.getOperand(1);
11373       if (N.Val->getValueType(0) == MVT::f64 &&
11374           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11375         Select(Tmp0, Tmp0);
11376         Select(Tmp1, Tmp1);
11377         Select(Tmp2, Tmp2);
11378         Select(Tmp3, Tmp3);
11379         Select(Tmp4, N1);
11380         Select(Chain0, Chain0);
11381         ResNode = CurDAG->getTargetNode(X86::FpSUBR64m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11382         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11383         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11384         AddHandleReplacement(N0.Val, 1, ResNode, 1);
11385         Result = SDOperand(ResNode, N.ResNo);
11386         return;
11387       }
11388     }
11389     {
11390       N1 = N.getOperand(1);
11391       if (N1.getOpcode() == X86ISD::FILD &&
11392           N1.hasOneUse() &&
11393           !CodeGenMap.count(N1.getValue(0)) &&
11394           !isNonImmUse(N.Val, N1.Val)) {
11395         Chain1 = N1.getOperand(0);
11396         N11 = N1.getOperand(1);
11397         N12 = N1.getOperand(2);
11398 
11399         // Pattern: (fsub:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i16:Other))
11400         // Emits: (FpISUB16m:f64 RFP:f64:$src1, addr:i32:$src2)
11401         // Pattern complexity = 14  cost = 1
11402         if (cast<VTSDNode>(N12)->getVT() == MVT::i16 &&
11403             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11404           Select(Tmp0, Tmp0);
11405           Select(Tmp1, Tmp1);
11406           Select(Tmp2, Tmp2);
11407           Select(Tmp3, Tmp3);
11408           Select(Tmp4, N0);
11409           Select(Chain1, Chain1);
11410           ResNode = CurDAG->getTargetNode(X86::FpISUB16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11411           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11412           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11413           AddHandleReplacement(N1.Val, 1, ResNode, 1);
11414           Result = SDOperand(ResNode, N.ResNo);
11415           return;
11416         }
11417 
11418         // Pattern: (fsub:f64 RFP:f64:$src1, (X86fild:f64 addr:i32:$src2, i32:Other))
11419         // Emits: (FpISUB32m:f64 RFP:f64:$src1, addr:i32:$src2)
11420         // Pattern complexity = 14  cost = 1
11421         if (cast<VTSDNode>(N12)->getVT() == MVT::i32 &&
11422             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11423           Select(Tmp0, Tmp0);
11424           Select(Tmp1, Tmp1);
11425           Select(Tmp2, Tmp2);
11426           Select(Tmp3, Tmp3);
11427           Select(Tmp4, N0);
11428           Select(Chain1, Chain1);
11429           ResNode = CurDAG->getTargetNode(X86::FpISUB32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11430           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11431           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11432           AddHandleReplacement(N1.Val, 1, ResNode, 1);
11433           Result = SDOperand(ResNode, N.ResNo);
11434           return;
11435         }
11436       }
11437     }
11438     if (N0.getOpcode() == X86ISD::FILD &&
11439         N0.hasOneUse() &&
11440         !CodeGenMap.count(N0.getValue(0)) &&
11441         !isNonImmUse(N.Val, N0.Val)) {
11442       Chain0 = N0.getOperand(0);
11443       N01 = N0.getOperand(1);
11444       N02 = N0.getOperand(2);
11445 
11446       // Pattern: (fsub:f64 (X86fild:f64 addr:i32:$src2, i16:Other), RFP:f64:$src1)
11447       // Emits: (FpISUBR16m:f64 RFP:f64:$src1, addr:i32:$src2)
11448       // Pattern complexity = 14  cost = 1
11449       if (cast<VTSDNode>(N02)->getVT() == MVT::i16) {
11450         N1 = N.getOperand(1);
11451         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11452           Select(Tmp0, Tmp0);
11453           Select(Tmp1, Tmp1);
11454           Select(Tmp2, Tmp2);
11455           Select(Tmp3, Tmp3);
11456           Select(Tmp4, N1);
11457           Select(Chain0, Chain0);
11458           ResNode = CurDAG->getTargetNode(X86::FpISUBR16m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11459           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11460           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11461           AddHandleReplacement(N0.Val, 1, ResNode, 1);
11462           Result = SDOperand(ResNode, N.ResNo);
11463           return;
11464         }
11465       }
11466 
11467       // Pattern: (fsub:f64 (X86fild:f64 addr:i32:$src2, i32:Other), RFP:f64:$src1)
11468       // Emits: (FpISUBR32m:f64 RFP:f64:$src1, addr:i32:$src2)
11469       // Pattern complexity = 14  cost = 1
11470       if (cast<VTSDNode>(N02)->getVT() == MVT::i32) {
11471         N1 = N.getOperand(1);
11472         if (SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
11473           Select(Tmp0, Tmp0);
11474           Select(Tmp1, Tmp1);
11475           Select(Tmp2, Tmp2);
11476           Select(Tmp3, Tmp3);
11477           Select(Tmp4, N1);
11478           Select(Chain0, Chain0);
11479           ResNode = CurDAG->getTargetNode(X86::FpISUBR32m, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
11480           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11481           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
11482           AddHandleReplacement(N0.Val, 1, ResNode, 1);
11483           Result = SDOperand(ResNode, N.ResNo);
11484           return;
11485         }
11486       }
11487     }
11488   }
11489 
11490   // Pattern: (fsub:f32 FR32:f32:$src1, (load:f32 addr:i32:$src2))
11491   // Emits: (SUBSSrm:f32 FR32:f32:$src1, addr:i32:$src2)
11492   // Pattern complexity = 14  cost = 1
11493   if ((Subtarget->hasSSE1())) {
11494     N0 = N.getOperand(0);
11495     N1 = N.getOperand(1);
11496     if (N1.getOpcode() == ISD::LOAD &&
11497         N1.hasOneUse() &&
11498         !CodeGenMap.count(N1.getValue(0)) &&
11499         !isNonImmUse(N.Val, N1.Val)) {
11500       Chain1 = N1.getOperand(0);
11501       N11 = N1.getOperand(1);
11502       if (N.Val->getValueType(0) == MVT::f32 &&
11503           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11504         Select(Tmp0, Tmp0);
11505         Select(Tmp1, Tmp1);
11506         Select(Tmp2, Tmp2);
11507         Select(Tmp3, Tmp3);
11508         Select(Tmp4, N0);
11509         Select(Chain1, Chain1);
11510         ResNode = CurDAG->getTargetNode(X86::SUBSSrm, MVT::f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11511         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11512         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11513         AddHandleReplacement(N1.Val, 1, ResNode, 1);
11514         Result = SDOperand(ResNode, N.ResNo);
11515         return;
11516       }
11517     }
11518   }
11519 
11520   // Pattern: (fsub:f64 FR64:f64:$src1, (load:f64 addr:i32:$src2))
11521   // Emits: (SUBSDrm:f64 FR64:f64:$src1, addr:i32:$src2)
11522   // Pattern complexity = 14  cost = 1
11523   if ((Subtarget->hasSSE2())) {
11524     N0 = N.getOperand(0);
11525     N1 = N.getOperand(1);
11526     if (N1.getOpcode() == ISD::LOAD &&
11527         N1.hasOneUse() &&
11528         !CodeGenMap.count(N1.getValue(0)) &&
11529         !isNonImmUse(N.Val, N1.Val)) {
11530       Chain1 = N1.getOperand(0);
11531       N11 = N1.getOperand(1);
11532       if (N.Val->getValueType(0) == MVT::f64 &&
11533           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11534         Select(Tmp0, Tmp0);
11535         Select(Tmp1, Tmp1);
11536         Select(Tmp2, Tmp2);
11537         Select(Tmp3, Tmp3);
11538         Select(Tmp4, N0);
11539         Select(Chain1, Chain1);
11540         ResNode = CurDAG->getTargetNode(X86::SUBSDrm, MVT::f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11541         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11542         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11543         AddHandleReplacement(N1.Val, 1, ResNode, 1);
11544         Result = SDOperand(ResNode, N.ResNo);
11545         return;
11546       }
11547     }
11548   }
11549 
11550   // Pattern: (fsub:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
11551   // Emits: (SUBPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
11552   // Pattern complexity = 14  cost = 1
11553   if ((Subtarget->hasSSE1())) {
11554     N0 = N.getOperand(0);
11555     N1 = N.getOperand(1);
11556     if (N1.getOpcode() == ISD::LOAD &&
11557         N1.hasOneUse() &&
11558         !CodeGenMap.count(N1.getValue(0)) &&
11559         !isNonImmUse(N.Val, N1.Val)) {
11560       Chain1 = N1.getOperand(0);
11561       N11 = N1.getOperand(1);
11562       if (N.Val->getValueType(0) == MVT::v4f32 &&
11563           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11564         Select(Tmp0, Tmp0);
11565         Select(Tmp1, Tmp1);
11566         Select(Tmp2, Tmp2);
11567         Select(Tmp3, Tmp3);
11568         Select(Tmp4, N0);
11569         Select(Chain1, Chain1);
11570         ResNode = CurDAG->getTargetNode(X86::SUBPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11571         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11572         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11573         AddHandleReplacement(N1.Val, 1, ResNode, 1);
11574         Result = SDOperand(ResNode, N.ResNo);
11575         return;
11576       }
11577     }
11578   }
11579 
11580   // Pattern: (fsub:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
11581   // Emits: (SUBPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
11582   // Pattern complexity = 14  cost = 1
11583   if ((Subtarget->hasSSE2())) {
11584     N0 = N.getOperand(0);
11585     N1 = N.getOperand(1);
11586     if (N1.getOpcode() == ISD::LOAD &&
11587         N1.hasOneUse() &&
11588         !CodeGenMap.count(N1.getValue(0)) &&
11589         !isNonImmUse(N.Val, N1.Val)) {
11590       Chain1 = N1.getOperand(0);
11591       N11 = N1.getOperand(1);
11592       if (N.Val->getValueType(0) == MVT::v2f64 &&
11593           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
11594         Select(Tmp0, Tmp0);
11595         Select(Tmp1, Tmp1);
11596         Select(Tmp2, Tmp2);
11597         Select(Tmp3, Tmp3);
11598         Select(Tmp4, N0);
11599         Select(Chain1, Chain1);
11600         ResNode = CurDAG->getTargetNode(X86::SUBPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
11601         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11602         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
11603         AddHandleReplacement(N1.Val, 1, ResNode, 1);
11604         Result = SDOperand(ResNode, N.ResNo);
11605         return;
11606       }
11607     }
11608   }
11609 
11610   // Pattern: (fsub:f64 RFP:f64:$src1, RFP:f64:$src2)
11611   // Emits: (FpSUB:f64 RFP:f64:$src1, RFP:f64:$src2)
11612   // Pattern complexity = 2  cost = 1
11613   if ((!Subtarget->hasSSE2())) {
11614     N0 = N.getOperand(0);
11615     N1 = N.getOperand(1);
11616     if (N.Val->getValueType(0) == MVT::f64) {
11617       Select(Tmp0, N0);
11618       Select(Tmp1, N1);
11619       if (N.Val->hasOneUse()) {
11620         Result = CurDAG->SelectNodeTo(N.Val, X86::FpSUB, MVT::f64, Tmp0, Tmp1);
11621       } else {
11622         ResNode = CurDAG->getTargetNode(X86::FpSUB, MVT::f64, Tmp0, Tmp1);
11623         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11624         Result = SDOperand(ResNode, 0);
11625       }
11626       return;
11627     }
11628   }
11629 
11630   // Pattern: (fsub:f32 FR32:f32:$src1, FR32:f32:$src2)
11631   // Emits: (SUBSSrr:f32 FR32:f32:$src1, FR32:f32:$src2)
11632   // Pattern complexity = 2  cost = 1
11633   if ((Subtarget->hasSSE1())) {
11634     N0 = N.getOperand(0);
11635     N1 = N.getOperand(1);
11636     if (N.Val->getValueType(0) == MVT::f32) {
11637       Select(Tmp0, N0);
11638       Select(Tmp1, N1);
11639       if (N.Val->hasOneUse()) {
11640         Result = CurDAG->SelectNodeTo(N.Val, X86::SUBSSrr, MVT::f32, Tmp0, Tmp1);
11641       } else {
11642         ResNode = CurDAG->getTargetNode(X86::SUBSSrr, MVT::f32, Tmp0, Tmp1);
11643         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11644         Result = SDOperand(ResNode, 0);
11645       }
11646       return;
11647     }
11648   }
11649 
11650   // Pattern: (fsub:f64 FR64:f64:$src1, FR64:f64:$src2)
11651   // Emits: (SUBSDrr:f64 FR64:f64:$src1, FR64:f64:$src2)
11652   // Pattern complexity = 2  cost = 1
11653   if ((Subtarget->hasSSE2())) {
11654     N0 = N.getOperand(0);
11655     N1 = N.getOperand(1);
11656     if (N.Val->getValueType(0) == MVT::f64) {
11657       Select(Tmp0, N0);
11658       Select(Tmp1, N1);
11659       if (N.Val->hasOneUse()) {
11660         Result = CurDAG->SelectNodeTo(N.Val, X86::SUBSDrr, MVT::f64, Tmp0, Tmp1);
11661       } else {
11662         ResNode = CurDAG->getTargetNode(X86::SUBSDrr, MVT::f64, Tmp0, Tmp1);
11663         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11664         Result = SDOperand(ResNode, 0);
11665       }
11666       return;
11667     }
11668   }
11669 
11670   // Pattern: (fsub:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
11671   // Emits: (SUBPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
11672   // Pattern complexity = 2  cost = 1
11673   if ((Subtarget->hasSSE1())) {
11674     N0 = N.getOperand(0);
11675     N1 = N.getOperand(1);
11676     if (N.Val->getValueType(0) == MVT::v4f32) {
11677       Select(Tmp0, N0);
11678       Select(Tmp1, N1);
11679       if (N.Val->hasOneUse()) {
11680         Result = CurDAG->SelectNodeTo(N.Val, X86::SUBPSrr, MVT::v4f32, Tmp0, Tmp1);
11681       } else {
11682         ResNode = CurDAG->getTargetNode(X86::SUBPSrr, MVT::v4f32, Tmp0, Tmp1);
11683         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11684         Result = SDOperand(ResNode, 0);
11685       }
11686       return;
11687     }
11688   }
11689 
11690   // Pattern: (fsub:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
11691   // Emits: (SUBPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
11692   // Pattern complexity = 2  cost = 1
11693   if ((Subtarget->hasSSE2())) {
11694     N0 = N.getOperand(0);
11695     N1 = N.getOperand(1);
11696     if (N.Val->getValueType(0) == MVT::v2f64) {
11697       Select(Tmp0, N0);
11698       Select(Tmp1, N1);
11699       if (N.Val->hasOneUse()) {
11700         Result = CurDAG->SelectNodeTo(N.Val, X86::SUBPDrr, MVT::v2f64, Tmp0, Tmp1);
11701       } else {
11702         ResNode = CurDAG->getTargetNode(X86::SUBPDrr, MVT::v2f64, Tmp0, Tmp1);
11703         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11704         Result = SDOperand(ResNode, 0);
11705       }
11706       return;
11707     }
11708   }
11709   std::cerr << "Cannot yet select: ";
11710   N.Val->dump(CurDAG);
11711   std::cerr << '\n';
11712   abort();
11713 }
11714 
11715 void Select_imm(SDOperand &Result, SDOperand N) {
11716   SDOperand Tmp0(0, 0);
11717   SDNode *ResNode;
11718   if (cast<ConstantSDNode>(N)->getSignExtended() == 0) {
11719 
11720     // Pattern: 0:i8
11721     // Emits: (MOV8r0:i8)
11722     // Pattern complexity = 3  cost = 1
11723     if (N.Val->getValueType(0) == MVT::i8) {
11724       if (N.Val->hasOneUse()) {
11725         Result = CurDAG->SelectNodeTo(N.Val, X86::MOV8r0, MVT::i8);
11726       } else {
11727         ResNode = CurDAG->getTargetNode(X86::MOV8r0, MVT::i8);
11728         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11729         Result = SDOperand(ResNode, 0);
11730       }
11731       return;
11732     }
11733 
11734     // Pattern: 0:i16
11735     // Emits: (MOV16r0:i16)
11736     // Pattern complexity = 3  cost = 1
11737     if (N.Val->getValueType(0) == MVT::i16) {
11738       if (N.Val->hasOneUse()) {
11739         Result = CurDAG->SelectNodeTo(N.Val, X86::MOV16r0, MVT::i16);
11740       } else {
11741         ResNode = CurDAG->getTargetNode(X86::MOV16r0, MVT::i16);
11742         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11743         Result = SDOperand(ResNode, 0);
11744       }
11745       return;
11746     }
11747 
11748     // Pattern: 0:i32
11749     // Emits: (MOV32r0:i32)
11750     // Pattern complexity = 3  cost = 1
11751     if (N.Val->getValueType(0) == MVT::i32) {
11752       if (N.Val->hasOneUse()) {
11753         Result = CurDAG->SelectNodeTo(N.Val, X86::MOV32r0, MVT::i32);
11754       } else {
11755         ResNode = CurDAG->getTargetNode(X86::MOV32r0, MVT::i32);
11756         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11757         Result = SDOperand(ResNode, 0);
11758       }
11759       return;
11760     }
11761   }
11762 
11763   // Pattern: (imm:i8):$src
11764   // Emits: (MOV8ri:i8 (imm:i8):$src)
11765   // Pattern complexity = 2  cost = 1
11766   if (N.Val->getValueType(0) == MVT::i8) {
11767     unsigned char Tmp0C = (unsigned char)cast<ConstantSDNode>(N)->getValue();
11768     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i8);
11769     if (N.Val->hasOneUse()) {
11770       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV8ri, MVT::i8, Tmp0);
11771     } else {
11772       ResNode = CurDAG->getTargetNode(X86::MOV8ri, MVT::i8, Tmp0);
11773       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11774       Result = SDOperand(ResNode, 0);
11775     }
11776     return;
11777   }
11778 
11779   // Pattern: (imm:i16):$src
11780   // Emits: (MOV16ri:i16 (imm:i16):$src)
11781   // Pattern complexity = 2  cost = 1
11782   if (N.Val->getValueType(0) == MVT::i16) {
11783     unsigned short Tmp0C = (unsigned short)cast<ConstantSDNode>(N)->getValue();
11784     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i16);
11785     if (N.Val->hasOneUse()) {
11786       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV16ri, MVT::i16, Tmp0);
11787     } else {
11788       ResNode = CurDAG->getTargetNode(X86::MOV16ri, MVT::i16, Tmp0);
11789       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11790       Result = SDOperand(ResNode, 0);
11791     }
11792     return;
11793   }
11794 
11795   // Pattern: (imm:i32):$src
11796   // Emits: (MOV32ri:i32 (imm:i32):$src)
11797   // Pattern complexity = 2  cost = 1
11798   if (N.Val->getValueType(0) == MVT::i32) {
11799     unsigned Tmp0C = (unsigned)cast<ConstantSDNode>(N)->getValue();
11800     Tmp0 = CurDAG->getTargetConstant(Tmp0C, MVT::i32);
11801     if (N.Val->hasOneUse()) {
11802       Result = CurDAG->SelectNodeTo(N.Val, X86::MOV32ri, MVT::i32, Tmp0);
11803     } else {
11804       ResNode = CurDAG->getTargetNode(X86::MOV32ri, MVT::i32, Tmp0);
11805       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
11806       Result = SDOperand(ResNode, 0);
11807     }
11808     return;
11809   }
11810   std::cerr << "Cannot yet select: ";
11811   N.Val->dump(CurDAG);
11812   std::cerr << '\n';
11813   abort();
11814 }
11815 
11816 void Select_intrinsic_void(SDOperand &Result, SDOperand N) {
11817   SDOperand Chain(0, 0);
11818   SDOperand InFlag(0, 0);
11819   SDOperand N1(0, 0);
11820   SDOperand N2(0, 0);
11821   SDOperand N3(0, 0);
11822   SDOperand N4(0, 0);
11823   SDOperand Tmp1(0, 0);
11824   SDOperand Tmp2(0, 0);
11825   SDOperand Tmp3(0, 0);
11826   SDOperand Tmp4(0, 0);
11827   SDOperand Tmp5(0, 0);
11828   SDNode *ResNode;
11829   if ((Subtarget->hasSSE2())) {
11830     Chain = N.getOperand(0);
11831     N1 = N.getOperand(1);
11832     if (isa<ConstantSDNode>(N1)) {
11833       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11834 
11835       // Pattern: (intrinsic_void:void 306:i32, addr:i32:$dst, VR128:v4f32:$src)
11836       // Emits: (MOVUPSmr:void addr:i32:$dst, VR128:v4f32:$src)
11837       // Pattern complexity = 15  cost = 1
11838       if (CN0 == 306) {
11839         N2 = N.getOperand(2);
11840         N3 = N.getOperand(3);
11841         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11842           Select(Tmp1, Tmp1);
11843           Select(Tmp2, Tmp2);
11844           Select(Tmp3, Tmp3);
11845           Select(Tmp4, Tmp4);
11846           Select(Tmp5, N3);
11847           Select(Chain, Chain);
11848           ResNode = CurDAG->getTargetNode(X86::MOVUPSmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11849           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11850           Result = SDOperand(ResNode, N.ResNo);
11851           return;
11852         }
11853       }
11854 
11855       // Pattern: (intrinsic_void:void 264:i32, addr:i32:$dst, VR128:v2f64:$src)
11856       // Emits: (MOVUPDmr:void addr:i32:$dst, VR128:v2f64:$src)
11857       // Pattern complexity = 15  cost = 1
11858       if (CN0 == 264) {
11859         N2 = N.getOperand(2);
11860         N3 = N.getOperand(3);
11861         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11862           Select(Tmp1, Tmp1);
11863           Select(Tmp2, Tmp2);
11864           Select(Tmp3, Tmp3);
11865           Select(Tmp4, Tmp4);
11866           Select(Tmp5, N3);
11867           Select(Chain, Chain);
11868           ResNode = CurDAG->getTargetNode(X86::MOVUPDmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11869           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11870           Result = SDOperand(ResNode, N.ResNo);
11871           return;
11872         }
11873       }
11874 
11875       // Pattern: (intrinsic_void:void 263:i32, addr:i32:$dst, VR128:v16i8:$src)
11876       // Emits: (MOVDQUmr:void addr:i32:$dst, VR128:v16i8:$src)
11877       // Pattern complexity = 15  cost = 1
11878       if (CN0 == 263) {
11879         N2 = N.getOperand(2);
11880         N3 = N.getOperand(3);
11881         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11882           Select(Tmp1, Tmp1);
11883           Select(Tmp2, Tmp2);
11884           Select(Tmp3, Tmp3);
11885           Select(Tmp4, Tmp4);
11886           Select(Tmp5, N3);
11887           Select(Chain, Chain);
11888           ResNode = CurDAG->getTargetNode(X86::MOVDQUmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11889           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11890           Result = SDOperand(ResNode, N.ResNo);
11891           return;
11892         }
11893       }
11894     }
11895   }
11896 
11897   // Pattern: (intrinsic_void:void 296:i32, addr:i32:$dst, VR128:v4f32:$src)
11898   // Emits: (MOVNTPSmr:void addr:i32:$dst, VR128:v4f32:$src)
11899   // Pattern complexity = 15  cost = 1
11900   if ((Subtarget->hasSSE1())) {
11901     Chain = N.getOperand(0);
11902     N1 = N.getOperand(1);
11903     if (isa<ConstantSDNode>(N1)) {
11904       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11905       if (CN0 == 296) {
11906         N2 = N.getOperand(2);
11907         N3 = N.getOperand(3);
11908         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11909           Select(Tmp1, Tmp1);
11910           Select(Tmp2, Tmp2);
11911           Select(Tmp3, Tmp3);
11912           Select(Tmp4, Tmp4);
11913           Select(Tmp5, N3);
11914           Select(Chain, Chain);
11915           ResNode = CurDAG->getTargetNode(X86::MOVNTPSmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11916           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11917           Result = SDOperand(ResNode, N.ResNo);
11918           return;
11919         }
11920       }
11921     }
11922   }
11923   if ((Subtarget->hasSSE2())) {
11924     Chain = N.getOperand(0);
11925     N1 = N.getOperand(1);
11926     if (isa<ConstantSDNode>(N1)) {
11927       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11928 
11929       // Pattern: (intrinsic_void:void 229:i32, addr:i32:$dst, VR128:v2f64:$src)
11930       // Emits: (MOVNTPDmr:void addr:i32:$dst, VR128:v2f64:$src)
11931       // Pattern complexity = 15  cost = 1
11932       if (CN0 == 229) {
11933         N2 = N.getOperand(2);
11934         N3 = N.getOperand(3);
11935         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11936           Select(Tmp1, Tmp1);
11937           Select(Tmp2, Tmp2);
11938           Select(Tmp3, Tmp3);
11939           Select(Tmp4, Tmp4);
11940           Select(Tmp5, N3);
11941           Select(Chain, Chain);
11942           ResNode = CurDAG->getTargetNode(X86::MOVNTPDmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11943           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11944           Result = SDOperand(ResNode, N.ResNo);
11945           return;
11946         }
11947       }
11948 
11949       // Pattern: (intrinsic_void:void 227:i32, addr:i32:$dst, VR128:v2i64:$src)
11950       // Emits: (MOVNTDQmr:void addr:i32:$dst, VR128:v2i64:$src)
11951       // Pattern complexity = 15  cost = 1
11952       if (CN0 == 227) {
11953         N2 = N.getOperand(2);
11954         N3 = N.getOperand(3);
11955         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11956           Select(Tmp1, Tmp1);
11957           Select(Tmp2, Tmp2);
11958           Select(Tmp3, Tmp3);
11959           Select(Tmp4, Tmp4);
11960           Select(Tmp5, N3);
11961           Select(Chain, Chain);
11962           ResNode = CurDAG->getTargetNode(X86::MOVNTDQmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11963           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11964           Result = SDOperand(ResNode, N.ResNo);
11965           return;
11966         }
11967       }
11968 
11969       // Pattern: (intrinsic_void:void 228:i32, addr:i32:$dst, R32:i32:$src)
11970       // Emits: (MOVNTImr:void addr:i32:$dst, R32:i32:$src)
11971       // Pattern complexity = 15  cost = 1
11972       if (CN0 == 228) {
11973         N2 = N.getOperand(2);
11974         N3 = N.getOperand(3);
11975         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
11976           Select(Tmp1, Tmp1);
11977           Select(Tmp2, Tmp2);
11978           Select(Tmp3, Tmp3);
11979           Select(Tmp4, Tmp4);
11980           Select(Tmp5, N3);
11981           Select(Chain, Chain);
11982           ResNode = CurDAG->getTargetNode(X86::MOVNTImr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
11983           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
11984           Result = SDOperand(ResNode, N.ResNo);
11985           return;
11986         }
11987       }
11988     }
11989   }
11990   if ((Subtarget->hasSSE1())) {
11991     Chain = N.getOperand(0);
11992     N1 = N.getOperand(1);
11993     if (isa<ConstantSDNode>(N1)) {
11994       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
11995 
11996       // Pattern: (intrinsic_void:void 289:i32, addr:i32:$src)
11997       // Emits: (LDMXCSR:void addr:i32:$src)
11998       // Pattern complexity = 15  cost = 1
11999       if (CN0 == 289) {
12000         N2 = N.getOperand(2);
12001         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12002           Select(Tmp1, Tmp1);
12003           Select(Tmp2, Tmp2);
12004           Select(Tmp3, Tmp3);
12005           Select(Tmp4, Tmp4);
12006           Select(Chain, Chain);
12007           ResNode = CurDAG->getTargetNode(X86::LDMXCSR, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
12008           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12009           Result = SDOperand(ResNode, N.ResNo);
12010           return;
12011         }
12012       }
12013 
12014       // Pattern: (intrinsic_void:void 305:i32, addr:i32:$dst)
12015       // Emits: (STMXCSR:void addr:i32:$dst)
12016       // Pattern complexity = 15  cost = 1
12017       if (CN0 == 305) {
12018         N2 = N.getOperand(2);
12019         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12020           Select(Tmp1, Tmp1);
12021           Select(Tmp2, Tmp2);
12022           Select(Tmp3, Tmp3);
12023           Select(Tmp4, Tmp4);
12024           Select(Chain, Chain);
12025           ResNode = CurDAG->getTargetNode(X86::STMXCSR, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
12026           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12027           Result = SDOperand(ResNode, N.ResNo);
12028           return;
12029         }
12030       }
12031     }
12032   }
12033   if ((Subtarget->hasSSE2())) {
12034     Chain = N.getOperand(0);
12035     N1 = N.getOperand(1);
12036     if (isa<ConstantSDNode>(N1)) {
12037       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12038 
12039       // Pattern: (intrinsic_void:void 262:i32, addr:i32:$dst, VR128:v4i32:$src)
12040       // Emits: (MOVLQ128mr:void addr:i32:$dst, VR128:v4i32:$src)
12041       // Pattern complexity = 15  cost = 1
12042       if (CN0 == 262) {
12043         N2 = N.getOperand(2);
12044         N3 = N.getOperand(3);
12045         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12046           Select(Tmp1, Tmp1);
12047           Select(Tmp2, Tmp2);
12048           Select(Tmp3, Tmp3);
12049           Select(Tmp4, Tmp4);
12050           Select(Tmp5, N3);
12051           Select(Chain, Chain);
12052           ResNode = CurDAG->getTargetNode(X86::MOVLQ128mr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
12053           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12054           Result = SDOperand(ResNode, N.ResNo);
12055           return;
12056         }
12057       }
12058 
12059       // Pattern: (intrinsic_void:void 220:i32, VR128:v16i8:$src, VR128:v16i8:$mask, EDI:i32)
12060       // Emits: (MASKMOVDQU:void VR128:v16i8:$src, VR128:v16i8:$mask)
12061       // Pattern complexity = 5  cost = 1
12062       if (CN0 == 220) {
12063         N2 = N.getOperand(2);
12064         N3 = N.getOperand(3);
12065         N4 = N.getOperand(4);
12066         Select(Tmp1, N2);
12067         Select(Tmp2, N3);
12068         Select(Chain, Chain);
12069         Select(N4, N4);
12070         ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::EDI, MVT::i32), N4, InFlag).Val;
12071         Chain = SDOperand(ResNode, 0);
12072         InFlag = SDOperand(ResNode, 1);
12073         ResNode = CurDAG->getTargetNode(X86::MASKMOVDQU, MVT::Other, Tmp1, Tmp2, Chain, InFlag);
12074         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12075         Result = SDOperand(ResNode, N.ResNo);
12076         return;
12077       }
12078     }
12079   }
12080 
12081   // Pattern: (intrinsic_void:void 302:i32)
12082   // Emits: (SFENCE:void)
12083   // Pattern complexity = 5  cost = 1
12084   if ((Subtarget->hasSSE1())) {
12085     Chain = N.getOperand(0);
12086     N1 = N.getOperand(1);
12087     if (isa<ConstantSDNode>(N1)) {
12088       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12089       if (CN0 == 302) {
12090         Select(Chain, Chain);
12091         ResNode = CurDAG->getTargetNode(X86::SFENCE, MVT::Other, Chain);
12092         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12093         Result = SDOperand(ResNode, N.ResNo);
12094         return;
12095       }
12096     }
12097   }
12098   std::cerr << "Cannot yet select: ";
12099   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
12100   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
12101   std::cerr << '\n';
12102   abort();
12103 }
12104 
12105 void Select_intrinsic_w_chain(SDOperand &Result, SDOperand N) {
12106   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
12107     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
12108     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
12109     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
12110     Result = Dummy;
12111     return;
12112   }
12113   SDOperand Chain(0, 0);
12114   SDOperand N1(0, 0);
12115   SDOperand N2(0, 0);
12116   SDOperand Tmp1(0, 0);
12117   SDOperand Tmp2(0, 0);
12118   SDOperand Tmp3(0, 0);
12119   SDOperand Tmp4(0, 0);
12120   SDNode *ResNode;
12121   if ((Subtarget->hasSSE2())) {
12122     Chain = N.getOperand(0);
12123     N1 = N.getOperand(1);
12124     if (isa<ConstantSDNode>(N1)) {
12125       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
12126 
12127       // Pattern: (intrinsic_w_chain:v4f32 290:i32, addr:i32:$src)
12128       // Emits: (MOVUPSrm:v4f32 addr:i32:$src)
12129       // Pattern complexity = 15  cost = 1
12130       if (CN0 == 290) {
12131         N2 = N.getOperand(2);
12132         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12133           Select(Tmp1, Tmp1);
12134           Select(Tmp2, Tmp2);
12135           Select(Tmp3, Tmp3);
12136           Select(Tmp4, Tmp4);
12137           Select(Chain, Chain);
12138           ResNode = CurDAG->getTargetNode(X86::MOVUPSrm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
12139           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12140           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
12141           if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
12142           Result = SDOperand(ResNode, N.ResNo);
12143           return;
12144         }
12145       }
12146 
12147       // Pattern: (intrinsic_w_chain:v2f64 219:i32, addr:i32:$src)
12148       // Emits: (MOVUPDrm:v2f64 addr:i32:$src)
12149       // Pattern complexity = 15  cost = 1
12150       if (CN0 == 219) {
12151         N2 = N.getOperand(2);
12152         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12153           Select(Tmp1, Tmp1);
12154           Select(Tmp2, Tmp2);
12155           Select(Tmp3, Tmp3);
12156           Select(Tmp4, Tmp4);
12157           Select(Chain, Chain);
12158           ResNode = CurDAG->getTargetNode(X86::MOVUPDrm, MVT::v2f64, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
12159           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12160           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
12161           if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
12162           Result = SDOperand(ResNode, N.ResNo);
12163           return;
12164         }
12165       }
12166 
12167       // Pattern: (intrinsic_w_chain:v16i8 218:i32, addr:i32:$src)
12168       // Emits: (MOVDQUrm:v16i8 addr:i32:$src)
12169       // Pattern complexity = 15  cost = 1
12170       if (CN0 == 218) {
12171         N2 = N.getOperand(2);
12172         if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
12173           Select(Tmp1, Tmp1);
12174           Select(Tmp2, Tmp2);
12175           Select(Tmp3, Tmp3);
12176           Select(Tmp4, Tmp4);
12177           Select(Chain, Chain);
12178           ResNode = CurDAG->getTargetNode(X86::MOVDQUrm, MVT::v16i8, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
12179           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12180           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
12181           if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
12182           Result = SDOperand(ResNode, N.ResNo);
12183           return;
12184         }
12185       }
12186     }
12187   }
12188   std::cerr << "Cannot yet select: ";
12189   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
12190   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
12191   std::cerr << '\n';
12192   abort();
12193 }
12194 
12195 void Select_intrinsic_wo_chain(SDOperand &Result, SDOperand N) {
12196   SDOperand Chain1(0, 0);
12197   SDOperand Chain10(0, 0);
12198   SDOperand Chain2(0, 0);
12199   SDOperand Chain20(0, 0);
12200   SDOperand N0(0, 0);
12201   SDOperand N1(0, 0);
12202   SDOperand N10(0, 0);
12203   SDOperand N101(0, 0);
12204   SDOperand N11(0, 0);
12205   SDOperand N2(0, 0);
12206   SDOperand N20(0, 0);
12207   SDOperand N201(0, 0);
12208   SDOperand N21(0, 0);
12209   SDOperand N3(0, 0);
12210   SDOperand Tmp1(0, 0);
12211   SDOperand Tmp2(0, 0);
12212   SDOperand Tmp3(0, 0);
12213   SDOperand Tmp4(0, 0);
12214   SDOperand Tmp5(0, 0);
12215   SDOperand Tmp6(0, 0);
12216   SDNode *ResNode;
12217 
12218   // Pattern: (intrinsic_wo_chain:v4f32 278:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src), (imm:i8):$cc)
12219   // Emits: (Int_CMPSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src, (imm:i8):$cc)
12220   // Pattern complexity = 19  cost = 1
12221   if ((Subtarget->hasSSE1())) {
12222     N0 = N.getOperand(0);
12223     if (isa<ConstantSDNode>(N0)) {
12224       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12225       if (CN0 == 278) {
12226         N1 = N.getOperand(1);
12227         N2 = N.getOperand(2);
12228         if (N2.getOpcode() == ISD::LOAD &&
12229             N2.hasOneUse() &&
12230             !CodeGenMap.count(N2.getValue(0))) {
12231           Chain2 = N2.getOperand(0);
12232           N21 = N2.getOperand(1);
12233           N3 = N.getOperand(3);
12234           if (N3.getOpcode() == ISD::Constant &&
12235               SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
12236             Select(Tmp1, Tmp1);
12237             Select(Tmp2, Tmp2);
12238             Select(Tmp3, Tmp3);
12239             Select(Tmp4, Tmp4);
12240             Select(Tmp5, N1);
12241             unsigned char Tmp6C = (unsigned char)cast<ConstantSDNode>(N3)->getValue();
12242             Tmp6 = CurDAG->getTargetConstant(Tmp6C, MVT::i8);
12243             Select(Chain2, Chain2);
12244             ResNode = CurDAG->getTargetNode(X86::Int_CMPSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Tmp6, Chain2);
12245             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12246             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
12247             AddHandleReplacement(N2.Val, 1, ResNode, 1);
12248             Result = SDOperand(ResNode, N.ResNo);
12249             return;
12250           }
12251         }
12252       }
12253     }
12254   }
12255   if ((Subtarget->hasSSE2())) {
12256     N0 = N.getOperand(0);
12257     if (isa<ConstantSDNode>(N0)) {
12258       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12259 
12260       // Pattern: (intrinsic_wo_chain:v4f32 205:i32, (bitconvert:v4i32 (load:v2i64 addr:i32:$src)))
12261       // Emits: (CVTDQ2PSrm:v4f32 addr:i32:$src)
12262       // Pattern complexity = 19  cost = 1
12263       if (CN0 == 205) {
12264         N1 = N.getOperand(1);
12265         if (N1.getOpcode() == ISD::BIT_CONVERT &&
12266             N1.hasOneUse() &&
12267             !CodeGenMap.count(N1.getValue(0))) {
12268           N10 = N1.getOperand(0);
12269           if (N10.getOpcode() == ISD::LOAD &&
12270               N10.hasOneUse() &&
12271               !CodeGenMap.count(N10.getValue(0))) {
12272             Chain10 = N10.getOperand(0);
12273             N101 = N10.getOperand(1);
12274             if (N10.Val->getValueType(0) == MVT::v2i64 &&
12275                 SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
12276               Select(Tmp1, Tmp1);
12277               Select(Tmp2, Tmp2);
12278               Select(Tmp3, Tmp3);
12279               Select(Tmp4, Tmp4);
12280               Select(Chain10, Chain10);
12281               ResNode = CurDAG->getTargetNode(X86::CVTDQ2PSrm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
12282               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12283               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
12284               AddHandleReplacement(N10.Val, 1, ResNode, 1);
12285               Result = SDOperand(ResNode, N.ResNo);
12286               return;
12287             }
12288           }
12289         }
12290       }
12291 
12292       // Pattern: (intrinsic_wo_chain:v2f64 204:i32, (bitconvert:v4i32 (load:v2i64 addr:i32:$src)))
12293       // Emits: (CVTDQ2PDrm:v2f64 addr:i32:$src)
12294       // Pattern complexity = 19  cost = 1
12295       if (CN0 == 204) {
12296         N1 = N.getOperand(1);
12297         if (N1.getOpcode() == ISD::BIT_CONVERT &&
12298             N1.hasOneUse() &&
12299             !CodeGenMap.count(N1.getValue(0))) {
12300           N10 = N1.getOperand(0);
12301           if (N10.getOpcode() == ISD::LOAD &&
12302               N10.hasOneUse() &&
12303               !CodeGenMap.count(N10.getValue(0))) {
12304             Chain10 = N10.getOperand(0);
12305             N101 = N10.getOperand(1);
12306             if (N10.Val->getValueType(0) == MVT::v2i64 &&
12307                 SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
12308               Select(Tmp1, Tmp1);
12309               Select(Tmp2, Tmp2);
12310               Select(Tmp3, Tmp3);
12311               Select(Tmp4, Tmp4);
12312               Select(Chain10, Chain10);
12313               ResNode = CurDAG->getTargetNode(X86::CVTDQ2PDrm, MVT::v2f64, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
12314               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12315               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
12316               AddHandleReplacement(N10.Val, 1, ResNode, 1);
12317               Result = SDOperand(ResNode, N.ResNo);
12318               return;
12319             }
12320           }
12321         }
12322       }
12323     }
12324   }
12325 
12326   // Pattern: (intrinsic_wo_chain:v4f32 277:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src), (imm:i8):$cc)
12327   // Emits: (CMPPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src, (imm:i8):$cc)
12328   // Pattern complexity = 19  cost = 1
12329   if ((Subtarget->hasSSE1())) {
12330     N0 = N.getOperand(0);
12331     if (isa<ConstantSDNode>(N0)) {
12332       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12333       if (CN0 == 277) {
12334         N1 = N.getOperand(1);
12335         N2 = N.getOperand(2);
12336         if (N2.getOpcode() == ISD::LOAD &&
12337             N2.hasOneUse() &&
12338             !CodeGenMap.count(N2.getValue(0))) {
12339           Chain2 = N2.getOperand(0);
12340           N21 = N2.getOperand(1);
12341           N3 = N.getOperand(3);
12342           if (N3.getOpcode() == ISD::Constant &&
12343               SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
12344             Select(Tmp1, Tmp1);
12345             Select(Tmp2, Tmp2);
12346             Select(Tmp3, Tmp3);
12347             Select(Tmp4, Tmp4);
12348             Select(Tmp5, N1);
12349             unsigned char Tmp6C = (unsigned char)cast<ConstantSDNode>(N3)->getValue();
12350             Tmp6 = CurDAG->getTargetConstant(Tmp6C, MVT::i8);
12351             Select(Chain2, Chain2);
12352             ResNode = CurDAG->getTargetNode(X86::CMPPSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Tmp6, Chain2);
12353             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12354             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
12355             AddHandleReplacement(N2.Val, 1, ResNode, 1);
12356             Result = SDOperand(ResNode, N.ResNo);
12357             return;
12358           }
12359         }
12360       }
12361     }
12362   }
12363   if ((Subtarget->hasSSE2())) {
12364     N0 = N.getOperand(0);
12365     if (isa<ConstantSDNode>(N0)) {
12366       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
12367 
12368       // Pattern: (intrinsic_wo_chain:v16i8 234:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12369       // Emits: (PADDSBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12370       // Pattern complexity = 19  cost = 1
12371       if (CN0 == 234) {
12372         N1 = N.getOperand(1);
12373         N2 = N.getOperand(2);
12374         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12375             N2.hasOneUse() &&
12376             !CodeGenMap.count(N2.getValue(0))) {
12377           N20 = N2.getOperand(0);
12378           if (N20.getOpcode() == ISD::LOAD &&
12379               N20.hasOneUse() &&
12380               !CodeGenMap.count(N20.getValue(0))) {
12381             Chain20 = N20.getOperand(0);
12382             N201 = N20.getOperand(1);
12383             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12384                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12385               Select(Tmp1, Tmp1);
12386               Select(Tmp2, Tmp2);
12387               Select(Tmp3, Tmp3);
12388               Select(Tmp4, Tmp4);
12389               Select(Tmp5, N1);
12390               Select(Chain20, Chain20);
12391               ResNode = CurDAG->getTargetNode(X86::PADDSBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12392               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12393               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12394               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12395               Result = SDOperand(ResNode, N.ResNo);
12396               return;
12397             }
12398           }
12399         }
12400       }
12401 
12402       // Pattern: (intrinsic_wo_chain:v8i16 235:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12403       // Emits: (PADDSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12404       // Pattern complexity = 19  cost = 1
12405       if (CN0 == 235) {
12406         N1 = N.getOperand(1);
12407         N2 = N.getOperand(2);
12408         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12409             N2.hasOneUse() &&
12410             !CodeGenMap.count(N2.getValue(0))) {
12411           N20 = N2.getOperand(0);
12412           if (N20.getOpcode() == ISD::LOAD &&
12413               N20.hasOneUse() &&
12414               !CodeGenMap.count(N20.getValue(0))) {
12415             Chain20 = N20.getOperand(0);
12416             N201 = N20.getOperand(1);
12417             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12418                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12419               Select(Tmp1, Tmp1);
12420               Select(Tmp2, Tmp2);
12421               Select(Tmp3, Tmp3);
12422               Select(Tmp4, Tmp4);
12423               Select(Tmp5, N1);
12424               Select(Chain20, Chain20);
12425               ResNode = CurDAG->getTargetNode(X86::PADDSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12426               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12427               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12428               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12429               Result = SDOperand(ResNode, N.ResNo);
12430               return;
12431             }
12432           }
12433         }
12434       }
12435 
12436       // Pattern: (intrinsic_wo_chain:v16i8 236:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12437       // Emits: (PADDUSBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12438       // Pattern complexity = 19  cost = 1
12439       if (CN0 == 236) {
12440         N1 = N.getOperand(1);
12441         N2 = N.getOperand(2);
12442         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12443             N2.hasOneUse() &&
12444             !CodeGenMap.count(N2.getValue(0))) {
12445           N20 = N2.getOperand(0);
12446           if (N20.getOpcode() == ISD::LOAD &&
12447               N20.hasOneUse() &&
12448               !CodeGenMap.count(N20.getValue(0))) {
12449             Chain20 = N20.getOperand(0);
12450             N201 = N20.getOperand(1);
12451             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12452                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12453               Select(Tmp1, Tmp1);
12454               Select(Tmp2, Tmp2);
12455               Select(Tmp3, Tmp3);
12456               Select(Tmp4, Tmp4);
12457               Select(Tmp5, N1);
12458               Select(Chain20, Chain20);
12459               ResNode = CurDAG->getTargetNode(X86::PADDUSBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12460               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12461               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12462               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12463               Result = SDOperand(ResNode, N.ResNo);
12464               return;
12465             }
12466           }
12467         }
12468       }
12469 
12470       // Pattern: (intrinsic_wo_chain:v8i16 237:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12471       // Emits: (PADDUSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12472       // Pattern complexity = 19  cost = 1
12473       if (CN0 == 237) {
12474         N1 = N.getOperand(1);
12475         N2 = N.getOperand(2);
12476         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12477             N2.hasOneUse() &&
12478             !CodeGenMap.count(N2.getValue(0))) {
12479           N20 = N2.getOperand(0);
12480           if (N20.getOpcode() == ISD::LOAD &&
12481               N20.hasOneUse() &&
12482               !CodeGenMap.count(N20.getValue(0))) {
12483             Chain20 = N20.getOperand(0);
12484             N201 = N20.getOperand(1);
12485             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12486                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12487               Select(Tmp1, Tmp1);
12488               Select(Tmp2, Tmp2);
12489               Select(Tmp3, Tmp3);
12490               Select(Tmp4, Tmp4);
12491               Select(Tmp5, N1);
12492               Select(Chain20, Chain20);
12493               ResNode = CurDAG->getTargetNode(X86::PADDUSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12494               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12495               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12496               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12497               Result = SDOperand(ResNode, N.ResNo);
12498               return;
12499             }
12500           }
12501         }
12502       }
12503 
12504       // Pattern: (intrinsic_wo_chain:v16i8 252:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12505       // Emits: (PSUBSBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12506       // Pattern complexity = 19  cost = 1
12507       if (CN0 == 252) {
12508         N1 = N.getOperand(1);
12509         N2 = N.getOperand(2);
12510         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12511             N2.hasOneUse() &&
12512             !CodeGenMap.count(N2.getValue(0))) {
12513           N20 = N2.getOperand(0);
12514           if (N20.getOpcode() == ISD::LOAD &&
12515               N20.hasOneUse() &&
12516               !CodeGenMap.count(N20.getValue(0))) {
12517             Chain20 = N20.getOperand(0);
12518             N201 = N20.getOperand(1);
12519             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12520                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12521               Select(Tmp1, Tmp1);
12522               Select(Tmp2, Tmp2);
12523               Select(Tmp3, Tmp3);
12524               Select(Tmp4, Tmp4);
12525               Select(Tmp5, N1);
12526               Select(Chain20, Chain20);
12527               ResNode = CurDAG->getTargetNode(X86::PSUBSBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12528               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12529               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12530               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12531               Result = SDOperand(ResNode, N.ResNo);
12532               return;
12533             }
12534           }
12535         }
12536       }
12537 
12538       // Pattern: (intrinsic_wo_chain:v8i16 253:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12539       // Emits: (PSUBSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12540       // Pattern complexity = 19  cost = 1
12541       if (CN0 == 253) {
12542         N1 = N.getOperand(1);
12543         N2 = N.getOperand(2);
12544         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12545             N2.hasOneUse() &&
12546             !CodeGenMap.count(N2.getValue(0))) {
12547           N20 = N2.getOperand(0);
12548           if (N20.getOpcode() == ISD::LOAD &&
12549               N20.hasOneUse() &&
12550               !CodeGenMap.count(N20.getValue(0))) {
12551             Chain20 = N20.getOperand(0);
12552             N201 = N20.getOperand(1);
12553             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12554                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12555               Select(Tmp1, Tmp1);
12556               Select(Tmp2, Tmp2);
12557               Select(Tmp3, Tmp3);
12558               Select(Tmp4, Tmp4);
12559               Select(Tmp5, N1);
12560               Select(Chain20, Chain20);
12561               ResNode = CurDAG->getTargetNode(X86::PSUBSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12562               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12563               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12564               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12565               Result = SDOperand(ResNode, N.ResNo);
12566               return;
12567             }
12568           }
12569         }
12570       }
12571 
12572       // Pattern: (intrinsic_wo_chain:v16i8 254:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12573       // Emits: (PSUBUSBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12574       // Pattern complexity = 19  cost = 1
12575       if (CN0 == 254) {
12576         N1 = N.getOperand(1);
12577         N2 = N.getOperand(2);
12578         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12579             N2.hasOneUse() &&
12580             !CodeGenMap.count(N2.getValue(0))) {
12581           N20 = N2.getOperand(0);
12582           if (N20.getOpcode() == ISD::LOAD &&
12583               N20.hasOneUse() &&
12584               !CodeGenMap.count(N20.getValue(0))) {
12585             Chain20 = N20.getOperand(0);
12586             N201 = N20.getOperand(1);
12587             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12588                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12589               Select(Tmp1, Tmp1);
12590               Select(Tmp2, Tmp2);
12591               Select(Tmp3, Tmp3);
12592               Select(Tmp4, Tmp4);
12593               Select(Tmp5, N1);
12594               Select(Chain20, Chain20);
12595               ResNode = CurDAG->getTargetNode(X86::PSUBUSBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12596               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12597               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12598               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12599               Result = SDOperand(ResNode, N.ResNo);
12600               return;
12601             }
12602           }
12603         }
12604       }
12605 
12606       // Pattern: (intrinsic_wo_chain:v8i16 255:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12607       // Emits: (PSUBUSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12608       // Pattern complexity = 19  cost = 1
12609       if (CN0 == 255) {
12610         N1 = N.getOperand(1);
12611         N2 = N.getOperand(2);
12612         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12613             N2.hasOneUse() &&
12614             !CodeGenMap.count(N2.getValue(0))) {
12615           N20 = N2.getOperand(0);
12616           if (N20.getOpcode() == ISD::LOAD &&
12617               N20.hasOneUse() &&
12618               !CodeGenMap.count(N20.getValue(0))) {
12619             Chain20 = N20.getOperand(0);
12620             N201 = N20.getOperand(1);
12621             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12622                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12623               Select(Tmp1, Tmp1);
12624               Select(Tmp2, Tmp2);
12625               Select(Tmp3, Tmp3);
12626               Select(Tmp4, Tmp4);
12627               Select(Tmp5, N1);
12628               Select(Chain20, Chain20);
12629               ResNode = CurDAG->getTargetNode(X86::PSUBUSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12630               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12631               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12632               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12633               Result = SDOperand(ResNode, N.ResNo);
12634               return;
12635             }
12636           }
12637         }
12638       }
12639 
12640       // Pattern: (intrinsic_wo_chain:v8i16 247:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12641       // Emits: (PMULHUWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12642       // Pattern complexity = 19  cost = 1
12643       if (CN0 == 247) {
12644         N1 = N.getOperand(1);
12645         N2 = N.getOperand(2);
12646         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12647             N2.hasOneUse() &&
12648             !CodeGenMap.count(N2.getValue(0))) {
12649           N20 = N2.getOperand(0);
12650           if (N20.getOpcode() == ISD::LOAD &&
12651               N20.hasOneUse() &&
12652               !CodeGenMap.count(N20.getValue(0))) {
12653             Chain20 = N20.getOperand(0);
12654             N201 = N20.getOperand(1);
12655             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12656                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12657               Select(Tmp1, Tmp1);
12658               Select(Tmp2, Tmp2);
12659               Select(Tmp3, Tmp3);
12660               Select(Tmp4, Tmp4);
12661               Select(Tmp5, N1);
12662               Select(Chain20, Chain20);
12663               ResNode = CurDAG->getTargetNode(X86::PMULHUWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12664               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12665               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12666               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12667               Result = SDOperand(ResNode, N.ResNo);
12668               return;
12669             }
12670           }
12671         }
12672       }
12673 
12674       // Pattern: (intrinsic_wo_chain:v8i16 246:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12675       // Emits: (PMULHWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12676       // Pattern complexity = 19  cost = 1
12677       if (CN0 == 246) {
12678         N1 = N.getOperand(1);
12679         N2 = N.getOperand(2);
12680         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12681             N2.hasOneUse() &&
12682             !CodeGenMap.count(N2.getValue(0))) {
12683           N20 = N2.getOperand(0);
12684           if (N20.getOpcode() == ISD::LOAD &&
12685               N20.hasOneUse() &&
12686               !CodeGenMap.count(N20.getValue(0))) {
12687             Chain20 = N20.getOperand(0);
12688             N201 = N20.getOperand(1);
12689             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12690                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12691               Select(Tmp1, Tmp1);
12692               Select(Tmp2, Tmp2);
12693               Select(Tmp3, Tmp3);
12694               Select(Tmp4, Tmp4);
12695               Select(Tmp5, N1);
12696               Select(Chain20, Chain20);
12697               ResNode = CurDAG->getTargetNode(X86::PMULHWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12698               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12699               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12700               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12701               Result = SDOperand(ResNode, N.ResNo);
12702               return;
12703             }
12704           }
12705         }
12706       }
12707 
12708       // Pattern: (intrinsic_wo_chain:v2i64 248:i32, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:i32:$src2)))
12709       // Emits: (PMULUDQrm:v2i64 VR128:v4i32:$src1, addr:i32:$src2)
12710       // Pattern complexity = 19  cost = 1
12711       if (CN0 == 248) {
12712         N1 = N.getOperand(1);
12713         N2 = N.getOperand(2);
12714         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12715             N2.hasOneUse() &&
12716             !CodeGenMap.count(N2.getValue(0))) {
12717           N20 = N2.getOperand(0);
12718           if (N20.getOpcode() == ISD::LOAD &&
12719               N20.hasOneUse() &&
12720               !CodeGenMap.count(N20.getValue(0))) {
12721             Chain20 = N20.getOperand(0);
12722             N201 = N20.getOperand(1);
12723             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12724                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12725               Select(Tmp1, Tmp1);
12726               Select(Tmp2, Tmp2);
12727               Select(Tmp3, Tmp3);
12728               Select(Tmp4, Tmp4);
12729               Select(Tmp5, N1);
12730               Select(Chain20, Chain20);
12731               ResNode = CurDAG->getTargetNode(X86::PMULUDQrm, MVT::v2i64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12732               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12733               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12734               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12735               Result = SDOperand(ResNode, N.ResNo);
12736               return;
12737             }
12738           }
12739         }
12740       }
12741 
12742       // Pattern: (intrinsic_wo_chain:v4i32 240:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12743       // Emits: (PMADDWDrm:v4i32 VR128:v8i16:$src1, addr:i32:$src2)
12744       // Pattern complexity = 19  cost = 1
12745       if (CN0 == 240) {
12746         N1 = N.getOperand(1);
12747         N2 = N.getOperand(2);
12748         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12749             N2.hasOneUse() &&
12750             !CodeGenMap.count(N2.getValue(0))) {
12751           N20 = N2.getOperand(0);
12752           if (N20.getOpcode() == ISD::LOAD &&
12753               N20.hasOneUse() &&
12754               !CodeGenMap.count(N20.getValue(0))) {
12755             Chain20 = N20.getOperand(0);
12756             N201 = N20.getOperand(1);
12757             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12758                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12759               Select(Tmp1, Tmp1);
12760               Select(Tmp2, Tmp2);
12761               Select(Tmp3, Tmp3);
12762               Select(Tmp4, Tmp4);
12763               Select(Tmp5, N1);
12764               Select(Chain20, Chain20);
12765               ResNode = CurDAG->getTargetNode(X86::PMADDWDrm, MVT::v4i32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12766               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12767               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12768               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12769               Result = SDOperand(ResNode, N.ResNo);
12770               return;
12771             }
12772           }
12773         }
12774       }
12775 
12776       // Pattern: (intrinsic_wo_chain:v16i8 238:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12777       // Emits: (PAVGBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12778       // Pattern complexity = 19  cost = 1
12779       if (CN0 == 238) {
12780         N1 = N.getOperand(1);
12781         N2 = N.getOperand(2);
12782         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12783             N2.hasOneUse() &&
12784             !CodeGenMap.count(N2.getValue(0))) {
12785           N20 = N2.getOperand(0);
12786           if (N20.getOpcode() == ISD::LOAD &&
12787               N20.hasOneUse() &&
12788               !CodeGenMap.count(N20.getValue(0))) {
12789             Chain20 = N20.getOperand(0);
12790             N201 = N20.getOperand(1);
12791             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12792                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12793               Select(Tmp1, Tmp1);
12794               Select(Tmp2, Tmp2);
12795               Select(Tmp3, Tmp3);
12796               Select(Tmp4, Tmp4);
12797               Select(Tmp5, N1);
12798               Select(Chain20, Chain20);
12799               ResNode = CurDAG->getTargetNode(X86::PAVGBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12800               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12801               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12802               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12803               Result = SDOperand(ResNode, N.ResNo);
12804               return;
12805             }
12806           }
12807         }
12808       }
12809 
12810       // Pattern: (intrinsic_wo_chain:v8i16 239:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12811       // Emits: (PAVGWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12812       // Pattern complexity = 19  cost = 1
12813       if (CN0 == 239) {
12814         N1 = N.getOperand(1);
12815         N2 = N.getOperand(2);
12816         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12817             N2.hasOneUse() &&
12818             !CodeGenMap.count(N2.getValue(0))) {
12819           N20 = N2.getOperand(0);
12820           if (N20.getOpcode() == ISD::LOAD &&
12821               N20.hasOneUse() &&
12822               !CodeGenMap.count(N20.getValue(0))) {
12823             Chain20 = N20.getOperand(0);
12824             N201 = N20.getOperand(1);
12825             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12826                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12827               Select(Tmp1, Tmp1);
12828               Select(Tmp2, Tmp2);
12829               Select(Tmp3, Tmp3);
12830               Select(Tmp4, Tmp4);
12831               Select(Tmp5, N1);
12832               Select(Chain20, Chain20);
12833               ResNode = CurDAG->getTargetNode(X86::PAVGWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12834               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12835               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12836               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12837               Result = SDOperand(ResNode, N.ResNo);
12838               return;
12839             }
12840           }
12841         }
12842       }
12843 
12844       // Pattern: (intrinsic_wo_chain:v16i8 242:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12845       // Emits: (PMAXUBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12846       // Pattern complexity = 19  cost = 1
12847       if (CN0 == 242) {
12848         N1 = N.getOperand(1);
12849         N2 = N.getOperand(2);
12850         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12851             N2.hasOneUse() &&
12852             !CodeGenMap.count(N2.getValue(0))) {
12853           N20 = N2.getOperand(0);
12854           if (N20.getOpcode() == ISD::LOAD &&
12855               N20.hasOneUse() &&
12856               !CodeGenMap.count(N20.getValue(0))) {
12857             Chain20 = N20.getOperand(0);
12858             N201 = N20.getOperand(1);
12859             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12860                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12861               Select(Tmp1, Tmp1);
12862               Select(Tmp2, Tmp2);
12863               Select(Tmp3, Tmp3);
12864               Select(Tmp4, Tmp4);
12865               Select(Tmp5, N1);
12866               Select(Chain20, Chain20);
12867               ResNode = CurDAG->getTargetNode(X86::PMAXUBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12868               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12869               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12870               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12871               Result = SDOperand(ResNode, N.ResNo);
12872               return;
12873             }
12874           }
12875         }
12876       }
12877 
12878       // Pattern: (intrinsic_wo_chain:v8i16 241:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12879       // Emits: (PMAXSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12880       // Pattern complexity = 19  cost = 1
12881       if (CN0 == 241) {
12882         N1 = N.getOperand(1);
12883         N2 = N.getOperand(2);
12884         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12885             N2.hasOneUse() &&
12886             !CodeGenMap.count(N2.getValue(0))) {
12887           N20 = N2.getOperand(0);
12888           if (N20.getOpcode() == ISD::LOAD &&
12889               N20.hasOneUse() &&
12890               !CodeGenMap.count(N20.getValue(0))) {
12891             Chain20 = N20.getOperand(0);
12892             N201 = N20.getOperand(1);
12893             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12894                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12895               Select(Tmp1, Tmp1);
12896               Select(Tmp2, Tmp2);
12897               Select(Tmp3, Tmp3);
12898               Select(Tmp4, Tmp4);
12899               Select(Tmp5, N1);
12900               Select(Chain20, Chain20);
12901               ResNode = CurDAG->getTargetNode(X86::PMAXSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12902               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12903               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12904               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12905               Result = SDOperand(ResNode, N.ResNo);
12906               return;
12907             }
12908           }
12909         }
12910       }
12911 
12912       // Pattern: (intrinsic_wo_chain:v16i8 244:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12913       // Emits: (PMINUBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
12914       // Pattern complexity = 19  cost = 1
12915       if (CN0 == 244) {
12916         N1 = N.getOperand(1);
12917         N2 = N.getOperand(2);
12918         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12919             N2.hasOneUse() &&
12920             !CodeGenMap.count(N2.getValue(0))) {
12921           N20 = N2.getOperand(0);
12922           if (N20.getOpcode() == ISD::LOAD &&
12923               N20.hasOneUse() &&
12924               !CodeGenMap.count(N20.getValue(0))) {
12925             Chain20 = N20.getOperand(0);
12926             N201 = N20.getOperand(1);
12927             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12928                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12929               Select(Tmp1, Tmp1);
12930               Select(Tmp2, Tmp2);
12931               Select(Tmp3, Tmp3);
12932               Select(Tmp4, Tmp4);
12933               Select(Tmp5, N1);
12934               Select(Chain20, Chain20);
12935               ResNode = CurDAG->getTargetNode(X86::PMINUBrm, MVT::v16i8, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12936               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12937               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12938               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12939               Result = SDOperand(ResNode, N.ResNo);
12940               return;
12941             }
12942           }
12943         }
12944       }
12945 
12946       // Pattern: (intrinsic_wo_chain:v8i16 243:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
12947       // Emits: (PMINSWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
12948       // Pattern complexity = 19  cost = 1
12949       if (CN0 == 243) {
12950         N1 = N.getOperand(1);
12951         N2 = N.getOperand(2);
12952         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12953             N2.hasOneUse() &&
12954             !CodeGenMap.count(N2.getValue(0))) {
12955           N20 = N2.getOperand(0);
12956           if (N20.getOpcode() == ISD::LOAD &&
12957               N20.hasOneUse() &&
12958               !CodeGenMap.count(N20.getValue(0))) {
12959             Chain20 = N20.getOperand(0);
12960             N201 = N20.getOperand(1);
12961             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12962                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12963               Select(Tmp1, Tmp1);
12964               Select(Tmp2, Tmp2);
12965               Select(Tmp3, Tmp3);
12966               Select(Tmp4, Tmp4);
12967               Select(Tmp5, N1);
12968               Select(Chain20, Chain20);
12969               ResNode = CurDAG->getTargetNode(X86::PMINSWrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
12970               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
12971               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
12972               AddHandleReplacement(N20.Val, 1, ResNode, 1);
12973               Result = SDOperand(ResNode, N.ResNo);
12974               return;
12975             }
12976           }
12977         }
12978       }
12979 
12980       // Pattern: (intrinsic_wo_chain:v2i64 249:i32, VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)))
12981       // Emits: (PSADBWrm:v2i64 VR128:v16i8:$src1, addr:i32:$src2)
12982       // Pattern complexity = 19  cost = 1
12983       if (CN0 == 249) {
12984         N1 = N.getOperand(1);
12985         N2 = N.getOperand(2);
12986         if (N2.getOpcode() == ISD::BIT_CONVERT &&
12987             N2.hasOneUse() &&
12988             !CodeGenMap.count(N2.getValue(0))) {
12989           N20 = N2.getOperand(0);
12990           if (N20.getOpcode() == ISD::LOAD &&
12991               N20.hasOneUse() &&
12992               !CodeGenMap.count(N20.getValue(0))) {
12993             Chain20 = N20.getOperand(0);
12994             N201 = N20.getOperand(1);
12995             if (N20.Val->getValueType(0) == MVT::v2i64 &&
12996                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
12997               Select(Tmp1, Tmp1);
12998               Select(Tmp2, Tmp2);
12999               Select(Tmp3, Tmp3);
13000               Select(Tmp4, Tmp4);
13001               Select(Tmp5, N1);
13002               Select(Chain20, Chain20);
13003               ResNode = CurDAG->getTargetNode(X86::PSADBWrm, MVT::v2i64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
13004               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13005               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
13006               AddHandleReplacement(N20.Val, 1, ResNode, 1);
13007               Result = SDOperand(ResNode, N.ResNo);
13008               return;
13009             }
13010           }
13011         }
13012       }
13013 
13014       // Pattern: (intrinsic_wo_chain:v8i16 232:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2f64 addr:i32:$src2)))
13015       // Emits: (PACKSSWBrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
13016       // Pattern complexity = 19  cost = 1
13017       if (CN0 == 232) {
13018         N1 = N.getOperand(1);
13019         N2 = N.getOperand(2);
13020         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13021             N2.hasOneUse() &&
13022             !CodeGenMap.count(N2.getValue(0))) {
13023           N20 = N2.getOperand(0);
13024           if (N20.getOpcode() == ISD::LOAD &&
13025               N20.hasOneUse() &&
13026               !CodeGenMap.count(N20.getValue(0))) {
13027             Chain20 = N20.getOperand(0);
13028             N201 = N20.getOperand(1);
13029             if (N20.Val->getValueType(0) == MVT::v2f64 &&
13030                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
13031               Select(Tmp1, Tmp1);
13032               Select(Tmp2, Tmp2);
13033               Select(Tmp3, Tmp3);
13034               Select(Tmp4, Tmp4);
13035               Select(Tmp5, N1);
13036               Select(Chain20, Chain20);
13037               ResNode = CurDAG->getTargetNode(X86::PACKSSWBrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
13038               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13039               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
13040               AddHandleReplacement(N20.Val, 1, ResNode, 1);
13041               Result = SDOperand(ResNode, N.ResNo);
13042               return;
13043             }
13044           }
13045         }
13046       }
13047 
13048       // Pattern: (intrinsic_wo_chain:v4i32 231:i32, VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:i32:$src2)))
13049       // Emits: (PACKSSDWrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
13050       // Pattern complexity = 19  cost = 1
13051       if (CN0 == 231) {
13052         N1 = N.getOperand(1);
13053         N2 = N.getOperand(2);
13054         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13055             N2.hasOneUse() &&
13056             !CodeGenMap.count(N2.getValue(0))) {
13057           N20 = N2.getOperand(0);
13058           if (N20.getOpcode() == ISD::LOAD &&
13059               N20.hasOneUse() &&
13060               !CodeGenMap.count(N20.getValue(0))) {
13061             Chain20 = N20.getOperand(0);
13062             N201 = N20.getOperand(1);
13063             if (N20.Val->getValueType(0) == MVT::v2i64 &&
13064                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
13065               Select(Tmp1, Tmp1);
13066               Select(Tmp2, Tmp2);
13067               Select(Tmp3, Tmp3);
13068               Select(Tmp4, Tmp4);
13069               Select(Tmp5, N1);
13070               Select(Chain20, Chain20);
13071               ResNode = CurDAG->getTargetNode(X86::PACKSSDWrm, MVT::v4i32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
13072               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13073               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
13074               AddHandleReplacement(N20.Val, 1, ResNode, 1);
13075               Result = SDOperand(ResNode, N.ResNo);
13076               return;
13077             }
13078           }
13079         }
13080       }
13081 
13082       // Pattern: (intrinsic_wo_chain:v8i16 233:i32, VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
13083       // Emits: (PACKUSWBrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
13084       // Pattern complexity = 19  cost = 1
13085       if (CN0 == 233) {
13086         N1 = N.getOperand(1);
13087         N2 = N.getOperand(2);
13088         if (N2.getOpcode() == ISD::BIT_CONVERT &&
13089             N2.hasOneUse() &&
13090             !CodeGenMap.count(N2.getValue(0))) {
13091           N20 = N2.getOperand(0);
13092           if (N20.getOpcode() == ISD::LOAD &&
13093               N20.hasOneUse() &&
13094               !CodeGenMap.count(N20.getValue(0))) {
13095             Chain20 = N20.getOperand(0);
13096             N201 = N20.getOperand(1);
13097             if (N20.Val->getValueType(0) == MVT::v2i64 &&
13098                 SelectAddr(N201, Tmp1, Tmp2, Tmp3, Tmp4)) {
13099               Select(Tmp1, Tmp1);
13100               Select(Tmp2, Tmp2);
13101               Select(Tmp3, Tmp3);
13102               Select(Tmp4, Tmp4);
13103               Select(Tmp5, N1);
13104               Select(Chain20, Chain20);
13105               ResNode = CurDAG->getTargetNode(X86::PACKUSWBrm, MVT::v8i16, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain20);
13106               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13107               SelectionDAG::InsertISelMapEntry(CodeGenMap, N20.Val, 1, ResNode, 1);
13108               AddHandleReplacement(N20.Val, 1, ResNode, 1);
13109               Result = SDOperand(ResNode, N.ResNo);
13110               return;
13111             }
13112           }
13113         }
13114       }
13115     }
13116   }
13117 
13118   // Pattern: (intrinsic_wo_chain:v4f32 276:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13119   // Emits: (Int_ADDSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13120   // Pattern complexity = 17  cost = 1
13121   if ((Subtarget->hasSSE1())) {
13122     N0 = N.getOperand(0);
13123     if (isa<ConstantSDNode>(N0)) {
13124       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13125       if (CN0 == 276) {
13126         N1 = N.getOperand(1);
13127         N2 = N.getOperand(2);
13128         if (N2.getOpcode() == ISD::LOAD &&
13129             N2.hasOneUse() &&
13130             !CodeGenMap.count(N2.getValue(0))) {
13131           Chain2 = N2.getOperand(0);
13132           N21 = N2.getOperand(1);
13133           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13134             Select(Tmp1, Tmp1);
13135             Select(Tmp2, Tmp2);
13136             Select(Tmp3, Tmp3);
13137             Select(Tmp4, Tmp4);
13138             Select(Tmp5, N1);
13139             Select(Chain2, Chain2);
13140             ResNode = CurDAG->getTargetNode(X86::Int_ADDSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13141             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13142             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13143             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13144             Result = SDOperand(ResNode, N.ResNo);
13145             return;
13146           }
13147         }
13148       }
13149     }
13150   }
13151 
13152   // Pattern: (intrinsic_wo_chain:v2f64 195:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13153   // Emits: (Int_ADDSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13154   // Pattern complexity = 17  cost = 1
13155   if ((Subtarget->hasSSE2())) {
13156     N0 = N.getOperand(0);
13157     if (isa<ConstantSDNode>(N0)) {
13158       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13159       if (CN0 == 195) {
13160         N1 = N.getOperand(1);
13161         N2 = N.getOperand(2);
13162         if (N2.getOpcode() == ISD::LOAD &&
13163             N2.hasOneUse() &&
13164             !CodeGenMap.count(N2.getValue(0))) {
13165           Chain2 = N2.getOperand(0);
13166           N21 = N2.getOperand(1);
13167           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13168             Select(Tmp1, Tmp1);
13169             Select(Tmp2, Tmp2);
13170             Select(Tmp3, Tmp3);
13171             Select(Tmp4, Tmp4);
13172             Select(Tmp5, N1);
13173             Select(Chain2, Chain2);
13174             ResNode = CurDAG->getTargetNode(X86::Int_ADDSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13175             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13176             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13177             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13178             Result = SDOperand(ResNode, N.ResNo);
13179             return;
13180           }
13181         }
13182       }
13183     }
13184   }
13185 
13186   // Pattern: (intrinsic_wo_chain:v4f32 297:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13187   // Emits: (Int_MULSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13188   // Pattern complexity = 17  cost = 1
13189   if ((Subtarget->hasSSE1())) {
13190     N0 = N.getOperand(0);
13191     if (isa<ConstantSDNode>(N0)) {
13192       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13193       if (CN0 == 297) {
13194         N1 = N.getOperand(1);
13195         N2 = N.getOperand(2);
13196         if (N2.getOpcode() == ISD::LOAD &&
13197             N2.hasOneUse() &&
13198             !CodeGenMap.count(N2.getValue(0))) {
13199           Chain2 = N2.getOperand(0);
13200           N21 = N2.getOperand(1);
13201           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13202             Select(Tmp1, Tmp1);
13203             Select(Tmp2, Tmp2);
13204             Select(Tmp3, Tmp3);
13205             Select(Tmp4, Tmp4);
13206             Select(Tmp5, N1);
13207             Select(Chain2, Chain2);
13208             ResNode = CurDAG->getTargetNode(X86::Int_MULSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13209             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13210             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13211             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13212             Result = SDOperand(ResNode, N.ResNo);
13213             return;
13214           }
13215         }
13216       }
13217     }
13218   }
13219 
13220   // Pattern: (intrinsic_wo_chain:v2f64 230:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13221   // Emits: (Int_MULSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13222   // Pattern complexity = 17  cost = 1
13223   if ((Subtarget->hasSSE2())) {
13224     N0 = N.getOperand(0);
13225     if (isa<ConstantSDNode>(N0)) {
13226       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13227       if (CN0 == 230) {
13228         N1 = N.getOperand(1);
13229         N2 = N.getOperand(2);
13230         if (N2.getOpcode() == ISD::LOAD &&
13231             N2.hasOneUse() &&
13232             !CodeGenMap.count(N2.getValue(0))) {
13233           Chain2 = N2.getOperand(0);
13234           N21 = N2.getOperand(1);
13235           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13236             Select(Tmp1, Tmp1);
13237             Select(Tmp2, Tmp2);
13238             Select(Tmp3, Tmp3);
13239             Select(Tmp4, Tmp4);
13240             Select(Tmp5, N1);
13241             Select(Chain2, Chain2);
13242             ResNode = CurDAG->getTargetNode(X86::Int_MULSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13243             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13244             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13245             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13246             Result = SDOperand(ResNode, N.ResNo);
13247             return;
13248           }
13249         }
13250       }
13251     }
13252   }
13253 
13254   // Pattern: (intrinsic_wo_chain:v4f32 288:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13255   // Emits: (Int_DIVSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13256   // Pattern complexity = 17  cost = 1
13257   if ((Subtarget->hasSSE1())) {
13258     N0 = N.getOperand(0);
13259     if (isa<ConstantSDNode>(N0)) {
13260       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13261       if (CN0 == 288) {
13262         N1 = N.getOperand(1);
13263         N2 = N.getOperand(2);
13264         if (N2.getOpcode() == ISD::LOAD &&
13265             N2.hasOneUse() &&
13266             !CodeGenMap.count(N2.getValue(0))) {
13267           Chain2 = N2.getOperand(0);
13268           N21 = N2.getOperand(1);
13269           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13270             Select(Tmp1, Tmp1);
13271             Select(Tmp2, Tmp2);
13272             Select(Tmp3, Tmp3);
13273             Select(Tmp4, Tmp4);
13274             Select(Tmp5, N1);
13275             Select(Chain2, Chain2);
13276             ResNode = CurDAG->getTargetNode(X86::Int_DIVSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13277             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13278             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13279             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13280             Result = SDOperand(ResNode, N.ResNo);
13281             return;
13282           }
13283         }
13284       }
13285     }
13286   }
13287 
13288   // Pattern: (intrinsic_wo_chain:v2f64 217:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13289   // Emits: (Int_DIVSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13290   // Pattern complexity = 17  cost = 1
13291   if ((Subtarget->hasSSE2())) {
13292     N0 = N.getOperand(0);
13293     if (isa<ConstantSDNode>(N0)) {
13294       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13295       if (CN0 == 217) {
13296         N1 = N.getOperand(1);
13297         N2 = N.getOperand(2);
13298         if (N2.getOpcode() == ISD::LOAD &&
13299             N2.hasOneUse() &&
13300             !CodeGenMap.count(N2.getValue(0))) {
13301           Chain2 = N2.getOperand(0);
13302           N21 = N2.getOperand(1);
13303           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13304             Select(Tmp1, Tmp1);
13305             Select(Tmp2, Tmp2);
13306             Select(Tmp3, Tmp3);
13307             Select(Tmp4, Tmp4);
13308             Select(Tmp5, N1);
13309             Select(Chain2, Chain2);
13310             ResNode = CurDAG->getTargetNode(X86::Int_DIVSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13311             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13312             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13313             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13314             Result = SDOperand(ResNode, N.ResNo);
13315             return;
13316           }
13317         }
13318       }
13319     }
13320   }
13321 
13322   // Pattern: (intrinsic_wo_chain:v4f32 307:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13323   // Emits: (Int_SUBSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13324   // Pattern complexity = 17  cost = 1
13325   if ((Subtarget->hasSSE1())) {
13326     N0 = N.getOperand(0);
13327     if (isa<ConstantSDNode>(N0)) {
13328       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13329       if (CN0 == 307) {
13330         N1 = N.getOperand(1);
13331         N2 = N.getOperand(2);
13332         if (N2.getOpcode() == ISD::LOAD &&
13333             N2.hasOneUse() &&
13334             !CodeGenMap.count(N2.getValue(0))) {
13335           Chain2 = N2.getOperand(0);
13336           N21 = N2.getOperand(1);
13337           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13338             Select(Tmp1, Tmp1);
13339             Select(Tmp2, Tmp2);
13340             Select(Tmp3, Tmp3);
13341             Select(Tmp4, Tmp4);
13342             Select(Tmp5, N1);
13343             Select(Chain2, Chain2);
13344             ResNode = CurDAG->getTargetNode(X86::Int_SUBSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13345             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13346             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13347             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13348             Result = SDOperand(ResNode, N.ResNo);
13349             return;
13350           }
13351         }
13352       }
13353     }
13354   }
13355 
13356   // Pattern: (intrinsic_wo_chain:v2f64 265:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13357   // Emits: (Int_SUBSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13358   // Pattern complexity = 17  cost = 1
13359   if ((Subtarget->hasSSE2())) {
13360     N0 = N.getOperand(0);
13361     if (isa<ConstantSDNode>(N0)) {
13362       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13363       if (CN0 == 265) {
13364         N1 = N.getOperand(1);
13365         N2 = N.getOperand(2);
13366         if (N2.getOpcode() == ISD::LOAD &&
13367             N2.hasOneUse() &&
13368             !CodeGenMap.count(N2.getValue(0))) {
13369           Chain2 = N2.getOperand(0);
13370           N21 = N2.getOperand(1);
13371           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13372             Select(Tmp1, Tmp1);
13373             Select(Tmp2, Tmp2);
13374             Select(Tmp3, Tmp3);
13375             Select(Tmp4, Tmp4);
13376             Select(Tmp5, N1);
13377             Select(Chain2, Chain2);
13378             ResNode = CurDAG->getTargetNode(X86::Int_SUBSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13379             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13380             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13381             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13382             Result = SDOperand(ResNode, N.ResNo);
13383             return;
13384           }
13385         }
13386       }
13387     }
13388   }
13389 
13390   // Pattern: (intrinsic_wo_chain:v4f32 304:i32, (load:v4f32 addr:i32:$src))
13391   // Emits: (Int_SQRTSSm:v4f32 addr:i32:$src)
13392   // Pattern complexity = 17  cost = 1
13393   if ((Subtarget->hasSSE1())) {
13394     N0 = N.getOperand(0);
13395     if (isa<ConstantSDNode>(N0)) {
13396       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13397       if (CN0 == 304) {
13398         N1 = N.getOperand(1);
13399         if (N1.getOpcode() == ISD::LOAD &&
13400             N1.hasOneUse() &&
13401             !CodeGenMap.count(N1.getValue(0))) {
13402           Chain1 = N1.getOperand(0);
13403           N11 = N1.getOperand(1);
13404           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13405             Select(Tmp1, Tmp1);
13406             Select(Tmp2, Tmp2);
13407             Select(Tmp3, Tmp3);
13408             Select(Tmp4, Tmp4);
13409             Select(Chain1, Chain1);
13410             ResNode = CurDAG->getTargetNode(X86::Int_SQRTSSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13411             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13412             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13413             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13414             Result = SDOperand(ResNode, N.ResNo);
13415             return;
13416           }
13417         }
13418       }
13419     }
13420   }
13421 
13422   // Pattern: (intrinsic_wo_chain:v2f64 261:i32, (load:v2f64 addr:i32:$src))
13423   // Emits: (Int_SQRTSDm:v2f64 addr:i32:$src)
13424   // Pattern complexity = 17  cost = 1
13425   if ((Subtarget->hasSSE2())) {
13426     N0 = N.getOperand(0);
13427     if (isa<ConstantSDNode>(N0)) {
13428       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13429       if (CN0 == 261) {
13430         N1 = N.getOperand(1);
13431         if (N1.getOpcode() == ISD::LOAD &&
13432             N1.hasOneUse() &&
13433             !CodeGenMap.count(N1.getValue(0))) {
13434           Chain1 = N1.getOperand(0);
13435           N11 = N1.getOperand(1);
13436           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13437             Select(Tmp1, Tmp1);
13438             Select(Tmp2, Tmp2);
13439             Select(Tmp3, Tmp3);
13440             Select(Tmp4, Tmp4);
13441             Select(Chain1, Chain1);
13442             ResNode = CurDAG->getTargetNode(X86::Int_SQRTSDm, MVT::v2f64, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13443             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13444             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13445             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13446             Result = SDOperand(ResNode, N.ResNo);
13447             return;
13448           }
13449         }
13450       }
13451     }
13452   }
13453   if ((Subtarget->hasSSE1())) {
13454     N0 = N.getOperand(0);
13455     if (isa<ConstantSDNode>(N0)) {
13456       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13457 
13458       // Pattern: (intrinsic_wo_chain:v4f32 301:i32, (load:v4f32 addr:i32:$src))
13459       // Emits: (Int_RSQRTSSm:v4f32 addr:i32:$src)
13460       // Pattern complexity = 17  cost = 1
13461       if (CN0 == 301) {
13462         N1 = N.getOperand(1);
13463         if (N1.getOpcode() == ISD::LOAD &&
13464             N1.hasOneUse() &&
13465             !CodeGenMap.count(N1.getValue(0))) {
13466           Chain1 = N1.getOperand(0);
13467           N11 = N1.getOperand(1);
13468           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13469             Select(Tmp1, Tmp1);
13470             Select(Tmp2, Tmp2);
13471             Select(Tmp3, Tmp3);
13472             Select(Tmp4, Tmp4);
13473             Select(Chain1, Chain1);
13474             ResNode = CurDAG->getTargetNode(X86::Int_RSQRTSSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13475             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13476             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13477             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13478             Result = SDOperand(ResNode, N.ResNo);
13479             return;
13480           }
13481         }
13482       }
13483 
13484       // Pattern: (intrinsic_wo_chain:v4f32 299:i32, (load:v4f32 addr:i32:$src))
13485       // Emits: (Int_RCPSSm:v4f32 addr:i32:$src)
13486       // Pattern complexity = 17  cost = 1
13487       if (CN0 == 299) {
13488         N1 = N.getOperand(1);
13489         if (N1.getOpcode() == ISD::LOAD &&
13490             N1.hasOneUse() &&
13491             !CodeGenMap.count(N1.getValue(0))) {
13492           Chain1 = N1.getOperand(0);
13493           N11 = N1.getOperand(1);
13494           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13495             Select(Tmp1, Tmp1);
13496             Select(Tmp2, Tmp2);
13497             Select(Tmp3, Tmp3);
13498             Select(Tmp4, Tmp4);
13499             Select(Chain1, Chain1);
13500             ResNode = CurDAG->getTargetNode(X86::Int_RCPSSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13501             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13502             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13503             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13504             Result = SDOperand(ResNode, N.ResNo);
13505             return;
13506           }
13507         }
13508       }
13509 
13510       // Pattern: (intrinsic_wo_chain:v4f32 292:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13511       // Emits: (Int_MAXSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13512       // Pattern complexity = 17  cost = 1
13513       if (CN0 == 292) {
13514         N1 = N.getOperand(1);
13515         N2 = N.getOperand(2);
13516         if (N2.getOpcode() == ISD::LOAD &&
13517             N2.hasOneUse() &&
13518             !CodeGenMap.count(N2.getValue(0))) {
13519           Chain2 = N2.getOperand(0);
13520           N21 = N2.getOperand(1);
13521           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13522             Select(Tmp1, Tmp1);
13523             Select(Tmp2, Tmp2);
13524             Select(Tmp3, Tmp3);
13525             Select(Tmp4, Tmp4);
13526             Select(Tmp5, N1);
13527             Select(Chain2, Chain2);
13528             ResNode = CurDAG->getTargetNode(X86::Int_MAXSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13529             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13530             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13531             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13532             Result = SDOperand(ResNode, N.ResNo);
13533             return;
13534           }
13535         }
13536       }
13537     }
13538   }
13539 
13540   // Pattern: (intrinsic_wo_chain:v2f64 222:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13541   // Emits: (Int_MAXSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13542   // Pattern complexity = 17  cost = 1
13543   if ((Subtarget->hasSSE2())) {
13544     N0 = N.getOperand(0);
13545     if (isa<ConstantSDNode>(N0)) {
13546       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13547       if (CN0 == 222) {
13548         N1 = N.getOperand(1);
13549         N2 = N.getOperand(2);
13550         if (N2.getOpcode() == ISD::LOAD &&
13551             N2.hasOneUse() &&
13552             !CodeGenMap.count(N2.getValue(0))) {
13553           Chain2 = N2.getOperand(0);
13554           N21 = N2.getOperand(1);
13555           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13556             Select(Tmp1, Tmp1);
13557             Select(Tmp2, Tmp2);
13558             Select(Tmp3, Tmp3);
13559             Select(Tmp4, Tmp4);
13560             Select(Tmp5, N1);
13561             Select(Chain2, Chain2);
13562             ResNode = CurDAG->getTargetNode(X86::Int_MAXSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13563             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13564             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13565             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13566             Result = SDOperand(ResNode, N.ResNo);
13567             return;
13568           }
13569         }
13570       }
13571     }
13572   }
13573 
13574   // Pattern: (intrinsic_wo_chain:v4f32 294:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
13575   // Emits: (Int_MINSSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13576   // Pattern complexity = 17  cost = 1
13577   if ((Subtarget->hasSSE1())) {
13578     N0 = N.getOperand(0);
13579     if (isa<ConstantSDNode>(N0)) {
13580       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13581       if (CN0 == 294) {
13582         N1 = N.getOperand(1);
13583         N2 = N.getOperand(2);
13584         if (N2.getOpcode() == ISD::LOAD &&
13585             N2.hasOneUse() &&
13586             !CodeGenMap.count(N2.getValue(0))) {
13587           Chain2 = N2.getOperand(0);
13588           N21 = N2.getOperand(1);
13589           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13590             Select(Tmp1, Tmp1);
13591             Select(Tmp2, Tmp2);
13592             Select(Tmp3, Tmp3);
13593             Select(Tmp4, Tmp4);
13594             Select(Tmp5, N1);
13595             Select(Chain2, Chain2);
13596             ResNode = CurDAG->getTargetNode(X86::Int_MINSSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13597             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13598             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13599             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13600             Result = SDOperand(ResNode, N.ResNo);
13601             return;
13602           }
13603         }
13604       }
13605     }
13606   }
13607 
13608   // Pattern: (intrinsic_wo_chain:v2f64 224:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
13609   // Emits: (Int_MINSDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13610   // Pattern complexity = 17  cost = 1
13611   if ((Subtarget->hasSSE2())) {
13612     N0 = N.getOperand(0);
13613     if (isa<ConstantSDNode>(N0)) {
13614       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13615       if (CN0 == 224) {
13616         N1 = N.getOperand(1);
13617         N2 = N.getOperand(2);
13618         if (N2.getOpcode() == ISD::LOAD &&
13619             N2.hasOneUse() &&
13620             !CodeGenMap.count(N2.getValue(0))) {
13621           Chain2 = N2.getOperand(0);
13622           N21 = N2.getOperand(1);
13623           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13624             Select(Tmp1, Tmp1);
13625             Select(Tmp2, Tmp2);
13626             Select(Tmp3, Tmp3);
13627             Select(Tmp4, Tmp4);
13628             Select(Tmp5, N1);
13629             Select(Chain2, Chain2);
13630             ResNode = CurDAG->getTargetNode(X86::Int_MINSDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13631             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13632             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13633             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13634             Result = SDOperand(ResNode, N.ResNo);
13635             return;
13636           }
13637         }
13638       }
13639     }
13640   }
13641   if ((Subtarget->hasSSE1())) {
13642     N0 = N.getOperand(0);
13643     if (isa<ConstantSDNode>(N0)) {
13644       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13645 
13646       // Pattern: (intrinsic_wo_chain:i32 286:i32, (load:v4f32 addr:i32:$src))
13647       // Emits: (CVTSS2SIrm:i32 addr:i32:$src)
13648       // Pattern complexity = 17  cost = 1
13649       if (CN0 == 286) {
13650         N1 = N.getOperand(1);
13651         if (N1.getOpcode() == ISD::LOAD &&
13652             N1.hasOneUse() &&
13653             !CodeGenMap.count(N1.getValue(0))) {
13654           Chain1 = N1.getOperand(0);
13655           N11 = N1.getOperand(1);
13656           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13657             Select(Tmp1, Tmp1);
13658             Select(Tmp2, Tmp2);
13659             Select(Tmp3, Tmp3);
13660             Select(Tmp4, Tmp4);
13661             Select(Chain1, Chain1);
13662             ResNode = CurDAG->getTargetNode(X86::CVTSS2SIrm, MVT::i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13663             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13664             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13665             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13666             Result = SDOperand(ResNode, N.ResNo);
13667             return;
13668           }
13669         }
13670       }
13671 
13672       // Pattern: (intrinsic_wo_chain:i32 287:i32, (load:v4f32 addr:i32:$src))
13673       // Emits: (Int_CVTTSS2SIrm:i32 addr:i32:$src)
13674       // Pattern complexity = 17  cost = 1
13675       if (CN0 == 287) {
13676         N1 = N.getOperand(1);
13677         if (N1.getOpcode() == ISD::LOAD &&
13678             N1.hasOneUse() &&
13679             !CodeGenMap.count(N1.getValue(0))) {
13680           Chain1 = N1.getOperand(0);
13681           N11 = N1.getOperand(1);
13682           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13683             Select(Tmp1, Tmp1);
13684             Select(Tmp2, Tmp2);
13685             Select(Tmp3, Tmp3);
13686             Select(Tmp4, Tmp4);
13687             Select(Chain1, Chain1);
13688             ResNode = CurDAG->getTargetNode(X86::Int_CVTTSS2SIrm, MVT::i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13689             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13690             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13691             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13692             Result = SDOperand(ResNode, N.ResNo);
13693             return;
13694           }
13695         }
13696       }
13697     }
13698   }
13699 
13700   // Pattern: (intrinsic_wo_chain:i32 216:i32, (load:v2f64 addr:i32:$src))
13701   // Emits: (Int_CVTTSD2SIrm:i32 addr:i32:$src)
13702   // Pattern complexity = 17  cost = 1
13703   if ((Subtarget->hasSSE2())) {
13704     N0 = N.getOperand(0);
13705     if (isa<ConstantSDNode>(N0)) {
13706       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13707       if (CN0 == 216) {
13708         N1 = N.getOperand(1);
13709         if (N1.getOpcode() == ISD::LOAD &&
13710             N1.hasOneUse() &&
13711             !CodeGenMap.count(N1.getValue(0))) {
13712           Chain1 = N1.getOperand(0);
13713           N11 = N1.getOperand(1);
13714           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13715             Select(Tmp1, Tmp1);
13716             Select(Tmp2, Tmp2);
13717             Select(Tmp3, Tmp3);
13718             Select(Tmp4, Tmp4);
13719             Select(Chain1, Chain1);
13720             ResNode = CurDAG->getTargetNode(X86::Int_CVTTSD2SIrm, MVT::i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13721             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13722             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13723             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13724             Result = SDOperand(ResNode, N.ResNo);
13725             return;
13726           }
13727         }
13728       }
13729     }
13730   }
13731 
13732   // Pattern: (intrinsic_wo_chain:v4f32 285:i32, VR128:v4f32:$src1, (load:i32 addr:i32:$src2))
13733   // Emits: (Int_CVTSI2SSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13734   // Pattern complexity = 17  cost = 1
13735   if ((Subtarget->hasSSE1())) {
13736     N0 = N.getOperand(0);
13737     if (isa<ConstantSDNode>(N0)) {
13738       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13739       if (CN0 == 285) {
13740         N1 = N.getOperand(1);
13741         N2 = N.getOperand(2);
13742         if (N2.getOpcode() == ISD::LOAD &&
13743             N2.hasOneUse() &&
13744             !CodeGenMap.count(N2.getValue(0))) {
13745           Chain2 = N2.getOperand(0);
13746           N21 = N2.getOperand(1);
13747           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13748             Select(Tmp1, Tmp1);
13749             Select(Tmp2, Tmp2);
13750             Select(Tmp3, Tmp3);
13751             Select(Tmp4, Tmp4);
13752             Select(Tmp5, N1);
13753             Select(Chain2, Chain2);
13754             ResNode = CurDAG->getTargetNode(X86::Int_CVTSI2SSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13755             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13756             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13757             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13758             Result = SDOperand(ResNode, N.ResNo);
13759             return;
13760           }
13761         }
13762       }
13763     }
13764   }
13765   if ((Subtarget->hasSSE2())) {
13766     N0 = N.getOperand(0);
13767     if (isa<ConstantSDNode>(N0)) {
13768       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
13769 
13770       // Pattern: (intrinsic_wo_chain:v4i32 208:i32, (load:v4f32 addr:i32:$src))
13771       // Emits: (CVTPS2DQrm:v4i32 addr:i32:$src)
13772       // Pattern complexity = 17  cost = 1
13773       if (CN0 == 208) {
13774         N1 = N.getOperand(1);
13775         if (N1.getOpcode() == ISD::LOAD &&
13776             N1.hasOneUse() &&
13777             !CodeGenMap.count(N1.getValue(0))) {
13778           Chain1 = N1.getOperand(0);
13779           N11 = N1.getOperand(1);
13780           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13781             Select(Tmp1, Tmp1);
13782             Select(Tmp2, Tmp2);
13783             Select(Tmp3, Tmp3);
13784             Select(Tmp4, Tmp4);
13785             Select(Chain1, Chain1);
13786             ResNode = CurDAG->getTargetNode(X86::CVTPS2DQrm, MVT::v4i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13787             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13788             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13789             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13790             Result = SDOperand(ResNode, N.ResNo);
13791             return;
13792           }
13793         }
13794       }
13795 
13796       // Pattern: (intrinsic_wo_chain:v4i32 215:i32, (load:v4f32 addr:i32:$src))
13797       // Emits: (CVTTPS2DQrm:v4i32 addr:i32:$src)
13798       // Pattern complexity = 17  cost = 1
13799       if (CN0 == 215) {
13800         N1 = N.getOperand(1);
13801         if (N1.getOpcode() == ISD::LOAD &&
13802             N1.hasOneUse() &&
13803             !CodeGenMap.count(N1.getValue(0))) {
13804           Chain1 = N1.getOperand(0);
13805           N11 = N1.getOperand(1);
13806           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13807             Select(Tmp1, Tmp1);
13808             Select(Tmp2, Tmp2);
13809             Select(Tmp3, Tmp3);
13810             Select(Tmp4, Tmp4);
13811             Select(Chain1, Chain1);
13812             ResNode = CurDAG->getTargetNode(X86::CVTTPS2DQrm, MVT::v4i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13813             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13814             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13815             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13816             Result = SDOperand(ResNode, N.ResNo);
13817             return;
13818           }
13819         }
13820       }
13821 
13822       // Pattern: (intrinsic_wo_chain:v4i32 206:i32, (load:v2f64 addr:i32:$src))
13823       // Emits: (CVTPD2DQrm:v4i32 addr:i32:$src)
13824       // Pattern complexity = 17  cost = 1
13825       if (CN0 == 206) {
13826         N1 = N.getOperand(1);
13827         if (N1.getOpcode() == ISD::LOAD &&
13828             N1.hasOneUse() &&
13829             !CodeGenMap.count(N1.getValue(0))) {
13830           Chain1 = N1.getOperand(0);
13831           N11 = N1.getOperand(1);
13832           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13833             Select(Tmp1, Tmp1);
13834             Select(Tmp2, Tmp2);
13835             Select(Tmp3, Tmp3);
13836             Select(Tmp4, Tmp4);
13837             Select(Chain1, Chain1);
13838             ResNode = CurDAG->getTargetNode(X86::CVTPD2DQrm, MVT::v4i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13839             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13840             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13841             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13842             Result = SDOperand(ResNode, N.ResNo);
13843             return;
13844           }
13845         }
13846       }
13847 
13848       // Pattern: (intrinsic_wo_chain:v4i32 214:i32, (load:v2f64 addr:i32:$src))
13849       // Emits: (CVTTPD2DQrm:v4i32 addr:i32:$src)
13850       // Pattern complexity = 17  cost = 1
13851       if (CN0 == 214) {
13852         N1 = N.getOperand(1);
13853         if (N1.getOpcode() == ISD::LOAD &&
13854             N1.hasOneUse() &&
13855             !CodeGenMap.count(N1.getValue(0))) {
13856           Chain1 = N1.getOperand(0);
13857           N11 = N1.getOperand(1);
13858           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13859             Select(Tmp1, Tmp1);
13860             Select(Tmp2, Tmp2);
13861             Select(Tmp3, Tmp3);
13862             Select(Tmp4, Tmp4);
13863             Select(Chain1, Chain1);
13864             ResNode = CurDAG->getTargetNode(X86::CVTTPD2DQrm, MVT::v4i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13865             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13866             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13867             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13868             Result = SDOperand(ResNode, N.ResNo);
13869             return;
13870           }
13871         }
13872       }
13873 
13874       // Pattern: (intrinsic_wo_chain:v2f64 209:i32, (load:v4f32 addr:i32:$src))
13875       // Emits: (CVTPS2PDrm:v2f64 addr:i32:$src)
13876       // Pattern complexity = 17  cost = 1
13877       if (CN0 == 209) {
13878         N1 = N.getOperand(1);
13879         if (N1.getOpcode() == ISD::LOAD &&
13880             N1.hasOneUse() &&
13881             !CodeGenMap.count(N1.getValue(0))) {
13882           Chain1 = N1.getOperand(0);
13883           N11 = N1.getOperand(1);
13884           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13885             Select(Tmp1, Tmp1);
13886             Select(Tmp2, Tmp2);
13887             Select(Tmp3, Tmp3);
13888             Select(Tmp4, Tmp4);
13889             Select(Chain1, Chain1);
13890             ResNode = CurDAG->getTargetNode(X86::CVTPS2PDrm, MVT::v2f64, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13891             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13892             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13893             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13894             Result = SDOperand(ResNode, N.ResNo);
13895             return;
13896           }
13897         }
13898       }
13899 
13900       // Pattern: (intrinsic_wo_chain:v4f32 207:i32, (load:v2f64 addr:i32:$src))
13901       // Emits: (CVTPD2PSrm:v4f32 addr:i32:$src)
13902       // Pattern complexity = 17  cost = 1
13903       if (CN0 == 207) {
13904         N1 = N.getOperand(1);
13905         if (N1.getOpcode() == ISD::LOAD &&
13906             N1.hasOneUse() &&
13907             !CodeGenMap.count(N1.getValue(0))) {
13908           Chain1 = N1.getOperand(0);
13909           N11 = N1.getOperand(1);
13910           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13911             Select(Tmp1, Tmp1);
13912             Select(Tmp2, Tmp2);
13913             Select(Tmp3, Tmp3);
13914             Select(Tmp4, Tmp4);
13915             Select(Chain1, Chain1);
13916             ResNode = CurDAG->getTargetNode(X86::CVTPD2PSrm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13917             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13918             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13919             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13920             Result = SDOperand(ResNode, N.ResNo);
13921             return;
13922           }
13923         }
13924       }
13925 
13926       // Pattern: (intrinsic_wo_chain:i32 210:i32, (load:v2f64 addr:i32:$src))
13927       // Emits: (CVTSD2SIrm:i32 addr:i32:$src)
13928       // Pattern complexity = 17  cost = 1
13929       if (CN0 == 210) {
13930         N1 = N.getOperand(1);
13931         if (N1.getOpcode() == ISD::LOAD &&
13932             N1.hasOneUse() &&
13933             !CodeGenMap.count(N1.getValue(0))) {
13934           Chain1 = N1.getOperand(0);
13935           N11 = N1.getOperand(1);
13936           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
13937             Select(Tmp1, Tmp1);
13938             Select(Tmp2, Tmp2);
13939             Select(Tmp3, Tmp3);
13940             Select(Tmp4, Tmp4);
13941             Select(Chain1, Chain1);
13942             ResNode = CurDAG->getTargetNode(X86::CVTSD2SIrm, MVT::i32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
13943             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13944             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
13945             AddHandleReplacement(N1.Val, 1, ResNode, 1);
13946             Result = SDOperand(ResNode, N.ResNo);
13947             return;
13948           }
13949         }
13950       }
13951 
13952       // Pattern: (intrinsic_wo_chain:v2f64 212:i32, VR128:v2f64:$src1, (load:i32 addr:i32:$src2))
13953       // Emits: (Int_CVTSI2SDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
13954       // Pattern complexity = 17  cost = 1
13955       if (CN0 == 212) {
13956         N1 = N.getOperand(1);
13957         N2 = N.getOperand(2);
13958         if (N2.getOpcode() == ISD::LOAD &&
13959             N2.hasOneUse() &&
13960             !CodeGenMap.count(N2.getValue(0))) {
13961           Chain2 = N2.getOperand(0);
13962           N21 = N2.getOperand(1);
13963           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13964             Select(Tmp1, Tmp1);
13965             Select(Tmp2, Tmp2);
13966             Select(Tmp3, Tmp3);
13967             Select(Tmp4, Tmp4);
13968             Select(Tmp5, N1);
13969             Select(Chain2, Chain2);
13970             ResNode = CurDAG->getTargetNode(X86::Int_CVTSI2SDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13971             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
13972             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
13973             AddHandleReplacement(N2.Val, 1, ResNode, 1);
13974             Result = SDOperand(ResNode, N.ResNo);
13975             return;
13976           }
13977         }
13978       }
13979 
13980       // Pattern: (intrinsic_wo_chain:v4f32 211:i32, VR128:v4f32:$src1, (load:v2f64 addr:i32:$src2))
13981       // Emits: (Int_CVTSD2SSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
13982       // Pattern complexity = 17  cost = 1
13983       if (CN0 == 211) {
13984         N1 = N.getOperand(1);
13985         N2 = N.getOperand(2);
13986         if (N2.getOpcode() == ISD::LOAD &&
13987             N2.hasOneUse() &&
13988             !CodeGenMap.count(N2.getValue(0))) {
13989           Chain2 = N2.getOperand(0);
13990           N21 = N2.getOperand(1);
13991           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
13992             Select(Tmp1, Tmp1);
13993             Select(Tmp2, Tmp2);
13994             Select(Tmp3, Tmp3);
13995             Select(Tmp4, Tmp4);
13996             Select(Tmp5, N1);
13997             Select(Chain2, Chain2);
13998             ResNode = CurDAG->getTargetNode(X86::Int_CVTSD2SSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
13999             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14000             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14001             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14002             Result = SDOperand(ResNode, N.ResNo);
14003             return;
14004           }
14005         }
14006       }
14007 
14008       // Pattern: (intrinsic_wo_chain:v2f64 213:i32, VR128:v2f64:$src1, (load:v4f32 addr:i32:$src2))
14009       // Emits: (Int_CVTSS2SDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
14010       // Pattern complexity = 17  cost = 1
14011       if (CN0 == 213) {
14012         N1 = N.getOperand(1);
14013         N2 = N.getOperand(2);
14014         if (N2.getOpcode() == ISD::LOAD &&
14015             N2.hasOneUse() &&
14016             !CodeGenMap.count(N2.getValue(0))) {
14017           Chain2 = N2.getOperand(0);
14018           N21 = N2.getOperand(1);
14019           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14020             Select(Tmp1, Tmp1);
14021             Select(Tmp2, Tmp2);
14022             Select(Tmp3, Tmp3);
14023             Select(Tmp4, Tmp4);
14024             Select(Tmp5, N1);
14025             Select(Chain2, Chain2);
14026             ResNode = CurDAG->getTargetNode(X86::Int_CVTSS2SDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14027             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14028             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14029             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14030             Result = SDOperand(ResNode, N.ResNo);
14031             return;
14032           }
14033         }
14034       }
14035     }
14036   }
14037 
14038   // Pattern: (intrinsic_wo_chain:v4f32 303:i32, (load:v4f32 addr:i32:$src))
14039   // Emits: (SQRTPSm:v4f32 addr:i32:$src)
14040   // Pattern complexity = 17  cost = 1
14041   if ((Subtarget->hasSSE1())) {
14042     N0 = N.getOperand(0);
14043     if (isa<ConstantSDNode>(N0)) {
14044       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14045       if (CN0 == 303) {
14046         N1 = N.getOperand(1);
14047         if (N1.getOpcode() == ISD::LOAD &&
14048             N1.hasOneUse() &&
14049             !CodeGenMap.count(N1.getValue(0))) {
14050           Chain1 = N1.getOperand(0);
14051           N11 = N1.getOperand(1);
14052           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
14053             Select(Tmp1, Tmp1);
14054             Select(Tmp2, Tmp2);
14055             Select(Tmp3, Tmp3);
14056             Select(Tmp4, Tmp4);
14057             Select(Chain1, Chain1);
14058             ResNode = CurDAG->getTargetNode(X86::SQRTPSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
14059             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14060             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
14061             AddHandleReplacement(N1.Val, 1, ResNode, 1);
14062             Result = SDOperand(ResNode, N.ResNo);
14063             return;
14064           }
14065         }
14066       }
14067     }
14068   }
14069 
14070   // Pattern: (intrinsic_wo_chain:v2f64 260:i32, (load:v2f64 addr:i32:$src))
14071   // Emits: (SQRTPDm:v2f64 addr:i32:$src)
14072   // Pattern complexity = 17  cost = 1
14073   if ((Subtarget->hasSSE2())) {
14074     N0 = N.getOperand(0);
14075     if (isa<ConstantSDNode>(N0)) {
14076       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14077       if (CN0 == 260) {
14078         N1 = N.getOperand(1);
14079         if (N1.getOpcode() == ISD::LOAD &&
14080             N1.hasOneUse() &&
14081             !CodeGenMap.count(N1.getValue(0))) {
14082           Chain1 = N1.getOperand(0);
14083           N11 = N1.getOperand(1);
14084           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
14085             Select(Tmp1, Tmp1);
14086             Select(Tmp2, Tmp2);
14087             Select(Tmp3, Tmp3);
14088             Select(Tmp4, Tmp4);
14089             Select(Chain1, Chain1);
14090             ResNode = CurDAG->getTargetNode(X86::SQRTPDm, MVT::v2f64, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
14091             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14092             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
14093             AddHandleReplacement(N1.Val, 1, ResNode, 1);
14094             Result = SDOperand(ResNode, N.ResNo);
14095             return;
14096           }
14097         }
14098       }
14099     }
14100   }
14101   if ((Subtarget->hasSSE1())) {
14102     N0 = N.getOperand(0);
14103     if (isa<ConstantSDNode>(N0)) {
14104       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14105 
14106       // Pattern: (intrinsic_wo_chain:v4f32 300:i32, (load:v4f32 addr:i32:$src))
14107       // Emits: (RSQRTPSm:v4f32 addr:i32:$src)
14108       // Pattern complexity = 17  cost = 1
14109       if (CN0 == 300) {
14110         N1 = N.getOperand(1);
14111         if (N1.getOpcode() == ISD::LOAD &&
14112             N1.hasOneUse() &&
14113             !CodeGenMap.count(N1.getValue(0))) {
14114           Chain1 = N1.getOperand(0);
14115           N11 = N1.getOperand(1);
14116           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
14117             Select(Tmp1, Tmp1);
14118             Select(Tmp2, Tmp2);
14119             Select(Tmp3, Tmp3);
14120             Select(Tmp4, Tmp4);
14121             Select(Chain1, Chain1);
14122             ResNode = CurDAG->getTargetNode(X86::RSQRTPSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
14123             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14124             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
14125             AddHandleReplacement(N1.Val, 1, ResNode, 1);
14126             Result = SDOperand(ResNode, N.ResNo);
14127             return;
14128           }
14129         }
14130       }
14131 
14132       // Pattern: (intrinsic_wo_chain:v4f32 298:i32, (load:v4f32 addr:i32:$src))
14133       // Emits: (RCPPSm:v4f32 addr:i32:$src)
14134       // Pattern complexity = 17  cost = 1
14135       if (CN0 == 298) {
14136         N1 = N.getOperand(1);
14137         if (N1.getOpcode() == ISD::LOAD &&
14138             N1.hasOneUse() &&
14139             !CodeGenMap.count(N1.getValue(0))) {
14140           Chain1 = N1.getOperand(0);
14141           N11 = N1.getOperand(1);
14142           if (SelectAddr(N11, Tmp1, Tmp2, Tmp3, Tmp4)) {
14143             Select(Tmp1, Tmp1);
14144             Select(Tmp2, Tmp2);
14145             Select(Tmp3, Tmp3);
14146             Select(Tmp4, Tmp4);
14147             Select(Chain1, Chain1);
14148             ResNode = CurDAG->getTargetNode(X86::RCPPSm, MVT::v4f32, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain1);
14149             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14150             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
14151             AddHandleReplacement(N1.Val, 1, ResNode, 1);
14152             Result = SDOperand(ResNode, N.ResNo);
14153             return;
14154           }
14155         }
14156       }
14157 
14158       // Pattern: (intrinsic_wo_chain:v4f32 291:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
14159       // Emits: (MAXPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
14160       // Pattern complexity = 17  cost = 1
14161       if (CN0 == 291) {
14162         N1 = N.getOperand(1);
14163         N2 = N.getOperand(2);
14164         if (N2.getOpcode() == ISD::LOAD &&
14165             N2.hasOneUse() &&
14166             !CodeGenMap.count(N2.getValue(0))) {
14167           Chain2 = N2.getOperand(0);
14168           N21 = N2.getOperand(1);
14169           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14170             Select(Tmp1, Tmp1);
14171             Select(Tmp2, Tmp2);
14172             Select(Tmp3, Tmp3);
14173             Select(Tmp4, Tmp4);
14174             Select(Tmp5, N1);
14175             Select(Chain2, Chain2);
14176             ResNode = CurDAG->getTargetNode(X86::MAXPSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14177             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14178             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14179             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14180             Result = SDOperand(ResNode, N.ResNo);
14181             return;
14182           }
14183         }
14184       }
14185     }
14186   }
14187 
14188   // Pattern: (intrinsic_wo_chain:v2f64 221:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
14189   // Emits: (MAXPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
14190   // Pattern complexity = 17  cost = 1
14191   if ((Subtarget->hasSSE2())) {
14192     N0 = N.getOperand(0);
14193     if (isa<ConstantSDNode>(N0)) {
14194       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14195       if (CN0 == 221) {
14196         N1 = N.getOperand(1);
14197         N2 = N.getOperand(2);
14198         if (N2.getOpcode() == ISD::LOAD &&
14199             N2.hasOneUse() &&
14200             !CodeGenMap.count(N2.getValue(0))) {
14201           Chain2 = N2.getOperand(0);
14202           N21 = N2.getOperand(1);
14203           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14204             Select(Tmp1, Tmp1);
14205             Select(Tmp2, Tmp2);
14206             Select(Tmp3, Tmp3);
14207             Select(Tmp4, Tmp4);
14208             Select(Tmp5, N1);
14209             Select(Chain2, Chain2);
14210             ResNode = CurDAG->getTargetNode(X86::MAXPDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14211             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14212             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14213             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14214             Result = SDOperand(ResNode, N.ResNo);
14215             return;
14216           }
14217         }
14218       }
14219     }
14220   }
14221 
14222   // Pattern: (intrinsic_wo_chain:v4f32 293:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
14223   // Emits: (MINPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
14224   // Pattern complexity = 17  cost = 1
14225   if ((Subtarget->hasSSE1())) {
14226     N0 = N.getOperand(0);
14227     if (isa<ConstantSDNode>(N0)) {
14228       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14229       if (CN0 == 293) {
14230         N1 = N.getOperand(1);
14231         N2 = N.getOperand(2);
14232         if (N2.getOpcode() == ISD::LOAD &&
14233             N2.hasOneUse() &&
14234             !CodeGenMap.count(N2.getValue(0))) {
14235           Chain2 = N2.getOperand(0);
14236           N21 = N2.getOperand(1);
14237           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14238             Select(Tmp1, Tmp1);
14239             Select(Tmp2, Tmp2);
14240             Select(Tmp3, Tmp3);
14241             Select(Tmp4, Tmp4);
14242             Select(Tmp5, N1);
14243             Select(Chain2, Chain2);
14244             ResNode = CurDAG->getTargetNode(X86::MINPSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14245             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14246             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14247             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14248             Result = SDOperand(ResNode, N.ResNo);
14249             return;
14250           }
14251         }
14252       }
14253     }
14254   }
14255 
14256   // Pattern: (intrinsic_wo_chain:v2f64 223:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
14257   // Emits: (MINPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
14258   // Pattern complexity = 17  cost = 1
14259   if ((Subtarget->hasSSE2())) {
14260     N0 = N.getOperand(0);
14261     if (isa<ConstantSDNode>(N0)) {
14262       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14263       if (CN0 == 223) {
14264         N1 = N.getOperand(1);
14265         N2 = N.getOperand(2);
14266         if (N2.getOpcode() == ISD::LOAD &&
14267             N2.hasOneUse() &&
14268             !CodeGenMap.count(N2.getValue(0))) {
14269           Chain2 = N2.getOperand(0);
14270           N21 = N2.getOperand(1);
14271           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14272             Select(Tmp1, Tmp1);
14273             Select(Tmp2, Tmp2);
14274             Select(Tmp3, Tmp3);
14275             Select(Tmp4, Tmp4);
14276             Select(Tmp5, N1);
14277             Select(Chain2, Chain2);
14278             ResNode = CurDAG->getTargetNode(X86::MINPDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14279             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14280             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14281             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14282             Result = SDOperand(ResNode, N.ResNo);
14283             return;
14284           }
14285         }
14286       }
14287     }
14288   }
14289   if ((Subtarget->hasSSE3())) {
14290     N0 = N.getOperand(0);
14291     if (isa<ConstantSDNode>(N0)) {
14292       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14293 
14294       // Pattern: (intrinsic_wo_chain:v4f32 273:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
14295       // Emits: (HADDPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
14296       // Pattern complexity = 17  cost = 1
14297       if (CN0 == 273) {
14298         N1 = N.getOperand(1);
14299         N2 = N.getOperand(2);
14300         if (N2.getOpcode() == ISD::LOAD &&
14301             N2.hasOneUse() &&
14302             !CodeGenMap.count(N2.getValue(0))) {
14303           Chain2 = N2.getOperand(0);
14304           N21 = N2.getOperand(1);
14305           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14306             Select(Tmp1, Tmp1);
14307             Select(Tmp2, Tmp2);
14308             Select(Tmp3, Tmp3);
14309             Select(Tmp4, Tmp4);
14310             Select(Tmp5, N1);
14311             Select(Chain2, Chain2);
14312             ResNode = CurDAG->getTargetNode(X86::HADDPSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14313             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14314             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14315             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14316             Result = SDOperand(ResNode, N.ResNo);
14317             return;
14318           }
14319         }
14320       }
14321 
14322       // Pattern: (intrinsic_wo_chain:v2f64 272:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
14323       // Emits: (HADDPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
14324       // Pattern complexity = 17  cost = 1
14325       if (CN0 == 272) {
14326         N1 = N.getOperand(1);
14327         N2 = N.getOperand(2);
14328         if (N2.getOpcode() == ISD::LOAD &&
14329             N2.hasOneUse() &&
14330             !CodeGenMap.count(N2.getValue(0))) {
14331           Chain2 = N2.getOperand(0);
14332           N21 = N2.getOperand(1);
14333           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14334             Select(Tmp1, Tmp1);
14335             Select(Tmp2, Tmp2);
14336             Select(Tmp3, Tmp3);
14337             Select(Tmp4, Tmp4);
14338             Select(Tmp5, N1);
14339             Select(Chain2, Chain2);
14340             ResNode = CurDAG->getTargetNode(X86::HADDPDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14341             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14342             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14343             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14344             Result = SDOperand(ResNode, N.ResNo);
14345             return;
14346           }
14347         }
14348       }
14349 
14350       // Pattern: (intrinsic_wo_chain:v4f32 275:i32, VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2))
14351       // Emits: (HSUBPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
14352       // Pattern complexity = 17  cost = 1
14353       if (CN0 == 275) {
14354         N1 = N.getOperand(1);
14355         N2 = N.getOperand(2);
14356         if (N2.getOpcode() == ISD::LOAD &&
14357             N2.hasOneUse() &&
14358             !CodeGenMap.count(N2.getValue(0))) {
14359           Chain2 = N2.getOperand(0);
14360           N21 = N2.getOperand(1);
14361           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14362             Select(Tmp1, Tmp1);
14363             Select(Tmp2, Tmp2);
14364             Select(Tmp3, Tmp3);
14365             Select(Tmp4, Tmp4);
14366             Select(Tmp5, N1);
14367             Select(Chain2, Chain2);
14368             ResNode = CurDAG->getTargetNode(X86::HSUBPSrm, MVT::v4f32, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14369             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14370             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14371             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14372             Result = SDOperand(ResNode, N.ResNo);
14373             return;
14374           }
14375         }
14376       }
14377 
14378       // Pattern: (intrinsic_wo_chain:v2f64 274:i32, VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2))
14379       // Emits: (HSUBPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
14380       // Pattern complexity = 17  cost = 1
14381       if (CN0 == 274) {
14382         N1 = N.getOperand(1);
14383         N2 = N.getOperand(2);
14384         if (N2.getOpcode() == ISD::LOAD &&
14385             N2.hasOneUse() &&
14386             !CodeGenMap.count(N2.getValue(0))) {
14387           Chain2 = N2.getOperand(0);
14388           N21 = N2.getOperand(1);
14389           if (SelectAddr(N21, Tmp1, Tmp2, Tmp3, Tmp4)) {
14390             Select(Tmp1, Tmp1);
14391             Select(Tmp2, Tmp2);
14392             Select(Tmp3, Tmp3);
14393             Select(Tmp4, Tmp4);
14394             Select(Tmp5, N1);
14395             Select(Chain2, Chain2);
14396             ResNode = CurDAG->getTargetNode(X86::HSUBPDrm, MVT::v2f64, MVT::Other, Tmp5, Tmp1, Tmp2, Tmp3, Tmp4, Chain2);
14397             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
14398             SelectionDAG::InsertISelMapEntry(CodeGenMap, N2.Val, 1, ResNode, 1);
14399             AddHandleReplacement(N2.Val, 1, ResNode, 1);
14400             Result = SDOperand(ResNode, N.ResNo);
14401             return;
14402           }
14403         }
14404       }
14405     }
14406   }
14407   if ((Subtarget->hasSSE1())) {
14408     N0 = N.getOperand(0);
14409     if (isa<ConstantSDNode>(N0)) {
14410       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14411 
14412       // Pattern: (intrinsic_wo_chain:v4f32 278:i32, VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
14413       // Emits: (Int_CMPSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
14414       // Pattern complexity = 7  cost = 1
14415       if (CN0 == 278) {
14416         N1 = N.getOperand(1);
14417         N2 = N.getOperand(2);
14418         N3 = N.getOperand(3);
14419         if (N3.getOpcode() == ISD::Constant) {
14420           Select(Tmp1, N1);
14421           Select(Tmp2, N2);
14422           unsigned char Tmp3C = (unsigned char)cast<ConstantSDNode>(N3)->getValue();
14423           Tmp3 = CurDAG->getTargetConstant(Tmp3C, MVT::i8);
14424           if (N.Val->hasOneUse()) {
14425             Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CMPSSrr, MVT::v4f32, Tmp1, Tmp2, Tmp3);
14426           } else {
14427             ResNode = CurDAG->getTargetNode(X86::Int_CMPSSrr, MVT::v4f32, Tmp1, Tmp2, Tmp3);
14428             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14429             Result = SDOperand(ResNode, 0);
14430           }
14431           return;
14432         }
14433       }
14434 
14435       // Pattern: (intrinsic_wo_chain:v4f32 277:i32, VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
14436       // Emits: (CMPPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src, (imm:i8):$cc)
14437       // Pattern complexity = 7  cost = 1
14438       if (CN0 == 277) {
14439         N1 = N.getOperand(1);
14440         N2 = N.getOperand(2);
14441         N3 = N.getOperand(3);
14442         if (N3.getOpcode() == ISD::Constant) {
14443           Select(Tmp1, N1);
14444           Select(Tmp2, N2);
14445           unsigned char Tmp3C = (unsigned char)cast<ConstantSDNode>(N3)->getValue();
14446           Tmp3 = CurDAG->getTargetConstant(Tmp3C, MVT::i8);
14447           if (N.Val->hasOneUse()) {
14448             Result = CurDAG->SelectNodeTo(N.Val, X86::CMPPSrr, MVT::v4f32, Tmp1, Tmp2, Tmp3);
14449           } else {
14450             ResNode = CurDAG->getTargetNode(X86::CMPPSrr, MVT::v4f32, Tmp1, Tmp2, Tmp3);
14451             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14452             Result = SDOperand(ResNode, 0);
14453           }
14454           return;
14455         }
14456       }
14457     }
14458   }
14459   if ((Subtarget->hasSSE2())) {
14460     N0 = N.getOperand(0);
14461     if (isa<ConstantSDNode>(N0)) {
14462       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14463 
14464       // Pattern: (intrinsic_wo_chain:v2i64 250:i32, VR128:v2i64:$src1, (imm:i32):$src2)
14465       // Emits: (PSLLDQri:v2i64 VR128:v16i8:$src1, (PSxLDQ_imm:i32 (imm:i32):$src2))
14466       // Pattern complexity = 7  cost = 1
14467       if (CN0 == 250) {
14468         N1 = N.getOperand(1);
14469         N2 = N.getOperand(2);
14470         if (N2.getOpcode() == ISD::Constant) {
14471           Select(Tmp1, N1);
14472           unsigned Tmp2C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
14473           Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i32);
14474           Tmp3 = Transform_PSxLDQ_imm(Tmp2.Val);
14475           if (N.Val->hasOneUse()) {
14476             Result = CurDAG->SelectNodeTo(N.Val, X86::PSLLDQri, MVT::v2i64, Tmp1, Tmp3);
14477           } else {
14478             ResNode = CurDAG->getTargetNode(X86::PSLLDQri, MVT::v2i64, Tmp1, Tmp3);
14479             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14480             Result = SDOperand(ResNode, 0);
14481           }
14482           return;
14483         }
14484       }
14485 
14486       // Pattern: (intrinsic_wo_chain:v2i64 251:i32, VR128:v2i64:$src1, (imm:i32):$src2)
14487       // Emits: (PSRLDQri:v2i64 VR128:v16i8:$src1, (PSxLDQ_imm:i32 (imm:i32):$src2))
14488       // Pattern complexity = 7  cost = 1
14489       if (CN0 == 251) {
14490         N1 = N.getOperand(1);
14491         N2 = N.getOperand(2);
14492         if (N2.getOpcode() == ISD::Constant) {
14493           Select(Tmp1, N1);
14494           unsigned Tmp2C = (unsigned)cast<ConstantSDNode>(N2)->getValue();
14495           Tmp2 = CurDAG->getTargetConstant(Tmp2C, MVT::i32);
14496           Tmp3 = Transform_PSxLDQ_imm(Tmp2.Val);
14497           if (N.Val->hasOneUse()) {
14498             Result = CurDAG->SelectNodeTo(N.Val, X86::PSRLDQri, MVT::v2i64, Tmp1, Tmp3);
14499           } else {
14500             ResNode = CurDAG->getTargetNode(X86::PSRLDQri, MVT::v2i64, Tmp1, Tmp3);
14501             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14502             Result = SDOperand(ResNode, 0);
14503           }
14504           return;
14505         }
14506       }
14507     }
14508   }
14509 
14510   // Pattern: (intrinsic_wo_chain:v4f32 276:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14511   // Emits: (Int_ADDSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14512   // Pattern complexity = 5  cost = 1
14513   if ((Subtarget->hasSSE1())) {
14514     N0 = N.getOperand(0);
14515     if (isa<ConstantSDNode>(N0)) {
14516       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14517       if (CN0 == 276) {
14518         N1 = N.getOperand(1);
14519         N2 = N.getOperand(2);
14520         Select(Tmp1, N1);
14521         Select(Tmp2, N2);
14522         if (N.Val->hasOneUse()) {
14523           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_ADDSSrr, MVT::v4f32, Tmp1, Tmp2);
14524         } else {
14525           ResNode = CurDAG->getTargetNode(X86::Int_ADDSSrr, MVT::v4f32, Tmp1, Tmp2);
14526           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14527           Result = SDOperand(ResNode, 0);
14528         }
14529         return;
14530       }
14531     }
14532   }
14533 
14534   // Pattern: (intrinsic_wo_chain:v2f64 195:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14535   // Emits: (Int_ADDSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14536   // Pattern complexity = 5  cost = 1
14537   if ((Subtarget->hasSSE2())) {
14538     N0 = N.getOperand(0);
14539     if (isa<ConstantSDNode>(N0)) {
14540       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14541       if (CN0 == 195) {
14542         N1 = N.getOperand(1);
14543         N2 = N.getOperand(2);
14544         Select(Tmp1, N1);
14545         Select(Tmp2, N2);
14546         if (N.Val->hasOneUse()) {
14547           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_ADDSDrr, MVT::v2f64, Tmp1, Tmp2);
14548         } else {
14549           ResNode = CurDAG->getTargetNode(X86::Int_ADDSDrr, MVT::v2f64, Tmp1, Tmp2);
14550           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14551           Result = SDOperand(ResNode, 0);
14552         }
14553         return;
14554       }
14555     }
14556   }
14557 
14558   // Pattern: (intrinsic_wo_chain:v4f32 297:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14559   // Emits: (Int_MULSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14560   // Pattern complexity = 5  cost = 1
14561   if ((Subtarget->hasSSE1())) {
14562     N0 = N.getOperand(0);
14563     if (isa<ConstantSDNode>(N0)) {
14564       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14565       if (CN0 == 297) {
14566         N1 = N.getOperand(1);
14567         N2 = N.getOperand(2);
14568         Select(Tmp1, N1);
14569         Select(Tmp2, N2);
14570         if (N.Val->hasOneUse()) {
14571           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MULSSrr, MVT::v4f32, Tmp1, Tmp2);
14572         } else {
14573           ResNode = CurDAG->getTargetNode(X86::Int_MULSSrr, MVT::v4f32, Tmp1, Tmp2);
14574           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14575           Result = SDOperand(ResNode, 0);
14576         }
14577         return;
14578       }
14579     }
14580   }
14581 
14582   // Pattern: (intrinsic_wo_chain:v2f64 230:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14583   // Emits: (Int_MULSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14584   // Pattern complexity = 5  cost = 1
14585   if ((Subtarget->hasSSE2())) {
14586     N0 = N.getOperand(0);
14587     if (isa<ConstantSDNode>(N0)) {
14588       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14589       if (CN0 == 230) {
14590         N1 = N.getOperand(1);
14591         N2 = N.getOperand(2);
14592         Select(Tmp1, N1);
14593         Select(Tmp2, N2);
14594         if (N.Val->hasOneUse()) {
14595           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MULSDrr, MVT::v2f64, Tmp1, Tmp2);
14596         } else {
14597           ResNode = CurDAG->getTargetNode(X86::Int_MULSDrr, MVT::v2f64, Tmp1, Tmp2);
14598           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14599           Result = SDOperand(ResNode, 0);
14600         }
14601         return;
14602       }
14603     }
14604   }
14605 
14606   // Pattern: (intrinsic_wo_chain:v4f32 288:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14607   // Emits: (Int_DIVSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14608   // Pattern complexity = 5  cost = 1
14609   if ((Subtarget->hasSSE1())) {
14610     N0 = N.getOperand(0);
14611     if (isa<ConstantSDNode>(N0)) {
14612       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14613       if (CN0 == 288) {
14614         N1 = N.getOperand(1);
14615         N2 = N.getOperand(2);
14616         Select(Tmp1, N1);
14617         Select(Tmp2, N2);
14618         if (N.Val->hasOneUse()) {
14619           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_DIVSSrr, MVT::v4f32, Tmp1, Tmp2);
14620         } else {
14621           ResNode = CurDAG->getTargetNode(X86::Int_DIVSSrr, MVT::v4f32, Tmp1, Tmp2);
14622           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14623           Result = SDOperand(ResNode, 0);
14624         }
14625         return;
14626       }
14627     }
14628   }
14629 
14630   // Pattern: (intrinsic_wo_chain:v2f64 217:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14631   // Emits: (Int_DIVSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14632   // Pattern complexity = 5  cost = 1
14633   if ((Subtarget->hasSSE2())) {
14634     N0 = N.getOperand(0);
14635     if (isa<ConstantSDNode>(N0)) {
14636       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14637       if (CN0 == 217) {
14638         N1 = N.getOperand(1);
14639         N2 = N.getOperand(2);
14640         Select(Tmp1, N1);
14641         Select(Tmp2, N2);
14642         if (N.Val->hasOneUse()) {
14643           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_DIVSDrr, MVT::v2f64, Tmp1, Tmp2);
14644         } else {
14645           ResNode = CurDAG->getTargetNode(X86::Int_DIVSDrr, MVT::v2f64, Tmp1, Tmp2);
14646           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14647           Result = SDOperand(ResNode, 0);
14648         }
14649         return;
14650       }
14651     }
14652   }
14653 
14654   // Pattern: (intrinsic_wo_chain:v4f32 307:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14655   // Emits: (Int_SUBSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14656   // Pattern complexity = 5  cost = 1
14657   if ((Subtarget->hasSSE1())) {
14658     N0 = N.getOperand(0);
14659     if (isa<ConstantSDNode>(N0)) {
14660       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14661       if (CN0 == 307) {
14662         N1 = N.getOperand(1);
14663         N2 = N.getOperand(2);
14664         Select(Tmp1, N1);
14665         Select(Tmp2, N2);
14666         if (N.Val->hasOneUse()) {
14667           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_SUBSSrr, MVT::v4f32, Tmp1, Tmp2);
14668         } else {
14669           ResNode = CurDAG->getTargetNode(X86::Int_SUBSSrr, MVT::v4f32, Tmp1, Tmp2);
14670           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14671           Result = SDOperand(ResNode, 0);
14672         }
14673         return;
14674       }
14675     }
14676   }
14677 
14678   // Pattern: (intrinsic_wo_chain:v2f64 265:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14679   // Emits: (Int_SUBSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14680   // Pattern complexity = 5  cost = 1
14681   if ((Subtarget->hasSSE2())) {
14682     N0 = N.getOperand(0);
14683     if (isa<ConstantSDNode>(N0)) {
14684       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14685       if (CN0 == 265) {
14686         N1 = N.getOperand(1);
14687         N2 = N.getOperand(2);
14688         Select(Tmp1, N1);
14689         Select(Tmp2, N2);
14690         if (N.Val->hasOneUse()) {
14691           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_SUBSDrr, MVT::v2f64, Tmp1, Tmp2);
14692         } else {
14693           ResNode = CurDAG->getTargetNode(X86::Int_SUBSDrr, MVT::v2f64, Tmp1, Tmp2);
14694           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14695           Result = SDOperand(ResNode, 0);
14696         }
14697         return;
14698       }
14699     }
14700   }
14701 
14702   // Pattern: (intrinsic_wo_chain:v4f32 304:i32, VR128:v4f32:$src)
14703   // Emits: (Int_SQRTSSr:v4f32 VR128:v4f32:$src)
14704   // Pattern complexity = 5  cost = 1
14705   if ((Subtarget->hasSSE1())) {
14706     N0 = N.getOperand(0);
14707     if (isa<ConstantSDNode>(N0)) {
14708       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14709       if (CN0 == 304) {
14710         N1 = N.getOperand(1);
14711         Select(Tmp1, N1);
14712         if (N.Val->hasOneUse()) {
14713           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_SQRTSSr, MVT::v4f32, Tmp1);
14714         } else {
14715           ResNode = CurDAG->getTargetNode(X86::Int_SQRTSSr, MVT::v4f32, Tmp1);
14716           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14717           Result = SDOperand(ResNode, 0);
14718         }
14719         return;
14720       }
14721     }
14722   }
14723 
14724   // Pattern: (intrinsic_wo_chain:v2f64 261:i32, VR128:v2f64:$src)
14725   // Emits: (Int_SQRTSDr:v2f64 VR128:v2f64:$src)
14726   // Pattern complexity = 5  cost = 1
14727   if ((Subtarget->hasSSE2())) {
14728     N0 = N.getOperand(0);
14729     if (isa<ConstantSDNode>(N0)) {
14730       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14731       if (CN0 == 261) {
14732         N1 = N.getOperand(1);
14733         Select(Tmp1, N1);
14734         if (N.Val->hasOneUse()) {
14735           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_SQRTSDr, MVT::v2f64, Tmp1);
14736         } else {
14737           ResNode = CurDAG->getTargetNode(X86::Int_SQRTSDr, MVT::v2f64, Tmp1);
14738           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14739           Result = SDOperand(ResNode, 0);
14740         }
14741         return;
14742       }
14743     }
14744   }
14745   if ((Subtarget->hasSSE1())) {
14746     N0 = N.getOperand(0);
14747     if (isa<ConstantSDNode>(N0)) {
14748       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14749 
14750       // Pattern: (intrinsic_wo_chain:v4f32 301:i32, VR128:v4f32:$src)
14751       // Emits: (Int_RSQRTSSr:v4f32 VR128:v4f32:$src)
14752       // Pattern complexity = 5  cost = 1
14753       if (CN0 == 301) {
14754         N1 = N.getOperand(1);
14755         Select(Tmp1, N1);
14756         if (N.Val->hasOneUse()) {
14757           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_RSQRTSSr, MVT::v4f32, Tmp1);
14758         } else {
14759           ResNode = CurDAG->getTargetNode(X86::Int_RSQRTSSr, MVT::v4f32, Tmp1);
14760           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14761           Result = SDOperand(ResNode, 0);
14762         }
14763         return;
14764       }
14765 
14766       // Pattern: (intrinsic_wo_chain:v4f32 299:i32, VR128:v4f32:$src)
14767       // Emits: (Int_RCPSSr:v4f32 VR128:v4f32:$src)
14768       // Pattern complexity = 5  cost = 1
14769       if (CN0 == 299) {
14770         N1 = N.getOperand(1);
14771         Select(Tmp1, N1);
14772         if (N.Val->hasOneUse()) {
14773           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_RCPSSr, MVT::v4f32, Tmp1);
14774         } else {
14775           ResNode = CurDAG->getTargetNode(X86::Int_RCPSSr, MVT::v4f32, Tmp1);
14776           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14777           Result = SDOperand(ResNode, 0);
14778         }
14779         return;
14780       }
14781 
14782       // Pattern: (intrinsic_wo_chain:v4f32 292:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14783       // Emits: (Int_MAXSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14784       // Pattern complexity = 5  cost = 1
14785       if (CN0 == 292) {
14786         N1 = N.getOperand(1);
14787         N2 = N.getOperand(2);
14788         Select(Tmp1, N1);
14789         Select(Tmp2, N2);
14790         if (N.Val->hasOneUse()) {
14791           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MAXSSrr, MVT::v4f32, Tmp1, Tmp2);
14792         } else {
14793           ResNode = CurDAG->getTargetNode(X86::Int_MAXSSrr, MVT::v4f32, Tmp1, Tmp2);
14794           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14795           Result = SDOperand(ResNode, 0);
14796         }
14797         return;
14798       }
14799     }
14800   }
14801 
14802   // Pattern: (intrinsic_wo_chain:v2f64 222:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14803   // Emits: (Int_MAXSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14804   // Pattern complexity = 5  cost = 1
14805   if ((Subtarget->hasSSE2())) {
14806     N0 = N.getOperand(0);
14807     if (isa<ConstantSDNode>(N0)) {
14808       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14809       if (CN0 == 222) {
14810         N1 = N.getOperand(1);
14811         N2 = N.getOperand(2);
14812         Select(Tmp1, N1);
14813         Select(Tmp2, N2);
14814         if (N.Val->hasOneUse()) {
14815           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MAXSDrr, MVT::v2f64, Tmp1, Tmp2);
14816         } else {
14817           ResNode = CurDAG->getTargetNode(X86::Int_MAXSDrr, MVT::v2f64, Tmp1, Tmp2);
14818           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14819           Result = SDOperand(ResNode, 0);
14820         }
14821         return;
14822       }
14823     }
14824   }
14825 
14826   // Pattern: (intrinsic_wo_chain:v4f32 294:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
14827   // Emits: (Int_MINSSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
14828   // Pattern complexity = 5  cost = 1
14829   if ((Subtarget->hasSSE1())) {
14830     N0 = N.getOperand(0);
14831     if (isa<ConstantSDNode>(N0)) {
14832       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14833       if (CN0 == 294) {
14834         N1 = N.getOperand(1);
14835         N2 = N.getOperand(2);
14836         Select(Tmp1, N1);
14837         Select(Tmp2, N2);
14838         if (N.Val->hasOneUse()) {
14839           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MINSSrr, MVT::v4f32, Tmp1, Tmp2);
14840         } else {
14841           ResNode = CurDAG->getTargetNode(X86::Int_MINSSrr, MVT::v4f32, Tmp1, Tmp2);
14842           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14843           Result = SDOperand(ResNode, 0);
14844         }
14845         return;
14846       }
14847     }
14848   }
14849 
14850   // Pattern: (intrinsic_wo_chain:v2f64 224:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
14851   // Emits: (Int_MINSDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
14852   // Pattern complexity = 5  cost = 1
14853   if ((Subtarget->hasSSE2())) {
14854     N0 = N.getOperand(0);
14855     if (isa<ConstantSDNode>(N0)) {
14856       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14857       if (CN0 == 224) {
14858         N1 = N.getOperand(1);
14859         N2 = N.getOperand(2);
14860         Select(Tmp1, N1);
14861         Select(Tmp2, N2);
14862         if (N.Val->hasOneUse()) {
14863           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_MINSDrr, MVT::v2f64, Tmp1, Tmp2);
14864         } else {
14865           ResNode = CurDAG->getTargetNode(X86::Int_MINSDrr, MVT::v2f64, Tmp1, Tmp2);
14866           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14867           Result = SDOperand(ResNode, 0);
14868         }
14869         return;
14870       }
14871     }
14872   }
14873   if ((Subtarget->hasSSE1())) {
14874     N0 = N.getOperand(0);
14875     if (isa<ConstantSDNode>(N0)) {
14876       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14877 
14878       // Pattern: (intrinsic_wo_chain:i32 286:i32, VR128:v4f32:$src)
14879       // Emits: (CVTSS2SIrr:i32 VR128:v4f32:$src)
14880       // Pattern complexity = 5  cost = 1
14881       if (CN0 == 286) {
14882         N1 = N.getOperand(1);
14883         Select(Tmp1, N1);
14884         if (N.Val->hasOneUse()) {
14885           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSS2SIrr, MVT::i32, Tmp1);
14886         } else {
14887           ResNode = CurDAG->getTargetNode(X86::CVTSS2SIrr, MVT::i32, Tmp1);
14888           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14889           Result = SDOperand(ResNode, 0);
14890         }
14891         return;
14892       }
14893 
14894       // Pattern: (intrinsic_wo_chain:i32 287:i32, VR128:v4f32:$src)
14895       // Emits: (Int_CVTTSS2SIrr:i32 VR128:v4f32:$src)
14896       // Pattern complexity = 5  cost = 1
14897       if (CN0 == 287) {
14898         N1 = N.getOperand(1);
14899         Select(Tmp1, N1);
14900         if (N.Val->hasOneUse()) {
14901           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTTSS2SIrr, MVT::i32, Tmp1);
14902         } else {
14903           ResNode = CurDAG->getTargetNode(X86::Int_CVTTSS2SIrr, MVT::i32, Tmp1);
14904           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14905           Result = SDOperand(ResNode, 0);
14906         }
14907         return;
14908       }
14909     }
14910   }
14911 
14912   // Pattern: (intrinsic_wo_chain:i32 216:i32, VR128:v2f64:$src)
14913   // Emits: (Int_CVTTSD2SIrr:i32 VR128:v2f64:$src)
14914   // Pattern complexity = 5  cost = 1
14915   if ((Subtarget->hasSSE2())) {
14916     N0 = N.getOperand(0);
14917     if (isa<ConstantSDNode>(N0)) {
14918       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14919       if (CN0 == 216) {
14920         N1 = N.getOperand(1);
14921         Select(Tmp1, N1);
14922         if (N.Val->hasOneUse()) {
14923           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTTSD2SIrr, MVT::i32, Tmp1);
14924         } else {
14925           ResNode = CurDAG->getTargetNode(X86::Int_CVTTSD2SIrr, MVT::i32, Tmp1);
14926           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14927           Result = SDOperand(ResNode, 0);
14928         }
14929         return;
14930       }
14931     }
14932   }
14933 
14934   // Pattern: (intrinsic_wo_chain:v4f32 285:i32, VR128:v4f32:$src1, R32:i32:$src2)
14935   // Emits: (Int_CVTSI2SSrr:v4f32 VR128:v4f32:$src1, R32:i32:$src2)
14936   // Pattern complexity = 5  cost = 1
14937   if ((Subtarget->hasSSE1())) {
14938     N0 = N.getOperand(0);
14939     if (isa<ConstantSDNode>(N0)) {
14940       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14941       if (CN0 == 285) {
14942         N1 = N.getOperand(1);
14943         N2 = N.getOperand(2);
14944         Select(Tmp1, N1);
14945         Select(Tmp2, N2);
14946         if (N.Val->hasOneUse()) {
14947           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTSI2SSrr, MVT::v4f32, Tmp1, Tmp2);
14948         } else {
14949           ResNode = CurDAG->getTargetNode(X86::Int_CVTSI2SSrr, MVT::v4f32, Tmp1, Tmp2);
14950           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14951           Result = SDOperand(ResNode, 0);
14952         }
14953         return;
14954       }
14955     }
14956   }
14957   if ((Subtarget->hasSSE2())) {
14958     N0 = N.getOperand(0);
14959     if (isa<ConstantSDNode>(N0)) {
14960       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
14961 
14962       // Pattern: (intrinsic_wo_chain:v4f32 205:i32, VR128:v4i32:$src)
14963       // Emits: (CVTDQ2PSrr:v4f32 VR128:v4i32:$src)
14964       // Pattern complexity = 5  cost = 1
14965       if (CN0 == 205) {
14966         N1 = N.getOperand(1);
14967         Select(Tmp1, N1);
14968         if (N.Val->hasOneUse()) {
14969           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTDQ2PSrr, MVT::v4f32, Tmp1);
14970         } else {
14971           ResNode = CurDAG->getTargetNode(X86::CVTDQ2PSrr, MVT::v4f32, Tmp1);
14972           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14973           Result = SDOperand(ResNode, 0);
14974         }
14975         return;
14976       }
14977 
14978       // Pattern: (intrinsic_wo_chain:v2f64 204:i32, VR128:v4i32:$src)
14979       // Emits: (CVTDQ2PDrr:v2f64 VR128:v4i32:$src)
14980       // Pattern complexity = 5  cost = 1
14981       if (CN0 == 204) {
14982         N1 = N.getOperand(1);
14983         Select(Tmp1, N1);
14984         if (N.Val->hasOneUse()) {
14985           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTDQ2PDrr, MVT::v2f64, Tmp1);
14986         } else {
14987           ResNode = CurDAG->getTargetNode(X86::CVTDQ2PDrr, MVT::v2f64, Tmp1);
14988           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
14989           Result = SDOperand(ResNode, 0);
14990         }
14991         return;
14992       }
14993 
14994       // Pattern: (intrinsic_wo_chain:v4i32 208:i32, VR128:v4f32:$src)
14995       // Emits: (CVTPS2DQrr:v4i32 VR128:v4f32:$src)
14996       // Pattern complexity = 5  cost = 1
14997       if (CN0 == 208) {
14998         N1 = N.getOperand(1);
14999         Select(Tmp1, N1);
15000         if (N.Val->hasOneUse()) {
15001           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTPS2DQrr, MVT::v4i32, Tmp1);
15002         } else {
15003           ResNode = CurDAG->getTargetNode(X86::CVTPS2DQrr, MVT::v4i32, Tmp1);
15004           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15005           Result = SDOperand(ResNode, 0);
15006         }
15007         return;
15008       }
15009 
15010       // Pattern: (intrinsic_wo_chain:v4i32 215:i32, VR128:v4f32:$src)
15011       // Emits: (CVTTPS2DQrr:v4i32 VR128:v4f32:$src)
15012       // Pattern complexity = 5  cost = 1
15013       if (CN0 == 215) {
15014         N1 = N.getOperand(1);
15015         Select(Tmp1, N1);
15016         if (N.Val->hasOneUse()) {
15017           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTTPS2DQrr, MVT::v4i32, Tmp1);
15018         } else {
15019           ResNode = CurDAG->getTargetNode(X86::CVTTPS2DQrr, MVT::v4i32, Tmp1);
15020           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15021           Result = SDOperand(ResNode, 0);
15022         }
15023         return;
15024       }
15025 
15026       // Pattern: (intrinsic_wo_chain:v4i32 206:i32, VR128:v2f64:$src)
15027       // Emits: (CVTPD2DQrr:v4i32 VR128:v2f64:$src)
15028       // Pattern complexity = 5  cost = 1
15029       if (CN0 == 206) {
15030         N1 = N.getOperand(1);
15031         Select(Tmp1, N1);
15032         if (N.Val->hasOneUse()) {
15033           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTPD2DQrr, MVT::v4i32, Tmp1);
15034         } else {
15035           ResNode = CurDAG->getTargetNode(X86::CVTPD2DQrr, MVT::v4i32, Tmp1);
15036           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15037           Result = SDOperand(ResNode, 0);
15038         }
15039         return;
15040       }
15041 
15042       // Pattern: (intrinsic_wo_chain:v4i32 214:i32, VR128:v2f64:$src)
15043       // Emits: (CVTTPD2DQrr:v4i32 VR128:v2f64:$src)
15044       // Pattern complexity = 5  cost = 1
15045       if (CN0 == 214) {
15046         N1 = N.getOperand(1);
15047         Select(Tmp1, N1);
15048         if (N.Val->hasOneUse()) {
15049           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTTPD2DQrr, MVT::v4i32, Tmp1);
15050         } else {
15051           ResNode = CurDAG->getTargetNode(X86::CVTTPD2DQrr, MVT::v4i32, Tmp1);
15052           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15053           Result = SDOperand(ResNode, 0);
15054         }
15055         return;
15056       }
15057 
15058       // Pattern: (intrinsic_wo_chain:v2f64 209:i32, VR128:v4f32:$src)
15059       // Emits: (CVTPS2PDrr:v2f64 VR128:v4f32:$src)
15060       // Pattern complexity = 5  cost = 1
15061       if (CN0 == 209) {
15062         N1 = N.getOperand(1);
15063         Select(Tmp1, N1);
15064         if (N.Val->hasOneUse()) {
15065           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTPS2PDrr, MVT::v2f64, Tmp1);
15066         } else {
15067           ResNode = CurDAG->getTargetNode(X86::CVTPS2PDrr, MVT::v2f64, Tmp1);
15068           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15069           Result = SDOperand(ResNode, 0);
15070         }
15071         return;
15072       }
15073 
15074       // Pattern: (intrinsic_wo_chain:v4f32 207:i32, VR128:v2f64:$src)
15075       // Emits: (CVTPD2PSrr:v4f32 VR128:v2f64:$src)
15076       // Pattern complexity = 5  cost = 1
15077       if (CN0 == 207) {
15078         N1 = N.getOperand(1);
15079         Select(Tmp1, N1);
15080         if (N.Val->hasOneUse()) {
15081           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTPD2PSrr, MVT::v4f32, Tmp1);
15082         } else {
15083           ResNode = CurDAG->getTargetNode(X86::CVTPD2PSrr, MVT::v4f32, Tmp1);
15084           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15085           Result = SDOperand(ResNode, 0);
15086         }
15087         return;
15088       }
15089 
15090       // Pattern: (intrinsic_wo_chain:i32 210:i32, VR128:v2f64:$src)
15091       // Emits: (CVTSD2SIrr:i32 VR128:v2f64:$src)
15092       // Pattern complexity = 5  cost = 1
15093       if (CN0 == 210) {
15094         N1 = N.getOperand(1);
15095         Select(Tmp1, N1);
15096         if (N.Val->hasOneUse()) {
15097           Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSD2SIrr, MVT::i32, Tmp1);
15098         } else {
15099           ResNode = CurDAG->getTargetNode(X86::CVTSD2SIrr, MVT::i32, Tmp1);
15100           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15101           Result = SDOperand(ResNode, 0);
15102         }
15103         return;
15104       }
15105 
15106       // Pattern: (intrinsic_wo_chain:v2f64 212:i32, VR128:v2f64:$src1, R32:i32:$src2)
15107       // Emits: (Int_CVTSI2SDrr:v2f64 VR128:v2f64:$src1, R32:i32:$src2)
15108       // Pattern complexity = 5  cost = 1
15109       if (CN0 == 212) {
15110         N1 = N.getOperand(1);
15111         N2 = N.getOperand(2);
15112         Select(Tmp1, N1);
15113         Select(Tmp2, N2);
15114         if (N.Val->hasOneUse()) {
15115           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTSI2SDrr, MVT::v2f64, Tmp1, Tmp2);
15116         } else {
15117           ResNode = CurDAG->getTargetNode(X86::Int_CVTSI2SDrr, MVT::v2f64, Tmp1, Tmp2);
15118           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15119           Result = SDOperand(ResNode, 0);
15120         }
15121         return;
15122       }
15123 
15124       // Pattern: (intrinsic_wo_chain:v4f32 211:i32, VR128:v4f32:$src1, VR128:v2f64:$src2)
15125       // Emits: (Int_CVTSD2SSrr:v4f32 VR128:v4f32:$src1, VR128:v2f64:$src2)
15126       // Pattern complexity = 5  cost = 1
15127       if (CN0 == 211) {
15128         N1 = N.getOperand(1);
15129         N2 = N.getOperand(2);
15130         Select(Tmp1, N1);
15131         Select(Tmp2, N2);
15132         if (N.Val->hasOneUse()) {
15133           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTSD2SSrr, MVT::v4f32, Tmp1, Tmp2);
15134         } else {
15135           ResNode = CurDAG->getTargetNode(X86::Int_CVTSD2SSrr, MVT::v4f32, Tmp1, Tmp2);
15136           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15137           Result = SDOperand(ResNode, 0);
15138         }
15139         return;
15140       }
15141 
15142       // Pattern: (intrinsic_wo_chain:v2f64 213:i32, VR128:v2f64:$src1, VR128:v4f32:$src2)
15143       // Emits: (Int_CVTSS2SDrr:v2f64 VR128:v2f64:$src1, VR128:v4f32:$src2)
15144       // Pattern complexity = 5  cost = 1
15145       if (CN0 == 213) {
15146         N1 = N.getOperand(1);
15147         N2 = N.getOperand(2);
15148         Select(Tmp1, N1);
15149         Select(Tmp2, N2);
15150         if (N.Val->hasOneUse()) {
15151           Result = CurDAG->SelectNodeTo(N.Val, X86::Int_CVTSS2SDrr, MVT::v2f64, Tmp1, Tmp2);
15152         } else {
15153           ResNode = CurDAG->getTargetNode(X86::Int_CVTSS2SDrr, MVT::v2f64, Tmp1, Tmp2);
15154           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15155           Result = SDOperand(ResNode, 0);
15156         }
15157         return;
15158       }
15159     }
15160   }
15161 
15162   // Pattern: (intrinsic_wo_chain:v4f32 303:i32, VR128:v4f32:$src)
15163   // Emits: (SQRTPSr:v4f32 VR128:v4f32:$src)
15164   // Pattern complexity = 5  cost = 1
15165   if ((Subtarget->hasSSE1())) {
15166     N0 = N.getOperand(0);
15167     if (isa<ConstantSDNode>(N0)) {
15168       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15169       if (CN0 == 303) {
15170         N1 = N.getOperand(1);
15171         Select(Tmp1, N1);
15172         if (N.Val->hasOneUse()) {
15173           Result = CurDAG->SelectNodeTo(N.Val, X86::SQRTPSr, MVT::v4f32, Tmp1);
15174         } else {
15175           ResNode = CurDAG->getTargetNode(X86::SQRTPSr, MVT::v4f32, Tmp1);
15176           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15177           Result = SDOperand(ResNode, 0);
15178         }
15179         return;
15180       }
15181     }
15182   }
15183 
15184   // Pattern: (intrinsic_wo_chain:v2f64 260:i32, VR128:v2f64:$src)
15185   // Emits: (SQRTPDr:v2f64 VR128:v2f64:$src)
15186   // Pattern complexity = 5  cost = 1
15187   if ((Subtarget->hasSSE2())) {
15188     N0 = N.getOperand(0);
15189     if (isa<ConstantSDNode>(N0)) {
15190       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15191       if (CN0 == 260) {
15192         N1 = N.getOperand(1);
15193         Select(Tmp1, N1);
15194         if (N.Val->hasOneUse()) {
15195           Result = CurDAG->SelectNodeTo(N.Val, X86::SQRTPDr, MVT::v2f64, Tmp1);
15196         } else {
15197           ResNode = CurDAG->getTargetNode(X86::SQRTPDr, MVT::v2f64, Tmp1);
15198           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15199           Result = SDOperand(ResNode, 0);
15200         }
15201         return;
15202       }
15203     }
15204   }
15205   if ((Subtarget->hasSSE1())) {
15206     N0 = N.getOperand(0);
15207     if (isa<ConstantSDNode>(N0)) {
15208       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15209 
15210       // Pattern: (intrinsic_wo_chain:v4f32 300:i32, VR128:v4f32:$src)
15211       // Emits: (RSQRTPSr:v4f32 VR128:v4f32:$src)
15212       // Pattern complexity = 5  cost = 1
15213       if (CN0 == 300) {
15214         N1 = N.getOperand(1);
15215         Select(Tmp1, N1);
15216         if (N.Val->hasOneUse()) {
15217           Result = CurDAG->SelectNodeTo(N.Val, X86::RSQRTPSr, MVT::v4f32, Tmp1);
15218         } else {
15219           ResNode = CurDAG->getTargetNode(X86::RSQRTPSr, MVT::v4f32, Tmp1);
15220           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15221           Result = SDOperand(ResNode, 0);
15222         }
15223         return;
15224       }
15225 
15226       // Pattern: (intrinsic_wo_chain:v4f32 298:i32, VR128:v4f32:$src)
15227       // Emits: (RCPPSr:v4f32 VR128:v4f32:$src)
15228       // Pattern complexity = 5  cost = 1
15229       if (CN0 == 298) {
15230         N1 = N.getOperand(1);
15231         Select(Tmp1, N1);
15232         if (N.Val->hasOneUse()) {
15233           Result = CurDAG->SelectNodeTo(N.Val, X86::RCPPSr, MVT::v4f32, Tmp1);
15234         } else {
15235           ResNode = CurDAG->getTargetNode(X86::RCPPSr, MVT::v4f32, Tmp1);
15236           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15237           Result = SDOperand(ResNode, 0);
15238         }
15239         return;
15240       }
15241 
15242       // Pattern: (intrinsic_wo_chain:v4f32 291:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
15243       // Emits: (MAXPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15244       // Pattern complexity = 5  cost = 1
15245       if (CN0 == 291) {
15246         N1 = N.getOperand(1);
15247         N2 = N.getOperand(2);
15248         Select(Tmp1, N1);
15249         Select(Tmp2, N2);
15250         if (N.Val->hasOneUse()) {
15251           Result = CurDAG->SelectNodeTo(N.Val, X86::MAXPSrr, MVT::v4f32, Tmp1, Tmp2);
15252         } else {
15253           ResNode = CurDAG->getTargetNode(X86::MAXPSrr, MVT::v4f32, Tmp1, Tmp2);
15254           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15255           Result = SDOperand(ResNode, 0);
15256         }
15257         return;
15258       }
15259     }
15260   }
15261 
15262   // Pattern: (intrinsic_wo_chain:v2f64 221:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
15263   // Emits: (MAXPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15264   // Pattern complexity = 5  cost = 1
15265   if ((Subtarget->hasSSE2())) {
15266     N0 = N.getOperand(0);
15267     if (isa<ConstantSDNode>(N0)) {
15268       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15269       if (CN0 == 221) {
15270         N1 = N.getOperand(1);
15271         N2 = N.getOperand(2);
15272         Select(Tmp1, N1);
15273         Select(Tmp2, N2);
15274         if (N.Val->hasOneUse()) {
15275           Result = CurDAG->SelectNodeTo(N.Val, X86::MAXPDrr, MVT::v2f64, Tmp1, Tmp2);
15276         } else {
15277           ResNode = CurDAG->getTargetNode(X86::MAXPDrr, MVT::v2f64, Tmp1, Tmp2);
15278           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15279           Result = SDOperand(ResNode, 0);
15280         }
15281         return;
15282       }
15283     }
15284   }
15285 
15286   // Pattern: (intrinsic_wo_chain:v4f32 293:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
15287   // Emits: (MINPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15288   // Pattern complexity = 5  cost = 1
15289   if ((Subtarget->hasSSE1())) {
15290     N0 = N.getOperand(0);
15291     if (isa<ConstantSDNode>(N0)) {
15292       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15293       if (CN0 == 293) {
15294         N1 = N.getOperand(1);
15295         N2 = N.getOperand(2);
15296         Select(Tmp1, N1);
15297         Select(Tmp2, N2);
15298         if (N.Val->hasOneUse()) {
15299           Result = CurDAG->SelectNodeTo(N.Val, X86::MINPSrr, MVT::v4f32, Tmp1, Tmp2);
15300         } else {
15301           ResNode = CurDAG->getTargetNode(X86::MINPSrr, MVT::v4f32, Tmp1, Tmp2);
15302           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15303           Result = SDOperand(ResNode, 0);
15304         }
15305         return;
15306       }
15307     }
15308   }
15309 
15310   // Pattern: (intrinsic_wo_chain:v2f64 223:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
15311   // Emits: (MINPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15312   // Pattern complexity = 5  cost = 1
15313   if ((Subtarget->hasSSE2())) {
15314     N0 = N.getOperand(0);
15315     if (isa<ConstantSDNode>(N0)) {
15316       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15317       if (CN0 == 223) {
15318         N1 = N.getOperand(1);
15319         N2 = N.getOperand(2);
15320         Select(Tmp1, N1);
15321         Select(Tmp2, N2);
15322         if (N.Val->hasOneUse()) {
15323           Result = CurDAG->SelectNodeTo(N.Val, X86::MINPDrr, MVT::v2f64, Tmp1, Tmp2);
15324         } else {
15325           ResNode = CurDAG->getTargetNode(X86::MINPDrr, MVT::v2f64, Tmp1, Tmp2);
15326           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15327           Result = SDOperand(ResNode, 0);
15328         }
15329         return;
15330       }
15331     }
15332   }
15333   if ((Subtarget->hasSSE3())) {
15334     N0 = N.getOperand(0);
15335     if (isa<ConstantSDNode>(N0)) {
15336       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15337 
15338       // Pattern: (intrinsic_wo_chain:v4f32 273:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
15339       // Emits: (HADDPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15340       // Pattern complexity = 5  cost = 1
15341       if (CN0 == 273) {
15342         N1 = N.getOperand(1);
15343         N2 = N.getOperand(2);
15344         Select(Tmp1, N1);
15345         Select(Tmp2, N2);
15346         if (N.Val->hasOneUse()) {
15347           Result = CurDAG->SelectNodeTo(N.Val, X86::HADDPSrr, MVT::v4f32, Tmp1, Tmp2);
15348         } else {
15349           ResNode = CurDAG->getTargetNode(X86::HADDPSrr, MVT::v4f32, Tmp1, Tmp2);
15350           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15351           Result = SDOperand(ResNode, 0);
15352         }
15353         return;
15354       }
15355 
15356       // Pattern: (intrinsic_wo_chain:v2f64 272:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
15357       // Emits: (HADDPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15358       // Pattern complexity = 5  cost = 1
15359       if (CN0 == 272) {
15360         N1 = N.getOperand(1);
15361         N2 = N.getOperand(2);
15362         Select(Tmp1, N1);
15363         Select(Tmp2, N2);
15364         if (N.Val->hasOneUse()) {
15365           Result = CurDAG->SelectNodeTo(N.Val, X86::HADDPDrr, MVT::v2f64, Tmp1, Tmp2);
15366         } else {
15367           ResNode = CurDAG->getTargetNode(X86::HADDPDrr, MVT::v2f64, Tmp1, Tmp2);
15368           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15369           Result = SDOperand(ResNode, 0);
15370         }
15371         return;
15372       }
15373 
15374       // Pattern: (intrinsic_wo_chain:v4f32 275:i32, VR128:v4f32:$src1, VR128:v4f32:$src2)
15375       // Emits: (HSUBPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
15376       // Pattern complexity = 5  cost = 1
15377       if (CN0 == 275) {
15378         N1 = N.getOperand(1);
15379         N2 = N.getOperand(2);
15380         Select(Tmp1, N1);
15381         Select(Tmp2, N2);
15382         if (N.Val->hasOneUse()) {
15383           Result = CurDAG->SelectNodeTo(N.Val, X86::HSUBPSrr, MVT::v4f32, Tmp1, Tmp2);
15384         } else {
15385           ResNode = CurDAG->getTargetNode(X86::HSUBPSrr, MVT::v4f32, Tmp1, Tmp2);
15386           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15387           Result = SDOperand(ResNode, 0);
15388         }
15389         return;
15390       }
15391 
15392       // Pattern: (intrinsic_wo_chain:v2f64 274:i32, VR128:v2f64:$src1, VR128:v2f64:$src2)
15393       // Emits: (HSUBPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
15394       // Pattern complexity = 5  cost = 1
15395       if (CN0 == 274) {
15396         N1 = N.getOperand(1);
15397         N2 = N.getOperand(2);
15398         Select(Tmp1, N1);
15399         Select(Tmp2, N2);
15400         if (N.Val->hasOneUse()) {
15401           Result = CurDAG->SelectNodeTo(N.Val, X86::HSUBPDrr, MVT::v2f64, Tmp1, Tmp2);
15402         } else {
15403           ResNode = CurDAG->getTargetNode(X86::HSUBPDrr, MVT::v2f64, Tmp1, Tmp2);
15404           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15405           Result = SDOperand(ResNode, 0);
15406         }
15407         return;
15408       }
15409     }
15410   }
15411   if ((Subtarget->hasSSE2())) {
15412     N0 = N.getOperand(0);
15413     if (isa<ConstantSDNode>(N0)) {
15414       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15415 
15416       // Pattern: (intrinsic_wo_chain:v16i8 234:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15417       // Emits: (PADDSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15418       // Pattern complexity = 5  cost = 1
15419       if (CN0 == 234) {
15420         N1 = N.getOperand(1);
15421         N2 = N.getOperand(2);
15422         Select(Tmp1, N1);
15423         Select(Tmp2, N2);
15424         if (N.Val->hasOneUse()) {
15425           Result = CurDAG->SelectNodeTo(N.Val, X86::PADDSBrr, MVT::v16i8, Tmp1, Tmp2);
15426         } else {
15427           ResNode = CurDAG->getTargetNode(X86::PADDSBrr, MVT::v16i8, Tmp1, Tmp2);
15428           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15429           Result = SDOperand(ResNode, 0);
15430         }
15431         return;
15432       }
15433 
15434       // Pattern: (intrinsic_wo_chain:v8i16 235:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15435       // Emits: (PADDSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15436       // Pattern complexity = 5  cost = 1
15437       if (CN0 == 235) {
15438         N1 = N.getOperand(1);
15439         N2 = N.getOperand(2);
15440         Select(Tmp1, N1);
15441         Select(Tmp2, N2);
15442         if (N.Val->hasOneUse()) {
15443           Result = CurDAG->SelectNodeTo(N.Val, X86::PADDSWrr, MVT::v8i16, Tmp1, Tmp2);
15444         } else {
15445           ResNode = CurDAG->getTargetNode(X86::PADDSWrr, MVT::v8i16, Tmp1, Tmp2);
15446           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15447           Result = SDOperand(ResNode, 0);
15448         }
15449         return;
15450       }
15451 
15452       // Pattern: (intrinsic_wo_chain:v16i8 236:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15453       // Emits: (PADDUSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15454       // Pattern complexity = 5  cost = 1
15455       if (CN0 == 236) {
15456         N1 = N.getOperand(1);
15457         N2 = N.getOperand(2);
15458         Select(Tmp1, N1);
15459         Select(Tmp2, N2);
15460         if (N.Val->hasOneUse()) {
15461           Result = CurDAG->SelectNodeTo(N.Val, X86::PADDUSBrr, MVT::v16i8, Tmp1, Tmp2);
15462         } else {
15463           ResNode = CurDAG->getTargetNode(X86::PADDUSBrr, MVT::v16i8, Tmp1, Tmp2);
15464           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15465           Result = SDOperand(ResNode, 0);
15466         }
15467         return;
15468       }
15469 
15470       // Pattern: (intrinsic_wo_chain:v8i16 237:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15471       // Emits: (PADDUSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15472       // Pattern complexity = 5  cost = 1
15473       if (CN0 == 237) {
15474         N1 = N.getOperand(1);
15475         N2 = N.getOperand(2);
15476         Select(Tmp1, N1);
15477         Select(Tmp2, N2);
15478         if (N.Val->hasOneUse()) {
15479           Result = CurDAG->SelectNodeTo(N.Val, X86::PADDUSWrr, MVT::v8i16, Tmp1, Tmp2);
15480         } else {
15481           ResNode = CurDAG->getTargetNode(X86::PADDUSWrr, MVT::v8i16, Tmp1, Tmp2);
15482           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15483           Result = SDOperand(ResNode, 0);
15484         }
15485         return;
15486       }
15487 
15488       // Pattern: (intrinsic_wo_chain:v16i8 252:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15489       // Emits: (PSUBSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15490       // Pattern complexity = 5  cost = 1
15491       if (CN0 == 252) {
15492         N1 = N.getOperand(1);
15493         N2 = N.getOperand(2);
15494         Select(Tmp1, N1);
15495         Select(Tmp2, N2);
15496         if (N.Val->hasOneUse()) {
15497           Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBSBrr, MVT::v16i8, Tmp1, Tmp2);
15498         } else {
15499           ResNode = CurDAG->getTargetNode(X86::PSUBSBrr, MVT::v16i8, Tmp1, Tmp2);
15500           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15501           Result = SDOperand(ResNode, 0);
15502         }
15503         return;
15504       }
15505 
15506       // Pattern: (intrinsic_wo_chain:v8i16 253:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15507       // Emits: (PSUBSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15508       // Pattern complexity = 5  cost = 1
15509       if (CN0 == 253) {
15510         N1 = N.getOperand(1);
15511         N2 = N.getOperand(2);
15512         Select(Tmp1, N1);
15513         Select(Tmp2, N2);
15514         if (N.Val->hasOneUse()) {
15515           Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBSWrr, MVT::v8i16, Tmp1, Tmp2);
15516         } else {
15517           ResNode = CurDAG->getTargetNode(X86::PSUBSWrr, MVT::v8i16, Tmp1, Tmp2);
15518           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15519           Result = SDOperand(ResNode, 0);
15520         }
15521         return;
15522       }
15523 
15524       // Pattern: (intrinsic_wo_chain:v16i8 254:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15525       // Emits: (PSUBUSBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15526       // Pattern complexity = 5  cost = 1
15527       if (CN0 == 254) {
15528         N1 = N.getOperand(1);
15529         N2 = N.getOperand(2);
15530         Select(Tmp1, N1);
15531         Select(Tmp2, N2);
15532         if (N.Val->hasOneUse()) {
15533           Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBUSBrr, MVT::v16i8, Tmp1, Tmp2);
15534         } else {
15535           ResNode = CurDAG->getTargetNode(X86::PSUBUSBrr, MVT::v16i8, Tmp1, Tmp2);
15536           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15537           Result = SDOperand(ResNode, 0);
15538         }
15539         return;
15540       }
15541 
15542       // Pattern: (intrinsic_wo_chain:v8i16 255:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15543       // Emits: (PSUBUSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15544       // Pattern complexity = 5  cost = 1
15545       if (CN0 == 255) {
15546         N1 = N.getOperand(1);
15547         N2 = N.getOperand(2);
15548         Select(Tmp1, N1);
15549         Select(Tmp2, N2);
15550         if (N.Val->hasOneUse()) {
15551           Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBUSWrr, MVT::v8i16, Tmp1, Tmp2);
15552         } else {
15553           ResNode = CurDAG->getTargetNode(X86::PSUBUSWrr, MVT::v8i16, Tmp1, Tmp2);
15554           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15555           Result = SDOperand(ResNode, 0);
15556         }
15557         return;
15558       }
15559 
15560       // Pattern: (intrinsic_wo_chain:v8i16 247:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15561       // Emits: (PMULHUWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15562       // Pattern complexity = 5  cost = 1
15563       if (CN0 == 247) {
15564         N1 = N.getOperand(1);
15565         N2 = N.getOperand(2);
15566         Select(Tmp1, N1);
15567         Select(Tmp2, N2);
15568         if (N.Val->hasOneUse()) {
15569           Result = CurDAG->SelectNodeTo(N.Val, X86::PMULHUWrr, MVT::v8i16, Tmp1, Tmp2);
15570         } else {
15571           ResNode = CurDAG->getTargetNode(X86::PMULHUWrr, MVT::v8i16, Tmp1, Tmp2);
15572           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15573           Result = SDOperand(ResNode, 0);
15574         }
15575         return;
15576       }
15577 
15578       // Pattern: (intrinsic_wo_chain:v8i16 246:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15579       // Emits: (PMULHWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15580       // Pattern complexity = 5  cost = 1
15581       if (CN0 == 246) {
15582         N1 = N.getOperand(1);
15583         N2 = N.getOperand(2);
15584         Select(Tmp1, N1);
15585         Select(Tmp2, N2);
15586         if (N.Val->hasOneUse()) {
15587           Result = CurDAG->SelectNodeTo(N.Val, X86::PMULHWrr, MVT::v8i16, Tmp1, Tmp2);
15588         } else {
15589           ResNode = CurDAG->getTargetNode(X86::PMULHWrr, MVT::v8i16, Tmp1, Tmp2);
15590           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15591           Result = SDOperand(ResNode, 0);
15592         }
15593         return;
15594       }
15595 
15596       // Pattern: (intrinsic_wo_chain:v2i64 248:i32, VR128:v4i32:$src1, VR128:v4i32:$src2)
15597       // Emits: (PMULUDQrr:v2i64 VR128:v4i32:$src1, VR128:v4i32:$src2)
15598       // Pattern complexity = 5  cost = 1
15599       if (CN0 == 248) {
15600         N1 = N.getOperand(1);
15601         N2 = N.getOperand(2);
15602         Select(Tmp1, N1);
15603         Select(Tmp2, N2);
15604         if (N.Val->hasOneUse()) {
15605           Result = CurDAG->SelectNodeTo(N.Val, X86::PMULUDQrr, MVT::v2i64, Tmp1, Tmp2);
15606         } else {
15607           ResNode = CurDAG->getTargetNode(X86::PMULUDQrr, MVT::v2i64, Tmp1, Tmp2);
15608           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15609           Result = SDOperand(ResNode, 0);
15610         }
15611         return;
15612       }
15613 
15614       // Pattern: (intrinsic_wo_chain:v4i32 240:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15615       // Emits: (PMADDWDrr:v4i32 VR128:v8i16:$src1, VR128:v8i16:$src2)
15616       // Pattern complexity = 5  cost = 1
15617       if (CN0 == 240) {
15618         N1 = N.getOperand(1);
15619         N2 = N.getOperand(2);
15620         Select(Tmp1, N1);
15621         Select(Tmp2, N2);
15622         if (N.Val->hasOneUse()) {
15623           Result = CurDAG->SelectNodeTo(N.Val, X86::PMADDWDrr, MVT::v4i32, Tmp1, Tmp2);
15624         } else {
15625           ResNode = CurDAG->getTargetNode(X86::PMADDWDrr, MVT::v4i32, Tmp1, Tmp2);
15626           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15627           Result = SDOperand(ResNode, 0);
15628         }
15629         return;
15630       }
15631 
15632       // Pattern: (intrinsic_wo_chain:v16i8 238:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15633       // Emits: (PAVGBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15634       // Pattern complexity = 5  cost = 1
15635       if (CN0 == 238) {
15636         N1 = N.getOperand(1);
15637         N2 = N.getOperand(2);
15638         Select(Tmp1, N1);
15639         Select(Tmp2, N2);
15640         if (N.Val->hasOneUse()) {
15641           Result = CurDAG->SelectNodeTo(N.Val, X86::PAVGBrr, MVT::v16i8, Tmp1, Tmp2);
15642         } else {
15643           ResNode = CurDAG->getTargetNode(X86::PAVGBrr, MVT::v16i8, Tmp1, Tmp2);
15644           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15645           Result = SDOperand(ResNode, 0);
15646         }
15647         return;
15648       }
15649 
15650       // Pattern: (intrinsic_wo_chain:v8i16 239:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15651       // Emits: (PAVGWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15652       // Pattern complexity = 5  cost = 1
15653       if (CN0 == 239) {
15654         N1 = N.getOperand(1);
15655         N2 = N.getOperand(2);
15656         Select(Tmp1, N1);
15657         Select(Tmp2, N2);
15658         if (N.Val->hasOneUse()) {
15659           Result = CurDAG->SelectNodeTo(N.Val, X86::PAVGWrr, MVT::v8i16, Tmp1, Tmp2);
15660         } else {
15661           ResNode = CurDAG->getTargetNode(X86::PAVGWrr, MVT::v8i16, Tmp1, Tmp2);
15662           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15663           Result = SDOperand(ResNode, 0);
15664         }
15665         return;
15666       }
15667 
15668       // Pattern: (intrinsic_wo_chain:v16i8 242:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15669       // Emits: (PMAXUBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15670       // Pattern complexity = 5  cost = 1
15671       if (CN0 == 242) {
15672         N1 = N.getOperand(1);
15673         N2 = N.getOperand(2);
15674         Select(Tmp1, N1);
15675         Select(Tmp2, N2);
15676         if (N.Val->hasOneUse()) {
15677           Result = CurDAG->SelectNodeTo(N.Val, X86::PMAXUBrr, MVT::v16i8, Tmp1, Tmp2);
15678         } else {
15679           ResNode = CurDAG->getTargetNode(X86::PMAXUBrr, MVT::v16i8, Tmp1, Tmp2);
15680           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15681           Result = SDOperand(ResNode, 0);
15682         }
15683         return;
15684       }
15685 
15686       // Pattern: (intrinsic_wo_chain:v8i16 241:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15687       // Emits: (PMAXSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15688       // Pattern complexity = 5  cost = 1
15689       if (CN0 == 241) {
15690         N1 = N.getOperand(1);
15691         N2 = N.getOperand(2);
15692         Select(Tmp1, N1);
15693         Select(Tmp2, N2);
15694         if (N.Val->hasOneUse()) {
15695           Result = CurDAG->SelectNodeTo(N.Val, X86::PMAXSWrr, MVT::v8i16, Tmp1, Tmp2);
15696         } else {
15697           ResNode = CurDAG->getTargetNode(X86::PMAXSWrr, MVT::v8i16, Tmp1, Tmp2);
15698           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15699           Result = SDOperand(ResNode, 0);
15700         }
15701         return;
15702       }
15703 
15704       // Pattern: (intrinsic_wo_chain:v16i8 244:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15705       // Emits: (PMINUBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
15706       // Pattern complexity = 5  cost = 1
15707       if (CN0 == 244) {
15708         N1 = N.getOperand(1);
15709         N2 = N.getOperand(2);
15710         Select(Tmp1, N1);
15711         Select(Tmp2, N2);
15712         if (N.Val->hasOneUse()) {
15713           Result = CurDAG->SelectNodeTo(N.Val, X86::PMINUBrr, MVT::v16i8, Tmp1, Tmp2);
15714         } else {
15715           ResNode = CurDAG->getTargetNode(X86::PMINUBrr, MVT::v16i8, Tmp1, Tmp2);
15716           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15717           Result = SDOperand(ResNode, 0);
15718         }
15719         return;
15720       }
15721 
15722       // Pattern: (intrinsic_wo_chain:v8i16 243:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15723       // Emits: (PMINSWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15724       // Pattern complexity = 5  cost = 1
15725       if (CN0 == 243) {
15726         N1 = N.getOperand(1);
15727         N2 = N.getOperand(2);
15728         Select(Tmp1, N1);
15729         Select(Tmp2, N2);
15730         if (N.Val->hasOneUse()) {
15731           Result = CurDAG->SelectNodeTo(N.Val, X86::PMINSWrr, MVT::v8i16, Tmp1, Tmp2);
15732         } else {
15733           ResNode = CurDAG->getTargetNode(X86::PMINSWrr, MVT::v8i16, Tmp1, Tmp2);
15734           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15735           Result = SDOperand(ResNode, 0);
15736         }
15737         return;
15738       }
15739 
15740       // Pattern: (intrinsic_wo_chain:v2i64 249:i32, VR128:v16i8:$src1, VR128:v16i8:$src2)
15741       // Emits: (PSADBWrr:v2i64 VR128:v16i8:$src1, VR128:v16i8:$src2)
15742       // Pattern complexity = 5  cost = 1
15743       if (CN0 == 249) {
15744         N1 = N.getOperand(1);
15745         N2 = N.getOperand(2);
15746         Select(Tmp1, N1);
15747         Select(Tmp2, N2);
15748         if (N.Val->hasOneUse()) {
15749           Result = CurDAG->SelectNodeTo(N.Val, X86::PSADBWrr, MVT::v2i64, Tmp1, Tmp2);
15750         } else {
15751           ResNode = CurDAG->getTargetNode(X86::PSADBWrr, MVT::v2i64, Tmp1, Tmp2);
15752           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15753           Result = SDOperand(ResNode, 0);
15754         }
15755         return;
15756       }
15757 
15758       // Pattern: (intrinsic_wo_chain:v8i16 232:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15759       // Emits: (PACKSSWBrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15760       // Pattern complexity = 5  cost = 1
15761       if (CN0 == 232) {
15762         N1 = N.getOperand(1);
15763         N2 = N.getOperand(2);
15764         Select(Tmp1, N1);
15765         Select(Tmp2, N2);
15766         if (N.Val->hasOneUse()) {
15767           Result = CurDAG->SelectNodeTo(N.Val, X86::PACKSSWBrr, MVT::v8i16, Tmp1, Tmp2);
15768         } else {
15769           ResNode = CurDAG->getTargetNode(X86::PACKSSWBrr, MVT::v8i16, Tmp1, Tmp2);
15770           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15771           Result = SDOperand(ResNode, 0);
15772         }
15773         return;
15774       }
15775 
15776       // Pattern: (intrinsic_wo_chain:v4i32 231:i32, VR128:v4i32:$src1, VR128:v4i32:$src2)
15777       // Emits: (PACKSSDWrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
15778       // Pattern complexity = 5  cost = 1
15779       if (CN0 == 231) {
15780         N1 = N.getOperand(1);
15781         N2 = N.getOperand(2);
15782         Select(Tmp1, N1);
15783         Select(Tmp2, N2);
15784         if (N.Val->hasOneUse()) {
15785           Result = CurDAG->SelectNodeTo(N.Val, X86::PACKSSDWrr, MVT::v4i32, Tmp1, Tmp2);
15786         } else {
15787           ResNode = CurDAG->getTargetNode(X86::PACKSSDWrr, MVT::v4i32, Tmp1, Tmp2);
15788           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15789           Result = SDOperand(ResNode, 0);
15790         }
15791         return;
15792       }
15793 
15794       // Pattern: (intrinsic_wo_chain:v8i16 233:i32, VR128:v8i16:$src1, VR128:v8i16:$src2)
15795       // Emits: (PACKUSWBrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
15796       // Pattern complexity = 5  cost = 1
15797       if (CN0 == 233) {
15798         N1 = N.getOperand(1);
15799         N2 = N.getOperand(2);
15800         Select(Tmp1, N1);
15801         Select(Tmp2, N2);
15802         if (N.Val->hasOneUse()) {
15803           Result = CurDAG->SelectNodeTo(N.Val, X86::PACKUSWBrr, MVT::v8i16, Tmp1, Tmp2);
15804         } else {
15805           ResNode = CurDAG->getTargetNode(X86::PACKUSWBrr, MVT::v8i16, Tmp1, Tmp2);
15806           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15807           Result = SDOperand(ResNode, 0);
15808         }
15809         return;
15810       }
15811     }
15812   }
15813   if ((Subtarget->hasSSE1())) {
15814     N0 = N.getOperand(0);
15815     if (isa<ConstantSDNode>(N0)) {
15816       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15817 
15818       // Pattern: (intrinsic_wo_chain:i32 295:i32, VR128:v4f32:$src)
15819       // Emits: (MOVMSKPSrr:i32 VR128:v4f32:$src)
15820       // Pattern complexity = 5  cost = 1
15821       if (CN0 == 295) {
15822         N1 = N.getOperand(1);
15823         Select(Tmp1, N1);
15824         if (N.Val->hasOneUse()) {
15825           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVMSKPSrr, MVT::i32, Tmp1);
15826         } else {
15827           ResNode = CurDAG->getTargetNode(X86::MOVMSKPSrr, MVT::i32, Tmp1);
15828           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15829           Result = SDOperand(ResNode, 0);
15830         }
15831         return;
15832       }
15833 
15834       // Pattern: (intrinsic_wo_chain:i32 226:i32, VR128:v2f64:$src)
15835       // Emits: (MOVMSKPDrr:i32 VR128:v2f64:$src)
15836       // Pattern complexity = 5  cost = 1
15837       if (CN0 == 226) {
15838         N1 = N.getOperand(1);
15839         Select(Tmp1, N1);
15840         if (N.Val->hasOneUse()) {
15841           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVMSKPDrr, MVT::i32, Tmp1);
15842         } else {
15843           ResNode = CurDAG->getTargetNode(X86::MOVMSKPDrr, MVT::i32, Tmp1);
15844           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15845           Result = SDOperand(ResNode, 0);
15846         }
15847         return;
15848       }
15849     }
15850   }
15851   if ((Subtarget->hasSSE2())) {
15852     N0 = N.getOperand(0);
15853     if (isa<ConstantSDNode>(N0)) {
15854       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
15855 
15856       // Pattern: (intrinsic_wo_chain:i32 245:i32, VR128:v16i8:$src)
15857       // Emits: (PMOVMSKBrr:i32 VR128:v16i8:$src)
15858       // Pattern complexity = 5  cost = 1
15859       if (CN0 == 245) {
15860         N1 = N.getOperand(1);
15861         Select(Tmp1, N1);
15862         if (N.Val->hasOneUse()) {
15863           Result = CurDAG->SelectNodeTo(N.Val, X86::PMOVMSKBrr, MVT::i32, Tmp1);
15864         } else {
15865           ResNode = CurDAG->getTargetNode(X86::PMOVMSKBrr, MVT::i32, Tmp1);
15866           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15867           Result = SDOperand(ResNode, 0);
15868         }
15869         return;
15870       }
15871 
15872       // Pattern: (intrinsic_wo_chain:v4i32 225:i32, VR128:v4i32:$src)
15873       // Emits: (MOVLQ128rr:v4i32 VR128:v4i32:$src)
15874       // Pattern complexity = 5  cost = 1
15875       if (CN0 == 225) {
15876         N1 = N.getOperand(1);
15877         Select(Tmp1, N1);
15878         if (N.Val->hasOneUse()) {
15879           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLQ128rr, MVT::v4i32, Tmp1);
15880         } else {
15881           ResNode = CurDAG->getTargetNode(X86::MOVLQ128rr, MVT::v4i32, Tmp1);
15882           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
15883           Result = SDOperand(ResNode, 0);
15884         }
15885         return;
15886       }
15887     }
15888   }
15889   std::cerr << "Cannot yet select: ";
15890   unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
15891   std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
15892   std::cerr << '\n';
15893   abort();
15894 }
15895 
15896 void Select_load(SDOperand &Result, SDOperand N) {
15897   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
15898     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
15899     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
15900     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
15901     Result = Dummy;
15902     return;
15903   }
15904   SDOperand Chain(0, 0);
15905   SDOperand N1(0, 0);
15906   SDOperand Tmp0(0, 0);
15907   SDOperand Tmp1(0, 0);
15908   SDOperand Tmp2(0, 0);
15909   SDOperand Tmp3(0, 0);
15910   SDNode *ResNode;
15911   {
15912     Chain = N.getOperand(0);
15913     N1 = N.getOperand(1);
15914 
15915     // Pattern: (load:i8 addr:i32:$src)
15916     // Emits: (MOV8rm:i8 addr:i32:$src)
15917     // Pattern complexity = 12  cost = 1
15918     if (N.Val->getValueType(0) == MVT::i8 &&
15919         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
15920       Select(Tmp0, Tmp0);
15921       Select(Tmp1, Tmp1);
15922       Select(Tmp2, Tmp2);
15923       Select(Tmp3, Tmp3);
15924       Select(Chain, Chain);
15925       ResNode = CurDAG->getTargetNode(X86::MOV8rm, MVT::i8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
15926       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
15927       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
15928       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
15929       Result = SDOperand(ResNode, N.ResNo);
15930       return;
15931     }
15932 
15933     // Pattern: (load:i16 addr:i32:$src)
15934     // Emits: (MOV16rm:i16 addr:i32:$src)
15935     // Pattern complexity = 12  cost = 1
15936     if (N.Val->getValueType(0) == MVT::i16 &&
15937         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
15938       Select(Tmp0, Tmp0);
15939       Select(Tmp1, Tmp1);
15940       Select(Tmp2, Tmp2);
15941       Select(Tmp3, Tmp3);
15942       Select(Chain, Chain);
15943       ResNode = CurDAG->getTargetNode(X86::MOV16rm, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
15944       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
15945       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
15946       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
15947       Result = SDOperand(ResNode, N.ResNo);
15948       return;
15949     }
15950 
15951     // Pattern: (load:i32 addr:i32:$src)
15952     // Emits: (MOV32rm:i32 addr:i32:$src)
15953     // Pattern complexity = 12  cost = 1
15954     if (N.Val->getValueType(0) == MVT::i32 &&
15955         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
15956       Select(Tmp0, Tmp0);
15957       Select(Tmp1, Tmp1);
15958       Select(Tmp2, Tmp2);
15959       Select(Tmp3, Tmp3);
15960       Select(Chain, Chain);
15961       ResNode = CurDAG->getTargetNode(X86::MOV32rm, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
15962       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
15963       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
15964       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
15965       Result = SDOperand(ResNode, N.ResNo);
15966       return;
15967     }
15968   }
15969 
15970   // Pattern: (load:f64 addr:i32:$src)
15971   // Emits: (FpLD64m:f64 addr:i32:$src)
15972   // Pattern complexity = 12  cost = 1
15973   if ((!Subtarget->hasSSE2())) {
15974     Chain = N.getOperand(0);
15975     N1 = N.getOperand(1);
15976     if (N.Val->getValueType(0) == MVT::f64 &&
15977         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
15978       Select(Tmp0, Tmp0);
15979       Select(Tmp1, Tmp1);
15980       Select(Tmp2, Tmp2);
15981       Select(Tmp3, Tmp3);
15982       Select(Chain, Chain);
15983       ResNode = CurDAG->getTargetNode(X86::FpLD64m, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
15984       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
15985       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
15986       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
15987       Result = SDOperand(ResNode, N.ResNo);
15988       return;
15989     }
15990   }
15991 
15992   // Pattern: (load:f32 addr:i32:$src)
15993   // Emits: (MOVSSrm:f32 addr:i32:$src)
15994   // Pattern complexity = 12  cost = 1
15995   if ((Subtarget->hasSSE1())) {
15996     Chain = N.getOperand(0);
15997     N1 = N.getOperand(1);
15998     if (N.Val->getValueType(0) == MVT::f32 &&
15999         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
16000       Select(Tmp0, Tmp0);
16001       Select(Tmp1, Tmp1);
16002       Select(Tmp2, Tmp2);
16003       Select(Tmp3, Tmp3);
16004       Select(Chain, Chain);
16005       ResNode = CurDAG->getTargetNode(X86::MOVSSrm, MVT::f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
16006       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16007       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
16008       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
16009       Result = SDOperand(ResNode, N.ResNo);
16010       return;
16011     }
16012   }
16013 
16014   // Pattern: (load:f64 addr:i32:$src)
16015   // Emits: (MOVSDrm:f64 addr:i32:$src)
16016   // Pattern complexity = 12  cost = 1
16017   if ((Subtarget->hasSSE2())) {
16018     Chain = N.getOperand(0);
16019     N1 = N.getOperand(1);
16020     if (N.Val->getValueType(0) == MVT::f64 &&
16021         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
16022       Select(Tmp0, Tmp0);
16023       Select(Tmp1, Tmp1);
16024       Select(Tmp2, Tmp2);
16025       Select(Tmp3, Tmp3);
16026       Select(Chain, Chain);
16027       ResNode = CurDAG->getTargetNode(X86::MOVSDrm, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
16028       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16029       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
16030       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
16031       Result = SDOperand(ResNode, N.ResNo);
16032       return;
16033     }
16034   }
16035 
16036   // Pattern: (load:v4f32 addr:i32:$src)
16037   // Emits: (MOVAPSrm:v4f32 addr:i32:$src)
16038   // Pattern complexity = 12  cost = 1
16039   if ((Subtarget->hasSSE1())) {
16040     Chain = N.getOperand(0);
16041     N1 = N.getOperand(1);
16042     if (N.Val->getValueType(0) == MVT::v4f32 &&
16043         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
16044       Select(Tmp0, Tmp0);
16045       Select(Tmp1, Tmp1);
16046       Select(Tmp2, Tmp2);
16047       Select(Tmp3, Tmp3);
16048       Select(Chain, Chain);
16049       ResNode = CurDAG->getTargetNode(X86::MOVAPSrm, MVT::v4f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
16050       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16051       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
16052       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
16053       Result = SDOperand(ResNode, N.ResNo);
16054       return;
16055     }
16056   }
16057   if ((Subtarget->hasSSE2())) {
16058     Chain = N.getOperand(0);
16059     N1 = N.getOperand(1);
16060 
16061     // Pattern: (load:v2f64 addr:i32:$src)
16062     // Emits: (MOVAPDrm:v2f64 addr:i32:$src)
16063     // Pattern complexity = 12  cost = 1
16064     if (N.Val->getValueType(0) == MVT::v2f64 &&
16065         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
16066       Select(Tmp0, Tmp0);
16067       Select(Tmp1, Tmp1);
16068       Select(Tmp2, Tmp2);
16069       Select(Tmp3, Tmp3);
16070       Select(Chain, Chain);
16071       ResNode = CurDAG->getTargetNode(X86::MOVAPDrm, MVT::v2f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
16072       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16073       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
16074       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
16075       Result = SDOperand(ResNode, N.ResNo);
16076       return;
16077     }
16078 
16079     // Pattern: (load:v2i64 addr:i32:$src)
16080     // Emits: (MOVDQArm:v2i64 addr:i32:$src)
16081     // Pattern complexity = 12  cost = 1
16082     if (N.Val->getValueType(0) == MVT::v2i64 &&
16083         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
16084       Select(Tmp0, Tmp0);
16085       Select(Tmp1, Tmp1);
16086       Select(Tmp2, Tmp2);
16087       Select(Tmp3, Tmp3);
16088       Select(Chain, Chain);
16089       ResNode = CurDAG->getTargetNode(X86::MOVDQArm, MVT::v2i64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
16090       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16091       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
16092       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
16093       Result = SDOperand(ResNode, N.ResNo);
16094       return;
16095     }
16096   }
16097   std::cerr << "Cannot yet select: ";
16098   N.Val->dump(CurDAG);
16099   std::cerr << '\n';
16100   abort();
16101 }
16102 
16103 void Select_mul(SDOperand &Result, SDOperand N) {
16104   SDOperand Chain(0, 0);
16105   SDOperand Chain0(0, 0);
16106   SDOperand Chain00(0, 0);
16107   SDOperand Chain1(0, 0);
16108   SDOperand Chain10(0, 0);
16109   SDOperand InFlag(0, 0);
16110   SDOperand N0(0, 0);
16111   SDOperand N00(0, 0);
16112   SDOperand N001(0, 0);
16113   SDOperand N01(0, 0);
16114   SDOperand N1(0, 0);
16115   SDOperand N10(0, 0);
16116   SDOperand N101(0, 0);
16117   SDOperand N11(0, 0);
16118   SDOperand Tmp0(0, 0);
16119   SDOperand Tmp1(0, 0);
16120   SDOperand Tmp2(0, 0);
16121   SDOperand Tmp3(0, 0);
16122   SDOperand Tmp4(0, 0);
16123   SDNode *ResNode;
16124   {
16125     N0 = N.getOperand(0);
16126     if (N0.getOpcode() == ISD::LOAD &&
16127         N0.hasOneUse() &&
16128         !CodeGenMap.count(N0.getValue(0)) &&
16129         !isNonImmUse(N.Val, N0.Val)) {
16130       Chain0 = N0.getOperand(0);
16131       N01 = N0.getOperand(1);
16132       N1 = N.getOperand(1);
16133       if (N1.getOpcode() == ISD::Constant) {
16134 
16135         // Pattern: (mul:i16 (load:i16 addr:i32:$src1), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
16136         // Emits: (IMUL16rmi8:i16 addr:i32:$src1, (imm:i16):$src2)
16137         // Pattern complexity = 17  cost = 1
16138         if (Predicate_i16immSExt8(N1.Val) &&
16139             N.Val->getValueType(0) == MVT::i16 &&
16140             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16141           Select(Tmp0, Tmp0);
16142           Select(Tmp1, Tmp1);
16143           Select(Tmp2, Tmp2);
16144           Select(Tmp3, Tmp3);
16145           unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
16146           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
16147           Select(Chain0, Chain0);
16148           ResNode = CurDAG->getTargetNode(X86::IMUL16rmi8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16149           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16150           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16151           AddHandleReplacement(N0.Val, 1, ResNode, 1);
16152           Result = SDOperand(ResNode, N.ResNo);
16153           return;
16154         }
16155 
16156         // Pattern: (mul:i32 (load:i32 addr:i32:$src1), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
16157         // Emits: (IMUL32rmi8:i32 addr:i32:$src1, (imm:i32):$src2)
16158         // Pattern complexity = 17  cost = 1
16159         if (Predicate_i32immSExt8(N1.Val) &&
16160             N.Val->getValueType(0) == MVT::i32 &&
16161             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16162           Select(Tmp0, Tmp0);
16163           Select(Tmp1, Tmp1);
16164           Select(Tmp2, Tmp2);
16165           Select(Tmp3, Tmp3);
16166           unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
16167           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
16168           Select(Chain0, Chain0);
16169           ResNode = CurDAG->getTargetNode(X86::IMUL32rmi8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16170           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16171           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16172           AddHandleReplacement(N0.Val, 1, ResNode, 1);
16173           Result = SDOperand(ResNode, N.ResNo);
16174           return;
16175         }
16176 
16177         // Pattern: (mul:i16 (load:i16 addr:i32:$src1), (imm:i16):$src2)
16178         // Emits: (IMUL16rmi:i16 addr:i32:$src1, (imm:i16):$src2)
16179         // Pattern complexity = 16  cost = 1
16180         if (N.Val->getValueType(0) == MVT::i16 &&
16181             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16182           Select(Tmp0, Tmp0);
16183           Select(Tmp1, Tmp1);
16184           Select(Tmp2, Tmp2);
16185           Select(Tmp3, Tmp3);
16186           unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
16187           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
16188           Select(Chain0, Chain0);
16189           ResNode = CurDAG->getTargetNode(X86::IMUL16rmi, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16190           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16191           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16192           AddHandleReplacement(N0.Val, 1, ResNode, 1);
16193           Result = SDOperand(ResNode, N.ResNo);
16194           return;
16195         }
16196 
16197         // Pattern: (mul:i32 (load:i32 addr:i32:$src1), (imm:i32):$src2)
16198         // Emits: (IMUL32rmi:i32 addr:i32:$src1, (imm:i32):$src2)
16199         // Pattern complexity = 16  cost = 1
16200         if (N.Val->getValueType(0) == MVT::i32 &&
16201             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16202           Select(Tmp0, Tmp0);
16203           Select(Tmp1, Tmp1);
16204           Select(Tmp2, Tmp2);
16205           Select(Tmp3, Tmp3);
16206           unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
16207           Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
16208           Select(Chain0, Chain0);
16209           ResNode = CurDAG->getTargetNode(X86::IMUL32rmi, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain0);
16210           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16211           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16212           AddHandleReplacement(N0.Val, 1, ResNode, 1);
16213           Result = SDOperand(ResNode, N.ResNo);
16214           return;
16215         }
16216       }
16217     }
16218   }
16219   if ((Subtarget->hasSSE2())) {
16220     N0 = N.getOperand(0);
16221 
16222     // Pattern: (mul:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)))
16223     // Emits: (PMULLWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
16224     // Pattern complexity = 16  cost = 1
16225     {
16226       N1 = N.getOperand(1);
16227       if (N1.getOpcode() == ISD::BIT_CONVERT &&
16228           N1.hasOneUse() &&
16229           !CodeGenMap.count(N1.getValue(0))) {
16230         N10 = N1.getOperand(0);
16231         if (N10.getOpcode() == ISD::LOAD &&
16232             N10.hasOneUse() &&
16233             !CodeGenMap.count(N10.getValue(0))) {
16234           Chain10 = N10.getOperand(0);
16235           N101 = N10.getOperand(1);
16236           if (N.Val->getValueType(0) == MVT::v8i16 &&
16237               N10.Val->getValueType(0) == MVT::v2i64 &&
16238               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
16239             Select(Tmp0, Tmp0);
16240             Select(Tmp1, Tmp1);
16241             Select(Tmp2, Tmp2);
16242             Select(Tmp3, Tmp3);
16243             Select(Tmp4, N0);
16244             Select(Chain10, Chain10);
16245             ResNode = CurDAG->getTargetNode(X86::PMULLWrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
16246             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16247             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
16248             AddHandleReplacement(N10.Val, 1, ResNode, 1);
16249             Result = SDOperand(ResNode, N.ResNo);
16250             return;
16251           }
16252         }
16253       }
16254     }
16255 
16256     // Pattern: (mul:v8i16 (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)), VR128:v8i16:$src1)
16257     // Emits: (PMULLWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
16258     // Pattern complexity = 16  cost = 1
16259     if (N0.getOpcode() == ISD::BIT_CONVERT &&
16260         N0.hasOneUse() &&
16261         !CodeGenMap.count(N0.getValue(0))) {
16262       N00 = N0.getOperand(0);
16263       if (N00.getOpcode() == ISD::LOAD &&
16264           N00.hasOneUse() &&
16265           !CodeGenMap.count(N00.getValue(0))) {
16266         Chain00 = N00.getOperand(0);
16267         N001 = N00.getOperand(1);
16268         N1 = N.getOperand(1);
16269         if (N.Val->getValueType(0) == MVT::v8i16 &&
16270             N00.Val->getValueType(0) == MVT::v2i64 &&
16271             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
16272           Select(Tmp0, Tmp0);
16273           Select(Tmp1, Tmp1);
16274           Select(Tmp2, Tmp2);
16275           Select(Tmp3, Tmp3);
16276           Select(Tmp4, N1);
16277           Select(Chain00, Chain00);
16278           ResNode = CurDAG->getTargetNode(X86::PMULLWrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
16279           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16280           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
16281           AddHandleReplacement(N00.Val, 1, ResNode, 1);
16282           Result = SDOperand(ResNode, N.ResNo);
16283           return;
16284         }
16285       }
16286     }
16287   }
16288   {
16289     N0 = N.getOperand(0);
16290     {
16291       N1 = N.getOperand(1);
16292       if (N1.getOpcode() == ISD::LOAD &&
16293           N1.hasOneUse() &&
16294           !CodeGenMap.count(N1.getValue(0)) &&
16295           !isNonImmUse(N.Val, N1.Val)) {
16296         Chain1 = N1.getOperand(0);
16297         N11 = N1.getOperand(1);
16298 
16299         // Pattern: (mul:i8 AL:i8, (load:i8 addr:i32:$src))
16300         // Emits: (MUL8m:void addr:i32:$src)
16301         // Pattern complexity = 14  cost = 1
16302         if (N.Val->getValueType(0) == MVT::i8 &&
16303             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16304           Select(Tmp0, Tmp0);
16305           Select(Tmp1, Tmp1);
16306           Select(Tmp2, Tmp2);
16307           Select(Tmp3, Tmp3);
16308           Select(Chain1, Chain1);
16309           Select(N0, N0);
16310           ResNode = CurDAG->getCopyToReg(Chain1, CurDAG->getRegister(X86::AL, MVT::i8), N0, InFlag).Val;
16311           Chain1 = SDOperand(ResNode, 0);
16312           InFlag = SDOperand(ResNode, 1);
16313           ResNode = CurDAG->getTargetNode(X86::MUL8m, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
16314           InFlag = SDOperand(ResNode, 1);
16315           ResNode = CurDAG->getCopyFromReg(Chain1, X86::AL, MVT::i8, InFlag).Val;
16316           Chain1 = SDOperand(ResNode, 1);
16317           InFlag = SDOperand(ResNode, 2);
16318           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16319           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16320           AddHandleReplacement(N1.Val, 1, ResNode, 1);
16321           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16322           if (N.ResNo < 1)
16323             Result = SDOperand(ResNode, N.ResNo);
16324           else
16325             Result = SDOperand(ResNode, N.ResNo+1);
16326           return;
16327         }
16328 
16329         // Pattern: (mul:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
16330         // Emits: (IMUL16rm:i16 R16:i16:$src1, addr:i32:$src2)
16331         // Pattern complexity = 14  cost = 1
16332         if (N.Val->getValueType(0) == MVT::i16 &&
16333             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16334           Select(Tmp0, Tmp0);
16335           Select(Tmp1, Tmp1);
16336           Select(Tmp2, Tmp2);
16337           Select(Tmp3, Tmp3);
16338           Select(Tmp4, N0);
16339           Select(Chain1, Chain1);
16340           ResNode = CurDAG->getTargetNode(X86::IMUL16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16341           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16342           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16343           AddHandleReplacement(N1.Val, 1, ResNode, 1);
16344           Result = SDOperand(ResNode, N.ResNo);
16345           return;
16346         }
16347 
16348         // Pattern: (mul:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
16349         // Emits: (IMUL32rm:i32 R32:i32:$src1, addr:i32:$src2)
16350         // Pattern complexity = 14  cost = 1
16351         if (N.Val->getValueType(0) == MVT::i32 &&
16352             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16353           Select(Tmp0, Tmp0);
16354           Select(Tmp1, Tmp1);
16355           Select(Tmp2, Tmp2);
16356           Select(Tmp3, Tmp3);
16357           Select(Tmp4, N0);
16358           Select(Chain1, Chain1);
16359           ResNode = CurDAG->getTargetNode(X86::IMUL32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16360           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16361           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16362           AddHandleReplacement(N1.Val, 1, ResNode, 1);
16363           Result = SDOperand(ResNode, N.ResNo);
16364           return;
16365         }
16366       }
16367     }
16368     if (N0.getOpcode() == ISD::LOAD &&
16369         N0.hasOneUse() &&
16370         !CodeGenMap.count(N0.getValue(0)) &&
16371         !isNonImmUse(N.Val, N0.Val)) {
16372       Chain0 = N0.getOperand(0);
16373       N01 = N0.getOperand(1);
16374       N1 = N.getOperand(1);
16375 
16376       // Pattern: (mul:i8 (load:i8 addr:i32:$src), AL:i8)
16377       // Emits: (MUL8m:void addr:i32:$src)
16378       // Pattern complexity = 14  cost = 1
16379       if (N.Val->getValueType(0) == MVT::i8 &&
16380           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16381         Select(Tmp0, Tmp0);
16382         Select(Tmp1, Tmp1);
16383         Select(Tmp2, Tmp2);
16384         Select(Tmp3, Tmp3);
16385         Select(Chain0, Chain0);
16386         Select(N1, N1);
16387         ResNode = CurDAG->getCopyToReg(Chain0, CurDAG->getRegister(X86::AL, MVT::i8), N1, InFlag).Val;
16388         Chain0 = SDOperand(ResNode, 0);
16389         InFlag = SDOperand(ResNode, 1);
16390         ResNode = CurDAG->getTargetNode(X86::MUL8m, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Chain0, InFlag);
16391         InFlag = SDOperand(ResNode, 1);
16392         ResNode = CurDAG->getCopyFromReg(Chain0, X86::AL, MVT::i8, InFlag).Val;
16393         Chain0 = SDOperand(ResNode, 1);
16394         InFlag = SDOperand(ResNode, 2);
16395         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16396         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16397         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16398         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16399         if (N.ResNo < 1)
16400           Result = SDOperand(ResNode, N.ResNo);
16401         else
16402           Result = SDOperand(ResNode, N.ResNo+1);
16403         return;
16404       }
16405 
16406       // Pattern: (mul:i16 (load:i16 addr:i32:$src2), R16:i16:$src1)
16407       // Emits: (IMUL16rm:i16 R16:i16:$src1, addr:i32:$src2)
16408       // Pattern complexity = 14  cost = 1
16409       if (N.Val->getValueType(0) == MVT::i16 &&
16410           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16411         Select(Tmp0, Tmp0);
16412         Select(Tmp1, Tmp1);
16413         Select(Tmp2, Tmp2);
16414         Select(Tmp3, Tmp3);
16415         Select(Tmp4, N1);
16416         Select(Chain0, Chain0);
16417         ResNode = CurDAG->getTargetNode(X86::IMUL16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
16418         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16419         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16420         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16421         Result = SDOperand(ResNode, N.ResNo);
16422         return;
16423       }
16424 
16425       // Pattern: (mul:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
16426       // Emits: (IMUL32rm:i32 R32:i32:$src1, addr:i32:$src2)
16427       // Pattern complexity = 14  cost = 1
16428       if (N.Val->getValueType(0) == MVT::i32 &&
16429           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16430         Select(Tmp0, Tmp0);
16431         Select(Tmp1, Tmp1);
16432         Select(Tmp2, Tmp2);
16433         Select(Tmp3, Tmp3);
16434         Select(Tmp4, N1);
16435         Select(Chain0, Chain0);
16436         ResNode = CurDAG->getTargetNode(X86::IMUL32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
16437         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16438         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16439         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16440         Result = SDOperand(ResNode, N.ResNo);
16441         return;
16442       }
16443     }
16444   }
16445 
16446   // Pattern: leaaddr:i32:$src
16447   // Emits: (LEA32r:i32 leaaddr:i32:$src)
16448   // Pattern complexity = 10  cost = 1
16449   if (N.Val->getValueType(0) == MVT::i32 &&
16450       SelectLEAAddr(N, Tmp0, Tmp1, Tmp2, Tmp3)) {
16451     Select(Tmp0, Tmp0);
16452     Select(Tmp1, Tmp1);
16453     Select(Tmp2, Tmp2);
16454     Select(Tmp3, Tmp3);
16455     if (N.Val->hasOneUse()) {
16456       Result = CurDAG->SelectNodeTo(N.Val, X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
16457     } else {
16458       ResNode = CurDAG->getTargetNode(X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
16459       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16460       Result = SDOperand(ResNode, 0);
16461     }
16462     return;
16463   }
16464   {
16465     N0 = N.getOperand(0);
16466     N1 = N.getOperand(1);
16467     if (N1.getOpcode() == ISD::Constant) {
16468 
16469       // Pattern: (mul:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
16470       // Emits: (IMUL16rri8:i16 R16:i16:$src1, (imm:i16):$src2)
16471       // Pattern complexity = 5  cost = 1
16472       if (Predicate_i16immSExt8(N1.Val) &&
16473           N.Val->getValueType(0) == MVT::i16) {
16474         Select(Tmp0, N0);
16475         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
16476         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
16477         if (N.Val->hasOneUse()) {
16478           Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL16rri8, MVT::i16, Tmp0, Tmp1);
16479         } else {
16480           ResNode = CurDAG->getTargetNode(X86::IMUL16rri8, MVT::i16, Tmp0, Tmp1);
16481           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16482           Result = SDOperand(ResNode, 0);
16483         }
16484         return;
16485       }
16486 
16487       // Pattern: (mul:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
16488       // Emits: (IMUL32rri8:i32 R32:i32:$src1, (imm:i32):$src2)
16489       // Pattern complexity = 5  cost = 1
16490       if (Predicate_i32immSExt8(N1.Val) &&
16491           N.Val->getValueType(0) == MVT::i32) {
16492         Select(Tmp0, N0);
16493         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
16494         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
16495         if (N.Val->hasOneUse()) {
16496           Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL32rri8, MVT::i32, Tmp0, Tmp1);
16497         } else {
16498           ResNode = CurDAG->getTargetNode(X86::IMUL32rri8, MVT::i32, Tmp0, Tmp1);
16499           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16500           Result = SDOperand(ResNode, 0);
16501         }
16502         return;
16503       }
16504 
16505       // Pattern: (mul:i16 R16:i16:$src1, (imm:i16):$src2)
16506       // Emits: (IMUL16rri:i16 R16:i16:$src1, (imm:i16):$src2)
16507       // Pattern complexity = 4  cost = 1
16508       if (N.Val->getValueType(0) == MVT::i16) {
16509         Select(Tmp0, N0);
16510         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
16511         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
16512         if (N.Val->hasOneUse()) {
16513           Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL16rri, MVT::i16, Tmp0, Tmp1);
16514         } else {
16515           ResNode = CurDAG->getTargetNode(X86::IMUL16rri, MVT::i16, Tmp0, Tmp1);
16516           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16517           Result = SDOperand(ResNode, 0);
16518         }
16519         return;
16520       }
16521 
16522       // Pattern: (mul:i32 R32:i32:$src1, (imm:i32):$src2)
16523       // Emits: (IMUL32rri:i32 R32:i32:$src1, (imm:i32):$src2)
16524       // Pattern complexity = 4  cost = 1
16525       if (N.Val->getValueType(0) == MVT::i32) {
16526         Select(Tmp0, N0);
16527         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
16528         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
16529         if (N.Val->hasOneUse()) {
16530           Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL32rri, MVT::i32, Tmp0, Tmp1);
16531         } else {
16532           ResNode = CurDAG->getTargetNode(X86::IMUL32rri, MVT::i32, Tmp0, Tmp1);
16533           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16534           Result = SDOperand(ResNode, 0);
16535         }
16536         return;
16537       }
16538     }
16539 
16540     // Pattern: (mul:i8 AL:i8, R8:i8:$src)
16541     // Emits: (MUL8r:void R8:i8:$src)
16542     // Pattern complexity = 2  cost = 1
16543     if (N.Val->getValueType(0) == MVT::i8) {
16544       Select(Tmp0, N1);
16545       Chain = CurDAG->getEntryNode();
16546       Select(N0, N0);
16547       ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::AL, MVT::i8), N0, InFlag).Val;
16548       Chain = SDOperand(ResNode, 0);
16549       InFlag = SDOperand(ResNode, 1);
16550       ResNode = CurDAG->getTargetNode(X86::MUL8r, MVT::Flag, Tmp0, InFlag);
16551       InFlag = SDOperand(ResNode, 0);
16552       ResNode = CurDAG->getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).Val;
16553       Chain = SDOperand(ResNode, 1);
16554       InFlag = SDOperand(ResNode, 2);
16555       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16556       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16557       Result = SDOperand(ResNode, N.ResNo);
16558       return;
16559     }
16560 
16561     // Pattern: (mul:i16 R16:i16:$src1, R16:i16:$src2)
16562     // Emits: (IMUL16rr:i16 R16:i16:$src1, R16:i16:$src2)
16563     // Pattern complexity = 2  cost = 1
16564     if (N.Val->getValueType(0) == MVT::i16) {
16565       Select(Tmp0, N0);
16566       Select(Tmp1, N1);
16567       if (N.Val->hasOneUse()) {
16568         Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL16rr, MVT::i16, Tmp0, Tmp1);
16569       } else {
16570         ResNode = CurDAG->getTargetNode(X86::IMUL16rr, MVT::i16, Tmp0, Tmp1);
16571         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16572         Result = SDOperand(ResNode, 0);
16573       }
16574       return;
16575     }
16576 
16577     // Pattern: (mul:i32 R32:i32:$src1, R32:i32:$src2)
16578     // Emits: (IMUL32rr:i32 R32:i32:$src1, R32:i32:$src2)
16579     // Pattern complexity = 2  cost = 1
16580     if (N.Val->getValueType(0) == MVT::i32) {
16581       Select(Tmp0, N0);
16582       Select(Tmp1, N1);
16583       if (N.Val->hasOneUse()) {
16584         Result = CurDAG->SelectNodeTo(N.Val, X86::IMUL32rr, MVT::i32, Tmp0, Tmp1);
16585       } else {
16586         ResNode = CurDAG->getTargetNode(X86::IMUL32rr, MVT::i32, Tmp0, Tmp1);
16587         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16588         Result = SDOperand(ResNode, 0);
16589       }
16590       return;
16591     }
16592   }
16593 
16594   // Pattern: (mul:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16595   // Emits: (PMULLWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
16596   // Pattern complexity = 2  cost = 1
16597   if ((Subtarget->hasSSE2())) {
16598     N0 = N.getOperand(0);
16599     N1 = N.getOperand(1);
16600     if (N.Val->getValueType(0) == MVT::v8i16) {
16601       Select(Tmp0, N0);
16602       Select(Tmp1, N1);
16603       if (N.Val->hasOneUse()) {
16604         Result = CurDAG->SelectNodeTo(N.Val, X86::PMULLWrr, MVT::v8i16, Tmp0, Tmp1);
16605       } else {
16606         ResNode = CurDAG->getTargetNode(X86::PMULLWrr, MVT::v8i16, Tmp0, Tmp1);
16607         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
16608         Result = SDOperand(ResNode, 0);
16609       }
16610       return;
16611     }
16612   }
16613 
16614   // Pattern: (mul:i8 R8:i8:$src, AL:i8)
16615   // Emits: (MUL8r:void R8:i8:$src)
16616   // Pattern complexity = 2  cost = 1
16617   N0 = N.getOperand(0);
16618   N1 = N.getOperand(1);
16619   if (N.Val->getValueType(0) == MVT::i8) {
16620     Select(Tmp0, N0);
16621     Chain = CurDAG->getEntryNode();
16622     Select(N1, N1);
16623     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::AL, MVT::i8), N1, InFlag).Val;
16624     Chain = SDOperand(ResNode, 0);
16625     InFlag = SDOperand(ResNode, 1);
16626     ResNode = CurDAG->getTargetNode(X86::MUL8r, MVT::Flag, Tmp0, InFlag);
16627     InFlag = SDOperand(ResNode, 0);
16628     ResNode = CurDAG->getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).Val;
16629     Chain = SDOperand(ResNode, 1);
16630     InFlag = SDOperand(ResNode, 2);
16631     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16632     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
16633     Result = SDOperand(ResNode, N.ResNo);
16634     return;
16635   }
16636   std::cerr << "Cannot yet select: ";
16637   N.Val->dump(CurDAG);
16638   std::cerr << '\n';
16639   abort();
16640 }
16641 
16642 void Select_or(SDOperand &Result, SDOperand N) {
16643   SDOperand Chain(0, 0);
16644   SDOperand Chain0(0, 0);
16645   SDOperand Chain00(0, 0);
16646   SDOperand Chain1(0, 0);
16647   SDOperand Chain10(0, 0);
16648   SDOperand InFlag(0, 0);
16649   SDOperand N0(0, 0);
16650   SDOperand N00(0, 0);
16651   SDOperand N001(0, 0);
16652   SDOperand N01(0, 0);
16653   SDOperand N010(0, 0);
16654   SDOperand N011(0, 0);
16655   SDOperand N1(0, 0);
16656   SDOperand N10(0, 0);
16657   SDOperand N101(0, 0);
16658   SDOperand N11(0, 0);
16659   SDOperand N110(0, 0);
16660   SDOperand N111(0, 0);
16661   SDOperand Tmp0(0, 0);
16662   SDOperand Tmp1(0, 0);
16663   SDOperand Tmp2(0, 0);
16664   SDOperand Tmp3(0, 0);
16665   SDOperand Tmp4(0, 0);
16666   SDNode *ResNode;
16667   if ((Subtarget->hasSSE2())) {
16668     N0 = N.getOperand(0);
16669     if (N0.getOpcode() == ISD::BIT_CONVERT) {
16670 
16671       // Pattern: (or:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)))
16672       // Emits: (ORPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
16673       // Pattern complexity = 18  cost = 1
16674       {
16675         N00 = N0.getOperand(0);
16676         N1 = N.getOperand(1);
16677         if (N1.getOpcode() == ISD::BIT_CONVERT &&
16678             N1.hasOneUse() &&
16679             !CodeGenMap.count(N1.getValue(0))) {
16680           N10 = N1.getOperand(0);
16681           if (N10.getOpcode() == ISD::LOAD &&
16682               N10.hasOneUse() &&
16683               !CodeGenMap.count(N10.getValue(0))) {
16684             Chain10 = N10.getOperand(0);
16685             N101 = N10.getOperand(1);
16686             if (N.Val->getValueType(0) == MVT::v2i64 &&
16687                 N00.Val->getValueType(0) == MVT::v2f64 &&
16688                 N10.Val->getValueType(0) == MVT::v2f64 &&
16689                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
16690               Select(Tmp0, Tmp0);
16691               Select(Tmp1, Tmp1);
16692               Select(Tmp2, Tmp2);
16693               Select(Tmp3, Tmp3);
16694               Select(Tmp4, N00);
16695               Select(Chain10, Chain10);
16696               ResNode = CurDAG->getTargetNode(X86::ORPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
16697               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16698               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
16699               AddHandleReplacement(N10.Val, 1, ResNode, 1);
16700               Result = SDOperand(ResNode, N.ResNo);
16701               return;
16702             }
16703           }
16704         }
16705       }
16706 
16707       // Pattern: (or:v2i64 (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
16708       // Emits: (ORPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
16709       // Pattern complexity = 18  cost = 1
16710       if (N0.hasOneUse() &&
16711           !CodeGenMap.count(N0.getValue(0))) {
16712         N00 = N0.getOperand(0);
16713         if (N00.getOpcode() == ISD::LOAD &&
16714             N00.hasOneUse() &&
16715             !CodeGenMap.count(N00.getValue(0))) {
16716           Chain00 = N00.getOperand(0);
16717           N001 = N00.getOperand(1);
16718           N1 = N.getOperand(1);
16719           if (N1.getOpcode() == ISD::BIT_CONVERT) {
16720             N10 = N1.getOperand(0);
16721             if (N.Val->getValueType(0) == MVT::v2i64 &&
16722                 N00.Val->getValueType(0) == MVT::v2f64 &&
16723                 N10.Val->getValueType(0) == MVT::v2f64 &&
16724                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
16725               Select(Tmp0, Tmp0);
16726               Select(Tmp1, Tmp1);
16727               Select(Tmp2, Tmp2);
16728               Select(Tmp3, Tmp3);
16729               Select(Tmp4, N10);
16730               Select(Chain00, Chain00);
16731               ResNode = CurDAG->getTargetNode(X86::ORPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
16732               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16733               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
16734               AddHandleReplacement(N00.Val, 1, ResNode, 1);
16735               Result = SDOperand(ResNode, N.ResNo);
16736               return;
16737             }
16738           }
16739         }
16740       }
16741     }
16742   }
16743   if ((Subtarget->hasSSE1())) {
16744     N0 = N.getOperand(0);
16745 
16746     // Pattern: (or:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)))
16747     // Emits: (ORPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
16748     // Pattern complexity = 16  cost = 1
16749     {
16750       N1 = N.getOperand(1);
16751       if (N1.getOpcode() == ISD::BIT_CONVERT &&
16752           N1.hasOneUse() &&
16753           !CodeGenMap.count(N1.getValue(0))) {
16754         N10 = N1.getOperand(0);
16755         if (N10.getOpcode() == ISD::LOAD &&
16756             N10.hasOneUse() &&
16757             !CodeGenMap.count(N10.getValue(0))) {
16758           Chain10 = N10.getOperand(0);
16759           N101 = N10.getOperand(1);
16760           if (N.Val->getValueType(0) == MVT::v2i64 &&
16761               N10.Val->getValueType(0) == MVT::v4f32 &&
16762               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
16763             Select(Tmp0, Tmp0);
16764             Select(Tmp1, Tmp1);
16765             Select(Tmp2, Tmp2);
16766             Select(Tmp3, Tmp3);
16767             Select(Tmp4, N0);
16768             Select(Chain10, Chain10);
16769             ResNode = CurDAG->getTargetNode(X86::ORPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
16770             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16771             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
16772             AddHandleReplacement(N10.Val, 1, ResNode, 1);
16773             Result = SDOperand(ResNode, N.ResNo);
16774             return;
16775           }
16776         }
16777       }
16778     }
16779 
16780     // Pattern: (or:v2i64 (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)), VR128:v2i64:$src1)
16781     // Emits: (ORPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
16782     // Pattern complexity = 16  cost = 1
16783     if (N0.getOpcode() == ISD::BIT_CONVERT &&
16784         N0.hasOneUse() &&
16785         !CodeGenMap.count(N0.getValue(0))) {
16786       N00 = N0.getOperand(0);
16787       if (N00.getOpcode() == ISD::LOAD &&
16788           N00.hasOneUse() &&
16789           !CodeGenMap.count(N00.getValue(0))) {
16790         Chain00 = N00.getOperand(0);
16791         N001 = N00.getOperand(1);
16792         N1 = N.getOperand(1);
16793         if (N.Val->getValueType(0) == MVT::v2i64 &&
16794             N00.Val->getValueType(0) == MVT::v4f32 &&
16795             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
16796           Select(Tmp0, Tmp0);
16797           Select(Tmp1, Tmp1);
16798           Select(Tmp2, Tmp2);
16799           Select(Tmp3, Tmp3);
16800           Select(Tmp4, N1);
16801           Select(Chain00, Chain00);
16802           ResNode = CurDAG->getTargetNode(X86::ORPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
16803           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16804           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
16805           AddHandleReplacement(N00.Val, 1, ResNode, 1);
16806           Result = SDOperand(ResNode, N.ResNo);
16807           return;
16808         }
16809       }
16810     }
16811   }
16812   {
16813     N0 = N.getOperand(0);
16814     N1 = N.getOperand(1);
16815     if (N1.getOpcode() == ISD::LOAD &&
16816         N1.hasOneUse() &&
16817         !CodeGenMap.count(N1.getValue(0)) &&
16818         !isNonImmUse(N.Val, N1.Val)) {
16819       Chain1 = N1.getOperand(0);
16820       N11 = N1.getOperand(1);
16821 
16822       // Pattern: (or:i8 R8:i8:$src1, (load:i8 addr:i32:$src2))
16823       // Emits: (OR8rm:i8 R8:i8:$src1, addr:i32:$src2)
16824       // Pattern complexity = 14  cost = 1
16825       if (N.Val->getValueType(0) == MVT::i8 &&
16826           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16827         Select(Tmp0, Tmp0);
16828         Select(Tmp1, Tmp1);
16829         Select(Tmp2, Tmp2);
16830         Select(Tmp3, Tmp3);
16831         Select(Tmp4, N0);
16832         Select(Chain1, Chain1);
16833         ResNode = CurDAG->getTargetNode(X86::OR8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16834         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16835         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16836         AddHandleReplacement(N1.Val, 1, ResNode, 1);
16837         Result = SDOperand(ResNode, N.ResNo);
16838         return;
16839       }
16840 
16841       // Pattern: (or:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
16842       // Emits: (OR16rm:i16 R16:i16:$src1, addr:i32:$src2)
16843       // Pattern complexity = 14  cost = 1
16844       if (N.Val->getValueType(0) == MVT::i16 &&
16845           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16846         Select(Tmp0, Tmp0);
16847         Select(Tmp1, Tmp1);
16848         Select(Tmp2, Tmp2);
16849         Select(Tmp3, Tmp3);
16850         Select(Tmp4, N0);
16851         Select(Chain1, Chain1);
16852         ResNode = CurDAG->getTargetNode(X86::OR16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16853         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16854         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16855         AddHandleReplacement(N1.Val, 1, ResNode, 1);
16856         Result = SDOperand(ResNode, N.ResNo);
16857         return;
16858       }
16859 
16860       // Pattern: (or:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
16861       // Emits: (OR32rm:i32 R32:i32:$src1, addr:i32:$src2)
16862       // Pattern complexity = 14  cost = 1
16863       if (N.Val->getValueType(0) == MVT::i32 &&
16864           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16865         Select(Tmp0, Tmp0);
16866         Select(Tmp1, Tmp1);
16867         Select(Tmp2, Tmp2);
16868         Select(Tmp3, Tmp3);
16869         Select(Tmp4, N0);
16870         Select(Chain1, Chain1);
16871         ResNode = CurDAG->getTargetNode(X86::OR32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16872         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16873         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16874         AddHandleReplacement(N1.Val, 1, ResNode, 1);
16875         Result = SDOperand(ResNode, N.ResNo);
16876         return;
16877       }
16878     }
16879   }
16880 
16881   // Pattern: (or:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2))
16882   // Emits: (PORrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
16883   // Pattern complexity = 14  cost = 1
16884   if ((Subtarget->hasSSE2())) {
16885     N0 = N.getOperand(0);
16886     N1 = N.getOperand(1);
16887     if (N1.getOpcode() == ISD::LOAD &&
16888         N1.hasOneUse() &&
16889         !CodeGenMap.count(N1.getValue(0)) &&
16890         !isNonImmUse(N.Val, N1.Val)) {
16891       Chain1 = N1.getOperand(0);
16892       N11 = N1.getOperand(1);
16893       if (N.Val->getValueType(0) == MVT::v2i64 &&
16894           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
16895         Select(Tmp0, Tmp0);
16896         Select(Tmp1, Tmp1);
16897         Select(Tmp2, Tmp2);
16898         Select(Tmp3, Tmp3);
16899         Select(Tmp4, N0);
16900         Select(Chain1, Chain1);
16901         ResNode = CurDAG->getTargetNode(X86::PORrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
16902         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16903         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
16904         AddHandleReplacement(N1.Val, 1, ResNode, 1);
16905         Result = SDOperand(ResNode, N.ResNo);
16906         return;
16907       }
16908     }
16909   }
16910   {
16911     N0 = N.getOperand(0);
16912     if (N0.getOpcode() == ISD::LOAD &&
16913         N0.hasOneUse() &&
16914         !CodeGenMap.count(N0.getValue(0)) &&
16915         !isNonImmUse(N.Val, N0.Val)) {
16916       Chain0 = N0.getOperand(0);
16917       N01 = N0.getOperand(1);
16918       N1 = N.getOperand(1);
16919 
16920       // Pattern: (or:i8 (load:i8 addr:i32:$src2), R8:i8:$src1)
16921       // Emits: (OR8rm:i8 R8:i8:$src1, addr:i32:$src2)
16922       // Pattern complexity = 14  cost = 1
16923       if (N.Val->getValueType(0) == MVT::i8 &&
16924           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16925         Select(Tmp0, Tmp0);
16926         Select(Tmp1, Tmp1);
16927         Select(Tmp2, Tmp2);
16928         Select(Tmp3, Tmp3);
16929         Select(Tmp4, N1);
16930         Select(Chain0, Chain0);
16931         ResNode = CurDAG->getTargetNode(X86::OR8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
16932         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16933         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16934         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16935         Result = SDOperand(ResNode, N.ResNo);
16936         return;
16937       }
16938 
16939       // Pattern: (or:i16 (load:i16 addr:i32:$src2), R16:i16:$src1)
16940       // Emits: (OR16rm:i16 R16:i16:$src1, addr:i32:$src2)
16941       // Pattern complexity = 14  cost = 1
16942       if (N.Val->getValueType(0) == MVT::i16 &&
16943           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16944         Select(Tmp0, Tmp0);
16945         Select(Tmp1, Tmp1);
16946         Select(Tmp2, Tmp2);
16947         Select(Tmp3, Tmp3);
16948         Select(Tmp4, N1);
16949         Select(Chain0, Chain0);
16950         ResNode = CurDAG->getTargetNode(X86::OR16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
16951         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16952         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16953         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16954         Result = SDOperand(ResNode, N.ResNo);
16955         return;
16956       }
16957 
16958       // Pattern: (or:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
16959       // Emits: (OR32rm:i32 R32:i32:$src1, addr:i32:$src2)
16960       // Pattern complexity = 14  cost = 1
16961       if (N.Val->getValueType(0) == MVT::i32 &&
16962           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16963         Select(Tmp0, Tmp0);
16964         Select(Tmp1, Tmp1);
16965         Select(Tmp2, Tmp2);
16966         Select(Tmp3, Tmp3);
16967         Select(Tmp4, N1);
16968         Select(Chain0, Chain0);
16969         ResNode = CurDAG->getTargetNode(X86::OR32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
16970         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
16971         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
16972         AddHandleReplacement(N0.Val, 1, ResNode, 1);
16973         Result = SDOperand(ResNode, N.ResNo);
16974         return;
16975       }
16976     }
16977   }
16978 
16979   // Pattern: (or:v2i64 (load:v2i64 addr:i32:$src2), VR128:v2i64:$src1)
16980   // Emits: (PORrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
16981   // Pattern complexity = 14  cost = 1
16982   if ((Subtarget->hasSSE2())) {
16983     N0 = N.getOperand(0);
16984     if (N0.getOpcode() == ISD::LOAD &&
16985         N0.hasOneUse() &&
16986         !CodeGenMap.count(N0.getValue(0)) &&
16987         !isNonImmUse(N.Val, N0.Val)) {
16988       Chain0 = N0.getOperand(0);
16989       N01 = N0.getOperand(1);
16990       N1 = N.getOperand(1);
16991       if (N.Val->getValueType(0) == MVT::v2i64 &&
16992           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
16993         Select(Tmp0, Tmp0);
16994         Select(Tmp1, Tmp1);
16995         Select(Tmp2, Tmp2);
16996         Select(Tmp3, Tmp3);
16997         Select(Tmp4, N1);
16998         Select(Chain0, Chain0);
16999         ResNode = CurDAG->getTargetNode(X86::PORrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
17000         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
17001         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
17002         AddHandleReplacement(N0.Val, 1, ResNode, 1);
17003         Result = SDOperand(ResNode, N.ResNo);
17004         return;
17005       }
17006     }
17007   }
17008   {
17009     N0 = N.getOperand(0);
17010 
17011     // Pattern: (or:i32 (srl:i32 R32:i32:$src1, CL:i8:$amt), (shl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)))
17012     // Emits: (SHRD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
17013     // Pattern complexity = 11  cost = 1
17014     if (N0.getOpcode() == ISD::SRL) {
17015       N00 = N0.getOperand(0);
17016       N01 = N0.getOperand(1);
17017       N1 = N.getOperand(1);
17018       if (N1.getOpcode() == ISD::SHL) {
17019         N10 = N1.getOperand(0);
17020         N11 = N1.getOperand(1);
17021         if (N11.getOpcode() == ISD::SUB) {
17022           N110 = N11.getOperand(0);
17023           if (isa<ConstantSDNode>(N110)) {
17024             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17025             if (CN0 == 32) {
17026               N111 = N11.getOperand(1);
17027               if (N01 == N111 &&
17028                   N.Val->getValueType(0) == MVT::i32 &&
17029                   N01.Val->getValueType(0) == MVT::i8 &&
17030                   N11.Val->getValueType(0) == MVT::i8) {
17031                 Select(Tmp1, N00);
17032                 Select(Tmp2, N10);
17033                 Chain = CurDAG->getEntryNode();
17034                 Select(N01, N01);
17035                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N01, InFlag).Val;
17036                 Chain = SDOperand(ResNode, 0);
17037                 InFlag = SDOperand(ResNode, 1);
17038                 if (N.Val->hasOneUse()) {
17039                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17040                 } else {
17041                   ResNode = CurDAG->getTargetNode(X86::SHRD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17042                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17043                   Result = SDOperand(ResNode, 0);
17044                 }
17045                 return;
17046               }
17047             }
17048           }
17049         }
17050       }
17051     }
17052 
17053     // Pattern: (or:i32 (shl:i32 R32:i32:$src1, CL:i8:$amt), (srl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)))
17054     // Emits: (SHLD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
17055     // Pattern complexity = 11  cost = 1
17056     if (N0.getOpcode() == ISD::SHL) {
17057       N00 = N0.getOperand(0);
17058       N01 = N0.getOperand(1);
17059       N1 = N.getOperand(1);
17060       if (N1.getOpcode() == ISD::SRL) {
17061         N10 = N1.getOperand(0);
17062         N11 = N1.getOperand(1);
17063         if (N11.getOpcode() == ISD::SUB) {
17064           N110 = N11.getOperand(0);
17065           if (isa<ConstantSDNode>(N110)) {
17066             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17067             if (CN0 == 32) {
17068               N111 = N11.getOperand(1);
17069               if (N01 == N111 &&
17070                   N.Val->getValueType(0) == MVT::i32 &&
17071                   N01.Val->getValueType(0) == MVT::i8 &&
17072                   N11.Val->getValueType(0) == MVT::i8) {
17073                 Select(Tmp1, N00);
17074                 Select(Tmp2, N10);
17075                 Chain = CurDAG->getEntryNode();
17076                 Select(N01, N01);
17077                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N01, InFlag).Val;
17078                 Chain = SDOperand(ResNode, 0);
17079                 InFlag = SDOperand(ResNode, 1);
17080                 if (N.Val->hasOneUse()) {
17081                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17082                 } else {
17083                   ResNode = CurDAG->getTargetNode(X86::SHLD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17084                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17085                   Result = SDOperand(ResNode, 0);
17086                 }
17087                 return;
17088               }
17089             }
17090           }
17091         }
17092       }
17093     }
17094 
17095     // Pattern: (or:i16 (srl:i16 R16:i16:$src1, CL:i8:$amt), (shl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)))
17096     // Emits: (SHRD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
17097     // Pattern complexity = 11  cost = 1
17098     if (N0.getOpcode() == ISD::SRL) {
17099       N00 = N0.getOperand(0);
17100       N01 = N0.getOperand(1);
17101       N1 = N.getOperand(1);
17102       if (N1.getOpcode() == ISD::SHL) {
17103         N10 = N1.getOperand(0);
17104         N11 = N1.getOperand(1);
17105         if (N11.getOpcode() == ISD::SUB) {
17106           N110 = N11.getOperand(0);
17107           if (isa<ConstantSDNode>(N110)) {
17108             int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17109             if (CN0 == 16) {
17110               N111 = N11.getOperand(1);
17111               if (N01 == N111 &&
17112                   N.Val->getValueType(0) == MVT::i16 &&
17113                   N01.Val->getValueType(0) == MVT::i8 &&
17114                   N11.Val->getValueType(0) == MVT::i8) {
17115                 Select(Tmp1, N00);
17116                 Select(Tmp2, N10);
17117                 Chain = CurDAG->getEntryNode();
17118                 Select(N01, N01);
17119                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N01, InFlag).Val;
17120                 Chain = SDOperand(ResNode, 0);
17121                 InFlag = SDOperand(ResNode, 1);
17122                 if (N.Val->hasOneUse()) {
17123                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17124                 } else {
17125                   ResNode = CurDAG->getTargetNode(X86::SHRD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17126                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17127                   Result = SDOperand(ResNode, 0);
17128                 }
17129                 return;
17130               }
17131             }
17132           }
17133         }
17134       }
17135     }
17136     if (N0.getOpcode() == ISD::SHL) {
17137       N00 = N0.getOperand(0);
17138       N01 = N0.getOperand(1);
17139 
17140       // Pattern: (or:i16 (shl:i16 R16:i16:$src1, CL:i8:$amt), (srl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)))
17141       // Emits: (SHLD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
17142       // Pattern complexity = 11  cost = 1
17143       {
17144         N1 = N.getOperand(1);
17145         if (N1.getOpcode() == ISD::SRL) {
17146           N10 = N1.getOperand(0);
17147           N11 = N1.getOperand(1);
17148           if (N11.getOpcode() == ISD::SUB) {
17149             N110 = N11.getOperand(0);
17150             if (isa<ConstantSDNode>(N110)) {
17151               int64_t CN0 = cast<ConstantSDNode>(N110)->getSignExtended();
17152               if (CN0 == 16) {
17153                 N111 = N11.getOperand(1);
17154                 if (N01 == N111 &&
17155                     N.Val->getValueType(0) == MVT::i16 &&
17156                     N01.Val->getValueType(0) == MVT::i8 &&
17157                     N11.Val->getValueType(0) == MVT::i8) {
17158                   Select(Tmp1, N00);
17159                   Select(Tmp2, N10);
17160                   Chain = CurDAG->getEntryNode();
17161                   Select(N01, N01);
17162                   ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N01, InFlag).Val;
17163                   Chain = SDOperand(ResNode, 0);
17164                   InFlag = SDOperand(ResNode, 1);
17165                   if (N.Val->hasOneUse()) {
17166                     Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17167                   } else {
17168                     ResNode = CurDAG->getTargetNode(X86::SHLD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17169                     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17170                     Result = SDOperand(ResNode, 0);
17171                   }
17172                   return;
17173                 }
17174               }
17175             }
17176           }
17177         }
17178       }
17179 
17180       // Pattern: (or:i32 (shl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (srl:i32 R32:i32:$src1, CL:i8:$amt))
17181       // Emits: (SHRD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
17182       // Pattern complexity = 11  cost = 1
17183       if (N01.getOpcode() == ISD::SUB) {
17184         N010 = N01.getOperand(0);
17185         if (isa<ConstantSDNode>(N010)) {
17186           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17187           if (CN0 == 32) {
17188             N011 = N01.getOperand(1);
17189             N1 = N.getOperand(1);
17190             if (N1.getOpcode() == ISD::SRL) {
17191               N10 = N1.getOperand(0);
17192               N11 = N1.getOperand(1);
17193               if (N011 == N11 &&
17194                   N.Val->getValueType(0) == MVT::i32 &&
17195                   N01.Val->getValueType(0) == MVT::i8 &&
17196                   N11.Val->getValueType(0) == MVT::i8) {
17197                 Select(Tmp1, N10);
17198                 Select(Tmp2, N00);
17199                 Chain = CurDAG->getEntryNode();
17200                 Select(N011, N011);
17201                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N011, InFlag).Val;
17202                 Chain = SDOperand(ResNode, 0);
17203                 InFlag = SDOperand(ResNode, 1);
17204                 if (N.Val->hasOneUse()) {
17205                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17206                 } else {
17207                   ResNode = CurDAG->getTargetNode(X86::SHRD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17208                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17209                   Result = SDOperand(ResNode, 0);
17210                 }
17211                 return;
17212               }
17213             }
17214           }
17215         }
17216       }
17217     }
17218 
17219     // Pattern: (or:i32 (srl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (shl:i32 R32:i32:$src1, CL:i8:$amt))
17220     // Emits: (SHLD32rrCL:i32 R32:i32:$src1, R32:i32:$src2)
17221     // Pattern complexity = 11  cost = 1
17222     if (N0.getOpcode() == ISD::SRL) {
17223       N00 = N0.getOperand(0);
17224       N01 = N0.getOperand(1);
17225       if (N01.getOpcode() == ISD::SUB) {
17226         N010 = N01.getOperand(0);
17227         if (isa<ConstantSDNode>(N010)) {
17228           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17229           if (CN0 == 32) {
17230             N011 = N01.getOperand(1);
17231             N1 = N.getOperand(1);
17232             if (N1.getOpcode() == ISD::SHL) {
17233               N10 = N1.getOperand(0);
17234               N11 = N1.getOperand(1);
17235               if (N011 == N11 &&
17236                   N.Val->getValueType(0) == MVT::i32 &&
17237                   N01.Val->getValueType(0) == MVT::i8 &&
17238                   N11.Val->getValueType(0) == MVT::i8) {
17239                 Select(Tmp1, N10);
17240                 Select(Tmp2, N00);
17241                 Chain = CurDAG->getEntryNode();
17242                 Select(N011, N011);
17243                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N011, InFlag).Val;
17244                 Chain = SDOperand(ResNode, 0);
17245                 InFlag = SDOperand(ResNode, 1);
17246                 if (N.Val->hasOneUse()) {
17247                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17248                 } else {
17249                   ResNode = CurDAG->getTargetNode(X86::SHLD32rrCL, MVT::i32, Tmp1, Tmp2, InFlag);
17250                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17251                   Result = SDOperand(ResNode, 0);
17252                 }
17253                 return;
17254               }
17255             }
17256           }
17257         }
17258       }
17259     }
17260 
17261     // Pattern: (or:i16 (shl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (srl:i16 R16:i16:$src1, CL:i8:$amt))
17262     // Emits: (SHRD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
17263     // Pattern complexity = 11  cost = 1
17264     if (N0.getOpcode() == ISD::SHL) {
17265       N00 = N0.getOperand(0);
17266       N01 = N0.getOperand(1);
17267       if (N01.getOpcode() == ISD::SUB) {
17268         N010 = N01.getOperand(0);
17269         if (isa<ConstantSDNode>(N010)) {
17270           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17271           if (CN0 == 16) {
17272             N011 = N01.getOperand(1);
17273             N1 = N.getOperand(1);
17274             if (N1.getOpcode() == ISD::SRL) {
17275               N10 = N1.getOperand(0);
17276               N11 = N1.getOperand(1);
17277               if (N011 == N11 &&
17278                   N.Val->getValueType(0) == MVT::i16 &&
17279                   N01.Val->getValueType(0) == MVT::i8 &&
17280                   N11.Val->getValueType(0) == MVT::i8) {
17281                 Select(Tmp1, N10);
17282                 Select(Tmp2, N00);
17283                 Chain = CurDAG->getEntryNode();
17284                 Select(N011, N011);
17285                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N011, InFlag).Val;
17286                 Chain = SDOperand(ResNode, 0);
17287                 InFlag = SDOperand(ResNode, 1);
17288                 if (N.Val->hasOneUse()) {
17289                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHRD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17290                 } else {
17291                   ResNode = CurDAG->getTargetNode(X86::SHRD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17292                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17293                   Result = SDOperand(ResNode, 0);
17294                 }
17295                 return;
17296               }
17297             }
17298           }
17299         }
17300       }
17301     }
17302 
17303     // Pattern: (or:i16 (srl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (shl:i16 R16:i16:$src1, CL:i8:$amt))
17304     // Emits: (SHLD16rrCL:i16 R16:i16:$src1, R16:i16:$src2)
17305     // Pattern complexity = 11  cost = 1
17306     if (N0.getOpcode() == ISD::SRL) {
17307       N00 = N0.getOperand(0);
17308       N01 = N0.getOperand(1);
17309       if (N01.getOpcode() == ISD::SUB) {
17310         N010 = N01.getOperand(0);
17311         if (isa<ConstantSDNode>(N010)) {
17312           int64_t CN0 = cast<ConstantSDNode>(N010)->getSignExtended();
17313           if (CN0 == 16) {
17314             N011 = N01.getOperand(1);
17315             N1 = N.getOperand(1);
17316             if (N1.getOpcode() == ISD::SHL) {
17317               N10 = N1.getOperand(0);
17318               N11 = N1.getOperand(1);
17319               if (N011 == N11 &&
17320                   N.Val->getValueType(0) == MVT::i16 &&
17321                   N01.Val->getValueType(0) == MVT::i8 &&
17322                   N11.Val->getValueType(0) == MVT::i8) {
17323                 Select(Tmp1, N10);
17324                 Select(Tmp2, N00);
17325                 Chain = CurDAG->getEntryNode();
17326                 Select(N011, N011);
17327                 ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N011, InFlag).Val;
17328                 Chain = SDOperand(ResNode, 0);
17329                 InFlag = SDOperand(ResNode, 1);
17330                 if (N.Val->hasOneUse()) {
17331                   Result = CurDAG->SelectNodeTo(N.Val, X86::SHLD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17332                 } else {
17333                   ResNode = CurDAG->getTargetNode(X86::SHLD16rrCL, MVT::i16, Tmp1, Tmp2, InFlag);
17334                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17335                   Result = SDOperand(ResNode, 0);
17336                 }
17337                 return;
17338               }
17339             }
17340           }
17341         }
17342       }
17343     }
17344   }
17345 
17346   // Pattern: (or:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
17347   // Emits: (ORPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
17348   // Pattern complexity = 6  cost = 1
17349   if ((Subtarget->hasSSE2())) {
17350     N0 = N.getOperand(0);
17351     if (N0.getOpcode() == ISD::BIT_CONVERT) {
17352       N00 = N0.getOperand(0);
17353       N1 = N.getOperand(1);
17354       if (N1.getOpcode() == ISD::BIT_CONVERT) {
17355         N10 = N1.getOperand(0);
17356         if (N.Val->getValueType(0) == MVT::v2i64 &&
17357             N00.Val->getValueType(0) == MVT::v2f64 &&
17358             N10.Val->getValueType(0) == MVT::v2f64) {
17359           Select(Tmp0, N00);
17360           Select(Tmp1, N10);
17361           if (N.Val->hasOneUse()) {
17362             Result = CurDAG->SelectNodeTo(N.Val, X86::ORPDrr, MVT::v2i64, Tmp0, Tmp1);
17363           } else {
17364             ResNode = CurDAG->getTargetNode(X86::ORPDrr, MVT::v2i64, Tmp0, Tmp1);
17365             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17366             Result = SDOperand(ResNode, 0);
17367           }
17368           return;
17369         }
17370       }
17371     }
17372   }
17373   {
17374     N0 = N.getOperand(0);
17375     N1 = N.getOperand(1);
17376     if (N1.getOpcode() == ISD::Constant) {
17377 
17378       // Pattern: (or:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
17379       // Emits: (OR16ri8:i16 R16:i16:$src1, (imm:i16):$src2)
17380       // Pattern complexity = 5  cost = 1
17381       if (Predicate_i16immSExt8(N1.Val) &&
17382           N.Val->getValueType(0) == MVT::i16) {
17383         Select(Tmp0, N0);
17384         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
17385         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
17386         if (N.Val->hasOneUse()) {
17387           Result = CurDAG->SelectNodeTo(N.Val, X86::OR16ri8, MVT::i16, Tmp0, Tmp1);
17388         } else {
17389           ResNode = CurDAG->getTargetNode(X86::OR16ri8, MVT::i16, Tmp0, Tmp1);
17390           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17391           Result = SDOperand(ResNode, 0);
17392         }
17393         return;
17394       }
17395 
17396       // Pattern: (or:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
17397       // Emits: (OR32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
17398       // Pattern complexity = 5  cost = 1
17399       if (Predicate_i32immSExt8(N1.Val) &&
17400           N.Val->getValueType(0) == MVT::i32) {
17401         Select(Tmp0, N0);
17402         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
17403         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
17404         if (N.Val->hasOneUse()) {
17405           Result = CurDAG->SelectNodeTo(N.Val, X86::OR32ri8, MVT::i32, Tmp0, Tmp1);
17406         } else {
17407           ResNode = CurDAG->getTargetNode(X86::OR32ri8, MVT::i32, Tmp0, Tmp1);
17408           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17409           Result = SDOperand(ResNode, 0);
17410         }
17411         return;
17412       }
17413 
17414       // Pattern: (or:i8 R8:i8:$src1, (imm:i8):$src2)
17415       // Emits: (OR8ri:i8 R8:i8:$src1, (imm:i8):$src2)
17416       // Pattern complexity = 4  cost = 1
17417       if (N.Val->getValueType(0) == MVT::i8) {
17418         Select(Tmp0, N0);
17419         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17420         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17421         if (N.Val->hasOneUse()) {
17422           Result = CurDAG->SelectNodeTo(N.Val, X86::OR8ri, MVT::i8, Tmp0, Tmp1);
17423         } else {
17424           ResNode = CurDAG->getTargetNode(X86::OR8ri, MVT::i8, Tmp0, Tmp1);
17425           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17426           Result = SDOperand(ResNode, 0);
17427         }
17428         return;
17429       }
17430 
17431       // Pattern: (or:i16 R16:i16:$src1, (imm:i16):$src2)
17432       // Emits: (OR16ri:i16 R16:i16:$src1, (imm:i16):$src2)
17433       // Pattern complexity = 4  cost = 1
17434       if (N.Val->getValueType(0) == MVT::i16) {
17435         Select(Tmp0, N0);
17436         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
17437         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
17438         if (N.Val->hasOneUse()) {
17439           Result = CurDAG->SelectNodeTo(N.Val, X86::OR16ri, MVT::i16, Tmp0, Tmp1);
17440         } else {
17441           ResNode = CurDAG->getTargetNode(X86::OR16ri, MVT::i16, Tmp0, Tmp1);
17442           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17443           Result = SDOperand(ResNode, 0);
17444         }
17445         return;
17446       }
17447 
17448       // Pattern: (or:i32 R32:i32:$src1, (imm:i32):$src2)
17449       // Emits: (OR32ri:i32 R32:i32:$src1, (imm:i32):$src2)
17450       // Pattern complexity = 4  cost = 1
17451       if (N.Val->getValueType(0) == MVT::i32) {
17452         Select(Tmp0, N0);
17453         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
17454         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
17455         if (N.Val->hasOneUse()) {
17456           Result = CurDAG->SelectNodeTo(N.Val, X86::OR32ri, MVT::i32, Tmp0, Tmp1);
17457         } else {
17458           ResNode = CurDAG->getTargetNode(X86::OR32ri, MVT::i32, Tmp0, Tmp1);
17459           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17460           Result = SDOperand(ResNode, 0);
17461         }
17462         return;
17463       }
17464     }
17465 
17466     // Pattern: (or:i8 R8:i8:$src1, R8:i8:$src2)
17467     // Emits: (OR8rr:i8 R8:i8:$src1, R8:i8:$src2)
17468     // Pattern complexity = 2  cost = 1
17469     if (N.Val->getValueType(0) == MVT::i8) {
17470       Select(Tmp0, N0);
17471       Select(Tmp1, N1);
17472       if (N.Val->hasOneUse()) {
17473         Result = CurDAG->SelectNodeTo(N.Val, X86::OR8rr, MVT::i8, Tmp0, Tmp1);
17474       } else {
17475         ResNode = CurDAG->getTargetNode(X86::OR8rr, MVT::i8, Tmp0, Tmp1);
17476         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17477         Result = SDOperand(ResNode, 0);
17478       }
17479       return;
17480     }
17481 
17482     // Pattern: (or:i16 R16:i16:$src1, R16:i16:$src2)
17483     // Emits: (OR16rr:i16 R16:i16:$src1, R16:i16:$src2)
17484     // Pattern complexity = 2  cost = 1
17485     if (N.Val->getValueType(0) == MVT::i16) {
17486       Select(Tmp0, N0);
17487       Select(Tmp1, N1);
17488       if (N.Val->hasOneUse()) {
17489         Result = CurDAG->SelectNodeTo(N.Val, X86::OR16rr, MVT::i16, Tmp0, Tmp1);
17490       } else {
17491         ResNode = CurDAG->getTargetNode(X86::OR16rr, MVT::i16, Tmp0, Tmp1);
17492         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17493         Result = SDOperand(ResNode, 0);
17494       }
17495       return;
17496     }
17497 
17498     // Pattern: (or:i32 R32:i32:$src1, R32:i32:$src2)
17499     // Emits: (OR32rr:i32 R32:i32:$src1, R32:i32:$src2)
17500     // Pattern complexity = 2  cost = 1
17501     if (N.Val->getValueType(0) == MVT::i32) {
17502       Select(Tmp0, N0);
17503       Select(Tmp1, N1);
17504       if (N.Val->hasOneUse()) {
17505         Result = CurDAG->SelectNodeTo(N.Val, X86::OR32rr, MVT::i32, Tmp0, Tmp1);
17506       } else {
17507         ResNode = CurDAG->getTargetNode(X86::OR32rr, MVT::i32, Tmp0, Tmp1);
17508         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17509         Result = SDOperand(ResNode, 0);
17510       }
17511       return;
17512     }
17513   }
17514 
17515   // Pattern: (or:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17516   // Emits: (ORPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17517   // Pattern complexity = 2  cost = 1
17518   if ((Subtarget->hasSSE1())) {
17519     N0 = N.getOperand(0);
17520     N1 = N.getOperand(1);
17521     if (N.Val->getValueType(0) == MVT::v2i64) {
17522       Select(Tmp0, N0);
17523       Select(Tmp1, N1);
17524       if (N.Val->hasOneUse()) {
17525         Result = CurDAG->SelectNodeTo(N.Val, X86::ORPSrr, MVT::v2i64, Tmp0, Tmp1);
17526       } else {
17527         ResNode = CurDAG->getTargetNode(X86::ORPSrr, MVT::v2i64, Tmp0, Tmp1);
17528         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17529         Result = SDOperand(ResNode, 0);
17530       }
17531       return;
17532     }
17533   }
17534 
17535   // Pattern: (or:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17536   // Emits: (PORrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
17537   // Pattern complexity = 2  cost = 1
17538   if ((Subtarget->hasSSE2())) {
17539     N0 = N.getOperand(0);
17540     N1 = N.getOperand(1);
17541     if (N.Val->getValueType(0) == MVT::v2i64) {
17542       Select(Tmp0, N0);
17543       Select(Tmp1, N1);
17544       if (N.Val->hasOneUse()) {
17545         Result = CurDAG->SelectNodeTo(N.Val, X86::PORrr, MVT::v2i64, Tmp0, Tmp1);
17546       } else {
17547         ResNode = CurDAG->getTargetNode(X86::PORrr, MVT::v2i64, Tmp0, Tmp1);
17548         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17549         Result = SDOperand(ResNode, 0);
17550       }
17551       return;
17552     }
17553   }
17554   std::cerr << "Cannot yet select: ";
17555   N.Val->dump(CurDAG);
17556   std::cerr << '\n';
17557   abort();
17558 }
17559 
17560 void Select_rotl(SDOperand &Result, SDOperand N) {
17561   SDOperand Chain(0, 0);
17562   SDOperand InFlag(0, 0);
17563   SDOperand N0(0, 0);
17564   SDOperand N1(0, 0);
17565   SDOperand Tmp0(0, 0);
17566   SDOperand Tmp1(0, 0);
17567   SDNode *ResNode;
17568   N0 = N.getOperand(0);
17569   N1 = N.getOperand(1);
17570   if (N1.getOpcode() == ISD::Constant) {
17571 
17572     // Pattern: (rotl:i8 R8:i8:$src1, (imm:i8):$src2)
17573     // Emits: (ROL8ri:i8 R8:i8:$src1, (imm:i8):$src2)
17574     // Pattern complexity = 4  cost = 1
17575     if (N.Val->getValueType(0) == MVT::i8 &&
17576         N1.Val->getValueType(0) == MVT::i8) {
17577       Select(Tmp0, N0);
17578       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17579       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17580       if (N.Val->hasOneUse()) {
17581         Result = CurDAG->SelectNodeTo(N.Val, X86::ROL8ri, MVT::i8, Tmp0, Tmp1);
17582       } else {
17583         ResNode = CurDAG->getTargetNode(X86::ROL8ri, MVT::i8, Tmp0, Tmp1);
17584         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17585         Result = SDOperand(ResNode, 0);
17586       }
17587       return;
17588     }
17589 
17590     // Pattern: (rotl:i16 R16:i16:$src1, (imm:i8):$src2)
17591     // Emits: (ROL16ri:i16 R16:i16:$src1, (imm:i8):$src2)
17592     // Pattern complexity = 4  cost = 1
17593     if (N.Val->getValueType(0) == MVT::i16 &&
17594         N1.Val->getValueType(0) == MVT::i8) {
17595       Select(Tmp0, N0);
17596       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17597       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17598       if (N.Val->hasOneUse()) {
17599         Result = CurDAG->SelectNodeTo(N.Val, X86::ROL16ri, MVT::i16, Tmp0, Tmp1);
17600       } else {
17601         ResNode = CurDAG->getTargetNode(X86::ROL16ri, MVT::i16, Tmp0, Tmp1);
17602         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17603         Result = SDOperand(ResNode, 0);
17604       }
17605       return;
17606     }
17607 
17608     // Pattern: (rotl:i32 R32:i32:$src1, (imm:i8):$src2)
17609     // Emits: (ROL32ri:i32 R32:i32:$src1, (imm:i8):$src2)
17610     // Pattern complexity = 4  cost = 1
17611     if (N.Val->getValueType(0) == MVT::i32 &&
17612         N1.Val->getValueType(0) == MVT::i8) {
17613       Select(Tmp0, N0);
17614       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17615       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17616       if (N.Val->hasOneUse()) {
17617         Result = CurDAG->SelectNodeTo(N.Val, X86::ROL32ri, MVT::i32, Tmp0, Tmp1);
17618       } else {
17619         ResNode = CurDAG->getTargetNode(X86::ROL32ri, MVT::i32, Tmp0, Tmp1);
17620         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17621         Result = SDOperand(ResNode, 0);
17622       }
17623       return;
17624     }
17625   }
17626 
17627   // Pattern: (rotl:i8 R8:i8:$src, CL:i8)
17628   // Emits: (ROL8rCL:i8 R8:i8:$src)
17629   // Pattern complexity = 2  cost = 1
17630   if (N.Val->getValueType(0) == MVT::i8 &&
17631       N1.Val->getValueType(0) == MVT::i8) {
17632     Select(Tmp0, N0);
17633     Chain = CurDAG->getEntryNode();
17634     Select(N1, N1);
17635     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17636     Chain = SDOperand(ResNode, 0);
17637     InFlag = SDOperand(ResNode, 1);
17638     if (N.Val->hasOneUse()) {
17639       Result = CurDAG->SelectNodeTo(N.Val, X86::ROL8rCL, MVT::i8, Tmp0, InFlag);
17640     } else {
17641       ResNode = CurDAG->getTargetNode(X86::ROL8rCL, MVT::i8, Tmp0, InFlag);
17642       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17643       Result = SDOperand(ResNode, 0);
17644     }
17645     return;
17646   }
17647 
17648   // Pattern: (rotl:i16 R16:i16:$src, CL:i8)
17649   // Emits: (ROL16rCL:i16 R16:i16:$src)
17650   // Pattern complexity = 2  cost = 1
17651   if (N.Val->getValueType(0) == MVT::i16 &&
17652       N1.Val->getValueType(0) == MVT::i8) {
17653     Select(Tmp0, N0);
17654     Chain = CurDAG->getEntryNode();
17655     Select(N1, N1);
17656     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17657     Chain = SDOperand(ResNode, 0);
17658     InFlag = SDOperand(ResNode, 1);
17659     if (N.Val->hasOneUse()) {
17660       Result = CurDAG->SelectNodeTo(N.Val, X86::ROL16rCL, MVT::i16, Tmp0, InFlag);
17661     } else {
17662       ResNode = CurDAG->getTargetNode(X86::ROL16rCL, MVT::i16, Tmp0, InFlag);
17663       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17664       Result = SDOperand(ResNode, 0);
17665     }
17666     return;
17667   }
17668 
17669   // Pattern: (rotl:i32 R32:i32:$src, CL:i8)
17670   // Emits: (ROL32rCL:i32 R32:i32:$src)
17671   // Pattern complexity = 2  cost = 1
17672   if (N.Val->getValueType(0) == MVT::i32 &&
17673       N1.Val->getValueType(0) == MVT::i8) {
17674     Select(Tmp0, N0);
17675     Chain = CurDAG->getEntryNode();
17676     Select(N1, N1);
17677     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17678     Chain = SDOperand(ResNode, 0);
17679     InFlag = SDOperand(ResNode, 1);
17680     if (N.Val->hasOneUse()) {
17681       Result = CurDAG->SelectNodeTo(N.Val, X86::ROL32rCL, MVT::i32, Tmp0, InFlag);
17682     } else {
17683       ResNode = CurDAG->getTargetNode(X86::ROL32rCL, MVT::i32, Tmp0, InFlag);
17684       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17685       Result = SDOperand(ResNode, 0);
17686     }
17687     return;
17688   }
17689   std::cerr << "Cannot yet select: ";
17690   N.Val->dump(CurDAG);
17691   std::cerr << '\n';
17692   abort();
17693 }
17694 
17695 void Select_rotr(SDOperand &Result, SDOperand N) {
17696   SDOperand Chain(0, 0);
17697   SDOperand InFlag(0, 0);
17698   SDOperand N0(0, 0);
17699   SDOperand N1(0, 0);
17700   SDOperand Tmp0(0, 0);
17701   SDOperand Tmp1(0, 0);
17702   SDNode *ResNode;
17703   N0 = N.getOperand(0);
17704   N1 = N.getOperand(1);
17705   if (N1.getOpcode() == ISD::Constant) {
17706 
17707     // Pattern: (rotr:i8 R8:i8:$src1, (imm:i8):$src2)
17708     // Emits: (ROR8ri:i8 R8:i8:$src1, (imm:i8):$src2)
17709     // Pattern complexity = 4  cost = 1
17710     if (N.Val->getValueType(0) == MVT::i8 &&
17711         N1.Val->getValueType(0) == MVT::i8) {
17712       Select(Tmp0, N0);
17713       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17714       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17715       if (N.Val->hasOneUse()) {
17716         Result = CurDAG->SelectNodeTo(N.Val, X86::ROR8ri, MVT::i8, Tmp0, Tmp1);
17717       } else {
17718         ResNode = CurDAG->getTargetNode(X86::ROR8ri, MVT::i8, Tmp0, Tmp1);
17719         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17720         Result = SDOperand(ResNode, 0);
17721       }
17722       return;
17723     }
17724 
17725     // Pattern: (rotr:i16 R16:i16:$src1, (imm:i8):$src2)
17726     // Emits: (ROR16ri:i16 R16:i16:$src1, (imm:i8):$src2)
17727     // Pattern complexity = 4  cost = 1
17728     if (N.Val->getValueType(0) == MVT::i16 &&
17729         N1.Val->getValueType(0) == MVT::i8) {
17730       Select(Tmp0, N0);
17731       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17732       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17733       if (N.Val->hasOneUse()) {
17734         Result = CurDAG->SelectNodeTo(N.Val, X86::ROR16ri, MVT::i16, Tmp0, Tmp1);
17735       } else {
17736         ResNode = CurDAG->getTargetNode(X86::ROR16ri, MVT::i16, Tmp0, Tmp1);
17737         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17738         Result = SDOperand(ResNode, 0);
17739       }
17740       return;
17741     }
17742 
17743     // Pattern: (rotr:i32 R32:i32:$src1, (imm:i8):$src2)
17744     // Emits: (ROR32ri:i32 R32:i32:$src1, (imm:i8):$src2)
17745     // Pattern complexity = 4  cost = 1
17746     if (N.Val->getValueType(0) == MVT::i32 &&
17747         N1.Val->getValueType(0) == MVT::i8) {
17748       Select(Tmp0, N0);
17749       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
17750       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
17751       if (N.Val->hasOneUse()) {
17752         Result = CurDAG->SelectNodeTo(N.Val, X86::ROR32ri, MVT::i32, Tmp0, Tmp1);
17753       } else {
17754         ResNode = CurDAG->getTargetNode(X86::ROR32ri, MVT::i32, Tmp0, Tmp1);
17755         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17756         Result = SDOperand(ResNode, 0);
17757       }
17758       return;
17759     }
17760   }
17761 
17762   // Pattern: (rotr:i8 R8:i8:$src, CL:i8)
17763   // Emits: (ROR8rCL:i8 R8:i8:$src)
17764   // Pattern complexity = 2  cost = 1
17765   if (N.Val->getValueType(0) == MVT::i8 &&
17766       N1.Val->getValueType(0) == MVT::i8) {
17767     Select(Tmp0, N0);
17768     Chain = CurDAG->getEntryNode();
17769     Select(N1, N1);
17770     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17771     Chain = SDOperand(ResNode, 0);
17772     InFlag = SDOperand(ResNode, 1);
17773     if (N.Val->hasOneUse()) {
17774       Result = CurDAG->SelectNodeTo(N.Val, X86::ROR8rCL, MVT::i8, Tmp0, InFlag);
17775     } else {
17776       ResNode = CurDAG->getTargetNode(X86::ROR8rCL, MVT::i8, Tmp0, InFlag);
17777       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17778       Result = SDOperand(ResNode, 0);
17779     }
17780     return;
17781   }
17782 
17783   // Pattern: (rotr:i16 R16:i16:$src, CL:i8)
17784   // Emits: (ROR16rCL:i16 R16:i16:$src)
17785   // Pattern complexity = 2  cost = 1
17786   if (N.Val->getValueType(0) == MVT::i16 &&
17787       N1.Val->getValueType(0) == MVT::i8) {
17788     Select(Tmp0, N0);
17789     Chain = CurDAG->getEntryNode();
17790     Select(N1, N1);
17791     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17792     Chain = SDOperand(ResNode, 0);
17793     InFlag = SDOperand(ResNode, 1);
17794     if (N.Val->hasOneUse()) {
17795       Result = CurDAG->SelectNodeTo(N.Val, X86::ROR16rCL, MVT::i16, Tmp0, InFlag);
17796     } else {
17797       ResNode = CurDAG->getTargetNode(X86::ROR16rCL, MVT::i16, Tmp0, InFlag);
17798       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17799       Result = SDOperand(ResNode, 0);
17800     }
17801     return;
17802   }
17803 
17804   // Pattern: (rotr:i32 R32:i32:$src, CL:i8)
17805   // Emits: (ROR32rCL:i32 R32:i32:$src)
17806   // Pattern complexity = 2  cost = 1
17807   if (N.Val->getValueType(0) == MVT::i32 &&
17808       N1.Val->getValueType(0) == MVT::i8) {
17809     Select(Tmp0, N0);
17810     Chain = CurDAG->getEntryNode();
17811     Select(N1, N1);
17812     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
17813     Chain = SDOperand(ResNode, 0);
17814     InFlag = SDOperand(ResNode, 1);
17815     if (N.Val->hasOneUse()) {
17816       Result = CurDAG->SelectNodeTo(N.Val, X86::ROR32rCL, MVT::i32, Tmp0, InFlag);
17817     } else {
17818       ResNode = CurDAG->getTargetNode(X86::ROR32rCL, MVT::i32, Tmp0, InFlag);
17819       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17820       Result = SDOperand(ResNode, 0);
17821     }
17822     return;
17823   }
17824   std::cerr << "Cannot yet select: ";
17825   N.Val->dump(CurDAG);
17826   std::cerr << '\n';
17827   abort();
17828 }
17829 
17830 void Select_scalar_to_vector(SDOperand &Result, SDOperand N) {
17831   SDOperand Chain0(0, 0);
17832   SDOperand N0(0, 0);
17833   SDOperand N01(0, 0);
17834   SDOperand Tmp0(0, 0);
17835   SDOperand Tmp1(0, 0);
17836   SDOperand Tmp2(0, 0);
17837   SDOperand Tmp3(0, 0);
17838   SDNode *ResNode;
17839 
17840   // Pattern: (scalar_to_vector:v4f32 (load:f32 addr:i32:$src))
17841   // Emits: (MOVSS2PSrm:v4f32 addr:i32:$src)
17842   // Pattern complexity = 14  cost = 1
17843   if ((Subtarget->hasSSE1())) {
17844     N0 = N.getOperand(0);
17845     if (N0.getOpcode() == ISD::LOAD &&
17846         N0.hasOneUse() &&
17847         !CodeGenMap.count(N0.getValue(0))) {
17848       Chain0 = N0.getOperand(0);
17849       N01 = N0.getOperand(1);
17850       if (N.Val->getValueType(0) == MVT::v4f32 &&
17851           N0.Val->getValueType(0) == MVT::f32 &&
17852           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
17853         Select(Tmp0, Tmp0);
17854         Select(Tmp1, Tmp1);
17855         Select(Tmp2, Tmp2);
17856         Select(Tmp3, Tmp3);
17857         Select(Chain0, Chain0);
17858         ResNode = CurDAG->getTargetNode(X86::MOVSS2PSrm, MVT::v4f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
17859         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
17860         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
17861         AddHandleReplacement(N0.Val, 1, ResNode, 1);
17862         Result = SDOperand(ResNode, N.ResNo);
17863         return;
17864       }
17865     }
17866   }
17867   if ((Subtarget->hasSSE2())) {
17868     N0 = N.getOperand(0);
17869     if (N0.getOpcode() == ISD::LOAD &&
17870         N0.hasOneUse() &&
17871         !CodeGenMap.count(N0.getValue(0))) {
17872       Chain0 = N0.getOperand(0);
17873       N01 = N0.getOperand(1);
17874 
17875       // Pattern: (scalar_to_vector:v2f64 (load:f64 addr:i32:$src))
17876       // Emits: (MOVSD2PDrm:v2f64 addr:i32:$src)
17877       // Pattern complexity = 14  cost = 1
17878       if (N.Val->getValueType(0) == MVT::v2f64 &&
17879           N0.Val->getValueType(0) == MVT::f64 &&
17880           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
17881         Select(Tmp0, Tmp0);
17882         Select(Tmp1, Tmp1);
17883         Select(Tmp2, Tmp2);
17884         Select(Tmp3, Tmp3);
17885         Select(Chain0, Chain0);
17886         ResNode = CurDAG->getTargetNode(X86::MOVSD2PDrm, MVT::v2f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
17887         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
17888         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
17889         AddHandleReplacement(N0.Val, 1, ResNode, 1);
17890         Result = SDOperand(ResNode, N.ResNo);
17891         return;
17892       }
17893 
17894       // Pattern: (scalar_to_vector:v4i32 (load:i32 addr:i32:$src))
17895       // Emits: (MOVDI2PDIrm:v4i32 addr:i32:$src)
17896       // Pattern complexity = 14  cost = 1
17897       if (N.Val->getValueType(0) == MVT::v4i32 &&
17898           N0.Val->getValueType(0) == MVT::i32 &&
17899           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
17900         Select(Tmp0, Tmp0);
17901         Select(Tmp1, Tmp1);
17902         Select(Tmp2, Tmp2);
17903         Select(Tmp3, Tmp3);
17904         Select(Chain0, Chain0);
17905         ResNode = CurDAG->getTargetNode(X86::MOVDI2PDIrm, MVT::v4i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
17906         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
17907         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
17908         AddHandleReplacement(N0.Val, 1, ResNode, 1);
17909         Result = SDOperand(ResNode, N.ResNo);
17910         return;
17911       }
17912 
17913       // Pattern: (scalar_to_vector:v2i64 (load:i64 addr:i32:$src))
17914       // Emits: (MOVQI2PQIrm:v2i64 addr:i32:$src)
17915       // Pattern complexity = 14  cost = 1
17916       if (N.Val->getValueType(0) == MVT::v2i64 &&
17917           N0.Val->getValueType(0) == MVT::i64 &&
17918           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
17919         Select(Tmp0, Tmp0);
17920         Select(Tmp1, Tmp1);
17921         Select(Tmp2, Tmp2);
17922         Select(Tmp3, Tmp3);
17923         Select(Chain0, Chain0);
17924         ResNode = CurDAG->getTargetNode(X86::MOVQI2PQIrm, MVT::v2i64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
17925         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
17926         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
17927         AddHandleReplacement(N0.Val, 1, ResNode, 1);
17928         Result = SDOperand(ResNode, N.ResNo);
17929         return;
17930       }
17931     }
17932   }
17933 
17934   // Pattern: (scalar_to_vector:v4f32 FR32:f32:$src)
17935   // Emits: (MOVSS2PSrr:v4f32 FR32:f32:$src)
17936   // Pattern complexity = 2  cost = 1
17937   if ((Subtarget->hasSSE1())) {
17938     N0 = N.getOperand(0);
17939     if (N.Val->getValueType(0) == MVT::v4f32 &&
17940         N0.Val->getValueType(0) == MVT::f32) {
17941       Select(Tmp0, N0);
17942       if (N.Val->hasOneUse()) {
17943         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVSS2PSrr, MVT::v4f32, Tmp0);
17944       } else {
17945         ResNode = CurDAG->getTargetNode(X86::MOVSS2PSrr, MVT::v4f32, Tmp0);
17946         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17947         Result = SDOperand(ResNode, 0);
17948       }
17949       return;
17950     }
17951   }
17952   if ((Subtarget->hasSSE2())) {
17953     N0 = N.getOperand(0);
17954 
17955     // Pattern: (scalar_to_vector:v2f64 FR64:f64:$src)
17956     // Emits: (MOVSD2PDrr:v2f64 FR64:f64:$src)
17957     // Pattern complexity = 2  cost = 1
17958     if (N.Val->getValueType(0) == MVT::v2f64 &&
17959         N0.Val->getValueType(0) == MVT::f64) {
17960       Select(Tmp0, N0);
17961       if (N.Val->hasOneUse()) {
17962         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVSD2PDrr, MVT::v2f64, Tmp0);
17963       } else {
17964         ResNode = CurDAG->getTargetNode(X86::MOVSD2PDrr, MVT::v2f64, Tmp0);
17965         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17966         Result = SDOperand(ResNode, 0);
17967       }
17968       return;
17969     }
17970 
17971     // Pattern: (scalar_to_vector:v4i32 R32:i32:$src)
17972     // Emits: (MOVDI2PDIrr:v4i32 R32:i32:$src)
17973     // Pattern complexity = 2  cost = 1
17974     if (N.Val->getValueType(0) == MVT::v4i32 &&
17975         N0.Val->getValueType(0) == MVT::i32) {
17976       Select(Tmp0, N0);
17977       if (N.Val->hasOneUse()) {
17978         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVDI2PDIrr, MVT::v4i32, Tmp0);
17979       } else {
17980         ResNode = CurDAG->getTargetNode(X86::MOVDI2PDIrr, MVT::v4i32, Tmp0);
17981         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17982         Result = SDOperand(ResNode, 0);
17983       }
17984       return;
17985     }
17986 
17987     // Pattern: (scalar_to_vector:v2i64 VR64:v8i8:$src)
17988     // Emits: (MOVQI2PQIrr:v2i64 VR64:v8i8:$src)
17989     // Pattern complexity = 2  cost = 1
17990     if (N.Val->getValueType(0) == MVT::v2i64 &&
17991         N0.Val->getValueType(0) == MVT::v8i8) {
17992       Select(Tmp0, N0);
17993       if (N.Val->hasOneUse()) {
17994         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVQI2PQIrr, MVT::v2i64, Tmp0);
17995       } else {
17996         ResNode = CurDAG->getTargetNode(X86::MOVQI2PQIrr, MVT::v2i64, Tmp0);
17997         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
17998         Result = SDOperand(ResNode, 0);
17999       }
18000       return;
18001     }
18002   }
18003   std::cerr << "Cannot yet select: ";
18004   N.Val->dump(CurDAG);
18005   std::cerr << '\n';
18006   abort();
18007 }
18008 
18009 void Select_sext(SDOperand &Result, SDOperand N) {
18010   SDOperand N0(0, 0);
18011   SDOperand Tmp0(0, 0);
18012   SDNode *ResNode;
18013   N0 = N.getOperand(0);
18014 
18015   // Pattern: (sext:i16 R8:i8:$src)
18016   // Emits: (MOVSX16rr8:i16 R8:i8:$src)
18017   // Pattern complexity = 2  cost = 1
18018   if (N.Val->getValueType(0) == MVT::i16 &&
18019       N0.Val->getValueType(0) == MVT::i8) {
18020     Select(Tmp0, N0);
18021     if (N.Val->hasOneUse()) {
18022       Result = CurDAG->SelectNodeTo(N.Val, X86::MOVSX16rr8, MVT::i16, Tmp0);
18023     } else {
18024       ResNode = CurDAG->getTargetNode(X86::MOVSX16rr8, MVT::i16, Tmp0);
18025       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18026       Result = SDOperand(ResNode, 0);
18027     }
18028     return;
18029   }
18030   if (N.Val->getValueType(0) == MVT::i32) {
18031 
18032     // Pattern: (sext:i32 R8:i8:$src)
18033     // Emits: (MOVSX32rr8:i32 R8:i8:$src)
18034     // Pattern complexity = 2  cost = 1
18035     if (N0.Val->getValueType(0) == MVT::i8) {
18036       Select(Tmp0, N0);
18037       if (N.Val->hasOneUse()) {
18038         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVSX32rr8, MVT::i32, Tmp0);
18039       } else {
18040         ResNode = CurDAG->getTargetNode(X86::MOVSX32rr8, MVT::i32, Tmp0);
18041         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18042         Result = SDOperand(ResNode, 0);
18043       }
18044       return;
18045     }
18046 
18047     // Pattern: (sext:i32 R16:i16:$src)
18048     // Emits: (MOVSX32rr16:i32 R16:i16:$src)
18049     // Pattern complexity = 2  cost = 1
18050     if (N0.Val->getValueType(0) == MVT::i16) {
18051       Select(Tmp0, N0);
18052       if (N.Val->hasOneUse()) {
18053         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVSX32rr16, MVT::i32, Tmp0);
18054       } else {
18055         ResNode = CurDAG->getTargetNode(X86::MOVSX32rr16, MVT::i32, Tmp0);
18056         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18057         Result = SDOperand(ResNode, 0);
18058       }
18059       return;
18060     }
18061   }
18062   std::cerr << "Cannot yet select: ";
18063   N.Val->dump(CurDAG);
18064   std::cerr << '\n';
18065   abort();
18066 }
18067 
18068 void Select_sextld(SDOperand &Result, SDOperand N) {
18069   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
18070     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
18071     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
18072     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
18073     Result = Dummy;
18074     return;
18075   }
18076   SDOperand Chain(0, 0);
18077   SDOperand N1(0, 0);
18078   SDOperand N2(0, 0);
18079   SDOperand N3(0, 0);
18080   SDOperand Tmp0(0, 0);
18081   SDOperand Tmp1(0, 0);
18082   SDOperand Tmp2(0, 0);
18083   SDOperand Tmp3(0, 0);
18084   SDNode *ResNode;
18085   Chain = N.getOperand(0);
18086   N1 = N.getOperand(1);
18087   N2 = N.getOperand(2);
18088   N3 = N.getOperand(3);
18089   if (cast<VTSDNode>(N3)->getVT() == MVT::i8) {
18090 
18091     // Pattern: (sextld:i16 addr:i32:$src, srcvalue:Other:$dummy, i8:Other)
18092     // Emits: (MOVSX16rm8:i16 addr:i32:$src)
18093     // Pattern complexity = 12  cost = 1
18094     if (N.Val->getValueType(0) == MVT::i16 &&
18095         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
18096       Select(Tmp0, Tmp0);
18097       Select(Tmp1, Tmp1);
18098       Select(Tmp2, Tmp2);
18099       Select(Tmp3, Tmp3);
18100       Select(Chain, Chain);
18101       ResNode = CurDAG->getTargetNode(X86::MOVSX16rm8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
18102       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18103       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
18104       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
18105       Result = SDOperand(ResNode, N.ResNo);
18106       return;
18107     }
18108 
18109     // Pattern: (sextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i8:Other)
18110     // Emits: (MOVSX32rm8:i32 addr:i32:$src)
18111     // Pattern complexity = 12  cost = 1
18112     if (N.Val->getValueType(0) == MVT::i32 &&
18113         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
18114       Select(Tmp0, Tmp0);
18115       Select(Tmp1, Tmp1);
18116       Select(Tmp2, Tmp2);
18117       Select(Tmp3, Tmp3);
18118       Select(Chain, Chain);
18119       ResNode = CurDAG->getTargetNode(X86::MOVSX32rm8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
18120       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18121       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
18122       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
18123       Result = SDOperand(ResNode, N.ResNo);
18124       return;
18125     }
18126   }
18127 
18128   // Pattern: (sextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i16:Other)
18129   // Emits: (MOVSX32rm16:i32 addr:i32:$src)
18130   // Pattern complexity = 12  cost = 1
18131   if (cast<VTSDNode>(N3)->getVT() == MVT::i16 &&
18132       N.Val->getValueType(0) == MVT::i32 &&
18133       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
18134     Select(Tmp0, Tmp0);
18135     Select(Tmp1, Tmp1);
18136     Select(Tmp2, Tmp2);
18137     Select(Tmp3, Tmp3);
18138     Select(Chain, Chain);
18139     ResNode = CurDAG->getTargetNode(X86::MOVSX32rm16, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
18140     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18141     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
18142     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
18143     Result = SDOperand(ResNode, N.ResNo);
18144     return;
18145   }
18146   if (cast<VTSDNode>(N3)->getVT() == MVT::i1) {
18147 
18148     // Pattern: (sextld:i16 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
18149     // Emits: (MOVSX16rm8:i16 addr:i32:$src)
18150     // Pattern complexity = 12  cost = 1
18151     if (N.Val->getValueType(0) == MVT::i16 &&
18152         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
18153       Select(Tmp0, Tmp0);
18154       Select(Tmp1, Tmp1);
18155       Select(Tmp2, Tmp2);
18156       Select(Tmp3, Tmp3);
18157       Select(Chain, Chain);
18158       ResNode = CurDAG->getTargetNode(X86::MOVSX16rm8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
18159       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18160       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
18161       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
18162       Result = SDOperand(ResNode, N.ResNo);
18163       return;
18164     }
18165 
18166     // Pattern: (sextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
18167     // Emits: (MOVSX32rm8:i32 addr:i32:$src)
18168     // Pattern complexity = 12  cost = 1
18169     if (N.Val->getValueType(0) == MVT::i32 &&
18170         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
18171       Select(Tmp0, Tmp0);
18172       Select(Tmp1, Tmp1);
18173       Select(Tmp2, Tmp2);
18174       Select(Tmp3, Tmp3);
18175       Select(Chain, Chain);
18176       ResNode = CurDAG->getTargetNode(X86::MOVSX32rm8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
18177       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18178       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
18179       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
18180       Result = SDOperand(ResNode, N.ResNo);
18181       return;
18182     }
18183   }
18184   std::cerr << "Cannot yet select: ";
18185   N.Val->dump(CurDAG);
18186   std::cerr << '\n';
18187   abort();
18188 }
18189 
18190 void Select_shl(SDOperand &Result, SDOperand N) {
18191   SDOperand Chain(0, 0);
18192   SDOperand InFlag(0, 0);
18193   SDOperand N0(0, 0);
18194   SDOperand N1(0, 0);
18195   SDOperand Tmp0(0, 0);
18196   SDOperand Tmp1(0, 0);
18197   SDOperand Tmp2(0, 0);
18198   SDOperand Tmp3(0, 0);
18199   SDNode *ResNode;
18200 
18201   // Pattern: leaaddr:i32:$src
18202   // Emits: (LEA32r:i32 leaaddr:i32:$src)
18203   // Pattern complexity = 10  cost = 1
18204   if (N.Val->getValueType(0) == MVT::i32 &&
18205       SelectLEAAddr(N, Tmp0, Tmp1, Tmp2, Tmp3)) {
18206     Select(Tmp0, Tmp0);
18207     Select(Tmp1, Tmp1);
18208     Select(Tmp2, Tmp2);
18209     Select(Tmp3, Tmp3);
18210     if (N.Val->hasOneUse()) {
18211       Result = CurDAG->SelectNodeTo(N.Val, X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
18212     } else {
18213       ResNode = CurDAG->getTargetNode(X86::LEA32r, MVT::i32, Tmp0, Tmp1, Tmp2, Tmp3);
18214       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18215       Result = SDOperand(ResNode, 0);
18216     }
18217     return;
18218   }
18219   N0 = N.getOperand(0);
18220   N1 = N.getOperand(1);
18221   if (isa<ConstantSDNode>(N1)) {
18222     int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
18223     if (CN0 == 1) {
18224 
18225       // Pattern: (shl:i8 R8:i8:$src1, 1:i8)
18226       // Emits: (ADD8rr:i8 R8:i8:$src1, R8:i8:$src1)
18227       // Pattern complexity = 5  cost = 1
18228       if (N.Val->getValueType(0) == MVT::i8 &&
18229           N1.Val->getValueType(0) == MVT::i8) {
18230         Select(Tmp1, N0);
18231         if (N.Val->hasOneUse()) {
18232           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD8rr, MVT::i8, Tmp1, Tmp1);
18233         } else {
18234           ResNode = CurDAG->getTargetNode(X86::ADD8rr, MVT::i8, Tmp1, Tmp1);
18235           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18236           Result = SDOperand(ResNode, 0);
18237         }
18238         return;
18239       }
18240 
18241       // Pattern: (shl:i16 R16:i16:$src1, 1:i8)
18242       // Emits: (ADD16rr:i16 R16:i16:$src1, R16:i16:$src1)
18243       // Pattern complexity = 5  cost = 1
18244       if (N.Val->getValueType(0) == MVT::i16 &&
18245           N1.Val->getValueType(0) == MVT::i8) {
18246         Select(Tmp1, N0);
18247         if (N.Val->hasOneUse()) {
18248           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD16rr, MVT::i16, Tmp1, Tmp1);
18249         } else {
18250           ResNode = CurDAG->getTargetNode(X86::ADD16rr, MVT::i16, Tmp1, Tmp1);
18251           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18252           Result = SDOperand(ResNode, 0);
18253         }
18254         return;
18255       }
18256 
18257       // Pattern: (shl:i32 R32:i32:$src1, 1:i8)
18258       // Emits: (ADD32rr:i32 R32:i32:$src1, R32:i32:$src1)
18259       // Pattern complexity = 5  cost = 1
18260       if (N.Val->getValueType(0) == MVT::i32 &&
18261           N1.Val->getValueType(0) == MVT::i8) {
18262         Select(Tmp1, N0);
18263         if (N.Val->hasOneUse()) {
18264           Result = CurDAG->SelectNodeTo(N.Val, X86::ADD32rr, MVT::i32, Tmp1, Tmp1);
18265         } else {
18266           ResNode = CurDAG->getTargetNode(X86::ADD32rr, MVT::i32, Tmp1, Tmp1);
18267           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18268           Result = SDOperand(ResNode, 0);
18269         }
18270         return;
18271       }
18272     }
18273   }
18274   if (N1.getOpcode() == ISD::Constant) {
18275 
18276     // Pattern: (shl:i8 R8:i8:$src1, (imm:i8):$src2)
18277     // Emits: (SHL8ri:i8 R8:i8:$src1, (imm:i8):$src2)
18278     // Pattern complexity = 4  cost = 1
18279     if (N.Val->getValueType(0) == MVT::i8 &&
18280         N1.Val->getValueType(0) == MVT::i8) {
18281       Select(Tmp0, N0);
18282       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18283       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18284       if (N.Val->hasOneUse()) {
18285         Result = CurDAG->SelectNodeTo(N.Val, X86::SHL8ri, MVT::i8, Tmp0, Tmp1);
18286       } else {
18287         ResNode = CurDAG->getTargetNode(X86::SHL8ri, MVT::i8, Tmp0, Tmp1);
18288         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18289         Result = SDOperand(ResNode, 0);
18290       }
18291       return;
18292     }
18293 
18294     // Pattern: (shl:i16 R16:i16:$src1, (imm:i8):$src2)
18295     // Emits: (SHL16ri:i16 R16:i16:$src1, (imm:i8):$src2)
18296     // Pattern complexity = 4  cost = 1
18297     if (N.Val->getValueType(0) == MVT::i16 &&
18298         N1.Val->getValueType(0) == MVT::i8) {
18299       Select(Tmp0, N0);
18300       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18301       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18302       if (N.Val->hasOneUse()) {
18303         Result = CurDAG->SelectNodeTo(N.Val, X86::SHL16ri, MVT::i16, Tmp0, Tmp1);
18304       } else {
18305         ResNode = CurDAG->getTargetNode(X86::SHL16ri, MVT::i16, Tmp0, Tmp1);
18306         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18307         Result = SDOperand(ResNode, 0);
18308       }
18309       return;
18310     }
18311 
18312     // Pattern: (shl:i32 R32:i32:$src1, (imm:i8):$src2)
18313     // Emits: (SHL32ri:i32 R32:i32:$src1, (imm:i8):$src2)
18314     // Pattern complexity = 4  cost = 1
18315     if (N.Val->getValueType(0) == MVT::i32 &&
18316         N1.Val->getValueType(0) == MVT::i8) {
18317       Select(Tmp0, N0);
18318       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18319       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18320       if (N.Val->hasOneUse()) {
18321         Result = CurDAG->SelectNodeTo(N.Val, X86::SHL32ri, MVT::i32, Tmp0, Tmp1);
18322       } else {
18323         ResNode = CurDAG->getTargetNode(X86::SHL32ri, MVT::i32, Tmp0, Tmp1);
18324         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18325         Result = SDOperand(ResNode, 0);
18326       }
18327       return;
18328     }
18329   }
18330 
18331   // Pattern: (shl:i8 R8:i8:$src, CL:i8)
18332   // Emits: (SHL8rCL:i8 R8:i8:$src)
18333   // Pattern complexity = 2  cost = 1
18334   if (N.Val->getValueType(0) == MVT::i8 &&
18335       N1.Val->getValueType(0) == MVT::i8) {
18336     Select(Tmp0, N0);
18337     Chain = CurDAG->getEntryNode();
18338     Select(N1, N1);
18339     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18340     Chain = SDOperand(ResNode, 0);
18341     InFlag = SDOperand(ResNode, 1);
18342     if (N.Val->hasOneUse()) {
18343       Result = CurDAG->SelectNodeTo(N.Val, X86::SHL8rCL, MVT::i8, Tmp0, InFlag);
18344     } else {
18345       ResNode = CurDAG->getTargetNode(X86::SHL8rCL, MVT::i8, Tmp0, InFlag);
18346       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18347       Result = SDOperand(ResNode, 0);
18348     }
18349     return;
18350   }
18351 
18352   // Pattern: (shl:i16 R16:i16:$src, CL:i8)
18353   // Emits: (SHL16rCL:i16 R16:i16:$src)
18354   // Pattern complexity = 2  cost = 1
18355   if (N.Val->getValueType(0) == MVT::i16 &&
18356       N1.Val->getValueType(0) == MVT::i8) {
18357     Select(Tmp0, N0);
18358     Chain = CurDAG->getEntryNode();
18359     Select(N1, N1);
18360     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18361     Chain = SDOperand(ResNode, 0);
18362     InFlag = SDOperand(ResNode, 1);
18363     if (N.Val->hasOneUse()) {
18364       Result = CurDAG->SelectNodeTo(N.Val, X86::SHL16rCL, MVT::i16, Tmp0, InFlag);
18365     } else {
18366       ResNode = CurDAG->getTargetNode(X86::SHL16rCL, MVT::i16, Tmp0, InFlag);
18367       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18368       Result = SDOperand(ResNode, 0);
18369     }
18370     return;
18371   }
18372 
18373   // Pattern: (shl:i32 R32:i32:$src, CL:i8)
18374   // Emits: (SHL32rCL:i32 R32:i32:$src)
18375   // Pattern complexity = 2  cost = 1
18376   if (N.Val->getValueType(0) == MVT::i32 &&
18377       N1.Val->getValueType(0) == MVT::i8) {
18378     Select(Tmp0, N0);
18379     Chain = CurDAG->getEntryNode();
18380     Select(N1, N1);
18381     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18382     Chain = SDOperand(ResNode, 0);
18383     InFlag = SDOperand(ResNode, 1);
18384     if (N.Val->hasOneUse()) {
18385       Result = CurDAG->SelectNodeTo(N.Val, X86::SHL32rCL, MVT::i32, Tmp0, InFlag);
18386     } else {
18387       ResNode = CurDAG->getTargetNode(X86::SHL32rCL, MVT::i32, Tmp0, InFlag);
18388       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18389       Result = SDOperand(ResNode, 0);
18390     }
18391     return;
18392   }
18393   std::cerr << "Cannot yet select: ";
18394   N.Val->dump(CurDAG);
18395   std::cerr << '\n';
18396   abort();
18397 }
18398 
18399 void Select_sint_to_fp(SDOperand &Result, SDOperand N) {
18400   SDOperand Chain0(0, 0);
18401   SDOperand N0(0, 0);
18402   SDOperand N01(0, 0);
18403   SDOperand Tmp0(0, 0);
18404   SDOperand Tmp1(0, 0);
18405   SDOperand Tmp2(0, 0);
18406   SDOperand Tmp3(0, 0);
18407   SDNode *ResNode;
18408 
18409   // Pattern: (sint_to_fp:f32 (load:i32 addr:i32:$src))
18410   // Emits: (CVTSI2SSrm:f32 addr:i32:$src)
18411   // Pattern complexity = 14  cost = 1
18412   if ((Subtarget->hasSSE1())) {
18413     N0 = N.getOperand(0);
18414     if (N0.getOpcode() == ISD::LOAD &&
18415         N0.hasOneUse() &&
18416         !CodeGenMap.count(N0.getValue(0))) {
18417       Chain0 = N0.getOperand(0);
18418       N01 = N0.getOperand(1);
18419       if (N.Val->getValueType(0) == MVT::f32 &&
18420           N0.Val->getValueType(0) == MVT::i32 &&
18421           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
18422         Select(Tmp0, Tmp0);
18423         Select(Tmp1, Tmp1);
18424         Select(Tmp2, Tmp2);
18425         Select(Tmp3, Tmp3);
18426         Select(Chain0, Chain0);
18427         ResNode = CurDAG->getTargetNode(X86::CVTSI2SSrm, MVT::f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
18428         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18429         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
18430         AddHandleReplacement(N0.Val, 1, ResNode, 1);
18431         Result = SDOperand(ResNode, N.ResNo);
18432         return;
18433       }
18434     }
18435   }
18436 
18437   // Pattern: (sint_to_fp:f64 (load:i32 addr:i32:$src))
18438   // Emits: (CVTSI2SDrm:f64 addr:i32:$src)
18439   // Pattern complexity = 14  cost = 1
18440   if ((Subtarget->hasSSE2())) {
18441     N0 = N.getOperand(0);
18442     if (N0.getOpcode() == ISD::LOAD &&
18443         N0.hasOneUse() &&
18444         !CodeGenMap.count(N0.getValue(0))) {
18445       Chain0 = N0.getOperand(0);
18446       N01 = N0.getOperand(1);
18447       if (N.Val->getValueType(0) == MVT::f64 &&
18448           N0.Val->getValueType(0) == MVT::i32 &&
18449           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
18450         Select(Tmp0, Tmp0);
18451         Select(Tmp1, Tmp1);
18452         Select(Tmp2, Tmp2);
18453         Select(Tmp3, Tmp3);
18454         Select(Chain0, Chain0);
18455         ResNode = CurDAG->getTargetNode(X86::CVTSI2SDrm, MVT::f64, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
18456         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18457         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
18458         AddHandleReplacement(N0.Val, 1, ResNode, 1);
18459         Result = SDOperand(ResNode, N.ResNo);
18460         return;
18461       }
18462     }
18463   }
18464 
18465   // Pattern: (sint_to_fp:f32 R32:i32:$src)
18466   // Emits: (CVTSI2SSrr:f32 R32:i32:$src)
18467   // Pattern complexity = 2  cost = 1
18468   if ((Subtarget->hasSSE1())) {
18469     N0 = N.getOperand(0);
18470     if (N.Val->getValueType(0) == MVT::f32 &&
18471         N0.Val->getValueType(0) == MVT::i32) {
18472       Select(Tmp0, N0);
18473       if (N.Val->hasOneUse()) {
18474         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSI2SSrr, MVT::f32, Tmp0);
18475       } else {
18476         ResNode = CurDAG->getTargetNode(X86::CVTSI2SSrr, MVT::f32, Tmp0);
18477         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18478         Result = SDOperand(ResNode, 0);
18479       }
18480       return;
18481     }
18482   }
18483 
18484   // Pattern: (sint_to_fp:f64 R32:i32:$src)
18485   // Emits: (CVTSI2SDrr:f64 R32:i32:$src)
18486   // Pattern complexity = 2  cost = 1
18487   if ((Subtarget->hasSSE2())) {
18488     N0 = N.getOperand(0);
18489     if (N.Val->getValueType(0) == MVT::f64 &&
18490         N0.Val->getValueType(0) == MVT::i32) {
18491       Select(Tmp0, N0);
18492       if (N.Val->hasOneUse()) {
18493         Result = CurDAG->SelectNodeTo(N.Val, X86::CVTSI2SDrr, MVT::f64, Tmp0);
18494       } else {
18495         ResNode = CurDAG->getTargetNode(X86::CVTSI2SDrr, MVT::f64, Tmp0);
18496         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18497         Result = SDOperand(ResNode, 0);
18498       }
18499       return;
18500     }
18501   }
18502   std::cerr << "Cannot yet select: ";
18503   N.Val->dump(CurDAG);
18504   std::cerr << '\n';
18505   abort();
18506 }
18507 
18508 void Select_sra(SDOperand &Result, SDOperand N) {
18509   SDOperand Chain(0, 0);
18510   SDOperand InFlag(0, 0);
18511   SDOperand N0(0, 0);
18512   SDOperand N1(0, 0);
18513   SDOperand Tmp0(0, 0);
18514   SDOperand Tmp1(0, 0);
18515   SDNode *ResNode;
18516   N0 = N.getOperand(0);
18517   N1 = N.getOperand(1);
18518   if (N1.getOpcode() == ISD::Constant) {
18519 
18520     // Pattern: (sra:i8 R8:i8:$src1, (imm:i8):$src2)
18521     // Emits: (SAR8ri:i8 R8:i8:$src1, (imm:i8):$src2)
18522     // Pattern complexity = 4  cost = 1
18523     if (N.Val->getValueType(0) == MVT::i8 &&
18524         N1.Val->getValueType(0) == MVT::i8) {
18525       Select(Tmp0, N0);
18526       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18527       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18528       if (N.Val->hasOneUse()) {
18529         Result = CurDAG->SelectNodeTo(N.Val, X86::SAR8ri, MVT::i8, Tmp0, Tmp1);
18530       } else {
18531         ResNode = CurDAG->getTargetNode(X86::SAR8ri, MVT::i8, Tmp0, Tmp1);
18532         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18533         Result = SDOperand(ResNode, 0);
18534       }
18535       return;
18536     }
18537 
18538     // Pattern: (sra:i16 R16:i16:$src1, (imm:i8):$src2)
18539     // Emits: (SAR16ri:i16 R16:i16:$src1, (imm:i8):$src2)
18540     // Pattern complexity = 4  cost = 1
18541     if (N.Val->getValueType(0) == MVT::i16 &&
18542         N1.Val->getValueType(0) == MVT::i8) {
18543       Select(Tmp0, N0);
18544       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18545       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18546       if (N.Val->hasOneUse()) {
18547         Result = CurDAG->SelectNodeTo(N.Val, X86::SAR16ri, MVT::i16, Tmp0, Tmp1);
18548       } else {
18549         ResNode = CurDAG->getTargetNode(X86::SAR16ri, MVT::i16, Tmp0, Tmp1);
18550         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18551         Result = SDOperand(ResNode, 0);
18552       }
18553       return;
18554     }
18555 
18556     // Pattern: (sra:i32 R32:i32:$src1, (imm:i8):$src2)
18557     // Emits: (SAR32ri:i32 R32:i32:$src1, (imm:i8):$src2)
18558     // Pattern complexity = 4  cost = 1
18559     if (N.Val->getValueType(0) == MVT::i32 &&
18560         N1.Val->getValueType(0) == MVT::i8) {
18561       Select(Tmp0, N0);
18562       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18563       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18564       if (N.Val->hasOneUse()) {
18565         Result = CurDAG->SelectNodeTo(N.Val, X86::SAR32ri, MVT::i32, Tmp0, Tmp1);
18566       } else {
18567         ResNode = CurDAG->getTargetNode(X86::SAR32ri, MVT::i32, Tmp0, Tmp1);
18568         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18569         Result = SDOperand(ResNode, 0);
18570       }
18571       return;
18572     }
18573   }
18574 
18575   // Pattern: (sra:i8 R8:i8:$src, CL:i8)
18576   // Emits: (SAR8rCL:i8 R8:i8:$src)
18577   // Pattern complexity = 2  cost = 1
18578   if (N.Val->getValueType(0) == MVT::i8 &&
18579       N1.Val->getValueType(0) == MVT::i8) {
18580     Select(Tmp0, N0);
18581     Chain = CurDAG->getEntryNode();
18582     Select(N1, N1);
18583     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18584     Chain = SDOperand(ResNode, 0);
18585     InFlag = SDOperand(ResNode, 1);
18586     if (N.Val->hasOneUse()) {
18587       Result = CurDAG->SelectNodeTo(N.Val, X86::SAR8rCL, MVT::i8, Tmp0, InFlag);
18588     } else {
18589       ResNode = CurDAG->getTargetNode(X86::SAR8rCL, MVT::i8, Tmp0, InFlag);
18590       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18591       Result = SDOperand(ResNode, 0);
18592     }
18593     return;
18594   }
18595 
18596   // Pattern: (sra:i16 R16:i16:$src, CL:i8)
18597   // Emits: (SAR16rCL:i16 R16:i16:$src)
18598   // Pattern complexity = 2  cost = 1
18599   if (N.Val->getValueType(0) == MVT::i16 &&
18600       N1.Val->getValueType(0) == MVT::i8) {
18601     Select(Tmp0, N0);
18602     Chain = CurDAG->getEntryNode();
18603     Select(N1, N1);
18604     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18605     Chain = SDOperand(ResNode, 0);
18606     InFlag = SDOperand(ResNode, 1);
18607     if (N.Val->hasOneUse()) {
18608       Result = CurDAG->SelectNodeTo(N.Val, X86::SAR16rCL, MVT::i16, Tmp0, InFlag);
18609     } else {
18610       ResNode = CurDAG->getTargetNode(X86::SAR16rCL, MVT::i16, Tmp0, InFlag);
18611       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18612       Result = SDOperand(ResNode, 0);
18613     }
18614     return;
18615   }
18616 
18617   // Pattern: (sra:i32 R32:i32:$src, CL:i8)
18618   // Emits: (SAR32rCL:i32 R32:i32:$src)
18619   // Pattern complexity = 2  cost = 1
18620   if (N.Val->getValueType(0) == MVT::i32 &&
18621       N1.Val->getValueType(0) == MVT::i8) {
18622     Select(Tmp0, N0);
18623     Chain = CurDAG->getEntryNode();
18624     Select(N1, N1);
18625     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18626     Chain = SDOperand(ResNode, 0);
18627     InFlag = SDOperand(ResNode, 1);
18628     if (N.Val->hasOneUse()) {
18629       Result = CurDAG->SelectNodeTo(N.Val, X86::SAR32rCL, MVT::i32, Tmp0, InFlag);
18630     } else {
18631       ResNode = CurDAG->getTargetNode(X86::SAR32rCL, MVT::i32, Tmp0, InFlag);
18632       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18633       Result = SDOperand(ResNode, 0);
18634     }
18635     return;
18636   }
18637   std::cerr << "Cannot yet select: ";
18638   N.Val->dump(CurDAG);
18639   std::cerr << '\n';
18640   abort();
18641 }
18642 
18643 void Select_srl(SDOperand &Result, SDOperand N) {
18644   SDOperand Chain(0, 0);
18645   SDOperand InFlag(0, 0);
18646   SDOperand N0(0, 0);
18647   SDOperand N1(0, 0);
18648   SDOperand Tmp0(0, 0);
18649   SDOperand Tmp1(0, 0);
18650   SDNode *ResNode;
18651   N0 = N.getOperand(0);
18652   N1 = N.getOperand(1);
18653   if (N1.getOpcode() == ISD::Constant) {
18654 
18655     // Pattern: (srl:i8 R8:i8:$src1, (imm:i8):$src2)
18656     // Emits: (SHR8ri:i8 R8:i8:$src1, (imm:i8):$src2)
18657     // Pattern complexity = 4  cost = 1
18658     if (N.Val->getValueType(0) == MVT::i8 &&
18659         N1.Val->getValueType(0) == MVT::i8) {
18660       Select(Tmp0, N0);
18661       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18662       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18663       if (N.Val->hasOneUse()) {
18664         Result = CurDAG->SelectNodeTo(N.Val, X86::SHR8ri, MVT::i8, Tmp0, Tmp1);
18665       } else {
18666         ResNode = CurDAG->getTargetNode(X86::SHR8ri, MVT::i8, Tmp0, Tmp1);
18667         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18668         Result = SDOperand(ResNode, 0);
18669       }
18670       return;
18671     }
18672 
18673     // Pattern: (srl:i16 R16:i16:$src1, (imm:i8):$src2)
18674     // Emits: (SHR16ri:i16 R16:i16:$src1, (imm:i8):$src2)
18675     // Pattern complexity = 4  cost = 1
18676     if (N.Val->getValueType(0) == MVT::i16 &&
18677         N1.Val->getValueType(0) == MVT::i8) {
18678       Select(Tmp0, N0);
18679       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18680       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18681       if (N.Val->hasOneUse()) {
18682         Result = CurDAG->SelectNodeTo(N.Val, X86::SHR16ri, MVT::i16, Tmp0, Tmp1);
18683       } else {
18684         ResNode = CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Tmp0, Tmp1);
18685         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18686         Result = SDOperand(ResNode, 0);
18687       }
18688       return;
18689     }
18690 
18691     // Pattern: (srl:i32 R32:i32:$src1, (imm:i8):$src2)
18692     // Emits: (SHR32ri:i32 R32:i32:$src1, (imm:i8):$src2)
18693     // Pattern complexity = 4  cost = 1
18694     if (N.Val->getValueType(0) == MVT::i32 &&
18695         N1.Val->getValueType(0) == MVT::i8) {
18696       Select(Tmp0, N0);
18697       unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
18698       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
18699       if (N.Val->hasOneUse()) {
18700         Result = CurDAG->SelectNodeTo(N.Val, X86::SHR32ri, MVT::i32, Tmp0, Tmp1);
18701       } else {
18702         ResNode = CurDAG->getTargetNode(X86::SHR32ri, MVT::i32, Tmp0, Tmp1);
18703         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18704         Result = SDOperand(ResNode, 0);
18705       }
18706       return;
18707     }
18708   }
18709 
18710   // Pattern: (srl:i8 R8:i8:$src, CL:i8)
18711   // Emits: (SHR8rCL:i8 R8:i8:$src)
18712   // Pattern complexity = 2  cost = 1
18713   if (N.Val->getValueType(0) == MVT::i8 &&
18714       N1.Val->getValueType(0) == MVT::i8) {
18715     Select(Tmp0, N0);
18716     Chain = CurDAG->getEntryNode();
18717     Select(N1, N1);
18718     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18719     Chain = SDOperand(ResNode, 0);
18720     InFlag = SDOperand(ResNode, 1);
18721     if (N.Val->hasOneUse()) {
18722       Result = CurDAG->SelectNodeTo(N.Val, X86::SHR8rCL, MVT::i8, Tmp0, InFlag);
18723     } else {
18724       ResNode = CurDAG->getTargetNode(X86::SHR8rCL, MVT::i8, Tmp0, InFlag);
18725       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18726       Result = SDOperand(ResNode, 0);
18727     }
18728     return;
18729   }
18730 
18731   // Pattern: (srl:i16 R16:i16:$src, CL:i8)
18732   // Emits: (SHR16rCL:i16 R16:i16:$src)
18733   // Pattern complexity = 2  cost = 1
18734   if (N.Val->getValueType(0) == MVT::i16 &&
18735       N1.Val->getValueType(0) == MVT::i8) {
18736     Select(Tmp0, N0);
18737     Chain = CurDAG->getEntryNode();
18738     Select(N1, N1);
18739     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18740     Chain = SDOperand(ResNode, 0);
18741     InFlag = SDOperand(ResNode, 1);
18742     if (N.Val->hasOneUse()) {
18743       Result = CurDAG->SelectNodeTo(N.Val, X86::SHR16rCL, MVT::i16, Tmp0, InFlag);
18744     } else {
18745       ResNode = CurDAG->getTargetNode(X86::SHR16rCL, MVT::i16, Tmp0, InFlag);
18746       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18747       Result = SDOperand(ResNode, 0);
18748     }
18749     return;
18750   }
18751 
18752   // Pattern: (srl:i32 R32:i32:$src, CL:i8)
18753   // Emits: (SHR32rCL:i32 R32:i32:$src)
18754   // Pattern complexity = 2  cost = 1
18755   if (N.Val->getValueType(0) == MVT::i32 &&
18756       N1.Val->getValueType(0) == MVT::i8) {
18757     Select(Tmp0, N0);
18758     Chain = CurDAG->getEntryNode();
18759     Select(N1, N1);
18760     ResNode = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(X86::CL, MVT::i8), N1, InFlag).Val;
18761     Chain = SDOperand(ResNode, 0);
18762     InFlag = SDOperand(ResNode, 1);
18763     if (N.Val->hasOneUse()) {
18764       Result = CurDAG->SelectNodeTo(N.Val, X86::SHR32rCL, MVT::i32, Tmp0, InFlag);
18765     } else {
18766       ResNode = CurDAG->getTargetNode(X86::SHR32rCL, MVT::i32, Tmp0, InFlag);
18767       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
18768       Result = SDOperand(ResNode, 0);
18769     }
18770     return;
18771   }
18772   std::cerr << "Cannot yet select: ";
18773   N.Val->dump(CurDAG);
18774   std::cerr << '\n';
18775   abort();
18776 }
18777 
18778 void Select_store(SDOperand &Result, SDOperand N) {
18779   SDOperand Chain(0, 0);
18780   SDOperand Chain10(0, 0);
18781   SDOperand Chain100(0, 0);
18782   SDOperand Chain11(0, 0);
18783   SDOperand Chain110(0, 0);
18784   SDOperand InFlag(0, 0);
18785   SDOperand N1(0, 0);
18786   SDOperand N10(0, 0);
18787   SDOperand N100(0, 0);
18788   SDOperand N1000(0, 0);
18789   SDOperand N1001(0, 0);
18790   SDOperand N101(0, 0);
18791   SDOperand N1010(0, 0);
18792   SDOperand N1011(0, 0);
18793   SDOperand N102(0, 0);
18794   SDOperand N11(0, 0);
18795   SDOperand N110(0, 0);
18796   SDOperand N1101(0, 0);
18797   SDOperand N111(0, 0);
18798   SDOperand N1110(0, 0);
18799   SDOperand N1111(0, 0);
18800   SDOperand N12(0, 0);
18801   SDOperand N2(0, 0);
18802   SDOperand Tmp0(0, 0);
18803   SDOperand Tmp1(0, 0);
18804   SDOperand Tmp2(0, 0);
18805   SDOperand Tmp3(0, 0);
18806   SDOperand Tmp4(0, 0);
18807   SDOperand Tmp5(0, 0);
18808   SDNode *OldTF;
18809   SDNode *ResNode;
18810   {
18811     Chain = N.getOperand(0);
18812     N1 = N.getOperand(1);
18813     if (N1.getOpcode() == ISD::OR &&
18814         N1.hasOneUse() &&
18815         !CodeGenMap.count(N1.getValue(0))) {
18816       N10 = N1.getOperand(0);
18817 
18818       // Pattern: (store:void (or:i32 (srl:i32 (load:i32 addr:i32:$dst), CL:i8:$amt), (shl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt))), addr:i32:$dst)
18819       // Emits: (SHRD32mrCL:void addr:i32:$dst, R32:i32:$src2)
18820       // Pattern complexity = 35  cost = 1
18821       if (N10.getOpcode() == ISD::SRL &&
18822           N10.hasOneUse() &&
18823           !CodeGenMap.count(N10.getValue(0))) {
18824         N100 = N10.getOperand(0);
18825         if (N100.getOpcode() == ISD::LOAD &&
18826             N100.hasOneUse() &&
18827             !CodeGenMap.count(N100.getValue(0)) &&
18828             !isNonImmUse(N10.Val, N100.Val) &&
18829             (Chain100 = UpdateFoldedChain(CurDAG, N100.Val, Chain.Val, OldTF)).Val) {
18830           N1001 = N100.getOperand(1);
18831           N101 = N10.getOperand(1);
18832           N11 = N1.getOperand(1);
18833           if (N11.getOpcode() == ISD::SHL) {
18834             N110 = N11.getOperand(0);
18835             N111 = N11.getOperand(1);
18836             if (N111.getOpcode() == ISD::SUB) {
18837               N1110 = N111.getOperand(0);
18838               if (isa<ConstantSDNode>(N1110)) {
18839                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
18840                 if (CN0 == 32) {
18841                   N1111 = N111.getOperand(1);
18842                   if (N101 == N1111) {
18843                     N2 = N.getOperand(2);
18844                     if (N1001 == N2 &&
18845                         N1.Val->getValueType(0) == MVT::i32 &&
18846                         N101.Val->getValueType(0) == MVT::i8 &&
18847                         N111.Val->getValueType(0) == MVT::i8 &&
18848                         SelectAddr(N1001, Tmp1, Tmp2, Tmp3, Tmp4)) {
18849                       Select(Tmp1, Tmp1);
18850                       Select(Tmp2, Tmp2);
18851                       Select(Tmp3, Tmp3);
18852                       Select(Tmp4, Tmp4);
18853                       Select(Tmp5, N110);
18854                       Select(Chain100, Chain100);
18855                       Select(N101, N101);
18856                       ResNode = CurDAG->getCopyToReg(Chain100, CurDAG->getRegister(X86::CL, MVT::i8), N101, InFlag).Val;
18857                       Chain100 = SDOperand(ResNode, 0);
18858                       InFlag = SDOperand(ResNode, 1);
18859                       ResNode = CurDAG->getTargetNode(X86::SHRD32mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain100, InFlag);
18860                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain100.Val, 0);
18861                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18862                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 0);
18863                       AddHandleReplacement(N100.Val, 1, ResNode, 0);
18864                       Result = SDOperand(ResNode, N.ResNo);
18865                       return;
18866                     }
18867                   }
18868                 }
18869               }
18870             }
18871           }
18872         }
18873       }
18874 
18875       // Pattern: (store:void (or:i32 (shl:i32 (load:i32 addr:i32:$dst), CL:i8:$amt), (srl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt))), addr:i32:$dst)
18876       // Emits: (SHLD32mrCL:void addr:i32:$dst, R32:i32:$src2)
18877       // Pattern complexity = 35  cost = 1
18878       if (N10.getOpcode() == ISD::SHL &&
18879           N10.hasOneUse() &&
18880           !CodeGenMap.count(N10.getValue(0))) {
18881         N100 = N10.getOperand(0);
18882         if (N100.getOpcode() == ISD::LOAD &&
18883             N100.hasOneUse() &&
18884             !CodeGenMap.count(N100.getValue(0)) &&
18885             !isNonImmUse(N10.Val, N100.Val) &&
18886             (Chain100 = UpdateFoldedChain(CurDAG, N100.Val, Chain.Val, OldTF)).Val) {
18887           N1001 = N100.getOperand(1);
18888           N101 = N10.getOperand(1);
18889           N11 = N1.getOperand(1);
18890           if (N11.getOpcode() == ISD::SRL) {
18891             N110 = N11.getOperand(0);
18892             N111 = N11.getOperand(1);
18893             if (N111.getOpcode() == ISD::SUB) {
18894               N1110 = N111.getOperand(0);
18895               if (isa<ConstantSDNode>(N1110)) {
18896                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
18897                 if (CN0 == 32) {
18898                   N1111 = N111.getOperand(1);
18899                   if (N101 == N1111) {
18900                     N2 = N.getOperand(2);
18901                     if (N1001 == N2 &&
18902                         N1.Val->getValueType(0) == MVT::i32 &&
18903                         N101.Val->getValueType(0) == MVT::i8 &&
18904                         N111.Val->getValueType(0) == MVT::i8 &&
18905                         SelectAddr(N1001, Tmp1, Tmp2, Tmp3, Tmp4)) {
18906                       Select(Tmp1, Tmp1);
18907                       Select(Tmp2, Tmp2);
18908                       Select(Tmp3, Tmp3);
18909                       Select(Tmp4, Tmp4);
18910                       Select(Tmp5, N110);
18911                       Select(Chain100, Chain100);
18912                       Select(N101, N101);
18913                       ResNode = CurDAG->getCopyToReg(Chain100, CurDAG->getRegister(X86::CL, MVT::i8), N101, InFlag).Val;
18914                       Chain100 = SDOperand(ResNode, 0);
18915                       InFlag = SDOperand(ResNode, 1);
18916                       ResNode = CurDAG->getTargetNode(X86::SHLD32mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain100, InFlag);
18917                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain100.Val, 0);
18918                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18919                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 0);
18920                       AddHandleReplacement(N100.Val, 1, ResNode, 0);
18921                       Result = SDOperand(ResNode, N.ResNo);
18922                       return;
18923                     }
18924                   }
18925                 }
18926               }
18927             }
18928           }
18929         }
18930       }
18931 
18932       // Pattern: (store:void (or:i16 (srl:i16 (load:i16 addr:i32:$dst), CL:i8:$amt), (shl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt))), addr:i32:$dst)
18933       // Emits: (SHRD16mrCL:void addr:i32:$dst, R16:i16:$src2)
18934       // Pattern complexity = 35  cost = 1
18935       if (N10.getOpcode() == ISD::SRL &&
18936           N10.hasOneUse() &&
18937           !CodeGenMap.count(N10.getValue(0))) {
18938         N100 = N10.getOperand(0);
18939         if (N100.getOpcode() == ISD::LOAD &&
18940             N100.hasOneUse() &&
18941             !CodeGenMap.count(N100.getValue(0)) &&
18942             !isNonImmUse(N10.Val, N100.Val) &&
18943             (Chain100 = UpdateFoldedChain(CurDAG, N100.Val, Chain.Val, OldTF)).Val) {
18944           N1001 = N100.getOperand(1);
18945           N101 = N10.getOperand(1);
18946           N11 = N1.getOperand(1);
18947           if (N11.getOpcode() == ISD::SHL) {
18948             N110 = N11.getOperand(0);
18949             N111 = N11.getOperand(1);
18950             if (N111.getOpcode() == ISD::SUB) {
18951               N1110 = N111.getOperand(0);
18952               if (isa<ConstantSDNode>(N1110)) {
18953                 int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
18954                 if (CN0 == 16) {
18955                   N1111 = N111.getOperand(1);
18956                   if (N101 == N1111) {
18957                     N2 = N.getOperand(2);
18958                     if (N1001 == N2 &&
18959                         N1.Val->getValueType(0) == MVT::i16 &&
18960                         N101.Val->getValueType(0) == MVT::i8 &&
18961                         N111.Val->getValueType(0) == MVT::i8 &&
18962                         SelectAddr(N1001, Tmp1, Tmp2, Tmp3, Tmp4)) {
18963                       Select(Tmp1, Tmp1);
18964                       Select(Tmp2, Tmp2);
18965                       Select(Tmp3, Tmp3);
18966                       Select(Tmp4, Tmp4);
18967                       Select(Tmp5, N110);
18968                       Select(Chain100, Chain100);
18969                       Select(N101, N101);
18970                       ResNode = CurDAG->getCopyToReg(Chain100, CurDAG->getRegister(X86::CL, MVT::i8), N101, InFlag).Val;
18971                       Chain100 = SDOperand(ResNode, 0);
18972                       InFlag = SDOperand(ResNode, 1);
18973                       ResNode = CurDAG->getTargetNode(X86::SHRD16mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain100, InFlag);
18974                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain100.Val, 0);
18975                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
18976                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 0);
18977                       AddHandleReplacement(N100.Val, 1, ResNode, 0);
18978                       Result = SDOperand(ResNode, N.ResNo);
18979                       return;
18980                     }
18981                   }
18982                 }
18983               }
18984             }
18985           }
18986         }
18987       }
18988       if (N10.getOpcode() == ISD::SHL) {
18989 
18990         // Pattern: (store:void (or:i16 (shl:i16 (load:i16 addr:i32:$dst), CL:i8:$amt), (srl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt))), addr:i32:$dst)
18991         // Emits: (SHLD16mrCL:void addr:i32:$dst, R16:i16:$src2)
18992         // Pattern complexity = 35  cost = 1
18993         if (N10.hasOneUse() &&
18994             !CodeGenMap.count(N10.getValue(0))) {
18995           N100 = N10.getOperand(0);
18996           if (N100.getOpcode() == ISD::LOAD &&
18997               N100.hasOneUse() &&
18998               !CodeGenMap.count(N100.getValue(0)) &&
18999               !isNonImmUse(N10.Val, N100.Val) &&
19000               (Chain100 = UpdateFoldedChain(CurDAG, N100.Val, Chain.Val, OldTF)).Val) {
19001             N1001 = N100.getOperand(1);
19002             N101 = N10.getOperand(1);
19003             N11 = N1.getOperand(1);
19004             if (N11.getOpcode() == ISD::SRL) {
19005               N110 = N11.getOperand(0);
19006               N111 = N11.getOperand(1);
19007               if (N111.getOpcode() == ISD::SUB) {
19008                 N1110 = N111.getOperand(0);
19009                 if (isa<ConstantSDNode>(N1110)) {
19010                   int64_t CN0 = cast<ConstantSDNode>(N1110)->getSignExtended();
19011                   if (CN0 == 16) {
19012                     N1111 = N111.getOperand(1);
19013                     if (N101 == N1111) {
19014                       N2 = N.getOperand(2);
19015                       if (N1001 == N2 &&
19016                           N1.Val->getValueType(0) == MVT::i16 &&
19017                           N101.Val->getValueType(0) == MVT::i8 &&
19018                           N111.Val->getValueType(0) == MVT::i8 &&
19019                           SelectAddr(N1001, Tmp1, Tmp2, Tmp3, Tmp4)) {
19020                         Select(Tmp1, Tmp1);
19021                         Select(Tmp2, Tmp2);
19022                         Select(Tmp3, Tmp3);
19023                         Select(Tmp4, Tmp4);
19024                         Select(Tmp5, N110);
19025                         Select(Chain100, Chain100);
19026                         Select(N101, N101);
19027                         ResNode = CurDAG->getCopyToReg(Chain100, CurDAG->getRegister(X86::CL, MVT::i8), N101, InFlag).Val;
19028                         Chain100 = SDOperand(ResNode, 0);
19029                         InFlag = SDOperand(ResNode, 1);
19030                         ResNode = CurDAG->getTargetNode(X86::SHLD16mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain100, InFlag);
19031                         if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain100.Val, 0);
19032                         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19033                         SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 0);
19034                         AddHandleReplacement(N100.Val, 1, ResNode, 0);
19035                         Result = SDOperand(ResNode, N.ResNo);
19036                         return;
19037                       }
19038                     }
19039                   }
19040                 }
19041               }
19042             }
19043           }
19044         }
19045 
19046         // Pattern: (store:void (or:i32 (shl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (srl:i32 (load:i32 addr:i32:$dst), CL:i8:$amt)), addr:i32:$dst)
19047         // Emits: (SHRD32mrCL:void addr:i32:$dst, R32:i32:$src2)
19048         // Pattern complexity = 35  cost = 1
19049         N100 = N10.getOperand(0);
19050         N101 = N10.getOperand(1);
19051         if (N101.getOpcode() == ISD::SUB) {
19052           N1010 = N101.getOperand(0);
19053           if (isa<ConstantSDNode>(N1010)) {
19054             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19055             if (CN0 == 32) {
19056               N1011 = N101.getOperand(1);
19057               N11 = N1.getOperand(1);
19058               if (N11.getOpcode() == ISD::SRL &&
19059                   N11.hasOneUse() &&
19060                   !CodeGenMap.count(N11.getValue(0))) {
19061                 N110 = N11.getOperand(0);
19062                 if (N110.getOpcode() == ISD::LOAD &&
19063                     N110.hasOneUse() &&
19064                     !CodeGenMap.count(N110.getValue(0)) &&
19065                     !isNonImmUse(N11.Val, N110.Val) &&
19066                     (Chain110 = UpdateFoldedChain(CurDAG, N110.Val, Chain.Val, OldTF)).Val) {
19067                   N1101 = N110.getOperand(1);
19068                   N111 = N11.getOperand(1);
19069                   if (N1011 == N111) {
19070                     N2 = N.getOperand(2);
19071                     if (N1101 == N2 &&
19072                         N1.Val->getValueType(0) == MVT::i32 &&
19073                         N101.Val->getValueType(0) == MVT::i8 &&
19074                         N111.Val->getValueType(0) == MVT::i8 &&
19075                         SelectAddr(N1101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19076                       Select(Tmp1, Tmp1);
19077                       Select(Tmp2, Tmp2);
19078                       Select(Tmp3, Tmp3);
19079                       Select(Tmp4, Tmp4);
19080                       Select(Tmp5, N100);
19081                       Select(Chain110, Chain110);
19082                       Select(N1011, N1011);
19083                       ResNode = CurDAG->getCopyToReg(Chain110, CurDAG->getRegister(X86::CL, MVT::i8), N1011, InFlag).Val;
19084                       Chain110 = SDOperand(ResNode, 0);
19085                       InFlag = SDOperand(ResNode, 1);
19086                       ResNode = CurDAG->getTargetNode(X86::SHRD32mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain110, InFlag);
19087                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain110.Val, 0);
19088                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19089                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N110.Val, 1, ResNode, 0);
19090                       AddHandleReplacement(N110.Val, 1, ResNode, 0);
19091                       Result = SDOperand(ResNode, N.ResNo);
19092                       return;
19093                     }
19094                   }
19095                 }
19096               }
19097             }
19098           }
19099         }
19100       }
19101 
19102       // Pattern: (store:void (or:i32 (srl:i32 R32:i32:$src2, (sub:i8 32:i8, CL:i8:$amt)), (shl:i32 (load:i32 addr:i32:$dst), CL:i8:$amt)), addr:i32:$dst)
19103       // Emits: (SHLD32mrCL:void addr:i32:$dst, R32:i32:$src2)
19104       // Pattern complexity = 35  cost = 1
19105       if (N10.getOpcode() == ISD::SRL) {
19106         N100 = N10.getOperand(0);
19107         N101 = N10.getOperand(1);
19108         if (N101.getOpcode() == ISD::SUB) {
19109           N1010 = N101.getOperand(0);
19110           if (isa<ConstantSDNode>(N1010)) {
19111             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19112             if (CN0 == 32) {
19113               N1011 = N101.getOperand(1);
19114               N11 = N1.getOperand(1);
19115               if (N11.getOpcode() == ISD::SHL &&
19116                   N11.hasOneUse() &&
19117                   !CodeGenMap.count(N11.getValue(0))) {
19118                 N110 = N11.getOperand(0);
19119                 if (N110.getOpcode() == ISD::LOAD &&
19120                     N110.hasOneUse() &&
19121                     !CodeGenMap.count(N110.getValue(0)) &&
19122                     !isNonImmUse(N11.Val, N110.Val) &&
19123                     (Chain110 = UpdateFoldedChain(CurDAG, N110.Val, Chain.Val, OldTF)).Val) {
19124                   N1101 = N110.getOperand(1);
19125                   N111 = N11.getOperand(1);
19126                   if (N1011 == N111) {
19127                     N2 = N.getOperand(2);
19128                     if (N1101 == N2 &&
19129                         N1.Val->getValueType(0) == MVT::i32 &&
19130                         N101.Val->getValueType(0) == MVT::i8 &&
19131                         N111.Val->getValueType(0) == MVT::i8 &&
19132                         SelectAddr(N1101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19133                       Select(Tmp1, Tmp1);
19134                       Select(Tmp2, Tmp2);
19135                       Select(Tmp3, Tmp3);
19136                       Select(Tmp4, Tmp4);
19137                       Select(Tmp5, N100);
19138                       Select(Chain110, Chain110);
19139                       Select(N1011, N1011);
19140                       ResNode = CurDAG->getCopyToReg(Chain110, CurDAG->getRegister(X86::CL, MVT::i8), N1011, InFlag).Val;
19141                       Chain110 = SDOperand(ResNode, 0);
19142                       InFlag = SDOperand(ResNode, 1);
19143                       ResNode = CurDAG->getTargetNode(X86::SHLD32mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain110, InFlag);
19144                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain110.Val, 0);
19145                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19146                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N110.Val, 1, ResNode, 0);
19147                       AddHandleReplacement(N110.Val, 1, ResNode, 0);
19148                       Result = SDOperand(ResNode, N.ResNo);
19149                       return;
19150                     }
19151                   }
19152                 }
19153               }
19154             }
19155           }
19156         }
19157       }
19158 
19159       // Pattern: (store:void (or:i16 (shl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (srl:i16 (load:i16 addr:i32:$dst), CL:i8:$amt)), addr:i32:$dst)
19160       // Emits: (SHRD16mrCL:void addr:i32:$dst, R16:i16:$src2)
19161       // Pattern complexity = 35  cost = 1
19162       if (N10.getOpcode() == ISD::SHL) {
19163         N100 = N10.getOperand(0);
19164         N101 = N10.getOperand(1);
19165         if (N101.getOpcode() == ISD::SUB) {
19166           N1010 = N101.getOperand(0);
19167           if (isa<ConstantSDNode>(N1010)) {
19168             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19169             if (CN0 == 16) {
19170               N1011 = N101.getOperand(1);
19171               N11 = N1.getOperand(1);
19172               if (N11.getOpcode() == ISD::SRL &&
19173                   N11.hasOneUse() &&
19174                   !CodeGenMap.count(N11.getValue(0))) {
19175                 N110 = N11.getOperand(0);
19176                 if (N110.getOpcode() == ISD::LOAD &&
19177                     N110.hasOneUse() &&
19178                     !CodeGenMap.count(N110.getValue(0)) &&
19179                     !isNonImmUse(N11.Val, N110.Val) &&
19180                     (Chain110 = UpdateFoldedChain(CurDAG, N110.Val, Chain.Val, OldTF)).Val) {
19181                   N1101 = N110.getOperand(1);
19182                   N111 = N11.getOperand(1);
19183                   if (N1011 == N111) {
19184                     N2 = N.getOperand(2);
19185                     if (N1101 == N2 &&
19186                         N1.Val->getValueType(0) == MVT::i16 &&
19187                         N101.Val->getValueType(0) == MVT::i8 &&
19188                         N111.Val->getValueType(0) == MVT::i8 &&
19189                         SelectAddr(N1101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19190                       Select(Tmp1, Tmp1);
19191                       Select(Tmp2, Tmp2);
19192                       Select(Tmp3, Tmp3);
19193                       Select(Tmp4, Tmp4);
19194                       Select(Tmp5, N100);
19195                       Select(Chain110, Chain110);
19196                       Select(N1011, N1011);
19197                       ResNode = CurDAG->getCopyToReg(Chain110, CurDAG->getRegister(X86::CL, MVT::i8), N1011, InFlag).Val;
19198                       Chain110 = SDOperand(ResNode, 0);
19199                       InFlag = SDOperand(ResNode, 1);
19200                       ResNode = CurDAG->getTargetNode(X86::SHRD16mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain110, InFlag);
19201                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain110.Val, 0);
19202                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19203                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N110.Val, 1, ResNode, 0);
19204                       AddHandleReplacement(N110.Val, 1, ResNode, 0);
19205                       Result = SDOperand(ResNode, N.ResNo);
19206                       return;
19207                     }
19208                   }
19209                 }
19210               }
19211             }
19212           }
19213         }
19214       }
19215 
19216       // Pattern: (store:void (or:i16 (srl:i16 R16:i16:$src2, (sub:i8 16:i8, CL:i8:$amt)), (shl:i16 (load:i16 addr:i32:$dst), CL:i8:$amt)), addr:i32:$dst)
19217       // Emits: (SHLD16mrCL:void addr:i32:$dst, R16:i16:$src2)
19218       // Pattern complexity = 35  cost = 1
19219       if (N10.getOpcode() == ISD::SRL) {
19220         N100 = N10.getOperand(0);
19221         N101 = N10.getOperand(1);
19222         if (N101.getOpcode() == ISD::SUB) {
19223           N1010 = N101.getOperand(0);
19224           if (isa<ConstantSDNode>(N1010)) {
19225             int64_t CN0 = cast<ConstantSDNode>(N1010)->getSignExtended();
19226             if (CN0 == 16) {
19227               N1011 = N101.getOperand(1);
19228               N11 = N1.getOperand(1);
19229               if (N11.getOpcode() == ISD::SHL &&
19230                   N11.hasOneUse() &&
19231                   !CodeGenMap.count(N11.getValue(0))) {
19232                 N110 = N11.getOperand(0);
19233                 if (N110.getOpcode() == ISD::LOAD &&
19234                     N110.hasOneUse() &&
19235                     !CodeGenMap.count(N110.getValue(0)) &&
19236                     !isNonImmUse(N11.Val, N110.Val) &&
19237                     (Chain110 = UpdateFoldedChain(CurDAG, N110.Val, Chain.Val, OldTF)).Val) {
19238                   N1101 = N110.getOperand(1);
19239                   N111 = N11.getOperand(1);
19240                   if (N1011 == N111) {
19241                     N2 = N.getOperand(2);
19242                     if (N1101 == N2 &&
19243                         N1.Val->getValueType(0) == MVT::i16 &&
19244                         N101.Val->getValueType(0) == MVT::i8 &&
19245                         N111.Val->getValueType(0) == MVT::i8 &&
19246                         SelectAddr(N1101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19247                       Select(Tmp1, Tmp1);
19248                       Select(Tmp2, Tmp2);
19249                       Select(Tmp3, Tmp3);
19250                       Select(Tmp4, Tmp4);
19251                       Select(Tmp5, N100);
19252                       Select(Chain110, Chain110);
19253                       Select(N1011, N1011);
19254                       ResNode = CurDAG->getCopyToReg(Chain110, CurDAG->getRegister(X86::CL, MVT::i8), N1011, InFlag).Val;
19255                       Chain110 = SDOperand(ResNode, 0);
19256                       InFlag = SDOperand(ResNode, 1);
19257                       ResNode = CurDAG->getTargetNode(X86::SHLD16mrCL, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain110, InFlag);
19258                       if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain110.Val, 0);
19259                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19260                       SelectionDAG::InsertISelMapEntry(CodeGenMap, N110.Val, 1, ResNode, 0);
19261                       AddHandleReplacement(N110.Val, 1, ResNode, 0);
19262                       Result = SDOperand(ResNode, N.ResNo);
19263                       return;
19264                     }
19265                   }
19266                 }
19267               }
19268             }
19269           }
19270         }
19271       }
19272     }
19273     if (N1.getOpcode() == ISD::SUB &&
19274         N1.hasOneUse() &&
19275         !CodeGenMap.count(N1.getValue(0))) {
19276       N10 = N1.getOperand(0);
19277       if (isa<ConstantSDNode>(N10)) {
19278         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
19279         if (CN0 == 0) {
19280           N11 = N1.getOperand(1);
19281           if (N11.getOpcode() == ISD::LOAD &&
19282               N11.hasOneUse() &&
19283               !CodeGenMap.count(N11.getValue(0)) &&
19284               !isNonImmUse(N1.Val, N11.Val) &&
19285               (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
19286             N111 = N11.getOperand(1);
19287             N2 = N.getOperand(2);
19288             if (N111 == N2) {
19289 
19290               // Pattern: (store:void (sub:i8 0:i8, (load:i8 addr:i32:$dst)), addr:i32:$dst)
19291               // Emits: (NEG8m:void addr:i32:$dst)
19292               // Pattern complexity = 29  cost = 1
19293               if (N1.Val->getValueType(0) == MVT::i8 &&
19294                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
19295                 Select(Tmp1, Tmp1);
19296                 Select(Tmp2, Tmp2);
19297                 Select(Tmp3, Tmp3);
19298                 Select(Tmp4, Tmp4);
19299                 Select(Chain11, Chain11);
19300                 ResNode = CurDAG->getTargetNode(X86::NEG8m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
19301                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
19302                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19303                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
19304                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
19305                 Result = SDOperand(ResNode, N.ResNo);
19306                 return;
19307               }
19308 
19309               // Pattern: (store:void (sub:i16 0:i16, (load:i16 addr:i32:$dst)), addr:i32:$dst)
19310               // Emits: (NEG16m:void addr:i32:$dst)
19311               // Pattern complexity = 29  cost = 1
19312               if (N1.Val->getValueType(0) == MVT::i16 &&
19313                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
19314                 Select(Tmp1, Tmp1);
19315                 Select(Tmp2, Tmp2);
19316                 Select(Tmp3, Tmp3);
19317                 Select(Tmp4, Tmp4);
19318                 Select(Chain11, Chain11);
19319                 ResNode = CurDAG->getTargetNode(X86::NEG16m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
19320                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
19321                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19322                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
19323                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
19324                 Result = SDOperand(ResNode, N.ResNo);
19325                 return;
19326               }
19327 
19328               // Pattern: (store:void (sub:i32 0:i32, (load:i32 addr:i32:$dst)), addr:i32:$dst)
19329               // Emits: (NEG32m:void addr:i32:$dst)
19330               // Pattern complexity = 29  cost = 1
19331               if (N1.Val->getValueType(0) == MVT::i32 &&
19332                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
19333                 Select(Tmp1, Tmp1);
19334                 Select(Tmp2, Tmp2);
19335                 Select(Tmp3, Tmp3);
19336                 Select(Tmp4, Tmp4);
19337                 Select(Chain11, Chain11);
19338                 ResNode = CurDAG->getTargetNode(X86::NEG32m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
19339                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
19340                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19341                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
19342                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
19343                 Result = SDOperand(ResNode, N.ResNo);
19344                 return;
19345               }
19346             }
19347           }
19348         }
19349       }
19350     }
19351     if (N1.getOpcode() == ISD::XOR &&
19352         N1.hasOneUse() &&
19353         !CodeGenMap.count(N1.getValue(0))) {
19354       N10 = N1.getOperand(0);
19355       if (N10.getOpcode() == ISD::LOAD &&
19356           N10.hasOneUse() &&
19357           !CodeGenMap.count(N10.getValue(0)) &&
19358           !isNonImmUse(N1.Val, N10.Val) &&
19359           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19360         N101 = N10.getOperand(1);
19361         N11 = N1.getOperand(1);
19362         if (N11.getOpcode() == ISD::Constant &&
19363             Predicate_immAllOnes(N11.Val)) {
19364           N2 = N.getOperand(2);
19365           if (N101 == N2) {
19366 
19367             // Pattern: (store:void (xor:i8 (load:i8 addr:i32:$dst), (imm:i8)<<P:Predicate_immAllOnes>>), addr:i32:$dst)
19368             // Emits: (NOT8m:void addr:i32:$dst)
19369             // Pattern complexity = 29  cost = 1
19370             if (N1.Val->getValueType(0) == MVT::i8 &&
19371                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19372               Select(Tmp0, Tmp0);
19373               Select(Tmp1, Tmp1);
19374               Select(Tmp2, Tmp2);
19375               Select(Tmp3, Tmp3);
19376               Select(Chain10, Chain10);
19377               ResNode = CurDAG->getTargetNode(X86::NOT8m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
19378               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19379               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19380               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19381               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19382               Result = SDOperand(ResNode, N.ResNo);
19383               return;
19384             }
19385 
19386             // Pattern: (store:void (xor:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_immAllOnes>>), addr:i32:$dst)
19387             // Emits: (NOT16m:void addr:i32:$dst)
19388             // Pattern complexity = 29  cost = 1
19389             if (N1.Val->getValueType(0) == MVT::i16 &&
19390                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19391               Select(Tmp0, Tmp0);
19392               Select(Tmp1, Tmp1);
19393               Select(Tmp2, Tmp2);
19394               Select(Tmp3, Tmp3);
19395               Select(Chain10, Chain10);
19396               ResNode = CurDAG->getTargetNode(X86::NOT16m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
19397               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19398               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19399               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19400               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19401               Result = SDOperand(ResNode, N.ResNo);
19402               return;
19403             }
19404 
19405             // Pattern: (store:void (xor:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_immAllOnes>>), addr:i32:$dst)
19406             // Emits: (NOT32m:void addr:i32:$dst)
19407             // Pattern complexity = 29  cost = 1
19408             if (N1.Val->getValueType(0) == MVT::i32 &&
19409                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19410               Select(Tmp0, Tmp0);
19411               Select(Tmp1, Tmp1);
19412               Select(Tmp2, Tmp2);
19413               Select(Tmp3, Tmp3);
19414               Select(Chain10, Chain10);
19415               ResNode = CurDAG->getTargetNode(X86::NOT32m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
19416               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19417               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19418               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19419               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19420               Result = SDOperand(ResNode, N.ResNo);
19421               return;
19422             }
19423           }
19424         }
19425       }
19426     }
19427     if (N1.getOpcode() == ISD::ADD &&
19428         N1.hasOneUse() &&
19429         !CodeGenMap.count(N1.getValue(0))) {
19430       N10 = N1.getOperand(0);
19431       if (N10.getOpcode() == ISD::LOAD &&
19432           N10.hasOneUse() &&
19433           !CodeGenMap.count(N10.getValue(0)) &&
19434           !isNonImmUse(N1.Val, N10.Val) &&
19435           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19436         N101 = N10.getOperand(1);
19437         N11 = N1.getOperand(1);
19438         if (isa<ConstantSDNode>(N11)) {
19439           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
19440           if (CN0 == 1) {
19441             N2 = N.getOperand(2);
19442             if (N101 == N2) {
19443 
19444               // Pattern: (store:void (add:i8 (load:i8 addr:i32:$dst), 1:i8), addr:i32:$dst)
19445               // Emits: (INC8m:void addr:i32:$dst)
19446               // Pattern complexity = 29  cost = 1
19447               if (N1.Val->getValueType(0) == MVT::i8 &&
19448                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19449                 Select(Tmp1, Tmp1);
19450                 Select(Tmp2, Tmp2);
19451                 Select(Tmp3, Tmp3);
19452                 Select(Tmp4, Tmp4);
19453                 Select(Chain10, Chain10);
19454                 ResNode = CurDAG->getTargetNode(X86::INC8m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19455                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19456                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19457                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19458                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19459                 Result = SDOperand(ResNode, N.ResNo);
19460                 return;
19461               }
19462 
19463               // Pattern: (store:void (add:i16 (load:i16 addr:i32:$dst), 1:i16), addr:i32:$dst)
19464               // Emits: (INC16m:void addr:i32:$dst)
19465               // Pattern complexity = 29  cost = 1
19466               if (N1.Val->getValueType(0) == MVT::i16 &&
19467                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19468                 Select(Tmp1, Tmp1);
19469                 Select(Tmp2, Tmp2);
19470                 Select(Tmp3, Tmp3);
19471                 Select(Tmp4, Tmp4);
19472                 Select(Chain10, Chain10);
19473                 ResNode = CurDAG->getTargetNode(X86::INC16m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19474                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19475                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19476                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19477                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19478                 Result = SDOperand(ResNode, N.ResNo);
19479                 return;
19480               }
19481 
19482               // Pattern: (store:void (add:i32 (load:i32 addr:i32:$dst), 1:i32), addr:i32:$dst)
19483               // Emits: (INC32m:void addr:i32:$dst)
19484               // Pattern complexity = 29  cost = 1
19485               if (N1.Val->getValueType(0) == MVT::i32 &&
19486                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19487                 Select(Tmp1, Tmp1);
19488                 Select(Tmp2, Tmp2);
19489                 Select(Tmp3, Tmp3);
19490                 Select(Tmp4, Tmp4);
19491                 Select(Chain10, Chain10);
19492                 ResNode = CurDAG->getTargetNode(X86::INC32m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19493                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19494                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19495                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19496                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19497                 Result = SDOperand(ResNode, N.ResNo);
19498                 return;
19499               }
19500             }
19501           }
19502           if (CN0 == -1) {
19503             N2 = N.getOperand(2);
19504             if (N101 == N2) {
19505 
19506               // Pattern: (store:void (add:i8 (load:i8 addr:i32:$dst), -1:i8), addr:i32:$dst)
19507               // Emits: (DEC8m:void addr:i32:$dst)
19508               // Pattern complexity = 29  cost = 1
19509               if (N1.Val->getValueType(0) == MVT::i8 &&
19510                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19511                 Select(Tmp1, Tmp1);
19512                 Select(Tmp2, Tmp2);
19513                 Select(Tmp3, Tmp3);
19514                 Select(Tmp4, Tmp4);
19515                 Select(Chain10, Chain10);
19516                 ResNode = CurDAG->getTargetNode(X86::DEC8m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19517                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19518                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19519                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19520                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19521                 Result = SDOperand(ResNode, N.ResNo);
19522                 return;
19523               }
19524 
19525               // Pattern: (store:void (add:i16 (load:i16 addr:i32:$dst), -1:i16), addr:i32:$dst)
19526               // Emits: (DEC16m:void addr:i32:$dst)
19527               // Pattern complexity = 29  cost = 1
19528               if (N1.Val->getValueType(0) == MVT::i16 &&
19529                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19530                 Select(Tmp1, Tmp1);
19531                 Select(Tmp2, Tmp2);
19532                 Select(Tmp3, Tmp3);
19533                 Select(Tmp4, Tmp4);
19534                 Select(Chain10, Chain10);
19535                 ResNode = CurDAG->getTargetNode(X86::DEC16m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19536                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19537                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19538                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19539                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19540                 Result = SDOperand(ResNode, N.ResNo);
19541                 return;
19542               }
19543 
19544               // Pattern: (store:void (add:i32 (load:i32 addr:i32:$dst), -1:i32), addr:i32:$dst)
19545               // Emits: (DEC32m:void addr:i32:$dst)
19546               // Pattern complexity = 29  cost = 1
19547               if (N1.Val->getValueType(0) == MVT::i32 &&
19548                   SelectAddr(N101, Tmp1, Tmp2, Tmp3, Tmp4)) {
19549                 Select(Tmp1, Tmp1);
19550                 Select(Tmp2, Tmp2);
19551                 Select(Tmp3, Tmp3);
19552                 Select(Tmp4, Tmp4);
19553                 Select(Chain10, Chain10);
19554                 ResNode = CurDAG->getTargetNode(X86::DEC32m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19555                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19556                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19557                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19558                 AddHandleReplacement(N10.Val, 1, ResNode, 0);
19559                 Result = SDOperand(ResNode, N.ResNo);
19560                 return;
19561               }
19562             }
19563           }
19564         }
19565       }
19566     }
19567     if (N1.getOpcode() == ISD::AND &&
19568         N1.hasOneUse() &&
19569         !CodeGenMap.count(N1.getValue(0))) {
19570       N10 = N1.getOperand(0);
19571       if (N10.getOpcode() == ISD::LOAD &&
19572           N10.hasOneUse() &&
19573           !CodeGenMap.count(N10.getValue(0)) &&
19574           !isNonImmUse(N1.Val, N10.Val) &&
19575           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19576         N101 = N10.getOperand(1);
19577         N11 = N1.getOperand(1);
19578         if (N11.getOpcode() == ISD::Constant) {
19579 
19580           // Pattern: (store:void (and:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:i32:$dst)
19581           // Emits: (AND16mi8:void addr:i32:$dst, (imm:i16):$src)
19582           // Pattern complexity = 29  cost = 1
19583           if (Predicate_i16immSExt8(N11.Val)) {
19584             N2 = N.getOperand(2);
19585             if (N101 == N2 &&
19586                 N1.Val->getValueType(0) == MVT::i16 &&
19587                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19588               Select(Tmp0, Tmp0);
19589               Select(Tmp1, Tmp1);
19590               Select(Tmp2, Tmp2);
19591               Select(Tmp3, Tmp3);
19592               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
19593               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
19594               Select(Chain10, Chain10);
19595               ResNode = CurDAG->getTargetNode(X86::AND16mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19596               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19597               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19598               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19599               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19600               Result = SDOperand(ResNode, N.ResNo);
19601               return;
19602             }
19603           }
19604 
19605           // Pattern: (store:void (and:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:i32:$dst)
19606           // Emits: (AND32mi8:void addr:i32:$dst, (imm:i32):$src)
19607           // Pattern complexity = 29  cost = 1
19608           if (Predicate_i32immSExt8(N11.Val)) {
19609             N2 = N.getOperand(2);
19610             if (N101 == N2 &&
19611                 N1.Val->getValueType(0) == MVT::i32 &&
19612                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19613               Select(Tmp0, Tmp0);
19614               Select(Tmp1, Tmp1);
19615               Select(Tmp2, Tmp2);
19616               Select(Tmp3, Tmp3);
19617               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19618               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19619               Select(Chain10, Chain10);
19620               ResNode = CurDAG->getTargetNode(X86::AND32mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19621               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19622               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19623               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19624               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19625               Result = SDOperand(ResNode, N.ResNo);
19626               return;
19627             }
19628           }
19629         }
19630       }
19631     }
19632     if (N1.getOpcode() == ISD::OR &&
19633         N1.hasOneUse() &&
19634         !CodeGenMap.count(N1.getValue(0))) {
19635       N10 = N1.getOperand(0);
19636       if (N10.getOpcode() == ISD::LOAD &&
19637           N10.hasOneUse() &&
19638           !CodeGenMap.count(N10.getValue(0)) &&
19639           !isNonImmUse(N1.Val, N10.Val) &&
19640           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19641         N101 = N10.getOperand(1);
19642         N11 = N1.getOperand(1);
19643         if (N11.getOpcode() == ISD::Constant) {
19644 
19645           // Pattern: (store:void (or:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:i32:$dst)
19646           // Emits: (OR16mi8:void addr:i32:$dst, (imm:i16):$src)
19647           // Pattern complexity = 29  cost = 1
19648           if (Predicate_i16immSExt8(N11.Val)) {
19649             N2 = N.getOperand(2);
19650             if (N101 == N2 &&
19651                 N1.Val->getValueType(0) == MVT::i16 &&
19652                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19653               Select(Tmp0, Tmp0);
19654               Select(Tmp1, Tmp1);
19655               Select(Tmp2, Tmp2);
19656               Select(Tmp3, Tmp3);
19657               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
19658               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
19659               Select(Chain10, Chain10);
19660               ResNode = CurDAG->getTargetNode(X86::OR16mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19661               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19662               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19663               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19664               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19665               Result = SDOperand(ResNode, N.ResNo);
19666               return;
19667             }
19668           }
19669 
19670           // Pattern: (store:void (or:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:i32:$dst)
19671           // Emits: (OR32mi8:void addr:i32:$dst, (imm:i32):$src)
19672           // Pattern complexity = 29  cost = 1
19673           if (Predicate_i32immSExt8(N11.Val)) {
19674             N2 = N.getOperand(2);
19675             if (N101 == N2 &&
19676                 N1.Val->getValueType(0) == MVT::i32 &&
19677                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19678               Select(Tmp0, Tmp0);
19679               Select(Tmp1, Tmp1);
19680               Select(Tmp2, Tmp2);
19681               Select(Tmp3, Tmp3);
19682               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19683               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19684               Select(Chain10, Chain10);
19685               ResNode = CurDAG->getTargetNode(X86::OR32mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19686               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19687               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19688               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19689               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19690               Result = SDOperand(ResNode, N.ResNo);
19691               return;
19692             }
19693           }
19694         }
19695       }
19696     }
19697     if (N1.getOpcode() == ISD::XOR &&
19698         N1.hasOneUse() &&
19699         !CodeGenMap.count(N1.getValue(0))) {
19700       N10 = N1.getOperand(0);
19701       if (N10.getOpcode() == ISD::LOAD &&
19702           N10.hasOneUse() &&
19703           !CodeGenMap.count(N10.getValue(0)) &&
19704           !isNonImmUse(N1.Val, N10.Val) &&
19705           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19706         N101 = N10.getOperand(1);
19707         N11 = N1.getOperand(1);
19708         if (N11.getOpcode() == ISD::Constant) {
19709 
19710           // Pattern: (store:void (xor:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src), addr:i32:$dst)
19711           // Emits: (XOR16mi8:void addr:i32:$dst, (imm:i16):$src)
19712           // Pattern complexity = 29  cost = 1
19713           if (Predicate_i16immSExt8(N11.Val)) {
19714             N2 = N.getOperand(2);
19715             if (N101 == N2 &&
19716                 N1.Val->getValueType(0) == MVT::i16 &&
19717                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19718               Select(Tmp0, Tmp0);
19719               Select(Tmp1, Tmp1);
19720               Select(Tmp2, Tmp2);
19721               Select(Tmp3, Tmp3);
19722               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
19723               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
19724               Select(Chain10, Chain10);
19725               ResNode = CurDAG->getTargetNode(X86::XOR16mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19726               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19727               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19728               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19729               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19730               Result = SDOperand(ResNode, N.ResNo);
19731               return;
19732             }
19733           }
19734 
19735           // Pattern: (store:void (xor:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src), addr:i32:$dst)
19736           // Emits: (XOR32mi8:void addr:i32:$dst, (imm:i32):$src)
19737           // Pattern complexity = 29  cost = 1
19738           if (Predicate_i32immSExt8(N11.Val)) {
19739             N2 = N.getOperand(2);
19740             if (N101 == N2 &&
19741                 N1.Val->getValueType(0) == MVT::i32 &&
19742                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19743               Select(Tmp0, Tmp0);
19744               Select(Tmp1, Tmp1);
19745               Select(Tmp2, Tmp2);
19746               Select(Tmp3, Tmp3);
19747               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19748               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19749               Select(Chain10, Chain10);
19750               ResNode = CurDAG->getTargetNode(X86::XOR32mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19751               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19752               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19753               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19754               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19755               Result = SDOperand(ResNode, N.ResNo);
19756               return;
19757             }
19758           }
19759         }
19760       }
19761     }
19762     if (N1.getOpcode() == ISD::ADD &&
19763         N1.hasOneUse() &&
19764         !CodeGenMap.count(N1.getValue(0))) {
19765       N10 = N1.getOperand(0);
19766       if (N10.getOpcode() == ISD::LOAD &&
19767           N10.hasOneUse() &&
19768           !CodeGenMap.count(N10.getValue(0)) &&
19769           !isNonImmUse(N1.Val, N10.Val) &&
19770           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19771         N101 = N10.getOperand(1);
19772         N11 = N1.getOperand(1);
19773         if (N11.getOpcode() == ISD::Constant) {
19774 
19775           // Pattern: (store:void (add:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2), addr:i32:$dst)
19776           // Emits: (ADD16mi8:void addr:i32:$dst, (imm:i16):$src2)
19777           // Pattern complexity = 29  cost = 1
19778           if (Predicate_i16immSExt8(N11.Val)) {
19779             N2 = N.getOperand(2);
19780             if (N101 == N2 &&
19781                 N1.Val->getValueType(0) == MVT::i16 &&
19782                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19783               Select(Tmp0, Tmp0);
19784               Select(Tmp1, Tmp1);
19785               Select(Tmp2, Tmp2);
19786               Select(Tmp3, Tmp3);
19787               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
19788               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
19789               Select(Chain10, Chain10);
19790               ResNode = CurDAG->getTargetNode(X86::ADD16mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19791               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19792               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19793               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19794               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19795               Result = SDOperand(ResNode, N.ResNo);
19796               return;
19797             }
19798           }
19799 
19800           // Pattern: (store:void (add:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:i32:$dst)
19801           // Emits: (ADD32mi8:void addr:i32:$dst, (imm:i32):$src2)
19802           // Pattern complexity = 29  cost = 1
19803           if (Predicate_i32immSExt8(N11.Val)) {
19804             N2 = N.getOperand(2);
19805             if (N101 == N2 &&
19806                 N1.Val->getValueType(0) == MVT::i32 &&
19807                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19808               Select(Tmp0, Tmp0);
19809               Select(Tmp1, Tmp1);
19810               Select(Tmp2, Tmp2);
19811               Select(Tmp3, Tmp3);
19812               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19813               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19814               Select(Chain10, Chain10);
19815               ResNode = CurDAG->getTargetNode(X86::ADD32mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19816               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19817               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19818               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19819               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19820               Result = SDOperand(ResNode, N.ResNo);
19821               return;
19822             }
19823           }
19824         }
19825       }
19826     }
19827 
19828     // Pattern: (store:void (adde:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:i32:$dst)
19829     // Emits: (ADC32mi8:void addr:i32:$dst, (imm:i32):$src2)
19830     // Pattern complexity = 29  cost = 1
19831     if (N1.getOpcode() == ISD::ADDE &&
19832         N1.hasOneUse() &&
19833         !CodeGenMap.count(N1.getValue(0))) {
19834       N10 = N1.getOperand(0);
19835       if (N10.getOpcode() == ISD::LOAD &&
19836           N10.hasOneUse() &&
19837           !CodeGenMap.count(N10.getValue(0)) &&
19838           N1.Val->isOnlyUse(N10.Val) &&
19839           !isNonImmUse(N1.Val, N10.Val) &&
19840           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19841         N101 = N10.getOperand(1);
19842         N11 = N1.getOperand(1);
19843         if (N11.getOpcode() == ISD::Constant &&
19844             Predicate_i32immSExt8(N11.Val)) {
19845           N2 = N.getOperand(2);
19846           if (N101 == N2 &&
19847               N1.Val->getValueType(0) == MVT::i32 &&
19848               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19849             Select(Tmp0, Tmp0);
19850             Select(Tmp1, Tmp1);
19851             Select(Tmp2, Tmp2);
19852             Select(Tmp3, Tmp3);
19853             unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19854             Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19855             Select(Chain10, Chain10);
19856             Select(InFlag, N1.getOperand(2));
19857             ResNode = CurDAG->getTargetNode(X86::ADC32mi8, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
19858             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19859             InFlag = SDOperand(ResNode, 1);
19860             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19861             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19862             AddHandleReplacement(N10.Val, 1, ResNode, 0);
19863             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
19864             Result = SDOperand(ResNode, N.ResNo);
19865             return;
19866           }
19867         }
19868       }
19869     }
19870     if (N1.getOpcode() == ISD::SUB &&
19871         N1.hasOneUse() &&
19872         !CodeGenMap.count(N1.getValue(0))) {
19873       N10 = N1.getOperand(0);
19874       if (N10.getOpcode() == ISD::LOAD &&
19875           N10.hasOneUse() &&
19876           !CodeGenMap.count(N10.getValue(0)) &&
19877           !isNonImmUse(N1.Val, N10.Val) &&
19878           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19879         N101 = N10.getOperand(1);
19880         N11 = N1.getOperand(1);
19881         if (N11.getOpcode() == ISD::Constant) {
19882 
19883           // Pattern: (store:void (sub:i16 (load:i16 addr:i32:$dst), (imm:i16)<<P:Predicate_i16immSExt8>>:$src2), addr:i32:$dst)
19884           // Emits: (SUB16mi8:void addr:i32:$dst, (imm:i16):$src2)
19885           // Pattern complexity = 29  cost = 1
19886           if (Predicate_i16immSExt8(N11.Val)) {
19887             N2 = N.getOperand(2);
19888             if (N101 == N2 &&
19889                 N1.Val->getValueType(0) == MVT::i16 &&
19890                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19891               Select(Tmp0, Tmp0);
19892               Select(Tmp1, Tmp1);
19893               Select(Tmp2, Tmp2);
19894               Select(Tmp3, Tmp3);
19895               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
19896               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
19897               Select(Chain10, Chain10);
19898               ResNode = CurDAG->getTargetNode(X86::SUB16mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19899               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19900               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19901               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19902               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19903               Result = SDOperand(ResNode, N.ResNo);
19904               return;
19905             }
19906           }
19907 
19908           // Pattern: (store:void (sub:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:i32:$dst)
19909           // Emits: (SUB32mi8:void addr:i32:$dst, (imm:i32):$src2)
19910           // Pattern complexity = 29  cost = 1
19911           if (Predicate_i32immSExt8(N11.Val)) {
19912             N2 = N.getOperand(2);
19913             if (N101 == N2 &&
19914                 N1.Val->getValueType(0) == MVT::i32 &&
19915                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19916               Select(Tmp0, Tmp0);
19917               Select(Tmp1, Tmp1);
19918               Select(Tmp2, Tmp2);
19919               Select(Tmp3, Tmp3);
19920               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19921               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19922               Select(Chain10, Chain10);
19923               ResNode = CurDAG->getTargetNode(X86::SUB32mi8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
19924               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19925               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19926               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19927               AddHandleReplacement(N10.Val, 1, ResNode, 0);
19928               Result = SDOperand(ResNode, N.ResNo);
19929               return;
19930             }
19931           }
19932         }
19933       }
19934     }
19935 
19936     // Pattern: (store:void (sube:i32 (load:i32 addr:i32:$dst), (imm:i32)<<P:Predicate_i32immSExt8>>:$src2), addr:i32:$dst)
19937     // Emits: (SBB32mi8:void addr:i32:$dst, (imm:i32):$src2)
19938     // Pattern complexity = 29  cost = 1
19939     if (N1.getOpcode() == ISD::SUBE &&
19940         N1.hasOneUse() &&
19941         !CodeGenMap.count(N1.getValue(0))) {
19942       N10 = N1.getOperand(0);
19943       if (N10.getOpcode() == ISD::LOAD &&
19944           N10.hasOneUse() &&
19945           !CodeGenMap.count(N10.getValue(0)) &&
19946           N1.Val->isOnlyUse(N10.Val) &&
19947           !isNonImmUse(N1.Val, N10.Val) &&
19948           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
19949         N101 = N10.getOperand(1);
19950         N11 = N1.getOperand(1);
19951         if (N11.getOpcode() == ISD::Constant &&
19952             Predicate_i32immSExt8(N11.Val)) {
19953           N2 = N.getOperand(2);
19954           if (N101 == N2 &&
19955               N1.Val->getValueType(0) == MVT::i32 &&
19956               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
19957             Select(Tmp0, Tmp0);
19958             Select(Tmp1, Tmp1);
19959             Select(Tmp2, Tmp2);
19960             Select(Tmp3, Tmp3);
19961             unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
19962             Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
19963             Select(Chain10, Chain10);
19964             Select(InFlag, N1.getOperand(2));
19965             ResNode = CurDAG->getTargetNode(X86::SBB32mi8, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
19966             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
19967             InFlag = SDOperand(ResNode, 1);
19968             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
19969             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
19970             AddHandleReplacement(N10.Val, 1, ResNode, 0);
19971             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
19972             Result = SDOperand(ResNode, N.ResNo);
19973             return;
19974           }
19975         }
19976       }
19977     }
19978     if (N1.getOpcode() == ISD::ADD &&
19979         N1.hasOneUse() &&
19980         !CodeGenMap.count(N1.getValue(0))) {
19981       N10 = N1.getOperand(0);
19982       if (isa<ConstantSDNode>(N10)) {
19983         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
19984         if (CN0 == 1) {
19985           N11 = N1.getOperand(1);
19986           if (N11.getOpcode() == ISD::LOAD &&
19987               N11.hasOneUse() &&
19988               !CodeGenMap.count(N11.getValue(0)) &&
19989               !isNonImmUse(N1.Val, N11.Val) &&
19990               (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
19991             N111 = N11.getOperand(1);
19992             N2 = N.getOperand(2);
19993             if (N111 == N2) {
19994 
19995               // Pattern: (store:void (add:i8 1:i8, (load:i8 addr:i32:$dst)), addr:i32:$dst)
19996               // Emits: (INC8m:void addr:i32:$dst)
19997               // Pattern complexity = 29  cost = 1
19998               if (N1.Val->getValueType(0) == MVT::i8 &&
19999                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20000                 Select(Tmp1, Tmp1);
20001                 Select(Tmp2, Tmp2);
20002                 Select(Tmp3, Tmp3);
20003                 Select(Tmp4, Tmp4);
20004                 Select(Chain11, Chain11);
20005                 ResNode = CurDAG->getTargetNode(X86::INC8m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20006                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20007                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20008                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20009                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20010                 Result = SDOperand(ResNode, N.ResNo);
20011                 return;
20012               }
20013 
20014               // Pattern: (store:void (add:i16 1:i16, (load:i16 addr:i32:$dst)), addr:i32:$dst)
20015               // Emits: (INC16m:void addr:i32:$dst)
20016               // Pattern complexity = 29  cost = 1
20017               if (N1.Val->getValueType(0) == MVT::i16 &&
20018                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20019                 Select(Tmp1, Tmp1);
20020                 Select(Tmp2, Tmp2);
20021                 Select(Tmp3, Tmp3);
20022                 Select(Tmp4, Tmp4);
20023                 Select(Chain11, Chain11);
20024                 ResNode = CurDAG->getTargetNode(X86::INC16m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20025                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20026                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20027                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20028                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20029                 Result = SDOperand(ResNode, N.ResNo);
20030                 return;
20031               }
20032 
20033               // Pattern: (store:void (add:i32 1:i32, (load:i32 addr:i32:$dst)), addr:i32:$dst)
20034               // Emits: (INC32m:void addr:i32:$dst)
20035               // Pattern complexity = 29  cost = 1
20036               if (N1.Val->getValueType(0) == MVT::i32 &&
20037                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20038                 Select(Tmp1, Tmp1);
20039                 Select(Tmp2, Tmp2);
20040                 Select(Tmp3, Tmp3);
20041                 Select(Tmp4, Tmp4);
20042                 Select(Chain11, Chain11);
20043                 ResNode = CurDAG->getTargetNode(X86::INC32m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20044                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20045                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20046                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20047                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20048                 Result = SDOperand(ResNode, N.ResNo);
20049                 return;
20050               }
20051             }
20052           }
20053         }
20054         if (CN0 == -1) {
20055           N11 = N1.getOperand(1);
20056           if (N11.getOpcode() == ISD::LOAD &&
20057               N11.hasOneUse() &&
20058               !CodeGenMap.count(N11.getValue(0)) &&
20059               !isNonImmUse(N1.Val, N11.Val) &&
20060               (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
20061             N111 = N11.getOperand(1);
20062             N2 = N.getOperand(2);
20063             if (N111 == N2) {
20064 
20065               // Pattern: (store:void (add:i8 -1:i8, (load:i8 addr:i32:$dst)), addr:i32:$dst)
20066               // Emits: (DEC8m:void addr:i32:$dst)
20067               // Pattern complexity = 29  cost = 1
20068               if (N1.Val->getValueType(0) == MVT::i8 &&
20069                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20070                 Select(Tmp1, Tmp1);
20071                 Select(Tmp2, Tmp2);
20072                 Select(Tmp3, Tmp3);
20073                 Select(Tmp4, Tmp4);
20074                 Select(Chain11, Chain11);
20075                 ResNode = CurDAG->getTargetNode(X86::DEC8m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20076                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20077                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20078                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20079                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20080                 Result = SDOperand(ResNode, N.ResNo);
20081                 return;
20082               }
20083 
20084               // Pattern: (store:void (add:i16 -1:i16, (load:i16 addr:i32:$dst)), addr:i32:$dst)
20085               // Emits: (DEC16m:void addr:i32:$dst)
20086               // Pattern complexity = 29  cost = 1
20087               if (N1.Val->getValueType(0) == MVT::i16 &&
20088                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20089                 Select(Tmp1, Tmp1);
20090                 Select(Tmp2, Tmp2);
20091                 Select(Tmp3, Tmp3);
20092                 Select(Tmp4, Tmp4);
20093                 Select(Chain11, Chain11);
20094                 ResNode = CurDAG->getTargetNode(X86::DEC16m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20095                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20096                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20097                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20098                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20099                 Result = SDOperand(ResNode, N.ResNo);
20100                 return;
20101               }
20102 
20103               // Pattern: (store:void (add:i32 -1:i32, (load:i32 addr:i32:$dst)), addr:i32:$dst)
20104               // Emits: (DEC32m:void addr:i32:$dst)
20105               // Pattern complexity = 29  cost = 1
20106               if (N1.Val->getValueType(0) == MVT::i32 &&
20107                   SelectAddr(N111, Tmp1, Tmp2, Tmp3, Tmp4)) {
20108                 Select(Tmp1, Tmp1);
20109                 Select(Tmp2, Tmp2);
20110                 Select(Tmp3, Tmp3);
20111                 Select(Tmp4, Tmp4);
20112                 Select(Chain11, Chain11);
20113                 ResNode = CurDAG->getTargetNode(X86::DEC32m, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
20114                 if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
20115                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20116                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
20117                 AddHandleReplacement(N11.Val, 1, ResNode, 0);
20118                 Result = SDOperand(ResNode, N.ResNo);
20119                 return;
20120               }
20121             }
20122           }
20123         }
20124       }
20125     }
20126     if (N1.getOpcode() == ISD::AND &&
20127         N1.hasOneUse() &&
20128         !CodeGenMap.count(N1.getValue(0))) {
20129       N10 = N1.getOperand(0);
20130       if (N10.getOpcode() == ISD::LOAD &&
20131           N10.hasOneUse() &&
20132           !CodeGenMap.count(N10.getValue(0)) &&
20133           !isNonImmUse(N1.Val, N10.Val) &&
20134           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20135         N101 = N10.getOperand(1);
20136         N11 = N1.getOperand(1);
20137         if (N11.getOpcode() == ISD::Constant) {
20138           N2 = N.getOperand(2);
20139           if (N101 == N2) {
20140 
20141             // Pattern: (store:void (and:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20142             // Emits: (AND8mi:void addr:i32:$dst, (imm:i8):$src)
20143             // Pattern complexity = 28  cost = 1
20144             if (N1.Val->getValueType(0) == MVT::i8 &&
20145                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20146               Select(Tmp0, Tmp0);
20147               Select(Tmp1, Tmp1);
20148               Select(Tmp2, Tmp2);
20149               Select(Tmp3, Tmp3);
20150               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20151               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20152               Select(Chain10, Chain10);
20153               ResNode = CurDAG->getTargetNode(X86::AND8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20154               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20155               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20156               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20157               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20158               Result = SDOperand(ResNode, N.ResNo);
20159               return;
20160             }
20161 
20162             // Pattern: (store:void (and:i16 (load:i16 addr:i32:$dst), (imm:i16):$src), addr:i32:$dst)
20163             // Emits: (AND16mi:void addr:i32:$dst, (imm:i16):$src)
20164             // Pattern complexity = 28  cost = 1
20165             if (N1.Val->getValueType(0) == MVT::i16 &&
20166                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20167               Select(Tmp0, Tmp0);
20168               Select(Tmp1, Tmp1);
20169               Select(Tmp2, Tmp2);
20170               Select(Tmp3, Tmp3);
20171               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
20172               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
20173               Select(Chain10, Chain10);
20174               ResNode = CurDAG->getTargetNode(X86::AND16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20175               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20176               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20177               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20178               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20179               Result = SDOperand(ResNode, N.ResNo);
20180               return;
20181             }
20182 
20183             // Pattern: (store:void (and:i32 (load:i32 addr:i32:$dst), (imm:i32):$src), addr:i32:$dst)
20184             // Emits: (AND32mi:void addr:i32:$dst, (imm:i32):$src)
20185             // Pattern complexity = 28  cost = 1
20186             if (N1.Val->getValueType(0) == MVT::i32 &&
20187                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20188               Select(Tmp0, Tmp0);
20189               Select(Tmp1, Tmp1);
20190               Select(Tmp2, Tmp2);
20191               Select(Tmp3, Tmp3);
20192               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
20193               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
20194               Select(Chain10, Chain10);
20195               ResNode = CurDAG->getTargetNode(X86::AND32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20196               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20197               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20198               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20199               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20200               Result = SDOperand(ResNode, N.ResNo);
20201               return;
20202             }
20203           }
20204         }
20205       }
20206     }
20207     if (N1.getOpcode() == ISD::OR &&
20208         N1.hasOneUse() &&
20209         !CodeGenMap.count(N1.getValue(0))) {
20210       N10 = N1.getOperand(0);
20211       if (N10.getOpcode() == ISD::LOAD &&
20212           N10.hasOneUse() &&
20213           !CodeGenMap.count(N10.getValue(0)) &&
20214           !isNonImmUse(N1.Val, N10.Val) &&
20215           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20216         N101 = N10.getOperand(1);
20217         N11 = N1.getOperand(1);
20218         if (N11.getOpcode() == ISD::Constant) {
20219           N2 = N.getOperand(2);
20220           if (N101 == N2) {
20221 
20222             // Pattern: (store:void (or:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20223             // Emits: (OR8mi:void addr:i32:$dst, (imm:i8):$src)
20224             // Pattern complexity = 28  cost = 1
20225             if (N1.Val->getValueType(0) == MVT::i8 &&
20226                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20227               Select(Tmp0, Tmp0);
20228               Select(Tmp1, Tmp1);
20229               Select(Tmp2, Tmp2);
20230               Select(Tmp3, Tmp3);
20231               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20232               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20233               Select(Chain10, Chain10);
20234               ResNode = CurDAG->getTargetNode(X86::OR8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20235               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20236               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20237               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20238               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20239               Result = SDOperand(ResNode, N.ResNo);
20240               return;
20241             }
20242 
20243             // Pattern: (store:void (or:i16 (load:i16 addr:i32:$dst), (imm:i16):$src), addr:i32:$dst)
20244             // Emits: (OR16mi:void addr:i32:$dst, (imm:i16):$src)
20245             // Pattern complexity = 28  cost = 1
20246             if (N1.Val->getValueType(0) == MVT::i16 &&
20247                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20248               Select(Tmp0, Tmp0);
20249               Select(Tmp1, Tmp1);
20250               Select(Tmp2, Tmp2);
20251               Select(Tmp3, Tmp3);
20252               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
20253               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
20254               Select(Chain10, Chain10);
20255               ResNode = CurDAG->getTargetNode(X86::OR16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20256               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20257               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20258               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20259               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20260               Result = SDOperand(ResNode, N.ResNo);
20261               return;
20262             }
20263 
20264             // Pattern: (store:void (or:i32 (load:i32 addr:i32:$dst), (imm:i32):$src), addr:i32:$dst)
20265             // Emits: (OR32mi:void addr:i32:$dst, (imm:i32):$src)
20266             // Pattern complexity = 28  cost = 1
20267             if (N1.Val->getValueType(0) == MVT::i32 &&
20268                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20269               Select(Tmp0, Tmp0);
20270               Select(Tmp1, Tmp1);
20271               Select(Tmp2, Tmp2);
20272               Select(Tmp3, Tmp3);
20273               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
20274               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
20275               Select(Chain10, Chain10);
20276               ResNode = CurDAG->getTargetNode(X86::OR32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20277               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20278               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20279               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20280               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20281               Result = SDOperand(ResNode, N.ResNo);
20282               return;
20283             }
20284           }
20285         }
20286       }
20287     }
20288     if (N1.getOpcode() == ISD::XOR &&
20289         N1.hasOneUse() &&
20290         !CodeGenMap.count(N1.getValue(0))) {
20291       N10 = N1.getOperand(0);
20292       if (N10.getOpcode() == ISD::LOAD &&
20293           N10.hasOneUse() &&
20294           !CodeGenMap.count(N10.getValue(0)) &&
20295           !isNonImmUse(N1.Val, N10.Val) &&
20296           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20297         N101 = N10.getOperand(1);
20298         N11 = N1.getOperand(1);
20299         if (N11.getOpcode() == ISD::Constant) {
20300           N2 = N.getOperand(2);
20301           if (N101 == N2) {
20302 
20303             // Pattern: (store:void (xor:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20304             // Emits: (XOR8mi:void addr:i32:$dst, (imm:i8):$src)
20305             // Pattern complexity = 28  cost = 1
20306             if (N1.Val->getValueType(0) == MVT::i8 &&
20307                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20308               Select(Tmp0, Tmp0);
20309               Select(Tmp1, Tmp1);
20310               Select(Tmp2, Tmp2);
20311               Select(Tmp3, Tmp3);
20312               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20313               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20314               Select(Chain10, Chain10);
20315               ResNode = CurDAG->getTargetNode(X86::XOR8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20316               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20317               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20318               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20319               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20320               Result = SDOperand(ResNode, N.ResNo);
20321               return;
20322             }
20323 
20324             // Pattern: (store:void (xor:i16 (load:i16 addr:i32:$dst), (imm:i16):$src), addr:i32:$dst)
20325             // Emits: (XOR16mi:void addr:i32:$dst, (imm:i16):$src)
20326             // Pattern complexity = 28  cost = 1
20327             if (N1.Val->getValueType(0) == MVT::i16 &&
20328                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20329               Select(Tmp0, Tmp0);
20330               Select(Tmp1, Tmp1);
20331               Select(Tmp2, Tmp2);
20332               Select(Tmp3, Tmp3);
20333               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
20334               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
20335               Select(Chain10, Chain10);
20336               ResNode = CurDAG->getTargetNode(X86::XOR16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20337               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20338               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20339               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20340               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20341               Result = SDOperand(ResNode, N.ResNo);
20342               return;
20343             }
20344 
20345             // Pattern: (store:void (xor:i32 (load:i32 addr:i32:$dst), (imm:i32):$src), addr:i32:$dst)
20346             // Emits: (XOR32mi:void addr:i32:$dst, (imm:i32):$src)
20347             // Pattern complexity = 28  cost = 1
20348             if (N1.Val->getValueType(0) == MVT::i32 &&
20349                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20350               Select(Tmp0, Tmp0);
20351               Select(Tmp1, Tmp1);
20352               Select(Tmp2, Tmp2);
20353               Select(Tmp3, Tmp3);
20354               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
20355               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
20356               Select(Chain10, Chain10);
20357               ResNode = CurDAG->getTargetNode(X86::XOR32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20358               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20359               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20360               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20361               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20362               Result = SDOperand(ResNode, N.ResNo);
20363               return;
20364             }
20365           }
20366         }
20367       }
20368     }
20369     if (N1.getOpcode() == ISD::SHL &&
20370         N1.hasOneUse() &&
20371         !CodeGenMap.count(N1.getValue(0))) {
20372       N10 = N1.getOperand(0);
20373       if (N10.getOpcode() == ISD::LOAD &&
20374           N10.hasOneUse() &&
20375           !CodeGenMap.count(N10.getValue(0)) &&
20376           !isNonImmUse(N1.Val, N10.Val) &&
20377           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20378         N101 = N10.getOperand(1);
20379         N11 = N1.getOperand(1);
20380         if (N11.getOpcode() == ISD::Constant) {
20381           N2 = N.getOperand(2);
20382           if (N101 == N2) {
20383 
20384             // Pattern: (store:void (shl:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20385             // Emits: (SHL8mi:void addr:i32:$dst, (imm:i8):$src)
20386             // Pattern complexity = 28  cost = 1
20387             if (N1.Val->getValueType(0) == MVT::i8 &&
20388                 N11.Val->getValueType(0) == MVT::i8 &&
20389                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20390               Select(Tmp0, Tmp0);
20391               Select(Tmp1, Tmp1);
20392               Select(Tmp2, Tmp2);
20393               Select(Tmp3, Tmp3);
20394               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20395               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20396               Select(Chain10, Chain10);
20397               ResNode = CurDAG->getTargetNode(X86::SHL8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20398               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20399               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20400               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20401               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20402               Result = SDOperand(ResNode, N.ResNo);
20403               return;
20404             }
20405 
20406             // Pattern: (store:void (shl:i16 (load:i16 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20407             // Emits: (SHL16mi:void addr:i32:$dst, (imm:i8):$src)
20408             // Pattern complexity = 28  cost = 1
20409             if (N1.Val->getValueType(0) == MVT::i16 &&
20410                 N11.Val->getValueType(0) == MVT::i8 &&
20411                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20412               Select(Tmp0, Tmp0);
20413               Select(Tmp1, Tmp1);
20414               Select(Tmp2, Tmp2);
20415               Select(Tmp3, Tmp3);
20416               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20417               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20418               Select(Chain10, Chain10);
20419               ResNode = CurDAG->getTargetNode(X86::SHL16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20420               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20421               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20422               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20423               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20424               Result = SDOperand(ResNode, N.ResNo);
20425               return;
20426             }
20427 
20428             // Pattern: (store:void (shl:i32 (load:i32 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20429             // Emits: (SHL32mi:void addr:i32:$dst, (imm:i8):$src)
20430             // Pattern complexity = 28  cost = 1
20431             if (N1.Val->getValueType(0) == MVT::i32 &&
20432                 N11.Val->getValueType(0) == MVT::i8 &&
20433                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20434               Select(Tmp0, Tmp0);
20435               Select(Tmp1, Tmp1);
20436               Select(Tmp2, Tmp2);
20437               Select(Tmp3, Tmp3);
20438               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20439               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20440               Select(Chain10, Chain10);
20441               ResNode = CurDAG->getTargetNode(X86::SHL32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20442               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20443               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20444               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20445               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20446               Result = SDOperand(ResNode, N.ResNo);
20447               return;
20448             }
20449           }
20450         }
20451       }
20452     }
20453     if (N1.getOpcode() == ISD::SRL &&
20454         N1.hasOneUse() &&
20455         !CodeGenMap.count(N1.getValue(0))) {
20456       N10 = N1.getOperand(0);
20457       if (N10.getOpcode() == ISD::LOAD &&
20458           N10.hasOneUse() &&
20459           !CodeGenMap.count(N10.getValue(0)) &&
20460           !isNonImmUse(N1.Val, N10.Val) &&
20461           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20462         N101 = N10.getOperand(1);
20463         N11 = N1.getOperand(1);
20464         if (N11.getOpcode() == ISD::Constant) {
20465           N2 = N.getOperand(2);
20466           if (N101 == N2) {
20467 
20468             // Pattern: (store:void (srl:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20469             // Emits: (SHR8mi:void addr:i32:$dst, (imm:i8):$src)
20470             // Pattern complexity = 28  cost = 1
20471             if (N1.Val->getValueType(0) == MVT::i8 &&
20472                 N11.Val->getValueType(0) == MVT::i8 &&
20473                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20474               Select(Tmp0, Tmp0);
20475               Select(Tmp1, Tmp1);
20476               Select(Tmp2, Tmp2);
20477               Select(Tmp3, Tmp3);
20478               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20479               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20480               Select(Chain10, Chain10);
20481               ResNode = CurDAG->getTargetNode(X86::SHR8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20482               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20483               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20484               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20485               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20486               Result = SDOperand(ResNode, N.ResNo);
20487               return;
20488             }
20489 
20490             // Pattern: (store:void (srl:i16 (load:i16 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20491             // Emits: (SHR16mi:void addr:i32:$dst, (imm:i8):$src)
20492             // Pattern complexity = 28  cost = 1
20493             if (N1.Val->getValueType(0) == MVT::i16 &&
20494                 N11.Val->getValueType(0) == MVT::i8 &&
20495                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20496               Select(Tmp0, Tmp0);
20497               Select(Tmp1, Tmp1);
20498               Select(Tmp2, Tmp2);
20499               Select(Tmp3, Tmp3);
20500               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20501               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20502               Select(Chain10, Chain10);
20503               ResNode = CurDAG->getTargetNode(X86::SHR16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20504               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20505               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20506               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20507               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20508               Result = SDOperand(ResNode, N.ResNo);
20509               return;
20510             }
20511 
20512             // Pattern: (store:void (srl:i32 (load:i32 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20513             // Emits: (SHR32mi:void addr:i32:$dst, (imm:i8):$src)
20514             // Pattern complexity = 28  cost = 1
20515             if (N1.Val->getValueType(0) == MVT::i32 &&
20516                 N11.Val->getValueType(0) == MVT::i8 &&
20517                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20518               Select(Tmp0, Tmp0);
20519               Select(Tmp1, Tmp1);
20520               Select(Tmp2, Tmp2);
20521               Select(Tmp3, Tmp3);
20522               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20523               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20524               Select(Chain10, Chain10);
20525               ResNode = CurDAG->getTargetNode(X86::SHR32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20526               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20527               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20528               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20529               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20530               Result = SDOperand(ResNode, N.ResNo);
20531               return;
20532             }
20533           }
20534         }
20535       }
20536     }
20537     if (N1.getOpcode() == ISD::SRA &&
20538         N1.hasOneUse() &&
20539         !CodeGenMap.count(N1.getValue(0))) {
20540       N10 = N1.getOperand(0);
20541       if (N10.getOpcode() == ISD::LOAD &&
20542           N10.hasOneUse() &&
20543           !CodeGenMap.count(N10.getValue(0)) &&
20544           !isNonImmUse(N1.Val, N10.Val) &&
20545           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20546         N101 = N10.getOperand(1);
20547         N11 = N1.getOperand(1);
20548         if (N11.getOpcode() == ISD::Constant) {
20549           N2 = N.getOperand(2);
20550           if (N101 == N2) {
20551 
20552             // Pattern: (store:void (sra:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20553             // Emits: (SAR8mi:void addr:i32:$dst, (imm:i8):$src)
20554             // Pattern complexity = 28  cost = 1
20555             if (N1.Val->getValueType(0) == MVT::i8 &&
20556                 N11.Val->getValueType(0) == MVT::i8 &&
20557                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20558               Select(Tmp0, Tmp0);
20559               Select(Tmp1, Tmp1);
20560               Select(Tmp2, Tmp2);
20561               Select(Tmp3, Tmp3);
20562               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20563               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20564               Select(Chain10, Chain10);
20565               ResNode = CurDAG->getTargetNode(X86::SAR8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20566               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20567               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20568               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20569               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20570               Result = SDOperand(ResNode, N.ResNo);
20571               return;
20572             }
20573 
20574             // Pattern: (store:void (sra:i16 (load:i16 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20575             // Emits: (SAR16mi:void addr:i32:$dst, (imm:i8):$src)
20576             // Pattern complexity = 28  cost = 1
20577             if (N1.Val->getValueType(0) == MVT::i16 &&
20578                 N11.Val->getValueType(0) == MVT::i8 &&
20579                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20580               Select(Tmp0, Tmp0);
20581               Select(Tmp1, Tmp1);
20582               Select(Tmp2, Tmp2);
20583               Select(Tmp3, Tmp3);
20584               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20585               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20586               Select(Chain10, Chain10);
20587               ResNode = CurDAG->getTargetNode(X86::SAR16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20588               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20589               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20590               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20591               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20592               Result = SDOperand(ResNode, N.ResNo);
20593               return;
20594             }
20595 
20596             // Pattern: (store:void (sra:i32 (load:i32 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20597             // Emits: (SAR32mi:void addr:i32:$dst, (imm:i8):$src)
20598             // Pattern complexity = 28  cost = 1
20599             if (N1.Val->getValueType(0) == MVT::i32 &&
20600                 N11.Val->getValueType(0) == MVT::i8 &&
20601                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20602               Select(Tmp0, Tmp0);
20603               Select(Tmp1, Tmp1);
20604               Select(Tmp2, Tmp2);
20605               Select(Tmp3, Tmp3);
20606               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20607               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20608               Select(Chain10, Chain10);
20609               ResNode = CurDAG->getTargetNode(X86::SAR32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20610               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20611               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20612               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20613               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20614               Result = SDOperand(ResNode, N.ResNo);
20615               return;
20616             }
20617           }
20618         }
20619       }
20620     }
20621     if (N1.getOpcode() == ISD::ROTL &&
20622         N1.hasOneUse() &&
20623         !CodeGenMap.count(N1.getValue(0))) {
20624       N10 = N1.getOperand(0);
20625       if (N10.getOpcode() == ISD::LOAD &&
20626           N10.hasOneUse() &&
20627           !CodeGenMap.count(N10.getValue(0)) &&
20628           !isNonImmUse(N1.Val, N10.Val) &&
20629           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20630         N101 = N10.getOperand(1);
20631         N11 = N1.getOperand(1);
20632         if (N11.getOpcode() == ISD::Constant) {
20633           N2 = N.getOperand(2);
20634           if (N101 == N2) {
20635 
20636             // Pattern: (store:void (rotl:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20637             // Emits: (ROL8mi:void addr:i32:$dst, (imm:i8):$src)
20638             // Pattern complexity = 28  cost = 1
20639             if (N1.Val->getValueType(0) == MVT::i8 &&
20640                 N11.Val->getValueType(0) == MVT::i8 &&
20641                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20642               Select(Tmp0, Tmp0);
20643               Select(Tmp1, Tmp1);
20644               Select(Tmp2, Tmp2);
20645               Select(Tmp3, Tmp3);
20646               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20647               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20648               Select(Chain10, Chain10);
20649               ResNode = CurDAG->getTargetNode(X86::ROL8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20650               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20651               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20652               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20653               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20654               Result = SDOperand(ResNode, N.ResNo);
20655               return;
20656             }
20657 
20658             // Pattern: (store:void (rotl:i16 (load:i16 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20659             // Emits: (ROL16mi:void addr:i32:$dst, (imm:i8):$src)
20660             // Pattern complexity = 28  cost = 1
20661             if (N1.Val->getValueType(0) == MVT::i16 &&
20662                 N11.Val->getValueType(0) == MVT::i8 &&
20663                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20664               Select(Tmp0, Tmp0);
20665               Select(Tmp1, Tmp1);
20666               Select(Tmp2, Tmp2);
20667               Select(Tmp3, Tmp3);
20668               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20669               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20670               Select(Chain10, Chain10);
20671               ResNode = CurDAG->getTargetNode(X86::ROL16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20672               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20673               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20674               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20675               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20676               Result = SDOperand(ResNode, N.ResNo);
20677               return;
20678             }
20679 
20680             // Pattern: (store:void (rotl:i32 (load:i32 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20681             // Emits: (ROL32mi:void addr:i32:$dst, (imm:i8):$src)
20682             // Pattern complexity = 28  cost = 1
20683             if (N1.Val->getValueType(0) == MVT::i32 &&
20684                 N11.Val->getValueType(0) == MVT::i8 &&
20685                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20686               Select(Tmp0, Tmp0);
20687               Select(Tmp1, Tmp1);
20688               Select(Tmp2, Tmp2);
20689               Select(Tmp3, Tmp3);
20690               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20691               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20692               Select(Chain10, Chain10);
20693               ResNode = CurDAG->getTargetNode(X86::ROL32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20694               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20695               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20696               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20697               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20698               Result = SDOperand(ResNode, N.ResNo);
20699               return;
20700             }
20701           }
20702         }
20703       }
20704     }
20705     if (N1.getOpcode() == ISD::ROTR &&
20706         N1.hasOneUse() &&
20707         !CodeGenMap.count(N1.getValue(0))) {
20708       N10 = N1.getOperand(0);
20709       if (N10.getOpcode() == ISD::LOAD &&
20710           N10.hasOneUse() &&
20711           !CodeGenMap.count(N10.getValue(0)) &&
20712           !isNonImmUse(N1.Val, N10.Val) &&
20713           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20714         N101 = N10.getOperand(1);
20715         N11 = N1.getOperand(1);
20716         if (N11.getOpcode() == ISD::Constant) {
20717           N2 = N.getOperand(2);
20718           if (N101 == N2) {
20719 
20720             // Pattern: (store:void (rotr:i8 (load:i8 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20721             // Emits: (ROR8mi:void addr:i32:$dst, (imm:i8):$src)
20722             // Pattern complexity = 28  cost = 1
20723             if (N1.Val->getValueType(0) == MVT::i8 &&
20724                 N11.Val->getValueType(0) == MVT::i8 &&
20725                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20726               Select(Tmp0, Tmp0);
20727               Select(Tmp1, Tmp1);
20728               Select(Tmp2, Tmp2);
20729               Select(Tmp3, Tmp3);
20730               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20731               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20732               Select(Chain10, Chain10);
20733               ResNode = CurDAG->getTargetNode(X86::ROR8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20734               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20735               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20736               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20737               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20738               Result = SDOperand(ResNode, N.ResNo);
20739               return;
20740             }
20741 
20742             // Pattern: (store:void (rotr:i16 (load:i16 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20743             // Emits: (ROR16mi:void addr:i32:$dst, (imm:i8):$src)
20744             // Pattern complexity = 28  cost = 1
20745             if (N1.Val->getValueType(0) == MVT::i16 &&
20746                 N11.Val->getValueType(0) == MVT::i8 &&
20747                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20748               Select(Tmp0, Tmp0);
20749               Select(Tmp1, Tmp1);
20750               Select(Tmp2, Tmp2);
20751               Select(Tmp3, Tmp3);
20752               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20753               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20754               Select(Chain10, Chain10);
20755               ResNode = CurDAG->getTargetNode(X86::ROR16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20756               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20757               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20758               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20759               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20760               Result = SDOperand(ResNode, N.ResNo);
20761               return;
20762             }
20763 
20764             // Pattern: (store:void (rotr:i32 (load:i32 addr:i32:$dst), (imm:i8):$src), addr:i32:$dst)
20765             // Emits: (ROR32mi:void addr:i32:$dst, (imm:i8):$src)
20766             // Pattern complexity = 28  cost = 1
20767             if (N1.Val->getValueType(0) == MVT::i32 &&
20768                 N11.Val->getValueType(0) == MVT::i8 &&
20769                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20770               Select(Tmp0, Tmp0);
20771               Select(Tmp1, Tmp1);
20772               Select(Tmp2, Tmp2);
20773               Select(Tmp3, Tmp3);
20774               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20775               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20776               Select(Chain10, Chain10);
20777               ResNode = CurDAG->getTargetNode(X86::ROR32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20778               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20779               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20780               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20781               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20782               Result = SDOperand(ResNode, N.ResNo);
20783               return;
20784             }
20785           }
20786         }
20787       }
20788     }
20789 
20790     // Pattern: (store:void (X86shld:i32 (load:i32 addr:i32:$dst), R32:i32:$src2, (imm:i8):$src3), addr:i32:$dst)
20791     // Emits: (SHLD32mri8:void addr:i32:$dst, R32:i32:$src2, (imm:i8):$src3)
20792     // Pattern complexity = 28  cost = 1
20793     if (N1.getOpcode() == X86ISD::SHLD &&
20794         N1.hasOneUse() &&
20795         !CodeGenMap.count(N1.getValue(0))) {
20796       N10 = N1.getOperand(0);
20797       if (N10.getOpcode() == ISD::LOAD &&
20798           N10.hasOneUse() &&
20799           !CodeGenMap.count(N10.getValue(0)) &&
20800           !isNonImmUse(N1.Val, N10.Val) &&
20801           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20802         N101 = N10.getOperand(1);
20803         N11 = N1.getOperand(1);
20804         N12 = N1.getOperand(2);
20805         if (N12.getOpcode() == ISD::Constant) {
20806           N2 = N.getOperand(2);
20807           if (N101 == N2 &&
20808               N1.Val->getValueType(0) == MVT::i32 &&
20809               N12.Val->getValueType(0) == MVT::i8 &&
20810               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20811             Select(Tmp0, Tmp0);
20812             Select(Tmp1, Tmp1);
20813             Select(Tmp2, Tmp2);
20814             Select(Tmp3, Tmp3);
20815             Select(Tmp4, N11);
20816             unsigned char Tmp5C = (unsigned char)cast<ConstantSDNode>(N12)->getValue();
20817             Tmp5 = CurDAG->getTargetConstant(Tmp5C, MVT::i8);
20818             Select(Chain10, Chain10);
20819             ResNode = CurDAG->getTargetNode(X86::SHLD32mri8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain10);
20820             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20821             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20822             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20823             AddHandleReplacement(N10.Val, 1, ResNode, 0);
20824             Result = SDOperand(ResNode, N.ResNo);
20825             return;
20826           }
20827         }
20828       }
20829     }
20830 
20831     // Pattern: (store:void (X86shrd:i32 (load:i32 addr:i32:$dst), R32:i32:$src2, (imm:i8):$src3), addr:i32:$dst)
20832     // Emits: (SHRD32mri8:void addr:i32:$dst, R32:i32:$src2, (imm:i8):$src3)
20833     // Pattern complexity = 28  cost = 1
20834     if (N1.getOpcode() == X86ISD::SHRD &&
20835         N1.hasOneUse() &&
20836         !CodeGenMap.count(N1.getValue(0))) {
20837       N10 = N1.getOperand(0);
20838       if (N10.getOpcode() == ISD::LOAD &&
20839           N10.hasOneUse() &&
20840           !CodeGenMap.count(N10.getValue(0)) &&
20841           !isNonImmUse(N1.Val, N10.Val) &&
20842           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20843         N101 = N10.getOperand(1);
20844         N11 = N1.getOperand(1);
20845         N12 = N1.getOperand(2);
20846         if (N12.getOpcode() == ISD::Constant) {
20847           N2 = N.getOperand(2);
20848           if (N101 == N2 &&
20849               N1.Val->getValueType(0) == MVT::i32 &&
20850               N12.Val->getValueType(0) == MVT::i8 &&
20851               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20852             Select(Tmp0, Tmp0);
20853             Select(Tmp1, Tmp1);
20854             Select(Tmp2, Tmp2);
20855             Select(Tmp3, Tmp3);
20856             Select(Tmp4, N11);
20857             unsigned char Tmp5C = (unsigned char)cast<ConstantSDNode>(N12)->getValue();
20858             Tmp5 = CurDAG->getTargetConstant(Tmp5C, MVT::i8);
20859             Select(Chain10, Chain10);
20860             ResNode = CurDAG->getTargetNode(X86::SHRD32mri8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain10);
20861             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20862             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20863             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20864             AddHandleReplacement(N10.Val, 1, ResNode, 0);
20865             Result = SDOperand(ResNode, N.ResNo);
20866             return;
20867           }
20868         }
20869       }
20870     }
20871 
20872     // Pattern: (store:void (X86shld:i16 (load:i16 addr:i32:$dst), R16:i16:$src2, (imm:i8):$src3), addr:i32:$dst)
20873     // Emits: (SHLD16mri8:void addr:i32:$dst, R16:i16:$src2, (imm:i8):$src3)
20874     // Pattern complexity = 28  cost = 1
20875     if (N1.getOpcode() == X86ISD::SHLD &&
20876         N1.hasOneUse() &&
20877         !CodeGenMap.count(N1.getValue(0))) {
20878       N10 = N1.getOperand(0);
20879       if (N10.getOpcode() == ISD::LOAD &&
20880           N10.hasOneUse() &&
20881           !CodeGenMap.count(N10.getValue(0)) &&
20882           !isNonImmUse(N1.Val, N10.Val) &&
20883           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20884         N101 = N10.getOperand(1);
20885         N11 = N1.getOperand(1);
20886         N12 = N1.getOperand(2);
20887         if (N12.getOpcode() == ISD::Constant) {
20888           N2 = N.getOperand(2);
20889           if (N101 == N2 &&
20890               N1.Val->getValueType(0) == MVT::i16 &&
20891               N12.Val->getValueType(0) == MVT::i8 &&
20892               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20893             Select(Tmp0, Tmp0);
20894             Select(Tmp1, Tmp1);
20895             Select(Tmp2, Tmp2);
20896             Select(Tmp3, Tmp3);
20897             Select(Tmp4, N11);
20898             unsigned char Tmp5C = (unsigned char)cast<ConstantSDNode>(N12)->getValue();
20899             Tmp5 = CurDAG->getTargetConstant(Tmp5C, MVT::i8);
20900             Select(Chain10, Chain10);
20901             ResNode = CurDAG->getTargetNode(X86::SHLD16mri8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain10);
20902             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20903             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20904             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20905             AddHandleReplacement(N10.Val, 1, ResNode, 0);
20906             Result = SDOperand(ResNode, N.ResNo);
20907             return;
20908           }
20909         }
20910       }
20911     }
20912 
20913     // Pattern: (store:void (X86shrd:i16 (load:i16 addr:i32:$dst), R16:i16:$src2, (imm:i8):$src3), addr:i32:$dst)
20914     // Emits: (SHRD16mri8:void addr:i32:$dst, R16:i16:$src2, (imm:i8):$src3)
20915     // Pattern complexity = 28  cost = 1
20916     if (N1.getOpcode() == X86ISD::SHRD &&
20917         N1.hasOneUse() &&
20918         !CodeGenMap.count(N1.getValue(0))) {
20919       N10 = N1.getOperand(0);
20920       if (N10.getOpcode() == ISD::LOAD &&
20921           N10.hasOneUse() &&
20922           !CodeGenMap.count(N10.getValue(0)) &&
20923           !isNonImmUse(N1.Val, N10.Val) &&
20924           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20925         N101 = N10.getOperand(1);
20926         N11 = N1.getOperand(1);
20927         N12 = N1.getOperand(2);
20928         if (N12.getOpcode() == ISD::Constant) {
20929           N2 = N.getOperand(2);
20930           if (N101 == N2 &&
20931               N1.Val->getValueType(0) == MVT::i16 &&
20932               N12.Val->getValueType(0) == MVT::i8 &&
20933               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20934             Select(Tmp0, Tmp0);
20935             Select(Tmp1, Tmp1);
20936             Select(Tmp2, Tmp2);
20937             Select(Tmp3, Tmp3);
20938             Select(Tmp4, N11);
20939             unsigned char Tmp5C = (unsigned char)cast<ConstantSDNode>(N12)->getValue();
20940             Tmp5 = CurDAG->getTargetConstant(Tmp5C, MVT::i8);
20941             Select(Chain10, Chain10);
20942             ResNode = CurDAG->getTargetNode(X86::SHRD16mri8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain10);
20943             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20944             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20945             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20946             AddHandleReplacement(N10.Val, 1, ResNode, 0);
20947             Result = SDOperand(ResNode, N.ResNo);
20948             return;
20949           }
20950         }
20951       }
20952     }
20953     if (N1.getOpcode() == ISD::ADD &&
20954         N1.hasOneUse() &&
20955         !CodeGenMap.count(N1.getValue(0))) {
20956       N10 = N1.getOperand(0);
20957       if (N10.getOpcode() == ISD::LOAD &&
20958           N10.hasOneUse() &&
20959           !CodeGenMap.count(N10.getValue(0)) &&
20960           !isNonImmUse(N1.Val, N10.Val) &&
20961           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
20962         N101 = N10.getOperand(1);
20963         N11 = N1.getOperand(1);
20964         if (N11.getOpcode() == ISD::Constant) {
20965           N2 = N.getOperand(2);
20966           if (N101 == N2) {
20967 
20968             // Pattern: (store:void (add:i8 (load:i8 addr:i32:$dst), (imm:i8):$src2), addr:i32:$dst)
20969             // Emits: (ADD8mi:void addr:i32:$dst, (imm:i8):$src2)
20970             // Pattern complexity = 28  cost = 1
20971             if (N1.Val->getValueType(0) == MVT::i8 &&
20972                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20973               Select(Tmp0, Tmp0);
20974               Select(Tmp1, Tmp1);
20975               Select(Tmp2, Tmp2);
20976               Select(Tmp3, Tmp3);
20977               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
20978               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
20979               Select(Chain10, Chain10);
20980               ResNode = CurDAG->getTargetNode(X86::ADD8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
20981               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
20982               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
20983               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
20984               AddHandleReplacement(N10.Val, 1, ResNode, 0);
20985               Result = SDOperand(ResNode, N.ResNo);
20986               return;
20987             }
20988 
20989             // Pattern: (store:void (add:i16 (load:i16 addr:i32:$dst), (imm:i16):$src2), addr:i32:$dst)
20990             // Emits: (ADD16mi:void addr:i32:$dst, (imm:i16):$src2)
20991             // Pattern complexity = 28  cost = 1
20992             if (N1.Val->getValueType(0) == MVT::i16 &&
20993                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
20994               Select(Tmp0, Tmp0);
20995               Select(Tmp1, Tmp1);
20996               Select(Tmp2, Tmp2);
20997               Select(Tmp3, Tmp3);
20998               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
20999               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
21000               Select(Chain10, Chain10);
21001               ResNode = CurDAG->getTargetNode(X86::ADD16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21002               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21003               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21004               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21005               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21006               Result = SDOperand(ResNode, N.ResNo);
21007               return;
21008             }
21009 
21010             // Pattern: (store:void (add:i32 (load:i32 addr:i32:$dst), (imm:i32):$src2), addr:i32:$dst)
21011             // Emits: (ADD32mi:void addr:i32:$dst, (imm:i32):$src2)
21012             // Pattern complexity = 28  cost = 1
21013             if (N1.Val->getValueType(0) == MVT::i32 &&
21014                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21015               Select(Tmp0, Tmp0);
21016               Select(Tmp1, Tmp1);
21017               Select(Tmp2, Tmp2);
21018               Select(Tmp3, Tmp3);
21019               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
21020               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
21021               Select(Chain10, Chain10);
21022               ResNode = CurDAG->getTargetNode(X86::ADD32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21023               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21024               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21025               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21026               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21027               Result = SDOperand(ResNode, N.ResNo);
21028               return;
21029             }
21030           }
21031         }
21032       }
21033     }
21034 
21035     // Pattern: (store:void (adde:i32 (load:i32 addr:i32:$dst), (imm:i32):$src2), addr:i32:$dst)
21036     // Emits: (ADC32mi:void addr:i32:$dst, (imm:i32):$src2)
21037     // Pattern complexity = 28  cost = 1
21038     if (N1.getOpcode() == ISD::ADDE &&
21039         N1.hasOneUse() &&
21040         !CodeGenMap.count(N1.getValue(0))) {
21041       N10 = N1.getOperand(0);
21042       if (N10.getOpcode() == ISD::LOAD &&
21043           N10.hasOneUse() &&
21044           !CodeGenMap.count(N10.getValue(0)) &&
21045           N1.Val->isOnlyUse(N10.Val) &&
21046           !isNonImmUse(N1.Val, N10.Val) &&
21047           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21048         N101 = N10.getOperand(1);
21049         N11 = N1.getOperand(1);
21050         if (N11.getOpcode() == ISD::Constant) {
21051           N2 = N.getOperand(2);
21052           if (N101 == N2 &&
21053               N1.Val->getValueType(0) == MVT::i32 &&
21054               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21055             Select(Tmp0, Tmp0);
21056             Select(Tmp1, Tmp1);
21057             Select(Tmp2, Tmp2);
21058             Select(Tmp3, Tmp3);
21059             unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
21060             Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
21061             Select(Chain10, Chain10);
21062             Select(InFlag, N1.getOperand(2));
21063             ResNode = CurDAG->getTargetNode(X86::ADC32mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
21064             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21065             InFlag = SDOperand(ResNode, 1);
21066             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21067             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21068             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21069             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
21070             Result = SDOperand(ResNode, N.ResNo);
21071             return;
21072           }
21073         }
21074       }
21075     }
21076     if (N1.getOpcode() == ISD::SUB &&
21077         N1.hasOneUse() &&
21078         !CodeGenMap.count(N1.getValue(0))) {
21079       N10 = N1.getOperand(0);
21080       if (N10.getOpcode() == ISD::LOAD &&
21081           N10.hasOneUse() &&
21082           !CodeGenMap.count(N10.getValue(0)) &&
21083           !isNonImmUse(N1.Val, N10.Val) &&
21084           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21085         N101 = N10.getOperand(1);
21086         N11 = N1.getOperand(1);
21087         if (N11.getOpcode() == ISD::Constant) {
21088           N2 = N.getOperand(2);
21089           if (N101 == N2) {
21090 
21091             // Pattern: (store:void (sub:i8 (load:i8 addr:i32:$dst), (imm:i8):$src2), addr:i32:$dst)
21092             // Emits: (SUB8mi:void addr:i32:$dst, (imm:i8):$src2)
21093             // Pattern complexity = 28  cost = 1
21094             if (N1.Val->getValueType(0) == MVT::i8 &&
21095                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21096               Select(Tmp0, Tmp0);
21097               Select(Tmp1, Tmp1);
21098               Select(Tmp2, Tmp2);
21099               Select(Tmp3, Tmp3);
21100               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
21101               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
21102               Select(Chain10, Chain10);
21103               ResNode = CurDAG->getTargetNode(X86::SUB8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21104               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21105               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21106               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21107               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21108               Result = SDOperand(ResNode, N.ResNo);
21109               return;
21110             }
21111 
21112             // Pattern: (store:void (sub:i16 (load:i16 addr:i32:$dst), (imm:i16):$src2), addr:i32:$dst)
21113             // Emits: (SUB16mi:void addr:i32:$dst, (imm:i16):$src2)
21114             // Pattern complexity = 28  cost = 1
21115             if (N1.Val->getValueType(0) == MVT::i16 &&
21116                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21117               Select(Tmp0, Tmp0);
21118               Select(Tmp1, Tmp1);
21119               Select(Tmp2, Tmp2);
21120               Select(Tmp3, Tmp3);
21121               unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N11)->getValue();
21122               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
21123               Select(Chain10, Chain10);
21124               ResNode = CurDAG->getTargetNode(X86::SUB16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21125               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21126               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21127               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21128               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21129               Result = SDOperand(ResNode, N.ResNo);
21130               return;
21131             }
21132 
21133             // Pattern: (store:void (sub:i32 (load:i32 addr:i32:$dst), (imm:i32):$src2), addr:i32:$dst)
21134             // Emits: (SUB32mi:void addr:i32:$dst, (imm:i32):$src2)
21135             // Pattern complexity = 28  cost = 1
21136             if (N1.Val->getValueType(0) == MVT::i32 &&
21137                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21138               Select(Tmp0, Tmp0);
21139               Select(Tmp1, Tmp1);
21140               Select(Tmp2, Tmp2);
21141               Select(Tmp3, Tmp3);
21142               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
21143               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
21144               Select(Chain10, Chain10);
21145               ResNode = CurDAG->getTargetNode(X86::SUB32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21146               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21147               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21148               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21149               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21150               Result = SDOperand(ResNode, N.ResNo);
21151               return;
21152             }
21153           }
21154         }
21155       }
21156     }
21157     if (N1.getOpcode() == ISD::SUBE &&
21158         N1.hasOneUse() &&
21159         !CodeGenMap.count(N1.getValue(0))) {
21160       N10 = N1.getOperand(0);
21161       if (N10.getOpcode() == ISD::LOAD &&
21162           N10.hasOneUse() &&
21163           !CodeGenMap.count(N10.getValue(0)) &&
21164           N1.Val->isOnlyUse(N10.Val) &&
21165           !isNonImmUse(N1.Val, N10.Val) &&
21166           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21167         N101 = N10.getOperand(1);
21168         N11 = N1.getOperand(1);
21169         if (N11.getOpcode() == ISD::Constant) {
21170           N2 = N.getOperand(2);
21171           if (N101 == N2) {
21172 
21173             // Pattern: (store:void (sube:i8 (load:i8 addr:i32:$dst), (imm:i8):$src2), addr:i32:$dst)
21174             // Emits: (SBB8mi:void addr:i32:$dst, (imm:i8):$src2)
21175             // Pattern complexity = 28  cost = 1
21176             if (N1.Val->getValueType(0) == MVT::i8 &&
21177                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21178               Select(Tmp0, Tmp0);
21179               Select(Tmp1, Tmp1);
21180               Select(Tmp2, Tmp2);
21181               Select(Tmp3, Tmp3);
21182               unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N11)->getValue();
21183               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
21184               Select(Chain10, Chain10);
21185               Select(InFlag, N1.getOperand(2));
21186               ResNode = CurDAG->getTargetNode(X86::SBB8mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
21187               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21188               InFlag = SDOperand(ResNode, 1);
21189               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21190               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21191               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21192               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
21193               Result = SDOperand(ResNode, N.ResNo);
21194               return;
21195             }
21196 
21197             // Pattern: (store:void (sube:i32 (load:i32 addr:i32:$dst), (imm:i32):$src2), addr:i32:$dst)
21198             // Emits: (SBB32mi:void addr:i32:$dst, (imm:i32):$src2)
21199             // Pattern complexity = 28  cost = 1
21200             if (N1.Val->getValueType(0) == MVT::i32 &&
21201                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21202               Select(Tmp0, Tmp0);
21203               Select(Tmp1, Tmp1);
21204               Select(Tmp2, Tmp2);
21205               Select(Tmp3, Tmp3);
21206               unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N11)->getValue();
21207               Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
21208               Select(Chain10, Chain10);
21209               Select(InFlag, N1.getOperand(2));
21210               ResNode = CurDAG->getTargetNode(X86::SBB32mi, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
21211               if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21212               InFlag = SDOperand(ResNode, 1);
21213               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21214               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21215               AddHandleReplacement(N10.Val, 1, ResNode, 0);
21216               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
21217               Result = SDOperand(ResNode, N.ResNo);
21218               return;
21219             }
21220           }
21221         }
21222       }
21223     }
21224     if (N1.getOpcode() == ISD::AND &&
21225         N1.hasOneUse() &&
21226         !CodeGenMap.count(N1.getValue(0))) {
21227       N10 = N1.getOperand(0);
21228       if (N10.getOpcode() == ISD::LOAD &&
21229           N10.hasOneUse() &&
21230           !CodeGenMap.count(N10.getValue(0)) &&
21231           !isNonImmUse(N1.Val, N10.Val) &&
21232           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21233         N101 = N10.getOperand(1);
21234         N11 = N1.getOperand(1);
21235         N2 = N.getOperand(2);
21236         if (N101 == N2) {
21237 
21238           // Pattern: (store:void (and:i8 (load:i8 addr:i32:$dst), R8:i8:$src), addr:i32:$dst)
21239           // Emits: (AND8mr:void addr:i32:$dst, R8:i8:$src)
21240           // Pattern complexity = 26  cost = 1
21241           if (N1.Val->getValueType(0) == MVT::i8 &&
21242               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21243             Select(Tmp0, Tmp0);
21244             Select(Tmp1, Tmp1);
21245             Select(Tmp2, Tmp2);
21246             Select(Tmp3, Tmp3);
21247             Select(Tmp4, N11);
21248             Select(Chain10, Chain10);
21249             ResNode = CurDAG->getTargetNode(X86::AND8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21250             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21251             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21252             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21253             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21254             Result = SDOperand(ResNode, N.ResNo);
21255             return;
21256           }
21257 
21258           // Pattern: (store:void (and:i16 (load:i16 addr:i32:$dst), R16:i16:$src), addr:i32:$dst)
21259           // Emits: (AND16mr:void addr:i32:$dst, R16:i16:$src)
21260           // Pattern complexity = 26  cost = 1
21261           if (N1.Val->getValueType(0) == MVT::i16 &&
21262               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21263             Select(Tmp0, Tmp0);
21264             Select(Tmp1, Tmp1);
21265             Select(Tmp2, Tmp2);
21266             Select(Tmp3, Tmp3);
21267             Select(Tmp4, N11);
21268             Select(Chain10, Chain10);
21269             ResNode = CurDAG->getTargetNode(X86::AND16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21270             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21271             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21272             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21273             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21274             Result = SDOperand(ResNode, N.ResNo);
21275             return;
21276           }
21277 
21278           // Pattern: (store:void (and:i32 (load:i32 addr:i32:$dst), R32:i32:$src), addr:i32:$dst)
21279           // Emits: (AND32mr:void addr:i32:$dst, R32:i32:$src)
21280           // Pattern complexity = 26  cost = 1
21281           if (N1.Val->getValueType(0) == MVT::i32 &&
21282               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21283             Select(Tmp0, Tmp0);
21284             Select(Tmp1, Tmp1);
21285             Select(Tmp2, Tmp2);
21286             Select(Tmp3, Tmp3);
21287             Select(Tmp4, N11);
21288             Select(Chain10, Chain10);
21289             ResNode = CurDAG->getTargetNode(X86::AND32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21290             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21291             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21292             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21293             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21294             Result = SDOperand(ResNode, N.ResNo);
21295             return;
21296           }
21297         }
21298       }
21299     }
21300     if (N1.getOpcode() == ISD::OR &&
21301         N1.hasOneUse() &&
21302         !CodeGenMap.count(N1.getValue(0))) {
21303       N10 = N1.getOperand(0);
21304       if (N10.getOpcode() == ISD::LOAD &&
21305           N10.hasOneUse() &&
21306           !CodeGenMap.count(N10.getValue(0)) &&
21307           !isNonImmUse(N1.Val, N10.Val) &&
21308           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21309         N101 = N10.getOperand(1);
21310         N11 = N1.getOperand(1);
21311         N2 = N.getOperand(2);
21312         if (N101 == N2) {
21313 
21314           // Pattern: (store:void (or:i8 (load:i8 addr:i32:$dst), R8:i8:$src), addr:i32:$dst)
21315           // Emits: (OR8mr:void addr:i32:$dst, R8:i8:$src)
21316           // Pattern complexity = 26  cost = 1
21317           if (N1.Val->getValueType(0) == MVT::i8 &&
21318               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21319             Select(Tmp0, Tmp0);
21320             Select(Tmp1, Tmp1);
21321             Select(Tmp2, Tmp2);
21322             Select(Tmp3, Tmp3);
21323             Select(Tmp4, N11);
21324             Select(Chain10, Chain10);
21325             ResNode = CurDAG->getTargetNode(X86::OR8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21326             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21327             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21328             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21329             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21330             Result = SDOperand(ResNode, N.ResNo);
21331             return;
21332           }
21333 
21334           // Pattern: (store:void (or:i16 (load:i16 addr:i32:$dst), R16:i16:$src), addr:i32:$dst)
21335           // Emits: (OR16mr:void addr:i32:$dst, R16:i16:$src)
21336           // Pattern complexity = 26  cost = 1
21337           if (N1.Val->getValueType(0) == MVT::i16 &&
21338               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21339             Select(Tmp0, Tmp0);
21340             Select(Tmp1, Tmp1);
21341             Select(Tmp2, Tmp2);
21342             Select(Tmp3, Tmp3);
21343             Select(Tmp4, N11);
21344             Select(Chain10, Chain10);
21345             ResNode = CurDAG->getTargetNode(X86::OR16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21346             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21347             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21348             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21349             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21350             Result = SDOperand(ResNode, N.ResNo);
21351             return;
21352           }
21353 
21354           // Pattern: (store:void (or:i32 (load:i32 addr:i32:$dst), R32:i32:$src), addr:i32:$dst)
21355           // Emits: (OR32mr:void addr:i32:$dst, R32:i32:$src)
21356           // Pattern complexity = 26  cost = 1
21357           if (N1.Val->getValueType(0) == MVT::i32 &&
21358               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21359             Select(Tmp0, Tmp0);
21360             Select(Tmp1, Tmp1);
21361             Select(Tmp2, Tmp2);
21362             Select(Tmp3, Tmp3);
21363             Select(Tmp4, N11);
21364             Select(Chain10, Chain10);
21365             ResNode = CurDAG->getTargetNode(X86::OR32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21366             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21367             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21368             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21369             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21370             Result = SDOperand(ResNode, N.ResNo);
21371             return;
21372           }
21373         }
21374       }
21375     }
21376     if (N1.getOpcode() == ISD::XOR &&
21377         N1.hasOneUse() &&
21378         !CodeGenMap.count(N1.getValue(0))) {
21379       N10 = N1.getOperand(0);
21380       if (N10.getOpcode() == ISD::LOAD &&
21381           N10.hasOneUse() &&
21382           !CodeGenMap.count(N10.getValue(0)) &&
21383           !isNonImmUse(N1.Val, N10.Val) &&
21384           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21385         N101 = N10.getOperand(1);
21386         N11 = N1.getOperand(1);
21387         N2 = N.getOperand(2);
21388         if (N101 == N2) {
21389 
21390           // Pattern: (store:void (xor:i8 (load:i8 addr:i32:$dst), R8:i8:$src), addr:i32:$dst)
21391           // Emits: (XOR8mr:void addr:i32:$dst, R8:i8:$src)
21392           // Pattern complexity = 26  cost = 1
21393           if (N1.Val->getValueType(0) == MVT::i8 &&
21394               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21395             Select(Tmp0, Tmp0);
21396             Select(Tmp1, Tmp1);
21397             Select(Tmp2, Tmp2);
21398             Select(Tmp3, Tmp3);
21399             Select(Tmp4, N11);
21400             Select(Chain10, Chain10);
21401             ResNode = CurDAG->getTargetNode(X86::XOR8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21402             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21403             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21404             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21405             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21406             Result = SDOperand(ResNode, N.ResNo);
21407             return;
21408           }
21409 
21410           // Pattern: (store:void (xor:i16 (load:i16 addr:i32:$dst), R16:i16:$src), addr:i32:$dst)
21411           // Emits: (XOR16mr:void addr:i32:$dst, R16:i16:$src)
21412           // Pattern complexity = 26  cost = 1
21413           if (N1.Val->getValueType(0) == MVT::i16 &&
21414               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21415             Select(Tmp0, Tmp0);
21416             Select(Tmp1, Tmp1);
21417             Select(Tmp2, Tmp2);
21418             Select(Tmp3, Tmp3);
21419             Select(Tmp4, N11);
21420             Select(Chain10, Chain10);
21421             ResNode = CurDAG->getTargetNode(X86::XOR16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21422             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21423             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21424             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21425             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21426             Result = SDOperand(ResNode, N.ResNo);
21427             return;
21428           }
21429 
21430           // Pattern: (store:void (xor:i32 (load:i32 addr:i32:$dst), R32:i32:$src), addr:i32:$dst)
21431           // Emits: (XOR32mr:void addr:i32:$dst, R32:i32:$src)
21432           // Pattern complexity = 26  cost = 1
21433           if (N1.Val->getValueType(0) == MVT::i32 &&
21434               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21435             Select(Tmp0, Tmp0);
21436             Select(Tmp1, Tmp1);
21437             Select(Tmp2, Tmp2);
21438             Select(Tmp3, Tmp3);
21439             Select(Tmp4, N11);
21440             Select(Chain10, Chain10);
21441             ResNode = CurDAG->getTargetNode(X86::XOR32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
21442             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21443             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21444             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21445             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21446             Result = SDOperand(ResNode, N.ResNo);
21447             return;
21448           }
21449         }
21450       }
21451     }
21452     if (N1.getOpcode() == ISD::SHL &&
21453         N1.hasOneUse() &&
21454         !CodeGenMap.count(N1.getValue(0))) {
21455       N10 = N1.getOperand(0);
21456       if (N10.getOpcode() == ISD::LOAD &&
21457           N10.hasOneUse() &&
21458           !CodeGenMap.count(N10.getValue(0)) &&
21459           !isNonImmUse(N1.Val, N10.Val) &&
21460           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21461         N101 = N10.getOperand(1);
21462         N11 = N1.getOperand(1);
21463         N2 = N.getOperand(2);
21464         if (N101 == N2) {
21465 
21466           // Pattern: (store:void (shl:i8 (load:i8 addr:i32:$dst), CL:i8), addr:i32:$dst)
21467           // Emits: (SHL8mCL:void addr:i32:$dst)
21468           // Pattern complexity = 26  cost = 1
21469           if (N1.Val->getValueType(0) == MVT::i8 &&
21470               N11.Val->getValueType(0) == MVT::i8 &&
21471               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21472             Select(Tmp0, Tmp0);
21473             Select(Tmp1, Tmp1);
21474             Select(Tmp2, Tmp2);
21475             Select(Tmp3, Tmp3);
21476             Select(Chain10, Chain10);
21477             Select(N11, N11);
21478             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21479             Chain10 = SDOperand(ResNode, 0);
21480             InFlag = SDOperand(ResNode, 1);
21481             ResNode = CurDAG->getTargetNode(X86::SHL8mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21482             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21483             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21484             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21485             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21486             Result = SDOperand(ResNode, N.ResNo);
21487             return;
21488           }
21489 
21490           // Pattern: (store:void (shl:i16 (load:i16 addr:i32:$dst), CL:i8), addr:i32:$dst)
21491           // Emits: (SHL16mCL:void addr:i32:$dst)
21492           // Pattern complexity = 26  cost = 1
21493           if (N1.Val->getValueType(0) == MVT::i16 &&
21494               N11.Val->getValueType(0) == MVT::i8 &&
21495               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21496             Select(Tmp0, Tmp0);
21497             Select(Tmp1, Tmp1);
21498             Select(Tmp2, Tmp2);
21499             Select(Tmp3, Tmp3);
21500             Select(Chain10, Chain10);
21501             Select(N11, N11);
21502             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21503             Chain10 = SDOperand(ResNode, 0);
21504             InFlag = SDOperand(ResNode, 1);
21505             ResNode = CurDAG->getTargetNode(X86::SHL16mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21506             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21507             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21508             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21509             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21510             Result = SDOperand(ResNode, N.ResNo);
21511             return;
21512           }
21513 
21514           // Pattern: (store:void (shl:i32 (load:i32 addr:i32:$dst), CL:i8), addr:i32:$dst)
21515           // Emits: (SHL32mCL:void addr:i32:$dst)
21516           // Pattern complexity = 26  cost = 1
21517           if (N1.Val->getValueType(0) == MVT::i32 &&
21518               N11.Val->getValueType(0) == MVT::i8 &&
21519               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21520             Select(Tmp0, Tmp0);
21521             Select(Tmp1, Tmp1);
21522             Select(Tmp2, Tmp2);
21523             Select(Tmp3, Tmp3);
21524             Select(Chain10, Chain10);
21525             Select(N11, N11);
21526             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21527             Chain10 = SDOperand(ResNode, 0);
21528             InFlag = SDOperand(ResNode, 1);
21529             ResNode = CurDAG->getTargetNode(X86::SHL32mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21530             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21531             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21532             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21533             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21534             Result = SDOperand(ResNode, N.ResNo);
21535             return;
21536           }
21537         }
21538       }
21539     }
21540     if (N1.getOpcode() == ISD::SRL &&
21541         N1.hasOneUse() &&
21542         !CodeGenMap.count(N1.getValue(0))) {
21543       N10 = N1.getOperand(0);
21544       if (N10.getOpcode() == ISD::LOAD &&
21545           N10.hasOneUse() &&
21546           !CodeGenMap.count(N10.getValue(0)) &&
21547           !isNonImmUse(N1.Val, N10.Val) &&
21548           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21549         N101 = N10.getOperand(1);
21550         N11 = N1.getOperand(1);
21551         N2 = N.getOperand(2);
21552         if (N101 == N2) {
21553 
21554           // Pattern: (store:void (srl:i8 (load:i8 addr:i32:$dst), CL:i8), addr:i32:$dst)
21555           // Emits: (SHR8mCL:void addr:i32:$dst)
21556           // Pattern complexity = 26  cost = 1
21557           if (N1.Val->getValueType(0) == MVT::i8 &&
21558               N11.Val->getValueType(0) == MVT::i8 &&
21559               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21560             Select(Tmp0, Tmp0);
21561             Select(Tmp1, Tmp1);
21562             Select(Tmp2, Tmp2);
21563             Select(Tmp3, Tmp3);
21564             Select(Chain10, Chain10);
21565             Select(N11, N11);
21566             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21567             Chain10 = SDOperand(ResNode, 0);
21568             InFlag = SDOperand(ResNode, 1);
21569             ResNode = CurDAG->getTargetNode(X86::SHR8mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21570             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21571             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21572             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21573             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21574             Result = SDOperand(ResNode, N.ResNo);
21575             return;
21576           }
21577 
21578           // Pattern: (store:void (srl:i16 (load:i16 addr:i32:$dst), CL:i8), addr:i32:$dst)
21579           // Emits: (SHR16mCL:void addr:i32:$dst)
21580           // Pattern complexity = 26  cost = 1
21581           if (N1.Val->getValueType(0) == MVT::i16 &&
21582               N11.Val->getValueType(0) == MVT::i8 &&
21583               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21584             Select(Tmp0, Tmp0);
21585             Select(Tmp1, Tmp1);
21586             Select(Tmp2, Tmp2);
21587             Select(Tmp3, Tmp3);
21588             Select(Chain10, Chain10);
21589             Select(N11, N11);
21590             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21591             Chain10 = SDOperand(ResNode, 0);
21592             InFlag = SDOperand(ResNode, 1);
21593             ResNode = CurDAG->getTargetNode(X86::SHR16mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21594             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21595             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21596             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21597             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21598             Result = SDOperand(ResNode, N.ResNo);
21599             return;
21600           }
21601 
21602           // Pattern: (store:void (srl:i32 (load:i32 addr:i32:$dst), CL:i8), addr:i32:$dst)
21603           // Emits: (SHR32mCL:void addr:i32:$dst)
21604           // Pattern complexity = 26  cost = 1
21605           if (N1.Val->getValueType(0) == MVT::i32 &&
21606               N11.Val->getValueType(0) == MVT::i8 &&
21607               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21608             Select(Tmp0, Tmp0);
21609             Select(Tmp1, Tmp1);
21610             Select(Tmp2, Tmp2);
21611             Select(Tmp3, Tmp3);
21612             Select(Chain10, Chain10);
21613             Select(N11, N11);
21614             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21615             Chain10 = SDOperand(ResNode, 0);
21616             InFlag = SDOperand(ResNode, 1);
21617             ResNode = CurDAG->getTargetNode(X86::SHR32mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21618             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21619             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21620             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21621             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21622             Result = SDOperand(ResNode, N.ResNo);
21623             return;
21624           }
21625         }
21626       }
21627     }
21628     if (N1.getOpcode() == ISD::SRA &&
21629         N1.hasOneUse() &&
21630         !CodeGenMap.count(N1.getValue(0))) {
21631       N10 = N1.getOperand(0);
21632       if (N10.getOpcode() == ISD::LOAD &&
21633           N10.hasOneUse() &&
21634           !CodeGenMap.count(N10.getValue(0)) &&
21635           !isNonImmUse(N1.Val, N10.Val) &&
21636           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21637         N101 = N10.getOperand(1);
21638         N11 = N1.getOperand(1);
21639         N2 = N.getOperand(2);
21640         if (N101 == N2) {
21641 
21642           // Pattern: (store:void (sra:i8 (load:i8 addr:i32:$dst), CL:i8), addr:i32:$dst)
21643           // Emits: (SAR8mCL:void addr:i32:$dst)
21644           // Pattern complexity = 26  cost = 1
21645           if (N1.Val->getValueType(0) == MVT::i8 &&
21646               N11.Val->getValueType(0) == MVT::i8 &&
21647               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21648             Select(Tmp0, Tmp0);
21649             Select(Tmp1, Tmp1);
21650             Select(Tmp2, Tmp2);
21651             Select(Tmp3, Tmp3);
21652             Select(Chain10, Chain10);
21653             Select(N11, N11);
21654             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21655             Chain10 = SDOperand(ResNode, 0);
21656             InFlag = SDOperand(ResNode, 1);
21657             ResNode = CurDAG->getTargetNode(X86::SAR8mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21658             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21659             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21660             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21661             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21662             Result = SDOperand(ResNode, N.ResNo);
21663             return;
21664           }
21665 
21666           // Pattern: (store:void (sra:i16 (load:i16 addr:i32:$dst), CL:i8), addr:i32:$dst)
21667           // Emits: (SAR16mCL:void addr:i32:$dst)
21668           // Pattern complexity = 26  cost = 1
21669           if (N1.Val->getValueType(0) == MVT::i16 &&
21670               N11.Val->getValueType(0) == MVT::i8 &&
21671               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21672             Select(Tmp0, Tmp0);
21673             Select(Tmp1, Tmp1);
21674             Select(Tmp2, Tmp2);
21675             Select(Tmp3, Tmp3);
21676             Select(Chain10, Chain10);
21677             Select(N11, N11);
21678             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21679             Chain10 = SDOperand(ResNode, 0);
21680             InFlag = SDOperand(ResNode, 1);
21681             ResNode = CurDAG->getTargetNode(X86::SAR16mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21682             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21683             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21684             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21685             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21686             Result = SDOperand(ResNode, N.ResNo);
21687             return;
21688           }
21689 
21690           // Pattern: (store:void (sra:i32 (load:i32 addr:i32:$dst), CL:i8), addr:i32:$dst)
21691           // Emits: (SAR32mCL:void addr:i32:$dst)
21692           // Pattern complexity = 26  cost = 1
21693           if (N1.Val->getValueType(0) == MVT::i32 &&
21694               N11.Val->getValueType(0) == MVT::i8 &&
21695               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21696             Select(Tmp0, Tmp0);
21697             Select(Tmp1, Tmp1);
21698             Select(Tmp2, Tmp2);
21699             Select(Tmp3, Tmp3);
21700             Select(Chain10, Chain10);
21701             Select(N11, N11);
21702             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21703             Chain10 = SDOperand(ResNode, 0);
21704             InFlag = SDOperand(ResNode, 1);
21705             ResNode = CurDAG->getTargetNode(X86::SAR32mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21706             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21707             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21708             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21709             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21710             Result = SDOperand(ResNode, N.ResNo);
21711             return;
21712           }
21713         }
21714       }
21715     }
21716     if (N1.getOpcode() == ISD::ROTL &&
21717         N1.hasOneUse() &&
21718         !CodeGenMap.count(N1.getValue(0))) {
21719       N10 = N1.getOperand(0);
21720       if (N10.getOpcode() == ISD::LOAD &&
21721           N10.hasOneUse() &&
21722           !CodeGenMap.count(N10.getValue(0)) &&
21723           !isNonImmUse(N1.Val, N10.Val) &&
21724           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21725         N101 = N10.getOperand(1);
21726         N11 = N1.getOperand(1);
21727         N2 = N.getOperand(2);
21728         if (N101 == N2) {
21729 
21730           // Pattern: (store:void (rotl:i8 (load:i8 addr:i32:$dst), CL:i8), addr:i32:$dst)
21731           // Emits: (ROL8mCL:void addr:i32:$dst)
21732           // Pattern complexity = 26  cost = 1
21733           if (N1.Val->getValueType(0) == MVT::i8 &&
21734               N11.Val->getValueType(0) == MVT::i8 &&
21735               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21736             Select(Tmp0, Tmp0);
21737             Select(Tmp1, Tmp1);
21738             Select(Tmp2, Tmp2);
21739             Select(Tmp3, Tmp3);
21740             Select(Chain10, Chain10);
21741             Select(N11, N11);
21742             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21743             Chain10 = SDOperand(ResNode, 0);
21744             InFlag = SDOperand(ResNode, 1);
21745             ResNode = CurDAG->getTargetNode(X86::ROL8mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21746             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21747             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21748             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21749             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21750             Result = SDOperand(ResNode, N.ResNo);
21751             return;
21752           }
21753 
21754           // Pattern: (store:void (rotl:i16 (load:i16 addr:i32:$dst), CL:i8), addr:i32:$dst)
21755           // Emits: (ROL16mCL:void addr:i32:$dst)
21756           // Pattern complexity = 26  cost = 1
21757           if (N1.Val->getValueType(0) == MVT::i16 &&
21758               N11.Val->getValueType(0) == MVT::i8 &&
21759               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21760             Select(Tmp0, Tmp0);
21761             Select(Tmp1, Tmp1);
21762             Select(Tmp2, Tmp2);
21763             Select(Tmp3, Tmp3);
21764             Select(Chain10, Chain10);
21765             Select(N11, N11);
21766             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21767             Chain10 = SDOperand(ResNode, 0);
21768             InFlag = SDOperand(ResNode, 1);
21769             ResNode = CurDAG->getTargetNode(X86::ROL16mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21770             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21771             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21772             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21773             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21774             Result = SDOperand(ResNode, N.ResNo);
21775             return;
21776           }
21777 
21778           // Pattern: (store:void (rotl:i32 (load:i32 addr:i32:$dst), CL:i8), addr:i32:$dst)
21779           // Emits: (ROL32mCL:void addr:i32:$dst)
21780           // Pattern complexity = 26  cost = 1
21781           if (N1.Val->getValueType(0) == MVT::i32 &&
21782               N11.Val->getValueType(0) == MVT::i8 &&
21783               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21784             Select(Tmp0, Tmp0);
21785             Select(Tmp1, Tmp1);
21786             Select(Tmp2, Tmp2);
21787             Select(Tmp3, Tmp3);
21788             Select(Chain10, Chain10);
21789             Select(N11, N11);
21790             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21791             Chain10 = SDOperand(ResNode, 0);
21792             InFlag = SDOperand(ResNode, 1);
21793             ResNode = CurDAG->getTargetNode(X86::ROL32mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21794             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21795             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21796             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21797             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21798             Result = SDOperand(ResNode, N.ResNo);
21799             return;
21800           }
21801         }
21802       }
21803     }
21804     if (N1.getOpcode() == ISD::ROTR &&
21805         N1.hasOneUse() &&
21806         !CodeGenMap.count(N1.getValue(0))) {
21807       N10 = N1.getOperand(0);
21808       if (N10.getOpcode() == ISD::LOAD &&
21809           N10.hasOneUse() &&
21810           !CodeGenMap.count(N10.getValue(0)) &&
21811           !isNonImmUse(N1.Val, N10.Val) &&
21812           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21813         N101 = N10.getOperand(1);
21814         N11 = N1.getOperand(1);
21815         N2 = N.getOperand(2);
21816         if (N101 == N2) {
21817 
21818           // Pattern: (store:void (rotr:i8 (load:i8 addr:i32:$dst), CL:i8), addr:i32:$dst)
21819           // Emits: (ROR8mCL:void addr:i32:$dst)
21820           // Pattern complexity = 26  cost = 1
21821           if (N1.Val->getValueType(0) == MVT::i8 &&
21822               N11.Val->getValueType(0) == MVT::i8 &&
21823               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21824             Select(Tmp0, Tmp0);
21825             Select(Tmp1, Tmp1);
21826             Select(Tmp2, Tmp2);
21827             Select(Tmp3, Tmp3);
21828             Select(Chain10, Chain10);
21829             Select(N11, N11);
21830             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21831             Chain10 = SDOperand(ResNode, 0);
21832             InFlag = SDOperand(ResNode, 1);
21833             ResNode = CurDAG->getTargetNode(X86::ROR8mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21834             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21835             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21836             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21837             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21838             Result = SDOperand(ResNode, N.ResNo);
21839             return;
21840           }
21841 
21842           // Pattern: (store:void (rotr:i16 (load:i16 addr:i32:$dst), CL:i8), addr:i32:$dst)
21843           // Emits: (ROR16mCL:void addr:i32:$dst)
21844           // Pattern complexity = 26  cost = 1
21845           if (N1.Val->getValueType(0) == MVT::i16 &&
21846               N11.Val->getValueType(0) == MVT::i8 &&
21847               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21848             Select(Tmp0, Tmp0);
21849             Select(Tmp1, Tmp1);
21850             Select(Tmp2, Tmp2);
21851             Select(Tmp3, Tmp3);
21852             Select(Chain10, Chain10);
21853             Select(N11, N11);
21854             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21855             Chain10 = SDOperand(ResNode, 0);
21856             InFlag = SDOperand(ResNode, 1);
21857             ResNode = CurDAG->getTargetNode(X86::ROR16mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21858             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21859             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21860             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21861             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21862             Result = SDOperand(ResNode, N.ResNo);
21863             return;
21864           }
21865 
21866           // Pattern: (store:void (rotr:i32 (load:i32 addr:i32:$dst), CL:i8), addr:i32:$dst)
21867           // Emits: (ROR32mCL:void addr:i32:$dst)
21868           // Pattern complexity = 26  cost = 1
21869           if (N1.Val->getValueType(0) == MVT::i32 &&
21870               N11.Val->getValueType(0) == MVT::i8 &&
21871               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21872             Select(Tmp0, Tmp0);
21873             Select(Tmp1, Tmp1);
21874             Select(Tmp2, Tmp2);
21875             Select(Tmp3, Tmp3);
21876             Select(Chain10, Chain10);
21877             Select(N11, N11);
21878             ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N11, InFlag).Val;
21879             Chain10 = SDOperand(ResNode, 0);
21880             InFlag = SDOperand(ResNode, 1);
21881             ResNode = CurDAG->getTargetNode(X86::ROR32mCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain10, InFlag);
21882             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21883             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21884             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21885             AddHandleReplacement(N10.Val, 1, ResNode, 0);
21886             Result = SDOperand(ResNode, N.ResNo);
21887             return;
21888           }
21889         }
21890       }
21891     }
21892 
21893     // Pattern: (store:void (X86shld:i32 (load:i32 addr:i32:$dst), R32:i32:$src2, CL:i8), addr:i32:$dst)
21894     // Emits: (SHLD32mrCL:void addr:i32:$dst, R32:i32:$src2)
21895     // Pattern complexity = 26  cost = 1
21896     if (N1.getOpcode() == X86ISD::SHLD &&
21897         N1.hasOneUse() &&
21898         !CodeGenMap.count(N1.getValue(0))) {
21899       N10 = N1.getOperand(0);
21900       if (N10.getOpcode() == ISD::LOAD &&
21901           N10.hasOneUse() &&
21902           !CodeGenMap.count(N10.getValue(0)) &&
21903           !isNonImmUse(N1.Val, N10.Val) &&
21904           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21905         N101 = N10.getOperand(1);
21906         N11 = N1.getOperand(1);
21907         N12 = N1.getOperand(2);
21908         N2 = N.getOperand(2);
21909         if (N101 == N2 &&
21910             N1.Val->getValueType(0) == MVT::i32 &&
21911             N12.Val->getValueType(0) == MVT::i8 &&
21912             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21913           Select(Tmp0, Tmp0);
21914           Select(Tmp1, Tmp1);
21915           Select(Tmp2, Tmp2);
21916           Select(Tmp3, Tmp3);
21917           Select(Tmp4, N11);
21918           Select(Chain10, Chain10);
21919           Select(N12, N12);
21920           ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N12, InFlag).Val;
21921           Chain10 = SDOperand(ResNode, 0);
21922           InFlag = SDOperand(ResNode, 1);
21923           ResNode = CurDAG->getTargetNode(X86::SHLD32mrCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
21924           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21925           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21926           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21927           AddHandleReplacement(N10.Val, 1, ResNode, 0);
21928           Result = SDOperand(ResNode, N.ResNo);
21929           return;
21930         }
21931       }
21932     }
21933 
21934     // Pattern: (store:void (X86shrd:i32 (load:i32 addr:i32:$dst), R32:i32:$src2, CL:i8), addr:i32:$dst)
21935     // Emits: (SHRD32mrCL:void addr:i32:$dst, R32:i32:$src2)
21936     // Pattern complexity = 26  cost = 1
21937     if (N1.getOpcode() == X86ISD::SHRD &&
21938         N1.hasOneUse() &&
21939         !CodeGenMap.count(N1.getValue(0))) {
21940       N10 = N1.getOperand(0);
21941       if (N10.getOpcode() == ISD::LOAD &&
21942           N10.hasOneUse() &&
21943           !CodeGenMap.count(N10.getValue(0)) &&
21944           !isNonImmUse(N1.Val, N10.Val) &&
21945           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21946         N101 = N10.getOperand(1);
21947         N11 = N1.getOperand(1);
21948         N12 = N1.getOperand(2);
21949         N2 = N.getOperand(2);
21950         if (N101 == N2 &&
21951             N1.Val->getValueType(0) == MVT::i32 &&
21952             N12.Val->getValueType(0) == MVT::i8 &&
21953             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21954           Select(Tmp0, Tmp0);
21955           Select(Tmp1, Tmp1);
21956           Select(Tmp2, Tmp2);
21957           Select(Tmp3, Tmp3);
21958           Select(Tmp4, N11);
21959           Select(Chain10, Chain10);
21960           Select(N12, N12);
21961           ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N12, InFlag).Val;
21962           Chain10 = SDOperand(ResNode, 0);
21963           InFlag = SDOperand(ResNode, 1);
21964           ResNode = CurDAG->getTargetNode(X86::SHRD32mrCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
21965           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
21966           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
21967           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
21968           AddHandleReplacement(N10.Val, 1, ResNode, 0);
21969           Result = SDOperand(ResNode, N.ResNo);
21970           return;
21971         }
21972       }
21973     }
21974 
21975     // Pattern: (store:void (X86shld:i16 (load:i16 addr:i32:$dst), R16:i16:$src2, CL:i8), addr:i32:$dst)
21976     // Emits: (SHLD16mrCL:void addr:i32:$dst, R16:i16:$src2)
21977     // Pattern complexity = 26  cost = 1
21978     if (N1.getOpcode() == X86ISD::SHLD &&
21979         N1.hasOneUse() &&
21980         !CodeGenMap.count(N1.getValue(0))) {
21981       N10 = N1.getOperand(0);
21982       if (N10.getOpcode() == ISD::LOAD &&
21983           N10.hasOneUse() &&
21984           !CodeGenMap.count(N10.getValue(0)) &&
21985           !isNonImmUse(N1.Val, N10.Val) &&
21986           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
21987         N101 = N10.getOperand(1);
21988         N11 = N1.getOperand(1);
21989         N12 = N1.getOperand(2);
21990         N2 = N.getOperand(2);
21991         if (N101 == N2 &&
21992             N1.Val->getValueType(0) == MVT::i16 &&
21993             N12.Val->getValueType(0) == MVT::i8 &&
21994             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
21995           Select(Tmp0, Tmp0);
21996           Select(Tmp1, Tmp1);
21997           Select(Tmp2, Tmp2);
21998           Select(Tmp3, Tmp3);
21999           Select(Tmp4, N11);
22000           Select(Chain10, Chain10);
22001           Select(N12, N12);
22002           ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N12, InFlag).Val;
22003           Chain10 = SDOperand(ResNode, 0);
22004           InFlag = SDOperand(ResNode, 1);
22005           ResNode = CurDAG->getTargetNode(X86::SHLD16mrCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
22006           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22007           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22008           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22009           AddHandleReplacement(N10.Val, 1, ResNode, 0);
22010           Result = SDOperand(ResNode, N.ResNo);
22011           return;
22012         }
22013       }
22014     }
22015 
22016     // Pattern: (store:void (X86shrd:i16 (load:i16 addr:i32:$dst), R16:i16:$src2, CL:i8), addr:i32:$dst)
22017     // Emits: (SHRD16mrCL:void addr:i32:$dst, R16:i16:$src2)
22018     // Pattern complexity = 26  cost = 1
22019     if (N1.getOpcode() == X86ISD::SHRD &&
22020         N1.hasOneUse() &&
22021         !CodeGenMap.count(N1.getValue(0))) {
22022       N10 = N1.getOperand(0);
22023       if (N10.getOpcode() == ISD::LOAD &&
22024           N10.hasOneUse() &&
22025           !CodeGenMap.count(N10.getValue(0)) &&
22026           !isNonImmUse(N1.Val, N10.Val) &&
22027           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
22028         N101 = N10.getOperand(1);
22029         N11 = N1.getOperand(1);
22030         N12 = N1.getOperand(2);
22031         N2 = N.getOperand(2);
22032         if (N101 == N2 &&
22033             N1.Val->getValueType(0) == MVT::i16 &&
22034             N12.Val->getValueType(0) == MVT::i8 &&
22035             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22036           Select(Tmp0, Tmp0);
22037           Select(Tmp1, Tmp1);
22038           Select(Tmp2, Tmp2);
22039           Select(Tmp3, Tmp3);
22040           Select(Tmp4, N11);
22041           Select(Chain10, Chain10);
22042           Select(N12, N12);
22043           ResNode = CurDAG->getCopyToReg(Chain10, CurDAG->getRegister(X86::CL, MVT::i8), N12, InFlag).Val;
22044           Chain10 = SDOperand(ResNode, 0);
22045           InFlag = SDOperand(ResNode, 1);
22046           ResNode = CurDAG->getTargetNode(X86::SHRD16mrCL, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
22047           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22048           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22049           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22050           AddHandleReplacement(N10.Val, 1, ResNode, 0);
22051           Result = SDOperand(ResNode, N.ResNo);
22052           return;
22053         }
22054       }
22055     }
22056     if (N1.getOpcode() == ISD::ADD &&
22057         N1.hasOneUse() &&
22058         !CodeGenMap.count(N1.getValue(0))) {
22059       N10 = N1.getOperand(0);
22060       if (N10.getOpcode() == ISD::LOAD &&
22061           N10.hasOneUse() &&
22062           !CodeGenMap.count(N10.getValue(0)) &&
22063           !isNonImmUse(N1.Val, N10.Val) &&
22064           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
22065         N101 = N10.getOperand(1);
22066         N11 = N1.getOperand(1);
22067         N2 = N.getOperand(2);
22068         if (N101 == N2) {
22069 
22070           // Pattern: (store:void (add:i8 (load:i8 addr:i32:$dst), R8:i8:$src2), addr:i32:$dst)
22071           // Emits: (ADD8mr:void addr:i32:$dst, R8:i8:$src2)
22072           // Pattern complexity = 26  cost = 1
22073           if (N1.Val->getValueType(0) == MVT::i8 &&
22074               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22075             Select(Tmp0, Tmp0);
22076             Select(Tmp1, Tmp1);
22077             Select(Tmp2, Tmp2);
22078             Select(Tmp3, Tmp3);
22079             Select(Tmp4, N11);
22080             Select(Chain10, Chain10);
22081             ResNode = CurDAG->getTargetNode(X86::ADD8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22082             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22083             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22084             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22085             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22086             Result = SDOperand(ResNode, N.ResNo);
22087             return;
22088           }
22089 
22090           // Pattern: (store:void (add:i16 (load:i16 addr:i32:$dst), R16:i16:$src2), addr:i32:$dst)
22091           // Emits: (ADD16mr:void addr:i32:$dst, R16:i16:$src2)
22092           // Pattern complexity = 26  cost = 1
22093           if (N1.Val->getValueType(0) == MVT::i16 &&
22094               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22095             Select(Tmp0, Tmp0);
22096             Select(Tmp1, Tmp1);
22097             Select(Tmp2, Tmp2);
22098             Select(Tmp3, Tmp3);
22099             Select(Tmp4, N11);
22100             Select(Chain10, Chain10);
22101             ResNode = CurDAG->getTargetNode(X86::ADD16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22102             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22103             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22104             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22105             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22106             Result = SDOperand(ResNode, N.ResNo);
22107             return;
22108           }
22109 
22110           // Pattern: (store:void (add:i32 (load:i32 addr:i32:$dst), R32:i32:$src2), addr:i32:$dst)
22111           // Emits: (ADD32mr:void addr:i32:$dst, R32:i32:$src2)
22112           // Pattern complexity = 26  cost = 1
22113           if (N1.Val->getValueType(0) == MVT::i32 &&
22114               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22115             Select(Tmp0, Tmp0);
22116             Select(Tmp1, Tmp1);
22117             Select(Tmp2, Tmp2);
22118             Select(Tmp3, Tmp3);
22119             Select(Tmp4, N11);
22120             Select(Chain10, Chain10);
22121             ResNode = CurDAG->getTargetNode(X86::ADD32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22122             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22123             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22124             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22125             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22126             Result = SDOperand(ResNode, N.ResNo);
22127             return;
22128           }
22129         }
22130       }
22131     }
22132 
22133     // Pattern: (store:void (adde:i32 (load:i32 addr:i32:$dst), R32:i32:$src2), addr:i32:$dst)
22134     // Emits: (ADC32mr:void addr:i32:$dst, R32:i32:$src2)
22135     // Pattern complexity = 26  cost = 1
22136     if (N1.getOpcode() == ISD::ADDE &&
22137         N1.hasOneUse() &&
22138         !CodeGenMap.count(N1.getValue(0))) {
22139       N10 = N1.getOperand(0);
22140       if (N10.getOpcode() == ISD::LOAD &&
22141           N10.hasOneUse() &&
22142           !CodeGenMap.count(N10.getValue(0)) &&
22143           N1.Val->isOnlyUse(N10.Val) &&
22144           !isNonImmUse(N1.Val, N10.Val) &&
22145           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
22146         N101 = N10.getOperand(1);
22147         N11 = N1.getOperand(1);
22148         N2 = N.getOperand(2);
22149         if (N101 == N2 &&
22150             N1.Val->getValueType(0) == MVT::i32 &&
22151             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22152           Select(Tmp0, Tmp0);
22153           Select(Tmp1, Tmp1);
22154           Select(Tmp2, Tmp2);
22155           Select(Tmp3, Tmp3);
22156           Select(Tmp4, N11);
22157           Select(Chain10, Chain10);
22158           Select(InFlag, N1.getOperand(2));
22159           ResNode = CurDAG->getTargetNode(X86::ADC32mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
22160           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22161           InFlag = SDOperand(ResNode, 1);
22162           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22163           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22164           AddHandleReplacement(N10.Val, 1, ResNode, 0);
22165           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22166           Result = SDOperand(ResNode, N.ResNo);
22167           return;
22168         }
22169       }
22170     }
22171     if (N1.getOpcode() == ISD::SUB &&
22172         N1.hasOneUse() &&
22173         !CodeGenMap.count(N1.getValue(0))) {
22174       N10 = N1.getOperand(0);
22175       if (N10.getOpcode() == ISD::LOAD &&
22176           N10.hasOneUse() &&
22177           !CodeGenMap.count(N10.getValue(0)) &&
22178           !isNonImmUse(N1.Val, N10.Val) &&
22179           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
22180         N101 = N10.getOperand(1);
22181         N11 = N1.getOperand(1);
22182         N2 = N.getOperand(2);
22183         if (N101 == N2) {
22184 
22185           // Pattern: (store:void (sub:i8 (load:i8 addr:i32:$dst), R8:i8:$src2), addr:i32:$dst)
22186           // Emits: (SUB8mr:void addr:i32:$dst, R8:i8:$src2)
22187           // Pattern complexity = 26  cost = 1
22188           if (N1.Val->getValueType(0) == MVT::i8 &&
22189               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22190             Select(Tmp0, Tmp0);
22191             Select(Tmp1, Tmp1);
22192             Select(Tmp2, Tmp2);
22193             Select(Tmp3, Tmp3);
22194             Select(Tmp4, N11);
22195             Select(Chain10, Chain10);
22196             ResNode = CurDAG->getTargetNode(X86::SUB8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22197             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22198             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22199             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22200             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22201             Result = SDOperand(ResNode, N.ResNo);
22202             return;
22203           }
22204 
22205           // Pattern: (store:void (sub:i16 (load:i16 addr:i32:$dst), R16:i16:$src2), addr:i32:$dst)
22206           // Emits: (SUB16mr:void addr:i32:$dst, R16:i16:$src2)
22207           // Pattern complexity = 26  cost = 1
22208           if (N1.Val->getValueType(0) == MVT::i16 &&
22209               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22210             Select(Tmp0, Tmp0);
22211             Select(Tmp1, Tmp1);
22212             Select(Tmp2, Tmp2);
22213             Select(Tmp3, Tmp3);
22214             Select(Tmp4, N11);
22215             Select(Chain10, Chain10);
22216             ResNode = CurDAG->getTargetNode(X86::SUB16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22217             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22218             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22219             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22220             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22221             Result = SDOperand(ResNode, N.ResNo);
22222             return;
22223           }
22224 
22225           // Pattern: (store:void (sub:i32 (load:i32 addr:i32:$dst), R32:i32:$src2), addr:i32:$dst)
22226           // Emits: (SUB32mr:void addr:i32:$dst, R32:i32:$src2)
22227           // Pattern complexity = 26  cost = 1
22228           if (N1.Val->getValueType(0) == MVT::i32 &&
22229               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22230             Select(Tmp0, Tmp0);
22231             Select(Tmp1, Tmp1);
22232             Select(Tmp2, Tmp2);
22233             Select(Tmp3, Tmp3);
22234             Select(Tmp4, N11);
22235             Select(Chain10, Chain10);
22236             ResNode = CurDAG->getTargetNode(X86::SUB32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10);
22237             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22238             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22239             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22240             AddHandleReplacement(N10.Val, 1, ResNode, 0);
22241             Result = SDOperand(ResNode, N.ResNo);
22242             return;
22243           }
22244         }
22245       }
22246     }
22247 
22248     // Pattern: (store:void (sube:i32 (load:i32 addr:i32:$dst), R32:i32:$src2), addr:i32:$dst)
22249     // Emits: (SBB32mr:void addr:i32:$dst, R32:i32:$src2)
22250     // Pattern complexity = 26  cost = 1
22251     if (N1.getOpcode() == ISD::SUBE &&
22252         N1.hasOneUse() &&
22253         !CodeGenMap.count(N1.getValue(0))) {
22254       N10 = N1.getOperand(0);
22255       if (N10.getOpcode() == ISD::LOAD &&
22256           N10.hasOneUse() &&
22257           !CodeGenMap.count(N10.getValue(0)) &&
22258           N1.Val->isOnlyUse(N10.Val) &&
22259           !isNonImmUse(N1.Val, N10.Val) &&
22260           (Chain10 = UpdateFoldedChain(CurDAG, N10.Val, Chain.Val, OldTF)).Val) {
22261         N101 = N10.getOperand(1);
22262         N11 = N1.getOperand(1);
22263         N2 = N.getOperand(2);
22264         if (N101 == N2 &&
22265             N1.Val->getValueType(0) == MVT::i32 &&
22266             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
22267           Select(Tmp0, Tmp0);
22268           Select(Tmp1, Tmp1);
22269           Select(Tmp2, Tmp2);
22270           Select(Tmp3, Tmp3);
22271           Select(Tmp4, N11);
22272           Select(Chain10, Chain10);
22273           Select(InFlag, N1.getOperand(2));
22274           ResNode = CurDAG->getTargetNode(X86::SBB32mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain10, InFlag);
22275           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain10.Val, 0);
22276           InFlag = SDOperand(ResNode, 1);
22277           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22278           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 0);
22279           AddHandleReplacement(N10.Val, 1, ResNode, 0);
22280           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22281           Result = SDOperand(ResNode, N.ResNo);
22282           return;
22283         }
22284       }
22285     }
22286   }
22287 
22288   // Pattern: (store:void (vector_extract:f64 (vector_shuffle:v2f64 (bitconvert:v2f64 VR128:v4f32:$src), (undef:v2f64), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>), 0:i32), addr:i32:$dst)
22289   // Emits: (MOVHPSmr:void addr:i32:$dst, VR128:v4f32:$src)
22290   // Pattern complexity = 26  cost = 1
22291   if ((Subtarget->hasSSE1())) {
22292     Chain = N.getOperand(0);
22293     N1 = N.getOperand(1);
22294     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22295       N10 = N1.getOperand(0);
22296       if (N10.getOpcode() == ISD::VECTOR_SHUFFLE) {
22297         N100 = N10.getOperand(0);
22298         if (N100.getOpcode() == ISD::BIT_CONVERT) {
22299           N1000 = N100.getOperand(0);
22300           N101 = N10.getOperand(1);
22301           if (N101.getOpcode() == ISD::UNDEF) {
22302             N102 = N10.getOperand(2);
22303             if (N102.getOpcode() == ISD::BUILD_VECTOR &&
22304                 Predicate_UNPCKH_shuffle_mask(N102.Val)) {
22305               N11 = N1.getOperand(1);
22306               if (isa<ConstantSDNode>(N11)) {
22307                 int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22308                 if (CN0 == 0) {
22309                   N2 = N.getOperand(2);
22310                   if (N1.Val->getValueType(0) == MVT::f64 &&
22311                       N10.Val->getValueType(0) == MVT::v2f64 &&
22312                       N1000.Val->getValueType(0) == MVT::v4f32 &&
22313                       N11.Val->getValueType(0) == MVT::i32 &&
22314                       SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22315                     Select(Tmp1, Tmp1);
22316                     Select(Tmp2, Tmp2);
22317                     Select(Tmp3, Tmp3);
22318                     Select(Tmp4, Tmp4);
22319                     Select(Tmp5, N1000);
22320                     Select(Chain, Chain);
22321                     ResNode = CurDAG->getTargetNode(X86::MOVHPSmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
22322                     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22323                     Result = SDOperand(ResNode, N.ResNo);
22324                     return;
22325                   }
22326                 }
22327               }
22328             }
22329           }
22330         }
22331       }
22332     }
22333   }
22334   {
22335     Chain = N.getOperand(0);
22336     N1 = N.getOperand(1);
22337     if (N1.getOpcode() == ISD::AND &&
22338         N1.hasOneUse() &&
22339         !CodeGenMap.count(N1.getValue(0))) {
22340       N10 = N1.getOperand(0);
22341       N11 = N1.getOperand(1);
22342       if (N11.getOpcode() == ISD::LOAD &&
22343           N11.hasOneUse() &&
22344           !CodeGenMap.count(N11.getValue(0)) &&
22345           !isNonImmUse(N1.Val, N11.Val) &&
22346           (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
22347         N111 = N11.getOperand(1);
22348         N2 = N.getOperand(2);
22349         if (N111 == N2) {
22350 
22351           // Pattern: (store:void (and:i8 R8:i8:$src, (load:i8 addr:i32:$dst)), addr:i32:$dst)
22352           // Emits: (AND8mr:void addr:i32:$dst, R8:i8:$src)
22353           // Pattern complexity = 26  cost = 1
22354           if (N1.Val->getValueType(0) == MVT::i8 &&
22355               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22356             Select(Tmp0, Tmp0);
22357             Select(Tmp1, Tmp1);
22358             Select(Tmp2, Tmp2);
22359             Select(Tmp3, Tmp3);
22360             Select(Tmp4, N10);
22361             Select(Chain11, Chain11);
22362             ResNode = CurDAG->getTargetNode(X86::AND8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22363             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22364             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22365             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22366             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22367             Result = SDOperand(ResNode, N.ResNo);
22368             return;
22369           }
22370 
22371           // Pattern: (store:void (and:i16 R16:i16:$src, (load:i16 addr:i32:$dst)), addr:i32:$dst)
22372           // Emits: (AND16mr:void addr:i32:$dst, R16:i16:$src)
22373           // Pattern complexity = 26  cost = 1
22374           if (N1.Val->getValueType(0) == MVT::i16 &&
22375               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22376             Select(Tmp0, Tmp0);
22377             Select(Tmp1, Tmp1);
22378             Select(Tmp2, Tmp2);
22379             Select(Tmp3, Tmp3);
22380             Select(Tmp4, N10);
22381             Select(Chain11, Chain11);
22382             ResNode = CurDAG->getTargetNode(X86::AND16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22383             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22384             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22385             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22386             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22387             Result = SDOperand(ResNode, N.ResNo);
22388             return;
22389           }
22390 
22391           // Pattern: (store:void (and:i32 R32:i32:$src, (load:i32 addr:i32:$dst)), addr:i32:$dst)
22392           // Emits: (AND32mr:void addr:i32:$dst, R32:i32:$src)
22393           // Pattern complexity = 26  cost = 1
22394           if (N1.Val->getValueType(0) == MVT::i32 &&
22395               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22396             Select(Tmp0, Tmp0);
22397             Select(Tmp1, Tmp1);
22398             Select(Tmp2, Tmp2);
22399             Select(Tmp3, Tmp3);
22400             Select(Tmp4, N10);
22401             Select(Chain11, Chain11);
22402             ResNode = CurDAG->getTargetNode(X86::AND32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22403             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22404             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22405             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22406             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22407             Result = SDOperand(ResNode, N.ResNo);
22408             return;
22409           }
22410         }
22411       }
22412     }
22413     if (N1.getOpcode() == ISD::OR &&
22414         N1.hasOneUse() &&
22415         !CodeGenMap.count(N1.getValue(0))) {
22416       N10 = N1.getOperand(0);
22417       N11 = N1.getOperand(1);
22418       if (N11.getOpcode() == ISD::LOAD &&
22419           N11.hasOneUse() &&
22420           !CodeGenMap.count(N11.getValue(0)) &&
22421           !isNonImmUse(N1.Val, N11.Val) &&
22422           (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
22423         N111 = N11.getOperand(1);
22424         N2 = N.getOperand(2);
22425         if (N111 == N2) {
22426 
22427           // Pattern: (store:void (or:i8 R8:i8:$src, (load:i8 addr:i32:$dst)), addr:i32:$dst)
22428           // Emits: (OR8mr:void addr:i32:$dst, R8:i8:$src)
22429           // Pattern complexity = 26  cost = 1
22430           if (N1.Val->getValueType(0) == MVT::i8 &&
22431               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22432             Select(Tmp0, Tmp0);
22433             Select(Tmp1, Tmp1);
22434             Select(Tmp2, Tmp2);
22435             Select(Tmp3, Tmp3);
22436             Select(Tmp4, N10);
22437             Select(Chain11, Chain11);
22438             ResNode = CurDAG->getTargetNode(X86::OR8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22439             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22440             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22441             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22442             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22443             Result = SDOperand(ResNode, N.ResNo);
22444             return;
22445           }
22446 
22447           // Pattern: (store:void (or:i16 R16:i16:$src, (load:i16 addr:i32:$dst)), addr:i32:$dst)
22448           // Emits: (OR16mr:void addr:i32:$dst, R16:i16:$src)
22449           // Pattern complexity = 26  cost = 1
22450           if (N1.Val->getValueType(0) == MVT::i16 &&
22451               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22452             Select(Tmp0, Tmp0);
22453             Select(Tmp1, Tmp1);
22454             Select(Tmp2, Tmp2);
22455             Select(Tmp3, Tmp3);
22456             Select(Tmp4, N10);
22457             Select(Chain11, Chain11);
22458             ResNode = CurDAG->getTargetNode(X86::OR16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22459             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22460             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22461             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22462             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22463             Result = SDOperand(ResNode, N.ResNo);
22464             return;
22465           }
22466 
22467           // Pattern: (store:void (or:i32 R32:i32:$src, (load:i32 addr:i32:$dst)), addr:i32:$dst)
22468           // Emits: (OR32mr:void addr:i32:$dst, R32:i32:$src)
22469           // Pattern complexity = 26  cost = 1
22470           if (N1.Val->getValueType(0) == MVT::i32 &&
22471               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22472             Select(Tmp0, Tmp0);
22473             Select(Tmp1, Tmp1);
22474             Select(Tmp2, Tmp2);
22475             Select(Tmp3, Tmp3);
22476             Select(Tmp4, N10);
22477             Select(Chain11, Chain11);
22478             ResNode = CurDAG->getTargetNode(X86::OR32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22479             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22480             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22481             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22482             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22483             Result = SDOperand(ResNode, N.ResNo);
22484             return;
22485           }
22486         }
22487       }
22488     }
22489     if (N1.getOpcode() == ISD::XOR &&
22490         N1.hasOneUse() &&
22491         !CodeGenMap.count(N1.getValue(0))) {
22492       N10 = N1.getOperand(0);
22493       N11 = N1.getOperand(1);
22494       if (N11.getOpcode() == ISD::LOAD &&
22495           N11.hasOneUse() &&
22496           !CodeGenMap.count(N11.getValue(0)) &&
22497           !isNonImmUse(N1.Val, N11.Val) &&
22498           (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
22499         N111 = N11.getOperand(1);
22500         N2 = N.getOperand(2);
22501         if (N111 == N2) {
22502 
22503           // Pattern: (store:void (xor:i8 R8:i8:$src, (load:i8 addr:i32:$dst)), addr:i32:$dst)
22504           // Emits: (XOR8mr:void addr:i32:$dst, R8:i8:$src)
22505           // Pattern complexity = 26  cost = 1
22506           if (N1.Val->getValueType(0) == MVT::i8 &&
22507               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22508             Select(Tmp0, Tmp0);
22509             Select(Tmp1, Tmp1);
22510             Select(Tmp2, Tmp2);
22511             Select(Tmp3, Tmp3);
22512             Select(Tmp4, N10);
22513             Select(Chain11, Chain11);
22514             ResNode = CurDAG->getTargetNode(X86::XOR8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22515             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22516             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22517             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22518             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22519             Result = SDOperand(ResNode, N.ResNo);
22520             return;
22521           }
22522 
22523           // Pattern: (store:void (xor:i16 R16:i16:$src, (load:i16 addr:i32:$dst)), addr:i32:$dst)
22524           // Emits: (XOR16mr:void addr:i32:$dst, R16:i16:$src)
22525           // Pattern complexity = 26  cost = 1
22526           if (N1.Val->getValueType(0) == MVT::i16 &&
22527               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22528             Select(Tmp0, Tmp0);
22529             Select(Tmp1, Tmp1);
22530             Select(Tmp2, Tmp2);
22531             Select(Tmp3, Tmp3);
22532             Select(Tmp4, N10);
22533             Select(Chain11, Chain11);
22534             ResNode = CurDAG->getTargetNode(X86::XOR16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22535             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22536             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22537             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22538             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22539             Result = SDOperand(ResNode, N.ResNo);
22540             return;
22541           }
22542 
22543           // Pattern: (store:void (xor:i32 R32:i32:$src, (load:i32 addr:i32:$dst)), addr:i32:$dst)
22544           // Emits: (XOR32mr:void addr:i32:$dst, R32:i32:$src)
22545           // Pattern complexity = 26  cost = 1
22546           if (N1.Val->getValueType(0) == MVT::i32 &&
22547               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22548             Select(Tmp0, Tmp0);
22549             Select(Tmp1, Tmp1);
22550             Select(Tmp2, Tmp2);
22551             Select(Tmp3, Tmp3);
22552             Select(Tmp4, N10);
22553             Select(Chain11, Chain11);
22554             ResNode = CurDAG->getTargetNode(X86::XOR32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22555             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22556             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22557             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22558             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22559             Result = SDOperand(ResNode, N.ResNo);
22560             return;
22561           }
22562         }
22563       }
22564     }
22565     if (N1.getOpcode() == ISD::ADD &&
22566         N1.hasOneUse() &&
22567         !CodeGenMap.count(N1.getValue(0))) {
22568       N10 = N1.getOperand(0);
22569       N11 = N1.getOperand(1);
22570       if (N11.getOpcode() == ISD::LOAD &&
22571           N11.hasOneUse() &&
22572           !CodeGenMap.count(N11.getValue(0)) &&
22573           !isNonImmUse(N1.Val, N11.Val) &&
22574           (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
22575         N111 = N11.getOperand(1);
22576         N2 = N.getOperand(2);
22577         if (N111 == N2) {
22578 
22579           // Pattern: (store:void (add:i8 R8:i8:$src2, (load:i8 addr:i32:$dst)), addr:i32:$dst)
22580           // Emits: (ADD8mr:void addr:i32:$dst, R8:i8:$src2)
22581           // Pattern complexity = 26  cost = 1
22582           if (N1.Val->getValueType(0) == MVT::i8 &&
22583               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22584             Select(Tmp0, Tmp0);
22585             Select(Tmp1, Tmp1);
22586             Select(Tmp2, Tmp2);
22587             Select(Tmp3, Tmp3);
22588             Select(Tmp4, N10);
22589             Select(Chain11, Chain11);
22590             ResNode = CurDAG->getTargetNode(X86::ADD8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22591             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22592             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22593             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22594             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22595             Result = SDOperand(ResNode, N.ResNo);
22596             return;
22597           }
22598 
22599           // Pattern: (store:void (add:i16 R16:i16:$src2, (load:i16 addr:i32:$dst)), addr:i32:$dst)
22600           // Emits: (ADD16mr:void addr:i32:$dst, R16:i16:$src2)
22601           // Pattern complexity = 26  cost = 1
22602           if (N1.Val->getValueType(0) == MVT::i16 &&
22603               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22604             Select(Tmp0, Tmp0);
22605             Select(Tmp1, Tmp1);
22606             Select(Tmp2, Tmp2);
22607             Select(Tmp3, Tmp3);
22608             Select(Tmp4, N10);
22609             Select(Chain11, Chain11);
22610             ResNode = CurDAG->getTargetNode(X86::ADD16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22611             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22612             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22613             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22614             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22615             Result = SDOperand(ResNode, N.ResNo);
22616             return;
22617           }
22618 
22619           // Pattern: (store:void (add:i32 R32:i32:$src2, (load:i32 addr:i32:$dst)), addr:i32:$dst)
22620           // Emits: (ADD32mr:void addr:i32:$dst, R32:i32:$src2)
22621           // Pattern complexity = 26  cost = 1
22622           if (N1.Val->getValueType(0) == MVT::i32 &&
22623               SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22624             Select(Tmp0, Tmp0);
22625             Select(Tmp1, Tmp1);
22626             Select(Tmp2, Tmp2);
22627             Select(Tmp3, Tmp3);
22628             Select(Tmp4, N10);
22629             Select(Chain11, Chain11);
22630             ResNode = CurDAG->getTargetNode(X86::ADD32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11);
22631             if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22632             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22633             SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22634             AddHandleReplacement(N11.Val, 1, ResNode, 0);
22635             Result = SDOperand(ResNode, N.ResNo);
22636             return;
22637           }
22638         }
22639       }
22640     }
22641 
22642     // Pattern: (store:void (adde:i32 R32:i32:$src2, (load:i32 addr:i32:$dst)), addr:i32:$dst)
22643     // Emits: (ADC32mr:void addr:i32:$dst, R32:i32:$src2)
22644     // Pattern complexity = 26  cost = 1
22645     if (N1.getOpcode() == ISD::ADDE &&
22646         N1.hasOneUse() &&
22647         !CodeGenMap.count(N1.getValue(0))) {
22648       N10 = N1.getOperand(0);
22649       N11 = N1.getOperand(1);
22650       if (N11.getOpcode() == ISD::LOAD &&
22651           N11.hasOneUse() &&
22652           !CodeGenMap.count(N11.getValue(0)) &&
22653           N1.Val->isOnlyUse(N11.Val) &&
22654           !isNonImmUse(N1.Val, N11.Val) &&
22655           (Chain11 = UpdateFoldedChain(CurDAG, N11.Val, Chain.Val, OldTF)).Val) {
22656         N111 = N11.getOperand(1);
22657         N2 = N.getOperand(2);
22658         if (N111 == N2 &&
22659             N1.Val->getValueType(0) == MVT::i32 &&
22660             SelectAddr(N111, Tmp0, Tmp1, Tmp2, Tmp3)) {
22661           Select(Tmp0, Tmp0);
22662           Select(Tmp1, Tmp1);
22663           Select(Tmp2, Tmp2);
22664           Select(Tmp3, Tmp3);
22665           Select(Tmp4, N10);
22666           Select(Chain11, Chain11);
22667           Select(InFlag, N1.getOperand(2));
22668           ResNode = CurDAG->getTargetNode(X86::ADC32mr, MVT::Other, MVT::Flag, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain11, InFlag);
22669           if (OldTF) SelectionDAG::InsertISelMapEntry(CodeGenMap, OldTF, 0, Chain11.Val, 0);
22670           InFlag = SDOperand(ResNode, 1);
22671           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22672           SelectionDAG::InsertISelMapEntry(CodeGenMap, N11.Val, 1, ResNode, 0);
22673           AddHandleReplacement(N11.Val, 1, ResNode, 0);
22674           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22675           Result = SDOperand(ResNode, N.ResNo);
22676           return;
22677         }
22678       }
22679     }
22680   }
22681 
22682   // Pattern: (store:void (vector_extract:f64 (vector_shuffle:v2f64 VR128:v2f64:$src, (undef:v2f64), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>), 0:i32), addr:i32:$dst)
22683   // Emits: (MOVHPDmr:void addr:i32:$dst, VR128:v2f64:$src)
22684   // Pattern complexity = 24  cost = 1
22685   if ((Subtarget->hasSSE2())) {
22686     Chain = N.getOperand(0);
22687     N1 = N.getOperand(1);
22688     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22689       N10 = N1.getOperand(0);
22690       if (N10.getOpcode() == ISD::VECTOR_SHUFFLE) {
22691         N100 = N10.getOperand(0);
22692         N101 = N10.getOperand(1);
22693         if (N101.getOpcode() == ISD::UNDEF) {
22694           N102 = N10.getOperand(2);
22695           if (N102.getOpcode() == ISD::BUILD_VECTOR &&
22696               Predicate_UNPCKH_shuffle_mask(N102.Val)) {
22697             N11 = N1.getOperand(1);
22698             if (isa<ConstantSDNode>(N11)) {
22699               int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22700               if (CN0 == 0) {
22701                 N2 = N.getOperand(2);
22702                 if (N1.Val->getValueType(0) == MVT::f64 &&
22703                     N10.Val->getValueType(0) == MVT::v2f64 &&
22704                     N11.Val->getValueType(0) == MVT::i32 &&
22705                     SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22706                   Select(Tmp1, Tmp1);
22707                   Select(Tmp2, Tmp2);
22708                   Select(Tmp3, Tmp3);
22709                   Select(Tmp4, Tmp4);
22710                   Select(Tmp5, N100);
22711                   Select(Chain, Chain);
22712                   ResNode = CurDAG->getTargetNode(X86::MOVHPDmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
22713                   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22714                   Result = SDOperand(ResNode, N.ResNo);
22715                   return;
22716                 }
22717               }
22718             }
22719           }
22720         }
22721       }
22722     }
22723   }
22724 
22725   // Pattern: (store:void (vector_extract:f64 (bitconvert:v2f64 VR128:v4f32:$src), 0:i32), addr:i32:$dst)
22726   // Emits: (MOVLPSmr:void addr:i32:$dst, VR128:v4f32:$src)
22727   // Pattern complexity = 19  cost = 1
22728   if ((Subtarget->hasSSE1())) {
22729     Chain = N.getOperand(0);
22730     N1 = N.getOperand(1);
22731     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22732       N10 = N1.getOperand(0);
22733       if (N10.getOpcode() == ISD::BIT_CONVERT) {
22734         N100 = N10.getOperand(0);
22735         N11 = N1.getOperand(1);
22736         if (isa<ConstantSDNode>(N11)) {
22737           int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
22738           if (CN0 == 0) {
22739             N2 = N.getOperand(2);
22740             if (N1.Val->getValueType(0) == MVT::f64 &&
22741                 N10.Val->getValueType(0) == MVT::v2f64 &&
22742                 N100.Val->getValueType(0) == MVT::v4f32 &&
22743                 N11.Val->getValueType(0) == MVT::i32 &&
22744                 SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22745               Select(Tmp1, Tmp1);
22746               Select(Tmp2, Tmp2);
22747               Select(Tmp3, Tmp3);
22748               Select(Tmp4, Tmp4);
22749               Select(Tmp5, N100);
22750               Select(Chain, Chain);
22751               ResNode = CurDAG->getTargetNode(X86::MOVLPSmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
22752               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22753               Result = SDOperand(ResNode, N.ResNo);
22754               return;
22755             }
22756           }
22757         }
22758       }
22759     }
22760   }
22761   {
22762     Chain = N.getOperand(0);
22763     N1 = N.getOperand(1);
22764     if (N1.getOpcode() == X86ISD::SETCC &&
22765         N1.hasOneUse() &&
22766         !CodeGenMap.count(N1.getValue(0))) {
22767       N10 = N1.getOperand(0);
22768       if (isa<ConstantSDNode>(N10)) {
22769         int64_t CN0 = cast<ConstantSDNode>(N10)->getSignExtended();
22770 
22771         // Pattern: (store:void (X86setcc:i8 4:i8), addr:i32:$dst)
22772         // Emits: (SETEm:void addr:i32:$dst)
22773         // Pattern complexity = 17  cost = 1
22774         if (CN0 == 4) {
22775           N2 = N.getOperand(2);
22776           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22777             Select(Tmp1, Tmp1);
22778             Select(Tmp2, Tmp2);
22779             Select(Tmp3, Tmp3);
22780             Select(Tmp4, Tmp4);
22781             Select(Chain, Chain);
22782             Select(InFlag, N1.getOperand(1));
22783             ResNode = CurDAG->getTargetNode(X86::SETEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22784             InFlag = SDOperand(ResNode, 1);
22785             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22786             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22787             Result = SDOperand(ResNode, N.ResNo);
22788             return;
22789           }
22790         }
22791 
22792         // Pattern: (store:void (X86setcc:i8 9:i8), addr:i32:$dst)
22793         // Emits: (SETNEm:void addr:i32:$dst)
22794         // Pattern complexity = 17  cost = 1
22795         if (CN0 == 9) {
22796           N2 = N.getOperand(2);
22797           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22798             Select(Tmp1, Tmp1);
22799             Select(Tmp2, Tmp2);
22800             Select(Tmp3, Tmp3);
22801             Select(Tmp4, Tmp4);
22802             Select(Chain, Chain);
22803             Select(InFlag, N1.getOperand(1));
22804             ResNode = CurDAG->getTargetNode(X86::SETNEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22805             InFlag = SDOperand(ResNode, 1);
22806             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22807             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22808             Result = SDOperand(ResNode, N.ResNo);
22809             return;
22810           }
22811         }
22812 
22813         // Pattern: (store:void (X86setcc:i8 7:i8), addr:i32:$dst)
22814         // Emits: (SETLm:void addr:i32:$dst)
22815         // Pattern complexity = 17  cost = 1
22816         if (CN0 == 7) {
22817           N2 = N.getOperand(2);
22818           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22819             Select(Tmp1, Tmp1);
22820             Select(Tmp2, Tmp2);
22821             Select(Tmp3, Tmp3);
22822             Select(Tmp4, Tmp4);
22823             Select(Chain, Chain);
22824             Select(InFlag, N1.getOperand(1));
22825             ResNode = CurDAG->getTargetNode(X86::SETLm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22826             InFlag = SDOperand(ResNode, 1);
22827             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22828             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22829             Result = SDOperand(ResNode, N.ResNo);
22830             return;
22831           }
22832         }
22833 
22834         // Pattern: (store:void (X86setcc:i8 6:i8), addr:i32:$dst)
22835         // Emits: (SETGEm:void addr:i32:$dst)
22836         // Pattern complexity = 17  cost = 1
22837         if (CN0 == 6) {
22838           N2 = N.getOperand(2);
22839           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22840             Select(Tmp1, Tmp1);
22841             Select(Tmp2, Tmp2);
22842             Select(Tmp3, Tmp3);
22843             Select(Tmp4, Tmp4);
22844             Select(Chain, Chain);
22845             Select(InFlag, N1.getOperand(1));
22846             ResNode = CurDAG->getTargetNode(X86::SETGEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22847             InFlag = SDOperand(ResNode, 1);
22848             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22849             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22850             Result = SDOperand(ResNode, N.ResNo);
22851             return;
22852           }
22853         }
22854 
22855         // Pattern: (store:void (X86setcc:i8 8:i8), addr:i32:$dst)
22856         // Emits: (SETLEm:void addr:i32:$dst)
22857         // Pattern complexity = 17  cost = 1
22858         if (CN0 == 8) {
22859           N2 = N.getOperand(2);
22860           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22861             Select(Tmp1, Tmp1);
22862             Select(Tmp2, Tmp2);
22863             Select(Tmp3, Tmp3);
22864             Select(Tmp4, Tmp4);
22865             Select(Chain, Chain);
22866             Select(InFlag, N1.getOperand(1));
22867             ResNode = CurDAG->getTargetNode(X86::SETLEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22868             InFlag = SDOperand(ResNode, 1);
22869             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22870             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22871             Result = SDOperand(ResNode, N.ResNo);
22872             return;
22873           }
22874         }
22875 
22876         // Pattern: (store:void (X86setcc:i8 5:i8), addr:i32:$dst)
22877         // Emits: (SETGm:void addr:i32:$dst)
22878         // Pattern complexity = 17  cost = 1
22879         if (CN0 == 5) {
22880           N2 = N.getOperand(2);
22881           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22882             Select(Tmp1, Tmp1);
22883             Select(Tmp2, Tmp2);
22884             Select(Tmp3, Tmp3);
22885             Select(Tmp4, Tmp4);
22886             Select(Chain, Chain);
22887             Select(InFlag, N1.getOperand(1));
22888             ResNode = CurDAG->getTargetNode(X86::SETGm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22889             InFlag = SDOperand(ResNode, 1);
22890             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22891             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22892             Result = SDOperand(ResNode, N.ResNo);
22893             return;
22894           }
22895         }
22896 
22897         // Pattern: (store:void (X86setcc:i8 2:i8), addr:i32:$dst)
22898         // Emits: (SETBm:void addr:i32:$dst)
22899         // Pattern complexity = 17  cost = 1
22900         if (CN0 == 2) {
22901           N2 = N.getOperand(2);
22902           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22903             Select(Tmp1, Tmp1);
22904             Select(Tmp2, Tmp2);
22905             Select(Tmp3, Tmp3);
22906             Select(Tmp4, Tmp4);
22907             Select(Chain, Chain);
22908             Select(InFlag, N1.getOperand(1));
22909             ResNode = CurDAG->getTargetNode(X86::SETBm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22910             InFlag = SDOperand(ResNode, 1);
22911             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22912             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22913             Result = SDOperand(ResNode, N.ResNo);
22914             return;
22915           }
22916         }
22917 
22918         // Pattern: (store:void (X86setcc:i8 1:i8), addr:i32:$dst)
22919         // Emits: (SETAEm:void addr:i32:$dst)
22920         // Pattern complexity = 17  cost = 1
22921         if (CN0 == 1) {
22922           N2 = N.getOperand(2);
22923           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22924             Select(Tmp1, Tmp1);
22925             Select(Tmp2, Tmp2);
22926             Select(Tmp3, Tmp3);
22927             Select(Tmp4, Tmp4);
22928             Select(Chain, Chain);
22929             Select(InFlag, N1.getOperand(1));
22930             ResNode = CurDAG->getTargetNode(X86::SETAEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22931             InFlag = SDOperand(ResNode, 1);
22932             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22933             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22934             Result = SDOperand(ResNode, N.ResNo);
22935             return;
22936           }
22937         }
22938 
22939         // Pattern: (store:void (X86setcc:i8 3:i8), addr:i32:$dst)
22940         // Emits: (SETBEm:void addr:i32:$dst)
22941         // Pattern complexity = 17  cost = 1
22942         if (CN0 == 3) {
22943           N2 = N.getOperand(2);
22944           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22945             Select(Tmp1, Tmp1);
22946             Select(Tmp2, Tmp2);
22947             Select(Tmp3, Tmp3);
22948             Select(Tmp4, Tmp4);
22949             Select(Chain, Chain);
22950             Select(InFlag, N1.getOperand(1));
22951             ResNode = CurDAG->getTargetNode(X86::SETBEm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22952             InFlag = SDOperand(ResNode, 1);
22953             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22954             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22955             Result = SDOperand(ResNode, N.ResNo);
22956             return;
22957           }
22958         }
22959 
22960         // Pattern: (store:void (X86setcc:i8 0:i8), addr:i32:$dst)
22961         // Emits: (SETAm:void addr:i32:$dst)
22962         // Pattern complexity = 17  cost = 1
22963         if (CN0 == 0) {
22964           N2 = N.getOperand(2);
22965           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22966             Select(Tmp1, Tmp1);
22967             Select(Tmp2, Tmp2);
22968             Select(Tmp3, Tmp3);
22969             Select(Tmp4, Tmp4);
22970             Select(Chain, Chain);
22971             Select(InFlag, N1.getOperand(1));
22972             ResNode = CurDAG->getTargetNode(X86::SETAm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22973             InFlag = SDOperand(ResNode, 1);
22974             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22975             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22976             Result = SDOperand(ResNode, N.ResNo);
22977             return;
22978           }
22979         }
22980 
22981         // Pattern: (store:void (X86setcc:i8 15:i8), addr:i32:$dst)
22982         // Emits: (SETSm:void addr:i32:$dst)
22983         // Pattern complexity = 17  cost = 1
22984         if (CN0 == 15) {
22985           N2 = N.getOperand(2);
22986           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
22987             Select(Tmp1, Tmp1);
22988             Select(Tmp2, Tmp2);
22989             Select(Tmp3, Tmp3);
22990             Select(Tmp4, Tmp4);
22991             Select(Chain, Chain);
22992             Select(InFlag, N1.getOperand(1));
22993             ResNode = CurDAG->getTargetNode(X86::SETSm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
22994             InFlag = SDOperand(ResNode, 1);
22995             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
22996             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
22997             Result = SDOperand(ResNode, N.ResNo);
22998             return;
22999           }
23000         }
23001 
23002         // Pattern: (store:void (X86setcc:i8 12:i8), addr:i32:$dst)
23003         // Emits: (SETNSm:void addr:i32:$dst)
23004         // Pattern complexity = 17  cost = 1
23005         if (CN0 == 12) {
23006           N2 = N.getOperand(2);
23007           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23008             Select(Tmp1, Tmp1);
23009             Select(Tmp2, Tmp2);
23010             Select(Tmp3, Tmp3);
23011             Select(Tmp4, Tmp4);
23012             Select(Chain, Chain);
23013             Select(InFlag, N1.getOperand(1));
23014             ResNode = CurDAG->getTargetNode(X86::SETNSm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
23015             InFlag = SDOperand(ResNode, 1);
23016             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23017             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
23018             Result = SDOperand(ResNode, N.ResNo);
23019             return;
23020           }
23021         }
23022 
23023         // Pattern: (store:void (X86setcc:i8 14:i8), addr:i32:$dst)
23024         // Emits: (SETPm:void addr:i32:$dst)
23025         // Pattern complexity = 17  cost = 1
23026         if (CN0 == 14) {
23027           N2 = N.getOperand(2);
23028           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23029             Select(Tmp1, Tmp1);
23030             Select(Tmp2, Tmp2);
23031             Select(Tmp3, Tmp3);
23032             Select(Tmp4, Tmp4);
23033             Select(Chain, Chain);
23034             Select(InFlag, N1.getOperand(1));
23035             ResNode = CurDAG->getTargetNode(X86::SETPm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
23036             InFlag = SDOperand(ResNode, 1);
23037             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23038             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
23039             Result = SDOperand(ResNode, N.ResNo);
23040             return;
23041           }
23042         }
23043 
23044         // Pattern: (store:void (X86setcc:i8 11:i8), addr:i32:$dst)
23045         // Emits: (SETNPm:void addr:i32:$dst)
23046         // Pattern complexity = 17  cost = 1
23047         if (CN0 == 11) {
23048           N2 = N.getOperand(2);
23049           if (SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23050             Select(Tmp1, Tmp1);
23051             Select(Tmp2, Tmp2);
23052             Select(Tmp3, Tmp3);
23053             Select(Tmp4, Tmp4);
23054             Select(Chain, Chain);
23055             Select(InFlag, N1.getOperand(1));
23056             ResNode = CurDAG->getTargetNode(X86::SETNPm, MVT::Other, MVT::Flag, Tmp1, Tmp2, Tmp3, Tmp4, Chain, InFlag);
23057             InFlag = SDOperand(ResNode, 1);
23058             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23059             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
23060             Result = SDOperand(ResNode, N.ResNo);
23061             return;
23062           }
23063         }
23064       }
23065     }
23066   }
23067 
23068   // Pattern: (store:void (vector_extract:f64 VR128:v2f64:$src, 0:i32), addr:i32:$dst)
23069   // Emits: (MOVLPDmr:void addr:i32:$dst, VR128:v2f64:$src)
23070   // Pattern complexity = 17  cost = 1
23071   if ((Subtarget->hasSSE2())) {
23072     Chain = N.getOperand(0);
23073     N1 = N.getOperand(1);
23074     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
23075       N10 = N1.getOperand(0);
23076       N11 = N1.getOperand(1);
23077       if (isa<ConstantSDNode>(N11)) {
23078         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
23079         if (CN0 == 0) {
23080           N2 = N.getOperand(2);
23081           if (N1.Val->getValueType(0) == MVT::f64 &&
23082               N10.Val->getValueType(0) == MVT::v2f64 &&
23083               N11.Val->getValueType(0) == MVT::i32 &&
23084               SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23085             Select(Tmp1, Tmp1);
23086             Select(Tmp2, Tmp2);
23087             Select(Tmp3, Tmp3);
23088             Select(Tmp4, Tmp4);
23089             Select(Tmp5, N10);
23090             Select(Chain, Chain);
23091             ResNode = CurDAG->getTargetNode(X86::MOVLPDmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
23092             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23093             Result = SDOperand(ResNode, N.ResNo);
23094             return;
23095           }
23096         }
23097       }
23098     }
23099   }
23100 
23101   // Pattern: (store:void (vector_extract:f32 VR128:v4f32:$src, 0:i32), addr:i32:$dst)
23102   // Emits: (MOVPS2SSmr:void addr:i32:$dst, VR128:v4f32:$src)
23103   // Pattern complexity = 17  cost = 1
23104   if ((Subtarget->hasSSE1())) {
23105     Chain = N.getOperand(0);
23106     N1 = N.getOperand(1);
23107     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
23108       N10 = N1.getOperand(0);
23109       N11 = N1.getOperand(1);
23110       if (isa<ConstantSDNode>(N11)) {
23111         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
23112         if (CN0 == 0) {
23113           N2 = N.getOperand(2);
23114           if (N1.Val->getValueType(0) == MVT::f32 &&
23115               N10.Val->getValueType(0) == MVT::v4f32 &&
23116               N11.Val->getValueType(0) == MVT::i32 &&
23117               SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23118             Select(Tmp1, Tmp1);
23119             Select(Tmp2, Tmp2);
23120             Select(Tmp3, Tmp3);
23121             Select(Tmp4, Tmp4);
23122             Select(Tmp5, N10);
23123             Select(Chain, Chain);
23124             ResNode = CurDAG->getTargetNode(X86::MOVPS2SSmr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
23125             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23126             Result = SDOperand(ResNode, N.ResNo);
23127             return;
23128           }
23129         }
23130       }
23131     }
23132   }
23133 
23134   // Pattern: (store:void (vector_extract:i32 VR128:v4i32:$src, 0:i32), addr:i32:$dst)
23135   // Emits: (MOVPDI2DImr:void addr:i32:$dst, VR128:v4i32:$src)
23136   // Pattern complexity = 17  cost = 1
23137   if ((Subtarget->hasSSE2())) {
23138     Chain = N.getOperand(0);
23139     N1 = N.getOperand(1);
23140     if (N1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
23141       N10 = N1.getOperand(0);
23142       N11 = N1.getOperand(1);
23143       if (isa<ConstantSDNode>(N11)) {
23144         int64_t CN0 = cast<ConstantSDNode>(N11)->getSignExtended();
23145         if (CN0 == 0) {
23146           N2 = N.getOperand(2);
23147           if (N1.Val->getValueType(0) == MVT::i32 &&
23148               N10.Val->getValueType(0) == MVT::v4i32 &&
23149               N11.Val->getValueType(0) == MVT::i32 &&
23150               SelectAddr(N2, Tmp1, Tmp2, Tmp3, Tmp4)) {
23151             Select(Tmp1, Tmp1);
23152             Select(Tmp2, Tmp2);
23153             Select(Tmp3, Tmp3);
23154             Select(Tmp4, Tmp4);
23155             Select(Tmp5, N10);
23156             Select(Chain, Chain);
23157             ResNode = CurDAG->getTargetNode(X86::MOVPDI2DImr, MVT::Other, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Chain);
23158             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23159             Result = SDOperand(ResNode, N.ResNo);
23160             return;
23161           }
23162         }
23163       }
23164     }
23165   }
23166   {
23167     Chain = N.getOperand(0);
23168     N1 = N.getOperand(1);
23169     if (N1.getOpcode() == X86ISD::Wrapper) {
23170       N10 = N1.getOperand(0);
23171 
23172       // Pattern: (store:void (X86Wrapper:i32 (tconstpool:i32):$src), addr:i32:$dst)
23173       // Emits: (MOV32mi:void addr:i32:$dst, (tconstpool:i32):$src)
23174       // Pattern complexity = 16  cost = 1
23175       if (N10.getOpcode() == ISD::TargetConstantPool) {
23176         N2 = N.getOperand(2);
23177         if (SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23178           Select(Tmp0, Tmp0);
23179           Select(Tmp1, Tmp1);
23180           Select(Tmp2, Tmp2);
23181           Select(Tmp3, Tmp3);
23182           Tmp4 = N10;
23183           Select(Chain, Chain);
23184           ResNode = CurDAG->getTargetNode(X86::MOV32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23185           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23186           Result = SDOperand(ResNode, N.ResNo);
23187           return;
23188         }
23189       }
23190 
23191       // Pattern: (store:void (X86Wrapper:i32 (tglobaladdr:i32):$src), addr:i32:$dst)
23192       // Emits: (MOV32mi:void addr:i32:$dst, (tglobaladdr:i32):$src)
23193       // Pattern complexity = 16  cost = 1
23194       if (N10.getOpcode() == ISD::TargetGlobalAddress) {
23195         N2 = N.getOperand(2);
23196         if (SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23197           Select(Tmp0, Tmp0);
23198           Select(Tmp1, Tmp1);
23199           Select(Tmp2, Tmp2);
23200           Select(Tmp3, Tmp3);
23201           Tmp4 = N10;
23202           Select(Chain, Chain);
23203           ResNode = CurDAG->getTargetNode(X86::MOV32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23204           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23205           Result = SDOperand(ResNode, N.ResNo);
23206           return;
23207         }
23208       }
23209 
23210       // Pattern: (store:void (X86Wrapper:i32 (texternalsym:i32):$src), addr:i32:$dst)
23211       // Emits: (MOV32mi:void addr:i32:$dst, (texternalsym:i32):$src)
23212       // Pattern complexity = 16  cost = 1
23213       if (N10.getOpcode() == ISD::TargetExternalSymbol) {
23214         N2 = N.getOperand(2);
23215         if (SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23216           Select(Tmp0, Tmp0);
23217           Select(Tmp1, Tmp1);
23218           Select(Tmp2, Tmp2);
23219           Select(Tmp3, Tmp3);
23220           Tmp4 = N10;
23221           Select(Chain, Chain);
23222           ResNode = CurDAG->getTargetNode(X86::MOV32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23223           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23224           Result = SDOperand(ResNode, N.ResNo);
23225           return;
23226         }
23227       }
23228     }
23229     if (N1.getOpcode() == ISD::Constant) {
23230       N2 = N.getOperand(2);
23231 
23232       // Pattern: (store:void (imm:i8):$src, addr:i32:$dst)
23233       // Emits: (MOV8mi:void addr:i32:$dst, (imm:i8):$src)
23234       // Pattern complexity = 14  cost = 1
23235       if (N1.Val->getValueType(0) == MVT::i8 &&
23236           SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23237         Select(Tmp0, Tmp0);
23238         Select(Tmp1, Tmp1);
23239         Select(Tmp2, Tmp2);
23240         Select(Tmp3, Tmp3);
23241         unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
23242         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
23243         Select(Chain, Chain);
23244         ResNode = CurDAG->getTargetNode(X86::MOV8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23245         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23246         Result = SDOperand(ResNode, N.ResNo);
23247         return;
23248       }
23249 
23250       // Pattern: (store:void (imm:i16):$src, addr:i32:$dst)
23251       // Emits: (MOV16mi:void addr:i32:$dst, (imm:i16):$src)
23252       // Pattern complexity = 14  cost = 1
23253       if (N1.Val->getValueType(0) == MVT::i16 &&
23254           SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23255         Select(Tmp0, Tmp0);
23256         Select(Tmp1, Tmp1);
23257         Select(Tmp2, Tmp2);
23258         Select(Tmp3, Tmp3);
23259         unsigned short Tmp4C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
23260         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i16);
23261         Select(Chain, Chain);
23262         ResNode = CurDAG->getTargetNode(X86::MOV16mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23263         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23264         Result = SDOperand(ResNode, N.ResNo);
23265         return;
23266       }
23267 
23268       // Pattern: (store:void (imm:i32):$src, addr:i32:$dst)
23269       // Emits: (MOV32mi:void addr:i32:$dst, (imm:i32):$src)
23270       // Pattern complexity = 14  cost = 1
23271       if (N1.Val->getValueType(0) == MVT::i32 &&
23272           SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23273         Select(Tmp0, Tmp0);
23274         Select(Tmp1, Tmp1);
23275         Select(Tmp2, Tmp2);
23276         Select(Tmp3, Tmp3);
23277         unsigned Tmp4C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
23278         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i32);
23279         Select(Chain, Chain);
23280         ResNode = CurDAG->getTargetNode(X86::MOV32mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23281         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23282         Result = SDOperand(ResNode, N.ResNo);
23283         return;
23284       }
23285     }
23286     N2 = N.getOperand(2);
23287 
23288     // Pattern: (store:void R8:i8:$src, addr:i32:$dst)
23289     // Emits: (MOV8mr:void addr:i32:$dst, R8:i8:$src)
23290     // Pattern complexity = 12  cost = 1
23291     if (N1.Val->getValueType(0) == MVT::i8 &&
23292         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23293       Select(Tmp0, Tmp0);
23294       Select(Tmp1, Tmp1);
23295       Select(Tmp2, Tmp2);
23296       Select(Tmp3, Tmp3);
23297       Select(Tmp4, N1);
23298       Select(Chain, Chain);
23299       ResNode = CurDAG->getTargetNode(X86::MOV8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23300       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23301       Result = SDOperand(ResNode, N.ResNo);
23302       return;
23303     }
23304 
23305     // Pattern: (store:void R16:i16:$src, addr:i32:$dst)
23306     // Emits: (MOV16mr:void addr:i32:$dst, R16:i16:$src)
23307     // Pattern complexity = 12  cost = 1
23308     if (N1.Val->getValueType(0) == MVT::i16 &&
23309         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23310       Select(Tmp0, Tmp0);
23311       Select(Tmp1, Tmp1);
23312       Select(Tmp2, Tmp2);
23313       Select(Tmp3, Tmp3);
23314       Select(Tmp4, N1);
23315       Select(Chain, Chain);
23316       ResNode = CurDAG->getTargetNode(X86::MOV16mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23317       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23318       Result = SDOperand(ResNode, N.ResNo);
23319       return;
23320     }
23321 
23322     // Pattern: (store:void R32:i32:$src, addr:i32:$dst)
23323     // Emits: (MOV32mr:void addr:i32:$dst, R32:i32:$src)
23324     // Pattern complexity = 12  cost = 1
23325     if (N1.Val->getValueType(0) == MVT::i32 &&
23326         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23327       Select(Tmp0, Tmp0);
23328       Select(Tmp1, Tmp1);
23329       Select(Tmp2, Tmp2);
23330       Select(Tmp3, Tmp3);
23331       Select(Tmp4, N1);
23332       Select(Chain, Chain);
23333       ResNode = CurDAG->getTargetNode(X86::MOV32mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23334       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23335       Result = SDOperand(ResNode, N.ResNo);
23336       return;
23337     }
23338   }
23339 
23340   // Pattern: (store:void RFP:f64:$src, addr:i32:$op)
23341   // Emits: (FpST64m:void addr:i32:$op, RFP:f64:$src)
23342   // Pattern complexity = 12  cost = 1
23343   if ((!Subtarget->hasSSE2())) {
23344     Chain = N.getOperand(0);
23345     N1 = N.getOperand(1);
23346     N2 = N.getOperand(2);
23347     if (N1.Val->getValueType(0) == MVT::f64 &&
23348         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23349       Select(Tmp0, Tmp0);
23350       Select(Tmp1, Tmp1);
23351       Select(Tmp2, Tmp2);
23352       Select(Tmp3, Tmp3);
23353       Select(Tmp4, N1);
23354       Select(Chain, Chain);
23355       ResNode = CurDAG->getTargetNode(X86::FpST64m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23356       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23357       Result = SDOperand(ResNode, N.ResNo);
23358       return;
23359     }
23360   }
23361 
23362   // Pattern: (store:void FR32:f32:$src, addr:i32:$dst)
23363   // Emits: (MOVSSmr:void addr:i32:$dst, FR32:f32:$src)
23364   // Pattern complexity = 12  cost = 1
23365   if ((Subtarget->hasSSE1())) {
23366     Chain = N.getOperand(0);
23367     N1 = N.getOperand(1);
23368     N2 = N.getOperand(2);
23369     if (N1.Val->getValueType(0) == MVT::f32 &&
23370         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23371       Select(Tmp0, Tmp0);
23372       Select(Tmp1, Tmp1);
23373       Select(Tmp2, Tmp2);
23374       Select(Tmp3, Tmp3);
23375       Select(Tmp4, N1);
23376       Select(Chain, Chain);
23377       ResNode = CurDAG->getTargetNode(X86::MOVSSmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23378       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23379       Result = SDOperand(ResNode, N.ResNo);
23380       return;
23381     }
23382   }
23383 
23384   // Pattern: (store:void FR64:f64:$src, addr:i32:$dst)
23385   // Emits: (MOVSDmr:void addr:i32:$dst, FR64:f64:$src)
23386   // Pattern complexity = 12  cost = 1
23387   if ((Subtarget->hasSSE2())) {
23388     Chain = N.getOperand(0);
23389     N1 = N.getOperand(1);
23390     N2 = N.getOperand(2);
23391     if (N1.Val->getValueType(0) == MVT::f64 &&
23392         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23393       Select(Tmp0, Tmp0);
23394       Select(Tmp1, Tmp1);
23395       Select(Tmp2, Tmp2);
23396       Select(Tmp3, Tmp3);
23397       Select(Tmp4, N1);
23398       Select(Chain, Chain);
23399       ResNode = CurDAG->getTargetNode(X86::MOVSDmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23400       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23401       Result = SDOperand(ResNode, N.ResNo);
23402       return;
23403     }
23404   }
23405 
23406   // Pattern: (store:void VR128:v4f32:$src, addr:i32:$dst)
23407   // Emits: (MOVAPSmr:void addr:i32:$dst, VR128:v4f32:$src)
23408   // Pattern complexity = 12  cost = 1
23409   if ((Subtarget->hasSSE1())) {
23410     Chain = N.getOperand(0);
23411     N1 = N.getOperand(1);
23412     N2 = N.getOperand(2);
23413     if (N1.Val->getValueType(0) == MVT::v4f32 &&
23414         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23415       Select(Tmp0, Tmp0);
23416       Select(Tmp1, Tmp1);
23417       Select(Tmp2, Tmp2);
23418       Select(Tmp3, Tmp3);
23419       Select(Tmp4, N1);
23420       Select(Chain, Chain);
23421       ResNode = CurDAG->getTargetNode(X86::MOVAPSmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23422       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23423       Result = SDOperand(ResNode, N.ResNo);
23424       return;
23425     }
23426   }
23427   if ((Subtarget->hasSSE2())) {
23428     Chain = N.getOperand(0);
23429     N1 = N.getOperand(1);
23430     N2 = N.getOperand(2);
23431 
23432     // Pattern: (store:void VR128:v2f64:$src, addr:i32:$dst)
23433     // Emits: (MOVAPDmr:void addr:i32:$dst, VR128:v2f64:$src)
23434     // Pattern complexity = 12  cost = 1
23435     if (N1.Val->getValueType(0) == MVT::v2f64 &&
23436         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23437       Select(Tmp0, Tmp0);
23438       Select(Tmp1, Tmp1);
23439       Select(Tmp2, Tmp2);
23440       Select(Tmp3, Tmp3);
23441       Select(Tmp4, N1);
23442       Select(Chain, Chain);
23443       ResNode = CurDAG->getTargetNode(X86::MOVAPDmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23444       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23445       Result = SDOperand(ResNode, N.ResNo);
23446       return;
23447     }
23448 
23449     // Pattern: (store:void VR128:v2i64:$src, addr:i32:$dst)
23450     // Emits: (MOVDQAmr:void addr:i32:$dst, VR128:v2i64:$src)
23451     // Pattern complexity = 12  cost = 1
23452     if (N1.Val->getValueType(0) == MVT::v2i64 &&
23453         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23454       Select(Tmp0, Tmp0);
23455       Select(Tmp1, Tmp1);
23456       Select(Tmp2, Tmp2);
23457       Select(Tmp3, Tmp3);
23458       Select(Tmp4, N1);
23459       Select(Chain, Chain);
23460       ResNode = CurDAG->getTargetNode(X86::MOVDQAmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23461       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23462       Result = SDOperand(ResNode, N.ResNo);
23463       return;
23464     }
23465 
23466     // Pattern: (store:void VR128:v16i8:$src, addr:i32:$dst)
23467     // Emits: (MOVDQAmr:void addr:i32:$dst, VR128:v16i8:$src)
23468     // Pattern complexity = 12  cost = 1
23469     if (N1.Val->getValueType(0) == MVT::v16i8 &&
23470         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23471       Select(Tmp0, Tmp0);
23472       Select(Tmp1, Tmp1);
23473       Select(Tmp2, Tmp2);
23474       Select(Tmp3, Tmp3);
23475       Select(Tmp4, N1);
23476       Select(Chain, Chain);
23477       ResNode = CurDAG->getTargetNode(X86::MOVDQAmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23478       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23479       Result = SDOperand(ResNode, N.ResNo);
23480       return;
23481     }
23482 
23483     // Pattern: (store:void VR128:v8i16:$src, addr:i32:$dst)
23484     // Emits: (MOVDQAmr:void addr:i32:$dst, VR128:v16i8:$src)
23485     // Pattern complexity = 12  cost = 1
23486     if (N1.Val->getValueType(0) == MVT::v8i16 &&
23487         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23488       Select(Tmp0, Tmp0);
23489       Select(Tmp1, Tmp1);
23490       Select(Tmp2, Tmp2);
23491       Select(Tmp3, Tmp3);
23492       Select(Tmp4, N1);
23493       Select(Chain, Chain);
23494       ResNode = CurDAG->getTargetNode(X86::MOVDQAmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23495       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23496       Result = SDOperand(ResNode, N.ResNo);
23497       return;
23498     }
23499 
23500     // Pattern: (store:void VR128:v4i32:$src, addr:i32:$dst)
23501     // Emits: (MOVDQAmr:void addr:i32:$dst, VR128:v16i8:$src)
23502     // Pattern complexity = 12  cost = 1
23503     if (N1.Val->getValueType(0) == MVT::v4i32 &&
23504         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
23505       Select(Tmp0, Tmp0);
23506       Select(Tmp1, Tmp1);
23507       Select(Tmp2, Tmp2);
23508       Select(Tmp3, Tmp3);
23509       Select(Tmp4, N1);
23510       Select(Chain, Chain);
23511       ResNode = CurDAG->getTargetNode(X86::MOVDQAmr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
23512       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23513       Result = SDOperand(ResNode, N.ResNo);
23514       return;
23515     }
23516   }
23517   std::cerr << "Cannot yet select: ";
23518   N.Val->dump(CurDAG);
23519   std::cerr << '\n';
23520   abort();
23521 }
23522 
23523 void Select_sub(SDOperand &Result, SDOperand N) {
23524   SDOperand Chain1(0, 0);
23525   SDOperand N0(0, 0);
23526   SDOperand N1(0, 0);
23527   SDOperand N11(0, 0);
23528   SDOperand Tmp0(0, 0);
23529   SDOperand Tmp1(0, 0);
23530   SDOperand Tmp2(0, 0);
23531   SDOperand Tmp3(0, 0);
23532   SDOperand Tmp4(0, 0);
23533   SDNode *ResNode;
23534   {
23535     N0 = N.getOperand(0);
23536     N1 = N.getOperand(1);
23537     if (N1.getOpcode() == ISD::LOAD &&
23538         N1.hasOneUse() &&
23539         !CodeGenMap.count(N1.getValue(0)) &&
23540         !isNonImmUse(N.Val, N1.Val)) {
23541       Chain1 = N1.getOperand(0);
23542       N11 = N1.getOperand(1);
23543 
23544       // Pattern: (sub:i8 R8:i8:$src1, (load:i8 addr:i32:$src2))
23545       // Emits: (SUB8rm:i8 R8:i8:$src1, addr:i32:$src2)
23546       // Pattern complexity = 14  cost = 1
23547       if (N.Val->getValueType(0) == MVT::i8 &&
23548           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23549         Select(Tmp0, Tmp0);
23550         Select(Tmp1, Tmp1);
23551         Select(Tmp2, Tmp2);
23552         Select(Tmp3, Tmp3);
23553         Select(Tmp4, N0);
23554         Select(Chain1, Chain1);
23555         ResNode = CurDAG->getTargetNode(X86::SUB8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23556         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23557         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23558         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23559         Result = SDOperand(ResNode, N.ResNo);
23560         return;
23561       }
23562 
23563       // Pattern: (sub:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
23564       // Emits: (SUB16rm:i16 R16:i16:$src1, addr:i32:$src2)
23565       // Pattern complexity = 14  cost = 1
23566       if (N.Val->getValueType(0) == MVT::i16 &&
23567           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23568         Select(Tmp0, Tmp0);
23569         Select(Tmp1, Tmp1);
23570         Select(Tmp2, Tmp2);
23571         Select(Tmp3, Tmp3);
23572         Select(Tmp4, N0);
23573         Select(Chain1, Chain1);
23574         ResNode = CurDAG->getTargetNode(X86::SUB16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23575         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23576         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23577         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23578         Result = SDOperand(ResNode, N.ResNo);
23579         return;
23580       }
23581 
23582       // Pattern: (sub:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
23583       // Emits: (SUB32rm:i32 R32:i32:$src1, addr:i32:$src2)
23584       // Pattern complexity = 14  cost = 1
23585       if (N.Val->getValueType(0) == MVT::i32 &&
23586           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23587         Select(Tmp0, Tmp0);
23588         Select(Tmp1, Tmp1);
23589         Select(Tmp2, Tmp2);
23590         Select(Tmp3, Tmp3);
23591         Select(Tmp4, N0);
23592         Select(Chain1, Chain1);
23593         ResNode = CurDAG->getTargetNode(X86::SUB32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23594         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23595         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23596         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23597         Result = SDOperand(ResNode, N.ResNo);
23598         return;
23599       }
23600     }
23601   }
23602   if ((Subtarget->hasSSE2())) {
23603     N0 = N.getOperand(0);
23604     N1 = N.getOperand(1);
23605     if (N1.getOpcode() == ISD::LOAD &&
23606         N1.hasOneUse() &&
23607         !CodeGenMap.count(N1.getValue(0)) &&
23608         !isNonImmUse(N.Val, N1.Val)) {
23609       Chain1 = N1.getOperand(0);
23610       N11 = N1.getOperand(1);
23611 
23612       // Pattern: (sub:v16i8 VR128:v16i8:$src1, (load:v16i8 addr:i32:$src2))
23613       // Emits: (PSUBBrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
23614       // Pattern complexity = 14  cost = 1
23615       if (N.Val->getValueType(0) == MVT::v16i8 &&
23616           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23617         Select(Tmp0, Tmp0);
23618         Select(Tmp1, Tmp1);
23619         Select(Tmp2, Tmp2);
23620         Select(Tmp3, Tmp3);
23621         Select(Tmp4, N0);
23622         Select(Chain1, Chain1);
23623         ResNode = CurDAG->getTargetNode(X86::PSUBBrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23624         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23625         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23626         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23627         Result = SDOperand(ResNode, N.ResNo);
23628         return;
23629       }
23630 
23631       // Pattern: (sub:v8i16 VR128:v8i16:$src1, (load:v8i16 addr:i32:$src2))
23632       // Emits: (PSUBWrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
23633       // Pattern complexity = 14  cost = 1
23634       if (N.Val->getValueType(0) == MVT::v8i16 &&
23635           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23636         Select(Tmp0, Tmp0);
23637         Select(Tmp1, Tmp1);
23638         Select(Tmp2, Tmp2);
23639         Select(Tmp3, Tmp3);
23640         Select(Tmp4, N0);
23641         Select(Chain1, Chain1);
23642         ResNode = CurDAG->getTargetNode(X86::PSUBWrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23643         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23644         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23645         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23646         Result = SDOperand(ResNode, N.ResNo);
23647         return;
23648       }
23649 
23650       // Pattern: (sub:v4i32 VR128:v4i32:$src1, (load:v4i32 addr:i32:$src2))
23651       // Emits: (PSUBDrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
23652       // Pattern complexity = 14  cost = 1
23653       if (N.Val->getValueType(0) == MVT::v4i32 &&
23654           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23655         Select(Tmp0, Tmp0);
23656         Select(Tmp1, Tmp1);
23657         Select(Tmp2, Tmp2);
23658         Select(Tmp3, Tmp3);
23659         Select(Tmp4, N0);
23660         Select(Chain1, Chain1);
23661         ResNode = CurDAG->getTargetNode(X86::PSUBDrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23662         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23663         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23664         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23665         Result = SDOperand(ResNode, N.ResNo);
23666         return;
23667       }
23668 
23669       // Pattern: (sub:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2))
23670       // Emits: (PSUBQrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
23671       // Pattern complexity = 14  cost = 1
23672       if (N.Val->getValueType(0) == MVT::v2i64 &&
23673           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23674         Select(Tmp0, Tmp0);
23675         Select(Tmp1, Tmp1);
23676         Select(Tmp2, Tmp2);
23677         Select(Tmp3, Tmp3);
23678         Select(Tmp4, N0);
23679         Select(Chain1, Chain1);
23680         ResNode = CurDAG->getTargetNode(X86::PSUBQrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23681         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23682         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23683         AddHandleReplacement(N1.Val, 1, ResNode, 1);
23684         Result = SDOperand(ResNode, N.ResNo);
23685         return;
23686       }
23687     }
23688   }
23689   {
23690     N0 = N.getOperand(0);
23691     if (isa<ConstantSDNode>(N0)) {
23692       int64_t CN0 = cast<ConstantSDNode>(N0)->getSignExtended();
23693       if (CN0 == 0) {
23694         N1 = N.getOperand(1);
23695 
23696         // Pattern: (sub:i8 0:i8, R8:i8:$src)
23697         // Emits: (NEG8r:i8 R8:i8:$src)
23698         // Pattern complexity = 5  cost = 1
23699         if (N.Val->getValueType(0) == MVT::i8) {
23700           Select(Tmp1, N1);
23701           if (N.Val->hasOneUse()) {
23702             Result = CurDAG->SelectNodeTo(N.Val, X86::NEG8r, MVT::i8, Tmp1);
23703           } else {
23704             ResNode = CurDAG->getTargetNode(X86::NEG8r, MVT::i8, Tmp1);
23705             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23706             Result = SDOperand(ResNode, 0);
23707           }
23708           return;
23709         }
23710 
23711         // Pattern: (sub:i16 0:i16, R16:i16:$src)
23712         // Emits: (NEG16r:i16 R16:i16:$src)
23713         // Pattern complexity = 5  cost = 1
23714         if (N.Val->getValueType(0) == MVT::i16) {
23715           Select(Tmp1, N1);
23716           if (N.Val->hasOneUse()) {
23717             Result = CurDAG->SelectNodeTo(N.Val, X86::NEG16r, MVT::i16, Tmp1);
23718           } else {
23719             ResNode = CurDAG->getTargetNode(X86::NEG16r, MVT::i16, Tmp1);
23720             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23721             Result = SDOperand(ResNode, 0);
23722           }
23723           return;
23724         }
23725 
23726         // Pattern: (sub:i32 0:i32, R32:i32:$src)
23727         // Emits: (NEG32r:i32 R32:i32:$src)
23728         // Pattern complexity = 5  cost = 1
23729         if (N.Val->getValueType(0) == MVT::i32) {
23730           Select(Tmp1, N1);
23731           if (N.Val->hasOneUse()) {
23732             Result = CurDAG->SelectNodeTo(N.Val, X86::NEG32r, MVT::i32, Tmp1);
23733           } else {
23734             ResNode = CurDAG->getTargetNode(X86::NEG32r, MVT::i32, Tmp1);
23735             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23736             Result = SDOperand(ResNode, 0);
23737           }
23738           return;
23739         }
23740       }
23741     }
23742     N1 = N.getOperand(1);
23743     if (N1.getOpcode() == ISD::Constant) {
23744 
23745       // Pattern: (sub:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
23746       // Emits: (SUB16ri8:i16 R16:i16:$src1, (imm:i16):$src2)
23747       // Pattern complexity = 5  cost = 1
23748       if (Predicate_i16immSExt8(N1.Val) &&
23749           N.Val->getValueType(0) == MVT::i16) {
23750         Select(Tmp0, N0);
23751         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
23752         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
23753         if (N.Val->hasOneUse()) {
23754           Result = CurDAG->SelectNodeTo(N.Val, X86::SUB16ri8, MVT::i16, Tmp0, Tmp1);
23755         } else {
23756           ResNode = CurDAG->getTargetNode(X86::SUB16ri8, MVT::i16, Tmp0, Tmp1);
23757           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23758           Result = SDOperand(ResNode, 0);
23759         }
23760         return;
23761       }
23762 
23763       // Pattern: (sub:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
23764       // Emits: (SUB32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
23765       // Pattern complexity = 5  cost = 1
23766       if (Predicate_i32immSExt8(N1.Val) &&
23767           N.Val->getValueType(0) == MVT::i32) {
23768         Select(Tmp0, N0);
23769         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
23770         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
23771         if (N.Val->hasOneUse()) {
23772           Result = CurDAG->SelectNodeTo(N.Val, X86::SUB32ri8, MVT::i32, Tmp0, Tmp1);
23773         } else {
23774           ResNode = CurDAG->getTargetNode(X86::SUB32ri8, MVT::i32, Tmp0, Tmp1);
23775           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23776           Result = SDOperand(ResNode, 0);
23777         }
23778         return;
23779       }
23780 
23781       // Pattern: (sub:i8 R8:i8:$src1, (imm:i8):$src2)
23782       // Emits: (SUB8ri:i8 R8:i8:$src1, (imm:i8):$src2)
23783       // Pattern complexity = 4  cost = 1
23784       if (N.Val->getValueType(0) == MVT::i8) {
23785         Select(Tmp0, N0);
23786         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
23787         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
23788         if (N.Val->hasOneUse()) {
23789           Result = CurDAG->SelectNodeTo(N.Val, X86::SUB8ri, MVT::i8, Tmp0, Tmp1);
23790         } else {
23791           ResNode = CurDAG->getTargetNode(X86::SUB8ri, MVT::i8, Tmp0, Tmp1);
23792           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23793           Result = SDOperand(ResNode, 0);
23794         }
23795         return;
23796       }
23797 
23798       // Pattern: (sub:i16 R16:i16:$src1, (imm:i16):$src2)
23799       // Emits: (SUB16ri:i16 R16:i16:$src1, (imm:i16):$src2)
23800       // Pattern complexity = 4  cost = 1
23801       if (N.Val->getValueType(0) == MVT::i16) {
23802         Select(Tmp0, N0);
23803         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
23804         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
23805         if (N.Val->hasOneUse()) {
23806           Result = CurDAG->SelectNodeTo(N.Val, X86::SUB16ri, MVT::i16, Tmp0, Tmp1);
23807         } else {
23808           ResNode = CurDAG->getTargetNode(X86::SUB16ri, MVT::i16, Tmp0, Tmp1);
23809           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23810           Result = SDOperand(ResNode, 0);
23811         }
23812         return;
23813       }
23814 
23815       // Pattern: (sub:i32 R32:i32:$src1, (imm:i32):$src2)
23816       // Emits: (SUB32ri:i32 R32:i32:$src1, (imm:i32):$src2)
23817       // Pattern complexity = 4  cost = 1
23818       if (N.Val->getValueType(0) == MVT::i32) {
23819         Select(Tmp0, N0);
23820         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
23821         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
23822         if (N.Val->hasOneUse()) {
23823           Result = CurDAG->SelectNodeTo(N.Val, X86::SUB32ri, MVT::i32, Tmp0, Tmp1);
23824         } else {
23825           ResNode = CurDAG->getTargetNode(X86::SUB32ri, MVT::i32, Tmp0, Tmp1);
23826           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23827           Result = SDOperand(ResNode, 0);
23828         }
23829         return;
23830       }
23831     }
23832 
23833     // Pattern: (sub:i8 R8:i8:$src1, R8:i8:$src2)
23834     // Emits: (SUB8rr:i8 R8:i8:$src1, R8:i8:$src2)
23835     // Pattern complexity = 2  cost = 1
23836     if (N.Val->getValueType(0) == MVT::i8) {
23837       Select(Tmp0, N0);
23838       Select(Tmp1, N1);
23839       if (N.Val->hasOneUse()) {
23840         Result = CurDAG->SelectNodeTo(N.Val, X86::SUB8rr, MVT::i8, Tmp0, Tmp1);
23841       } else {
23842         ResNode = CurDAG->getTargetNode(X86::SUB8rr, MVT::i8, Tmp0, Tmp1);
23843         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23844         Result = SDOperand(ResNode, 0);
23845       }
23846       return;
23847     }
23848 
23849     // Pattern: (sub:i16 R16:i16:$src1, R16:i16:$src2)
23850     // Emits: (SUB16rr:i16 R16:i16:$src1, R16:i16:$src2)
23851     // Pattern complexity = 2  cost = 1
23852     if (N.Val->getValueType(0) == MVT::i16) {
23853       Select(Tmp0, N0);
23854       Select(Tmp1, N1);
23855       if (N.Val->hasOneUse()) {
23856         Result = CurDAG->SelectNodeTo(N.Val, X86::SUB16rr, MVT::i16, Tmp0, Tmp1);
23857       } else {
23858         ResNode = CurDAG->getTargetNode(X86::SUB16rr, MVT::i16, Tmp0, Tmp1);
23859         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23860         Result = SDOperand(ResNode, 0);
23861       }
23862       return;
23863     }
23864 
23865     // Pattern: (sub:i32 R32:i32:$src1, R32:i32:$src2)
23866     // Emits: (SUB32rr:i32 R32:i32:$src1, R32:i32:$src2)
23867     // Pattern complexity = 2  cost = 1
23868     if (N.Val->getValueType(0) == MVT::i32) {
23869       Select(Tmp0, N0);
23870       Select(Tmp1, N1);
23871       if (N.Val->hasOneUse()) {
23872         Result = CurDAG->SelectNodeTo(N.Val, X86::SUB32rr, MVT::i32, Tmp0, Tmp1);
23873       } else {
23874         ResNode = CurDAG->getTargetNode(X86::SUB32rr, MVT::i32, Tmp0, Tmp1);
23875         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23876         Result = SDOperand(ResNode, 0);
23877       }
23878       return;
23879     }
23880   }
23881   if ((Subtarget->hasSSE2())) {
23882     N0 = N.getOperand(0);
23883     N1 = N.getOperand(1);
23884 
23885     // Pattern: (sub:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
23886     // Emits: (PSUBBrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
23887     // Pattern complexity = 2  cost = 1
23888     if (N.Val->getValueType(0) == MVT::v16i8) {
23889       Select(Tmp0, N0);
23890       Select(Tmp1, N1);
23891       if (N.Val->hasOneUse()) {
23892         Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBBrr, MVT::v16i8, Tmp0, Tmp1);
23893       } else {
23894         ResNode = CurDAG->getTargetNode(X86::PSUBBrr, MVT::v16i8, Tmp0, Tmp1);
23895         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23896         Result = SDOperand(ResNode, 0);
23897       }
23898       return;
23899     }
23900 
23901     // Pattern: (sub:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
23902     // Emits: (PSUBWrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
23903     // Pattern complexity = 2  cost = 1
23904     if (N.Val->getValueType(0) == MVT::v8i16) {
23905       Select(Tmp0, N0);
23906       Select(Tmp1, N1);
23907       if (N.Val->hasOneUse()) {
23908         Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBWrr, MVT::v8i16, Tmp0, Tmp1);
23909       } else {
23910         ResNode = CurDAG->getTargetNode(X86::PSUBWrr, MVT::v8i16, Tmp0, Tmp1);
23911         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23912         Result = SDOperand(ResNode, 0);
23913       }
23914       return;
23915     }
23916 
23917     // Pattern: (sub:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
23918     // Emits: (PSUBDrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
23919     // Pattern complexity = 2  cost = 1
23920     if (N.Val->getValueType(0) == MVT::v4i32) {
23921       Select(Tmp0, N0);
23922       Select(Tmp1, N1);
23923       if (N.Val->hasOneUse()) {
23924         Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBDrr, MVT::v4i32, Tmp0, Tmp1);
23925       } else {
23926         ResNode = CurDAG->getTargetNode(X86::PSUBDrr, MVT::v4i32, Tmp0, Tmp1);
23927         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23928         Result = SDOperand(ResNode, 0);
23929       }
23930       return;
23931     }
23932 
23933     // Pattern: (sub:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
23934     // Emits: (PSUBQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
23935     // Pattern complexity = 2  cost = 1
23936     if (N.Val->getValueType(0) == MVT::v2i64) {
23937       Select(Tmp0, N0);
23938       Select(Tmp1, N1);
23939       if (N.Val->hasOneUse()) {
23940         Result = CurDAG->SelectNodeTo(N.Val, X86::PSUBQrr, MVT::v2i64, Tmp0, Tmp1);
23941       } else {
23942         ResNode = CurDAG->getTargetNode(X86::PSUBQrr, MVT::v2i64, Tmp0, Tmp1);
23943         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
23944         Result = SDOperand(ResNode, 0);
23945       }
23946       return;
23947     }
23948   }
23949   std::cerr << "Cannot yet select: ";
23950   N.Val->dump(CurDAG);
23951   std::cerr << '\n';
23952   abort();
23953 }
23954 
23955 void Select_subc(SDOperand &Result, SDOperand N) {
23956   SDOperand Chain1(0, 0);
23957   SDOperand InFlag(0, 0);
23958   SDOperand N0(0, 0);
23959   SDOperand N1(0, 0);
23960   SDOperand N11(0, 0);
23961   SDOperand Tmp0(0, 0);
23962   SDOperand Tmp1(0, 0);
23963   SDOperand Tmp2(0, 0);
23964   SDOperand Tmp3(0, 0);
23965   SDOperand Tmp4(0, 0);
23966   SDNode *ResNode;
23967   N0 = N.getOperand(0);
23968   N1 = N.getOperand(1);
23969 
23970   // Pattern: (subc:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
23971   // Emits: (SUB32rm:i32 R32:i32:$src1, addr:i32:$src2)
23972   // Pattern complexity = 14  cost = 1
23973   if (N1.getOpcode() == ISD::LOAD &&
23974       N1.hasOneUse() &&
23975       !CodeGenMap.count(N1.getValue(0)) &&
23976       N.Val->isOnlyUse(N1.Val) &&
23977       !isNonImmUse(N.Val, N1.Val)) {
23978     Chain1 = N1.getOperand(0);
23979     N11 = N1.getOperand(1);
23980     if (N.Val->getValueType(0) == MVT::i32 &&
23981         SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
23982       Select(Tmp0, Tmp0);
23983       Select(Tmp1, Tmp1);
23984       Select(Tmp2, Tmp2);
23985       Select(Tmp3, Tmp3);
23986       Select(Tmp4, N0);
23987       Select(Chain1, Chain1);
23988       ResNode = CurDAG->getTargetNode(X86::SUB32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
23989       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
23990       InFlag = SDOperand(ResNode, 2);
23991       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
23992       AddHandleReplacement(N1.Val, 1, ResNode, 1);
23993       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
23994       if (N.ResNo < 1)
23995         Result = SDOperand(ResNode, N.ResNo);
23996       else
23997         Result = SDOperand(ResNode, N.ResNo+1);
23998       return;
23999     }
24000   }
24001   if (N1.getOpcode() == ISD::Constant) {
24002 
24003     // Pattern: (subc:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
24004     // Emits: (SUB32ri8:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
24005     // Pattern complexity = 5  cost = 1
24006     if (Predicate_i32immSExt8(N1.Val) &&
24007         N.Val->getValueType(0) == MVT::i32) {
24008       Select(Tmp0, N0);
24009       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
24010       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
24011       ResNode = CurDAG->getTargetNode(X86::SUB32ri8, MVT::i32, MVT::Flag, Tmp0, Tmp1);
24012       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24013       InFlag = SDOperand(ResNode, 1);
24014       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24015       Result = SDOperand(ResNode, N.ResNo);
24016       return;
24017     }
24018 
24019     // Pattern: (subc:i32 R32:i32:$src1, (imm:i32):$src2)
24020     // Emits: (SUB32ri:i32 R32:i32:$src1, (imm:i32):$src2)
24021     // Pattern complexity = 4  cost = 1
24022     if (N.Val->getValueType(0) == MVT::i32) {
24023       Select(Tmp0, N0);
24024       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
24025       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
24026       ResNode = CurDAG->getTargetNode(X86::SUB32ri, MVT::i32, MVT::Flag, Tmp0, Tmp1);
24027       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24028       InFlag = SDOperand(ResNode, 1);
24029       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24030       Result = SDOperand(ResNode, N.ResNo);
24031       return;
24032     }
24033   }
24034 
24035   // Pattern: (subc:i32 R32:i32:$src1, R32:i32:$src2)
24036   // Emits: (SUB32rr:i32 R32:i32:$src1, R32:i32:$src2)
24037   // Pattern complexity = 2  cost = 1
24038   if (N.Val->getValueType(0) == MVT::i32) {
24039     Select(Tmp0, N0);
24040     Select(Tmp1, N1);
24041     ResNode = CurDAG->getTargetNode(X86::SUB32rr, MVT::i32, MVT::Flag, Tmp0, Tmp1);
24042     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24043     InFlag = SDOperand(ResNode, 1);
24044     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24045     Result = SDOperand(ResNode, N.ResNo);
24046     return;
24047   }
24048   std::cerr << "Cannot yet select: ";
24049   N.Val->dump(CurDAG);
24050   std::cerr << '\n';
24051   abort();
24052 }
24053 
24054 void Select_sube(SDOperand &Result, SDOperand N) {
24055   SDOperand Chain1(0, 0);
24056   SDOperand InFlag(0, 0);
24057   SDOperand N0(0, 0);
24058   SDOperand N1(0, 0);
24059   SDOperand N11(0, 0);
24060   SDOperand Tmp0(0, 0);
24061   SDOperand Tmp1(0, 0);
24062   SDOperand Tmp2(0, 0);
24063   SDOperand Tmp3(0, 0);
24064   SDOperand Tmp4(0, 0);
24065   SDNode *ResNode;
24066   N0 = N.getOperand(0);
24067   N1 = N.getOperand(1);
24068 
24069   // Pattern: (sube:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
24070   // Emits: (SBB32rm:i32 R32:i32:$src1, addr:i32:$src2)
24071   // Pattern complexity = 14  cost = 1
24072   if (N1.getOpcode() == ISD::LOAD &&
24073       N1.hasOneUse() &&
24074       !CodeGenMap.count(N1.getValue(0)) &&
24075       N.Val->isOnlyUse(N1.Val) &&
24076       !isNonImmUse(N.Val, N1.Val)) {
24077     Chain1 = N1.getOperand(0);
24078     N11 = N1.getOperand(1);
24079     if (N.Val->getValueType(0) == MVT::i32 &&
24080         SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
24081       Select(Tmp0, Tmp0);
24082       Select(Tmp1, Tmp1);
24083       Select(Tmp2, Tmp2);
24084       Select(Tmp3, Tmp3);
24085       Select(Tmp4, N0);
24086       Select(Chain1, Chain1);
24087       Select(InFlag, N.getOperand(2));
24088       ResNode = CurDAG->getTargetNode(X86::SBB32rm, MVT::i32, MVT::Other, MVT::Flag, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1, InFlag);
24089       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24090       InFlag = SDOperand(ResNode, 2);
24091       SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
24092       AddHandleReplacement(N1.Val, 1, ResNode, 1);
24093       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24094       if (N.ResNo < 1)
24095         Result = SDOperand(ResNode, N.ResNo);
24096       else
24097         Result = SDOperand(ResNode, N.ResNo+1);
24098       return;
24099     }
24100   }
24101   if (N1.getOpcode() == ISD::Constant) {
24102 
24103     // Pattern: (sube:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
24104     // Emits: (SBB32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
24105     // Pattern complexity = 5  cost = 1
24106     if (Predicate_i32immSExt8(N1.Val) &&
24107         N.Val->getValueType(0) == MVT::i32) {
24108       Select(Tmp0, N0);
24109       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
24110       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
24111       Select(InFlag, N.getOperand(2));
24112       ResNode = CurDAG->getTargetNode(X86::SBB32ri8, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
24113       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24114       InFlag = SDOperand(ResNode, 1);
24115       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24116       Result = SDOperand(ResNode, N.ResNo);
24117       return;
24118     }
24119 
24120     // Pattern: (sube:i32 R32:i32:$src1, (imm:i32):$src2)
24121     // Emits: (SBB32ri:i32 R32:i32:$src1, (imm:i32):$src2)
24122     // Pattern complexity = 4  cost = 1
24123     if (N.Val->getValueType(0) == MVT::i32) {
24124       Select(Tmp0, N0);
24125       unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
24126       Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
24127       Select(InFlag, N.getOperand(2));
24128       ResNode = CurDAG->getTargetNode(X86::SBB32ri, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
24129       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24130       InFlag = SDOperand(ResNode, 1);
24131       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24132       Result = SDOperand(ResNode, N.ResNo);
24133       return;
24134     }
24135   }
24136 
24137   // Pattern: (sube:i32 R32:i32:$src1, R32:i32:$src2)
24138   // Emits: (SBB32rr:i32 R32:i32:$src1, R32:i32:$src2)
24139   // Pattern complexity = 2  cost = 1
24140   if (N.Val->getValueType(0) == MVT::i32) {
24141     Select(Tmp0, N0);
24142     Select(Tmp1, N1);
24143     Select(InFlag, N.getOperand(2));
24144     ResNode = CurDAG->getTargetNode(X86::SBB32rr, MVT::i32, MVT::Flag, Tmp0, Tmp1, InFlag);
24145     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24146     InFlag = SDOperand(ResNode, 1);
24147     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, InFlag.Val, InFlag.ResNo);
24148     Result = SDOperand(ResNode, N.ResNo);
24149     return;
24150   }
24151   std::cerr << "Cannot yet select: ";
24152   N.Val->dump(CurDAG);
24153   std::cerr << '\n';
24154   abort();
24155 }
24156 
24157 void Select_truncst(SDOperand &Result, SDOperand N) {
24158   SDOperand Chain(0, 0);
24159   SDOperand N1(0, 0);
24160   SDOperand N2(0, 0);
24161   SDOperand N3(0, 0);
24162   SDOperand N4(0, 0);
24163   SDOperand Tmp0(0, 0);
24164   SDOperand Tmp1(0, 0);
24165   SDOperand Tmp2(0, 0);
24166   SDOperand Tmp3(0, 0);
24167   SDOperand Tmp4(0, 0);
24168   SDNode *ResNode;
24169 
24170   // Pattern: (truncst:void (imm:i8):$src, addr:i32:$dst, srcvalue:Other:$dummy, i1:Other)
24171   // Emits: (MOV8mi:void addr:i32:$dst, (imm:i8):$src)
24172   // Pattern complexity = 14  cost = 1
24173   {
24174     Chain = N.getOperand(0);
24175     N1 = N.getOperand(1);
24176     if (N1.getOpcode() == ISD::Constant) {
24177       N2 = N.getOperand(2);
24178       N3 = N.getOperand(3);
24179       N4 = N.getOperand(4);
24180       if (cast<VTSDNode>(N4)->getVT() == MVT::i1 &&
24181           N1.Val->getValueType(0) == MVT::i8 &&
24182           SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
24183         Select(Tmp0, Tmp0);
24184         Select(Tmp1, Tmp1);
24185         Select(Tmp2, Tmp2);
24186         Select(Tmp3, Tmp3);
24187         unsigned char Tmp4C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
24188         Tmp4 = CurDAG->getTargetConstant(Tmp4C, MVT::i8);
24189         Select(Chain, Chain);
24190         ResNode = CurDAG->getTargetNode(X86::MOV8mi, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
24191         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24192         Result = SDOperand(ResNode, N.ResNo);
24193         return;
24194       }
24195     }
24196   }
24197 
24198   // Pattern: (truncst:void RFP:f64:$src, addr:i32:$op, srcvalue:Other:$dummy, f32:Other)
24199   // Emits: (FpST32m:void addr:i32:$op, RFP:f64:$src)
24200   // Pattern complexity = 12  cost = 1
24201   if ((!Subtarget->hasSSE2())) {
24202     Chain = N.getOperand(0);
24203     N1 = N.getOperand(1);
24204     N2 = N.getOperand(2);
24205     N3 = N.getOperand(3);
24206     N4 = N.getOperand(4);
24207     if (cast<VTSDNode>(N4)->getVT() == MVT::f32 &&
24208         N1.Val->getValueType(0) == MVT::f64 &&
24209         SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
24210       Select(Tmp0, Tmp0);
24211       Select(Tmp1, Tmp1);
24212       Select(Tmp2, Tmp2);
24213       Select(Tmp3, Tmp3);
24214       Select(Tmp4, N1);
24215       Select(Chain, Chain);
24216       ResNode = CurDAG->getTargetNode(X86::FpST32m, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
24217       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24218       Result = SDOperand(ResNode, N.ResNo);
24219       return;
24220     }
24221   }
24222 
24223   // Pattern: (truncst:void R8:i8:$src, addr:i32:$dst, srcvalue:Other:$dummy, i1:Other)
24224   // Emits: (MOV8mr:void addr:i32:$dst, R8:i8:$src)
24225   // Pattern complexity = 12  cost = 1
24226   Chain = N.getOperand(0);
24227   N1 = N.getOperand(1);
24228   N2 = N.getOperand(2);
24229   N3 = N.getOperand(3);
24230   N4 = N.getOperand(4);
24231   if (cast<VTSDNode>(N4)->getVT() == MVT::i1 &&
24232       N1.Val->getValueType(0) == MVT::i8 &&
24233       SelectAddr(N2, Tmp0, Tmp1, Tmp2, Tmp3)) {
24234     Select(Tmp0, Tmp0);
24235     Select(Tmp1, Tmp1);
24236     Select(Tmp2, Tmp2);
24237     Select(Tmp3, Tmp3);
24238     Select(Tmp4, N1);
24239     Select(Chain, Chain);
24240     ResNode = CurDAG->getTargetNode(X86::MOV8mr, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain);
24241     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24242     Result = SDOperand(ResNode, N.ResNo);
24243     return;
24244   }
24245   std::cerr << "Cannot yet select: ";
24246   N.Val->dump(CurDAG);
24247   std::cerr << '\n';
24248   abort();
24249 }
24250 
24251 void Select_undef(SDOperand &Result, SDOperand N) {
24252   SDNode *ResNode;
24253 
24254   // Pattern: (undef:i8)
24255   // Emits: (IMPLICIT_DEF_R8:i8)
24256   // Pattern complexity = 2  cost = 1
24257   if (N.Val->getValueType(0) == MVT::i8) {
24258     if (N.Val->hasOneUse()) {
24259       Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_R8, MVT::i8);
24260     } else {
24261       ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_R8, MVT::i8);
24262       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24263       Result = SDOperand(ResNode, 0);
24264     }
24265     return;
24266   }
24267 
24268   // Pattern: (undef:i16)
24269   // Emits: (IMPLICIT_DEF_R16:i16)
24270   // Pattern complexity = 2  cost = 1
24271   if (N.Val->getValueType(0) == MVT::i16) {
24272     if (N.Val->hasOneUse()) {
24273       Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_R16, MVT::i16);
24274     } else {
24275       ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_R16, MVT::i16);
24276       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24277       Result = SDOperand(ResNode, 0);
24278     }
24279     return;
24280   }
24281 
24282   // Pattern: (undef:i32)
24283   // Emits: (IMPLICIT_DEF_R32:i32)
24284   // Pattern complexity = 2  cost = 1
24285   if (N.Val->getValueType(0) == MVT::i32) {
24286     if (N.Val->hasOneUse()) {
24287       Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_R32, MVT::i32);
24288     } else {
24289       ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_R32, MVT::i32);
24290       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24291       Result = SDOperand(ResNode, 0);
24292     }
24293     return;
24294   }
24295 
24296   // Pattern: (undef:v8i8)
24297   // Emits: (IMPLICIT_DEF_VR64:v8i8)
24298   // Pattern complexity = 2  cost = 1
24299   if ((Subtarget->hasMMX()) &&
24300       N.Val->getValueType(0) == MVT::v8i8) {
24301     if (N.Val->hasOneUse()) {
24302       Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24303     } else {
24304       ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24305       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24306       Result = SDOperand(ResNode, 0);
24307     }
24308     return;
24309   }
24310   if ((Subtarget->hasSSE2())) {
24311 
24312     // Pattern: (undef:f32)
24313     // Emits: (IMPLICIT_DEF_FR32:f32)
24314     // Pattern complexity = 2  cost = 1
24315     if (N.Val->getValueType(0) == MVT::f32) {
24316       if (N.Val->hasOneUse()) {
24317         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_FR32, MVT::f32);
24318       } else {
24319         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_FR32, MVT::f32);
24320         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24321         Result = SDOperand(ResNode, 0);
24322       }
24323       return;
24324     }
24325 
24326     // Pattern: (undef:f64)
24327     // Emits: (IMPLICIT_DEF_FR64:f64)
24328     // Pattern complexity = 2  cost = 1
24329     if (N.Val->getValueType(0) == MVT::f64) {
24330       if (N.Val->hasOneUse()) {
24331         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_FR64, MVT::f64);
24332       } else {
24333         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_FR64, MVT::f64);
24334         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24335         Result = SDOperand(ResNode, 0);
24336       }
24337       return;
24338     }
24339   }
24340 
24341   // Pattern: (undef:v4f32)
24342   // Emits: (IMPLICIT_DEF_VR128:v4f32)
24343   // Pattern complexity = 2  cost = 1
24344   if ((Subtarget->hasSSE1()) &&
24345       N.Val->getValueType(0) == MVT::v4f32) {
24346     if (N.Val->hasOneUse()) {
24347       Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v4f32);
24348     } else {
24349       ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v4f32);
24350       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24351       Result = SDOperand(ResNode, 0);
24352     }
24353     return;
24354   }
24355   if ((Subtarget->hasMMX())) {
24356 
24357     // Pattern: (undef:v4i16)
24358     // Emits: (IMPLICIT_DEF_VR64:v8i8)
24359     // Pattern complexity = 2  cost = 1
24360     if (N.Val->getValueType(0) == MVT::v4i16) {
24361       if (N.Val->hasOneUse()) {
24362         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24363       } else {
24364         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24365         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24366         Result = SDOperand(ResNode, 0);
24367       }
24368       return;
24369     }
24370 
24371     // Pattern: (undef:v2i32)
24372     // Emits: (IMPLICIT_DEF_VR64:v8i8)
24373     // Pattern complexity = 2  cost = 1
24374     if (N.Val->getValueType(0) == MVT::v2i32) {
24375       if (N.Val->hasOneUse()) {
24376         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24377       } else {
24378         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR64, MVT::v8i8);
24379         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24380         Result = SDOperand(ResNode, 0);
24381       }
24382       return;
24383     }
24384   }
24385   if ((Subtarget->hasSSE2())) {
24386 
24387     // Pattern: (undef:v2f64)
24388     // Emits: (IMPLICIT_DEF_VR128:v16i8)
24389     // Pattern complexity = 2  cost = 1
24390     if (N.Val->getValueType(0) == MVT::v2f64) {
24391       if (N.Val->hasOneUse()) {
24392         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24393       } else {
24394         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24395         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24396         Result = SDOperand(ResNode, 0);
24397       }
24398       return;
24399     }
24400 
24401     // Pattern: (undef:v16i8)
24402     // Emits: (IMPLICIT_DEF_VR128:v16i8)
24403     // Pattern complexity = 2  cost = 1
24404     if (N.Val->getValueType(0) == MVT::v16i8) {
24405       if (N.Val->hasOneUse()) {
24406         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24407       } else {
24408         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24409         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24410         Result = SDOperand(ResNode, 0);
24411       }
24412       return;
24413     }
24414 
24415     // Pattern: (undef:v8i16)
24416     // Emits: (IMPLICIT_DEF_VR128:v16i8)
24417     // Pattern complexity = 2  cost = 1
24418     if (N.Val->getValueType(0) == MVT::v8i16) {
24419       if (N.Val->hasOneUse()) {
24420         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24421       } else {
24422         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24423         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24424         Result = SDOperand(ResNode, 0);
24425       }
24426       return;
24427     }
24428 
24429     // Pattern: (undef:v4i32)
24430     // Emits: (IMPLICIT_DEF_VR128:v16i8)
24431     // Pattern complexity = 2  cost = 1
24432     if (N.Val->getValueType(0) == MVT::v4i32) {
24433       if (N.Val->hasOneUse()) {
24434         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24435       } else {
24436         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24437         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24438         Result = SDOperand(ResNode, 0);
24439       }
24440       return;
24441     }
24442 
24443     // Pattern: (undef:v2i64)
24444     // Emits: (IMPLICIT_DEF_VR128:v16i8)
24445     // Pattern complexity = 2  cost = 1
24446     if (N.Val->getValueType(0) == MVT::v2i64) {
24447       if (N.Val->hasOneUse()) {
24448         Result = CurDAG->SelectNodeTo(N.Val, X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24449       } else {
24450         ResNode = CurDAG->getTargetNode(X86::IMPLICIT_DEF_VR128, MVT::v16i8);
24451         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24452         Result = SDOperand(ResNode, 0);
24453       }
24454       return;
24455     }
24456   }
24457   std::cerr << "Cannot yet select: ";
24458   N.Val->dump(CurDAG);
24459   std::cerr << '\n';
24460   abort();
24461 }
24462 
24463 void Select_vector_extract(SDOperand &Result, SDOperand N) {
24464   SDOperand N0(0, 0);
24465   SDOperand N1(0, 0);
24466   SDOperand Tmp1(0, 0);
24467   SDNode *ResNode;
24468 
24469   // Pattern: (vector_extract:f32 VR128:v4f32:$src, 0:i32)
24470   // Emits: (MOVPS2SSrr:f32 VR128:v4f32:$src)
24471   // Pattern complexity = 5  cost = 1
24472   if ((Subtarget->hasSSE1())) {
24473     N0 = N.getOperand(0);
24474     N1 = N.getOperand(1);
24475     if (isa<ConstantSDNode>(N1)) {
24476       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
24477       if (CN0 == 0 &&
24478           N.Val->getValueType(0) == MVT::f32 &&
24479           N0.Val->getValueType(0) == MVT::v4f32 &&
24480           N1.Val->getValueType(0) == MVT::i32) {
24481         Select(Tmp1, N0);
24482         if (N.Val->hasOneUse()) {
24483           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVPS2SSrr, MVT::f32, Tmp1);
24484         } else {
24485           ResNode = CurDAG->getTargetNode(X86::MOVPS2SSrr, MVT::f32, Tmp1);
24486           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24487           Result = SDOperand(ResNode, 0);
24488         }
24489         return;
24490       }
24491     }
24492   }
24493   if ((Subtarget->hasSSE2())) {
24494     N0 = N.getOperand(0);
24495     N1 = N.getOperand(1);
24496     if (isa<ConstantSDNode>(N1)) {
24497       int64_t CN0 = cast<ConstantSDNode>(N1)->getSignExtended();
24498       if (CN0 == 0) {
24499 
24500         // Pattern: (vector_extract:f64 VR128:v2f64:$src, 0:i32)
24501         // Emits: (MOVPD2SDrr:f64 VR128:v2f64:$src)
24502         // Pattern complexity = 5  cost = 1
24503         if (N.Val->getValueType(0) == MVT::f64 &&
24504             N0.Val->getValueType(0) == MVT::v2f64 &&
24505             N1.Val->getValueType(0) == MVT::i32) {
24506           Select(Tmp1, N0);
24507           if (N.Val->hasOneUse()) {
24508             Result = CurDAG->SelectNodeTo(N.Val, X86::MOVPD2SDrr, MVT::f64, Tmp1);
24509           } else {
24510             ResNode = CurDAG->getTargetNode(X86::MOVPD2SDrr, MVT::f64, Tmp1);
24511             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24512             Result = SDOperand(ResNode, 0);
24513           }
24514           return;
24515         }
24516 
24517         // Pattern: (vector_extract:i32 VR128:v4i32:$src, 0:i32)
24518         // Emits: (MOVPDI2DIrr:i32 VR128:v4i32:$src)
24519         // Pattern complexity = 5  cost = 1
24520         if (N.Val->getValueType(0) == MVT::i32 &&
24521             N0.Val->getValueType(0) == MVT::v4i32 &&
24522             N1.Val->getValueType(0) == MVT::i32) {
24523           Select(Tmp1, N0);
24524           if (N.Val->hasOneUse()) {
24525             Result = CurDAG->SelectNodeTo(N.Val, X86::MOVPDI2DIrr, MVT::i32, Tmp1);
24526           } else {
24527             ResNode = CurDAG->getTargetNode(X86::MOVPDI2DIrr, MVT::i32, Tmp1);
24528             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
24529             Result = SDOperand(ResNode, 0);
24530           }
24531           return;
24532         }
24533       }
24534     }
24535   }
24536   std::cerr << "Cannot yet select: ";
24537   N.Val->dump(CurDAG);
24538   std::cerr << '\n';
24539   abort();
24540 }
24541 
24542 void Select_vector_shuffle(SDOperand &Result, SDOperand N) {
24543   SDOperand Chain0(0, 0);
24544   SDOperand Chain00(0, 0);
24545   SDOperand Chain1(0, 0);
24546   SDOperand Chain10(0, 0);
24547   SDOperand Chain100(0, 0);
24548   SDOperand N0(0, 0);
24549   SDOperand N00(0, 0);
24550   SDOperand N001(0, 0);
24551   SDOperand N01(0, 0);
24552   SDOperand N1(0, 0);
24553   SDOperand N10(0, 0);
24554   SDOperand N100(0, 0);
24555   SDOperand N1001(0, 0);
24556   SDOperand N101(0, 0);
24557   SDOperand N11(0, 0);
24558   SDOperand N2(0, 0);
24559   SDOperand Tmp0(0, 0);
24560   SDOperand Tmp1(0, 0);
24561   SDOperand Tmp2(0, 0);
24562   SDOperand Tmp3(0, 0);
24563   SDOperand Tmp4(0, 0);
24564   SDOperand Tmp5(0, 0);
24565   SDOperand Tmp6(0, 0);
24566   SDNode *ResNode;
24567   if ((Subtarget->hasSSE1())) {
24568     N0 = N.getOperand(0);
24569     N1 = N.getOperand(1);
24570     if (N1.getOpcode() == ISD::BIT_CONVERT &&
24571         N1.hasOneUse() &&
24572         !CodeGenMap.count(N1.getValue(0))) {
24573       N10 = N1.getOperand(0);
24574       if (N10.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24575           N10.hasOneUse() &&
24576           !CodeGenMap.count(N10.getValue(0))) {
24577         N100 = N10.getOperand(0);
24578         if (N100.getOpcode() == ISD::LOAD &&
24579             N100.hasOneUse() &&
24580             !CodeGenMap.count(N100.getValue(0))) {
24581           Chain100 = N100.getOperand(0);
24582           N1001 = N100.getOperand(1);
24583           N2 = N.getOperand(2);
24584           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24585 
24586             // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (bitconvert:v4f32 (scalar_to_vector:v2f64 (load:f64 addr:i32:$src2))), (build_vector:v4i16)<<P:Predicate_MOVLP_shuffle_mask>>)
24587             // Emits: (MOVLPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
24588             // Pattern complexity = 21  cost = 1
24589             if (Predicate_MOVLP_shuffle_mask(N2.Val) &&
24590                 N.Val->getValueType(0) == MVT::v4f32 &&
24591                 N10.Val->getValueType(0) == MVT::v2f64 &&
24592                 N100.Val->getValueType(0) == MVT::f64 &&
24593                 SelectAddr(N1001, Tmp0, Tmp1, Tmp2, Tmp3)) {
24594               Select(Tmp0, Tmp0);
24595               Select(Tmp1, Tmp1);
24596               Select(Tmp2, Tmp2);
24597               Select(Tmp3, Tmp3);
24598               Select(Tmp4, N0);
24599               Select(Chain100, Chain100);
24600               ResNode = CurDAG->getTargetNode(X86::MOVLPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain100);
24601               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24602               SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 1);
24603               AddHandleReplacement(N100.Val, 1, ResNode, 1);
24604               Result = SDOperand(ResNode, N.ResNo);
24605               return;
24606             }
24607 
24608             // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (bitconvert:v4f32 (scalar_to_vector:v2f64 (load:f64 addr:i32:$src2))), (build_vector:v4i16)<<P:Predicate_MOVHP_shuffle_mask>>)
24609             // Emits: (MOVHPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
24610             // Pattern complexity = 21  cost = 1
24611             if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
24612                 N.Val->getValueType(0) == MVT::v4f32 &&
24613                 N10.Val->getValueType(0) == MVT::v2f64 &&
24614                 N100.Val->getValueType(0) == MVT::f64 &&
24615                 SelectAddr(N1001, Tmp0, Tmp1, Tmp2, Tmp3)) {
24616               Select(Tmp0, Tmp0);
24617               Select(Tmp1, Tmp1);
24618               Select(Tmp2, Tmp2);
24619               Select(Tmp3, Tmp3);
24620               Select(Tmp4, N0);
24621               Select(Chain100, Chain100);
24622               ResNode = CurDAG->getTargetNode(X86::MOVHPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain100);
24623               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24624               SelectionDAG::InsertISelMapEntry(CodeGenMap, N100.Val, 1, ResNode, 1);
24625               AddHandleReplacement(N100.Val, 1, ResNode, 1);
24626               Result = SDOperand(ResNode, N.ResNo);
24627               return;
24628             }
24629           }
24630         }
24631       }
24632     }
24633   }
24634   if ((Subtarget->hasSSE2())) {
24635     N0 = N.getOperand(0);
24636     if (N0.getOpcode() == ISD::BIT_CONVERT &&
24637         N0.hasOneUse() &&
24638         !CodeGenMap.count(N0.getValue(0))) {
24639       N00 = N0.getOperand(0);
24640       if (N00.getOpcode() == ISD::LOAD &&
24641           N00.hasOneUse() &&
24642           !CodeGenMap.count(N00.getValue(0))) {
24643         Chain00 = N00.getOperand(0);
24644         N001 = N00.getOperand(1);
24645         N1 = N.getOperand(1);
24646         if (N1.getOpcode() == ISD::UNDEF) {
24647           N2 = N.getOperand(2);
24648           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24649 
24650             // Pattern: (vector_shuffle:v4i32 (bitconvert:v4i32 (load:v2i64 addr:i32:$src1)), (undef:v4i32), (build_vector:v4i16)<<P:Predicate_PSHUFD_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src2)
24651             // Emits: (PSHUFDmi:v4i32 addr:i32:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:v4i16):$src2))
24652             // Pattern complexity = 21  cost = 1
24653             if (Predicate_PSHUFD_shuffle_mask(N2.Val) &&
24654                 N.Val->getValueType(0) == MVT::v4i32 &&
24655                 N00.Val->getValueType(0) == MVT::v2i64 &&
24656                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
24657               Select(Tmp0, Tmp0);
24658               Select(Tmp1, Tmp1);
24659               Select(Tmp2, Tmp2);
24660               Select(Tmp3, Tmp3);
24661               Tmp4 = N2;
24662               Tmp5 = Transform_SHUFFLE_get_shuf_imm(Tmp4.Val);
24663               Select(Chain00, Chain00);
24664               ResNode = CurDAG->getTargetNode(X86::PSHUFDmi, MVT::v4i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24665               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24666               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24667               AddHandleReplacement(N00.Val, 1, ResNode, 1);
24668               Result = SDOperand(ResNode, N.ResNo);
24669               return;
24670             }
24671 
24672             // Pattern: (vector_shuffle:v8i16 (bitconvert:v8i16 (load:v2i64 addr:i32:$src1)), (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFHW_shuffle_mask>><<X:SHUFFLE_get_pshufhw_imm>>:$src2)
24673             // Emits: (PSHUFHWmi:v8i16 addr:i32:$src1, (SHUFFLE_get_pshufhw_imm:i8 (build_vector:v8i8):$src2))
24674             // Pattern complexity = 21  cost = 1
24675             if (Predicate_PSHUFHW_shuffle_mask(N2.Val) &&
24676                 N.Val->getValueType(0) == MVT::v8i16 &&
24677                 N00.Val->getValueType(0) == MVT::v2i64 &&
24678                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
24679               Select(Tmp0, Tmp0);
24680               Select(Tmp1, Tmp1);
24681               Select(Tmp2, Tmp2);
24682               Select(Tmp3, Tmp3);
24683               Tmp4 = N2;
24684               Tmp5 = Transform_SHUFFLE_get_pshufhw_imm(Tmp4.Val);
24685               Select(Chain00, Chain00);
24686               ResNode = CurDAG->getTargetNode(X86::PSHUFHWmi, MVT::v8i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24687               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24688               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24689               AddHandleReplacement(N00.Val, 1, ResNode, 1);
24690               Result = SDOperand(ResNode, N.ResNo);
24691               return;
24692             }
24693 
24694             // Pattern: (vector_shuffle:v8i16 (bitconvert:v8i16 (load:v2i64 addr:i32:$src1)), (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFLW_shuffle_mask>><<X:SHUFFLE_get_pshuflw_imm>>:$src2)
24695             // Emits: (PSHUFLWmi:v8i16 addr:i32:$src1, (SHUFFLE_get_pshuflw_imm:i32 (build_vector:v8i8):$src2))
24696             // Pattern complexity = 21  cost = 1
24697             if (Predicate_PSHUFLW_shuffle_mask(N2.Val) &&
24698                 N.Val->getValueType(0) == MVT::v8i16 &&
24699                 N00.Val->getValueType(0) == MVT::v2i64 &&
24700                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
24701               Select(Tmp0, Tmp0);
24702               Select(Tmp1, Tmp1);
24703               Select(Tmp2, Tmp2);
24704               Select(Tmp3, Tmp3);
24705               Tmp4 = N2;
24706               Tmp5 = Transform_SHUFFLE_get_pshuflw_imm(Tmp4.Val);
24707               Select(Chain00, Chain00);
24708               ResNode = CurDAG->getTargetNode(X86::PSHUFLWmi, MVT::v8i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain00);
24709               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24710               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
24711               AddHandleReplacement(N00.Val, 1, ResNode, 1);
24712               Result = SDOperand(ResNode, N.ResNo);
24713               return;
24714             }
24715           }
24716         }
24717       }
24718     }
24719     {
24720       N1 = N.getOperand(1);
24721       if (N1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
24722           N1.hasOneUse() &&
24723           !CodeGenMap.count(N1.getValue(0))) {
24724         N10 = N1.getOperand(0);
24725         if (N10.getOpcode() == ISD::LOAD &&
24726             N10.hasOneUse() &&
24727             !CodeGenMap.count(N10.getValue(0))) {
24728           Chain10 = N10.getOperand(0);
24729           N101 = N10.getOperand(1);
24730           N2 = N.getOperand(2);
24731           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24732 
24733             // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (scalar_to_vector:v2f64 (load:f64 addr:i32:$src2)), (build_vector:v2i32)<<P:Predicate_MOVLP_shuffle_mask>>)
24734             // Emits: (MOVLPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
24735             // Pattern complexity = 19  cost = 1
24736             if (Predicate_MOVLP_shuffle_mask(N2.Val) &&
24737                 N.Val->getValueType(0) == MVT::v2f64 &&
24738                 N10.Val->getValueType(0) == MVT::f64 &&
24739                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24740               Select(Tmp0, Tmp0);
24741               Select(Tmp1, Tmp1);
24742               Select(Tmp2, Tmp2);
24743               Select(Tmp3, Tmp3);
24744               Select(Tmp4, N0);
24745               Select(Chain10, Chain10);
24746               ResNode = CurDAG->getTargetNode(X86::MOVLPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24747               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24748               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24749               AddHandleReplacement(N10.Val, 1, ResNode, 1);
24750               Result = SDOperand(ResNode, N.ResNo);
24751               return;
24752             }
24753 
24754             // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (scalar_to_vector:v2f64 (load:f64 addr:i32:$src2)), (build_vector:v2i32)<<P:Predicate_MOVHP_shuffle_mask>>)
24755             // Emits: (MOVHPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
24756             // Pattern complexity = 19  cost = 1
24757             if (Predicate_MOVHP_shuffle_mask(N2.Val) &&
24758                 N.Val->getValueType(0) == MVT::v2f64 &&
24759                 N10.Val->getValueType(0) == MVT::f64 &&
24760                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24761               Select(Tmp0, Tmp0);
24762               Select(Tmp1, Tmp1);
24763               Select(Tmp2, Tmp2);
24764               Select(Tmp3, Tmp3);
24765               Select(Tmp4, N0);
24766               Select(Chain10, Chain10);
24767               ResNode = CurDAG->getTargetNode(X86::MOVHPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24768               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24769               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24770               AddHandleReplacement(N10.Val, 1, ResNode, 1);
24771               Result = SDOperand(ResNode, N.ResNo);
24772               return;
24773             }
24774           }
24775         }
24776       }
24777       if (N1.getOpcode() == ISD::BIT_CONVERT &&
24778           N1.hasOneUse() &&
24779           !CodeGenMap.count(N1.getValue(0))) {
24780         N10 = N1.getOperand(0);
24781         if (N10.getOpcode() == ISD::LOAD &&
24782             N10.hasOneUse() &&
24783             !CodeGenMap.count(N10.getValue(0))) {
24784           Chain10 = N10.getOperand(0);
24785           N101 = N10.getOperand(1);
24786           N2 = N.getOperand(2);
24787           if (N2.getOpcode() == ISD::BUILD_VECTOR) {
24788             if (Predicate_UNPCKL_shuffle_mask(N2.Val)) {
24789 
24790               // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)), (build_vector:v16i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
24791               // Emits: (PUNPCKLBWrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
24792               // Pattern complexity = 19  cost = 1
24793               if (N.Val->getValueType(0) == MVT::v16i8 &&
24794                   N10.Val->getValueType(0) == MVT::v2i64 &&
24795                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24796                 Select(Tmp0, Tmp0);
24797                 Select(Tmp1, Tmp1);
24798                 Select(Tmp2, Tmp2);
24799                 Select(Tmp3, Tmp3);
24800                 Select(Tmp4, N0);
24801                 Select(Chain10, Chain10);
24802                 ResNode = CurDAG->getTargetNode(X86::PUNPCKLBWrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24803                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24804                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24805                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24806                 Result = SDOperand(ResNode, N.ResNo);
24807                 return;
24808               }
24809 
24810               // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)), (build_vector:v8i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
24811               // Emits: (PUNPCKLWDrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
24812               // Pattern complexity = 19  cost = 1
24813               if (N.Val->getValueType(0) == MVT::v8i16 &&
24814                   N10.Val->getValueType(0) == MVT::v2i64 &&
24815                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24816                 Select(Tmp0, Tmp0);
24817                 Select(Tmp1, Tmp1);
24818                 Select(Tmp2, Tmp2);
24819                 Select(Tmp3, Tmp3);
24820                 Select(Tmp4, N0);
24821                 Select(Chain10, Chain10);
24822                 ResNode = CurDAG->getTargetNode(X86::PUNPCKLWDrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24823                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24824                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24825                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24826                 Result = SDOperand(ResNode, N.ResNo);
24827                 return;
24828               }
24829 
24830               // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:i32:$src2)), (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
24831               // Emits: (PUNPCKLDQrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
24832               // Pattern complexity = 19  cost = 1
24833               if (N.Val->getValueType(0) == MVT::v4i32 &&
24834                   N10.Val->getValueType(0) == MVT::v2i64 &&
24835                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24836                 Select(Tmp0, Tmp0);
24837                 Select(Tmp1, Tmp1);
24838                 Select(Tmp2, Tmp2);
24839                 Select(Tmp3, Tmp3);
24840                 Select(Tmp4, N0);
24841                 Select(Chain10, Chain10);
24842                 ResNode = CurDAG->getTargetNode(X86::PUNPCKLDQrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24843                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24844                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24845                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24846                 Result = SDOperand(ResNode, N.ResNo);
24847                 return;
24848               }
24849             }
24850             if (Predicate_UNPCKH_shuffle_mask(N2.Val)) {
24851 
24852               // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, (bitconvert:v16i8 (load:v2i64 addr:i32:$src2)), (build_vector:v16i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
24853               // Emits: (PUNPCKHBWrm:v16i8 VR128:v16i8:$src1, addr:i32:$src2)
24854               // Pattern complexity = 19  cost = 1
24855               if (N.Val->getValueType(0) == MVT::v16i8 &&
24856                   N10.Val->getValueType(0) == MVT::v2i64 &&
24857                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24858                 Select(Tmp0, Tmp0);
24859                 Select(Tmp1, Tmp1);
24860                 Select(Tmp2, Tmp2);
24861                 Select(Tmp3, Tmp3);
24862                 Select(Tmp4, N0);
24863                 Select(Chain10, Chain10);
24864                 ResNode = CurDAG->getTargetNode(X86::PUNPCKHBWrm, MVT::v16i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24865                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24866                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24867                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24868                 Result = SDOperand(ResNode, N.ResNo);
24869                 return;
24870               }
24871 
24872               // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (bitconvert:v8i16 (load:v2i64 addr:i32:$src2)), (build_vector:v8i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
24873               // Emits: (PUNPCKHWDrm:v8i16 VR128:v8i16:$src1, addr:i32:$src2)
24874               // Pattern complexity = 19  cost = 1
24875               if (N.Val->getValueType(0) == MVT::v8i16 &&
24876                   N10.Val->getValueType(0) == MVT::v2i64 &&
24877                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24878                 Select(Tmp0, Tmp0);
24879                 Select(Tmp1, Tmp1);
24880                 Select(Tmp2, Tmp2);
24881                 Select(Tmp3, Tmp3);
24882                 Select(Tmp4, N0);
24883                 Select(Chain10, Chain10);
24884                 ResNode = CurDAG->getTargetNode(X86::PUNPCKHWDrm, MVT::v8i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24885                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24886                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24887                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24888                 Result = SDOperand(ResNode, N.ResNo);
24889                 return;
24890               }
24891 
24892               // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:i32:$src2)), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
24893               // Emits: (PUNPCKHDQrm:v4i32 VR128:v4i32:$src1, addr:i32:$src2)
24894               // Pattern complexity = 19  cost = 1
24895               if (N.Val->getValueType(0) == MVT::v4i32 &&
24896                   N10.Val->getValueType(0) == MVT::v2i64 &&
24897                   SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24898                 Select(Tmp0, Tmp0);
24899                 Select(Tmp1, Tmp1);
24900                 Select(Tmp2, Tmp2);
24901                 Select(Tmp3, Tmp3);
24902                 Select(Tmp4, N0);
24903                 Select(Chain10, Chain10);
24904                 ResNode = CurDAG->getTargetNode(X86::PUNPCKHDQrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
24905                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24906                 SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24907                 AddHandleReplacement(N10.Val, 1, ResNode, 1);
24908                 Result = SDOperand(ResNode, N.ResNo);
24909                 return;
24910               }
24911             }
24912           }
24913         }
24914       }
24915     }
24916 
24917     // Pattern: (vector_shuffle:v4f32 (load:v4f32 addr:i32:$src1), (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SHUFP_unary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
24918     // Emits: (PSHUFDmi:v4f32 addr:i32:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:i8)<<P:Predicate_SHUFP_unary_shuffle_mask>>:$sm))
24919     // Pattern complexity = 19  cost = 1
24920     if (N0.getOpcode() == ISD::LOAD &&
24921         N0.hasOneUse() &&
24922         !CodeGenMap.count(N0.getValue(0)) &&
24923         !isNonImmUse(N.Val, N0.Val)) {
24924       Chain0 = N0.getOperand(0);
24925       N01 = N0.getOperand(1);
24926       N1 = N.getOperand(1);
24927       if (N1.getOpcode() == ISD::UNDEF) {
24928         N2 = N.getOperand(2);
24929         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24930             Predicate_SHUFP_unary_shuffle_mask(N2.Val) &&
24931             N.Val->getValueType(0) == MVT::v4f32 &&
24932             SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
24933           Select(Tmp0, Tmp0);
24934           Select(Tmp1, Tmp1);
24935           Select(Tmp2, Tmp2);
24936           Select(Tmp3, Tmp3);
24937           Tmp4 = N2;
24938           Tmp5 = Transform_SHUFFLE_get_shuf_imm(Tmp4.Val);
24939           Select(Chain0, Chain0);
24940           ResNode = CurDAG->getTargetNode(X86::PSHUFDmi, MVT::v4f32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Tmp5, Chain0);
24941           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24942           SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
24943           AddHandleReplacement(N0.Val, 1, ResNode, 1);
24944           Result = SDOperand(ResNode, N.ResNo);
24945           return;
24946         }
24947       }
24948     }
24949 
24950     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (bitconvert:v4i32 (load:v2i64 addr:i32:$src2)), (build_vector:v4i16)<<P:Predicate_PSHUFD_binary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
24951     // Emits: (SHUFPSrm:v4i32 VR128:v16i8:$src1, addr:i32:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_PSHUFD_binary_shuffle_mask>>:$sm))
24952     // Pattern complexity = 19  cost = 1
24953     N1 = N.getOperand(1);
24954     if (N1.getOpcode() == ISD::BIT_CONVERT &&
24955         N1.hasOneUse() &&
24956         !CodeGenMap.count(N1.getValue(0))) {
24957       N10 = N1.getOperand(0);
24958       if (N10.getOpcode() == ISD::LOAD &&
24959           N10.hasOneUse() &&
24960           !CodeGenMap.count(N10.getValue(0))) {
24961         Chain10 = N10.getOperand(0);
24962         N101 = N10.getOperand(1);
24963         N2 = N.getOperand(2);
24964         if (N2.getOpcode() == ISD::BUILD_VECTOR &&
24965             Predicate_PSHUFD_binary_shuffle_mask(N2.Val) &&
24966             N.Val->getValueType(0) == MVT::v4i32 &&
24967             N10.Val->getValueType(0) == MVT::v2i64 &&
24968             SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
24969           Select(Tmp0, Tmp0);
24970           Select(Tmp1, Tmp1);
24971           Select(Tmp2, Tmp2);
24972           Select(Tmp3, Tmp3);
24973           Select(Tmp4, N0);
24974           Tmp5 = N2;
24975           Tmp6 = Transform_SHUFFLE_get_shuf_imm(Tmp5.Val);
24976           Select(Chain10, Chain10);
24977           ResNode = CurDAG->getTargetNode(X86::SHUFPSrm, MVT::v4i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp6, Chain10);
24978           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
24979           SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
24980           AddHandleReplacement(N10.Val, 1, ResNode, 1);
24981           Result = SDOperand(ResNode, N.ResNo);
24982           return;
24983         }
24984       }
24985     }
24986   }
24987 
24988   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2), (build_vector:v4i16)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
24989   // Emits: (SHUFPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:v4i16):$src3))
24990   // Pattern complexity = 17  cost = 1
24991   if ((Subtarget->hasSSE1())) {
24992     N0 = N.getOperand(0);
24993     N1 = N.getOperand(1);
24994     if (N1.getOpcode() == ISD::LOAD &&
24995         N1.hasOneUse() &&
24996         !CodeGenMap.count(N1.getValue(0)) &&
24997         !isNonImmUse(N.Val, N1.Val)) {
24998       Chain1 = N1.getOperand(0);
24999       N11 = N1.getOperand(1);
25000       N2 = N.getOperand(2);
25001       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25002           Predicate_SHUFP_shuffle_mask(N2.Val) &&
25003           N.Val->getValueType(0) == MVT::v4f32 &&
25004           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25005         Select(Tmp0, Tmp0);
25006         Select(Tmp1, Tmp1);
25007         Select(Tmp2, Tmp2);
25008         Select(Tmp3, Tmp3);
25009         Select(Tmp4, N0);
25010         Tmp5 = N2;
25011         Tmp6 = Transform_SHUFFLE_get_shuf_imm(Tmp5.Val);
25012         Select(Chain1, Chain1);
25013         ResNode = CurDAG->getTargetNode(X86::SHUFPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp6, Chain1);
25014         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25015         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25016         AddHandleReplacement(N1.Val, 1, ResNode, 1);
25017         Result = SDOperand(ResNode, N.ResNo);
25018         return;
25019       }
25020     }
25021   }
25022 
25023   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2), (build_vector:v2i32)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
25024   // Emits: (SHUFPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2, (SHUFFLE_get_shuf_imm:i8 (build_vector:v2i32):$src3))
25025   // Pattern complexity = 17  cost = 1
25026   if ((Subtarget->hasSSE2())) {
25027     N0 = N.getOperand(0);
25028     N1 = N.getOperand(1);
25029     if (N1.getOpcode() == ISD::LOAD &&
25030         N1.hasOneUse() &&
25031         !CodeGenMap.count(N1.getValue(0)) &&
25032         !isNonImmUse(N.Val, N1.Val)) {
25033       Chain1 = N1.getOperand(0);
25034       N11 = N1.getOperand(1);
25035       N2 = N.getOperand(2);
25036       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25037           Predicate_SHUFP_shuffle_mask(N2.Val) &&
25038           N.Val->getValueType(0) == MVT::v2f64 &&
25039           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25040         Select(Tmp0, Tmp0);
25041         Select(Tmp1, Tmp1);
25042         Select(Tmp2, Tmp2);
25043         Select(Tmp3, Tmp3);
25044         Select(Tmp4, N0);
25045         Tmp5 = N2;
25046         Tmp6 = Transform_SHUFFLE_get_shuf_imm(Tmp5.Val);
25047         Select(Chain1, Chain1);
25048         ResNode = CurDAG->getTargetNode(X86::SHUFPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Tmp6, Chain1);
25049         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25050         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25051         AddHandleReplacement(N1.Val, 1, ResNode, 1);
25052         Result = SDOperand(ResNode, N.ResNo);
25053         return;
25054       }
25055     }
25056   }
25057 
25058   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2), (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25059   // Emits: (UNPCKHPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
25060   // Pattern complexity = 17  cost = 1
25061   if ((Subtarget->hasSSE1())) {
25062     N0 = N.getOperand(0);
25063     N1 = N.getOperand(1);
25064     if (N1.getOpcode() == ISD::LOAD &&
25065         N1.hasOneUse() &&
25066         !CodeGenMap.count(N1.getValue(0)) &&
25067         !isNonImmUse(N.Val, N1.Val)) {
25068       Chain1 = N1.getOperand(0);
25069       N11 = N1.getOperand(1);
25070       N2 = N.getOperand(2);
25071       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25072           Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25073           N.Val->getValueType(0) == MVT::v4f32 &&
25074           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25075         Select(Tmp0, Tmp0);
25076         Select(Tmp1, Tmp1);
25077         Select(Tmp2, Tmp2);
25078         Select(Tmp3, Tmp3);
25079         Select(Tmp4, N0);
25080         Select(Chain1, Chain1);
25081         ResNode = CurDAG->getTargetNode(X86::UNPCKHPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25082         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25083         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25084         AddHandleReplacement(N1.Val, 1, ResNode, 1);
25085         Result = SDOperand(ResNode, N.ResNo);
25086         return;
25087       }
25088     }
25089   }
25090 
25091   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25092   // Emits: (UNPCKHPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
25093   // Pattern complexity = 17  cost = 1
25094   if ((Subtarget->hasSSE2())) {
25095     N0 = N.getOperand(0);
25096     N1 = N.getOperand(1);
25097     if (N1.getOpcode() == ISD::LOAD &&
25098         N1.hasOneUse() &&
25099         !CodeGenMap.count(N1.getValue(0)) &&
25100         !isNonImmUse(N.Val, N1.Val)) {
25101       Chain1 = N1.getOperand(0);
25102       N11 = N1.getOperand(1);
25103       N2 = N.getOperand(2);
25104       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25105           Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25106           N.Val->getValueType(0) == MVT::v2f64 &&
25107           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25108         Select(Tmp0, Tmp0);
25109         Select(Tmp1, Tmp1);
25110         Select(Tmp2, Tmp2);
25111         Select(Tmp3, Tmp3);
25112         Select(Tmp4, N0);
25113         Select(Chain1, Chain1);
25114         ResNode = CurDAG->getTargetNode(X86::UNPCKHPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25115         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25116         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25117         AddHandleReplacement(N1.Val, 1, ResNode, 1);
25118         Result = SDOperand(ResNode, N.ResNo);
25119         return;
25120       }
25121     }
25122   }
25123 
25124   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (load:v4f32 addr:i32:$src2), (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25125   // Emits: (UNPCKLPSrm:v4f32 VR128:v4f32:$src1, addr:i32:$src2)
25126   // Pattern complexity = 17  cost = 1
25127   if ((Subtarget->hasSSE1())) {
25128     N0 = N.getOperand(0);
25129     N1 = N.getOperand(1);
25130     if (N1.getOpcode() == ISD::LOAD &&
25131         N1.hasOneUse() &&
25132         !CodeGenMap.count(N1.getValue(0)) &&
25133         !isNonImmUse(N.Val, N1.Val)) {
25134       Chain1 = N1.getOperand(0);
25135       N11 = N1.getOperand(1);
25136       N2 = N.getOperand(2);
25137       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25138           Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25139           N.Val->getValueType(0) == MVT::v4f32 &&
25140           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25141         Select(Tmp0, Tmp0);
25142         Select(Tmp1, Tmp1);
25143         Select(Tmp2, Tmp2);
25144         Select(Tmp3, Tmp3);
25145         Select(Tmp4, N0);
25146         Select(Chain1, Chain1);
25147         ResNode = CurDAG->getTargetNode(X86::UNPCKLPSrm, MVT::v4f32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25148         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25149         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25150         AddHandleReplacement(N1.Val, 1, ResNode, 1);
25151         Result = SDOperand(ResNode, N.ResNo);
25152         return;
25153       }
25154     }
25155   }
25156   if ((Subtarget->hasSSE2())) {
25157     N0 = N.getOperand(0);
25158     N1 = N.getOperand(1);
25159     if (N1.getOpcode() == ISD::LOAD &&
25160         N1.hasOneUse() &&
25161         !CodeGenMap.count(N1.getValue(0)) &&
25162         !isNonImmUse(N.Val, N1.Val)) {
25163       Chain1 = N1.getOperand(0);
25164       N11 = N1.getOperand(1);
25165       N2 = N.getOperand(2);
25166       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25167         if (Predicate_UNPCKL_shuffle_mask(N2.Val)) {
25168 
25169           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, (load:v2f64 addr:i32:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25170           // Emits: (UNPCKLPDrm:v2f64 VR128:v2f64:$src1, addr:i32:$src2)
25171           // Pattern complexity = 17  cost = 1
25172           if (N.Val->getValueType(0) == MVT::v2f64 &&
25173               SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25174             Select(Tmp0, Tmp0);
25175             Select(Tmp1, Tmp1);
25176             Select(Tmp2, Tmp2);
25177             Select(Tmp3, Tmp3);
25178             Select(Tmp4, N0);
25179             Select(Chain1, Chain1);
25180             ResNode = CurDAG->getTargetNode(X86::UNPCKLPDrm, MVT::v2f64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25181             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25182             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25183             AddHandleReplacement(N1.Val, 1, ResNode, 1);
25184             Result = SDOperand(ResNode, N.ResNo);
25185             return;
25186           }
25187 
25188           // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25189           // Emits: (PUNPCKLQDQrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
25190           // Pattern complexity = 17  cost = 1
25191           if (N.Val->getValueType(0) == MVT::v2i64 &&
25192               SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25193             Select(Tmp0, Tmp0);
25194             Select(Tmp1, Tmp1);
25195             Select(Tmp2, Tmp2);
25196             Select(Tmp3, Tmp3);
25197             Select(Tmp4, N0);
25198             Select(Chain1, Chain1);
25199             ResNode = CurDAG->getTargetNode(X86::PUNPCKLQDQrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25200             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25201             SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25202             AddHandleReplacement(N1.Val, 1, ResNode, 1);
25203             Result = SDOperand(ResNode, N.ResNo);
25204             return;
25205           }
25206         }
25207 
25208         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2), (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25209         // Emits: (PUNPCKHQDQrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
25210         // Pattern complexity = 17  cost = 1
25211         if (Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25212             N.Val->getValueType(0) == MVT::v2i64 &&
25213             SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
25214           Select(Tmp0, Tmp0);
25215           Select(Tmp1, Tmp1);
25216           Select(Tmp2, Tmp2);
25217           Select(Tmp3, Tmp3);
25218           Select(Tmp4, N0);
25219           Select(Chain1, Chain1);
25220           ResNode = CurDAG->getTargetNode(X86::PUNPCKHQDQrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
25221           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25222           SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
25223           AddHandleReplacement(N1.Val, 1, ResNode, 1);
25224           Result = SDOperand(ResNode, N.ResNo);
25225           return;
25226         }
25227       }
25228     }
25229     if (N1.getOpcode() == ISD::UNDEF) {
25230       N2 = N.getOperand(2);
25231       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25232 
25233         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_PSHUFD_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src2)
25234         // Emits: (PSHUFDri:v4i32 VR128:v4i32:$src1, (SHUFFLE_get_shuf_imm:i8 (build_vector:v4i16):$src2))
25235         // Pattern complexity = 7  cost = 1
25236         if (Predicate_PSHUFD_shuffle_mask(N2.Val) &&
25237             N.Val->getValueType(0) == MVT::v4i32) {
25238           Select(Tmp0, N0);
25239           Tmp1 = N2;
25240           Tmp2 = Transform_SHUFFLE_get_shuf_imm(Tmp1.Val);
25241           if (N.Val->hasOneUse()) {
25242             Result = CurDAG->SelectNodeTo(N.Val, X86::PSHUFDri, MVT::v4i32, Tmp0, Tmp2);
25243           } else {
25244             ResNode = CurDAG->getTargetNode(X86::PSHUFDri, MVT::v4i32, Tmp0, Tmp2);
25245             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25246             Result = SDOperand(ResNode, 0);
25247           }
25248           return;
25249         }
25250 
25251         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFHW_shuffle_mask>><<X:SHUFFLE_get_pshufhw_imm>>:$src2)
25252         // Emits: (PSHUFHWri:v8i16 VR128:v8i16:$src1, (SHUFFLE_get_pshufhw_imm:i8 (build_vector:v8i8):$src2))
25253         // Pattern complexity = 7  cost = 1
25254         if (Predicate_PSHUFHW_shuffle_mask(N2.Val) &&
25255             N.Val->getValueType(0) == MVT::v8i16) {
25256           Select(Tmp0, N0);
25257           Tmp1 = N2;
25258           Tmp2 = Transform_SHUFFLE_get_pshufhw_imm(Tmp1.Val);
25259           if (N.Val->hasOneUse()) {
25260             Result = CurDAG->SelectNodeTo(N.Val, X86::PSHUFHWri, MVT::v8i16, Tmp0, Tmp2);
25261           } else {
25262             ResNode = CurDAG->getTargetNode(X86::PSHUFHWri, MVT::v8i16, Tmp0, Tmp2);
25263             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25264             Result = SDOperand(ResNode, 0);
25265           }
25266           return;
25267         }
25268 
25269         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_PSHUFLW_shuffle_mask>><<X:SHUFFLE_get_pshuflw_imm>>:$src2)
25270         // Emits: (PSHUFLWri:v8i16 VR128:v8i16:$src1, (SHUFFLE_get_pshuflw_imm:i32 (build_vector:v8i8):$src2))
25271         // Pattern complexity = 7  cost = 1
25272         if (Predicate_PSHUFLW_shuffle_mask(N2.Val) &&
25273             N.Val->getValueType(0) == MVT::v8i16) {
25274           Select(Tmp0, N0);
25275           Tmp1 = N2;
25276           Tmp2 = Transform_SHUFFLE_get_pshuflw_imm(Tmp1.Val);
25277           if (N.Val->hasOneUse()) {
25278             Result = CurDAG->SelectNodeTo(N.Val, X86::PSHUFLWri, MVT::v8i16, Tmp0, Tmp2);
25279           } else {
25280             ResNode = CurDAG->getTargetNode(X86::PSHUFLWri, MVT::v8i16, Tmp0, Tmp2);
25281             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25282             Result = SDOperand(ResNode, 0);
25283           }
25284           return;
25285         }
25286         if (Predicate_SSE_splat_mask(N2.Val)) {
25287 
25288           // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src, (undef:v2f64), (build_vector:v2i32)<<P:Predicate_SSE_splat_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25289           // Emits: (UNPCKLPDrr:v2f64 VR128:v16i8:$src, VR128:v16i8:$src)
25290           // Pattern complexity = 7  cost = 1
25291           if (N.Val->getValueType(0) == MVT::v2f64) {
25292             Select(Tmp0, N0);
25293             if (N.Val->hasOneUse()) {
25294               Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKLPDrr, MVT::v2f64, Tmp0, Tmp0);
25295             } else {
25296               ResNode = CurDAG->getTargetNode(X86::UNPCKLPDrr, MVT::v2f64, Tmp0, Tmp0);
25297               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25298               Result = SDOperand(ResNode, 0);
25299             }
25300             return;
25301           }
25302 
25303           // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src, (undef:v2i64), (build_vector:v2i32)<<P:Predicate_SSE_splat_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25304           // Emits: (PUNPCKLQDQrr:v2i64 VR128:v16i8:$src, VR128:v16i8:$src)
25305           // Pattern complexity = 7  cost = 1
25306           if (N.Val->getValueType(0) == MVT::v2i64) {
25307             Select(Tmp0, N0);
25308             if (N.Val->hasOneUse()) {
25309               Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLQDQrr, MVT::v2i64, Tmp0, Tmp0);
25310             } else {
25311               ResNode = CurDAG->getTargetNode(X86::PUNPCKLQDQrr, MVT::v2i64, Tmp0, Tmp0);
25312               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25313               Result = SDOperand(ResNode, 0);
25314             }
25315             return;
25316           }
25317         }
25318       }
25319     }
25320   }
25321   if ((Subtarget->hasSSE1())) {
25322     N0 = N.getOperand(0);
25323     N1 = N.getOperand(1);
25324     if (N1.getOpcode() == ISD::UNDEF) {
25325       N2 = N.getOperand(2);
25326       if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25327 
25328         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SSE_splat_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25329         // Emits: (SHUFPSrr:v4f32 VR128:v16i8:$src, VR128:v16i8:$src, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_SSE_splat_mask>>:$sm))
25330         // Pattern complexity = 7  cost = 1
25331         if (Predicate_SSE_splat_mask(N2.Val) &&
25332             N.Val->getValueType(0) == MVT::v4f32) {
25333           Select(Tmp0, N0);
25334           Tmp1 = N2;
25335           Tmp2 = Transform_SHUFFLE_get_shuf_imm(Tmp1.Val);
25336           if (N.Val->hasOneUse()) {
25337             Result = CurDAG->SelectNodeTo(N.Val, X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp0, Tmp2);
25338           } else {
25339             ResNode = CurDAG->getTargetNode(X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp0, Tmp2);
25340             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25341             Result = SDOperand(ResNode, 0);
25342           }
25343           return;
25344         }
25345 
25346         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_SHUFP_unary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25347         // Emits: (SHUFPSrr:v4f32 VR128:v16i8:$src1, VR128:v16i8:$src1, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_SHUFP_unary_shuffle_mask>>:$sm))
25348         // Pattern complexity = 7  cost = 1
25349         if (Predicate_SHUFP_unary_shuffle_mask(N2.Val) &&
25350             N.Val->getValueType(0) == MVT::v4f32) {
25351           Select(Tmp0, N0);
25352           Tmp1 = N2;
25353           Tmp2 = Transform_SHUFFLE_get_shuf_imm(Tmp1.Val);
25354           if (N.Val->hasOneUse()) {
25355             Result = CurDAG->SelectNodeTo(N.Val, X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp0, Tmp2);
25356           } else {
25357             ResNode = CurDAG->getTargetNode(X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp0, Tmp2);
25358             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25359             Result = SDOperand(ResNode, 0);
25360           }
25361           return;
25362         }
25363       }
25364     }
25365   }
25366   if ((Subtarget->hasSSE2())) {
25367     N0 = N.getOperand(0);
25368     N1 = N.getOperand(1);
25369     if (N1.getOpcode() == ISD::UNDEF) {
25370       N2 = N.getOperand(2);
25371       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25372           Predicate_UNPCKL_v_undef_shuffle_mask(N2.Val)) {
25373 
25374         // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src, (undef:v4f32), (build_vector:v4i16)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25375         // Emits: (UNPCKLPSrr:v16i8 VR128:v16i8:$src, VR128:v16i8:$src)
25376         // Pattern complexity = 7  cost = 1
25377         if (N.Val->getValueType(0) == MVT::v4f32) {
25378           Select(Tmp0, N0);
25379           if (N.Val->hasOneUse()) {
25380             Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKLPSrr, MVT::v16i8, Tmp0, Tmp0);
25381           } else {
25382             ResNode = CurDAG->getTargetNode(X86::UNPCKLPSrr, MVT::v16i8, Tmp0, Tmp0);
25383             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25384             Result = SDOperand(ResNode, 0);
25385           }
25386           return;
25387         }
25388 
25389         // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src, (undef:v16i8), (build_vector:v16i8)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25390         // Emits: (PUNPCKLBWrr:v16i8 VR128:v16i8:$src, VR128:v16i8:$src)
25391         // Pattern complexity = 7  cost = 1
25392         if (N.Val->getValueType(0) == MVT::v16i8) {
25393           Select(Tmp0, N0);
25394           if (N.Val->hasOneUse()) {
25395             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLBWrr, MVT::v16i8, Tmp0, Tmp0);
25396           } else {
25397             ResNode = CurDAG->getTargetNode(X86::PUNPCKLBWrr, MVT::v16i8, Tmp0, Tmp0);
25398             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25399             Result = SDOperand(ResNode, 0);
25400           }
25401           return;
25402         }
25403 
25404         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src, (undef:v8i16), (build_vector:v8i8)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25405         // Emits: (PUNPCKLWDrr:v16i8 VR128:v16i8:$src, VR128:v16i8:$src)
25406         // Pattern complexity = 7  cost = 1
25407         if (N.Val->getValueType(0) == MVT::v8i16) {
25408           Select(Tmp0, N0);
25409           if (N.Val->hasOneUse()) {
25410             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLWDrr, MVT::v16i8, Tmp0, Tmp0);
25411           } else {
25412             ResNode = CurDAG->getTargetNode(X86::PUNPCKLWDrr, MVT::v16i8, Tmp0, Tmp0);
25413             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25414             Result = SDOperand(ResNode, 0);
25415           }
25416           return;
25417         }
25418       }
25419     }
25420   }
25421   if ((Subtarget->hasSSE1())) {
25422     N0 = N.getOperand(0);
25423     N1 = N.getOperand(1);
25424 
25425     // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src, (undef:v4i32), (build_vector:v4i16)<<P:Predicate_UNPCKL_v_undef_shuffle_mask>>)
25426     // Emits: (PUNPCKLDQrr:v16i8 VR128:v16i8:$src, VR128:v16i8:$src)
25427     // Pattern complexity = 7  cost = 1
25428     if (N1.getOpcode() == ISD::UNDEF) {
25429       N2 = N.getOperand(2);
25430       if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25431           Predicate_UNPCKL_v_undef_shuffle_mask(N2.Val) &&
25432           N.Val->getValueType(0) == MVT::v4i32) {
25433         Select(Tmp0, N0);
25434         if (N.Val->hasOneUse()) {
25435           Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLDQrr, MVT::v16i8, Tmp0, Tmp0);
25436         } else {
25437           ResNode = CurDAG->getTargetNode(X86::PUNPCKLDQrr, MVT::v16i8, Tmp0, Tmp0);
25438           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25439           Result = SDOperand(ResNode, 0);
25440         }
25441         return;
25442       }
25443     }
25444     N2 = N.getOperand(2);
25445     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25446 
25447       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVLHPS_shuffle_mask>>)
25448       // Emits: (MOVLHPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25449       // Pattern complexity = 5  cost = 1
25450       if (Predicate_MOVLHPS_shuffle_mask(N2.Val) &&
25451           N.Val->getValueType(0) == MVT::v4f32) {
25452         Select(Tmp0, N0);
25453         Select(Tmp1, N1);
25454         if (N.Val->hasOneUse()) {
25455           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLHPSrr, MVT::v4f32, Tmp0, Tmp1);
25456         } else {
25457           ResNode = CurDAG->getTargetNode(X86::MOVLHPSrr, MVT::v4f32, Tmp0, Tmp1);
25458           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25459           Result = SDOperand(ResNode, 0);
25460         }
25461         return;
25462       }
25463 
25464       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVHLPS_shuffle_mask>>)
25465       // Emits: (MOVHLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25466       // Pattern complexity = 5  cost = 1
25467       if (Predicate_MOVHLPS_shuffle_mask(N2.Val) &&
25468           N.Val->getValueType(0) == MVT::v4f32) {
25469         Select(Tmp0, N0);
25470         Select(Tmp1, N1);
25471         if (N.Val->hasOneUse()) {
25472           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVHLPSrr, MVT::v4f32, Tmp0, Tmp1);
25473         } else {
25474           ResNode = CurDAG->getTargetNode(X86::MOVHLPSrr, MVT::v4f32, Tmp0, Tmp1);
25475           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25476           Result = SDOperand(ResNode, 0);
25477         }
25478         return;
25479       }
25480 
25481       // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
25482       // Emits: (SHUFPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:v4i16):$src3))
25483       // Pattern complexity = 5  cost = 1
25484       if (Predicate_SHUFP_shuffle_mask(N2.Val) &&
25485           N.Val->getValueType(0) == MVT::v4f32) {
25486         Select(Tmp0, N0);
25487         Select(Tmp1, N1);
25488         Tmp2 = N2;
25489         Tmp3 = Transform_SHUFFLE_get_shuf_imm(Tmp2.Val);
25490         if (N.Val->hasOneUse()) {
25491           Result = CurDAG->SelectNodeTo(N.Val, X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp1, Tmp3);
25492         } else {
25493           ResNode = CurDAG->getTargetNode(X86::SHUFPSrr, MVT::v4f32, Tmp0, Tmp1, Tmp3);
25494           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25495           Result = SDOperand(ResNode, 0);
25496         }
25497         return;
25498       }
25499     }
25500   }
25501 
25502   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_SHUFP_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$src3)
25503   // Emits: (SHUFPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (SHUFFLE_get_shuf_imm:i8 (build_vector:v2i32):$src3))
25504   // Pattern complexity = 5  cost = 1
25505   if ((Subtarget->hasSSE2())) {
25506     N0 = N.getOperand(0);
25507     N1 = N.getOperand(1);
25508     N2 = N.getOperand(2);
25509     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25510         Predicate_SHUFP_shuffle_mask(N2.Val) &&
25511         N.Val->getValueType(0) == MVT::v2f64) {
25512       Select(Tmp0, N0);
25513       Select(Tmp1, N1);
25514       Tmp2 = N2;
25515       Tmp3 = Transform_SHUFFLE_get_shuf_imm(Tmp2.Val);
25516       if (N.Val->hasOneUse()) {
25517         Result = CurDAG->SelectNodeTo(N.Val, X86::SHUFPDrr, MVT::v2f64, Tmp0, Tmp1, Tmp3);
25518       } else {
25519         ResNode = CurDAG->getTargetNode(X86::SHUFPDrr, MVT::v2f64, Tmp0, Tmp1, Tmp3);
25520         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25521         Result = SDOperand(ResNode, 0);
25522       }
25523       return;
25524     }
25525   }
25526 
25527   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25528   // Emits: (UNPCKHPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25529   // Pattern complexity = 5  cost = 1
25530   if ((Subtarget->hasSSE1())) {
25531     N0 = N.getOperand(0);
25532     N1 = N.getOperand(1);
25533     N2 = N.getOperand(2);
25534     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25535         Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25536         N.Val->getValueType(0) == MVT::v4f32) {
25537       Select(Tmp0, N0);
25538       Select(Tmp1, N1);
25539       if (N.Val->hasOneUse()) {
25540         Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKHPSrr, MVT::v4f32, Tmp0, Tmp1);
25541       } else {
25542         ResNode = CurDAG->getTargetNode(X86::UNPCKHPSrr, MVT::v4f32, Tmp0, Tmp1);
25543         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25544         Result = SDOperand(ResNode, 0);
25545       }
25546       return;
25547     }
25548   }
25549 
25550   // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25551   // Emits: (UNPCKHPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25552   // Pattern complexity = 5  cost = 1
25553   if ((Subtarget->hasSSE2())) {
25554     N0 = N.getOperand(0);
25555     N1 = N.getOperand(1);
25556     N2 = N.getOperand(2);
25557     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25558         Predicate_UNPCKH_shuffle_mask(N2.Val) &&
25559         N.Val->getValueType(0) == MVT::v2f64) {
25560       Select(Tmp0, N0);
25561       Select(Tmp1, N1);
25562       if (N.Val->hasOneUse()) {
25563         Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKHPDrr, MVT::v2f64, Tmp0, Tmp1);
25564       } else {
25565         ResNode = CurDAG->getTargetNode(X86::UNPCKHPDrr, MVT::v2f64, Tmp0, Tmp1);
25566         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25567         Result = SDOperand(ResNode, 0);
25568       }
25569       return;
25570     }
25571   }
25572 
25573   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25574   // Emits: (UNPCKLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25575   // Pattern complexity = 5  cost = 1
25576   if ((Subtarget->hasSSE1())) {
25577     N0 = N.getOperand(0);
25578     N1 = N.getOperand(1);
25579     N2 = N.getOperand(2);
25580     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25581         Predicate_UNPCKL_shuffle_mask(N2.Val) &&
25582         N.Val->getValueType(0) == MVT::v4f32) {
25583       Select(Tmp0, N0);
25584       Select(Tmp1, N1);
25585       if (N.Val->hasOneUse()) {
25586         Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKLPSrr, MVT::v4f32, Tmp0, Tmp1);
25587       } else {
25588         ResNode = CurDAG->getTargetNode(X86::UNPCKLPSrr, MVT::v4f32, Tmp0, Tmp1);
25589         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25590         Result = SDOperand(ResNode, 0);
25591       }
25592       return;
25593     }
25594   }
25595   if ((Subtarget->hasSSE2())) {
25596     N0 = N.getOperand(0);
25597     N1 = N.getOperand(1);
25598     N2 = N.getOperand(2);
25599     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25600       if (Predicate_UNPCKL_shuffle_mask(N2.Val)) {
25601 
25602         // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25603         // Emits: (UNPCKLPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25604         // Pattern complexity = 5  cost = 1
25605         if (N.Val->getValueType(0) == MVT::v2f64) {
25606           Select(Tmp0, N0);
25607           Select(Tmp1, N1);
25608           if (N.Val->hasOneUse()) {
25609             Result = CurDAG->SelectNodeTo(N.Val, X86::UNPCKLPDrr, MVT::v2f64, Tmp0, Tmp1);
25610           } else {
25611             ResNode = CurDAG->getTargetNode(X86::UNPCKLPDrr, MVT::v2f64, Tmp0, Tmp1);
25612             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25613             Result = SDOperand(ResNode, 0);
25614           }
25615           return;
25616         }
25617 
25618         // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2, (build_vector:v16i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
25619         // Emits: (PUNPCKLBWrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
25620         // Pattern complexity = 5  cost = 1
25621         if (N.Val->getValueType(0) == MVT::v16i8) {
25622           Select(Tmp0, N0);
25623           Select(Tmp1, N1);
25624           if (N.Val->hasOneUse()) {
25625             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLBWrr, MVT::v16i8, Tmp0, Tmp1);
25626           } else {
25627             ResNode = CurDAG->getTargetNode(X86::PUNPCKLBWrr, MVT::v16i8, Tmp0, Tmp1);
25628             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25629             Result = SDOperand(ResNode, 0);
25630           }
25631           return;
25632         }
25633 
25634         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2, (build_vector:v8i8)<<P:Predicate_UNPCKL_shuffle_mask>>)
25635         // Emits: (PUNPCKLWDrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
25636         // Pattern complexity = 5  cost = 1
25637         if (N.Val->getValueType(0) == MVT::v8i16) {
25638           Select(Tmp0, N0);
25639           Select(Tmp1, N1);
25640           if (N.Val->hasOneUse()) {
25641             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLWDrr, MVT::v8i16, Tmp0, Tmp1);
25642           } else {
25643             ResNode = CurDAG->getTargetNode(X86::PUNPCKLWDrr, MVT::v8i16, Tmp0, Tmp1);
25644             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25645             Result = SDOperand(ResNode, 0);
25646           }
25647           return;
25648         }
25649 
25650         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKL_shuffle_mask>>)
25651         // Emits: (PUNPCKLDQrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
25652         // Pattern complexity = 5  cost = 1
25653         if (N.Val->getValueType(0) == MVT::v4i32) {
25654           Select(Tmp0, N0);
25655           Select(Tmp1, N1);
25656           if (N.Val->hasOneUse()) {
25657             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLDQrr, MVT::v4i32, Tmp0, Tmp1);
25658           } else {
25659             ResNode = CurDAG->getTargetNode(X86::PUNPCKLDQrr, MVT::v4i32, Tmp0, Tmp1);
25660             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25661             Result = SDOperand(ResNode, 0);
25662           }
25663           return;
25664         }
25665 
25666         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKL_shuffle_mask>>)
25667         // Emits: (PUNPCKLQDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
25668         // Pattern complexity = 5  cost = 1
25669         if (N.Val->getValueType(0) == MVT::v2i64) {
25670           Select(Tmp0, N0);
25671           Select(Tmp1, N1);
25672           if (N.Val->hasOneUse()) {
25673             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKLQDQrr, MVT::v2i64, Tmp0, Tmp1);
25674           } else {
25675             ResNode = CurDAG->getTargetNode(X86::PUNPCKLQDQrr, MVT::v2i64, Tmp0, Tmp1);
25676             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25677             Result = SDOperand(ResNode, 0);
25678           }
25679           return;
25680         }
25681       }
25682       if (Predicate_UNPCKH_shuffle_mask(N2.Val)) {
25683 
25684         // Pattern: (vector_shuffle:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2, (build_vector:v16i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
25685         // Emits: (PUNPCKHBWrr:v16i8 VR128:v16i8:$src1, VR128:v16i8:$src2)
25686         // Pattern complexity = 5  cost = 1
25687         if (N.Val->getValueType(0) == MVT::v16i8) {
25688           Select(Tmp0, N0);
25689           Select(Tmp1, N1);
25690           if (N.Val->hasOneUse()) {
25691             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKHBWrr, MVT::v16i8, Tmp0, Tmp1);
25692           } else {
25693             ResNode = CurDAG->getTargetNode(X86::PUNPCKHBWrr, MVT::v16i8, Tmp0, Tmp1);
25694             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25695             Result = SDOperand(ResNode, 0);
25696           }
25697           return;
25698         }
25699 
25700         // Pattern: (vector_shuffle:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2, (build_vector:v8i8)<<P:Predicate_UNPCKH_shuffle_mask>>)
25701         // Emits: (PUNPCKHWDrr:v8i16 VR128:v8i16:$src1, VR128:v8i16:$src2)
25702         // Pattern complexity = 5  cost = 1
25703         if (N.Val->getValueType(0) == MVT::v8i16) {
25704           Select(Tmp0, N0);
25705           Select(Tmp1, N1);
25706           if (N.Val->hasOneUse()) {
25707             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKHWDrr, MVT::v8i16, Tmp0, Tmp1);
25708           } else {
25709             ResNode = CurDAG->getTargetNode(X86::PUNPCKHWDrr, MVT::v8i16, Tmp0, Tmp1);
25710             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25711             Result = SDOperand(ResNode, 0);
25712           }
25713           return;
25714         }
25715 
25716         // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_UNPCKH_shuffle_mask>>)
25717         // Emits: (PUNPCKHDQrr:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2)
25718         // Pattern complexity = 5  cost = 1
25719         if (N.Val->getValueType(0) == MVT::v4i32) {
25720           Select(Tmp0, N0);
25721           Select(Tmp1, N1);
25722           if (N.Val->hasOneUse()) {
25723             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKHDQrr, MVT::v4i32, Tmp0, Tmp1);
25724           } else {
25725             ResNode = CurDAG->getTargetNode(X86::PUNPCKHDQrr, MVT::v4i32, Tmp0, Tmp1);
25726             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25727             Result = SDOperand(ResNode, 0);
25728           }
25729           return;
25730         }
25731 
25732         // Pattern: (vector_shuffle:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2, (build_vector:v2i32)<<P:Predicate_UNPCKH_shuffle_mask>>)
25733         // Emits: (PUNPCKHQDQrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
25734         // Pattern complexity = 5  cost = 1
25735         if (N.Val->getValueType(0) == MVT::v2i64) {
25736           Select(Tmp0, N0);
25737           Select(Tmp1, N1);
25738           if (N.Val->hasOneUse()) {
25739             Result = CurDAG->SelectNodeTo(N.Val, X86::PUNPCKHQDQrr, MVT::v2i64, Tmp0, Tmp1);
25740           } else {
25741             ResNode = CurDAG->getTargetNode(X86::PUNPCKHQDQrr, MVT::v2i64, Tmp0, Tmp1);
25742             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25743             Result = SDOperand(ResNode, 0);
25744           }
25745           return;
25746         }
25747       }
25748     }
25749   }
25750 
25751   // Pattern: (vector_shuffle:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2, (build_vector:v4i16)<<P:Predicate_MOVS_shuffle_mask>>)
25752   // Emits: (MOVLPSrr:v4f32 VR128:v4f32:$src1, VR128:v4f32:$src2)
25753   // Pattern complexity = 5  cost = 1
25754   if ((Subtarget->hasSSE1())) {
25755     N0 = N.getOperand(0);
25756     N1 = N.getOperand(1);
25757     N2 = N.getOperand(2);
25758     if (N2.getOpcode() == ISD::BUILD_VECTOR &&
25759         Predicate_MOVS_shuffle_mask(N2.Val) &&
25760         N.Val->getValueType(0) == MVT::v4f32) {
25761       Select(Tmp0, N0);
25762       Select(Tmp1, N1);
25763       if (N.Val->hasOneUse()) {
25764         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLPSrr, MVT::v4f32, Tmp0, Tmp1);
25765       } else {
25766         ResNode = CurDAG->getTargetNode(X86::MOVLPSrr, MVT::v4f32, Tmp0, Tmp1);
25767         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25768         Result = SDOperand(ResNode, 0);
25769       }
25770       return;
25771     }
25772   }
25773   if ((Subtarget->hasSSE2())) {
25774     N0 = N.getOperand(0);
25775     N1 = N.getOperand(1);
25776     N2 = N.getOperand(2);
25777     if (N2.getOpcode() == ISD::BUILD_VECTOR) {
25778 
25779       // Pattern: (vector_shuffle:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2, (build_vector:v2i32)<<P:Predicate_MOVS_shuffle_mask>>)
25780       // Emits: (MOVLPDrr:v2f64 VR128:v2f64:$src1, VR128:v2f64:$src2)
25781       // Pattern complexity = 5  cost = 1
25782       if (Predicate_MOVS_shuffle_mask(N2.Val) &&
25783           N.Val->getValueType(0) == MVT::v2f64) {
25784         Select(Tmp0, N0);
25785         Select(Tmp1, N1);
25786         if (N.Val->hasOneUse()) {
25787           Result = CurDAG->SelectNodeTo(N.Val, X86::MOVLPDrr, MVT::v2f64, Tmp0, Tmp1);
25788         } else {
25789           ResNode = CurDAG->getTargetNode(X86::MOVLPDrr, MVT::v2f64, Tmp0, Tmp1);
25790           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25791           Result = SDOperand(ResNode, 0);
25792         }
25793         return;
25794       }
25795 
25796       // Pattern: (vector_shuffle:v4i32 VR128:v4i32:$src1, VR128:v4i32:$src2, (build_vector:v4i16)<<P:Predicate_PSHUFD_binary_shuffle_mask>><<X:SHUFFLE_get_shuf_imm>>:$sm)
25797       // Emits: (SHUFPSrr:v4i32 VR128:v16i8:$src1, VR128:v16i8:$src2, (SHUFFLE_get_shuf_imm:i32 (build_vector:i32)<<P:Predicate_PSHUFD_binary_shuffle_mask>>:$sm))
25798       // Pattern complexity = 5  cost = 1
25799       if (Predicate_PSHUFD_binary_shuffle_mask(N2.Val) &&
25800           N.Val->getValueType(0) == MVT::v4i32) {
25801         Select(Tmp0, N0);
25802         Select(Tmp1, N1);
25803         Tmp2 = N2;
25804         Tmp3 = Transform_SHUFFLE_get_shuf_imm(Tmp2.Val);
25805         if (N.Val->hasOneUse()) {
25806           Result = CurDAG->SelectNodeTo(N.Val, X86::SHUFPSrr, MVT::v4i32, Tmp0, Tmp1, Tmp3);
25807         } else {
25808           ResNode = CurDAG->getTargetNode(X86::SHUFPSrr, MVT::v4i32, Tmp0, Tmp1, Tmp3);
25809           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
25810           Result = SDOperand(ResNode, 0);
25811         }
25812         return;
25813       }
25814     }
25815   }
25816   std::cerr << "Cannot yet select: ";
25817   N.Val->dump(CurDAG);
25818   std::cerr << '\n';
25819   abort();
25820 }
25821 
25822 void Select_xor(SDOperand &Result, SDOperand N) {
25823   SDOperand Chain0(0, 0);
25824   SDOperand Chain00(0, 0);
25825   SDOperand Chain1(0, 0);
25826   SDOperand Chain10(0, 0);
25827   SDOperand N0(0, 0);
25828   SDOperand N00(0, 0);
25829   SDOperand N001(0, 0);
25830   SDOperand N01(0, 0);
25831   SDOperand N1(0, 0);
25832   SDOperand N10(0, 0);
25833   SDOperand N101(0, 0);
25834   SDOperand N11(0, 0);
25835   SDOperand Tmp0(0, 0);
25836   SDOperand Tmp1(0, 0);
25837   SDOperand Tmp2(0, 0);
25838   SDOperand Tmp3(0, 0);
25839   SDOperand Tmp4(0, 0);
25840   SDNode *ResNode;
25841   if ((Subtarget->hasSSE2())) {
25842     N0 = N.getOperand(0);
25843     if (N0.getOpcode() == ISD::BIT_CONVERT) {
25844 
25845       // Pattern: (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)))
25846       // Emits: (XORPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
25847       // Pattern complexity = 18  cost = 1
25848       {
25849         N00 = N0.getOperand(0);
25850         N1 = N.getOperand(1);
25851         if (N1.getOpcode() == ISD::BIT_CONVERT &&
25852             N1.hasOneUse() &&
25853             !CodeGenMap.count(N1.getValue(0))) {
25854           N10 = N1.getOperand(0);
25855           if (N10.getOpcode() == ISD::LOAD &&
25856               N10.hasOneUse() &&
25857               !CodeGenMap.count(N10.getValue(0))) {
25858             Chain10 = N10.getOperand(0);
25859             N101 = N10.getOperand(1);
25860             if (N.Val->getValueType(0) == MVT::v2i64 &&
25861                 N00.Val->getValueType(0) == MVT::v2f64 &&
25862                 N10.Val->getValueType(0) == MVT::v2f64 &&
25863                 SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
25864               Select(Tmp0, Tmp0);
25865               Select(Tmp1, Tmp1);
25866               Select(Tmp2, Tmp2);
25867               Select(Tmp3, Tmp3);
25868               Select(Tmp4, N00);
25869               Select(Chain10, Chain10);
25870               ResNode = CurDAG->getTargetNode(X86::XORPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
25871               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25872               SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
25873               AddHandleReplacement(N10.Val, 1, ResNode, 1);
25874               Result = SDOperand(ResNode, N.ResNo);
25875               return;
25876             }
25877           }
25878         }
25879       }
25880 
25881       // Pattern: (xor:v2i64 (bitconvert:v2i64 (load:v2f64 addr:i32:$src2)), (bitconvert:v2i64 VR128:v2f64:$src1))
25882       // Emits: (XORPDrm:v2i64 VR128:v2f64:$src1, addr:i32:$src2)
25883       // Pattern complexity = 18  cost = 1
25884       if (N0.hasOneUse() &&
25885           !CodeGenMap.count(N0.getValue(0))) {
25886         N00 = N0.getOperand(0);
25887         if (N00.getOpcode() == ISD::LOAD &&
25888             N00.hasOneUse() &&
25889             !CodeGenMap.count(N00.getValue(0))) {
25890           Chain00 = N00.getOperand(0);
25891           N001 = N00.getOperand(1);
25892           N1 = N.getOperand(1);
25893           if (N1.getOpcode() == ISD::BIT_CONVERT) {
25894             N10 = N1.getOperand(0);
25895             if (N.Val->getValueType(0) == MVT::v2i64 &&
25896                 N00.Val->getValueType(0) == MVT::v2f64 &&
25897                 N10.Val->getValueType(0) == MVT::v2f64 &&
25898                 SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
25899               Select(Tmp0, Tmp0);
25900               Select(Tmp1, Tmp1);
25901               Select(Tmp2, Tmp2);
25902               Select(Tmp3, Tmp3);
25903               Select(Tmp4, N10);
25904               Select(Chain00, Chain00);
25905               ResNode = CurDAG->getTargetNode(X86::XORPDrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
25906               SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25907               SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
25908               AddHandleReplacement(N00.Val, 1, ResNode, 1);
25909               Result = SDOperand(ResNode, N.ResNo);
25910               return;
25911             }
25912           }
25913         }
25914       }
25915     }
25916   }
25917   if ((Subtarget->hasSSE1())) {
25918     N0 = N.getOperand(0);
25919 
25920     // Pattern: (xor:v2i64 VR128:v2i64:$src1, (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)))
25921     // Emits: (XORPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
25922     // Pattern complexity = 16  cost = 1
25923     {
25924       N1 = N.getOperand(1);
25925       if (N1.getOpcode() == ISD::BIT_CONVERT &&
25926           N1.hasOneUse() &&
25927           !CodeGenMap.count(N1.getValue(0))) {
25928         N10 = N1.getOperand(0);
25929         if (N10.getOpcode() == ISD::LOAD &&
25930             N10.hasOneUse() &&
25931             !CodeGenMap.count(N10.getValue(0))) {
25932           Chain10 = N10.getOperand(0);
25933           N101 = N10.getOperand(1);
25934           if (N.Val->getValueType(0) == MVT::v2i64 &&
25935               N10.Val->getValueType(0) == MVT::v4f32 &&
25936               SelectAddr(N101, Tmp0, Tmp1, Tmp2, Tmp3)) {
25937             Select(Tmp0, Tmp0);
25938             Select(Tmp1, Tmp1);
25939             Select(Tmp2, Tmp2);
25940             Select(Tmp3, Tmp3);
25941             Select(Tmp4, N0);
25942             Select(Chain10, Chain10);
25943             ResNode = CurDAG->getTargetNode(X86::XORPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain10);
25944             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25945             SelectionDAG::InsertISelMapEntry(CodeGenMap, N10.Val, 1, ResNode, 1);
25946             AddHandleReplacement(N10.Val, 1, ResNode, 1);
25947             Result = SDOperand(ResNode, N.ResNo);
25948             return;
25949           }
25950         }
25951       }
25952     }
25953 
25954     // Pattern: (xor:v2i64 (bitconvert:v2i64 (load:v4f32 addr:i32:$src2)), VR128:v2i64:$src1)
25955     // Emits: (XORPSrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
25956     // Pattern complexity = 16  cost = 1
25957     if (N0.getOpcode() == ISD::BIT_CONVERT &&
25958         N0.hasOneUse() &&
25959         !CodeGenMap.count(N0.getValue(0))) {
25960       N00 = N0.getOperand(0);
25961       if (N00.getOpcode() == ISD::LOAD &&
25962           N00.hasOneUse() &&
25963           !CodeGenMap.count(N00.getValue(0))) {
25964         Chain00 = N00.getOperand(0);
25965         N001 = N00.getOperand(1);
25966         N1 = N.getOperand(1);
25967         if (N.Val->getValueType(0) == MVT::v2i64 &&
25968             N00.Val->getValueType(0) == MVT::v4f32 &&
25969             SelectAddr(N001, Tmp0, Tmp1, Tmp2, Tmp3)) {
25970           Select(Tmp0, Tmp0);
25971           Select(Tmp1, Tmp1);
25972           Select(Tmp2, Tmp2);
25973           Select(Tmp3, Tmp3);
25974           Select(Tmp4, N1);
25975           Select(Chain00, Chain00);
25976           ResNode = CurDAG->getTargetNode(X86::XORPSrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain00);
25977           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
25978           SelectionDAG::InsertISelMapEntry(CodeGenMap, N00.Val, 1, ResNode, 1);
25979           AddHandleReplacement(N00.Val, 1, ResNode, 1);
25980           Result = SDOperand(ResNode, N.ResNo);
25981           return;
25982         }
25983       }
25984     }
25985   }
25986   {
25987     N0 = N.getOperand(0);
25988     N1 = N.getOperand(1);
25989     if (N1.getOpcode() == ISD::LOAD &&
25990         N1.hasOneUse() &&
25991         !CodeGenMap.count(N1.getValue(0)) &&
25992         !isNonImmUse(N.Val, N1.Val)) {
25993       Chain1 = N1.getOperand(0);
25994       N11 = N1.getOperand(1);
25995 
25996       // Pattern: (xor:i8 R8:i8:$src1, (load:i8 addr:i32:$src2))
25997       // Emits: (XOR8rm:i8 R8:i8:$src1, addr:i32:$src2)
25998       // Pattern complexity = 14  cost = 1
25999       if (N.Val->getValueType(0) == MVT::i8 &&
26000           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
26001         Select(Tmp0, Tmp0);
26002         Select(Tmp1, Tmp1);
26003         Select(Tmp2, Tmp2);
26004         Select(Tmp3, Tmp3);
26005         Select(Tmp4, N0);
26006         Select(Chain1, Chain1);
26007         ResNode = CurDAG->getTargetNode(X86::XOR8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
26008         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26009         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
26010         AddHandleReplacement(N1.Val, 1, ResNode, 1);
26011         Result = SDOperand(ResNode, N.ResNo);
26012         return;
26013       }
26014 
26015       // Pattern: (xor:i16 R16:i16:$src1, (load:i16 addr:i32:$src2))
26016       // Emits: (XOR16rm:i16 R16:i16:$src1, addr:i32:$src2)
26017       // Pattern complexity = 14  cost = 1
26018       if (N.Val->getValueType(0) == MVT::i16 &&
26019           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
26020         Select(Tmp0, Tmp0);
26021         Select(Tmp1, Tmp1);
26022         Select(Tmp2, Tmp2);
26023         Select(Tmp3, Tmp3);
26024         Select(Tmp4, N0);
26025         Select(Chain1, Chain1);
26026         ResNode = CurDAG->getTargetNode(X86::XOR16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
26027         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26028         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
26029         AddHandleReplacement(N1.Val, 1, ResNode, 1);
26030         Result = SDOperand(ResNode, N.ResNo);
26031         return;
26032       }
26033 
26034       // Pattern: (xor:i32 R32:i32:$src1, (load:i32 addr:i32:$src2))
26035       // Emits: (XOR32rm:i32 R32:i32:$src1, addr:i32:$src2)
26036       // Pattern complexity = 14  cost = 1
26037       if (N.Val->getValueType(0) == MVT::i32 &&
26038           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
26039         Select(Tmp0, Tmp0);
26040         Select(Tmp1, Tmp1);
26041         Select(Tmp2, Tmp2);
26042         Select(Tmp3, Tmp3);
26043         Select(Tmp4, N0);
26044         Select(Chain1, Chain1);
26045         ResNode = CurDAG->getTargetNode(X86::XOR32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
26046         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26047         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
26048         AddHandleReplacement(N1.Val, 1, ResNode, 1);
26049         Result = SDOperand(ResNode, N.ResNo);
26050         return;
26051       }
26052     }
26053   }
26054 
26055   // Pattern: (xor:v2i64 VR128:v2i64:$src1, (load:v2i64 addr:i32:$src2))
26056   // Emits: (PXORrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
26057   // Pattern complexity = 14  cost = 1
26058   if ((Subtarget->hasSSE2())) {
26059     N0 = N.getOperand(0);
26060     N1 = N.getOperand(1);
26061     if (N1.getOpcode() == ISD::LOAD &&
26062         N1.hasOneUse() &&
26063         !CodeGenMap.count(N1.getValue(0)) &&
26064         !isNonImmUse(N.Val, N1.Val)) {
26065       Chain1 = N1.getOperand(0);
26066       N11 = N1.getOperand(1);
26067       if (N.Val->getValueType(0) == MVT::v2i64 &&
26068           SelectAddr(N11, Tmp0, Tmp1, Tmp2, Tmp3)) {
26069         Select(Tmp0, Tmp0);
26070         Select(Tmp1, Tmp1);
26071         Select(Tmp2, Tmp2);
26072         Select(Tmp3, Tmp3);
26073         Select(Tmp4, N0);
26074         Select(Chain1, Chain1);
26075         ResNode = CurDAG->getTargetNode(X86::PXORrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain1);
26076         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26077         SelectionDAG::InsertISelMapEntry(CodeGenMap, N1.Val, 1, ResNode, 1);
26078         AddHandleReplacement(N1.Val, 1, ResNode, 1);
26079         Result = SDOperand(ResNode, N.ResNo);
26080         return;
26081       }
26082     }
26083   }
26084   {
26085     N0 = N.getOperand(0);
26086     if (N0.getOpcode() == ISD::LOAD &&
26087         N0.hasOneUse() &&
26088         !CodeGenMap.count(N0.getValue(0)) &&
26089         !isNonImmUse(N.Val, N0.Val)) {
26090       Chain0 = N0.getOperand(0);
26091       N01 = N0.getOperand(1);
26092       N1 = N.getOperand(1);
26093 
26094       // Pattern: (xor:i8 (load:i8 addr:i32:$src2), R8:i8:$src1)
26095       // Emits: (XOR8rm:i8 R8:i8:$src1, addr:i32:$src2)
26096       // Pattern complexity = 14  cost = 1
26097       if (N.Val->getValueType(0) == MVT::i8 &&
26098           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
26099         Select(Tmp0, Tmp0);
26100         Select(Tmp1, Tmp1);
26101         Select(Tmp2, Tmp2);
26102         Select(Tmp3, Tmp3);
26103         Select(Tmp4, N1);
26104         Select(Chain0, Chain0);
26105         ResNode = CurDAG->getTargetNode(X86::XOR8rm, MVT::i8, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
26106         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26107         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
26108         AddHandleReplacement(N0.Val, 1, ResNode, 1);
26109         Result = SDOperand(ResNode, N.ResNo);
26110         return;
26111       }
26112 
26113       // Pattern: (xor:i16 (load:i16 addr:i32:$src2), R16:i16:$src1)
26114       // Emits: (XOR16rm:i16 R16:i16:$src1, addr:i32:$src2)
26115       // Pattern complexity = 14  cost = 1
26116       if (N.Val->getValueType(0) == MVT::i16 &&
26117           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
26118         Select(Tmp0, Tmp0);
26119         Select(Tmp1, Tmp1);
26120         Select(Tmp2, Tmp2);
26121         Select(Tmp3, Tmp3);
26122         Select(Tmp4, N1);
26123         Select(Chain0, Chain0);
26124         ResNode = CurDAG->getTargetNode(X86::XOR16rm, MVT::i16, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
26125         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26126         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
26127         AddHandleReplacement(N0.Val, 1, ResNode, 1);
26128         Result = SDOperand(ResNode, N.ResNo);
26129         return;
26130       }
26131 
26132       // Pattern: (xor:i32 (load:i32 addr:i32:$src2), R32:i32:$src1)
26133       // Emits: (XOR32rm:i32 R32:i32:$src1, addr:i32:$src2)
26134       // Pattern complexity = 14  cost = 1
26135       if (N.Val->getValueType(0) == MVT::i32 &&
26136           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
26137         Select(Tmp0, Tmp0);
26138         Select(Tmp1, Tmp1);
26139         Select(Tmp2, Tmp2);
26140         Select(Tmp3, Tmp3);
26141         Select(Tmp4, N1);
26142         Select(Chain0, Chain0);
26143         ResNode = CurDAG->getTargetNode(X86::XOR32rm, MVT::i32, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
26144         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26145         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
26146         AddHandleReplacement(N0.Val, 1, ResNode, 1);
26147         Result = SDOperand(ResNode, N.ResNo);
26148         return;
26149       }
26150     }
26151   }
26152   if ((Subtarget->hasSSE2())) {
26153     N0 = N.getOperand(0);
26154 
26155     // Pattern: (xor:v2i64 (load:v2i64 addr:i32:$src2), VR128:v2i64:$src1)
26156     // Emits: (PXORrm:v2i64 VR128:v2i64:$src1, addr:i32:$src2)
26157     // Pattern complexity = 14  cost = 1
26158     if (N0.getOpcode() == ISD::LOAD &&
26159         N0.hasOneUse() &&
26160         !CodeGenMap.count(N0.getValue(0)) &&
26161         !isNonImmUse(N.Val, N0.Val)) {
26162       Chain0 = N0.getOperand(0);
26163       N01 = N0.getOperand(1);
26164       N1 = N.getOperand(1);
26165       if (N.Val->getValueType(0) == MVT::v2i64 &&
26166           SelectAddr(N01, Tmp0, Tmp1, Tmp2, Tmp3)) {
26167         Select(Tmp0, Tmp0);
26168         Select(Tmp1, Tmp1);
26169         Select(Tmp2, Tmp2);
26170         Select(Tmp3, Tmp3);
26171         Select(Tmp4, N1);
26172         Select(Chain0, Chain0);
26173         ResNode = CurDAG->getTargetNode(X86::PXORrm, MVT::v2i64, MVT::Other, Tmp4, Tmp0, Tmp1, Tmp2, Tmp3, Chain0);
26174         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26175         SelectionDAG::InsertISelMapEntry(CodeGenMap, N0.Val, 1, ResNode, 1);
26176         AddHandleReplacement(N0.Val, 1, ResNode, 1);
26177         Result = SDOperand(ResNode, N.ResNo);
26178         return;
26179       }
26180     }
26181 
26182     // Pattern: (xor:v2i64 (bitconvert:v2i64 VR128:v2f64:$src1), (bitconvert:v2i64 VR128:v2f64:$src2))
26183     // Emits: (XORPDrr:v2i64 VR128:v2f64:$src1, VR128:v2f64:$src2)
26184     // Pattern complexity = 6  cost = 1
26185     if (N0.getOpcode() == ISD::BIT_CONVERT) {
26186       N00 = N0.getOperand(0);
26187       N1 = N.getOperand(1);
26188       if (N1.getOpcode() == ISD::BIT_CONVERT) {
26189         N10 = N1.getOperand(0);
26190         if (N.Val->getValueType(0) == MVT::v2i64 &&
26191             N00.Val->getValueType(0) == MVT::v2f64 &&
26192             N10.Val->getValueType(0) == MVT::v2f64) {
26193           Select(Tmp0, N00);
26194           Select(Tmp1, N10);
26195           if (N.Val->hasOneUse()) {
26196             Result = CurDAG->SelectNodeTo(N.Val, X86::XORPDrr, MVT::v2i64, Tmp0, Tmp1);
26197           } else {
26198             ResNode = CurDAG->getTargetNode(X86::XORPDrr, MVT::v2i64, Tmp0, Tmp1);
26199             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26200             Result = SDOperand(ResNode, 0);
26201           }
26202           return;
26203         }
26204       }
26205     }
26206   }
26207   {
26208     N0 = N.getOperand(0);
26209     N1 = N.getOperand(1);
26210     if (N1.getOpcode() == ISD::Constant) {
26211       if (Predicate_immAllOnes(N1.Val)) {
26212 
26213         // Pattern: (xor:i8 R8:i8:$src, (imm:i8)<<P:Predicate_immAllOnes>>)
26214         // Emits: (NOT8r:i8 R8:i8:$src)
26215         // Pattern complexity = 5  cost = 1
26216         if (N.Val->getValueType(0) == MVT::i8) {
26217           Select(Tmp0, N0);
26218           if (N.Val->hasOneUse()) {
26219             Result = CurDAG->SelectNodeTo(N.Val, X86::NOT8r, MVT::i8, Tmp0);
26220           } else {
26221             ResNode = CurDAG->getTargetNode(X86::NOT8r, MVT::i8, Tmp0);
26222             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26223             Result = SDOperand(ResNode, 0);
26224           }
26225           return;
26226         }
26227 
26228         // Pattern: (xor:i16 R16:i16:$src, (imm:i16)<<P:Predicate_immAllOnes>>)
26229         // Emits: (NOT16r:i16 R16:i16:$src)
26230         // Pattern complexity = 5  cost = 1
26231         if (N.Val->getValueType(0) == MVT::i16) {
26232           Select(Tmp0, N0);
26233           if (N.Val->hasOneUse()) {
26234             Result = CurDAG->SelectNodeTo(N.Val, X86::NOT16r, MVT::i16, Tmp0);
26235           } else {
26236             ResNode = CurDAG->getTargetNode(X86::NOT16r, MVT::i16, Tmp0);
26237             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26238             Result = SDOperand(ResNode, 0);
26239           }
26240           return;
26241         }
26242 
26243         // Pattern: (xor:i32 R32:i32:$src, (imm:i32)<<P:Predicate_immAllOnes>>)
26244         // Emits: (NOT32r:i32 R32:i32:$src)
26245         // Pattern complexity = 5  cost = 1
26246         if (N.Val->getValueType(0) == MVT::i32) {
26247           Select(Tmp0, N0);
26248           if (N.Val->hasOneUse()) {
26249             Result = CurDAG->SelectNodeTo(N.Val, X86::NOT32r, MVT::i32, Tmp0);
26250           } else {
26251             ResNode = CurDAG->getTargetNode(X86::NOT32r, MVT::i32, Tmp0);
26252             SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26253             Result = SDOperand(ResNode, 0);
26254           }
26255           return;
26256         }
26257       }
26258 
26259       // Pattern: (xor:i16 R16:i16:$src1, (imm:i16)<<P:Predicate_i16immSExt8>>:$src2)
26260       // Emits: (XOR16ri8:i16 R16:i16:$src1, (imm:i16):$src2)
26261       // Pattern complexity = 5  cost = 1
26262       if (Predicate_i16immSExt8(N1.Val) &&
26263           N.Val->getValueType(0) == MVT::i16) {
26264         Select(Tmp0, N0);
26265         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
26266         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
26267         if (N.Val->hasOneUse()) {
26268           Result = CurDAG->SelectNodeTo(N.Val, X86::XOR16ri8, MVT::i16, Tmp0, Tmp1);
26269         } else {
26270           ResNode = CurDAG->getTargetNode(X86::XOR16ri8, MVT::i16, Tmp0, Tmp1);
26271           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26272           Result = SDOperand(ResNode, 0);
26273         }
26274         return;
26275       }
26276 
26277       // Pattern: (xor:i32 R32:i32:$src1, (imm:i32)<<P:Predicate_i32immSExt8>>:$src2)
26278       // Emits: (XOR32ri8:i32 R32:i32:$src1, (imm:i32):$src2)
26279       // Pattern complexity = 5  cost = 1
26280       if (Predicate_i32immSExt8(N1.Val) &&
26281           N.Val->getValueType(0) == MVT::i32) {
26282         Select(Tmp0, N0);
26283         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
26284         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
26285         if (N.Val->hasOneUse()) {
26286           Result = CurDAG->SelectNodeTo(N.Val, X86::XOR32ri8, MVT::i32, Tmp0, Tmp1);
26287         } else {
26288           ResNode = CurDAG->getTargetNode(X86::XOR32ri8, MVT::i32, Tmp0, Tmp1);
26289           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26290           Result = SDOperand(ResNode, 0);
26291         }
26292         return;
26293       }
26294 
26295       // Pattern: (xor:i8 R8:i8:$src1, (imm:i8):$src2)
26296       // Emits: (XOR8ri:i8 R8:i8:$src1, (imm:i8):$src2)
26297       // Pattern complexity = 4  cost = 1
26298       if (N.Val->getValueType(0) == MVT::i8) {
26299         Select(Tmp0, N0);
26300         unsigned char Tmp1C = (unsigned char)cast<ConstantSDNode>(N1)->getValue();
26301         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i8);
26302         if (N.Val->hasOneUse()) {
26303           Result = CurDAG->SelectNodeTo(N.Val, X86::XOR8ri, MVT::i8, Tmp0, Tmp1);
26304         } else {
26305           ResNode = CurDAG->getTargetNode(X86::XOR8ri, MVT::i8, Tmp0, Tmp1);
26306           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26307           Result = SDOperand(ResNode, 0);
26308         }
26309         return;
26310       }
26311 
26312       // Pattern: (xor:i16 R16:i16:$src1, (imm:i16):$src2)
26313       // Emits: (XOR16ri:i16 R16:i16:$src1, (imm:i16):$src2)
26314       // Pattern complexity = 4  cost = 1
26315       if (N.Val->getValueType(0) == MVT::i16) {
26316         Select(Tmp0, N0);
26317         unsigned short Tmp1C = (unsigned short)cast<ConstantSDNode>(N1)->getValue();
26318         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i16);
26319         if (N.Val->hasOneUse()) {
26320           Result = CurDAG->SelectNodeTo(N.Val, X86::XOR16ri, MVT::i16, Tmp0, Tmp1);
26321         } else {
26322           ResNode = CurDAG->getTargetNode(X86::XOR16ri, MVT::i16, Tmp0, Tmp1);
26323           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26324           Result = SDOperand(ResNode, 0);
26325         }
26326         return;
26327       }
26328 
26329       // Pattern: (xor:i32 R32:i32:$src1, (imm:i32):$src2)
26330       // Emits: (XOR32ri:i32 R32:i32:$src1, (imm:i32):$src2)
26331       // Pattern complexity = 4  cost = 1
26332       if (N.Val->getValueType(0) == MVT::i32) {
26333         Select(Tmp0, N0);
26334         unsigned Tmp1C = (unsigned)cast<ConstantSDNode>(N1)->getValue();
26335         Tmp1 = CurDAG->getTargetConstant(Tmp1C, MVT::i32);
26336         if (N.Val->hasOneUse()) {
26337           Result = CurDAG->SelectNodeTo(N.Val, X86::XOR32ri, MVT::i32, Tmp0, Tmp1);
26338         } else {
26339           ResNode = CurDAG->getTargetNode(X86::XOR32ri, MVT::i32, Tmp0, Tmp1);
26340           SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26341           Result = SDOperand(ResNode, 0);
26342         }
26343         return;
26344       }
26345     }
26346 
26347     // Pattern: (xor:i8 R8:i8:$src1, R8:i8:$src2)
26348     // Emits: (XOR8rr:i8 R8:i8:$src1, R8:i8:$src2)
26349     // Pattern complexity = 2  cost = 1
26350     if (N.Val->getValueType(0) == MVT::i8) {
26351       Select(Tmp0, N0);
26352       Select(Tmp1, N1);
26353       if (N.Val->hasOneUse()) {
26354         Result = CurDAG->SelectNodeTo(N.Val, X86::XOR8rr, MVT::i8, Tmp0, Tmp1);
26355       } else {
26356         ResNode = CurDAG->getTargetNode(X86::XOR8rr, MVT::i8, Tmp0, Tmp1);
26357         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26358         Result = SDOperand(ResNode, 0);
26359       }
26360       return;
26361     }
26362 
26363     // Pattern: (xor:i16 R16:i16:$src1, R16:i16:$src2)
26364     // Emits: (XOR16rr:i16 R16:i16:$src1, R16:i16:$src2)
26365     // Pattern complexity = 2  cost = 1
26366     if (N.Val->getValueType(0) == MVT::i16) {
26367       Select(Tmp0, N0);
26368       Select(Tmp1, N1);
26369       if (N.Val->hasOneUse()) {
26370         Result = CurDAG->SelectNodeTo(N.Val, X86::XOR16rr, MVT::i16, Tmp0, Tmp1);
26371       } else {
26372         ResNode = CurDAG->getTargetNode(X86::XOR16rr, MVT::i16, Tmp0, Tmp1);
26373         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26374         Result = SDOperand(ResNode, 0);
26375       }
26376       return;
26377     }
26378 
26379     // Pattern: (xor:i32 R32:i32:$src1, R32:i32:$src2)
26380     // Emits: (XOR32rr:i32 R32:i32:$src1, R32:i32:$src2)
26381     // Pattern complexity = 2  cost = 1
26382     if (N.Val->getValueType(0) == MVT::i32) {
26383       Select(Tmp0, N0);
26384       Select(Tmp1, N1);
26385       if (N.Val->hasOneUse()) {
26386         Result = CurDAG->SelectNodeTo(N.Val, X86::XOR32rr, MVT::i32, Tmp0, Tmp1);
26387       } else {
26388         ResNode = CurDAG->getTargetNode(X86::XOR32rr, MVT::i32, Tmp0, Tmp1);
26389         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26390         Result = SDOperand(ResNode, 0);
26391       }
26392       return;
26393     }
26394   }
26395 
26396   // Pattern: (xor:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26397   // Emits: (XORPSrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26398   // Pattern complexity = 2  cost = 1
26399   if ((Subtarget->hasSSE1())) {
26400     N0 = N.getOperand(0);
26401     N1 = N.getOperand(1);
26402     if (N.Val->getValueType(0) == MVT::v2i64) {
26403       Select(Tmp0, N0);
26404       Select(Tmp1, N1);
26405       if (N.Val->hasOneUse()) {
26406         Result = CurDAG->SelectNodeTo(N.Val, X86::XORPSrr, MVT::v2i64, Tmp0, Tmp1);
26407       } else {
26408         ResNode = CurDAG->getTargetNode(X86::XORPSrr, MVT::v2i64, Tmp0, Tmp1);
26409         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26410         Result = SDOperand(ResNode, 0);
26411       }
26412       return;
26413     }
26414   }
26415 
26416   // Pattern: (xor:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26417   // Emits: (PXORrr:v2i64 VR128:v2i64:$src1, VR128:v2i64:$src2)
26418   // Pattern complexity = 2  cost = 1
26419   if ((Subtarget->hasSSE2())) {
26420     N0 = N.getOperand(0);
26421     N1 = N.getOperand(1);
26422     if (N.Val->getValueType(0) == MVT::v2i64) {
26423       Select(Tmp0, N0);
26424       Select(Tmp1, N1);
26425       if (N.Val->hasOneUse()) {
26426         Result = CurDAG->SelectNodeTo(N.Val, X86::PXORrr, MVT::v2i64, Tmp0, Tmp1);
26427       } else {
26428         ResNode = CurDAG->getTargetNode(X86::PXORrr, MVT::v2i64, Tmp0, Tmp1);
26429         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26430         Result = SDOperand(ResNode, 0);
26431       }
26432       return;
26433     }
26434   }
26435   std::cerr << "Cannot yet select: ";
26436   N.Val->dump(CurDAG);
26437   std::cerr << '\n';
26438   abort();
26439 }
26440 
26441 void Select_zext(SDOperand &Result, SDOperand N) {
26442   SDOperand N0(0, 0);
26443   SDOperand Tmp0(0, 0);
26444   SDNode *ResNode;
26445   N0 = N.getOperand(0);
26446 
26447   // Pattern: (zext:i16 R8:i8:$src)
26448   // Emits: (MOVZX16rr8:i16 R8:i8:$src)
26449   // Pattern complexity = 2  cost = 1
26450   if (N.Val->getValueType(0) == MVT::i16 &&
26451       N0.Val->getValueType(0) == MVT::i8) {
26452     Select(Tmp0, N0);
26453     if (N.Val->hasOneUse()) {
26454       Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX16rr8, MVT::i16, Tmp0);
26455     } else {
26456       ResNode = CurDAG->getTargetNode(X86::MOVZX16rr8, MVT::i16, Tmp0);
26457       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26458       Result = SDOperand(ResNode, 0);
26459     }
26460     return;
26461   }
26462   if (N.Val->getValueType(0) == MVT::i32) {
26463 
26464     // Pattern: (zext:i32 R8:i8:$src)
26465     // Emits: (MOVZX32rr8:i32 R8:i8:$src)
26466     // Pattern complexity = 2  cost = 1
26467     if (N0.Val->getValueType(0) == MVT::i8) {
26468       Select(Tmp0, N0);
26469       if (N.Val->hasOneUse()) {
26470         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX32rr8, MVT::i32, Tmp0);
26471       } else {
26472         ResNode = CurDAG->getTargetNode(X86::MOVZX32rr8, MVT::i32, Tmp0);
26473         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26474         Result = SDOperand(ResNode, 0);
26475       }
26476       return;
26477     }
26478 
26479     // Pattern: (zext:i32 R16:i16:$src)
26480     // Emits: (MOVZX32rr16:i32 R16:i16:$src)
26481     // Pattern complexity = 2  cost = 1
26482     if (N0.Val->getValueType(0) == MVT::i16) {
26483       Select(Tmp0, N0);
26484       if (N.Val->hasOneUse()) {
26485         Result = CurDAG->SelectNodeTo(N.Val, X86::MOVZX32rr16, MVT::i32, Tmp0);
26486       } else {
26487         ResNode = CurDAG->getTargetNode(X86::MOVZX32rr16, MVT::i32, Tmp0);
26488         SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, ResNode, 0);
26489         Result = SDOperand(ResNode, 0);
26490       }
26491       return;
26492     }
26493   }
26494   std::cerr << "Cannot yet select: ";
26495   N.Val->dump(CurDAG);
26496   std::cerr << '\n';
26497   abort();
26498 }
26499 
26500 void Select_zextld(SDOperand &Result, SDOperand N) {
26501   if (N.ResNo == 1 && N.getValue(0).hasOneUse()) {
26502     SDOperand Dummy = CurDAG->getNode(ISD::HANDLENODE, MVT::Other, N);
26503     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Dummy.Val, 0);
26504     SelectionDAG::InsertISelMapEntry(HandleMap, N.Val, 1, Dummy.Val, 0);
26505     Result = Dummy;
26506     return;
26507   }
26508   SDOperand Chain(0, 0);
26509   SDOperand N1(0, 0);
26510   SDOperand N2(0, 0);
26511   SDOperand N3(0, 0);
26512   SDOperand Tmp0(0, 0);
26513   SDOperand Tmp1(0, 0);
26514   SDOperand Tmp2(0, 0);
26515   SDOperand Tmp3(0, 0);
26516   SDNode *ResNode;
26517   Chain = N.getOperand(0);
26518   N1 = N.getOperand(1);
26519   N2 = N.getOperand(2);
26520   N3 = N.getOperand(3);
26521   if (cast<VTSDNode>(N3)->getVT() == MVT::i8) {
26522 
26523     // Pattern: (zextld:i16 addr:i32:$src, srcvalue:Other:$dummy, i8:Other)
26524     // Emits: (MOVZX16rm8:i16 addr:i32:$src)
26525     // Pattern complexity = 12  cost = 1
26526     if (N.Val->getValueType(0) == MVT::i16 &&
26527         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26528       Select(Tmp0, Tmp0);
26529       Select(Tmp1, Tmp1);
26530       Select(Tmp2, Tmp2);
26531       Select(Tmp3, Tmp3);
26532       Select(Chain, Chain);
26533       ResNode = CurDAG->getTargetNode(X86::MOVZX16rm8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26534       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26535       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26536       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26537       Result = SDOperand(ResNode, N.ResNo);
26538       return;
26539     }
26540 
26541     // Pattern: (zextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i8:Other)
26542     // Emits: (MOVZX32rm8:i32 addr:i32:$src)
26543     // Pattern complexity = 12  cost = 1
26544     if (N.Val->getValueType(0) == MVT::i32 &&
26545         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26546       Select(Tmp0, Tmp0);
26547       Select(Tmp1, Tmp1);
26548       Select(Tmp2, Tmp2);
26549       Select(Tmp3, Tmp3);
26550       Select(Chain, Chain);
26551       ResNode = CurDAG->getTargetNode(X86::MOVZX32rm8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26552       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26553       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26554       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26555       Result = SDOperand(ResNode, N.ResNo);
26556       return;
26557     }
26558   }
26559 
26560   // Pattern: (zextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i16:Other)
26561   // Emits: (MOVZX32rm16:i32 addr:i32:$src)
26562   // Pattern complexity = 12  cost = 1
26563   if (cast<VTSDNode>(N3)->getVT() == MVT::i16 &&
26564       N.Val->getValueType(0) == MVT::i32 &&
26565       SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26566     Select(Tmp0, Tmp0);
26567     Select(Tmp1, Tmp1);
26568     Select(Tmp2, Tmp2);
26569     Select(Tmp3, Tmp3);
26570     Select(Chain, Chain);
26571     ResNode = CurDAG->getTargetNode(X86::MOVZX32rm16, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26572     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26573     SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26574     if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26575     Result = SDOperand(ResNode, N.ResNo);
26576     return;
26577   }
26578   if (cast<VTSDNode>(N3)->getVT() == MVT::i1) {
26579 
26580     // Pattern: (zextld:i8 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
26581     // Emits: (MOV8rm:i8 addr:i32:$src)
26582     // Pattern complexity = 12  cost = 1
26583     if (N.Val->getValueType(0) == MVT::i8 &&
26584         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26585       Select(Tmp0, Tmp0);
26586       Select(Tmp1, Tmp1);
26587       Select(Tmp2, Tmp2);
26588       Select(Tmp3, Tmp3);
26589       Select(Chain, Chain);
26590       ResNode = CurDAG->getTargetNode(X86::MOV8rm, MVT::i8, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26591       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26592       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26593       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26594       Result = SDOperand(ResNode, N.ResNo);
26595       return;
26596     }
26597 
26598     // Pattern: (zextld:i16 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
26599     // Emits: (MOVZX16rm8:i16 addr:i32:$src)
26600     // Pattern complexity = 12  cost = 1
26601     if (N.Val->getValueType(0) == MVT::i16 &&
26602         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26603       Select(Tmp0, Tmp0);
26604       Select(Tmp1, Tmp1);
26605       Select(Tmp2, Tmp2);
26606       Select(Tmp3, Tmp3);
26607       Select(Chain, Chain);
26608       ResNode = CurDAG->getTargetNode(X86::MOVZX16rm8, MVT::i16, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26609       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26610       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26611       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26612       Result = SDOperand(ResNode, N.ResNo);
26613       return;
26614     }
26615 
26616     // Pattern: (zextld:i32 addr:i32:$src, srcvalue:Other:$dummy, i1:Other)
26617     // Emits: (MOVZX32rm8:i32 addr:i32:$src)
26618     // Pattern complexity = 12  cost = 1
26619     if (N.Val->getValueType(0) == MVT::i32 &&
26620         SelectAddr(N1, Tmp0, Tmp1, Tmp2, Tmp3)) {
26621       Select(Tmp0, Tmp0);
26622       Select(Tmp1, Tmp1);
26623       Select(Tmp2, Tmp2);
26624       Select(Tmp3, Tmp3);
26625       Select(Chain, Chain);
26626       ResNode = CurDAG->getTargetNode(X86::MOVZX32rm8, MVT::i32, MVT::Other, Tmp0, Tmp1, Tmp2, Tmp3, Chain);
26627       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, ResNode, 0);
26628       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, ResNode, 1);
26629       if (N.ResNo == 0) AddHandleReplacement(N.Val, 1, ResNode, 1);
26630       Result = SDOperand(ResNode, N.ResNo);
26631       return;
26632     }
26633   }
26634   std::cerr << "Cannot yet select: ";
26635   N.Val->dump(CurDAG);
26636   std::cerr << '\n';
26637   abort();
26638 }
26639 
26640 void Select_INLINEASM(SDOperand& Result, SDOperand N) {
26641   std::vector<SDOperand> Ops(N.Val->op_begin(), N.Val->op_end());
26642   Select(Ops[0], N.getOperand(0)); // Select the chain.
26643 
26644   // Select the flag operand.
26645   if (Ops.back().getValueType() == MVT::Flag)
26646     Select(Ops.back(), Ops.back());
26647   SelectInlineAsmMemoryOperands(Ops, *CurDAG);
26648   std::vector<MVT::ValueType> VTs;
26649   VTs.push_back(MVT::Other);
26650   VTs.push_back(MVT::Flag);
26651   SDOperand New = CurDAG->getNode(ISD::INLINEASM, VTs, Ops);
26652   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26653   SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26654   Result = New.getValue(N.ResNo);
26655   return;
26656 }
26657 
26658 // The main instruction selector code.
26659 void SelectCode(SDOperand &Result, SDOperand N) {
26660   if (N.getOpcode() >= ISD::BUILTIN_OP_END &&
26661       N.getOpcode() < (ISD::BUILTIN_OP_END+X86::INSTRUCTION_LIST_END)) {
26662     Result = N;
26663     return;   // Already selected.
26664   }
26665 
26666   std::map<SDOperand, SDOperand>::iterator CGMI = CodeGenMap.find(N);
26667   if (CGMI != CodeGenMap.end()) {
26668     Result = CGMI->second;
26669     return;
26670   }
26671 
26672   switch (N.getOpcode()) {
26673   default: break;
26674   case ISD::EntryToken:       // These leaves remain the same.
26675   case ISD::BasicBlock:
26676   case ISD::Register:
26677   case ISD::HANDLENODE:
26678   case ISD::TargetConstant:
26679   case ISD::TargetConstantPool:
26680   case ISD::TargetFrameIndex:
26681   case ISD::TargetGlobalAddress: {
26682     Result = N;
26683     return;
26684   }
26685   case ISD::AssertSext:
26686   case ISD::AssertZext: {
26687     SDOperand Tmp0;
26688     Select(Tmp0, N.getOperand(0));
26689     if (!N.Val->hasOneUse())
26690       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Tmp0.Val, Tmp0.ResNo);
26691     Result = Tmp0;
26692     return;
26693   }
26694   case ISD::TokenFactor:
26695     if (N.getNumOperands() == 2) {
26696       SDOperand Op0, Op1;
26697       Select(Op0, N.getOperand(0));
26698       Select(Op1, N.getOperand(1));
26699       Result = 
26700           CurDAG->getNode(ISD::TokenFactor, MVT::Other, Op0, Op1);
26701       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Result.Val, Result.ResNo);
26702     } else {
26703       std::vector<SDOperand> Ops;
26704       for (unsigned i = 0, e = N.getNumOperands(); i != e; ++i) {
26705         SDOperand Val;
26706         Select(Val, N.getOperand(i));
26707         Ops.push_back(Val);
26708       }
26709       Result = 
26710           CurDAG->getNode(ISD::TokenFactor, MVT::Other, Ops);
26711       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, N.ResNo, Result.Val, Result.ResNo);
26712     }
26713     return;
26714   case ISD::CopyFromReg: {
26715     SDOperand Chain;
26716     Select(Chain, N.getOperand(0));
26717     unsigned Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
26718     MVT::ValueType VT = N.Val->getValueType(0);
26719     if (N.Val->getNumValues() == 2) {
26720       if (Chain == N.getOperand(0)) {
26721         Result = N; // No change
26722         return;
26723       }
26724       SDOperand New = CurDAG->getCopyFromReg(Chain, Reg, VT);
26725       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26726       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26727       Result = New.getValue(N.ResNo);
26728       return;
26729     } else {
26730       SDOperand Flag;
26731       if (N.getNumOperands() == 3) Select(Flag, N.getOperand(2));
26732       if (Chain == N.getOperand(0) &&
26733           (N.getNumOperands() == 2 || Flag == N.getOperand(2))) {
26734         Result = N; // No change
26735         return;
26736       }
26737       SDOperand New = CurDAG->getCopyFromReg(Chain, Reg, VT, Flag);
26738       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, New.Val, 0);
26739       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, New.Val, 1);
26740       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 2, New.Val, 2);
26741       Result = New.getValue(N.ResNo);
26742       return;
26743     }
26744   }
26745   case ISD::CopyToReg: {
26746     SDOperand Chain;
26747     Select(Chain, N.getOperand(0));
26748     unsigned Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
26749     SDOperand Val;
26750     Select(Val, N.getOperand(2));
26751     Result = N;
26752     if (N.Val->getNumValues() == 1) {
26753       if (Chain != N.getOperand(0) || Val != N.getOperand(2))
26754         Result = CurDAG->getCopyToReg(Chain, Reg, Val);
26755       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Result.Val, 0);
26756     } else {
26757       SDOperand Flag(0, 0);
26758       if (N.getNumOperands() == 4) Select(Flag, N.getOperand(3));
26759       if (Chain != N.getOperand(0) || Val != N.getOperand(2) ||
26760           (N.getNumOperands() == 4 && Flag != N.getOperand(3)))
26761         Result = CurDAG->getCopyToReg(Chain, Reg, Val, Flag);
26762       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 0, Result.Val, 0);
26763       SelectionDAG::InsertISelMapEntry(CodeGenMap, N.Val, 1, Result.Val, 1);
26764       Result = Result.getValue(N.ResNo);
26765     }
26766     return;
26767   }
26768   case ISD::INLINEASM:  Select_INLINEASM(Result, N); return;
26769   case X86ISD::Wrapper:          Select_X86Wrapper(Result, N); return;
26770   case X86ISD::BRCOND:           Select_X86brcond(Result, N); return;
26771   case X86ISD::CALL:             Select_X86call(Result, N); return;
26772   case ISD::CALLSEQ_END:         Select_X86callseq_end(Result, N); return;
26773   case ISD::CALLSEQ_START:       Select_X86callseq_start(Result, N); return;
26774   case X86ISD::CMOV:             Select_X86cmov(Result, N); return;
26775   case X86ISD::CMP:              Select_X86cmp(Result, N); return;
26776   case X86ISD::COMI:             Select_X86comi(Result, N); return;
26777   case X86ISD::FAND:             Select_X86fand(Result, N); return;
26778   case X86ISD::FILD:             Select_X86fild(Result, N); return;
26779   case X86ISD::FILD_FLAG:        Select_X86fildflag(Result, N); return;
26780   case X86ISD::FLD:              Select_X86fld(Result, N); return;
26781   case X86ISD::FP_TO_INT16_IN_MEM: Select_X86fp_to_i16mem(Result, N); return;
26782   case X86ISD::FP_TO_INT32_IN_MEM: Select_X86fp_to_i32mem(Result, N); return;
26783   case X86ISD::FP_TO_INT64_IN_MEM: Select_X86fp_to_i64mem(Result, N); return;
26784   case X86ISD::FP_GET_RESULT:    Select_X86fpget(Result, N); return;
26785   case X86ISD::FP_SET_RESULT:    Select_X86fpset(Result, N); return;
26786   case X86ISD::FST:              Select_X86fst(Result, N); return;
26787   case X86ISD::FXOR:             Select_X86fxor(Result, N); return;
26788   case X86ISD::LOAD_PACK:        Select_X86loadp(Result, N); return;
26789   case X86ISD::PEXTRW:           Select_X86pextrw(Result, N); return;
26790   case X86ISD::PINSRW:           Select_X86pinsrw(Result, N); return;
26791   case X86ISD::RDTSC_DAG:        Select_X86rdtsc(Result, N); return;
26792   case X86ISD::REP_MOVS:         Select_X86rep_movs(Result, N); return;
26793   case X86ISD::REP_STOS:         Select_X86rep_stos(Result, N); return;
26794   case X86ISD::RET_FLAG:         Select_X86retflag(Result, N); return;
26795   case X86ISD::S2VEC:            Select_X86s2vec(Result, N); return;
26796   case X86ISD::SETCC:            Select_X86setcc(Result, N); return;
26797   case X86ISD::SHLD:             Select_X86shld(Result, N); return;
26798   case X86ISD::SHRD:             Select_X86shrd(Result, N); return;
26799   case X86ISD::TEST:             Select_X86test(Result, N); return;
26800   case X86ISD::UCOMI:            Select_X86ucomi(Result, N); return;
26801   case X86ISD::ZEXT_S2VEC:       Select_X86zexts2vec(Result, N); return;
26802   case ISD::ADD:                 Select_add(Result, N); return;
26803   case ISD::ADDC:                Select_addc(Result, N); return;
26804   case ISD::ADDE:                Select_adde(Result, N); return;
26805   case ISD::AND:                 Select_and(Result, N); return;
26806   case ISD::ANY_EXTEND:          Select_anyext(Result, N); return;
26807   case ISD::BIT_CONVERT:         Select_bitconvert(Result, N); return;
26808   case ISD::BR:                  Select_br(Result, N); return;
26809   case ISD::BSWAP:               Select_bswap(Result, N); return;
26810   case ISD::BUILD_VECTOR:        Select_build_vector(Result, N); return;
26811   case ISD::DEBUG_LABEL:         Select_dwarf_label(Result, N); return;
26812   case ISD::DEBUG_LOC:           Select_dwarf_loc(Result, N); return;
26813   case ISD::EXTLOAD:             Select_extld(Result, N); return;
26814   case ISD::FABS:                Select_fabs(Result, N); return;
26815   case ISD::FADD:                Select_fadd(Result, N); return;
26816   case ISD::FCOS:                Select_fcos(Result, N); return;
26817   case ISD::FDIV:                Select_fdiv(Result, N); return;
26818   case ISD::FP_EXTEND:           Select_fextend(Result, N); return;
26819   case ISD::FMUL:                Select_fmul(Result, N); return;
26820   case ISD::FNEG:                Select_fneg(Result, N); return;
26821   case ISD::FP_TO_SINT:          Select_fp_to_sint(Result, N); return;
26822   case ISD::TargetConstantFP:    Select_fpimm(Result, N); return;
26823   case ISD::FrameIndex:          Select_frameindex(Result, N); return;
26824   case ISD::FP_ROUND:            Select_fround(Result, N); return;
26825   case ISD::FSIN:                Select_fsin(Result, N); return;
26826   case ISD::FSQRT:               Select_fsqrt(Result, N); return;
26827   case ISD::FSUB:                Select_fsub(Result, N); return;
26828   case ISD::Constant:            Select_imm(Result, N); return;
26829   case ISD::INTRINSIC_VOID:      Select_intrinsic_void(Result, N); return;
26830   case ISD::INTRINSIC_W_CHAIN:   Select_intrinsic_w_chain(Result, N); return;
26831   case ISD::INTRINSIC_WO_CHAIN:  Select_intrinsic_wo_chain(Result, N); return;
26832   case ISD::LOAD:                Select_load(Result, N); return;
26833   case ISD::MUL:                 Select_mul(Result, N); return;
26834   case ISD::OR:                  Select_or(Result, N); return;
26835   case ISD::ROTL:                Select_rotl(Result, N); return;
26836   case ISD::ROTR:                Select_rotr(Result, N); return;
26837   case ISD::SCALAR_TO_VECTOR:    Select_scalar_to_vector(Result, N); return;
26838   case ISD::SIGN_EXTEND:         Select_sext(Result, N); return;
26839   case ISD::SEXTLOAD:            Select_sextld(Result, N); return;
26840   case ISD::SHL:                 Select_shl(Result, N); return;
26841   case ISD::SINT_TO_FP:          Select_sint_to_fp(Result, N); return;
26842   case ISD::SRA:                 Select_sra(Result, N); return;
26843   case ISD::SRL:                 Select_srl(Result, N); return;
26844   case ISD::STORE:               Select_store(Result, N); return;
26845   case ISD::SUB:                 Select_sub(Result, N); return;
26846   case ISD::SUBC:                Select_subc(Result, N); return;
26847   case ISD::SUBE:                Select_sube(Result, N); return;
26848   case ISD::TRUNCSTORE:          Select_truncst(Result, N); return;
26849   case ISD::UNDEF:               Select_undef(Result, N); return;
26850   case ISD::EXTRACT_VECTOR_ELT:  Select_vector_extract(Result, N); return;
26851   case ISD::VECTOR_SHUFFLE:      Select_vector_shuffle(Result, N); return;
26852   case ISD::XOR:                 Select_xor(Result, N); return;
26853   case ISD::ZERO_EXTEND:         Select_zext(Result, N); return;
26854   case ISD::ZEXTLOAD:            Select_zextld(Result, N); return;
26855   } // end of big switch.
26856 
26857   std::cerr << "Cannot yet select: ";
26858   if (N.getOpcode() != ISD::INTRINSIC_W_CHAIN &&
26859       N.getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
26860       N.getOpcode() != ISD::INTRINSIC_VOID) {
26861     N.Val->dump(CurDAG);
26862   } else {
26863     unsigned iid = cast<ConstantSDNode>(N.getOperand(N.getOperand(0).getValueType() == MVT::Other))->getValue();
26864     std::cerr << "intrinsic %"<< Intrinsic::getName((Intrinsic::ID)iid);
26865   }
26866   std::cerr << '\n';
26867   abort();
26868 }