head 1.1; branch 1.1.1; access; symbols netbsd-11-0-RC4:1.1.1.10 netbsd-11-0-RC3:1.1.1.10 netbsd-11-0-RC2:1.1.1.10 netbsd-11-0-RC1:1.1.1.10 perseant-exfatfs-base-20250801:1.1.1.10 netbsd-11:1.1.1.10.0.10 netbsd-11-base:1.1.1.10 netbsd-10-1-RELEASE:1.1.1.10 perseant-exfatfs-base-20240630:1.1.1.10 perseant-exfatfs:1.1.1.10.0.8 perseant-exfatfs-base:1.1.1.10 netbsd-8-3-RELEASE:1.1.1.6 netbsd-9-4-RELEASE:1.1.1.9 netbsd-10-0-RELEASE:1.1.1.10 netbsd-10-0-RC6:1.1.1.10 netbsd-10-0-RC5:1.1.1.10 netbsd-10-0-RC4:1.1.1.10 netbsd-10-0-RC3:1.1.1.10 netbsd-10-0-RC2:1.1.1.10 netbsd-10-0-RC1:1.1.1.10 netbsd-10:1.1.1.10.0.6 netbsd-10-base:1.1.1.10 netbsd-9-3-RELEASE:1.1.1.9 cjep_sun2x:1.1.1.10.0.4 cjep_sun2x-base:1.1.1.10 cjep_staticlib_x-base1:1.1.1.10 netbsd-9-2-RELEASE:1.1.1.9 cjep_staticlib_x:1.1.1.10.0.2 cjep_staticlib_x-base:1.1.1.10 netbsd-9-1-RELEASE:1.1.1.9 phil-wifi-20200421:1.1.1.10 phil-wifi-20200411:1.1.1.10 phil-wifi-20200406:1.1.1.10 netbsd-8-2-RELEASE:1.1.1.6 netbsd-9-0-RELEASE:1.1.1.9 netbsd-9-0-RC2:1.1.1.9 netbsd-9-0-RC1:1.1.1.9 netbsd-9:1.1.1.9.0.2 netbsd-9-base:1.1.1.9 phil-wifi-20190609:1.1.1.9 netbsd-8-1-RELEASE:1.1.1.6 netbsd-8-1-RC1:1.1.1.6 pgoyette-compat-merge-20190127:1.1.1.8.2.1 pgoyette-compat-20190127:1.1.1.9 pgoyette-compat-20190118:1.1.1.9 pgoyette-compat-1226:1.1.1.9 pgoyette-compat-1126:1.1.1.9 pgoyette-compat-1020:1.1.1.9 pgoyette-compat-0930:1.1.1.9 pgoyette-compat-0906:1.1.1.9 netbsd-7-2-RELEASE:1.1.1.2.2.1 pgoyette-compat-0728:1.1.1.9 clang-337282:1.1.1.9 netbsd-8-0-RELEASE:1.1.1.6 phil-wifi:1.1.1.8.0.4 phil-wifi-base:1.1.1.8 pgoyette-compat-0625:1.1.1.8 netbsd-8-0-RC2:1.1.1.6 pgoyette-compat-0521:1.1.1.8 pgoyette-compat-0502:1.1.1.8 pgoyette-compat-0422:1.1.1.8 netbsd-8-0-RC1:1.1.1.6 pgoyette-compat-0415:1.1.1.8 pgoyette-compat-0407:1.1.1.8 pgoyette-compat-0330:1.1.1.8 pgoyette-compat-0322:1.1.1.8 pgoyette-compat-0315:1.1.1.8 netbsd-7-1-2-RELEASE:1.1.1.2.2.1 pgoyette-compat:1.1.1.8.0.2 pgoyette-compat-base:1.1.1.8 netbsd-7-1-1-RELEASE:1.1.1.2.2.1 clang-319952:1.1.1.8 matt-nb8-mediatek:1.1.1.6.0.8 matt-nb8-mediatek-base:1.1.1.6 clang-309604:1.1.1.7 perseant-stdc-iso10646:1.1.1.6.0.6 perseant-stdc-iso10646-base:1.1.1.6 netbsd-8:1.1.1.6.0.4 netbsd-8-base:1.1.1.6 prg-localcount2-base3:1.1.1.6 prg-localcount2-base2:1.1.1.6 prg-localcount2-base1:1.1.1.6 prg-localcount2:1.1.1.6.0.2 prg-localcount2-base:1.1.1.6 pgoyette-localcount-20170426:1.1.1.6 bouyer-socketcan-base1:1.1.1.6 pgoyette-localcount-20170320:1.1.1.6 netbsd-7-1:1.1.1.2.2.1.0.6 netbsd-7-1-RELEASE:1.1.1.2.2.1 netbsd-7-1-RC2:1.1.1.2.2.1 clang-294123:1.1.1.6 netbsd-7-nhusb-base-20170116:1.1.1.2.2.1 bouyer-socketcan:1.1.1.5.0.2 bouyer-socketcan-base:1.1.1.5 clang-291444:1.1.1.5 pgoyette-localcount-20170107:1.1.1.4 netbsd-7-1-RC1:1.1.1.2.2.1 pgoyette-localcount-20161104:1.1.1.4 netbsd-7-0-2-RELEASE:1.1.1.2.2.1 localcount-20160914:1.1.1.4 netbsd-7-nhusb:1.1.1.2.2.1.0.4 netbsd-7-nhusb-base:1.1.1.2.2.1 clang-280599:1.1.1.4 pgoyette-localcount-20160806:1.1.1.4 pgoyette-localcount-20160726:1.1.1.4 pgoyette-localcount:1.1.1.4.0.2 pgoyette-localcount-base:1.1.1.4 netbsd-7-0-1-RELEASE:1.1.1.2.2.1 clang-261930:1.1.1.4 netbsd-7-0:1.1.1.2.2.1.0.2 netbsd-7-0-RELEASE:1.1.1.2.2.1 netbsd-7-0-RC3:1.1.1.2.2.1 netbsd-7-0-RC2:1.1.1.2.2.1 netbsd-7-0-RC1:1.1.1.2.2.1 clang-237755:1.1.1.3 clang-232565:1.1.1.3 clang-227398:1.1.1.3 tls-maxphys-base:1.1.1.2 tls-maxphys:1.1.1.2.0.4 netbsd-7:1.1.1.2.0.2 netbsd-7-base:1.1.1.2 clang-215315:1.1.1.2 tls-earlyentropy:1.1.1.1.0.2 tls-earlyentropy-base:1.1.1.1 clang-209886:1.1.1.1 LLVM:1.1.1; locks; strict; comment @// @; 1.1 date 2014.05.30.18.14.41; author joerg; state Exp; branches 1.1.1.1; next ; commitid 8q0kdlBlCn09GACx; 1.1.1.1 date 2014.05.30.18.14.41; author joerg; state Exp; branches 1.1.1.1.2.1; next 1.1.1.2; commitid 8q0kdlBlCn09GACx; 1.1.1.2 date 2014.08.10.17.08.35; author joerg; state Exp; branches 1.1.1.2.2.1 1.1.1.2.4.1; next 1.1.1.3; commitid N85tXAN6Ex9VZPLx; 1.1.1.3 date 2015.01.29.19.57.31; author joerg; state Exp; branches; next 1.1.1.4; commitid mlISSizlPKvepX7y; 1.1.1.4 date 2016.02.27.22.12.08; author joerg; state Exp; branches 1.1.1.4.2.1; next 1.1.1.5; commitid tIimz3oDlh1NpBWy; 1.1.1.5 date 2017.01.11.10.33.25; author joerg; state Exp; branches 1.1.1.5.2.1; next 1.1.1.6; commitid CNnUNfII1jgNmxBz; 1.1.1.6 date 2017.02.09.17.38.01; author joerg; state Exp; branches; next 1.1.1.7; commitid UxB8JMFWM7xyMiFz; 1.1.1.7 date 2017.08.01.19.35.20; author joerg; state Exp; branches; next 1.1.1.8; commitid pMuDy65V0VicSx1A; 1.1.1.8 date 2017.12.17.20.59.06; author joerg; state Exp; branches 1.1.1.8.2.1 1.1.1.8.4.1; next 1.1.1.9; commitid T7p9sjiUJyY1UhjA; 1.1.1.9 date 2018.07.17.18.31.01; author joerg; state Exp; branches; next 1.1.1.10; commitid wDzL46ALjrCZgwKA; 1.1.1.10 date 2019.11.13.22.19.21; author joerg; state dead; branches; next ; commitid QD8YATxuNG34YJKB; 1.1.1.1.2.1 date 2014.05.30.18.14.41; author tls; state dead; branches; next 1.1.1.1.2.2; commitid t01A1TLTYxkpGMLx; 1.1.1.1.2.2 date 2014.08.10.07.08.07; author tls; state Exp; branches; next ; commitid t01A1TLTYxkpGMLx; 1.1.1.2.2.1 date 2015.06.04.20.04.28; author snj; state Exp; branches; next ; commitid yRnjq9fueSo6n9oy; 1.1.1.2.4.1 date 2014.08.10.17.08.35; author tls; state dead; branches; next 1.1.1.2.4.2; commitid jTnpym9Qu0o4R1Nx; 1.1.1.2.4.2 date 2014.08.19.23.47.27; author tls; state Exp; branches; next ; commitid jTnpym9Qu0o4R1Nx; 1.1.1.4.2.1 date 2017.03.20.06.52.37; author pgoyette; state Exp; branches; next ; commitid jjw7cAwgyKq7RfKz; 1.1.1.5.2.1 date 2017.04.21.16.51.38; author bouyer; state Exp; branches; next ; commitid dUG7nkTKALCadqOz; 1.1.1.8.2.1 date 2018.07.28.04.33.18; author pgoyette; state Exp; branches; next ; commitid 1UP1xAIUxv1ZgRLA; 1.1.1.8.4.1 date 2019.06.10.21.45.21; author christos; state Exp; branches; next 1.1.1.8.4.2; commitid jtc8rnCzWiEEHGqB; 1.1.1.8.4.2 date 2020.04.13.07.46.32; author martin; state dead; branches; next ; commitid X01YhRUPVUDaec4C; desc @@ 1.1 log @Initial revision @ text @//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This contains code to emit OpenMP nodes as LLVM code. // //===----------------------------------------------------------------------===// #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "clang/AST/Stmt.h" #include "clang/AST/StmtOpenMP.h" using namespace clang; using namespace CodeGen; //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); llvm::Value *CapturedStruct = GenerateCapturedStmtArgument(*CS); llvm::Value *OutlinedFn; { CodeGenFunction CGF(CGM, true); CGCapturedStmtInfo CGInfo(*CS, CS->getCapturedRegionKind()); CGF.CapturedStmtInfo = &CGInfo; OutlinedFn = CGF.GenerateCapturedStmtFunction( CS->getCapturedDecl(), CS->getCapturedRecordDecl(), CS->getLocStart()); } // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) llvm::Value *Args[] = { CGM.getOpenMPRuntime().EmitOpenMPUpdateLocation(*this, S.getLocStart()), Builder.getInt32(1), // Number of arguments after 'microtask' argument // (there is only one additional argument - 'context') Builder.CreateBitCast(OutlinedFn, CGM.getOpenMPRuntime().getKmpc_MicroPointerTy()), EmitCastToVoidPtr(CapturedStruct) }; llvm::Constant *RTLFn = CGM.getOpenMPRuntime().CreateRuntimeFunction( CGOpenMPRuntime::OMPRTL__kmpc_fork_call); EmitRuntimeCall(RTLFn, Args); } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); const Stmt *Body = CS->getCapturedStmt(); LoopStack.setParallel(); LoopStack.setVectorizerEnable(true); for (auto C : S.clauses()) { switch (C->getClauseKind()) { case OMPC_safelen: { RValue Len = EmitAnyExpr(cast(C)->getSafelen(), AggValueSlot::ignored(), true); llvm::ConstantInt *Val = cast(Len.getScalarVal()); LoopStack.setVectorizerWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. LoopStack.setParallel(false); break; } default: // Not handled yet ; } } EmitStmt(Body); } @ 1.1.1.1 log @Import Clang 3.5svn r209886. @ text @@ 1.1.1.2 log @Import clang 3.6svn r215315. @ text @d35 2 a36 1 OutlinedFn = CGF.GenerateCapturedStmtFunction(*CS); d41 7 a47 6 CGM.getOpenMPRuntime().EmitOpenMPUpdateLocation(*this, S.getLocStart()), Builder.getInt32(1), // Number of arguments after 'microtask' argument // (there is only one additional argument - 'context') Builder.CreateBitCast(OutlinedFn, CGM.getOpenMPRuntime().getKmpc_MicroPointerTy()), EmitCastToVoidPtr(CapturedStruct)}; a78 62 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &) { llvm_unreachable("CodeGen for 'omp for' is not supported yet."); } void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &) { llvm_unreachable("CodeGen for 'omp sections' is not supported yet."); } void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &) { llvm_unreachable("CodeGen for 'omp section' is not supported yet."); } void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &) { llvm_unreachable("CodeGen for 'omp single' is not supported yet."); } void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &) { llvm_unreachable("CodeGen for 'omp master' is not supported yet."); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &) { llvm_unreachable("CodeGen for 'omp critical' is not supported yet."); } void CodeGenFunction::EmitOMPParallelForDirective(const OMPParallelForDirective &) { llvm_unreachable("CodeGen for 'omp parallel for' is not supported yet."); } void CodeGenFunction::EmitOMPParallelSectionsDirective( const OMPParallelSectionsDirective &) { llvm_unreachable("CodeGen for 'omp parallel sections' is not supported yet."); } void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &) { llvm_unreachable("CodeGen for 'omp task' is not supported yet."); } void CodeGenFunction::EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &) { llvm_unreachable("CodeGen for 'omp taskyield' is not supported yet."); } void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &) { llvm_unreachable("CodeGen for 'omp barrier' is not supported yet."); } void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) { llvm_unreachable("CodeGen for 'omp taskwait' is not supported yet."); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &) { llvm_unreachable("CodeGen for 'omp flush' is not supported yet."); } void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) { llvm_unreachable("CodeGen for 'omp ordered' is not supported yet."); } void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &) { llvm_unreachable("CodeGen for 'omp atomic' is not supported yet."); } @ 1.1.1.2.2.1 log @Update LLVM to 3.6.1, requested by joerg in ticket 824. @ text @a16 1 #include "TargetInfo.h" a21 29 namespace { /// \brief RAII for emitting code of CapturedStmt without function outlining. class InlinedOpenMPRegion { CodeGenFunction &CGF; CodeGenFunction::CGCapturedStmtInfo *PrevCapturedStmtInfo; const Decl *StoredCurCodeDecl; /// \brief A class to emit CapturedStmt construct as inlined statement without /// generating a function for outlined code. class CGInlinedOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { public: CGInlinedOpenMPRegionInfo() : CGCapturedStmtInfo() {} }; public: InlinedOpenMPRegion(CodeGenFunction &CGF, const Stmt *S) : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo), StoredCurCodeDecl(CGF.CurCodeDecl) { CGF.CurCodeDecl = cast(S)->getCapturedDecl(); CGF.CapturedStmtInfo = new CGInlinedOpenMPRegionInfo(); } ~InlinedOpenMPRegion() { delete CGF.CapturedStmtInfo; CGF.CapturedStmtInfo = PrevCapturedStmtInfo; CGF.CurCodeDecl = StoredCurCodeDecl; } }; } // namespace d26 3 a28 18 /// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen /// function. Here is the logic: /// if (Cond) { /// CodeGen(true); /// } else { /// CodeGen(false); /// } static void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const std::function &CodeGen) { CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. bool CondConstant; if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { CodeGen(CondConstant); return; } d30 1 a30 12 // Otherwise, the condition did not fold, or we couldn't elide it. Just // emit the conditional branch. auto ThenBlock = CGF.createBasicBlock(/*name*/ "omp_if.then"); auto ElseBlock = CGF.createBasicBlock(/*name*/ "omp_if.else"); auto ContBlock = CGF.createBasicBlock(/*name*/ "omp_if.end"); CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount*/ 0); // Emit the 'then' code. CGF.EmitBlock(ThenBlock); CodeGen(/*ThenBlock*/ true); CGF.EmitBranch(ContBlock); // Emit the 'else' code if present. d32 17 a48 314 // There is no need to emit line number for unconditional branch. ApplyDebugLocation DL(CGF); CGF.EmitBlock(ElseBlock); } CodeGen(/*ThenBlock*/ false); { // There is no need to emit line number for unconditional branch. ApplyDebugLocation DL(CGF); CGF.EmitBranch(ContBlock); } // Emit the continuation block for code after the if. CGF.EmitBlock(ContBlock, /*IsFinished*/ true); } void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr, const Expr *AssignExpr, QualType OriginalType, const VarDecl *VDInit) { EmitBlock(createBasicBlock(".omp.assign.begin.")); if (!isa(AssignExpr) || isTrivialInitializer(AssignExpr)) { // Perform simple memcpy. EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(), AssignExpr->getType()); } else { // Perform element-by-element initialization. QualType ElementTy; auto SrcBegin = OriginalAddr.getAddress(); auto DestBegin = PrivateAddr; auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin); auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements); auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements); // The basic structure here is a do-while loop, because we don't // need to check for the zero-element case. auto BodyBB = createBasicBlock("omp.arraycpy.body"); auto DoneBB = createBasicBlock("omp.arraycpy.done"); auto IsEmpty = Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Enter the loop body, making that address the current address. auto EntryBB = Builder.GetInsertBlock(); EmitBlock(BodyBB); auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); SrcElementPast->addIncoming(SrcEnd, EntryBB); auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); DestElementPast->addIncoming(DestEnd, EntryBB); // Shift the address back by one element. auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true); auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne, "omp.arraycpy.dest.element"); auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne, "omp.arraycpy.src.element"); { // Create RunCleanScope to cleanup possible temps. CodeGenFunction::RunCleanupsScope Init(*this); // Emit initialization for single element. LocalDeclMap[VDInit] = SrcElement; EmitAnyExprToMem(AssignExpr, DestElement, AssignExpr->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); } // Check whether we've reached the end. auto Done = Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done"); Builder.CreateCondBr(Done, DoneBB, BodyBB); DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock()); SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock()); // Done. EmitBlock(DoneBB, true); } EmitBlock(createBasicBlock(".omp.assign.end.")); } void CodeGenFunction::EmitOMPFirstprivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { auto PrivateFilter = [](const OMPClause *C) -> bool { return C->getClauseKind() == OMPC_firstprivate; }; for (OMPExecutableDirective::filtered_clause_iterator I(D.clauses(), PrivateFilter); I; ++I) { auto *C = cast(*I); auto IRef = C->varlist_begin(); auto InitsRef = C->inits().begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered; if (*InitsRef != nullptr) { // Emit VarDecl with copy init for arrays. auto *FD = CapturedStmtInfo->lookup(OrigVD); LValue Base = MakeNaturalAlignAddrLValue( CapturedStmtInfo->getContextValue(), getContext().getTagDeclType(FD->getParent())); auto OriginalAddr = EmitLValueForField(Base, FD); auto VDInit = cast(cast(*InitsRef)->getDecl()); IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { auto Emission = EmitAutoVarAlloca(*VD); // Emit initialization of aggregate firstprivate vars. EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(), VD->getInit(), (*IRef)->getType(), VDInit); EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); } else IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++IRef, ++InitsRef; } } } void CodeGenFunction::EmitOMPPrivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { auto PrivateFilter = [](const OMPClause *C) -> bool { return C->getClauseKind() == OMPC_private; }; for (OMPExecutableDirective::filtered_clause_iterator I(D.clauses(), PrivateFilter); I; ++I) { auto *C = cast(*I); auto IRef = C->varlist_begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); auto VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++IRef; } } } /// \brief Emits code for OpenMP parallel directive in the parallel region. static void EmitOMPParallelCall(CodeGenFunction &CGF, const OMPParallelDirective &S, llvm::Value *OutlinedFn, llvm::Value *CapturedStruct) { if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); auto NumThreadsClause = cast(C); auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign*/ true); CGF.CGM.getOpenMPRuntime().EmitOMPNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } CGF.CGM.getOpenMPRuntime().EmitOMPParallelCall(CGF, S.getLocStart(), OutlinedFn, CapturedStruct); } void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { auto CS = cast(S.getAssociatedStmt()); auto CapturedStruct = GenerateCapturedStmtArgument(*CS); auto OutlinedFn = CGM.getOpenMPRuntime().EmitOpenMPOutlinedFunction( S, *CS->getCapturedDecl()->param_begin()); if (auto C = S.getSingleClause(/*K*/ OMPC_if)) { auto Cond = cast(C)->getCondition(); EmitOMPIfClause(*this, Cond, [&](bool ThenBlock) { if (ThenBlock) EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); else CGM.getOpenMPRuntime().EmitOMPSerialCall(*this, S.getLocStart(), OutlinedFn, CapturedStruct); }); } else EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); } void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, bool SeparateIter) { RunCleanupsScope BodyScope(*this); // Update counters values on current iteration. for (auto I : S.updates()) { EmitIgnoredExpr(I); } // On a continue in the body, jump to the end. auto Continue = getJumpDestInCurrentScope("omp.body.continue"); BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); // Emit loop body. EmitStmt(S.getBody()); // The end (updates/cleanups). EmitBlock(Continue.getBlock()); BreakContinueStack.pop_back(); if (SeparateIter) { // TODO: Update lastprivates if the SeparateIter flag is true. // This will be implemented in a follow-up OMPLastprivateClause patch, but // result should be still correct without it, as we do not make these // variables private yet. } } void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool SeparateIter) { auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); auto Cnt = getPGORegionCounter(&S); // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.inner.for.cond"); EmitBlock(CondBlock); LoopStack.push(CondBlock); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. auto ExitBlock = LoopExit.getBlock(); if (LoopScope.requiresCleanups()) ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); auto LoopBody = createBasicBlock("omp.inner.for.body"); // Emit condition: "IV < LastIteration + 1 [ - 1]" // ("- 1" when lastprivate clause is present - separate one iteration). llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond(SeparateIter)); Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, PGO.createLoopWeights(S.getCond(SeparateIter), Cnt)); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } EmitBlock(LoopBody); Cnt.beginRegion(Builder); // Create a block for the increment. auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); EmitOMPLoopBody(S); EmitStopPoint(&S); // Emit "IV = IV + 1" and a back-edge to the condition block. EmitBlock(Continue.getBlock()); EmitIgnoredExpr(S.getInc()); BreakContinueStack.pop_back(); EmitBranch(CondBlock); LoopStack.pop(); // Emit the fall-through block. EmitBlock(LoopExit.getBlock()); } void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { auto IC = S.counters().begin(); for (auto F : S.finals()) { if (LocalDeclMap.lookup(cast((*IC))->getDecl())) { EmitIgnoredExpr(F); } ++IC; } } static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, const OMPAlignedClause &Clause) { unsigned ClauseAlignment = 0; if (auto AlignmentExpr = Clause.getAlignment()) { auto AlignmentCI = cast(CGF.EmitScalarExpr(AlignmentExpr)); ClauseAlignment = static_cast(AlignmentCI->getZExtValue()); } for (auto E : Clause.varlists()) { unsigned Alignment = ClauseAlignment; if (Alignment == 0) { // OpenMP [2.8.1, Description] // If no optional parameter is specified, implementation-defined default // alignments for SIMD instructions on the target platforms are assumed. Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( E->getType()); } assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && "alignment is not power of 2"); if (Alignment != 0) { llvm::Value *PtrValue = CGF.EmitScalarExpr(E); CGF.EmitAlignmentAssumption(PtrValue, Alignment); } } } static void EmitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, ArrayRef Counters) { for (auto *E : Counters) { auto VD = cast(cast(E)->getDecl()); bool IsRegistered = LoopScope.addPrivate(VD, [&]() -> llvm::Value * { // Emit var without initialization. auto VarEmission = CGF.EmitAutoVarAlloca(*VD); CGF.EmitAutoVarCleanups(VarEmission); return VarEmission.getAllocatedAddress(); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } (void)LoopScope.Privatize(); d52 2 a53 17 // Pragma 'simd' code depends on presence of 'lastprivate'. // If present, we have to separate last iteration of the loop: // // if (LastIteration != 0) { // for (IV in 0..LastIteration-1) BODY; // BODY with updates of lastprivate vars; // ; // } // // otherwise (when there's no lastprivate): // // for (IV in 0..LastIteration) BODY; // ; // // Walk clauses and process safelen/lastprivate. bool SeparateIter = false; a68 6 case OMPC_aligned: EmitOMPAlignedClause(*this, CGM, cast(*C)); break; case OMPC_lastprivate: SeparateIter = true; break; d74 1 a74 53 InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope DirectiveScope(*this); CGDebugInfo *DI = getDebugInfo(); if (DI) DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); EmitVarDecl(*IVDecl); EmitIgnoredExpr(S.getInit()); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } if (SeparateIter) { // Emit: if (LastIteration > 0) - begin. RegionCounter Cnt = getPGORegionCounter(&S); auto ThenBlock = createBasicBlock("simd.if.then"); auto ContBlock = createBasicBlock("simd.if.end"); EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); EmitBlock(ThenBlock); Cnt.beginRegion(Builder); // Emit 'then' code. { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); EmitOMPInnerLoop(S, LoopScope, /* SeparateIter */ true); EmitOMPLoopBody(S, /* SeparateIter */ true); } EmitOMPSimdFinal(S); // Emit: if (LastIteration != 0) - end. EmitBranch(ContBlock); EmitBlock(ContBlock, true); } else { { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); EmitOMPInnerLoop(S, LoopScope); } EmitOMPSimdFinal(S); } if (DI) DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); d77 2 a78 109 /// \brief Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast(S.getIterationVariable()); auto IVDecl = cast(IVExpr->getDecl()); EmitVarDecl(*IVDecl); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } auto &RT = CGM.getOpenMPRuntime(); // Check pre-condition. { // Skip the entire loop if we don't meet the precondition. RegionCounter Cnt = getPGORegionCounter(&S); auto ThenBlock = createBasicBlock("omp.precond.then"); auto ContBlock = createBasicBlock("omp.precond.end"); EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); EmitBlock(ThenBlock); Cnt.beginRegion(Builder); // Emit 'then' code. { // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); // Detect the loop schedule kind and chunk. auto ScheduleKind = OMPC_SCHEDULE_unknown; llvm::Value *Chunk = nullptr; if (auto C = cast_or_null( S.getSingleClause(OMPC_schedule))) { ScheduleKind = C->getScheduleKind(); if (auto Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType()); } } const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr)) { // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When no chunk_size is specified, the iteration space is divided into // chunks that are approximately equal in size, and at most one chunk is // distributed to each thread. Note that the size of the chunks is // unspecified in this case. RT.EmitOMPForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } EmitOMPInnerLoop(S, LoopScope); // Tell the runtime we are done. RT.EmitOMPForFinish(*this, S.getLocStart(), ScheduleKind); } else ErrorUnsupported(&S, "OpenMP loop with requested schedule"); } // We're now done with the loop, so jump to the continuation block. EmitBranch(ContBlock); EmitBlock(ContBlock, true); } } void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope DirectiveScope(*this); CGDebugInfo *DI = getDebugInfo(); if (DI) DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); EmitOMPWorksharingLoop(S); // Emit an implicit barrier at the end. CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart(), /*IsExplicit*/ false); if (DI) DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); } void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); d93 6 a98 18 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { CGM.getOpenMPRuntime().EmitOMPMasterRegion(*this, [&]() -> void { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope Scope(*this); EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); EnsureInsertPoint(); }, S.getLocStart()); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { CGM.getOpenMPRuntime().EmitOMPCriticalRegion( *this, S.getDirectiveName().getAsString(), [&]() -> void { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope Scope(*this); EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); EnsureInsertPoint(); }, S.getLocStart()); a105 5 void CodeGenFunction::EmitOMPParallelForSimdDirective( const OMPParallelForSimdDirective &) { llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet."); } d119 2 a120 2 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart()); d127 2 a128 11 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().EmitOMPFlush( *this, [&]() -> ArrayRef { if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { auto FlushClause = cast(C); return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } return llvm::None; }(), S.getLocStart()); a138 8 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { llvm_unreachable("CodeGen for 'omp target' is not supported yet."); } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { llvm_unreachable("CodeGen for 'omp teams' is not supported yet."); } @ 1.1.1.3 log @Import Clang 3.6RC1 r227398. @ text @a16 1 #include "TargetInfo.h" a21 29 namespace { /// \brief RAII for emitting code of CapturedStmt without function outlining. class InlinedOpenMPRegion { CodeGenFunction &CGF; CodeGenFunction::CGCapturedStmtInfo *PrevCapturedStmtInfo; const Decl *StoredCurCodeDecl; /// \brief A class to emit CapturedStmt construct as inlined statement without /// generating a function for outlined code. class CGInlinedOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { public: CGInlinedOpenMPRegionInfo() : CGCapturedStmtInfo() {} }; public: InlinedOpenMPRegion(CodeGenFunction &CGF, const Stmt *S) : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo), StoredCurCodeDecl(CGF.CurCodeDecl) { CGF.CurCodeDecl = cast(S)->getCapturedDecl(); CGF.CapturedStmtInfo = new CGInlinedOpenMPRegionInfo(); } ~InlinedOpenMPRegion() { delete CGF.CapturedStmtInfo; CGF.CapturedStmtInfo = PrevCapturedStmtInfo; CGF.CurCodeDecl = StoredCurCodeDecl; } }; } // namespace d26 3 a28 18 /// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen /// function. Here is the logic: /// if (Cond) { /// CodeGen(true); /// } else { /// CodeGen(false); /// } static void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const std::function &CodeGen) { CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. bool CondConstant; if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { CodeGen(CondConstant); return; } d30 1 a30 12 // Otherwise, the condition did not fold, or we couldn't elide it. Just // emit the conditional branch. auto ThenBlock = CGF.createBasicBlock(/*name*/ "omp_if.then"); auto ElseBlock = CGF.createBasicBlock(/*name*/ "omp_if.else"); auto ContBlock = CGF.createBasicBlock(/*name*/ "omp_if.end"); CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount*/ 0); // Emit the 'then' code. CGF.EmitBlock(ThenBlock); CodeGen(/*ThenBlock*/ true); CGF.EmitBranch(ContBlock); // Emit the 'else' code if present. d32 17 a48 314 // There is no need to emit line number for unconditional branch. ApplyDebugLocation DL(CGF); CGF.EmitBlock(ElseBlock); } CodeGen(/*ThenBlock*/ false); { // There is no need to emit line number for unconditional branch. ApplyDebugLocation DL(CGF); CGF.EmitBranch(ContBlock); } // Emit the continuation block for code after the if. CGF.EmitBlock(ContBlock, /*IsFinished*/ true); } void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr, const Expr *AssignExpr, QualType OriginalType, const VarDecl *VDInit) { EmitBlock(createBasicBlock(".omp.assign.begin.")); if (!isa(AssignExpr) || isTrivialInitializer(AssignExpr)) { // Perform simple memcpy. EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(), AssignExpr->getType()); } else { // Perform element-by-element initialization. QualType ElementTy; auto SrcBegin = OriginalAddr.getAddress(); auto DestBegin = PrivateAddr; auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin); auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements); auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements); // The basic structure here is a do-while loop, because we don't // need to check for the zero-element case. auto BodyBB = createBasicBlock("omp.arraycpy.body"); auto DoneBB = createBasicBlock("omp.arraycpy.done"); auto IsEmpty = Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Enter the loop body, making that address the current address. auto EntryBB = Builder.GetInsertBlock(); EmitBlock(BodyBB); auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); SrcElementPast->addIncoming(SrcEnd, EntryBB); auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); DestElementPast->addIncoming(DestEnd, EntryBB); // Shift the address back by one element. auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true); auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne, "omp.arraycpy.dest.element"); auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne, "omp.arraycpy.src.element"); { // Create RunCleanScope to cleanup possible temps. CodeGenFunction::RunCleanupsScope Init(*this); // Emit initialization for single element. LocalDeclMap[VDInit] = SrcElement; EmitAnyExprToMem(AssignExpr, DestElement, AssignExpr->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); } // Check whether we've reached the end. auto Done = Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done"); Builder.CreateCondBr(Done, DoneBB, BodyBB); DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock()); SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock()); // Done. EmitBlock(DoneBB, true); } EmitBlock(createBasicBlock(".omp.assign.end.")); } void CodeGenFunction::EmitOMPFirstprivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { auto PrivateFilter = [](const OMPClause *C) -> bool { return C->getClauseKind() == OMPC_firstprivate; }; for (OMPExecutableDirective::filtered_clause_iterator I(D.clauses(), PrivateFilter); I; ++I) { auto *C = cast(*I); auto IRef = C->varlist_begin(); auto InitsRef = C->inits().begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered; if (*InitsRef != nullptr) { // Emit VarDecl with copy init for arrays. auto *FD = CapturedStmtInfo->lookup(OrigVD); LValue Base = MakeNaturalAlignAddrLValue( CapturedStmtInfo->getContextValue(), getContext().getTagDeclType(FD->getParent())); auto OriginalAddr = EmitLValueForField(Base, FD); auto VDInit = cast(cast(*InitsRef)->getDecl()); IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { auto Emission = EmitAutoVarAlloca(*VD); // Emit initialization of aggregate firstprivate vars. EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(), VD->getInit(), (*IRef)->getType(), VDInit); EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); } else IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++IRef, ++InitsRef; } } } void CodeGenFunction::EmitOMPPrivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { auto PrivateFilter = [](const OMPClause *C) -> bool { return C->getClauseKind() == OMPC_private; }; for (OMPExecutableDirective::filtered_clause_iterator I(D.clauses(), PrivateFilter); I; ++I) { auto *C = cast(*I); auto IRef = C->varlist_begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); auto VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++IRef; } } } /// \brief Emits code for OpenMP parallel directive in the parallel region. static void EmitOMPParallelCall(CodeGenFunction &CGF, const OMPParallelDirective &S, llvm::Value *OutlinedFn, llvm::Value *CapturedStruct) { if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); auto NumThreadsClause = cast(C); auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign*/ true); CGF.CGM.getOpenMPRuntime().EmitOMPNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } CGF.CGM.getOpenMPRuntime().EmitOMPParallelCall(CGF, S.getLocStart(), OutlinedFn, CapturedStruct); } void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { auto CS = cast(S.getAssociatedStmt()); auto CapturedStruct = GenerateCapturedStmtArgument(*CS); auto OutlinedFn = CGM.getOpenMPRuntime().EmitOpenMPOutlinedFunction( S, *CS->getCapturedDecl()->param_begin()); if (auto C = S.getSingleClause(/*K*/ OMPC_if)) { auto Cond = cast(C)->getCondition(); EmitOMPIfClause(*this, Cond, [&](bool ThenBlock) { if (ThenBlock) EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); else CGM.getOpenMPRuntime().EmitOMPSerialCall(*this, S.getLocStart(), OutlinedFn, CapturedStruct); }); } else EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); } void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, bool SeparateIter) { RunCleanupsScope BodyScope(*this); // Update counters values on current iteration. for (auto I : S.updates()) { EmitIgnoredExpr(I); } // On a continue in the body, jump to the end. auto Continue = getJumpDestInCurrentScope("omp.body.continue"); BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); // Emit loop body. EmitStmt(S.getBody()); // The end (updates/cleanups). EmitBlock(Continue.getBlock()); BreakContinueStack.pop_back(); if (SeparateIter) { // TODO: Update lastprivates if the SeparateIter flag is true. // This will be implemented in a follow-up OMPLastprivateClause patch, but // result should be still correct without it, as we do not make these // variables private yet. } } void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool SeparateIter) { auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); auto Cnt = getPGORegionCounter(&S); // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.inner.for.cond"); EmitBlock(CondBlock); LoopStack.push(CondBlock); // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. auto ExitBlock = LoopExit.getBlock(); if (LoopScope.requiresCleanups()) ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); auto LoopBody = createBasicBlock("omp.inner.for.body"); // Emit condition: "IV < LastIteration + 1 [ - 1]" // ("- 1" when lastprivate clause is present - separate one iteration). llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond(SeparateIter)); Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, PGO.createLoopWeights(S.getCond(SeparateIter), Cnt)); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } EmitBlock(LoopBody); Cnt.beginRegion(Builder); // Create a block for the increment. auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); EmitOMPLoopBody(S); EmitStopPoint(&S); // Emit "IV = IV + 1" and a back-edge to the condition block. EmitBlock(Continue.getBlock()); EmitIgnoredExpr(S.getInc()); BreakContinueStack.pop_back(); EmitBranch(CondBlock); LoopStack.pop(); // Emit the fall-through block. EmitBlock(LoopExit.getBlock()); } void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { auto IC = S.counters().begin(); for (auto F : S.finals()) { if (LocalDeclMap.lookup(cast((*IC))->getDecl())) { EmitIgnoredExpr(F); } ++IC; } } static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, const OMPAlignedClause &Clause) { unsigned ClauseAlignment = 0; if (auto AlignmentExpr = Clause.getAlignment()) { auto AlignmentCI = cast(CGF.EmitScalarExpr(AlignmentExpr)); ClauseAlignment = static_cast(AlignmentCI->getZExtValue()); } for (auto E : Clause.varlists()) { unsigned Alignment = ClauseAlignment; if (Alignment == 0) { // OpenMP [2.8.1, Description] // If no optional parameter is specified, implementation-defined default // alignments for SIMD instructions on the target platforms are assumed. Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment( E->getType()); } assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && "alignment is not power of 2"); if (Alignment != 0) { llvm::Value *PtrValue = CGF.EmitScalarExpr(E); CGF.EmitAlignmentAssumption(PtrValue, Alignment); } } } static void EmitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, ArrayRef Counters) { for (auto *E : Counters) { auto VD = cast(cast(E)->getDecl()); bool IsRegistered = LoopScope.addPrivate(VD, [&]() -> llvm::Value * { // Emit var without initialization. auto VarEmission = CGF.EmitAutoVarAlloca(*VD); CGF.EmitAutoVarCleanups(VarEmission); return VarEmission.getAllocatedAddress(); }); assert(IsRegistered && "counter already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } (void)LoopScope.Privatize(); d52 2 a53 17 // Pragma 'simd' code depends on presence of 'lastprivate'. // If present, we have to separate last iteration of the loop: // // if (LastIteration != 0) { // for (IV in 0..LastIteration-1) BODY; // BODY with updates of lastprivate vars; // ; // } // // otherwise (when there's no lastprivate): // // for (IV in 0..LastIteration) BODY; // ; // // Walk clauses and process safelen/lastprivate. bool SeparateIter = false; a68 6 case OMPC_aligned: EmitOMPAlignedClause(*this, CGM, cast(*C)); break; case OMPC_lastprivate: SeparateIter = true; break; d74 1 a74 53 InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope DirectiveScope(*this); CGDebugInfo *DI = getDebugInfo(); if (DI) DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); EmitVarDecl(*IVDecl); EmitIgnoredExpr(S.getInit()); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } if (SeparateIter) { // Emit: if (LastIteration > 0) - begin. RegionCounter Cnt = getPGORegionCounter(&S); auto ThenBlock = createBasicBlock("simd.if.then"); auto ContBlock = createBasicBlock("simd.if.end"); EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); EmitBlock(ThenBlock); Cnt.beginRegion(Builder); // Emit 'then' code. { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); EmitOMPInnerLoop(S, LoopScope, /* SeparateIter */ true); EmitOMPLoopBody(S, /* SeparateIter */ true); } EmitOMPSimdFinal(S); // Emit: if (LastIteration != 0) - end. EmitBranch(ContBlock); EmitBlock(ContBlock, true); } else { { OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); EmitOMPInnerLoop(S, LoopScope); } EmitOMPSimdFinal(S); } if (DI) DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); d77 2 a78 109 /// \brief Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast(S.getIterationVariable()); auto IVDecl = cast(IVExpr->getDecl()); EmitVarDecl(*IVDecl); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } auto &RT = CGM.getOpenMPRuntime(); // Check pre-condition. { // Skip the entire loop if we don't meet the precondition. RegionCounter Cnt = getPGORegionCounter(&S); auto ThenBlock = createBasicBlock("omp.precond.then"); auto ContBlock = createBasicBlock("omp.precond.end"); EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); EmitBlock(ThenBlock); Cnt.beginRegion(Builder); // Emit 'then' code. { // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); OMPPrivateScope LoopScope(*this); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); // Detect the loop schedule kind and chunk. auto ScheduleKind = OMPC_SCHEDULE_unknown; llvm::Value *Chunk = nullptr; if (auto C = cast_or_null( S.getSingleClause(OMPC_schedule))) { ScheduleKind = C->getScheduleKind(); if (auto Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType()); } } const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr)) { // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When no chunk_size is specified, the iteration space is divided into // chunks that are approximately equal in size, and at most one chunk is // distributed to each thread. Note that the size of the chunks is // unspecified in this case. RT.EmitOMPForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } EmitOMPInnerLoop(S, LoopScope); // Tell the runtime we are done. RT.EmitOMPForFinish(*this, S.getLocStart(), ScheduleKind); } else ErrorUnsupported(&S, "OpenMP loop with requested schedule"); } // We're now done with the loop, so jump to the continuation block. EmitBranch(ContBlock); EmitBlock(ContBlock, true); } } void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope DirectiveScope(*this); CGDebugInfo *DI = getDebugInfo(); if (DI) DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); EmitOMPWorksharingLoop(S); // Emit an implicit barrier at the end. CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart(), /*IsExplicit*/ false); if (DI) DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); } void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); d93 6 a98 18 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { CGM.getOpenMPRuntime().EmitOMPMasterRegion(*this, [&]() -> void { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope Scope(*this); EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); EnsureInsertPoint(); }, S.getLocStart()); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { CGM.getOpenMPRuntime().EmitOMPCriticalRegion( *this, S.getDirectiveName().getAsString(), [&]() -> void { InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); RunCleanupsScope Scope(*this); EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); EnsureInsertPoint(); }, S.getLocStart()); a105 5 void CodeGenFunction::EmitOMPParallelForSimdDirective( const OMPParallelForSimdDirective &) { llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet."); } d119 2 a120 2 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart()); d127 2 a128 11 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().EmitOMPFlush( *this, [&]() -> ArrayRef { if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { auto FlushClause = cast(C); return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } return llvm::None; }(), S.getLocStart()); a138 8 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { llvm_unreachable("CodeGen for 'omp target' is not supported yet."); } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { llvm_unreachable("CodeGen for 'omp teams' is not supported yet."); } @ 1.1.1.4 log @Import Clang 3.8.0rc3 r261930. @ text @d23 25 a47 21 void CodeGenFunction::GenerateOpenMPCapturedVars( const CapturedStmt &S, SmallVectorImpl &CapturedVars) { const RecordDecl *RD = S.getCapturedRecordDecl(); auto CurField = RD->field_begin(); auto CurCap = S.captures().begin(); for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(), E = S.capture_init_end(); I != E; ++I, ++CurField, ++CurCap) { if (CurField->hasCapturedVLAType()) { auto VAT = CurField->getCapturedVLAType(); auto *Val = VLASizeMap[VAT->getSizeExpr()]; CapturedVars.push_back(Val); } else if (CurCap->capturesThis()) CapturedVars.push_back(CXXThisValue); else if (CurCap->capturesVariableByCopy()) CapturedVars.push_back( EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal()); else { assert(CurCap->capturesVariable() && "Expected capture by reference."); CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer()); } d49 2 a50 149 } static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType = false) { ASTContext &Ctx = CGF.getContext(); auto *CastedPtr = CGF.EmitScalarConversion( AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), SourceLocation()); auto TmpAddr = CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) .getAddress(); // If we are dealing with references we need to return the address of the // reference instead of the reference of the value. if (isReferenceType) { QualType RefType = Ctx.getLValueReferenceType(DstType); auto *RefVal = TmpAddr.getPointer(); TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref"); auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); CGF.EmitScalarInit(RefVal, TmpLVal); } return TmpAddr; } llvm::Function * CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { assert( CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function"); const CapturedDecl *CD = S.getCapturedDecl(); const RecordDecl *RD = S.getCapturedRecordDecl(); assert(CD->hasBody() && "missing CapturedDecl body"); // Build the argument list. ASTContext &Ctx = CGM.getContext(); FunctionArgList Args; Args.append(CD->param_begin(), std::next(CD->param_begin(), CD->getContextParamPosition())); auto I = S.captures().begin(); for (auto *FD : RD->fields()) { QualType ArgType = FD->getType(); IdentifierInfo *II = nullptr; VarDecl *CapVar = nullptr; // If this is a capture by copy and the type is not a pointer, the outlined // function argument type should be uintptr and the value properly casted to // uintptr. This is necessary given that the runtime library is only able to // deal with pointers. We can pass in the same way the VLA type sizes to the // outlined function. if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || I->capturesVariableArrayType()) ArgType = Ctx.getUIntPtrType(); if (I->capturesVariable() || I->capturesVariableByCopy()) { CapVar = I->getCapturedVar(); II = CapVar->getIdentifier(); } else if (I->capturesThis()) II = &getContext().Idents.get("this"); else { assert(I->capturesVariableArrayType()); II = &getContext().Idents.get("vla"); } if (ArgType->isVariablyModifiedType()) ArgType = getContext().getVariableArrayDecayedType(ArgType); Args.push_back(ImplicitParamDecl::Create(getContext(), nullptr, FD->getLocation(), II, ArgType)); ++I; } Args.append( std::next(CD->param_begin(), CD->getContextParamPosition() + 1), CD->param_end()); // Create the function declaration. FunctionType::ExtInfo ExtInfo; const CGFunctionInfo &FuncInfo = CGM.getTypes().arrangeFreeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo, /*IsVariadic=*/false); llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo); llvm::Function *F = llvm::Function::Create( FuncLLVMTy, llvm::GlobalValue::InternalLinkage, CapturedStmtInfo->getHelperName(), &CGM.getModule()); CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); if (CD->isNothrow()) F->addFnAttr(llvm::Attribute::NoUnwind); // Generate the function. StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(), CD->getBody()->getLocStart()); unsigned Cnt = CD->getContextParamPosition(); I = S.captures().begin(); for (auto *FD : RD->fields()) { // If we are capturing a pointer by copy we don't need to do anything, just // use the value that we get from the arguments. if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) { setAddrOfLocalVar(I->getCapturedVar(), GetAddrOfLocalVar(Args[Cnt])); ++Cnt, ++I; continue; } LValue ArgLVal = MakeAddrLValue(GetAddrOfLocalVar(Args[Cnt]), Args[Cnt]->getType(), AlignmentSource::Decl); if (FD->hasCapturedVLAType()) { LValue CastedArgLVal = MakeAddrLValue(castValueFromUintptr(*this, FD->getType(), Args[Cnt]->getName(), ArgLVal), FD->getType(), AlignmentSource::Decl); auto *ExprArg = EmitLoadOfLValue(CastedArgLVal, SourceLocation()).getScalarVal(); auto VAT = FD->getCapturedVLAType(); VLASizeMap[VAT->getSizeExpr()] = ExprArg; } else if (I->capturesVariable()) { auto *Var = I->getCapturedVar(); QualType VarTy = Var->getType(); Address ArgAddr = ArgLVal.getAddress(); if (!VarTy->isReferenceType()) { ArgAddr = EmitLoadOfReference( ArgAddr, ArgLVal.getType()->castAs()); } setAddrOfLocalVar( Var, Address(ArgAddr.getPointer(), getContext().getDeclAlign(Var))); } else if (I->capturesVariableByCopy()) { assert(!FD->getType()->isAnyPointerType() && "Not expecting a captured pointer."); auto *Var = I->getCapturedVar(); QualType VarTy = Var->getType(); setAddrOfLocalVar(I->getCapturedVar(), castValueFromUintptr(*this, FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType())); } else { // If 'this' is captured, load it into CXXThisValue. assert(I->capturesThis()); CXXThisValue = EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation()).getScalarVal(); } ++Cnt, ++I; } PGO.assignRegionCounters(GlobalDecl(CD), F); CapturedStmtInfo->EmitBody(*this, CD->getBody()); FinishFunction(CD->getBodyRBrace()); return F; } a54 98 void CodeGenFunction::EmitOMPAggregateAssign( Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref &CopyGen) { // Perform element-by-element initialization. QualType ElementTy; // Drill down to the base element type on both arrays. auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); auto SrcBegin = SrcAddr.getPointer(); auto DestBegin = DestAddr.getPointer(); // Cast from pointer to array type to pointer to single element. auto DestEnd = Builder.CreateGEP(DestBegin, NumElements); // The basic structure here is a while-do loop. auto BodyBB = createBasicBlock("omp.arraycpy.body"); auto DoneBB = createBasicBlock("omp.arraycpy.done"); auto IsEmpty = Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Enter the loop body, making that address the current address. auto EntryBB = Builder.GetInsertBlock(); EmitBlock(BodyBB); CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy); llvm::PHINode *SrcElementPHI = Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); SrcElementPHI->addIncoming(SrcBegin, EntryBB); Address SrcElementCurrent = Address(SrcElementPHI, SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); llvm::PHINode *DestElementPHI = Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); DestElementPHI->addIncoming(DestBegin, EntryBB); Address DestElementCurrent = Address(DestElementPHI, DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); // Emit copy. CopyGen(DestElementCurrent, SrcElementCurrent); // Shift the address forward by one element. auto DestElementNext = Builder.CreateConstGEP1_32( DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); auto SrcElementNext = Builder.CreateConstGEP1_32( SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); // Check whether we've reached the end. auto Done = Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); Builder.CreateCondBr(Done, DoneBB, BodyBB); DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock()); SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock()); // Done. EmitBlock(DoneBB, /*IsFinished=*/true); } /// \brief Emit initialization of arrays of complex types. /// \param DestAddr Address of the array. /// \param Type Type of array. /// \param Init Initial expression of array. static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, const Expr *Init) { // Perform element-by-element initialization. QualType ElementTy; // Drill down to the base element type on both arrays. auto ArrayTy = Type->getAsArrayTypeUnsafe(); auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); DestAddr = CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); auto DestBegin = DestAddr.getPointer(); // Cast from pointer to array type to pointer to single element. auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements); // The basic structure here is a while-do loop. auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); auto IsEmpty = CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Enter the loop body, making that address the current address. auto EntryBB = CGF.Builder.GetInsertBlock(); CGF.EmitBlock(BodyBB); CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); DestElementPHI->addIncoming(DestBegin, EntryBB); Address DestElementCurrent = Address(DestElementPHI, DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); d56 31 a86 1 // Emit copy. d88 24 a111 45 CodeGenFunction::RunCleanupsScope InitScope(CGF); CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), /*IsInitializer=*/false); } // Shift the address forward by one element. auto DestElementNext = CGF.Builder.CreateConstGEP1_32( DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); // Check whether we've reached the end. auto Done = CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); // Done. CGF.EmitBlock(DoneBB, /*IsFinished=*/true); } void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy) { if (OriginalType->isArrayType()) { auto *BO = dyn_cast(Copy); if (BO && BO->getOpcode() == BO_Assign) { // Perform simple memcpy for simple copying. EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); } else { // For arrays with complex element types perform element by element // copying. EmitOMPAggregateAssign( DestAddr, SrcAddr, OriginalType, [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) { // Working with the single array element, so have to remap // destination and source variables to corresponding array // elements. CodeGenFunction::OMPPrivateScope Remap(*this); Remap.addPrivate(DestVD, [DestElement]() -> Address { return DestElement; }); Remap.addPrivate( SrcVD, [SrcElement]() -> Address { return SrcElement; }); (void)Remap.Privatize(); EmitIgnoredExpr(Copy); }); } d113 53 a165 7 // Remap pseudo source variable to private copy. CodeGenFunction::OMPPrivateScope Remap(*this); Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; }); Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; }); (void)Remap.Privatize(); // Emit copying of the whole variable. EmitIgnoredExpr(Copy); d167 1 d170 9 a178 6 bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { if (!HaveInsertPoint()) return false; llvm::DenseSet EmittedAsFirstprivate; for (const auto *C : D.getClausesOfKind()) { d183 27 a209 58 if (EmittedAsFirstprivate.count(OrigVD) == 0) { EmittedAsFirstprivate.insert(OrigVD); auto *VD = cast(cast(IInit)->getDecl()); auto *VDInit = cast(cast(*InitsRef)->getDecl()); bool IsRegistered; DeclRefExpr DRE( const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( OrigVD) != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); Address OriginalAddr = EmitLValue(&DRE).getAddress(); QualType Type = OrigVD->getType(); if (Type->isArrayType()) { // Emit VarDecl with copy init for arrays. // Get the address of the original variable captured in current // captured region. IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { auto Emission = EmitAutoVarAlloca(*VD); auto *Init = VD->getInit(); if (!isa(Init) || isTrivialInitializer(Init)) { // Perform simple memcpy. EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr, Type); } else { EmitOMPAggregateAssign( Emission.getAllocatedAddress(), OriginalAddr, Type, [this, VDInit, Init](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the initialization. RunCleanupsScope InitScope(*this); // Emit initialization for single element. setAddrOfLocalVar(VDInit, SrcElement); EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); }); } EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); } else { IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable // (for proper handling of captured global variables). setAddrOfLocalVar(VDInit, OriginalAddr); EmitDecl(*VD); LocalDeclMap.erase(VDInit); return GetAddrOfLocalVar(VD); }); } assert(IsRegistered && "firstprivate var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } a212 1 return !EmittedAsFirstprivate.empty(); d218 6 a223 4 if (!HaveInsertPoint()) return; llvm::DenseSet EmittedAsPrivate; for (const auto *C : D.getClausesOfKind()) { d227 10 a236 184 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { auto VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } ++IRef; } } } bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { if (!HaveInsertPoint()) return false; // threadprivate_var1 = master_threadprivate_var1; // operator=(threadprivate_var2, master_threadprivate_var2); // ... // __kmpc_barrier(&loc, global_tid); llvm::DenseSet CopiedVars; llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; for (const auto *C : D.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto ISrcRef = C->source_exprs().begin(); auto IDestRef = C->destination_exprs().begin(); for (auto *AssignOp : C->assignment_ops()) { auto *VD = cast(cast(*IRef)->getDecl()); QualType Type = VD->getType(); if (CopiedVars.insert(VD->getCanonicalDecl()).second) { // Get the address of the master variable. If we are emitting code with // TLS support, the address is passed from the master as field in the // captured declaration. Address MasterAddr = Address::invalid(); if (getLangOpts().OpenMPUseTLS && getContext().getTargetInfo().isTLSSupported()) { assert(CapturedStmtInfo->lookup(VD) && "Copyin threadprivates should have been captured!"); DeclRefExpr DRE(const_cast(VD), true, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); MasterAddr = EmitLValue(&DRE).getAddress(); LocalDeclMap.erase(VD); } else { MasterAddr = Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) : CGM.GetAddrOfGlobal(VD), getContext().getDeclAlign(VD)); } // Get the address of the threadprivate variable. Address PrivateAddr = EmitLValue(*IRef).getAddress(); if (CopiedVars.size() == 1) { // At first check if current thread is a master thread. If it is, no // need to copy data. CopyBegin = createBasicBlock("copyin.not.master"); CopyEnd = createBasicBlock("copyin.not.master.end"); Builder.CreateCondBr( Builder.CreateICmpNE( Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy), Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)), CopyBegin, CopyEnd); EmitBlock(CopyBegin); } auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); auto *DestVD = cast(cast(*IDestRef)->getDecl()); EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp); } ++IRef; ++ISrcRef; ++IDestRef; } } if (CopyEnd) { // Exit out of copying procedure for non-master thread. EmitBlock(CopyEnd, /*IsFinished=*/true); return true; } return false; } bool CodeGenFunction::EmitOMPLastprivateClauseInit( const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { if (!HaveInsertPoint()) return false; bool HasAtLeastOneLastprivate = false; llvm::DenseSet AlreadyEmittedVars; for (const auto *C : D.getClausesOfKind()) { HasAtLeastOneLastprivate = true; auto IRef = C->varlist_begin(); auto IDestRef = C->destination_exprs().begin(); for (auto *IInit : C->private_copies()) { // Keep the address of the original variable for future update at the end // of the loop. auto *OrigVD = cast(cast(*IRef)->getDecl()); if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { auto *DestVD = cast(cast(*IDestRef)->getDecl()); PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address { DeclRefExpr DRE( const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( OrigVD) != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); return EmitLValue(&DRE).getAddress(); }); // Check if the variable is also a firstprivate: in this case IInit is // not generated. Initialization of this variable will happen in codegen // for 'firstprivate' clause. if (IInit) { auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); assert(IsRegistered && "lastprivate var already registered as private"); (void)IsRegistered; } } ++IRef, ++IDestRef; } } return HasAtLeastOneLastprivate; } void CodeGenFunction::EmitOMPLastprivateClauseFinal( const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) { if (!HaveInsertPoint()) return; // Emit following code: // if () { // orig_var1 = private_orig_var1; // ... // orig_varn = private_orig_varn; // } llvm::BasicBlock *ThenBB = nullptr; llvm::BasicBlock *DoneBB = nullptr; if (IsLastIterCond) { ThenBB = createBasicBlock(".omp.lastprivate.then"); DoneBB = createBasicBlock(".omp.lastprivate.done"); Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); EmitBlock(ThenBB); } llvm::DenseMap LoopCountersAndUpdates; if (auto *LoopDirective = dyn_cast(&D)) { auto IC = LoopDirective->counters().begin(); for (auto F : LoopDirective->finals()) { auto *D = cast(*IC)->getDecl()->getCanonicalDecl(); LoopCountersAndUpdates[D] = F; ++IC; } } llvm::DenseSet AlreadyEmittedVars; for (const auto *C : D.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto ISrcRef = C->source_exprs().begin(); auto IDestRef = C->destination_exprs().begin(); for (auto *AssignOp : C->assignment_ops()) { auto *PrivateVD = cast(cast(*IRef)->getDecl()); QualType Type = PrivateVD->getType(); auto *CanonicalVD = PrivateVD->getCanonicalDecl(); if (AlreadyEmittedVars.insert(CanonicalVD).second) { // If lastprivate variable is a loop control variable for loop-based // directive, update its value before copyin back to original // variable. if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) EmitIgnoredExpr(UpExpr); auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); auto *DestVD = cast(cast(*IDestRef)->getDecl()); // Get the address of the original variable. Address OriginalAddr = GetAddrOfLocalVar(DestVD); // Get the address of the private variable. Address PrivateAddr = GetAddrOfLocalVar(PrivateVD); if (auto RefTy = PrivateVD->getType()->getAs()) PrivateAddr = Address(Builder.CreateLoad(PrivateAddr), getNaturalTypeAlignment(RefTy->getPointeeType())); EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp); } a237 163 ++ISrcRef; ++IDestRef; } } if (IsLastIterCond) EmitBlock(DoneBB, /*IsFinished=*/true); } void CodeGenFunction::EmitOMPReductionClauseInit( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { if (!HaveInsertPoint()) return; for (const auto *C : D.getClausesOfKind()) { auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); auto IPriv = C->privates().begin(); for (auto IRef : C->varlists()) { auto *LHSVD = cast(cast(*ILHS)->getDecl()); auto *RHSVD = cast(cast(*IRHS)->getDecl()); auto *PrivateVD = cast(cast(*IPriv)->getDecl()); if (auto *OASE = dyn_cast(IRef)) { auto *Base = OASE->getBase()->IgnoreParenImpCasts(); while (auto *TempOASE = dyn_cast(Base)) Base = TempOASE->getBase()->IgnoreParenImpCasts(); while (auto *TempASE = dyn_cast(Base)) Base = TempASE->getBase()->IgnoreParenImpCasts(); auto *DE = cast(Base); auto *OrigVD = cast(DE->getDecl()); auto OASELValueLB = EmitOMPArraySectionExpr(OASE); auto OASELValueUB = EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); auto OriginalBaseLValue = EmitLValue(DE); auto BaseLValue = OriginalBaseLValue; auto *Zero = Builder.getInt64(/*C=*/0); llvm::SmallVector Indexes; Indexes.push_back(Zero); auto *ItemTy = OASELValueLB.getPointer()->getType()->getPointerElementType(); auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType(); while (Ty != ItemTy) { Indexes.push_back(Zero); Ty = Ty->getPointerElementType(); } BaseLValue = MakeAddrLValue( Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes), OASELValueLB.getAlignment()), OASELValueLB.getType(), OASELValueLB.getAlignmentSource()); // Store the address of the original variable associated with the LHS // implicit variable. PrivateScope.addPrivate(LHSVD, [this, OASELValueLB]() -> Address { return OASELValueLB.getAddress(); }); // Emit reduction copy. bool IsRegistered = PrivateScope.addPrivate( OrigVD, [this, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB, OriginalBaseLValue]() -> Address { // Emit VarDecl with copy init for arrays. // Get the address of the original variable captured in current // captured region. auto *Size = Builder.CreatePtrDiff(OASELValueUB.getPointer(), OASELValueLB.getPointer()); Size = Builder.CreateNUWAdd( Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1)); CodeGenFunction::OpaqueValueMapping OpaqueMap( *this, cast( getContext() .getAsVariableArrayType(PrivateVD->getType()) ->getSizeExpr()), RValue::get(Size)); EmitVariablyModifiedType(PrivateVD->getType()); auto Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); auto *Init = PrivateVD->getInit(); EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init); EmitAutoVarCleanups(Emission); // Emit private VarDecl with reduction init. auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(), OASELValueLB.getPointer()); auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset); Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast( Ptr, OriginalBaseLValue.getPointer()->getType()); return Address(Ptr, OriginalBaseLValue.getAlignment()); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } else if (auto *ASE = dyn_cast(IRef)) { auto *Base = ASE->getBase()->IgnoreParenImpCasts(); while (auto *TempASE = dyn_cast(Base)) Base = TempASE->getBase()->IgnoreParenImpCasts(); auto *DE = cast(Base); auto *OrigVD = cast(DE->getDecl()); auto ASELValue = EmitLValue(ASE); auto OriginalBaseLValue = EmitLValue(DE); auto BaseLValue = OriginalBaseLValue; auto *Zero = Builder.getInt64(/*C=*/0); llvm::SmallVector Indexes; Indexes.push_back(Zero); auto *ItemTy = ASELValue.getPointer()->getType()->getPointerElementType(); auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType(); while (Ty != ItemTy) { Indexes.push_back(Zero); Ty = Ty->getPointerElementType(); } BaseLValue = MakeAddrLValue( Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes), ASELValue.getAlignment()), ASELValue.getType(), ASELValue.getAlignmentSource()); // Store the address of the original variable associated with the LHS // implicit variable. PrivateScope.addPrivate(LHSVD, [this, ASELValue]() -> Address { return ASELValue.getAddress(); }); // Emit reduction copy. bool IsRegistered = PrivateScope.addPrivate( OrigVD, [this, PrivateVD, BaseLValue, ASELValue, OriginalBaseLValue]() -> Address { // Emit private VarDecl with reduction init. EmitDecl(*PrivateVD); auto Addr = GetAddrOfLocalVar(PrivateVD); auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(), ASELValue.getPointer()); auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset); Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast( Ptr, OriginalBaseLValue.getPointer()->getType()); return Address(Ptr, OriginalBaseLValue.getAlignment()); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } else { auto *OrigVD = cast(cast(IRef)->getDecl()); // Store the address of the original variable associated with the LHS // implicit variable. PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> Address { DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, IRef->getType(), VK_LValue, IRef->getExprLoc()); return EmitLValue(&DRE).getAddress(); }); // Emit reduction copy. bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> Address { // Emit private VarDecl with reduction init. EmitDecl(*PrivateVD); return GetAddrOfLocalVar(PrivateVD); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } ++ILHS, ++IRHS, ++IPriv; d242 6 a247 38 void CodeGenFunction::EmitOMPReductionClauseFinal( const OMPExecutableDirective &D) { if (!HaveInsertPoint()) return; llvm::SmallVector Privates; llvm::SmallVector LHSExprs; llvm::SmallVector RHSExprs; llvm::SmallVector ReductionOps; bool HasAtLeastOneReduction = false; for (const auto *C : D.getClausesOfKind()) { HasAtLeastOneReduction = true; Privates.append(C->privates().begin(), C->privates().end()); LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); } if (HasAtLeastOneReduction) { // Emit nowait reduction if nowait clause is present or directive is a // parallel directive (it always has implicit barrier). CGM.getOpenMPRuntime().emitReduction( *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps, D.getSingleClause() || isOpenMPParallelDirective(D.getDirectiveKind()) || D.getDirectiveKind() == OMPD_simd, D.getDirectiveKind() == OMPD_simd); } } static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { auto CS = cast(S.getAssociatedStmt()); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); if (const auto *NumThreadsClause = S.getSingleClause()) { d249 1 d252 1 a252 1 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( d255 2 a256 15 if (const auto *ProcBindClause = S.getSingleClause()) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); CGF.CGM.getOpenMPRuntime().emitProcBindClause( CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart()); } const Expr *IfCond = nullptr; for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_parallel) { IfCond = C->getCondition(); break; } } CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn, CapturedVars, IfCond); d260 15 a274 22 LexicalScope Scope(*this, S.getSourceRange()); // Emit parallel region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF) { OMPPrivateScope PrivateScope(CGF); bool Copyins = CGF.EmitOMPCopyinClause(S); bool Firstprivates = CGF.EmitOMPFirstprivateClause(S, PrivateScope); if (Copyins || Firstprivates) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables or propagation master's thread // values of threadprivate variables to local instances of that variables // of all other implicit threads. CGF.CGM.getOpenMPRuntime().emitBarrierCall( CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); } CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); CGF.EmitOMPReductionClauseFinal(S); }; emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen); d277 2 a278 2 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit) { d281 1 a281 1 for (auto I : D.updates()) { a283 7 // Update the linear variables. for (const auto *C : D.getClausesOfKind()) { for (auto U : C->updates()) { EmitIgnoredExpr(U); } } d286 1 a286 1 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); d288 1 a288 1 EmitStmt(D.getBody()); d292 6 d300 3 a302 5 void CodeGenFunction::EmitOMPInnerLoop( const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref &BodyGen, const llvm::function_ref &PostIncGen) { d304 1 d314 1 a314 1 if (RequiresCleanup) d319 6 a324 2 // Emit condition. EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S)); d331 1 a331 1 incrementProfileCounter(&S); d337 2 a338 1 BodyGen(*this); d342 1 a342 2 EmitIgnoredExpr(IncExpr); PostIncGen(*this); d350 5 a354 18 void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { if (!HaveInsertPoint()) return; // Emit inits for the linear variables. for (const auto *C : D.getClausesOfKind()) { for (auto Init : C->inits()) { auto *VD = cast(cast(Init)->getDecl()); auto *OrigVD = cast( cast(VD->getInit()->IgnoreImpCasts())->getDecl()); DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, VD->getInit()->getType(), VK_LValue, VD->getInit()->getExprLoc()); AutoVarEmission Emission = EmitAutoVarAlloca(*VD); EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()), /*capturedByInit=*/false); EmitAutoVarCleanups(Emission); d356 1 a356 8 // Emit the linear steps for the linear clauses. // If a step is not constant, it is pre-calculated before the loop. if (auto CS = cast_or_null(C->getCalcStep())) if (auto SaveRef = cast(CS->getLHS())) { EmitVarDecl(*cast(SaveRef->getDecl())); // Emit calculation of the linear step. EmitIgnoredExpr(CS); } d360 22 a381 52 static void emitLinearClauseFinal(CodeGenFunction &CGF, const OMPLoopDirective &D) { if (!CGF.HaveInsertPoint()) return; // Emit the final values of the linear variables. for (const auto *C : D.getClausesOfKind()) { auto IC = C->varlist_begin(); for (auto F : C->finals()) { auto *OrigVD = cast(cast(*IC)->getDecl()); DeclRefExpr DRE(const_cast(OrigVD), CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); Address OrigAddr = CGF.EmitLValue(&DRE).getAddress(); CodeGenFunction::OMPPrivateScope VarScope(CGF); VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; }); (void)VarScope.Privatize(); CGF.EmitIgnoredExpr(F); ++IC; } } } static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D) { if (!CGF.HaveInsertPoint()) return; for (const auto *Clause : D.getClausesOfKind()) { unsigned ClauseAlignment = 0; if (auto AlignmentExpr = Clause->getAlignment()) { auto AlignmentCI = cast(CGF.EmitScalarExpr(AlignmentExpr)); ClauseAlignment = static_cast(AlignmentCI->getZExtValue()); } for (auto E : Clause->varlists()) { unsigned Alignment = ClauseAlignment; if (Alignment == 0) { // OpenMP [2.8.1, Description] // If no optional parameter is specified, implementation-defined default // alignments for SIMD instructions on the target platforms are assumed. Alignment = CGF.getContext() .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( E->getType()->getPointeeType())) .getQuantity(); } assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) && "alignment is not power of 2"); if (Alignment != 0) { llvm::Value *PtrValue = CGF.EmitScalarExpr(E); CGF.EmitAlignmentAssumption(PtrValue, Alignment); } d386 1 a386 1 static void emitPrivateLoopCounters(CodeGenFunction &CGF, d388 1 a388 5 ArrayRef Counters, ArrayRef PrivateCounters) { if (!CGF.HaveInsertPoint()) return; auto I = PrivateCounters.begin(); d390 2 a391 4 auto *VD = cast(cast(E)->getDecl()); auto *PrivateVD = cast(cast(*I)->getDecl()); Address Addr = Address::invalid(); (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address { d393 1 a393 1 auto VarEmission = CGF.EmitAutoVarAlloca(*PrivateVD); d395 1 a395 2 Addr = VarEmission.getAllocatedAddress(); return Addr; d397 3 a399 2 (void)LoopScope.addPrivate(VD, [&]() -> Address { return Addr; }); ++I; d401 1 a401 100 } static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { if (!CGF.HaveInsertPoint()) return; { CodeGenFunction::OMPPrivateScope PreCondScope(CGF); emitPrivateLoopCounters(CGF, PreCondScope, S.counters(), S.private_counters()); (void)PreCondScope.Privatize(); // Get initial values of real counters. for (auto I : S.inits()) { CGF.EmitIgnoredExpr(I); } } // Check that loop is executed at least one time. CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); } static void emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { if (!CGF.HaveInsertPoint()) return; for (const auto *C : D.getClausesOfKind()) { auto CurPrivate = C->privates().begin(); for (auto *E : C->varlists()) { auto *VD = cast(cast(E)->getDecl()); auto *PrivateVD = cast(cast(*CurPrivate)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address { // Emit private VarDecl with copy init. CGF.EmitVarDecl(*PrivateVD); return CGF.GetAddrOfLocalVar(PrivateVD); }); assert(IsRegistered && "linear var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++CurPrivate; } } } static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic) { if (!CGF.HaveInsertPoint()) return; if (const auto *C = D.getSingleClause()) { RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), /*ignoreResult=*/true); llvm::ConstantInt *Val = cast(Len.getScalarVal()); CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. if (!IsMonotonic) CGF.LoopStack.setParallel(!D.getSingleClause()); } else if (const auto *C = D.getSingleClause()) { RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), /*ignoreResult=*/true); llvm::ConstantInt *Val = cast(Len.getScalarVal()); CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. CGF.LoopStack.setParallel(false); } } void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic) { // Walk clauses and process safelen/lastprivate. LoopStack.setParallel(!IsMonotonic); LoopStack.setVectorizeEnable(true); emitSimdlenSafelenClause(*this, D, IsMonotonic); } void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) { if (!HaveInsertPoint()) return; auto IC = D.counters().begin(); for (auto F : D.finals()) { auto *OrigVD = cast(cast((*IC))->getDecl()); if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) { DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); Address OrigAddr = EmitLValue(&DRE).getAddress(); OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; }); (void)VarScope.Privatize(); EmitIgnoredExpr(F); } ++IC; } emitLinearClauseFinal(*this, D); d405 2 a406 109 auto &&CodeGen = [&S](CodeGenFunction &CGF) { // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { auto *ThenBlock = CGF.createBasicBlock("simd.if.then"); ContBlock = CGF.createBasicBlock("simd.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } CGF.EmitOMPSimdInit(S); emitAlignedClause(CGF, S); CGF.EmitOMPLinearClauseInit(S); bool HasLastprivateClause; { OMPPrivateScope LoopScope(CGF); emitPrivateLoopCounters(CGF, LoopScope, S.counters(), S.private_counters()); emitPrivateLinearVars(CGF, S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); // Emit final copy of the lastprivate variables at the end of loops. if (HasLastprivateClause) { CGF.EmitOMPLastprivateClauseFinal(S); } CGF.EmitOMPReductionClauseFinal(S); } CGF.EmitOMPSimdFinal(S); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } }; CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); } void CodeGenFunction::EmitOMPForOuterLoop( OpenMPScheduleClauseKind ScheduleKind, bool IsMonotonic, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind); assert((Ordered || !RT.isStaticNonchunked(ScheduleKind, /*Chunked=*/Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"); // Emit outer loop. // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(dynamic,chunk_size) is specified, the iterations are // distributed to threads in the team in chunks as the threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be distributed. Each chunk contains chunk_size // iterations, except for the last chunk to be distributed, which may have // fewer iterations. When no chunk_size is specified, it defaults to 1. // // When schedule(guided,chunk_size) is specified, the iterations are assigned // to threads in the team in chunks as the executing threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be assigned. For a chunk_size of 1, the size of // each chunk is proportional to the number of unassigned iterations divided // by the number of threads in the team, decreasing to 1. For a chunk_size // with value k (greater than 1), the size of each chunk is determined in the // same way, with the restriction that the chunks do not contain fewer than k // iterations (except for the last chunk to be assigned, which may have fewer // than k iterations). d408 4 a411 15 // When schedule(auto) is specified, the decision regarding scheduling is // delegated to the compiler and/or runtime system. The programmer gives the // implementation the freedom to choose any possible mapping of iterations to // threads in the team. // // When schedule(runtime) is specified, the decision regarding scheduling is // deferred until run time, and the schedule and chunk size are taken from the // run-sched-var ICV. If the ICV is set to auto, the schedule is // implementation defined // // while(__kmpc_dispatch_next(&LB, &UB)) { // idx = LB; // while (idx <= UB) { BODY; ++idx; // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. // } // inner loop d414 1 a414 4 // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(static, chunk_size) is specified, iterations are divided into // chunks of size chunk_size, and the chunks are assigned to the threads in // the team in a round-robin fashion in the order of the thread number. d416 2 a417 5 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { // while (idx <= UB) { BODY; ++idx; } // inner loop // LB = LB + ST; // UB = UB + ST; // } d420 27 a446 11 const Expr *IVExpr = S.getIterationVariable(); const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); if (DynamicOrOrdered) { llvm::Value *UBVal = EmitScalarExpr(S.getLastIteration()); RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, UBVal, Chunk); } else { RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk); d449 2 a450 1 auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); d452 3 a454 4 // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.dispatch.cond"); EmitBlock(CondBlock); LoopStack.push(CondBlock); d456 5 a460 12 llvm::Value *BoolCondVal = nullptr; if (!DynamicOrOrdered) { // UB = min(UB, GlobalUB) EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB EmitIgnoredExpr(S.getInit()); // IV < UB BoolCondVal = EvaluateExprAsBool(S.getCond()); } else { BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, IL, LB, UB, ST); } d462 7 a468 11 // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. auto ExitBlock = LoopExit.getBlock(); if (LoopScope.requiresCleanups()) ExitBlock = createBasicBlock("omp.dispatch.cleanup"); auto LoopBody = createBasicBlock("omp.dispatch.body"); Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); a469 1 EmitBlock(LoopBody); d471 26 a496 35 // Emit "IV = LB" (in case of static schedule, we have already calculated new // LB for loop condition and emitted it above). if (DynamicOrOrdered) EmitIgnoredExpr(S.getInit()); // Create a block for the increment. auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); // Generate !llvm.loop.parallel metadata for loads and stores for loops // with dynamic/guided scheduling and without ordered clause. if (!isOpenMPSimdDirective(S.getDirectiveKind())) LoopStack.setParallel(!IsMonotonic); else EmitOMPSimdInit(S, IsMonotonic); SourceLocation Loc = S.getLocStart(); EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S, LoopExit](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, LoopExit); CGF.EmitStopPoint(&S); }, [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) { if (Ordered) { CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd( CGF, Loc, IVSize, IVSigned); } }); EmitBlock(Continue.getBlock()); BreakContinueStack.pop_back(); if (!DynamicOrOrdered) { // Emit "LB = LB + Stride", "UB = UB + Stride". EmitIgnoredExpr(S.getNextLowerBound()); EmitIgnoredExpr(S.getNextUpperBound()); d499 2 a500 8 EmitBranch(CondBlock); LoopStack.pop(); // Emit the fall-through block. EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. if (!DynamicOrOrdered) RT.emitForStaticFinish(*this, S.getLocEnd()); d511 1 a511 49 namespace { struct ScheduleKindModifiersTy { OpenMPScheduleClauseKind Kind; OpenMPScheduleClauseModifier M1; OpenMPScheduleClauseModifier M2; ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2) : Kind(Kind), M1(M1), M2(M2) {} }; } // namespace static std::pair emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, bool OuterRegion) { // Detect the loop schedule kind and chunk. auto ScheduleKind = OMPC_SCHEDULE_unknown; OpenMPScheduleClauseModifier M1 = OMPC_SCHEDULE_MODIFIER_unknown; OpenMPScheduleClauseModifier M2 = OMPC_SCHEDULE_MODIFIER_unknown; llvm::Value *Chunk = nullptr; if (const auto *C = S.getSingleClause()) { ScheduleKind = C->getScheduleKind(); M1 = C->getFirstScheduleModifier(); M2 = C->getSecondScheduleModifier(); if (const auto *Ch = C->getChunkSize()) { if (auto *ImpRef = cast_or_null(C->getHelperChunkSize())) { if (OuterRegion) { const VarDecl *ImpVar = cast(ImpRef->getDecl()); CGF.EmitVarDecl(*ImpVar); CGF.EmitStoreThroughLValue( CGF.EmitAnyExpr(Ch), CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(ImpVar), ImpVar->getType())); } else { Ch = ImpRef; } } if (!C->getHelperChunkSize() || !OuterRegion) { Chunk = CGF.EmitScalarExpr(Ch); Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType(), S.getLocStart()); } } } return std::make_pair(Chunk, ScheduleKindModifiersTy(ScheduleKind, M1, M2)); } bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { a527 1 bool HasLastprivateClause; d531 6 a536 18 // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return false; } else { auto *ThenBlock = createBasicBlock("omp.precond.then"); ContBlock = createBasicBlock("omp.precond.end"); emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, getProfileCount(&S)); EmitBlock(ThenBlock); incrementProfileCounter(&S); } emitAlignedClause(*this, S); EmitOMPLinearClauseInit(S); d550 1 a550 14 if (EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. CGM.getOpenMPRuntime().emitBarrierCall( *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); } EmitOMPPrivateClause(S, LoopScope); HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPReductionClauseInit(S, LoopScope); emitPrivateLoopCounters(*this, LoopScope, S.counters(), S.private_counters()); emitPrivateLinearVars(*this, S, LoopScope); (void)LoopScope.Privatize(); d553 11 a563 8 llvm::Value *Chunk; OpenMPScheduleClauseKind ScheduleKind; auto ScheduleInfo = emitScheduleClause(*this, S, /*OuterRegion=*/false); Chunk = ScheduleInfo.first; ScheduleKind = ScheduleInfo.second.Kind; const OpenMPScheduleClauseModifier M1 = ScheduleInfo.second.M1; const OpenMPScheduleClauseModifier M2 = ScheduleInfo.second.M2; a565 5 const bool Ordered = S.getSingleClause() != nullptr; // OpenMP 4.5, 2.7.1 Loop Construct, Description. // If the static schedule kind is specified or if the ordered clause is // specified, and if no monotonic modifier is specified, the effect will // be as if the monotonic modifier was specified. d567 1 a567 4 /* Chunked */ Chunk != nullptr) && !Ordered) { if (isOpenMPSimdDirective(S.getDirectiveKind())) EmitOMPSimdInit(S, /*IsMonotonic=*/true); d573 3 a575 6 RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); d581 1 a581 8 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S, LoopExit](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, LoopExit); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); EmitBlock(LoopExit.getBlock()); d583 3 a585 21 RT.emitForStaticFinish(*this, S.getLocStart()); } else { const bool IsMonotonic = Ordered || ScheduleKind == OMPC_SCHEDULE_static || ScheduleKind == OMPC_SCHEDULE_unknown || M1 == OMPC_SCHEDULE_MODIFIER_monotonic || M2 == OMPC_SCHEDULE_MODIFIER_monotonic; // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered, LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk); } EmitOMPReductionClauseFinal(S); // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivateClause) EmitOMPLastprivateClauseFinal( S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S); d588 2 a589 4 if (ContBlock) { EmitBranch(ContBlock); EmitBlock(ContBlock, true); } a590 1 return HasLastprivateClause; d594 2 a595 7 LexicalScope Scope(*this, S.getSourceRange()); bool HasLastprivates = false; auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { HasLastprivates = CGF.EmitOMPWorksharingLoop(S); }; CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, S.hasCancel()); d597 3 a599 5 // Emit an implicit barrier at the end. if (!S.getSingleClause() || HasLastprivates) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); } } d601 1 a601 7 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); bool HasLastprivates = false; auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { HasLastprivates = CGF.EmitOMPWorksharingLoop(S); }; CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); d604 4 a607 3 if (!S.getSingleClause() || HasLastprivates) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); } d610 3 a612 115 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init = nullptr) { auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); if (Init) CGF.EmitScalarInit(Init, LVal); return LVal; } OpenMPDirectiveKind CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { auto *Stmt = cast(S.getAssociatedStmt())->getCapturedStmt(); auto *CS = dyn_cast(Stmt); bool HasLastprivates = false; auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF) { auto &C = CGF.CGM.getContext(); auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); // Emit helper vars inits. LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", CGF.Builder.getInt32(0)); auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1) : CGF.Builder.getInt32(0); LValue UB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", CGF.Builder.getInt32(1)); LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", CGF.Builder.getInt32(0)); // Loop counter. LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); // Generate condition for loop. BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, OK_Ordinary, S.getLocStart(), /*fpContractable=*/false); // Increment for loop counter. UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary, S.getLocStart()); auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) { // Iterate through all sections and emit a switch construct: // switch (IV) { // case 0: // ; // break; // ... // case - 1: // - 1]>; // break; // } // .omp.sections.exit: auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); auto *SwitchStmt = CGF.Builder.CreateSwitch( CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB, CS == nullptr ? 1 : CS->size()); if (CS) { unsigned CaseNumber = 0; for (auto *SubStmt : CS->children()) { auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); CGF.EmitBlock(CaseBB); SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); CGF.EmitStmt(SubStmt); CGF.EmitBranch(ExitBB); ++CaseNumber; } } else { auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); CGF.EmitBlock(CaseBB); SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); CGF.EmitStmt(Stmt); CGF.EmitBranch(ExitBB); } CGF.EmitBlock(ExitBB, /*IsFinished=*/true); }; CodeGenFunction::OMPPrivateScope LoopScope(CGF); if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. CGF.CGM.getOpenMPRuntime().emitBarrierCall( CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); } CGF.EmitOMPPrivateClause(S, LoopScope); HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); (void)LoopScope.Privatize(); // Emit static non-chunked loop. CGF.CGM.getOpenMPRuntime().emitForStaticInit( CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); // UB = min(UB, GlobalUB); auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); auto *MinUBGlobalUB = CGF.Builder.CreateSelect( CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); // IV = LB; CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV); // while (idx <= UB) { BODY; ++idx; } CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, [](CodeGenFunction &) {}); // Tell the runtime we are done. CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart()); CGF.EmitOMPReductionClauseFinal(S); // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivates) CGF.EmitOMPLastprivateClauseFinal( S, CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart()))); }; d614 2 a615 26 bool HasCancel = false; if (auto *OSD = dyn_cast(&S)) HasCancel = OSD->hasCancel(); else if (auto *OPSD = dyn_cast(&S)) HasCancel = OPSD->hasCancel(); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen, HasCancel); // Emit barrier for lastprivates only if 'sections' directive has 'nowait' // clause. Otherwise the barrier will be generated by the codegen for the // directive. if (HasLastprivates && S.getSingleClause()) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_unknown); } return OMPD_sections; } void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); OpenMPDirectiveKind EmittedAs = EmitSections(S); // Emit an implicit barrier at the end. if (!S.getSingleClause()) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs); } d618 2 a619 7 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen, S.hasCancel()); d622 2 a623 40 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { llvm::SmallVector CopyprivateVars; llvm::SmallVector DestExprs; llvm::SmallVector SrcExprs; llvm::SmallVector AssignmentOps; // Check if there are any 'copyprivate' clauses associated with this // 'single' // construct. // Build a list of copyprivate variables along with helper expressions // (, , = expressions) for (const auto *C : S.getClausesOfKind()) { CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); DestExprs.append(C->destination_exprs().begin(), C->destination_exprs().end()); SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); AssignmentOps.append(C->assignment_ops().begin(), C->assignment_ops().end()); } LexicalScope Scope(*this, S.getSourceRange()); // Emit code for 'single' region along with 'copyprivate' clauses bool HasFirstprivates; auto &&CodeGen = [&S, &HasFirstprivates](CodeGenFunction &CGF) { CodeGenFunction::OMPPrivateScope SingleScope(CGF); HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); CGF.EmitOMPPrivateClause(S, SingleScope); (void)SingleScope.Privatize(); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), CopyprivateVars, DestExprs, SrcExprs, AssignmentOps); // Emit an implicit barrier at the end (to avoid data race on firstprivate // init or if no 'nowait' clause was specified and no 'copyprivate' clause). if ((!S.getSingleClause() || HasFirstprivates) && CopyprivateVars.empty()) { CGM.getOpenMPRuntime().emitBarrierCall( *this, S.getLocStart(), S.getSingleClause() ? OMPD_unknown : OMPD_single); } d627 6 a632 5 LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart()); d636 13 a648 22 LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; Expr *Hint = nullptr; if (auto *HintClause = S.getSingleClause()) Hint = HintClause->getHint(); CGM.getOpenMPRuntime().emitCriticalRegion(*this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart(), Hint); } void CodeGenFunction::EmitOMPParallelForDirective( const OMPParallelForDirective &S) { // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. LexicalScope Scope(*this, S.getSourceRange()); (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitOMPWorksharingLoop(S); }; emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen); d652 2 a653 9 const OMPParallelForSimdDirective &S) { // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. LexicalScope Scope(*this, S.getSourceRange()); (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitOMPWorksharingLoop(S); }; emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen); d657 2 a658 8 const OMPParallelSectionsDirective &S) { // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'sections' directive. LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { (void)CGF.EmitSections(S); }; emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen); d661 2 a662 124 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { // Emit outlined function for task construct. LexicalScope Scope(*this, S.getSourceRange()); auto CS = cast(S.getAssociatedStmt()); auto CapturedStruct = GenerateCapturedStmtArgument(*CS); auto *I = CS->getCapturedDecl()->param_begin(); auto *PartId = std::next(I); // The first function argument for tasks is a thread id, the second one is a // part id (0 for tied tasks, >=0 for untied task). llvm::DenseSet EmittedAsPrivate; // Get list of private variables. llvm::SmallVector PrivateVars; llvm::SmallVector PrivateCopies; for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); for (auto *IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { PrivateVars.push_back(*IRef); PrivateCopies.push_back(IInit); } ++IRef; } } EmittedAsPrivate.clear(); // Get list of firstprivate variables. llvm::SmallVector FirstprivateVars; llvm::SmallVector FirstprivateCopies; llvm::SmallVector FirstprivateInits; for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto IElemInitRef = C->inits().begin(); for (auto *IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { FirstprivateVars.push_back(*IRef); FirstprivateCopies.push_back(IInit); FirstprivateInits.push_back(*IElemInitRef); } ++IRef, ++IElemInitRef; } } // Build list of dependences. llvm::SmallVector, 8> Dependences; for (const auto *C : S.getClausesOfKind()) { for (auto *IRef : C->varlists()) { Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef)); } } auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars]( CodeGenFunction &CGF) { // Set proper addresses for generated private copies. auto *CS = cast(S.getAssociatedStmt()); OMPPrivateScope Scope(CGF); if (!PrivateVars.empty() || !FirstprivateVars.empty()) { auto *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3))); auto *PrivatesPtr = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2))); // Map privates. llvm::SmallVector, 16> PrivatePtrs; llvm::SmallVector CallArgs; CallArgs.push_back(PrivatesPtr); for (auto *E : PrivateVars) { auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); CallArgs.push_back(PrivatePtr.getPointer()); } for (auto *E : FirstprivateVars) { auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); CallArgs.push_back(PrivatePtr.getPointer()); } CGF.EmitRuntimeCall(CopyFn, CallArgs); for (auto &&Pair : PrivatePtrs) { Address Replacement(CGF.Builder.CreateLoad(Pair.second), CGF.getContext().getDeclAlign(Pair.first)); Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); } } (void)Scope.Privatize(); if (*PartId) { // TODO: emit code for untied tasks. } CGF.EmitStmt(CS->getCapturedStmt()); }; auto OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, OMPD_task, CodeGen); // Check if we should emit tied or untied task. bool Tied = !S.getSingleClause(); // Check if the task is final llvm::PointerIntPair Final; if (const auto *Clause = S.getSingleClause()) { // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. auto *Cond = Clause->getCondition(); bool CondConstant; if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) Final.setInt(CondConstant); else Final.setPointer(EvaluateExprAsBool(Cond)); } else { // By default the task is not final. Final.setInt(/*IntVal=*/false); } auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); const Expr *IfCond = nullptr; for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_task) { IfCond = C->getCondition(); break; } } CGM.getOpenMPRuntime().emitTaskCall( *this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy, CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars, FirstprivateCopies, FirstprivateInits, Dependences); d665 2 a666 3 void CodeGenFunction::EmitOMPTaskyieldDirective( const OMPTaskyieldDirective &S) { CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); d670 1 a670 1 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); d673 2 a674 11 void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPTaskgroupDirective( const OMPTaskgroupDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); d678 10 a687 12 CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) { return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } return llvm::None; }(), S.getLocStart()); } void CodeGenFunction::EmitOMPDistributeDirective( const OMPDistributeDirective &S) { llvm_unreachable("CodeGen for 'omp distribute' is not supported yet."); d690 2 a691 8 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S) { CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); CodeGenFunction::CGCapturedStmtInfo CapStmtInfo; CGF.CapturedStmtInfo = &CapStmtInfo; auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); Fn->addFnAttr(llvm::Attribute::NoInline); return Fn; d694 2 a695 18 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { if (!S.getAssociatedStmt()) return; LexicalScope Scope(*this, S.getSourceRange()); auto *C = S.getSingleClause(); auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF) { if (C) { auto CS = cast(S.getAssociatedStmt()); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars); } else { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); } }; CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C); d698 2 a699 502 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc) { assert(CGF.hasScalarEvaluationKind(DestType) && "DestType must have scalar evaluation kind."); assert(!Val.isAggregate() && "Must be a scalar or complex."); return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType, Loc) : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType, DestType, Loc); } static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc) { assert(CGF.getEvaluationKind(DestType) == TEK_Complex && "DestType must have complex evaluation kind."); CodeGenFunction::ComplexPairTy ComplexVal; if (Val.isScalar()) { // Convert the input element to the element type of the complex. auto DestElementType = DestType->castAs()->getElementType(); auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType, Loc); ComplexVal = CodeGenFunction::ComplexPairTy( ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); } else { assert(Val.isComplex() && "Must be a scalar or complex."); auto SrcElementType = SrcType->castAs()->getElementType(); auto DestElementType = DestType->castAs()->getElementType(); ComplexVal.first = CGF.EmitScalarConversion( Val.getComplexVal().first, SrcElementType, DestElementType, Loc); ComplexVal.second = CGF.EmitScalarConversion( Val.getComplexVal().second, SrcElementType, DestElementType, Loc); } return ComplexVal; } static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal) { if (LVal.isGlobalReg()) { CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); } else { CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic, LVal.isVolatile(), /*IsInit=*/false); } } void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc) { switch (getEvaluationKind(LVal.getType())) { case TEK_Scalar: EmitStoreThroughLValue(RValue::get(convertToScalarValue( *this, RVal, RValTy, LVal.getType(), Loc)), LVal); break; case TEK_Complex: EmitStoreOfComplex( convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal, /*isInit=*/false); break; case TEK_Aggregate: llvm_unreachable("Must be a scalar or complex."); } } static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc) { // v = x; assert(V->isLValue() && "V of 'omp atomic read' is not lvalue"); assert(X->isLValue() && "X of 'omp atomic read' is not lvalue"); LValue XLValue = CGF.EmitLValue(X); LValue VLValue = CGF.EmitLValue(V); RValue Res = XLValue.isGlobalReg() ? CGF.EmitLoadOfLValue(XLValue, Loc) : CGF.EmitAtomicLoad(XLValue, Loc, IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic, XLValue.isVolatile()); // OpenMP, 2.12.6, atomic Construct // Any atomic construct with a seq_cst clause forces the atomically // performed operation to include an implicit flush operation without a // list. if (IsSeqCst) CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); } static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc) { // x = expr; assert(X->isLValue() && "X of 'omp atomic write' is not lvalue"); emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); // OpenMP, 2.12.6, atomic Construct // Any atomic construct with a seq_cst clause forces the atomically // performed operation to include an implicit flush operation without a // list. if (IsSeqCst) CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); } static std::pair emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart) { auto &Context = CGF.CGM.getContext(); // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' // expression is simple and atomic is allowed for the given type for the // target platform. if (BO == BO_Comma || !Update.isScalar() || !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() || (!isa(Update.getScalarVal()) && (Update.getScalarVal()->getType() != X.getAddress().getElementType())) || !X.getAddress().getElementType()->isIntegerTy() || !Context.getTargetInfo().hasBuiltinAtomic( Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) return std::make_pair(false, RValue::get(nullptr)); llvm::AtomicRMWInst::BinOp RMWOp; switch (BO) { case BO_Add: RMWOp = llvm::AtomicRMWInst::Add; break; case BO_Sub: if (!IsXLHSInRHSPart) return std::make_pair(false, RValue::get(nullptr)); RMWOp = llvm::AtomicRMWInst::Sub; break; case BO_And: RMWOp = llvm::AtomicRMWInst::And; break; case BO_Or: RMWOp = llvm::AtomicRMWInst::Or; break; case BO_Xor: RMWOp = llvm::AtomicRMWInst::Xor; break; case BO_LT: RMWOp = X.getType()->hasSignedIntegerRepresentation() ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min : llvm::AtomicRMWInst::Max) : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin : llvm::AtomicRMWInst::UMax); break; case BO_GT: RMWOp = X.getType()->hasSignedIntegerRepresentation() ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max : llvm::AtomicRMWInst::Min) : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax : llvm::AtomicRMWInst::UMin); break; case BO_Assign: RMWOp = llvm::AtomicRMWInst::Xchg; break; case BO_Mul: case BO_Div: case BO_Rem: case BO_Shl: case BO_Shr: case BO_LAnd: case BO_LOr: return std::make_pair(false, RValue::get(nullptr)); case BO_PtrMemD: case BO_PtrMemI: case BO_LE: case BO_GE: case BO_EQ: case BO_NE: case BO_AddAssign: case BO_SubAssign: case BO_AndAssign: case BO_OrAssign: case BO_XorAssign: case BO_MulAssign: case BO_DivAssign: case BO_RemAssign: case BO_ShlAssign: case BO_ShrAssign: case BO_Comma: llvm_unreachable("Unsupported atomic update operation"); } auto *UpdateVal = Update.getScalarVal(); if (auto *IC = dyn_cast(UpdateVal)) { UpdateVal = CGF.Builder.CreateIntCast( IC, X.getAddress().getElementType(), X.getType()->hasSignedIntegerRepresentation()); } auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); return std::make_pair(true, RValue::get(Res)); } std::pair CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref &CommonGen) { // Update expressions are allowed to have the following forms: // x binop= expr; -> xrval + expr; // x++, ++x -> xrval + 1; // x--, --x -> xrval - 1; // x = x binop expr; -> xrval binop expr // x = expr Op x; - > expr binop xrval; auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart); if (!Res.first) { if (X.isGlobalReg()) { // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop // 'xrval'. EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); } else { // Perform compare-and-swap procedure. EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); } } return Res; } static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc) { assert(isa(UE->IgnoreImpCasts()) && "Update expr in 'atomic update' must be a binary operator."); auto *BOUE = cast(UE->IgnoreImpCasts()); // Update expressions are allowed to have the following forms: // x binop= expr; -> xrval + expr; // x++, ++x -> xrval + 1; // x--, --x -> xrval - 1; // x = x binop expr; -> xrval binop expr // x = expr Op x; - > expr binop xrval; assert(X->isLValue() && "X of 'omp atomic update' is not lvalue"); LValue XLValue = CGF.EmitLValue(X); RValue ExprRValue = CGF.EmitAnyExpr(E); auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; auto Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue { CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); return CGF.EmitAnyExpr(UE); }; (void)CGF.EmitOMPAtomicSimpleUpdateExpr( XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); // OpenMP, 2.12.6, atomic Construct // Any atomic construct with a seq_cst clause forces the atomically // performed operation to include an implicit flush operation without a // list. if (IsSeqCst) CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); } static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc) { switch (CGF.getEvaluationKind(ResType)) { case TEK_Scalar: return RValue::get( convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); case TEK_Complex: { auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); return RValue::getComplex(Res.first, Res.second); } case TEK_Aggregate: break; } llvm_unreachable("Must be a scalar or complex."); } static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc) { assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue"); assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue"); RValue NewVVal; LValue VLValue = CGF.EmitLValue(V); LValue XLValue = CGF.EmitLValue(X); RValue ExprRValue = CGF.EmitAnyExpr(E); auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; QualType NewVValType; if (UE) { // 'x' is updated with some additional value. assert(isa(UE->IgnoreImpCasts()) && "Update expr in 'atomic capture' must be a binary operator."); auto *BOUE = cast(UE->IgnoreImpCasts()); // Update expressions are allowed to have the following forms: // x binop= expr; -> xrval + expr; // x++, ++x -> xrval + 1; // x--, --x -> xrval - 1; // x = x binop expr; -> xrval binop expr // x = expr Op x; - > expr binop xrval; auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; NewVValType = XRValExpr->getType(); auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, IsSeqCst, IsPostfixUpdate](RValue XRValue) -> RValue { CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); RValue Res = CGF.EmitAnyExpr(UE); NewVVal = IsPostfixUpdate ? XRValue : Res; return Res; }; auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); if (Res.first) { // 'atomicrmw' instruction was generated. if (IsPostfixUpdate) { // Use old value from 'atomicrmw'. NewVVal = Res.second; } else { // 'atomicrmw' does not provide new value, so evaluate it using old // value of 'x'. CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); NewVVal = CGF.EmitAnyExpr(UE); } } } else { // 'x' is simply rewritten with some 'expr'. NewVValType = X->getType().getNonReferenceType(); ExprRValue = convertToType(CGF, ExprRValue, E->getType(), X->getType().getNonReferenceType(), Loc); auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue { NewVVal = XRValue; return ExprRValue; }; // Try to perform atomicrmw xchg, otherwise simple exchange. auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO, Loc, Gen); if (Res.first) { // 'atomicrmw' instruction was generated. NewVVal = IsPostfixUpdate ? Res.second : ExprRValue; } } // Emit post-update store to 'v' of old/new 'x' value. CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); // OpenMP, 2.12.6, atomic Construct // Any atomic construct with a seq_cst clause forces the atomically // performed operation to include an implicit flush operation without a // list. if (IsSeqCst) CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); } static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc) { switch (Kind) { case OMPC_read: EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); break; case OMPC_write: EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); break; case OMPC_unknown: case OMPC_update: EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); break; case OMPC_capture: EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, IsXLHSInRHSPart, Loc); break; case OMPC_if: case OMPC_final: case OMPC_num_threads: case OMPC_private: case OMPC_firstprivate: case OMPC_lastprivate: case OMPC_reduction: case OMPC_safelen: case OMPC_simdlen: case OMPC_collapse: case OMPC_default: case OMPC_seq_cst: case OMPC_shared: case OMPC_linear: case OMPC_aligned: case OMPC_copyin: case OMPC_copyprivate: case OMPC_flush: case OMPC_proc_bind: case OMPC_schedule: case OMPC_ordered: case OMPC_nowait: case OMPC_untied: case OMPC_threadprivate: case OMPC_depend: case OMPC_mergeable: case OMPC_device: case OMPC_threads: case OMPC_simd: case OMPC_map: case OMPC_num_teams: case OMPC_thread_limit: case OMPC_priority: case OMPC_grainsize: case OMPC_nogroup: case OMPC_num_tasks: case OMPC_hint: llvm_unreachable("Clause is not allowed in 'omp atomic'."); } } void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { bool IsSeqCst = S.getSingleClause(); OpenMPClauseKind Kind = OMPC_unknown; for (auto *C : S.clauses()) { // Find first clause (skip seq_cst clause, if it is first). if (C->getClauseKind() != OMPC_seq_cst) { Kind = C->getClauseKind(); break; } } const auto *CS = S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); if (const auto *EWC = dyn_cast(CS)) { enterFullExpression(EWC); } // Processing for statements under 'atomic capture'. if (const auto *Compound = dyn_cast(CS)) { for (const auto *C : Compound->body()) { if (const auto *EWC = dyn_cast(C)) { enterFullExpression(EWC); } } } LexicalScope Scope(*this, S.getSourceRange()); auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF) { CGF.EmitStopPoint(CS); EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), S.getV(), S.getExpr(), S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart()); }; CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen); } void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) { LexicalScope Scope(*this, S.getSourceRange()); const CapturedStmt &CS = *cast(S.getAssociatedStmt()); llvm::SmallVector CapturedVars; GenerateOpenMPCapturedVars(CS, CapturedVars); llvm::Function *Fn = nullptr; llvm::Constant *FnID = nullptr; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) { IfCond = C->getCondition(); } // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) { Device = C->getDevice(); } // Check if we have an if clause whose conditional always evaluates to false // or if we do not have any targets specified. If so the target region is not // an offload entry point. bool IsOffloadEntry = true; if (IfCond) { bool Val; if (ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) IsOffloadEntry = false; } if (CGM.getLangOpts().OMPTargetTriples.empty()) IsOffloadEntry = false; assert(CurFuncDecl && "No parent declaration for target region!"); StringRef ParentName; // In case we have Ctors/Dtors we use the complete type variant to produce // the mangling of the device outlined kernel. if (auto *D = dyn_cast(CurFuncDecl)) ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete)); else if (auto *D = dyn_cast(CurFuncDecl)) ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete)); else ParentName = CGM.getMangledName(GlobalDecl(cast(CurFuncDecl))); CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID, IsOffloadEntry); CGM.getOpenMPRuntime().emitTargetCall(*this, S, Fn, FnID, IfCond, Device, CapturedVars); a705 55 void CodeGenFunction::EmitOMPCancellationPointDirective( const OMPCancellationPointDirective &S) { CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(), S.getCancelRegion()); } void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { const Expr *IfCond = nullptr; for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_cancel) { IfCond = C->getCondition(); break; } } CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond, S.getCancelRegion()); } CodeGenFunction::JumpDest CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { if (Kind == OMPD_parallel || Kind == OMPD_task) return ReturnBlock; assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for); return BreakContinueStack.back().BreakBlock; } // Generate the instructions for '#pragma omp target data' directive. void CodeGenFunction::EmitOMPTargetDataDirective( const OMPTargetDataDirective &S) { // emit the code inside the construct for now auto CS = cast(S.getAssociatedStmt()); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_data, [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) { // emit the code inside the construct for now auto CS = cast(S.getAssociatedStmt()); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_taskloop, [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTaskLoopSimdDirective( const OMPTaskLoopSimdDirective &S) { // emit the code inside the construct for now auto CS = cast(S.getAssociatedStmt()); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_taskloop_simd, [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); }); } @ 1.1.1.4.2.1 log @Sync with HEAD @ text @a13 1 #include "CGCleanup.h" a19 2 #include "clang/AST/DeclOpenMP.h" #include "llvm/IR/CallSite.h" a22 99 namespace { /// Lexical scope for OpenMP executable constructs, that handles correct codegen /// for captured expressions. class OMPLexicalScope final : public CodeGenFunction::LexicalScope { void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { for (const auto *C : S.clauses()) { if (auto *CPI = OMPClauseWithPreInit::get(C)) { if (auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { for (const auto *I : PreInit->decls()) { if (!I->hasAttr()) CGF.EmitVarDecl(cast(*I)); else { CodeGenFunction::AutoVarEmission Emission = CGF.EmitAutoVarAlloca(cast(*I)); CGF.EmitAutoVarCleanups(Emission); } } } } } } CodeGenFunction::OMPPrivateScope InlinedShareds; static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || (CGF.CurCodeDecl && isa(CGF.CurCodeDecl)); } public: OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S, bool AsInlined = false) : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), InlinedShareds(CGF) { emitPreInitStmt(CGF, S); if (AsInlined) { if (S.hasAssociatedStmt()) { auto *CS = cast(S.getAssociatedStmt()); for (auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); DeclRefExpr DRE(const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, SourceLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); } } (void)InlinedShareds.Privatize(); } } } }; /// Private scope for OpenMP loop-based directives, that supports capturing /// of used expression from loop statement. class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { if (auto *LD = dyn_cast(&S)) { if (auto *PreInits = cast_or_null(LD->getPreInits())) { for (const auto *I : PreInits->decls()) CGF.EmitVarDecl(cast(*I)); } } } public: OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) : CodeGenFunction::RunCleanupsScope(CGF) { emitPreInitStmt(CGF, S); } }; } // namespace llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) { auto &C = getContext(); llvm::Value *Size = nullptr; auto SizeInChars = C.getTypeSizeInChars(Ty); if (SizeInChars.isZero()) { // getTypeSizeInChars() returns 0 for a VLA. while (auto *VAT = C.getAsVariableArrayType(Ty)) { llvm::Value *ArraySize; std::tie(ArraySize, Ty) = getVLASize(VAT); Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize; } SizeInChars = C.getTypeSizeInChars(Ty); if (SizeInChars.isZero()) return llvm::ConstantInt::get(SizeTy, /*V=*/0); Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); } else Size = CGM.getSize(SizeInChars); return Size; } d37 4 a40 27 else if (CurCap->capturesVariableByCopy()) { llvm::Value *CV = EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal(); // If the field is not a pointer, we need to save the actual value // and load it as a void pointer. if (!CurField->getType()->isAnyPointerType()) { auto &Ctx = getContext(); auto DstAddr = CreateMemTemp( Ctx.getUIntPtrType(), Twine(CurCap->getCapturedVar()->getName()) + ".casted"); LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); auto *SrcAddrVal = EmitScalarConversion( DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), SourceLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); // Store the value using the source type pointer. EmitStoreThroughLValue(RValue::get(CV), SrcLV); // Load the value using the destination type pointer. CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal(); } CapturedVars.push_back(CV); } else { d66 1 a66 1 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true); d110 2 a111 9 if (ArgType->isVariablyModifiedType()) { bool IsReference = ArgType->isLValueReferenceType(); ArgType = getContext().getCanonicalParamType(ArgType.getNonReferenceType()); if (IsReference && !ArgType->isPointerType()) { ArgType = getContext().getLValueReferenceType( ArgType, /*SpelledAsLValue=*/false); } } d123 2 a124 1 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args); d143 2 a144 13 const VarDecl *CurVD = I->getCapturedVar(); Address LocalAddr = GetAddrOfLocalVar(Args[Cnt]); // If the variable is a reference we need to materialize it here. if (CurVD->getType()->isReferenceType()) { Address RefAddr = CreateMemTemp(CurVD->getType(), getPointerAlign(), ".materialized_ref"); EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr, /*Volatile=*/false, CurVD->getType()); LocalAddr = RefAddr; } setAddrOfLocalVar(CurVD, LocalAddr); ++Cnt; ++I; d165 2 a166 8 if (ArgLVal.getType()->isLValueReferenceType()) { ArgAddr = EmitLoadOfReference( ArgAddr, ArgLVal.getType()->castAs()); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { assert(ArgLVal.getType()->isPointerType()); ArgAddr = EmitLoadOfPointer( ArgAddr, ArgLVal.getType()->castAs()); } d175 4 a178 3 setAddrOfLocalVar(Var, castValueFromUintptr(*this, FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType())); d185 1 a185 2 ++Cnt; ++I; a258 62 /// Check if the combiner is a call to UDR combiner and if it is so return the /// UDR decl used for reduction. static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp) { if (auto *CE = dyn_cast(ReductionOp)) if (auto *OVE = dyn_cast(CE->getCallee())) if (auto *DRE = dyn_cast(OVE->getSourceExpr()->IgnoreImpCasts())) if (auto *DRD = dyn_cast(DRE->getDecl())) return DRD; return nullptr; } static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty) { if (DRD->getInitializer()) { std::pair Reduction = CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); auto *CE = cast(InitOp); auto *OVE = cast(CE->getCallee()); const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts(); const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts(); auto *LHSDRE = cast(cast(LHS)->getSubExpr()); auto *RHSDRE = cast(cast(RHS)->getSubExpr()); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); PrivateScope.addPrivate(cast(LHSDRE->getDecl()), [=]() -> Address { return Private; }); PrivateScope.addPrivate(cast(RHSDRE->getDecl()), [=]() -> Address { return Original; }); (void)PrivateScope.Privatize(); RValue Func = RValue::get(Reduction.second); CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); CGF.EmitIgnoredExpr(InitOp); } else { llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, ".init"); LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: InitRVal = CGF.EmitLoadOfLValue(LV, SourceLocation()); break; case TEK_Complex: InitRVal = RValue::getComplex(CGF.EmitLoadOfComplex(LV, SourceLocation())); break; case TEK_Aggregate: InitRVal = RValue::getAggregate(LV.getAddress()); break; } OpaqueValueExpr OVE(SourceLocation(), Ty, VK_RValue); CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), /*IsInitializer=*/false); } } a262 1 /// \param SrcAddr Address of the original array. d264 1 a264 3 QualType Type, const Expr *Init, Address SrcAddr = Address::invalid()) { auto *DRD = getReductionInit(Init); d273 1 a273 7 if (DRD) SrcAddr = CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); llvm::Value *SrcBegin = nullptr; if (DRD) SrcBegin = SrcAddr.getPointer(); a289 10 llvm::PHINode *SrcElementPHI = nullptr; Address SrcElementCurrent = Address::invalid(); if (DRD) { SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); SrcElementPHI->addIncoming(SrcBegin, EntryBB); SrcElementCurrent = Address(SrcElementPHI, SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); } d300 2 a301 13 if (DRD && (DRD->getInitializer() || !Init)) { emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, SrcElementCurrent, ElementTy); } else CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), /*IsInitializer=*/false); } if (DRD) { // Shift the address forward by one element. auto SrcElementNext = CGF.Builder.CreateConstGEP1_32( SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); a358 7 bool FirstprivateIsLastprivate = false; llvm::DenseSet Lastprivates; for (const auto *C : D.getClausesOfKind()) { for (const auto *D : C->varlists()) Lastprivates.insert( cast(cast(D)->getDecl())->getCanonicalDecl()); } a359 1 CGCapturedStmtInfo CapturesInfo(cast(*D.getAssociatedStmt())); d365 2 a366 14 bool ThisFirstprivateIsLastprivate = Lastprivates.count(OrigVD->getCanonicalDecl()) > 0; auto *CapFD = CapturesInfo.lookup(OrigVD); auto *FD = CapturedStmtInfo->lookup(OrigVD); if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) && !FD->getType()->isReferenceType()) { EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()); ++IRef; ++InitsRef; continue; } FirstprivateIsLastprivate = FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate; if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) { d370 5 a374 3 DeclRefExpr DRE(const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/FD != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); d376 1 a376 1 QualType Type = VD->getType(); d423 1 a423 2 ++IRef; ++InitsRef; d426 1 a426 1 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty(); d473 1 a527 8 llvm::DenseSet SIMDLCVs; if (isOpenMPSimdDirective(D.getDirectiveKind())) { auto *LoopDirective = cast(&D); for (auto *C : LoopDirective->counters()) { SIMDLCVs.insert( cast(cast(C)->getDecl())->getCanonicalDecl()); } } a530 2 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) break; a536 2 // Taskloops do not require additional initialization, it is done in // runtime support library. d550 1 a550 1 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) { d552 6 a557 5 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); d563 1 a563 2 ++IRef; ++IDestRef; d570 1 a570 2 const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond) { d587 1 a587 2 llvm::DenseSet AlreadyEmittedVars; llvm::DenseMap LoopCountersAndUpdates; d591 2 a592 6 auto *D = cast(cast(*IC)->getDecl())->getCanonicalDecl(); if (NoFinals) AlreadyEmittedVars.insert(D); else LoopCountersAndUpdates[D] = F; d596 1 d609 2 a610 2 if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) EmitIgnoredExpr(FinalExpr); a626 2 if (auto *PostUpdate = C->getPostUpdateExpr()) EmitIgnoredExpr(PostUpdate); a631 48 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV, llvm::Value *Addr) { Address Tmp = Address::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && !CGF.getContext().hasSameType(BaseTy, ElTy)) { Tmp = CGF.CreateMemTemp(BaseTy); if (TopTmp.isValid()) CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); else MostTopTmp = Tmp; TopTmp = Tmp; BaseTy = BaseTy->getPointeeType(); } llvm::Type *Ty = BaseLV.getPointer()->getType(); if (Tmp.isValid()) Ty = Tmp.getElementType(); Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); if (Tmp.isValid()) { CGF.Builder.CreateStore(Addr, Tmp); return MostTopTmp; } return Address(Addr, BaseLV.getAlignment()); } static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV) { BaseTy = BaseTy.getNonReferenceType(); while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && !CGF.getContext().hasSameType(BaseTy, ElTy)) { if (auto *PtrTy = BaseTy->getAs()) BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy); else { BaseLV = CGF.EmitLoadOfReferenceLValue(BaseLV.getAddress(), BaseTy->castAs()); } BaseTy = BaseTy->getPointeeType(); } return CGF.MakeAddrLValue( Address( CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( BaseLV.getPointer(), CGF.ConvertTypeForMem(ElTy)->getPointerTo()), BaseLV.getAlignment()), BaseLV.getType(), BaseLV.getAlignmentSource()); } a640 1 auto IRed = C->reduction_ops().begin(); a644 1 auto *DRD = getReductionInit(*IRed); d657 15 a671 3 LValue BaseLValue = loadToBegin(*this, OrigVD->getType(), OASELValueLB.getType(), OriginalBaseLValue); d679 2 a680 2 OrigVD, [this, OrigVD, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB, OriginalBaseLValue, DRD, IRed]() -> Address { d698 1 a698 3 EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), DRD ? *IRed : Init, OASELValueLB.getAddress()); d704 3 a706 3 return castToBase(*this, OrigVD->getType(), OASELValueLB.getType(), OriginalBaseLValue, Ptr); d722 15 a736 2 LValue BaseLValue = loadToBegin( *this, OrigVD->getType(), ASELValue.getType(), OriginalBaseLValue); d744 2 a745 2 OrigVD, [this, OrigVD, PrivateVD, BaseLValue, ASELValue, OriginalBaseLValue, DRD, IRed]() -> Address { d747 2 a748 9 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { emitInitWithReductionInitializer(*this, DRD, *IRed, Addr, ASELValue.getAddress(), ASELValue.getType()); } else EmitAutoVarInit(Emission); EmitAutoVarCleanups(Emission); d752 3 a754 2 return castToBase(*this, OrigVD->getType(), ASELValue.getType(), OriginalBaseLValue, Ptr); d759 2 a760 4 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address { return Builder.CreateElementBitCast( GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin"); d764 3 a766 4 QualType Type = PrivateVD->getType(); if (getContext().getAsArrayType(Type)) { // Store the address of the original variable associated with the LHS // implicit variable. d770 15 a784 68 Address OriginalAddr = EmitLValue(&DRE).getAddress(); PrivateScope.addPrivate(LHSVD, [this, &OriginalAddr, LHSVD]() -> Address { OriginalAddr = Builder.CreateElementBitCast( OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin"); return OriginalAddr; }); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { if (Type->isVariablyModifiedType()) { CodeGenFunction::OpaqueValueMapping OpaqueMap( *this, cast( getContext() .getAsVariableArrayType(PrivateVD->getType()) ->getSizeExpr()), RValue::get( getTypeSize(OrigVD->getType().getNonReferenceType()))); EmitVariablyModifiedType(Type); } auto Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); auto *Init = PrivateVD->getInit(); EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), DRD ? *IRed : Init, OriginalAddr); EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address { return Builder.CreateElementBitCast( GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin"); }); } else { // Store the address of the original variable associated with the LHS // implicit variable. Address OriginalAddr = Address::invalid(); PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef, &OriginalAddr]() -> Address { DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, IRef->getType(), VK_LValue, IRef->getExprLoc()); OriginalAddr = EmitLValue(&DRE).getAddress(); return OriginalAddr; }); // Emit reduction copy. bool IsRegistered = PrivateScope.addPrivate( OrigVD, [this, PrivateVD, OriginalAddr, DRD, IRed]() -> Address { // Emit private VarDecl with reduction init. AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { emitInitWithReductionInitializer(*this, DRD, *IRed, Addr, OriginalAddr, PrivateVD->getType()); } else EmitAutoVarInit(Emission); EmitAutoVarCleanups(Emission); return Addr; }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } d786 1 a786 4 ++ILHS; ++IRHS; ++IPriv; ++IRed; a818 25 static void emitPostUpdateForReductionClause( CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref &CondGen) { if (!CGF.HaveInsertPoint()) return; llvm::BasicBlock *DoneBB = nullptr; for (const auto *C : D.getClausesOfKind()) { if (auto *PostUpdate = C->getPostUpdateExpr()) { if (!DoneBB) { if (auto *Cond = CondGen(CGF)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); CGF.EmitBlock(ThenBB); } } CGF.EmitIgnoredExpr(PostUpdate); } } if (DoneBB) CGF.EmitBlock(DoneBB, /*IsFinished=*/true); } d824 4 a827 3 auto OutlinedFn = CGF.CGM.getOpenMPRuntime(). emitParallelOrTeamsOutlinedFunction(S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d836 1 a836 1 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); a847 4 OMPLexicalScope Scope(CGF, S); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); d853 1 d855 1 a855 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { d858 2 a859 2 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); if (Copyins) { d861 3 a863 2 // propagation master's thread values of threadprivate variables to local // instances of that variables of all other implicit threads. a874 2 emitPostUpdateForReductionClause( *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d886 1 a886 1 for (auto *U : C->updates()) d888 1 d911 1 a911 3 const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd())); d953 1 a953 1 for (auto *Init : C->inits()) { d955 11 a965 13 if (auto *Ref = dyn_cast(VD->getInit()->IgnoreImpCasts())) { AutoVarEmission Emission = EmitAutoVarAlloca(*VD); auto *OrigVD = cast(Ref->getDecl()); DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, VD->getInit()->getType(), VK_LValue, VD->getInit()->getExprLoc()); EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()), /*capturedByInit=*/false); EmitAutoVarCleanups(Emission); } else EmitVarDecl(*VD); d978 3 a980 4 void CodeGenFunction::EmitOMPLinearClauseFinal( const OMPLoopDirective &D, const llvm::function_ref &CondGen) { if (!HaveInsertPoint()) a981 1 llvm::BasicBlock *DoneBB = nullptr; d985 1 a985 11 for (auto *F : C->finals()) { if (!DoneBB) { if (auto *Cond = CondGen(*this)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = createBasicBlock(".omp.linear.pu"); DoneBB = createBasicBlock(".omp.linear.pu.done"); Builder.CreateCondBr(Cond, ThenBB, DoneBB); EmitBlock(ThenBB); } } d988 1 a988 1 CapturedStmtInfo->lookup(OrigVD) != nullptr, d990 4 a993 3 Address OrigAddr = EmitLValue(&DRE).getAddress(); CodeGenFunction::OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; }); d995 1 a995 1 EmitIgnoredExpr(F); a997 2 if (auto *PostUpdate = C->getPostUpdateExpr()) EmitIgnoredExpr(PostUpdate); a998 2 if (DoneBB) EmitBlock(DoneBB, /*IsFinished=*/true); d1034 5 a1038 3 void CodeGenFunction::EmitOMPPrivateLoopCounters( const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) { if (!HaveInsertPoint()) d1040 2 a1041 2 auto I = S.private_counters().begin(); for (auto *E : S.counters()) { d1044 2 a1045 1 (void)LoopScope.addPrivate(VD, [&]() -> Address { d1047 4 a1050 8 if (!LocalDeclMap.count(PrivateVD)) { auto VarEmission = EmitAutoVarAlloca(*PrivateVD); EmitAutoVarCleanups(VarEmission); } DeclRefExpr DRE(const_cast(PrivateVD), /*RefersToEnclosingVariableOrCapture=*/false, (*I)->getType(), VK_LValue, (*I)->getExprLoc()); return EmitLValue(&DRE).getAddress(); d1052 1 a1052 9 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) || VD->hasGlobalStorage()) { (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address { DeclRefExpr DRE(const_cast(VD), LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD), E->getType(), VK_LValue, E->getExprLoc()); return EmitLValue(&DRE).getAddress(); }); } d1064 2 a1065 1 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); d1076 4 a1079 3 void CodeGenFunction::EmitOMPLinearClause( const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { if (!HaveInsertPoint()) a1080 8 llvm::DenseSet SIMDLCVs; if (isOpenMPSimdDirective(D.getDirectiveKind())) { auto *LoopDirective = cast(&D); for (auto *C : LoopDirective->counters()) { SIMDLCVs.insert( cast(cast(C)->getDecl())->getCanonicalDecl()); } } d1087 8 a1094 11 if (!SIMDLCVs.count(VD->getCanonicalDecl())) { bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address { // Emit private VarDecl with copy init. EmitVarDecl(*PrivateVD); return GetAddrOfLocalVar(PrivateVD); }); assert(IsRegistered && "linear var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } else EmitVarDecl(*PrivateVD); d1135 1 a1135 3 void CodeGenFunction::EmitOMPSimdFinal( const OMPLoopDirective &D, const llvm::function_ref &CondGen) { a1137 1 llvm::BasicBlock *DoneBB = nullptr; a1138 1 auto IPC = D.private_counters().begin(); d1141 5 a1145 23 auto *PrivateVD = cast(cast((*IPC))->getDecl()); auto *CED = dyn_cast(OrigVD); if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) || OrigVD->hasGlobalStorage() || CED) { if (!DoneBB) { if (auto *Cond = CondGen(*this)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = createBasicBlock(".omp.final.then"); DoneBB = createBasicBlock(".omp.final.done"); Builder.CreateCondBr(Cond, ThenBB, DoneBB); EmitBlock(ThenBB); } } Address OrigAddr = Address::invalid(); if (CED) OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(); else { DeclRefExpr DRE(const_cast(PrivateVD), /*RefersToEnclosingVariableOrCapture=*/false, (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc()); OrigAddr = EmitLValue(&DRE).getAddress(); } a1152 1 ++IPC; d1154 1 a1154 2 if (DoneBB) EmitBlock(DoneBB, /*IsFinished=*/true); d1158 1 a1158 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); d1201 1 d1204 3 a1206 2 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); CGF.EmitOMPLinearClause(S, LoopScope); d1209 1 a1209 2 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); a1217 2 CGF.EmitOMPSimdFinal( S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d1219 3 a1221 2 if (HasLastprivateClause) CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); a1222 2 emitPostUpdateForReductionClause( CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d1224 1 a1224 2 CGF.EmitOMPLinearClauseFinal( S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); a1230 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1234 2 a1235 1 void CodeGenFunction::EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic, d1240 57 d1301 9 d1315 1 a1315 3 const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd())); d1326 2 a1327 2 BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, IL, LB, UB, ST); d1387 2 a1388 5 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { if (!DynamicOrOrdered) CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); d1391 7 a1397 5 void CodeGenFunction::EmitOMPForOuterLoop( const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); d1399 11 a1409 3 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule); d1411 35 a1445 273 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, /*Chunked=*/Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"); // Emit outer loop. // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(dynamic,chunk_size) is specified, the iterations are // distributed to threads in the team in chunks as the threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be distributed. Each chunk contains chunk_size // iterations, except for the last chunk to be distributed, which may have // fewer iterations. When no chunk_size is specified, it defaults to 1. // // When schedule(guided,chunk_size) is specified, the iterations are assigned // to threads in the team in chunks as the executing threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be assigned. For a chunk_size of 1, the size of // each chunk is proportional to the number of unassigned iterations divided // by the number of threads in the team, decreasing to 1. For a chunk_size // with value k (greater than 1), the size of each chunk is determined in the // same way, with the restriction that the chunks do not contain fewer than k // iterations (except for the last chunk to be assigned, which may have fewer // than k iterations). // // When schedule(auto) is specified, the decision regarding scheduling is // delegated to the compiler and/or runtime system. The programmer gives the // implementation the freedom to choose any possible mapping of iterations to // threads in the team. // // When schedule(runtime) is specified, the decision regarding scheduling is // deferred until run time, and the schedule and chunk size are taken from the // run-sched-var ICV. If the ICV is set to auto, the schedule is // implementation defined // // while(__kmpc_dispatch_next(&LB, &UB)) { // idx = LB; // while (idx <= UB) { BODY; ++idx; // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. // } // inner loop // } // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(static, chunk_size) is specified, iterations are divided into // chunks of size chunk_size, and the chunks are assigned to the threads in // the team in a round-robin fashion in the order of the thread number. // // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { // while (idx <= UB) { BODY; ++idx; } // inner loop // LB = LB + ST; // UB = UB + ST; // } // const Expr *IVExpr = S.getIterationVariable(); const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); if (DynamicOrOrdered) { llvm::Value *UBVal = EmitScalarExpr(S.getLastIteration()); RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, UBVal, Chunk); } else { RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk); } EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, Ordered, LB, UB, ST, IL, Chunk); } void CodeGenFunction::EmitOMPDistributeOuterLoop( OpenMPDistScheduleClauseKind ScheduleKind, const OMPDistributeDirective &S, OMPPrivateScope &LoopScope, Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); // Emit outer loop. // Same behavior as a OMPForOuterLoop, except that schedule cannot be // dynamic // const Expr *IVExpr = S.getIterationVariable(); const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, /* Ordered = */ false, IL, LB, UB, ST, Chunk); EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, LoopScope, /* Ordered = */ false, LB, UB, ST, IL, Chunk); } void CodeGenFunction::EmitOMPDistributeParallelForDirective( const OMPDistributeParallelForDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); OMPCancelStackRAII CancelRegion(CGF, OMPD_distribute_parallel_for, /*HasCancel=*/false); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective( const OMPDistributeParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPDistributeSimdDirective( const OMPDistributeSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetSimdDirective( const OMPTargetSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeDirective( const OMPTeamsDistributeDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( const OMPTeamsDistributeSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( const OMPTeamsDistributeParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDirective( const OMPTargetTeamsDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( const OMPTargetTeamsDistributeDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( const OMPTargetTeamsDistributeParallelForSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } /// \brief Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } namespace { struct ScheduleKindModifiersTy { OpenMPScheduleClauseKind Kind; OpenMPScheduleClauseModifier M1; OpenMPScheduleClauseModifier M2; ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2) : Kind(Kind), M1(M1), M2(M2) {} }; } // namespace a1466 1 OMPLoopScope PreInitScope(*this, S); a1483 9 bool Ordered = false; if (auto *OrderedClause = S.getSingleClause()) { if (OrderedClause->getNumForLoops()) RT.emitDoacrossInit(*this, S); else Ordered = true; } llvm::DenseSet EmittedFinals; a1485 10 // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); d1488 10 d1501 1 a1501 2 // initialization of firstprivate variables and post-update of // lastprivate variables. d1509 3 a1511 2 EmitOMPPrivateLoopCounters(S, LoopScope); EmitOMPLinearClause(S, LoopScope); d1515 8 a1522 13 llvm::Value *Chunk = nullptr; OpenMPScheduleTy ScheduleKind; if (auto *C = S.getSingleClause()) { ScheduleKind.Schedule = C->getScheduleKind(); ScheduleKind.M1 = C->getFirstScheduleModifier(); ScheduleKind.M2 = C->getSecondScheduleModifier(); if (const auto *Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType(), S.getLocStart()); } } d1525 1 d1530 1 a1530 1 if (RT.isStaticNonchunked(ScheduleKind.Schedule, d1560 1 a1560 4 auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); d1562 5 a1566 5 const bool IsMonotonic = Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static || ScheduleKind.Schedule == OMPC_SCHEDULE_unknown || ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic || ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic; a1572 7 if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } a1573 6 // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1577 4 a1580 2 S, isOpenMPSimdDirective(S.getDirectiveKind()), Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); a1581 4 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1592 1 d1594 1 a1594 3 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel()); d1597 2 a1598 5 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, S.hasCancel()); } d1607 1 d1609 1 a1609 2 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { d1612 1 a1612 4 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); } d1625 1 a1625 1 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); d1629 2 a1630 1 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { d1634 1 a1634 2 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { d1700 1 a1700 2 // initialization of firstprivate variables and post-update of lastprivate // variables. a1710 2 OpenMPScheduleTy ScheduleKind; ScheduleKind.Schedule = OMPC_SCHEDULE_static; d1712 1 a1712 1 CGF, S.getLocStart(), ScheduleKind, /*IVSize=*/32, d1726 1 a1726 4 auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); a1727 6 // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1732 2 a1733 3 S, /*NoFinals=*/false, CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart()))); a1740 1 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel); d1752 1 d1756 2 a1757 4 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); EmitSections(S); } d1760 1 a1760 2 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_sections); d1765 2 a1766 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { a1768 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1779 2 a1780 1 // 'single' construct. d1791 1 d1793 4 a1796 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope SingleScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); d1799 1 d1802 3 a1804 6 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), CopyprivateVars, DestExprs, SrcExprs, AssignmentOps); } d1807 2 a1808 1 if (!S.getSingleClause() && CopyprivateVars.empty()) { d1816 2 a1817 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); a1819 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1824 2 a1825 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); a1830 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1840 3 a1842 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel()); d1852 3 a1854 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { d1864 3 a1866 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitSections(S); d1871 1 a1871 4 void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data) { d1873 1 d1875 1 a1877 24 auto *TaskT = std::next(I, 4); // Check if the task is final if (const auto *Clause = S.getSingleClause()) { // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. auto *Cond = Clause->getCondition(); bool CondConstant; if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) Data.Final.setInt(CondConstant); else Data.Final.setPointer(EvaluateExprAsBool(Cond)); } else { // By default the task is not final. Data.Final.setInt(/*IntVal=*/false); } // Check if the task has 'priority' clause. if (const auto *Clause = S.getSingleClause()) { auto *Prio = Clause->getPriority(); Data.Priority.setInt(/*IntVal=*/true); Data.Priority.setPointer(EmitScalarConversion( EmitScalarExpr(Prio), Prio->getType(), getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), Prio->getExprLoc())); } d1882 2 d1889 2 a1890 2 Data.PrivateVars.push_back(*IRef); Data.PrivateCopies.push_back(IInit); d1897 3 d1906 3 a1908 3 Data.FirstprivateVars.push_back(*IRef); Data.FirstprivateCopies.push_back(IInit); Data.FirstprivateInits.push_back(*IElemInitRef); d1910 1 a1910 2 ++IRef; ++IElemInitRef; d1913 6 a1918 16 // Get list of lastprivate variables (for taskloops). llvm::DenseMap LastprivateDstsOrigs; for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto ID = C->destination_exprs().begin(); for (auto *IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { Data.LastprivateVars.push_back(*IRef); Data.LastprivateCopies.push_back(IInit); } LastprivateDstsOrigs.insert( {cast(cast(*ID)->getDecl()), cast(*IRef)}); ++IRef; ++ID; d1921 2 a1922 6 // Build list of dependences. for (const auto *C : S.getClausesOfKind()) for (auto *IRef : C->varlists()) Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef)); auto &&CodeGen = [PartId, &S, &Data, CS, &BodyGen, &LastprivateDstsOrigs]( CodeGenFunction &CGF, PrePostActionTy &Action) { d1924 1 d1926 1 a1926 2 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() || !Data.LastprivateVars.empty()) { d1932 2 a1933 1 llvm::SmallVector, 16> PrivatePtrs; d1936 1 a1936 8 for (auto *E : Data.PrivateVars) { auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); CallArgs.push_back(PrivatePtr.getPointer()); } for (auto *E : Data.FirstprivateVars) { d1939 1 a1939 2 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); d1943 1 a1943 1 for (auto *E : Data.LastprivateVars) { d1946 1 a1946 2 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); a1950 11 for (auto &&Pair : LastprivateDstsOrigs) { auto *OrigVD = cast(Pair.second->getDecl()); DeclRefExpr DRE( const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup( OrigVD) != nullptr, Pair.second->getType(), VK_LValue, Pair.second->getExprLoc()); Scope.addPrivate(Pair.first, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); }); } d1958 4 a1961 3 Action.Enter(CGF); BodyGen(CGF); d1963 19 a1981 11 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied, Data.NumberOfParts); OMPLexicalScope Scope(*this, S); TaskGen(*this, OutlinedFn, Data); } void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { // Emit outlined function for task construct. auto CS = cast(S.getAssociatedStmt()); auto CapturedStruct = GenerateCapturedStmtArgument(*CS); d1991 4 a1994 48 OMPTaskDataTy Data; // Check if we should emit tied or untied task. Data.Tied = !S.getSingleClause(); auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt(CS->getCapturedStmt()); }; auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, const OMPTaskDataTy &Data) { CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, Data); }; EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); } void CodeGenFunction::EmitOMPTaskyieldDirective( const OMPTaskyieldDirective &S) { CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); } void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPTaskgroupDirective( const OMPTaskgroupDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) { return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } return llvm::None; }(), S.getLocStart()); d1997 4 a2000 49 void CodeGenFunction::EmitOMPDistributeLoop(const OMPDistributeDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast(S.getIterationVariable()); auto IVDecl = cast(IVExpr->getDecl()); EmitVarDecl(*IVDecl); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } auto &RT = CGM.getOpenMPRuntime(); bool HasLastprivateClause = false; // Check pre-condition. { OMPLoopScope PreInitScope(*this, S); // Skip the entire loop if we don't meet the precondition. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { auto *ThenBlock = createBasicBlock("omp.precond.then"); ContBlock = createBasicBlock("omp.precond.end"); emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, getProfileCount(&S)); EmitBlock(ThenBlock); incrementProfileCounter(&S); } // Emit 'then' code. { // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); d2002 3 a2004 13 OMPPrivateScope LoopScope(*this); if (EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables and post-update of // lastprivate variables. CGM.getOpenMPRuntime().emitBarrierCall( *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); } EmitOMPPrivateClause(S, LoopScope); HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPPrivateLoopCounters(S, LoopScope); (void)LoopScope.Privatize(); d2006 3 a2008 14 // Detect the distribute schedule kind and chunk. llvm::Value *Chunk = nullptr; OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown; if (auto *C = S.getSingleClause()) { ScheduleKind = C->getDistScheduleKind(); if (const auto *Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType(), S.getLocStart()); } } const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); d2010 8 a2017 38 // OpenMP [2.10.8, distribute Construct, Description] // If dist_schedule is specified, kind must be static. If specified, // iterations are divided into chunks of size chunk_size, chunks are // assigned to the teams of the league in a round-robin fashion in the // order of the team number. When no chunk_size is specified, the // iteration space is divided into chunks that are approximately equal // in size, and at most one chunk is distributed to each team of the // league. The size of the chunks is unspecified in this case. if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr)) { RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S, LoopExit](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, LoopExit); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. RT.emitForStaticFinish(*this, S.getLocStart()); } else { // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk); } d2019 5 a2023 6 // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivateClause) EmitOMPLastprivateClauseFinal( S, /*NoFinals=*/false, Builder.CreateIsNotNull( EmitLoadOfScalar(IL, S.getLocStart()))); d2025 2 a2026 7 // We're now done with the loop, so jump to the continuation block. if (ContBlock) { EmitBranch(ContBlock); EmitBlock(ContBlock, true); } } d2031 1 a2031 6 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S); }; OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen, false); d2045 1 a2045 3 if (!S.getAssociatedStmt()) { for (const auto *DC : S.getClausesOfKind()) CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); d2047 1 a2047 1 } d2049 1 a2049 2 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, PrePostActionTy &Action) { a2056 1 Action.Enter(CGF); a2060 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d2107 2 a2108 3 CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic, d2141 4 a2144 5 : CGF.EmitAtomicLoad( XLValue, Loc, IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic, XLValue.isVolatile()); d2300 1 a2300 2 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; d2349 1 a2349 2 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; a2474 7 case OMPC_dist_schedule: case OMPC_defaultmap: case OMPC_uniform: case OMPC_to: case OMPC_from: case OMPC_use_device_ptr: case OMPC_is_device_ptr: d2504 2 a2505 2 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF, PrePostActionTy &) { a2510 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); a2513 21 std::pair CodeGenFunction::EmitOMPTargetDirectiveOutlinedFunction( CodeGenModule &CGM, const OMPTargetDirective &S, StringRef ParentName, bool IsOffloadEntry) { llvm::Function *OutlinedFn = nullptr; llvm::Constant *OutlinedFnID = nullptr; auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); (void)PrivateScope.Privatize(); Action.Enter(CGF); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry, CodeGen); return std::make_pair(OutlinedFn, OutlinedFnID); } d2515 1 d2561 3 a2563 3 std::tie(Fn, FnID) = EmitOMPTargetDirectiveOutlinedFunction( CGM, S, ParentName, IsOffloadEntry); OMPLexicalScope Scope(*this, S); d2568 2 a2569 37 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { auto CS = cast(S.getAssociatedStmt()); auto OutlinedFn = CGF.CGM.getOpenMPRuntime(). emitParallelOrTeamsOutlinedFunction(S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); const OMPTeamsDirective &TD = *dyn_cast(&S); const OMPNumTeamsClause *NT = TD.getSingleClause(); const OMPThreadLimitClause *TL = TD.getSingleClause(); if (NT || TL) { Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr; Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr; CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, S.getLocStart()); } OMPLexicalScope Scope(CGF, S); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn, CapturedVars); } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) { // Emit teams region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; emitCommonOMPTeamsDirective(*this, S, OMPD_teams, CodeGen); d2593 1 a2593 2 if (Kind == OMPD_parallel || Kind == OMPD_task || Kind == OMPD_target_parallel) d2596 2 a2597 66 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for); return OMPCancelStack.getExitBlock(); } void CodeGenFunction::EmitOMPUseDevicePtrClause( const OMPClause &NC, OMPPrivateScope &PrivateScope, const llvm::DenseMap &CaptureDeviceAddrMap) { const auto &C = cast(NC); auto OrigVarIt = C.varlist_begin(); auto InitIt = C.inits().begin(); for (auto PvtVarIt : C.private_copies()) { auto *OrigVD = cast(cast(*OrigVarIt)->getDecl()); auto *InitVD = cast(cast(*InitIt)->getDecl()); auto *PvtVD = cast(cast(PvtVarIt)->getDecl()); // In order to identify the right initializer we need to match the // declaration used by the mapping logic. In some cases we may get // OMPCapturedExprDecl that refers to the original declaration. const ValueDecl *MatchingVD = OrigVD; if (auto *OED = dyn_cast(MatchingVD)) { // OMPCapturedExprDecl are used to privative fields of the current // structure. auto *ME = cast(OED->getInit()); assert(isa(ME->getBase()) && "Base should be the current struct!"); MatchingVD = ME->getMemberDecl(); } // If we don't have information about the current list item, move on to // the next one. auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD); if (InitAddrIt == CaptureDeviceAddrMap.end()) continue; bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Initialize the temporary initialization variable with the address we // get from the runtime library. We have to cast the source address // because it is always a void *. References are materialized in the // privatization scope, so the initialization here disregards the fact // the original variable is a reference. QualType AddrQTy = getContext().getPointerType(OrigVD->getType().getNonReferenceType()); llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy); Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy); setAddrOfLocalVar(InitVD, InitAddr); // Emit private declaration, it will be initialized by the value we // declaration we just added to the local declarations map. EmitDecl(*PvtVD); // The initialization variables reached its purpose in the emission // ofthe previous declaration, so we don't need it anymore. LocalDeclMap.erase(InitVD); // Return the address of the private variable. return GetAddrOfLocalVar(PvtVD); }); assert(IsRegistered && "firstprivate var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++OrigVarIt; ++InitIt; } d2603 1 a2603 149 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true); // Create a pre/post action to signal the privatization of the device pointer. // This action can be replaced by the OpenMP runtime code generation to // deactivate privatization. bool PrivatizeDevicePointers = false; class DevicePointerPrivActionTy : public PrePostActionTy { bool &PrivatizeDevicePointers; public: explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers) : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {} void Enter(CodeGenFunction &CGF) override { PrivatizeDevicePointers = true; } }; DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers); auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers]( CodeGenFunction &CGF, PrePostActionTy &Action) { auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }; // Codegen that selects wheather to generate the privatization code or not. auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers, &InnermostCodeGen](CodeGenFunction &CGF, PrePostActionTy &Action) { RegionCodeGenTy RCG(InnermostCodeGen); PrivatizeDevicePointers = false; // Call the pre-action to change the status of PrivatizeDevicePointers if // needed. Action.Enter(CGF); if (PrivatizeDevicePointers) { OMPPrivateScope PrivateScope(CGF); // Emit all instances of the use_device_ptr clause. for (const auto *C : S.getClausesOfKind()) CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, Info.CaptureDeviceAddrMap); (void)PrivateScope.Privatize(); RCG(CGF); } else RCG(CGF); }; // Forward the provided action to the privatization codegen. RegionCodeGenTy PrivRCG(PrivCodeGen); PrivRCG.setAction(Action); // Notwithstanding the body of the region is emitted as inlined directive, // we don't use an inline scope as changes in the references inside the // region are expected to be visible outside, so we do not privative them. OMPLexicalScope Scope(CGF, S); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, PrivRCG); }; RegionCodeGenTy RCG(CodeGen); // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) { RCG(*this); return; } // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); // Set the action to signal privatization of device pointers. RCG.setAction(PrivAction); // Emit region code. CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG, Info); } void CodeGenFunction::EmitOMPTargetEnterDataDirective( const OMPTargetEnterDataDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } void CodeGenFunction::EmitOMPTargetExitDataDirective( const OMPTargetExitDataDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } void CodeGenFunction::EmitOMPTargetParallelDirective( const OMPTargetParallelDirective &S) { // TODO: codegen for target parallel. } void CodeGenFunction::EmitOMPTargetParallelForDirective( const OMPTargetParallelForDirective &S) { // TODO: codegen for target parallel for. } /// Emit a helper variable and return corresponding lvalue. static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates) { auto *VDecl = cast(Helper->getDecl()); Privates.addPrivate( VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); }); } void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) { assert(isOpenMPTaskLoopDirective(S.getDirectiveKind())); // Emit outlined function for task construct. d2605 3 a2607 122 auto CapturedStruct = GenerateCapturedStmtArgument(*CS); auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); const Expr *IfCond = nullptr; for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_taskloop) { IfCond = C->getCondition(); break; } } OMPTaskDataTy Data; // Check if taskloop must be emitted without taskgroup. Data.Nogroup = S.getSingleClause(); // TODO: Check if we should emit tied or untied task. Data.Tied = true; // Set scheduling for taskloop if (const auto* Clause = S.getSingleClause()) { // grainsize clause Data.Schedule.setInt(/*IntVal=*/false); Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize())); } else if (const auto* Clause = S.getSingleClause()) { // num_tasks clause Data.Schedule.setInt(/*IntVal=*/true); Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks())); } auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; OMPLoopScope PreInitScope(CGF, S); if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); ContBlock = CGF.createBasicBlock("taskloop.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } if (isOpenMPSimdDirective(S.getDirectiveKind())) CGF.EmitOMPSimdInit(S); OMPPrivateScope LoopScope(CGF); // Emit helper vars inits. enum { LowerBound = 5, UpperBound, Stride, LastIter }; auto *I = CS->getCapturedDecl()->param_begin(); auto *LBP = std::next(I, LowerBound); auto *UBP = std::next(I, UpperBound); auto *STP = std::next(I, Stride); auto *LIP = std::next(I, LastIter); mapParam(CGF, cast(S.getLowerBoundVariable()), *LBP, LoopScope); mapParam(CGF, cast(S.getUpperBoundVariable()), *UBP, LoopScope); mapParam(CGF, cast(S.getStrideVariable()), *STP, LoopScope); mapParam(CGF, cast(S.getIsLastIterVariable()), *LIP, LoopScope); CGF.EmitOMPPrivateLoopCounters(S, LoopScope); bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivateClause) { CGF.EmitOMPLastprivateClauseFinal( S, isOpenMPSimdDirective(S.getDirectiveKind()), CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, (*LIP)->getType(), S.getLocStart()))); } }; auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, const OMPTaskDataTy &Data) { auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, Data); }; CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, CodeGen); }; EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); d2611 5 a2615 1 EmitOMPTaskLoopBasedDirective(S); d2620 5 a2624 1 EmitOMPTaskLoopBasedDirective(S); a2626 20 // Generate the instructions for '#pragma omp target update' directive. void CodeGenFunction::EmitOMPTargetUpdateDirective( const OMPTargetUpdateDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } @ 1.1.1.5 log @Import Clang pre-4.0.0 r291444. @ text @a13 1 #include "CGCleanup.h" a19 2 #include "clang/AST/DeclOpenMP.h" #include "llvm/IR/CallSite.h" a22 99 namespace { /// Lexical scope for OpenMP executable constructs, that handles correct codegen /// for captured expressions. class OMPLexicalScope final : public CodeGenFunction::LexicalScope { void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { for (const auto *C : S.clauses()) { if (auto *CPI = OMPClauseWithPreInit::get(C)) { if (auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { for (const auto *I : PreInit->decls()) { if (!I->hasAttr()) CGF.EmitVarDecl(cast(*I)); else { CodeGenFunction::AutoVarEmission Emission = CGF.EmitAutoVarAlloca(cast(*I)); CGF.EmitAutoVarCleanups(Emission); } } } } } } CodeGenFunction::OMPPrivateScope InlinedShareds; static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || (CGF.CurCodeDecl && isa(CGF.CurCodeDecl)); } public: OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S, bool AsInlined = false) : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), InlinedShareds(CGF) { emitPreInitStmt(CGF, S); if (AsInlined) { if (S.hasAssociatedStmt()) { auto *CS = cast(S.getAssociatedStmt()); for (auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); DeclRefExpr DRE(const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, SourceLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); } } (void)InlinedShareds.Privatize(); } } } }; /// Private scope for OpenMP loop-based directives, that supports capturing /// of used expression from loop statement. class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { if (auto *LD = dyn_cast(&S)) { if (auto *PreInits = cast_or_null(LD->getPreInits())) { for (const auto *I : PreInits->decls()) CGF.EmitVarDecl(cast(*I)); } } } public: OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) : CodeGenFunction::RunCleanupsScope(CGF) { emitPreInitStmt(CGF, S); } }; } // namespace llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) { auto &C = getContext(); llvm::Value *Size = nullptr; auto SizeInChars = C.getTypeSizeInChars(Ty); if (SizeInChars.isZero()) { // getTypeSizeInChars() returns 0 for a VLA. while (auto *VAT = C.getAsVariableArrayType(Ty)) { llvm::Value *ArraySize; std::tie(ArraySize, Ty) = getVLASize(VAT); Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize; } SizeInChars = C.getTypeSizeInChars(Ty); if (SizeInChars.isZero()) return llvm::ConstantInt::get(SizeTy, /*V=*/0); Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); } else Size = CGM.getSize(SizeInChars); return Size; } d37 4 a40 27 else if (CurCap->capturesVariableByCopy()) { llvm::Value *CV = EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal(); // If the field is not a pointer, we need to save the actual value // and load it as a void pointer. if (!CurField->getType()->isAnyPointerType()) { auto &Ctx = getContext(); auto DstAddr = CreateMemTemp( Ctx.getUIntPtrType(), Twine(CurCap->getCapturedVar()->getName()) + ".casted"); LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); auto *SrcAddrVal = EmitScalarConversion( DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), SourceLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); // Store the value using the source type pointer. EmitStoreThroughLValue(RValue::get(CV), SrcLV); // Load the value using the destination type pointer. CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal(); } CapturedVars.push_back(CV); } else { d66 1 a66 1 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true); d110 2 a111 9 if (ArgType->isVariablyModifiedType()) { bool IsReference = ArgType->isLValueReferenceType(); ArgType = getContext().getCanonicalParamType(ArgType.getNonReferenceType()); if (IsReference && !ArgType->isPointerType()) { ArgType = getContext().getLValueReferenceType( ArgType, /*SpelledAsLValue=*/false); } } d123 2 a124 1 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args); d143 2 a144 13 const VarDecl *CurVD = I->getCapturedVar(); Address LocalAddr = GetAddrOfLocalVar(Args[Cnt]); // If the variable is a reference we need to materialize it here. if (CurVD->getType()->isReferenceType()) { Address RefAddr = CreateMemTemp(CurVD->getType(), getPointerAlign(), ".materialized_ref"); EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr, /*Volatile=*/false, CurVD->getType()); LocalAddr = RefAddr; } setAddrOfLocalVar(CurVD, LocalAddr); ++Cnt; ++I; d165 2 a166 8 if (ArgLVal.getType()->isLValueReferenceType()) { ArgAddr = EmitLoadOfReference( ArgAddr, ArgLVal.getType()->castAs()); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { assert(ArgLVal.getType()->isPointerType()); ArgAddr = EmitLoadOfPointer( ArgAddr, ArgLVal.getType()->castAs()); } d175 4 a178 3 setAddrOfLocalVar(Var, castValueFromUintptr(*this, FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType())); d185 1 a185 2 ++Cnt; ++I; a258 62 /// Check if the combiner is a call to UDR combiner and if it is so return the /// UDR decl used for reduction. static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp) { if (auto *CE = dyn_cast(ReductionOp)) if (auto *OVE = dyn_cast(CE->getCallee())) if (auto *DRE = dyn_cast(OVE->getSourceExpr()->IgnoreImpCasts())) if (auto *DRD = dyn_cast(DRE->getDecl())) return DRD; return nullptr; } static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty) { if (DRD->getInitializer()) { std::pair Reduction = CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); auto *CE = cast(InitOp); auto *OVE = cast(CE->getCallee()); const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts(); const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts(); auto *LHSDRE = cast(cast(LHS)->getSubExpr()); auto *RHSDRE = cast(cast(RHS)->getSubExpr()); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); PrivateScope.addPrivate(cast(LHSDRE->getDecl()), [=]() -> Address { return Private; }); PrivateScope.addPrivate(cast(RHSDRE->getDecl()), [=]() -> Address { return Original; }); (void)PrivateScope.Privatize(); RValue Func = RValue::get(Reduction.second); CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); CGF.EmitIgnoredExpr(InitOp); } else { llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, ".init"); LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: InitRVal = CGF.EmitLoadOfLValue(LV, SourceLocation()); break; case TEK_Complex: InitRVal = RValue::getComplex(CGF.EmitLoadOfComplex(LV, SourceLocation())); break; case TEK_Aggregate: InitRVal = RValue::getAggregate(LV.getAddress()); break; } OpaqueValueExpr OVE(SourceLocation(), Ty, VK_RValue); CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), /*IsInitializer=*/false); } } a262 1 /// \param SrcAddr Address of the original array. d264 1 a264 3 QualType Type, const Expr *Init, Address SrcAddr = Address::invalid()) { auto *DRD = getReductionInit(Init); d273 1 a273 7 if (DRD) SrcAddr = CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); llvm::Value *SrcBegin = nullptr; if (DRD) SrcBegin = SrcAddr.getPointer(); a289 10 llvm::PHINode *SrcElementPHI = nullptr; Address SrcElementCurrent = Address::invalid(); if (DRD) { SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); SrcElementPHI->addIncoming(SrcBegin, EntryBB); SrcElementCurrent = Address(SrcElementPHI, SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); } d300 2 a301 13 if (DRD && (DRD->getInitializer() || !Init)) { emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, SrcElementCurrent, ElementTy); } else CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), /*IsInitializer=*/false); } if (DRD) { // Shift the address forward by one element. auto SrcElementNext = CGF.Builder.CreateConstGEP1_32( SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); a358 7 bool FirstprivateIsLastprivate = false; llvm::DenseSet Lastprivates; for (const auto *C : D.getClausesOfKind()) { for (const auto *D : C->varlists()) Lastprivates.insert( cast(cast(D)->getDecl())->getCanonicalDecl()); } a359 1 CGCapturedStmtInfo CapturesInfo(cast(*D.getAssociatedStmt())); d365 2 a366 14 bool ThisFirstprivateIsLastprivate = Lastprivates.count(OrigVD->getCanonicalDecl()) > 0; auto *CapFD = CapturesInfo.lookup(OrigVD); auto *FD = CapturedStmtInfo->lookup(OrigVD); if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) && !FD->getType()->isReferenceType()) { EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()); ++IRef; ++InitsRef; continue; } FirstprivateIsLastprivate = FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate; if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) { d370 5 a374 3 DeclRefExpr DRE(const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/FD != nullptr, (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); d376 1 a376 1 QualType Type = VD->getType(); d423 1 a423 2 ++IRef; ++InitsRef; d426 1 a426 1 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty(); d473 1 a527 8 llvm::DenseSet SIMDLCVs; if (isOpenMPSimdDirective(D.getDirectiveKind())) { auto *LoopDirective = cast(&D); for (auto *C : LoopDirective->counters()) { SIMDLCVs.insert( cast(cast(C)->getDecl())->getCanonicalDecl()); } } a530 2 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) break; a536 2 // Taskloops do not require additional initialization, it is done in // runtime support library. d550 1 a550 1 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) { d552 6 a557 5 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); d563 1 a563 2 ++IRef; ++IDestRef; d570 1 a570 2 const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond) { d587 1 a587 2 llvm::DenseSet AlreadyEmittedVars; llvm::DenseMap LoopCountersAndUpdates; d591 2 a592 6 auto *D = cast(cast(*IC)->getDecl())->getCanonicalDecl(); if (NoFinals) AlreadyEmittedVars.insert(D); else LoopCountersAndUpdates[D] = F; d596 1 d609 2 a610 2 if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) EmitIgnoredExpr(FinalExpr); a626 2 if (auto *PostUpdate = C->getPostUpdateExpr()) EmitIgnoredExpr(PostUpdate); a631 48 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV, llvm::Value *Addr) { Address Tmp = Address::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && !CGF.getContext().hasSameType(BaseTy, ElTy)) { Tmp = CGF.CreateMemTemp(BaseTy); if (TopTmp.isValid()) CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); else MostTopTmp = Tmp; TopTmp = Tmp; BaseTy = BaseTy->getPointeeType(); } llvm::Type *Ty = BaseLV.getPointer()->getType(); if (Tmp.isValid()) Ty = Tmp.getElementType(); Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); if (Tmp.isValid()) { CGF.Builder.CreateStore(Addr, Tmp); return MostTopTmp; } return Address(Addr, BaseLV.getAlignment()); } static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV) { BaseTy = BaseTy.getNonReferenceType(); while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && !CGF.getContext().hasSameType(BaseTy, ElTy)) { if (auto *PtrTy = BaseTy->getAs()) BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy); else { BaseLV = CGF.EmitLoadOfReferenceLValue(BaseLV.getAddress(), BaseTy->castAs()); } BaseTy = BaseTy->getPointeeType(); } return CGF.MakeAddrLValue( Address( CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( BaseLV.getPointer(), CGF.ConvertTypeForMem(ElTy)->getPointerTo()), BaseLV.getAlignment()), BaseLV.getType(), BaseLV.getAlignmentSource()); } a640 1 auto IRed = C->reduction_ops().begin(); a644 1 auto *DRD = getReductionInit(*IRed); d657 15 a671 3 LValue BaseLValue = loadToBegin(*this, OrigVD->getType(), OASELValueLB.getType(), OriginalBaseLValue); d679 2 a680 2 OrigVD, [this, OrigVD, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB, OriginalBaseLValue, DRD, IRed]() -> Address { d698 1 a698 3 EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), DRD ? *IRed : Init, OASELValueLB.getAddress()); d704 3 a706 3 return castToBase(*this, OrigVD->getType(), OASELValueLB.getType(), OriginalBaseLValue, Ptr); d722 15 a736 2 LValue BaseLValue = loadToBegin( *this, OrigVD->getType(), ASELValue.getType(), OriginalBaseLValue); d744 2 a745 2 OrigVD, [this, OrigVD, PrivateVD, BaseLValue, ASELValue, OriginalBaseLValue, DRD, IRed]() -> Address { d747 2 a748 9 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { emitInitWithReductionInitializer(*this, DRD, *IRed, Addr, ASELValue.getAddress(), ASELValue.getType()); } else EmitAutoVarInit(Emission); EmitAutoVarCleanups(Emission); d752 3 a754 2 return castToBase(*this, OrigVD->getType(), ASELValue.getType(), OriginalBaseLValue, Ptr); d759 2 a760 4 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address { return Builder.CreateElementBitCast( GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin"); d764 3 a766 4 QualType Type = PrivateVD->getType(); if (getContext().getAsArrayType(Type)) { // Store the address of the original variable associated with the LHS // implicit variable. d770 15 a784 68 Address OriginalAddr = EmitLValue(&DRE).getAddress(); PrivateScope.addPrivate(LHSVD, [this, &OriginalAddr, LHSVD]() -> Address { OriginalAddr = Builder.CreateElementBitCast( OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin"); return OriginalAddr; }); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { if (Type->isVariablyModifiedType()) { CodeGenFunction::OpaqueValueMapping OpaqueMap( *this, cast( getContext() .getAsVariableArrayType(PrivateVD->getType()) ->getSizeExpr()), RValue::get( getTypeSize(OrigVD->getType().getNonReferenceType()))); EmitVariablyModifiedType(Type); } auto Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); auto *Init = PrivateVD->getInit(); EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), DRD ? *IRed : Init, OriginalAddr); EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address { return Builder.CreateElementBitCast( GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin"); }); } else { // Store the address of the original variable associated with the LHS // implicit variable. Address OriginalAddr = Address::invalid(); PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef, &OriginalAddr]() -> Address { DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, IRef->getType(), VK_LValue, IRef->getExprLoc()); OriginalAddr = EmitLValue(&DRE).getAddress(); return OriginalAddr; }); // Emit reduction copy. bool IsRegistered = PrivateScope.addPrivate( OrigVD, [this, PrivateVD, OriginalAddr, DRD, IRed]() -> Address { // Emit private VarDecl with reduction init. AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); auto Addr = Emission.getAllocatedAddress(); if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { emitInitWithReductionInitializer(*this, DRD, *IRed, Addr, OriginalAddr, PrivateVD->getType()); } else EmitAutoVarInit(Emission); EmitAutoVarCleanups(Emission); return Addr; }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } d786 1 a786 4 ++ILHS; ++IRHS; ++IPriv; ++IRed; a818 25 static void emitPostUpdateForReductionClause( CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref &CondGen) { if (!CGF.HaveInsertPoint()) return; llvm::BasicBlock *DoneBB = nullptr; for (const auto *C : D.getClausesOfKind()) { if (auto *PostUpdate = C->getPostUpdateExpr()) { if (!DoneBB) { if (auto *Cond = CondGen(CGF)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); CGF.EmitBlock(ThenBB); } } CGF.EmitIgnoredExpr(PostUpdate); } } if (DoneBB) CGF.EmitBlock(DoneBB, /*IsFinished=*/true); } d824 4 a827 3 auto OutlinedFn = CGF.CGM.getOpenMPRuntime(). emitParallelOrTeamsOutlinedFunction(S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d836 1 a836 1 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); a847 4 OMPLexicalScope Scope(CGF, S); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); d853 1 d855 1 a855 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { d858 2 a859 2 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); if (Copyins) { d861 3 a863 2 // propagation master's thread values of threadprivate variables to local // instances of that variables of all other implicit threads. a874 2 emitPostUpdateForReductionClause( *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d886 1 a886 1 for (auto *U : C->updates()) d888 1 d911 1 a911 3 const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd())); d953 1 a953 1 for (auto *Init : C->inits()) { d955 11 a965 13 if (auto *Ref = dyn_cast(VD->getInit()->IgnoreImpCasts())) { AutoVarEmission Emission = EmitAutoVarAlloca(*VD); auto *OrigVD = cast(Ref->getDecl()); DeclRefExpr DRE(const_cast(OrigVD), CapturedStmtInfo->lookup(OrigVD) != nullptr, VD->getInit()->getType(), VK_LValue, VD->getInit()->getExprLoc()); EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()), /*capturedByInit=*/false); EmitAutoVarCleanups(Emission); } else EmitVarDecl(*VD); d978 3 a980 4 void CodeGenFunction::EmitOMPLinearClauseFinal( const OMPLoopDirective &D, const llvm::function_ref &CondGen) { if (!HaveInsertPoint()) a981 1 llvm::BasicBlock *DoneBB = nullptr; d985 1 a985 11 for (auto *F : C->finals()) { if (!DoneBB) { if (auto *Cond = CondGen(*this)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = createBasicBlock(".omp.linear.pu"); DoneBB = createBasicBlock(".omp.linear.pu.done"); Builder.CreateCondBr(Cond, ThenBB, DoneBB); EmitBlock(ThenBB); } } d988 1 a988 1 CapturedStmtInfo->lookup(OrigVD) != nullptr, d990 4 a993 3 Address OrigAddr = EmitLValue(&DRE).getAddress(); CodeGenFunction::OMPPrivateScope VarScope(*this); VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; }); d995 1 a995 1 EmitIgnoredExpr(F); a997 2 if (auto *PostUpdate = C->getPostUpdateExpr()) EmitIgnoredExpr(PostUpdate); a998 2 if (DoneBB) EmitBlock(DoneBB, /*IsFinished=*/true); d1034 5 a1038 3 void CodeGenFunction::EmitOMPPrivateLoopCounters( const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) { if (!HaveInsertPoint()) d1040 2 a1041 2 auto I = S.private_counters().begin(); for (auto *E : S.counters()) { d1044 2 a1045 1 (void)LoopScope.addPrivate(VD, [&]() -> Address { d1047 4 a1050 8 if (!LocalDeclMap.count(PrivateVD)) { auto VarEmission = EmitAutoVarAlloca(*PrivateVD); EmitAutoVarCleanups(VarEmission); } DeclRefExpr DRE(const_cast(PrivateVD), /*RefersToEnclosingVariableOrCapture=*/false, (*I)->getType(), VK_LValue, (*I)->getExprLoc()); return EmitLValue(&DRE).getAddress(); d1052 1 a1052 9 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) || VD->hasGlobalStorage()) { (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address { DeclRefExpr DRE(const_cast(VD), LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD), E->getType(), VK_LValue, E->getExprLoc()); return EmitLValue(&DRE).getAddress(); }); } d1064 2 a1065 1 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); d1076 4 a1079 3 void CodeGenFunction::EmitOMPLinearClause( const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { if (!HaveInsertPoint()) a1080 8 llvm::DenseSet SIMDLCVs; if (isOpenMPSimdDirective(D.getDirectiveKind())) { auto *LoopDirective = cast(&D); for (auto *C : LoopDirective->counters()) { SIMDLCVs.insert( cast(cast(C)->getDecl())->getCanonicalDecl()); } } d1087 8 a1094 11 if (!SIMDLCVs.count(VD->getCanonicalDecl())) { bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address { // Emit private VarDecl with copy init. EmitVarDecl(*PrivateVD); return GetAddrOfLocalVar(PrivateVD); }); assert(IsRegistered && "linear var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; } else EmitVarDecl(*PrivateVD); d1135 1 a1135 3 void CodeGenFunction::EmitOMPSimdFinal( const OMPLoopDirective &D, const llvm::function_ref &CondGen) { a1137 1 llvm::BasicBlock *DoneBB = nullptr; a1138 1 auto IPC = D.private_counters().begin(); d1141 5 a1145 23 auto *PrivateVD = cast(cast((*IPC))->getDecl()); auto *CED = dyn_cast(OrigVD); if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) || OrigVD->hasGlobalStorage() || CED) { if (!DoneBB) { if (auto *Cond = CondGen(*this)) { // If the first post-update expression is found, emit conditional // block if it was requested. auto *ThenBB = createBasicBlock(".omp.final.then"); DoneBB = createBasicBlock(".omp.final.done"); Builder.CreateCondBr(Cond, ThenBB, DoneBB); EmitBlock(ThenBB); } } Address OrigAddr = Address::invalid(); if (CED) OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(); else { DeclRefExpr DRE(const_cast(PrivateVD), /*RefersToEnclosingVariableOrCapture=*/false, (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc()); OrigAddr = EmitLValue(&DRE).getAddress(); } a1152 1 ++IPC; d1154 1 a1154 2 if (DoneBB) EmitBlock(DoneBB, /*IsFinished=*/true); d1158 1 a1158 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); d1201 1 d1204 3 a1206 2 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); CGF.EmitOMPLinearClause(S, LoopScope); d1209 1 a1209 2 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); a1217 2 CGF.EmitOMPSimdFinal( S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d1219 3 a1221 2 if (HasLastprivateClause) CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); a1222 2 emitPostUpdateForReductionClause( CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); d1224 1 a1224 2 CGF.EmitOMPLinearClauseFinal( S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); a1230 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1234 2 a1235 1 void CodeGenFunction::EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic, d1240 57 d1301 9 d1315 1 a1315 3 const SourceRange &R = S.getSourceRange(); LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), SourceLocToDebugLoc(R.getEnd())); d1326 2 a1327 2 BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, IL, LB, UB, ST); d1387 2 a1388 5 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { if (!DynamicOrOrdered) CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); d1391 7 a1397 5 void CodeGenFunction::EmitOMPForOuterLoop( const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); d1399 11 a1409 3 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule); d1411 35 a1445 263 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule, /*Chunked=*/Chunk != nullptr)) && "static non-chunked schedule does not need outer loop"); // Emit outer loop. // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(dynamic,chunk_size) is specified, the iterations are // distributed to threads in the team in chunks as the threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be distributed. Each chunk contains chunk_size // iterations, except for the last chunk to be distributed, which may have // fewer iterations. When no chunk_size is specified, it defaults to 1. // // When schedule(guided,chunk_size) is specified, the iterations are assigned // to threads in the team in chunks as the executing threads request them. // Each thread executes a chunk of iterations, then requests another chunk, // until no chunks remain to be assigned. For a chunk_size of 1, the size of // each chunk is proportional to the number of unassigned iterations divided // by the number of threads in the team, decreasing to 1. For a chunk_size // with value k (greater than 1), the size of each chunk is determined in the // same way, with the restriction that the chunks do not contain fewer than k // iterations (except for the last chunk to be assigned, which may have fewer // than k iterations). // // When schedule(auto) is specified, the decision regarding scheduling is // delegated to the compiler and/or runtime system. The programmer gives the // implementation the freedom to choose any possible mapping of iterations to // threads in the team. // // When schedule(runtime) is specified, the decision regarding scheduling is // deferred until run time, and the schedule and chunk size are taken from the // run-sched-var ICV. If the ICV is set to auto, the schedule is // implementation defined // // while(__kmpc_dispatch_next(&LB, &UB)) { // idx = LB; // while (idx <= UB) { BODY; ++idx; // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. // } // inner loop // } // // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When schedule(static, chunk_size) is specified, iterations are divided into // chunks of size chunk_size, and the chunks are assigned to the threads in // the team in a round-robin fashion in the order of the thread number. // // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { // while (idx <= UB) { BODY; ++idx; } // inner loop // LB = LB + ST; // UB = UB + ST; // } // const Expr *IVExpr = S.getIterationVariable(); const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); if (DynamicOrOrdered) { llvm::Value *UBVal = EmitScalarExpr(S.getLastIteration()); RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, UBVal, Chunk); } else { RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk); } EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, Ordered, LB, UB, ST, IL, Chunk); } void CodeGenFunction::EmitOMPDistributeOuterLoop( OpenMPDistScheduleClauseKind ScheduleKind, const OMPDistributeDirective &S, OMPPrivateScope &LoopScope, Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) { auto &RT = CGM.getOpenMPRuntime(); // Emit outer loop. // Same behavior as a OMPForOuterLoop, except that schedule cannot be // dynamic // const Expr *IVExpr = S.getIterationVariable(); const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, /* Ordered = */ false, IL, LB, UB, ST, Chunk); EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, LoopScope, /* Ordered = */ false, LB, UB, ST, IL, Chunk); } void CodeGenFunction::EmitOMPDistributeParallelForDirective( const OMPDistributeParallelForDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); OMPCancelStackRAII CancelRegion(CGF, OMPD_distribute_parallel_for, /*HasCancel=*/false); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective( const OMPDistributeParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPDistributeSimdDirective( const OMPDistributeSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetSimdDirective( const OMPTargetSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeDirective( const OMPTeamsDistributeDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( const OMPTeamsDistributeSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( const OMPTeamsDistributeParallelForSimdDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &S) { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_teams_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDirective( const OMPTargetTeamsDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( const OMPTargetTeamsDistributeDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_parallel_for, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( const OMPTargetTeamsDistributeParallelForSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_parallel_for_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } /// \brief Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } namespace { struct ScheduleKindModifiersTy { OpenMPScheduleClauseKind Kind; OpenMPScheduleClauseModifier M1; OpenMPScheduleClauseModifier M2; ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2) : Kind(Kind), M1(M1), M2(M2) {} }; } // namespace a1466 1 OMPLoopScope PreInitScope(*this, S); a1483 9 bool Ordered = false; if (auto *OrderedClause = S.getSingleClause()) { if (OrderedClause->getNumForLoops()) RT.emitDoacrossInit(*this, S); else Ordered = true; } llvm::DenseSet EmittedFinals; a1485 10 // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); d1488 10 d1501 1 a1501 2 // initialization of firstprivate variables and post-update of // lastprivate variables. d1509 3 a1511 2 EmitOMPPrivateLoopCounters(S, LoopScope); EmitOMPLinearClause(S, LoopScope); d1515 8 a1522 13 llvm::Value *Chunk = nullptr; OpenMPScheduleTy ScheduleKind; if (auto *C = S.getSingleClause()) { ScheduleKind.Schedule = C->getScheduleKind(); ScheduleKind.M1 = C->getFirstScheduleModifier(); ScheduleKind.M2 = C->getSecondScheduleModifier(); if (const auto *Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType(), S.getLocStart()); } } d1525 1 d1530 1 a1530 1 if (RT.isStaticNonchunked(ScheduleKind.Schedule, d1560 1 a1560 4 auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); d1562 5 a1566 5 const bool IsMonotonic = Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static || ScheduleKind.Schedule == OMPC_SCHEDULE_unknown || ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic || ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic; a1572 7 if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } a1573 6 // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1577 4 a1580 2 S, isOpenMPSimdDirective(S.getDirectiveKind()), Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); a1581 4 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1592 1 d1594 1 a1594 3 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel()); d1597 2 a1598 5 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, S.hasCancel()); } d1607 1 d1609 1 a1609 2 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { d1612 1 a1612 4 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); } d1625 1 a1625 1 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); d1629 2 a1630 1 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { d1634 1 a1634 2 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { d1700 1 a1700 2 // initialization of firstprivate variables and post-update of lastprivate // variables. a1710 2 OpenMPScheduleTy ScheduleKind; ScheduleKind.Schedule = OMPC_SCHEDULE_static; d1712 1 a1712 1 CGF, S.getLocStart(), ScheduleKind, /*IVSize=*/32, d1726 1 a1726 4 auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd()); }; CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); a1727 6 // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d1732 2 a1733 3 S, /*NoFinals=*/false, CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart()))); a1740 1 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel); d1752 1 d1756 2 a1757 4 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); EmitSections(S); } d1760 1 a1760 2 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_sections); d1765 2 a1766 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { a1768 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1779 2 a1780 1 // 'single' construct. d1791 1 d1793 4 a1796 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope SingleScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); d1799 1 d1802 3 a1804 6 { OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), CopyprivateVars, DestExprs, SrcExprs, AssignmentOps); } d1807 2 a1808 1 if (!S.getSingleClause() && CopyprivateVars.empty()) { d1816 2 a1817 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); a1819 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1824 2 a1825 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); a1830 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d1840 3 a1842 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel()); d1852 3 a1854 1 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { d1864 3 a1866 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitSections(S); d1871 1 a1871 4 void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data) { d1873 1 d1875 1 a1877 24 auto *TaskT = std::next(I, 4); // Check if the task is final if (const auto *Clause = S.getSingleClause()) { // If the condition constant folds and can be elided, try to avoid emitting // the condition and the dead arm of the if/else. auto *Cond = Clause->getCondition(); bool CondConstant; if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) Data.Final.setInt(CondConstant); else Data.Final.setPointer(EvaluateExprAsBool(Cond)); } else { // By default the task is not final. Data.Final.setInt(/*IntVal=*/false); } // Check if the task has 'priority' clause. if (const auto *Clause = S.getSingleClause()) { auto *Prio = Clause->getPriority(); Data.Priority.setInt(/*IntVal=*/true); Data.Priority.setPointer(EmitScalarConversion( EmitScalarExpr(Prio), Prio->getType(), getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), Prio->getExprLoc())); } d1882 2 d1889 2 a1890 2 Data.PrivateVars.push_back(*IRef); Data.PrivateCopies.push_back(IInit); d1897 3 d1906 3 a1908 3 Data.FirstprivateVars.push_back(*IRef); Data.FirstprivateCopies.push_back(IInit); Data.FirstprivateInits.push_back(*IElemInitRef); d1910 1 a1910 2 ++IRef; ++IElemInitRef; d1913 6 a1918 16 // Get list of lastprivate variables (for taskloops). llvm::DenseMap LastprivateDstsOrigs; for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto ID = C->destination_exprs().begin(); for (auto *IInit : C->private_copies()) { auto *OrigVD = cast(cast(*IRef)->getDecl()); if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { Data.LastprivateVars.push_back(*IRef); Data.LastprivateCopies.push_back(IInit); } LastprivateDstsOrigs.insert( {cast(cast(*ID)->getDecl()), cast(*IRef)}); ++IRef; ++ID; d1921 2 a1922 6 // Build list of dependences. for (const auto *C : S.getClausesOfKind()) for (auto *IRef : C->varlists()) Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef)); auto &&CodeGen = [PartId, &S, &Data, CS, &BodyGen, &LastprivateDstsOrigs]( CodeGenFunction &CGF, PrePostActionTy &Action) { d1924 1 d1926 1 a1926 2 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() || !Data.LastprivateVars.empty()) { d1932 2 a1933 1 llvm::SmallVector, 16> PrivatePtrs; d1936 1 a1936 8 for (auto *E : Data.PrivateVars) { auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); CallArgs.push_back(PrivatePtr.getPointer()); } for (auto *E : Data.FirstprivateVars) { d1939 1 a1939 2 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); d1943 1 a1943 1 for (auto *E : Data.LastprivateVars) { d1946 1 a1946 2 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); a1950 11 for (auto &&Pair : LastprivateDstsOrigs) { auto *OrigVD = cast(Pair.second->getDecl()); DeclRefExpr DRE( const_cast(OrigVD), /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup( OrigVD) != nullptr, Pair.second->getType(), VK_LValue, Pair.second->getExprLoc()); Scope.addPrivate(Pair.first, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); }); } d1958 4 a1961 3 Action.Enter(CGF); BodyGen(CGF); d1963 19 a1981 11 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied, Data.NumberOfParts); OMPLexicalScope Scope(*this, S); TaskGen(*this, OutlinedFn, Data); } void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { // Emit outlined function for task construct. auto CS = cast(S.getAssociatedStmt()); auto CapturedStruct = GenerateCapturedStmtArgument(*CS); d1991 4 a1994 20 OMPTaskDataTy Data; // Check if we should emit tied or untied task. Data.Tied = !S.getSingleClause(); auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt(CS->getCapturedStmt()); }; auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, const OMPTaskDataTy &Data) { CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, Data); }; EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); } void CodeGenFunction::EmitOMPTaskyieldDirective( const OMPTaskyieldDirective &S) { CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); d1997 4 a2000 77 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); } void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPTaskgroupDirective( const OMPTaskgroupDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) { return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); } return llvm::None; }(), S.getLocStart()); } void CodeGenFunction::EmitOMPDistributeLoop(const OMPDistributeDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast(S.getIterationVariable()); auto IVDecl = cast(IVExpr->getDecl()); EmitVarDecl(*IVDecl); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on each // iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. EmitIgnoredExpr(S.getCalcLastIteration()); } auto &RT = CGM.getOpenMPRuntime(); bool HasLastprivateClause = false; // Check pre-condition. { OMPLoopScope PreInitScope(*this, S); // Skip the entire loop if we don't meet the precondition. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { auto *ThenBlock = createBasicBlock("omp.precond.then"); ContBlock = createBasicBlock("omp.precond.end"); emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, getProfileCount(&S)); EmitBlock(ThenBlock); incrementProfileCounter(&S); } // Emit 'then' code. { // Emit helper vars inits. LValue LB = EmitOMPHelperVar(*this, cast(S.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(*this, cast(S.getUpperBoundVariable())); LValue ST = EmitOMPHelperVar(*this, cast(S.getStrideVariable())); LValue IL = EmitOMPHelperVar(*this, cast(S.getIsLastIterVariable())); d2002 3 a2004 13 OMPPrivateScope LoopScope(*this); if (EmitOMPFirstprivateClause(S, LoopScope)) { // Emit implicit barrier to synchronize threads and avoid data races on // initialization of firstprivate variables and post-update of // lastprivate variables. CGM.getOpenMPRuntime().emitBarrierCall( *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); } EmitOMPPrivateClause(S, LoopScope); HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); EmitOMPPrivateLoopCounters(S, LoopScope); (void)LoopScope.Privatize(); d2006 3 a2008 14 // Detect the distribute schedule kind and chunk. llvm::Value *Chunk = nullptr; OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown; if (auto *C = S.getSingleClause()) { ScheduleKind = C->getDistScheduleKind(); if (const auto *Ch = C->getChunkSize()) { Chunk = EmitScalarExpr(Ch); Chunk = EmitScalarConversion(Chunk, Ch->getType(), S.getIterationVariable()->getType(), S.getLocStart()); } } const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); d2010 8 a2017 38 // OpenMP [2.10.8, distribute Construct, Description] // If dist_schedule is specified, kind must be static. If specified, // iterations are divided into chunks of size chunk_size, chunks are // assigned to the teams of the league in a round-robin fashion in the // order of the team number. When no chunk_size is specified, the // iteration space is divided into chunks that are approximately equal // in size, and at most one chunk is distributed to each team of the // league. The size of the chunks is unspecified in this case. if (RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr)) { RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S, LoopExit](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, LoopExit); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. RT.emitForStaticFinish(*this, S.getLocStart()); } else { // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk); } d2019 5 a2023 6 // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivateClause) EmitOMPLastprivateClauseFinal( S, /*NoFinals=*/false, Builder.CreateIsNotNull( EmitLoadOfScalar(IL, S.getLocStart()))); d2025 2 a2026 7 // We're now done with the loop, so jump to the continuation block. if (ContBlock) { EmitBranch(ContBlock); EmitBlock(ContBlock, true); } } d2031 1 a2031 6 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S); }; OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen, false); d2045 1 a2045 3 if (!S.getAssociatedStmt()) { for (const auto *DC : S.getClausesOfKind()) CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); d2047 1 a2047 1 } d2049 1 a2049 2 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, PrePostActionTy &Action) { a2056 1 Action.Enter(CGF); a2060 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); d2107 2 a2108 3 CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic, d2141 4 a2144 5 : CGF.EmitAtomicLoad( XLValue, Loc, IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic, XLValue.isVolatile()); d2300 1 a2300 2 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; d2349 1 a2349 2 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; a2474 7 case OMPC_dist_schedule: case OMPC_defaultmap: case OMPC_uniform: case OMPC_to: case OMPC_from: case OMPC_use_device_ptr: case OMPC_is_device_ptr: d2504 2 a2505 2 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF, PrePostActionTy &) { a2510 1 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true); a2513 21 std::pair CodeGenFunction::EmitOMPTargetDirectiveOutlinedFunction( CodeGenModule &CGM, const OMPTargetDirective &S, StringRef ParentName, bool IsOffloadEntry) { llvm::Function *OutlinedFn = nullptr; llvm::Constant *OutlinedFnID = nullptr; auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); (void)PrivateScope.Privatize(); Action.Enter(CGF); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry, CodeGen); return std::make_pair(OutlinedFn, OutlinedFnID); } d2515 1 d2561 3 a2563 3 std::tie(Fn, FnID) = EmitOMPTargetDirectiveOutlinedFunction( CGM, S, ParentName, IsOffloadEntry); OMPLexicalScope Scope(*this, S); d2568 2 a2569 37 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { auto CS = cast(S.getAssociatedStmt()); auto OutlinedFn = CGF.CGM.getOpenMPRuntime(). emitParallelOrTeamsOutlinedFunction(S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); const OMPTeamsDirective &TD = *dyn_cast(&S); const OMPNumTeamsClause *NT = TD.getSingleClause(); const OMPThreadLimitClause *TL = TD.getSingleClause(); if (NT || TL) { Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr; Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr; CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, S.getLocStart()); } OMPLexicalScope Scope(CGF, S); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn, CapturedVars); } void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) { // Emit teams region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); }; emitCommonOMPTeamsDirective(*this, S, OMPD_teams, CodeGen); d2593 1 a2593 2 if (Kind == OMPD_parallel || Kind == OMPD_task || Kind == OMPD_target_parallel) d2596 2 a2597 66 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || Kind == OMPD_distribute_parallel_for || Kind == OMPD_target_parallel_for); return OMPCancelStack.getExitBlock(); } void CodeGenFunction::EmitOMPUseDevicePtrClause( const OMPClause &NC, OMPPrivateScope &PrivateScope, const llvm::DenseMap &CaptureDeviceAddrMap) { const auto &C = cast(NC); auto OrigVarIt = C.varlist_begin(); auto InitIt = C.inits().begin(); for (auto PvtVarIt : C.private_copies()) { auto *OrigVD = cast(cast(*OrigVarIt)->getDecl()); auto *InitVD = cast(cast(*InitIt)->getDecl()); auto *PvtVD = cast(cast(PvtVarIt)->getDecl()); // In order to identify the right initializer we need to match the // declaration used by the mapping logic. In some cases we may get // OMPCapturedExprDecl that refers to the original declaration. const ValueDecl *MatchingVD = OrigVD; if (auto *OED = dyn_cast(MatchingVD)) { // OMPCapturedExprDecl are used to privative fields of the current // structure. auto *ME = cast(OED->getInit()); assert(isa(ME->getBase()) && "Base should be the current struct!"); MatchingVD = ME->getMemberDecl(); } // If we don't have information about the current list item, move on to // the next one. auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD); if (InitAddrIt == CaptureDeviceAddrMap.end()) continue; bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address { // Initialize the temporary initialization variable with the address we // get from the runtime library. We have to cast the source address // because it is always a void *. References are materialized in the // privatization scope, so the initialization here disregards the fact // the original variable is a reference. QualType AddrQTy = getContext().getPointerType(OrigVD->getType().getNonReferenceType()); llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy); Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy); setAddrOfLocalVar(InitVD, InitAddr); // Emit private declaration, it will be initialized by the value we // declaration we just added to the local declarations map. EmitDecl(*PvtVD); // The initialization variables reached its purpose in the emission // ofthe previous declaration, so we don't need it anymore. LocalDeclMap.erase(InitVD); // Return the address of the private variable. return GetAddrOfLocalVar(PvtVD); }); assert(IsRegistered && "firstprivate var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; ++OrigVarIt; ++InitIt; } d2603 1 a2603 149 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true); // Create a pre/post action to signal the privatization of the device pointer. // This action can be replaced by the OpenMP runtime code generation to // deactivate privatization. bool PrivatizeDevicePointers = false; class DevicePointerPrivActionTy : public PrePostActionTy { bool &PrivatizeDevicePointers; public: explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers) : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {} void Enter(CodeGenFunction &CGF) override { PrivatizeDevicePointers = true; } }; DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers); auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers]( CodeGenFunction &CGF, PrePostActionTy &Action) { auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }; // Codegen that selects wheather to generate the privatization code or not. auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers, &InnermostCodeGen](CodeGenFunction &CGF, PrePostActionTy &Action) { RegionCodeGenTy RCG(InnermostCodeGen); PrivatizeDevicePointers = false; // Call the pre-action to change the status of PrivatizeDevicePointers if // needed. Action.Enter(CGF); if (PrivatizeDevicePointers) { OMPPrivateScope PrivateScope(CGF); // Emit all instances of the use_device_ptr clause. for (const auto *C : S.getClausesOfKind()) CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, Info.CaptureDeviceAddrMap); (void)PrivateScope.Privatize(); RCG(CGF); } else RCG(CGF); }; // Forward the provided action to the privatization codegen. RegionCodeGenTy PrivRCG(PrivCodeGen); PrivRCG.setAction(Action); // Notwithstanding the body of the region is emitted as inlined directive, // we don't use an inline scope as changes in the references inside the // region are expected to be visible outside, so we do not privative them. OMPLexicalScope Scope(CGF, S); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, PrivRCG); }; RegionCodeGenTy RCG(CodeGen); // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) { RCG(*this); return; } // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); // Set the action to signal privatization of device pointers. RCG.setAction(PrivAction); // Emit region code. CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG, Info); } void CodeGenFunction::EmitOMPTargetEnterDataDirective( const OMPTargetEnterDataDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } void CodeGenFunction::EmitOMPTargetExitDataDirective( const OMPTargetExitDataDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } void CodeGenFunction::EmitOMPTargetParallelDirective( const OMPTargetParallelDirective &S) { // TODO: codegen for target parallel. } void CodeGenFunction::EmitOMPTargetParallelForDirective( const OMPTargetParallelForDirective &S) { // TODO: codegen for target parallel for. } /// Emit a helper variable and return corresponding lvalue. static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates) { auto *VDecl = cast(Helper->getDecl()); Privates.addPrivate( VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); }); } void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) { assert(isOpenMPTaskLoopDirective(S.getDirectiveKind())); // Emit outlined function for task construct. d2605 3 a2607 122 auto CapturedStruct = GenerateCapturedStmtArgument(*CS); auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); const Expr *IfCond = nullptr; for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_taskloop) { IfCond = C->getCondition(); break; } } OMPTaskDataTy Data; // Check if taskloop must be emitted without taskgroup. Data.Nogroup = S.getSingleClause(); // TODO: Check if we should emit tied or untied task. Data.Tied = true; // Set scheduling for taskloop if (const auto* Clause = S.getSingleClause()) { // grainsize clause Data.Schedule.setInt(/*IntVal=*/false); Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize())); } else if (const auto* Clause = S.getSingleClause()) { // num_tasks clause Data.Schedule.setInt(/*IntVal=*/true); Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks())); } auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; OMPLoopScope PreInitScope(CGF, S); if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); ContBlock = CGF.createBasicBlock("taskloop.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } if (isOpenMPSimdDirective(S.getDirectiveKind())) CGF.EmitOMPSimdInit(S); OMPPrivateScope LoopScope(CGF); // Emit helper vars inits. enum { LowerBound = 5, UpperBound, Stride, LastIter }; auto *I = CS->getCapturedDecl()->param_begin(); auto *LBP = std::next(I, LowerBound); auto *UBP = std::next(I, UpperBound); auto *STP = std::next(I, Stride); auto *LIP = std::next(I, LastIter); mapParam(CGF, cast(S.getLowerBoundVariable()), *LBP, LoopScope); mapParam(CGF, cast(S.getUpperBoundVariable()), *UBP, LoopScope); mapParam(CGF, cast(S.getStrideVariable()), *STP, LoopScope); mapParam(CGF, cast(S.getIsLastIterVariable()), *LIP, LoopScope); CGF.EmitOMPPrivateLoopCounters(S, LoopScope); bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const VarDecl *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (auto LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } // Emit final copy of the lastprivate variables if IsLastIter != 0. if (HasLastprivateClause) { CGF.EmitOMPLastprivateClauseFinal( S, isOpenMPSimdDirective(S.getDirectiveKind()), CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, (*LIP)->getType(), S.getLocStart()))); } }; auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, const OMPTaskDataTy &Data) { auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) { OMPLoopScope PreInitScope(CGF, S); CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, Data); }; CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, CodeGen); }; EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); d2611 5 a2615 1 EmitOMPTaskLoopBasedDirective(S); d2620 5 a2624 1 EmitOMPTaskLoopBasedDirective(S); a2626 20 // Generate the instructions for '#pragma omp target update' directive. void CodeGenFunction::EmitOMPTargetUpdateDirective( const OMPTargetUpdateDirective &S) { // If we don't have target devices, don't bother emitting the data mapping // code. if (CGM.getLangOpts().OMPTargetTriples.empty()) return; // Check if we have any if clause associated with the directive. const Expr *IfCond = nullptr; if (auto *C = S.getSingleClause()) IfCond = C->getCondition(); // Check if we have any device clause associated with the directive. const Expr *Device = nullptr; if (auto *C = S.getSingleClause()) Device = C->getDevice(); CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); } @ 1.1.1.5.2.1 log @Sync with HEAD @ text @a2044 10 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } @ 1.1.1.6 log @Import Clang 4.0RC1 r294123. @ text @a2044 10 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { CGM.getOpenMPRuntime().emitInlinedDirective( *this, OMPD_target_teams_distribute_simd, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt( cast(S.getAssociatedStmt())->getCapturedStmt()); }); } @ 1.1.1.7 log @Import clang r309604 from branches/release_50 @ text @d29 1 a29 1 class OMPLexicalScope : public CodeGenFunction::LexicalScope { d57 1 a57 1 bool AsInlined = false, bool EmitPreInitStmt = true) d60 1 a60 2 if (EmitPreInitStmt) emitPreInitStmt(CGF, S); a83 33 /// Lexical scope for OpenMP parallel construct, that handles correct codegen /// for captured expressions. class OMPParallelScope final : public OMPLexicalScope { bool EmitPreInitStmt(const OMPExecutableDirective &S) { OpenMPDirectiveKind Kind = S.getDirectiveKind(); return !(isOpenMPTargetExecutionDirective(Kind) || isOpenMPLoopBoundSharingDirective(Kind)) && isOpenMPParallelDirective(Kind); } public: OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) : OMPLexicalScope(CGF, S, /*AsInlined=*/false, /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {} }; /// Lexical scope for OpenMP teams construct, that handles correct codegen /// for captured expressions. class OMPTeamsScope final : public OMPLexicalScope { bool EmitPreInitStmt(const OMPExecutableDirective &S) { OpenMPDirectiveKind Kind = S.getDirectiveKind(); return !isOpenMPTargetExecutionDirective(Kind) && isOpenMPTeamsDirective(Kind); } public: OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) : OMPLexicalScope(CGF, S, /*AsInlined=*/false, /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {} }; d197 7 a203 42 static QualType getCanonicalParamType(ASTContext &C, QualType T) { if (T->isLValueReferenceType()) { return C.getLValueReferenceType( getCanonicalParamType(C, T.getNonReferenceType()), /*SpelledAsLValue=*/false); } if (T->isPointerType()) return C.getPointerType(getCanonicalParamType(C, T->getPointeeType())); return C.getCanonicalParamType(T); } namespace { /// Contains required data for proper outlined function codegen. struct FunctionOptions { /// Captured statement for which the function is generated. const CapturedStmt *S = nullptr; /// true if cast to/from UIntPtr is required for variables captured by /// value. bool UIntPtrCastRequired = true; /// true if only casted argumefnts must be registered as local args or VLA /// sizes. bool RegisterCastedArgsOnly = false; /// Name of the generated function. StringRef FunctionName; explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired, bool RegisterCastedArgsOnly, StringRef FunctionName) : S(S), UIntPtrCastRequired(UIntPtrCastRequired), RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly), FunctionName(FunctionName) {} }; } static std::pair emitOutlinedFunctionPrologue( CodeGenFunction &CGF, FunctionArgList &Args, llvm::DenseMap> &LocalAddrs, llvm::DenseMap> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO) { const CapturedDecl *CD = FO.S->getCapturedDecl(); const RecordDecl *RD = FO.S->getCapturedRecordDecl(); a205 1 CXXThisValue = nullptr; a206 1 CodeGenModule &CGM = CGF.CGM; d208 1 a208 1 bool HasUIntPtrArgs = false; d211 1 a211 1 auto I = FO.S->captures().begin(); d223 2 a224 5 I->capturesVariableArrayType()) { HasUIntPtrArgs = true; if (FO.UIntPtrCastRequired) ArgType = Ctx.getUIntPtrType(); } d230 1 a230 1 II = &Ctx.Idents.get("this"); d233 10 a242 1 II = &Ctx.Idents.get("vla"); d244 2 a245 5 if (ArgType->isVariablyModifiedType()) ArgType = getCanonicalParamType(Ctx, ArgType.getNonReferenceType()); Args.push_back(ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), II, ArgType, ImplicitParamDecl::Other)); d258 3 a260 3 llvm::Function *F = llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage, FO.FunctionName, &CGM.getModule()); d266 2 a267 2 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(), CD->getBody()->getLocStart()); d269 1 a269 1 I = FO.S->captures().begin(); d275 1 a275 1 Address LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); d278 4 a281 4 Address RefAddr = CGF.CreateMemTemp( CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref"); CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr, /*Volatile=*/false, CurVD->getType()); d284 1 a284 2 if (!FO.RegisterCastedArgsOnly) LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}}); d290 3 a292 3 LValueBaseInfo BaseInfo(AlignmentSource::Decl, false); LValue ArgLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(Args[Cnt]), Args[Cnt]->getType(), BaseInfo); d294 4 a297 6 if (FO.UIntPtrCastRequired) { ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(), ArgLVal), FD->getType(), BaseInfo); } d299 1 a299 1 CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal(); d301 1 a301 1 VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}}); d308 1 a308 1 ArgAddr = CGF.EmitLoadOfReference( d312 1 a312 1 ArgAddr = CGF.EmitLoadOfPointer( d316 2 a317 5 if (!FO.RegisterCastedArgsOnly) { LocalAddrs.insert( {Args[Cnt], {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}}); } d323 3 a325 7 LocalAddrs.insert( {Args[Cnt], {Var, FO.UIntPtrCastRequired ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType()) : ArgLVal.getAddress()}}); d329 2 a330 3 CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation()) .getScalarVal(); LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}}); a335 30 return {F, HasUIntPtrArgs}; } llvm::Function * CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { assert( CapturedStmtInfo && "CapturedStmtInfo should be set when generating the captured function"); const CapturedDecl *CD = S.getCapturedDecl(); // Build the argument list. bool NeedWrapperFunction = getDebugInfo() && CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo; FunctionArgList Args; llvm::DenseMap> LocalAddrs; llvm::DenseMap> VLASizes; FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false, CapturedStmtInfo->getHelperName()); llvm::Function *F; bool HasUIntPtrArgs; std::tie(F, HasUIntPtrArgs) = emitOutlinedFunctionPrologue( *this, Args, LocalAddrs, VLASizes, CXXThisValue, FO); for (const auto &LocalAddrPair : LocalAddrs) { if (LocalAddrPair.second.first) { setAddrOfLocalVar(LocalAddrPair.second.first, LocalAddrPair.second.second); } } for (const auto &VLASizePair : VLASizes) VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second; a338 2 if (!NeedWrapperFunction || !HasUIntPtrArgs) return F; d340 1 a340 35 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true, /*RegisterCastedArgsOnly=*/true, ".nondebug_wrapper."); CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true); WrapperCGF.disableDebugInfo(); Args.clear(); LocalAddrs.clear(); VLASizes.clear(); llvm::Function *WrapperF = emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes, WrapperCGF.CXXThisValue, WrapperFO).first; LValueBaseInfo BaseInfo(AlignmentSource::Decl, false); llvm::SmallVector CallArgs; for (const auto *Arg : Args) { llvm::Value *CallArg; auto I = LocalAddrs.find(Arg); if (I != LocalAddrs.end()) { LValue LV = WrapperCGF.MakeAddrLValue(I->second.second, Arg->getType(), BaseInfo); CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation()); } else { auto EI = VLASizes.find(Arg); if (EI != VLASizes.end()) CallArg = EI->second.second; else { LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg), Arg->getType(), BaseInfo); CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation()); } } CallArgs.emplace_back(CallArg); } WrapperCGF.Builder.CreateCall(F, CallArgs); WrapperCGF.FinishFunction(); return WrapperF; d407 150 d909 48 a961 5 SmallVector Shareds; SmallVector Privates; SmallVector ReductionOps; SmallVector LHSs; SmallVector RHSs; d963 2 d967 184 a1150 71 auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); for (const auto *Ref : C->varlists()) { Shareds.emplace_back(Ref); Privates.emplace_back(*IPriv); ReductionOps.emplace_back(*IRed); LHSs.emplace_back(*ILHS); RHSs.emplace_back(*IRHS); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ILHS, 1); std::advance(IRHS, 1); } } ReductionCodeGen RedCG(Shareds, Privates, ReductionOps); unsigned Count = 0; auto ILHS = LHSs.begin(); auto IRHS = RHSs.begin(); auto IPriv = Privates.begin(); for (const auto *IRef : Shareds) { auto *PrivateVD = cast(cast(*IPriv)->getDecl()); // Emit private VarDecl with reduction init. RedCG.emitSharedLValue(*this, Count); RedCG.emitAggregateType(*this, Count); auto Emission = EmitAutoVarAlloca(*PrivateVD); RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(), RedCG.getSharedLValue(Count), [&Emission](CodeGenFunction &CGF) { CGF.EmitAutoVarInit(Emission); return true; }); EmitAutoVarCleanups(Emission); Address BaseAddr = RedCG.adjustPrivateAddress( *this, Count, Emission.getAllocatedAddress()); bool IsRegistered = PrivateScope.addPrivate( RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; }); assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; auto *LHSVD = cast(cast(*ILHS)->getDecl()); auto *RHSVD = cast(cast(*IRHS)->getDecl()); if (isa(IRef)) { // Store the address of the original variable associated with the LHS // implicit variable. PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address { return RedCG.getSharedLValue(Count).getAddress(); }); PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address { return GetAddrOfLocalVar(PrivateVD); }); } else if (isa(IRef)) { // Store the address of the original variable associated with the LHS // implicit variable. PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address { return RedCG.getSharedLValue(Count).getAddress(); }); PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address { return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin"); }); } else { QualType Type = PrivateVD->getType(); bool IsArray = getContext().getAsArrayType(Type) != nullptr; Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(); // Store the address of the original variable associated with the LHS // implicit variable. if (IsArray) { OriginalAddr = Builder.CreateElementBitCast( OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin"); d1152 4 a1155 10 PrivateScope.addPrivate( LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; }); PrivateScope.addPrivate( RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address { return IsArray ? Builder.CreateElementBitCast( GetAddrOfLocalVar(PrivateVD), ConvertTypeForMem(RHSVD->getType()), "rhs.begin") : GetAddrOfLocalVar(PrivateVD); }); a1156 4 ++ILHS; ++IRHS; ++IPriv; ++Count; d1161 1 a1161 1 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) { a1176 4 bool WithNowait = D.getSingleClause() || isOpenMPParallelDirective(D.getDirectiveKind()) || D.getDirectiveKind() == OMPD_simd; bool SimpleReduction = D.getDirectiveKind() == OMPD_simd; d1181 4 a1184 1 {WithNowait, SimpleReduction, ReductionKind}); d1213 8 a1220 17 namespace { /// Codegen lambda for appending distribute lower and upper bounds to outlined /// parallel function. This is necessary for combined constructs such as /// 'distribute parallel for' typedef llvm::function_ref &)> CodeGenBoundParametersTy; } // anonymous namespace static void emitCommonOMPParallelDirective( CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters) { const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d1242 1 a1242 1 OMPParallelScope Scope(CGF, S); a1243 5 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk // lower and upper bounds with the pragma 'for' chunking mechanism. // The following lambda takes care of appending the lower and upper bound // parameters when necessary CodeGenBoundParameters(CGF, S, CapturedVars); a1248 4 static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl &) {} d1267 1 a1267 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); d1269 1 a1269 2 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen, emitEmptyBoundParameters); a1613 7 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit) { CGF.EmitOMPLoopBody(S, LoopExit); CGF.EmitStopPoint(&S); } d1680 1 a1680 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); d1696 3 a1698 6 void CodeGenFunction::EmitOMPOuterLoop( bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope, const CodeGenFunction::OMPLoopArguments &LoopArgs, const CodeGenFunction::CodeGenLoopTy &CodeGenLoop, const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) { d1716 2 a1717 4 // UB = min(UB, GlobalUB) or // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g. // 'distribute parallel for') EmitIgnoredExpr(LoopArgs.EUB); d1719 1 a1719 1 EmitIgnoredExpr(LoopArgs.Init); d1721 1 a1721 1 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond); d1723 2 a1724 3 BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST); d1744 1 a1744 1 EmitIgnoredExpr(LoopArgs.Init); d1758 11 a1768 14 // when 'distribute' is not combined with a 'for': // while (idx <= UB) { BODY; ++idx; } // when 'distribute' is combined with a 'for' // (e.g. 'distribute parallel for') // while (idx <= UB) { ; idx += ST; } EmitOMPInnerLoop( S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr, [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { CodeGenLoop(CGF, S, LoopExit); }, [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { CodeGenOrdered(CGF, Loc, IVSize, IVSigned); }); d1774 2 a1775 2 EmitIgnoredExpr(LoopArgs.NextLB); EmitIgnoredExpr(LoopArgs.NextUB); d1794 1 a1794 2 const OMPLoopArguments &LoopArgs, const CodeGenDispatchBoundsTy &CGDispatchBounds) { d1803 1 a1803 1 LoopArgs.Chunk != nullptr)) && d1861 1 a1861 5 auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); llvm::Value *LBVal = DispatchBounds.first; llvm::Value *UBVal = DispatchBounds.second; CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal, LoopArgs.Chunk}; d1863 1 a1863 1 IVSigned, Ordered, DipatchRTInputValues); d1866 1 a1866 2 Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); d1869 2 a1870 18 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, const unsigned IVSize, const bool IVSigned) { if (Ordered) { CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, IVSigned); } }; OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB); OuterLoopArgs.IncExpr = S.getInc(); OuterLoopArgs.Init = S.getInit(); OuterLoopArgs.Cond = S.getCond(); OuterLoopArgs.NextLB = S.getNextLowerBound(); OuterLoopArgs.NextUB = S.getNextUpperBound(); EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs, emitOMPLoopBodyWithStopPoint, CodeGenOrdered); a1872 3 static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned) {} d1874 3 a1876 3 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S, OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs, const CodeGenLoopTy &CodeGenLoopContent) { d1889 3 a1891 12 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); // for combined 'distribute' and 'for' the increment expression of distribute // is store in DistInc. For 'distribute' alone, it is in Inc. Expr *IncExpr; if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())) IncExpr = S.getDistInc(); else IncExpr = S.getInc(); d1893 2 a1894 127 // this routine is shared by 'omp distribute parallel for' and // 'omp distribute': select the right EUB expression depending on the // directive OMPLoopArguments OuterLoopArgs; OuterLoopArgs.LB = LoopArgs.LB; OuterLoopArgs.UB = LoopArgs.UB; OuterLoopArgs.ST = LoopArgs.ST; OuterLoopArgs.IL = LoopArgs.IL; OuterLoopArgs.Chunk = LoopArgs.Chunk; OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedEnsureUpperBound() : S.getEnsureUpperBound(); OuterLoopArgs.IncExpr = IncExpr; OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedInit() : S.getInit(); OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedCond() : S.getCond(); OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedNextLowerBound() : S.getNextLowerBound(); OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedNextUpperBound() : S.getNextUpperBound(); EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, LoopScope, OuterLoopArgs, CodeGenLoopContent, emitEmptyOrdered); } /// Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } static std::pair emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { const OMPLoopDirective &LS = cast(S); LValue LB = EmitOMPHelperVar(CGF, cast(LS.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(CGF, cast(LS.getUpperBoundVariable())); // When composing 'distribute' with 'for' (e.g. as in 'distribute // parallel for') we need to use the 'distribute' // chunk lower and upper bounds rather than the whole loop iteration // space. These are parameters to the outlined function for 'parallel' // and we copy the bounds of the previous schedule into the // the current ones. LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation()); PrevLBVal = CGF.EmitScalarConversion( PrevLBVal, LS.getPrevLowerBoundVariable()->getType(), LS.getIterationVariable()->getType(), SourceLocation()); llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation()); PrevUBVal = CGF.EmitScalarConversion( PrevUBVal, LS.getPrevUpperBoundVariable()->getType(), LS.getIterationVariable()->getType(), SourceLocation()); CGF.EmitStoreOfScalar(PrevLBVal, LB); CGF.EmitStoreOfScalar(PrevUBVal, UB); return {LB, UB}; } /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then /// we need to use the LB and UB expressions generated by the worksharing /// code generation support, whereas in non combined situations we would /// just emit 0 and the LastIteration expression /// This function is necessary due to the difference of the LB and UB /// types for the RT emission routines for 'for_static_init' and /// 'for_dispatch_init' static std::pair emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB) { const OMPLoopDirective &LS = cast(S); const Expr *IVExpr = LS.getIterationVariable(); // when implementing a dynamic schedule for a 'for' combined with a // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop // is not normalized as each team only executes its own assigned // distribute chunk QualType IteratorTy = IVExpr->getType(); llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, SourceLocation()); llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, SourceLocation()); return {LBVal, UBVal}; } static void emitDistributeParallelForDistributeInnerBoundParams( CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl &CapturedVars) { const auto &Dir = cast(S); LValue LB = CGF.EmitLValue(cast(Dir.getCombinedLowerBoundVariable())); auto LBCast = CGF.Builder.CreateIntCast( CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false); CapturedVars.push_back(LBCast); LValue UB = CGF.EmitLValue(cast(Dir.getCombinedUpperBoundVariable())); auto UBCast = CGF.Builder.CreateIntCast( CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false); CapturedVars.push_back(UBCast); } static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit) { auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), emitDistributeParallelForInnerBounds, emitDistributeParallelForDispatchBounds); }; emitCommonOMPParallelDirective( CGF, S, OMPD_for, CGInlinedWorksharingLoop, emitDistributeParallelForDistributeInnerBoundParams); a1898 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; d1900 9 a1908 4 OMPCancelStackRAII CancelRegion(*this, OMPD_distribute_parallel_for, /*HasCancel=*/false); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen, /*HasCancel=*/false); d2006 9 d2055 8 d2075 1 a2075 4 bool CodeGenFunction::EmitOMPWorksharingLoop( const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds) { d2125 4 a2128 4 std::pair Bounds = CodeGenLoopBounds(*this, S); LValue LB = Bounds.first; LValue UB = Bounds.second; a2213 3 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk, EUB); d2215 2 a2216 1 LoopArguments, CGDispatchBounds); d2225 1 a2225 4 EmitOMPReductionClauseFinal( S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind()) ? /*Parallel and Simd*/ OMPD_parallel_for_simd : /*Parallel only*/ OMPD_parallel); a2250 28 /// The following two functions generate expressions for the loop lower /// and upper bounds in case of static and dynamic (dispatch) schedule /// of the associated 'for' or 'distribute' loop. static std::pair emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { const OMPLoopDirective &LS = cast(S); LValue LB = EmitOMPHelperVar(CGF, cast(LS.getLowerBoundVariable())); LValue UB = EmitOMPHelperVar(CGF, cast(LS.getUpperBoundVariable())); return {LB, UB}; } /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not /// consider the lower and upper bound expressions generated by the /// worksharing loop support, but we use 0 and the iteration space size as /// constants static std::pair emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB) { const OMPLoopDirective &LS = cast(S); const Expr *IVExpr = LS.getIterationVariable(); const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); return {LBVal, UBVal}; } d2256 1 a2256 3 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); d2274 1 a2274 3 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); d2323 2 a2324 1 OK_Ordinary, S.getLocStart(), FPOptions()); d2400 1 a2400 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); d2526 1 a2526 2 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); d2528 1 a2528 2 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen, emitEmptyBoundParameters); d2536 1 a2536 2 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); d2538 1 a2538 2 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen, emitEmptyBoundParameters); d2548 1 a2548 2 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen, emitEmptyBoundParameters); a2631 21 SmallVector LHSs; SmallVector RHSs; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); for (const auto *Ref : C->varlists()) { Data.ReductionVars.emplace_back(Ref); Data.ReductionCopies.emplace_back(*IPriv); Data.ReductionOps.emplace_back(*IRed); LHSs.emplace_back(*ILHS); RHSs.emplace_back(*IRHS); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ILHS, 1); std::advance(IRHS, 1); } } Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit( *this, S.getLocStart(), LHSs, RHSs, Data); d2636 1 a2636 1 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs]( a2690 28 if (Data.Reductions) { OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true); ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies, Data.ReductionOps); llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) { RedCG.emitSharedLValue(CGF, Cnt); RedCG.emitAggregateType(CGF, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address(CGF.EmitScalarConversion( Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType( Data.ReductionCopies[Cnt]->getType()), SourceLocation()), Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), [Replacement]() { return Replacement; }); // FIXME: This must removed once the runtime library is fixed. // Emit required threadprivate variables for // initilizer/combiner/finalizer. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(), RedCG, Cnt); } } d2766 1 a2766 3 void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr) { d2807 4 a2810 11 LValue LB = EmitOMPHelperVar( *this, cast( (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedLowerBoundVariable() : S.getLowerBoundVariable()))); LValue UB = EmitOMPHelperVar( *this, cast( (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedUpperBoundVariable() : S.getUpperBoundVariable()))); d2862 1 a2862 3 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedEnsureUpperBound() : S.getEnsureUpperBound()); d2864 1 a2864 9 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedInit() : S.getInit()); Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedCond() : S.getCond(); // for distribute alone, codegen d2866 5 a2870 5 // when combined with 'for' (e.g. as in 'distribute parallel for') // while (idx <= UB) { ; idx += ST; } EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr, [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { CodeGenLoop(CGF, S, LoopExit); d2879 3 a2881 5 const OMPLoopArguments LoopArguments = { LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(), Chunk}; EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments, CodeGenLoop); d2903 1 a2903 2 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); d3253 1 a3253 1 IsPostfixUpdate](RValue XRValue) -> RValue { d3280 1 a3280 1 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue { a3329 1 case OMPC_task_reduction: d3407 22 a3428 5 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen) { assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind())); CodeGenModule &CGM = CGF.CGM; d3431 3 d3437 1 d3439 3 a3441 7 // Check for the at most one if clause associated with the target region. for (const auto *C : S.getClausesOfKind()) { if (C->getNameModifier() == OMPD_unknown || C->getNameModifier() == OMPD_target) { IfCond = C->getCondition(); break; } d3456 1 a3456 1 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) d3462 1 a3462 1 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); d3466 1 a3466 1 if (auto *D = dyn_cast(CGF.CurFuncDecl)) d3468 1 a3468 1 else if (auto *D = dyn_cast(CGF.CurFuncDecl)) d3472 1 a3472 1 CGM.getMangledName(GlobalDecl(cast(CGF.CurFuncDecl))); d3474 4 a3477 7 // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID, IsOffloadEntry, CodeGen); OMPLexicalScope Scope(CGF, S); llvm::SmallVector CapturedVars; CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, a3480 32 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action) { CodeGenFunction::OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); (void)PrivateScope.Privatize(); Action.Enter(CGF); CGF.EmitStmt(cast(S.getAssociatedStmt())->getCapturedStmt()); } void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } d3485 8 a3492 6 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams); auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); const OMPNumTeamsClause *NT = S.getSingleClause(); const OMPThreadLimitClause *TL = S.getSingleClause(); d3501 1 a3501 1 OMPTeamsScope Scope(CGF, S); a3513 1 CGF.EmitOMPReductionClauseInit(S, PrivateScope); a3515 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); a3517 35 emitPostUpdateForReductionClause( *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); } static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S) { auto *CS = S.getCapturedStmt(OMPD_teams); Action.Enter(CGF); auto &&CodeGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { // TODO: Add support for clauses. CGF.EmitStmt(CS->getCapturedStmt()); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); } void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDirective( const OMPTargetTeamsDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); a3742 36 static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action) { // Get the captured statement associated with the 'parallel' region. auto *CS = S.getCapturedStmt(OMPD_parallel); Action.Enter(CGF); auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) { CodeGenFunction::OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); // TODO: Add support for clauses. CGF.EmitStmt(CS->getCapturedStmt()); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); }; emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, emitEmptyBoundParameters); emitPostUpdateForReductionClause( CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; }); } void CodeGenFunction::EmitOMPTargetParallelDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } d3745 1 a3745 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d3887 1 a3887 12 if (Data.Nogroup) EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); else { CGM.getOpenMPRuntime().emitTaskgroupRegion( *this, [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data); }, S.getLocStart()); } @ 1.1.1.8 log @Import clang r319952 from branches/release_50 @ text @d1213 1 a1213 1 bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { d1215 1 a1215 1 return false; a1216 1 bool HasLinears = false; a1218 1 HasLinears = true; a1242 1 return HasLinears; d1532 1 a1532 1 (void)CGF.EmitOMPLinearClauseInit(S); d2150 1 a2150 1 bool HasLinears = EmitOMPLinearClauseInit(S); d2164 1 a2164 1 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) { @ 1.1.1.8.4.1 log @Sync with HEAD @ text @d32 2 a33 3 if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { d35 1 a35 1 if (!I->hasAttr()) { d37 1 a37 1 } else { d56 2 a57 4 OMPLexicalScope( CodeGenFunction &CGF, const OMPExecutableDirective &S, const llvm::Optional CapturedRegion = llvm::None, const bool EmitPreInitStmt = true) d62 18 a79 18 if (!CapturedRegion.hasValue()) return; assert(S.hasAssociatedStmt() && "Expected associated statement for inlined directive."); const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion); for (const auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE( const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); a81 1 (void)InlinedShareds.Privatize(); d97 3 a99 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d113 3 a115 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d122 5 a126 10 CodeGenFunction::OMPMapVars PreCondVars; for (const auto *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); (void)PreCondVars.setVarAddr( CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); } (void)PreCondVars.apply(CGF); if (const auto *PreInits = cast_or_null(S.getPreInits())) { for (const auto *I : PreInits->decls()) CGF.EmitVarDecl(cast(*I)); a127 1 PreCondVars.restore(CGF); a136 66 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { CodeGenFunction::OMPPrivateScope InlinedShareds; static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || (CGF.CurCodeDecl && isa(CGF.CurCodeDecl) && cast(CGF.CurCodeDecl)->capturesVariable(VD)); } public: OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), InlinedShareds(CGF) { for (const auto *C : S.clauses()) { if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { for (const auto *I : PreInit->decls()) { if (!I->hasAttr()) { CGF.EmitVarDecl(cast(*I)); } else { CodeGenFunction::AutoVarEmission Emission = CGF.EmitAutoVarAlloca(cast(*I)); CGF.EmitAutoVarCleanups(Emission); } } } } else if (const auto *UDP = dyn_cast(C)) { for (const Expr *E : UDP->varlists()) { const Decl *D = cast(E)->getDecl(); if (const auto *OED = dyn_cast(D)) CGF.EmitVarDecl(*OED); } } } if (!isOpenMPSimdDirective(S.getDirectiveKind())) CGF.EmitOMPPrivateClause(S, InlinedShareds); if (const auto *TG = dyn_cast(&S)) { if (const Expr *E = TG->getReductionRef()) CGF.EmitVarDecl(*cast(cast(E)->getDecl())); } const auto *CS = cast_or_null(S.getAssociatedStmt()); while (CS) { for (auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE(const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); } } CS = dyn_cast(CS->getCapturedStmt()); } (void)InlinedShareds.Privatize(); } }; a138 20 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen); LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { if (const auto *OrigDRE = dyn_cast(E)) { if (const auto *OrigVD = dyn_cast(OrigDRE->getDecl())) { OrigVD = OrigVD->getCanonicalDecl(); bool IsCaptured = LambdaCaptureFields.lookup(OrigVD) || (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || (CurCodeDecl && isa(CurCodeDecl)); DeclRefExpr DRE(const_cast(OrigVD), IsCaptured, OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); return EmitLValue(&DRE); } } return EmitLValue(E); } d140 1 a140 1 ASTContext &C = getContext(); d145 4 a148 5 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) { VlaSizePair VlaSize = getVLASize(VAT); Ty = VlaSize.Type; Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts; d153 4 a156 3 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); } return CGM.getSize(SizeInChars); d168 2 a169 2 const VariableArrayType *VAT = CurField->getCapturedVLAType(); llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()]; d171 1 a171 1 } else if (CurCap->capturesThis()) { d173 3 a175 2 } else if (CurCap->capturesVariableByCopy()) { llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation()); d180 2 a181 2 ASTContext &Ctx = getContext(); Address DstAddr = CreateMemTemp( d183 1 a183 1 Twine(CurCap->getCapturedVar()->getName(), ".casted")); d186 1 a186 1 llvm::Value *SrcAddrVal = EmitScalarConversion( d188 1 a188 1 Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); d196 1 a196 1 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation()); d206 2 a207 3 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, d211 1 a211 1 llvm::Value *CastedPtr = CGF.EmitScalarConversion( d213 2 a214 2 Ctx.getPointerType(DstType), Loc); Address TmpAddr = d222 4 a225 4 llvm::Value *RefVal = TmpAddr.getPointer(); TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref")); LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true); d232 1 a232 1 if (T->isLValueReferenceType()) d236 1 a238 6 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) { if (const auto *VLA = dyn_cast(A)) return getCanonicalParamType(C, VLA->getElementType()); if (!A->isVariablyModifiedType()) return C.getCanonicalType(T); } d249 2 a250 2 const bool UIntPtrCastRequired = true; /// true if only casted arguments must be registered as local args or VLA d252 1 a252 1 const bool RegisterCastedArgsOnly = false; d254 1 a254 1 const StringRef FunctionName; d264 1 a264 1 static llvm::Function *emitOutlinedFunctionPrologue( d266 1 a266 1 llvm::MapVector> d279 1 a279 1 FunctionArgList TargetArgs; a281 3 TargetArgs.append( CD->param_begin(), std::next(CD->param_begin(), CD->getContextParamPosition())); d283 1 a283 11 FunctionDecl *DebugFunctionDecl = nullptr; if (!FO.UIntPtrCastRequired) { FunctionProtoType::ExtProtoInfo EPI; DebugFunctionDecl = FunctionDecl::Create( Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(), SourceLocation(), DeclarationName(), Ctx.VoidTy, Ctx.getTrivialTypeSourceInfo( Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)), SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false); } for (const FieldDecl *FD : RD->fields()) { d293 6 a298 4 if (FO.UIntPtrCastRequired && ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || I->capturesVariableArrayType())) ArgType = Ctx.getUIntPtrType(); d303 1 a303 1 } else if (I->capturesThis()) { d305 1 a305 1 } else { d310 4 a313 18 ArgType = getCanonicalParamType(Ctx, ArgType); VarDecl *Arg; if (DebugFunctionDecl && (CapVar || I->capturesThis())) { Arg = ParmVarDecl::Create( Ctx, DebugFunctionDecl, CapVar ? CapVar->getLocStart() : FD->getLocStart(), CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType, /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr); } else { Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), II, ArgType, ImplicitParamDecl::Other); } Args.emplace_back(Arg); // Do not cast arguments if we emit function with non-original types. TargetArgs.emplace_back( FO.UIntPtrCastRequired ? Arg : CGM.getOpenMPRuntime().translateParameter(FD, Arg)); a318 3 TargetArgs.append( std::next(CD->param_begin(), CD->getContextParamPosition() + 1), CD->param_end()); d321 1 d323 1 a323 1 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs); d326 1 a326 1 auto *F = d331 1 a331 2 F->setDoesNotThrow(); F->setDoesNotRecurse(); d334 2 a335 2 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, FO.S->getLocStart(), CD->getBody()->getLocStart()); d338 1 a338 9 for (const FieldDecl *FD : RD->fields()) { // Do not map arguments if we emit function with non-original types. Address LocalAddr(Address::invalid()); if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) { LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], TargetArgs[Cnt]); } else { LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); } d343 1 d359 3 a361 2 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), AlignmentSource::Decl); d364 9 a372 8 ArgLVal = CGF.MakeAddrLValue( castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal), FD->getType(), AlignmentSource::Decl); } llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); const VariableArrayType *VAT = FD->getCapturedVLAType(); VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg); d374 1 a374 1 const VarDecl *Var = I->getCapturedVar(); d379 3 a381 3 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { d395 1 a395 1 const VarDecl *Var = I->getCapturedVar(); d399 5 a403 5 {Var, FO.UIntPtrCastRequired ? castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType()) : ArgLVal.getAddress()}}); d407 2 a408 1 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); d415 1 a415 1 return F; d429 1 a429 1 llvm::MapVector> LocalAddrs; a430 5 SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); Out << CapturedStmtInfo->getHelperName(); if (NeedWrapperFunction) Out << "_debug__"; d432 5 a436 3 Out.str()); llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, VLASizes, CXXThisValue, FO); d448 1 a448 1 if (!NeedWrapperFunction) d453 1 a453 1 CapturedStmtInfo->getHelperName()); d455 1 a455 1 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo; d461 2 a462 1 WrapperCGF.CXXThisValue, WrapperFO); d468 3 a470 5 LValue LV = WrapperCGF.MakeAddrLValue( I->second.second, I->second.first ? I->second.first->getType() : Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d473 1 a473 1 if (EI != VLASizes.end()) { d475 1 a475 1 } else { d477 2 a478 3 Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d481 1 a481 1 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType())); d483 1 a483 2 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(), F, CallArgs); d493 1 a493 1 const llvm::function_ref CopyGen) { d498 2 a499 2 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe(); llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); d502 2 a503 2 llvm::Value *SrcBegin = SrcAddr.getPointer(); llvm::Value *DestBegin = DestAddr.getPointer(); d505 1 a505 1 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements); d507 3 a509 3 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body"); llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done"); llvm::Value *IsEmpty = d514 1 a514 1 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); d537 1 a537 1 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32( d539 1 a539 1 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32( d542 1 a542 1 llvm::Value *Done = d556 1 a556 1 const auto *BO = dyn_cast(Copy); d559 1 a559 3 LValue Dest = MakeAddrLValue(DestAddr, OriginalType); LValue Src = MakeAddrLValue(SrcAddr, OriginalType); EmitAggregateAssign(Dest, Src, OriginalType); d570 5 a574 2 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; }); Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; }); d582 2 a583 2 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; }); Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; }); d602 1 a602 6 llvm::SmallVector CaptureRegions; getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); // Force emission of the firstprivate copy if the directive does not emit // outlined function, like omp for, omp simd, omp distribute etc. bool MustEmitFirstprivateCopy = CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown; d606 2 a607 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d610 3 a612 2 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD); if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD && d622 2 a623 3 const auto *VD = cast(cast(IInit)->getDecl()); const auto *VDInit = cast(cast(*InitsRef)->getDecl()); d628 1 a628 1 LValue OriginalLVal = EmitLValue(&DRE); d634 25 a658 30 IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VD, Type, OriginalLVal, VDInit]() { AutoVarEmission Emission = EmitAutoVarAlloca(*VD); const Expr *Init = VD->getInit(); if (!isa(Init) || isTrivialInitializer(Init)) { // Perform simple memcpy. LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type); EmitAggregateAssign(Dest, OriginalLVal, Type); } else { EmitOMPAggregateAssign( Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type, [this, VDInit, Init](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the // initialization. RunCleanupsScope InitScope(*this); // Emit initialization for single element. setAddrOfLocalVar(VDInit, SrcElement); EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); }); } EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); d660 10 a669 11 Address OriginalAddr = OriginalLVal.getAddress(); IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VDInit, OriginalAddr, VD]() { // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable (for proper handling of captured global variables). setAddrOfLocalVar(VDInit, OriginalAddr); EmitDecl(*VD); LocalDeclMap.erase(VDInit); return GetAddrOfLocalVar(VD); }); d691 2 a692 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d694 7 a700 6 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); d723 2 a724 2 for (const Expr *AssignOp : C->assignment_ops()) { const auto *VD = cast(cast(*IRef)->getDecl()); d755 1 a755 2 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)), d759 2 a760 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d783 2 a784 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d792 1 a792 2 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && !getLangOpts().OpenMPSimd) d796 1 a796 1 for (const Expr *IInit : C->private_copies()) { d799 1 a799 1 const auto *OrigVD = cast(cast(*IRef)->getDecl()); d803 2 a804 3 const auto *DestVD = cast(cast(*IDestRef)->getDecl()); PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() { d816 2 a817 2 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { d855 1 a855 1 if (const auto *LoopDirective = dyn_cast(&D)) { d857 2 a858 2 for (const Expr *F : LoopDirective->finals()) { const auto *D = d871 2 a872 3 for (const Expr *AssignOp : C->assignment_ops()) { const auto *PrivateVD = cast(cast(*IRef)->getDecl()); d874 1 a874 1 const auto *CanonicalVD = PrivateVD->getCanonicalDecl(); d879 1 a879 1 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) d881 2 a882 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d887 1 a887 1 if (const auto *RefTy = PrivateVD->getType()->getAs()) d897 1 a897 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d919 1 a919 1 for (const Expr *Ref : C->varlists()) { d936 2 a937 2 for (const Expr *IRef : Shareds) { const auto *PrivateVD = cast(cast(*IPriv)->getDecl()); d941 1 a941 1 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); d952 1 a952 1 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; }); d957 3 a959 5 const auto *LHSVD = cast(cast(*ILHS)->getDecl()); const auto *RHSVD = cast(cast(*IRHS)->getDecl()); QualType Type = PrivateVD->getType(); bool isaOMPArraySectionExpr = isa(IRef); if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { d962 1 a962 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d965 4 a968 4 PrivateScope.addPrivate( RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); }); } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || isa(IRef)) { d971 1 a971 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d974 1 a974 1 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() { a988 1 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; }); d990 3 a992 1 RHSVD, [this, PrivateVD, RHSVD, IsArray]() { d1026 2 a1027 2 ReductionKind == OMPD_simd; bool SimpleReduction = ReductionKind == OMPD_simd; d1038 1 a1038 1 const llvm::function_ref CondGen) { d1043 1 a1043 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) { d1045 1 a1045 1 if (llvm::Value *Cond = CondGen(CGF)) { d1048 1 a1048 1 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); d1076 2 a1077 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d1080 2 a1081 3 llvm::Value *NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true); d1117 1 a1117 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d1132 1 a1132 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); d1137 2 a1138 2 emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d1145 3 a1147 2 for (const Expr *UE : D.updates()) EmitIgnoredExpr(UE); d1149 3 a1151 7 // In distribute directives only loop counters may be marked as linear, no // need to generate the code for them. if (!isOpenMPDistributeDirective(D.getDirectiveKind())) { for (const auto *C : D.getClausesOfKind()) { for (const Expr *UE : C->updates()) EmitIgnoredExpr(UE); } d1155 1 a1155 1 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue"); d1167 2 a1168 2 const llvm::function_ref BodyGen, const llvm::function_ref PostIncGen) { d1174 1 a1174 1 const SourceRange R = S.getSourceRange(); d1180 1 a1180 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1184 1 a1184 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body"); d1197 1 a1197 1 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); d1219 1 a1219 1 for (const Expr *Init : C->inits()) { d1221 2 a1222 3 const auto *VD = cast(cast(Init)->getDecl()); if (const auto *Ref = dyn_cast(VD->getInit()->IgnoreImpCasts())) { d1224 1 a1224 1 const auto *OrigVD = cast(Ref->getDecl()); d1233 1 a1233 1 } else { a1234 1 } d1238 2 a1239 2 if (const auto *CS = cast_or_null(C->getCalcStep())) if (const auto *SaveRef = cast(CS->getLHS())) { d1250 1 a1250 1 const llvm::function_ref CondGen) { d1257 1 a1257 1 for (const Expr *F : C->finals()) { d1259 1 a1259 1 if (llvm::Value *Cond = CondGen(*this)) { d1262 1 a1262 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu"); d1268 1 a1268 1 const auto *OrigVD = cast(cast(*IC)->getDecl()); d1274 1 a1274 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1279 1 a1279 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d1292 2 a1293 2 if (const Expr *AlignmentExpr = Clause->getAlignment()) { auto *AlignmentCI = d1297 1 a1297 1 for (const Expr *E : Clause->varlists()) { d1324 13 a1336 9 for (const Expr *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = cast(cast(*I)->getDecl()); // Emit var without initialization. AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD); EmitAutoVarCleanups(VarEmission); LocalDeclMap.erase(PrivateVD); (void)LoopScope.addPrivate(VD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); d1340 1 a1340 1 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() { a1345 4 } else { (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); }); d1361 1 a1361 1 for (const Expr *I : S.inits()) { d1375 2 a1376 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d1383 3 a1385 3 for (const Expr *E : C->varlists()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = d1388 1 a1388 1 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() { d1396 1 a1396 1 } else { a1397 1 } d1411 1 a1411 1 auto *Val = cast(Len.getScalarVal()); d1421 1 a1421 1 auto *Val = cast(Len.getScalarVal()); d1426 1 a1426 1 CGF.LoopStack.setParallel(/*Enable=*/false); d1434 1 a1434 1 LoopStack.setVectorizeEnable(); d1440 1 a1440 1 const llvm::function_ref CondGen) { d1446 4 a1449 4 for (const Expr *F : D.finals()) { const auto *OrigVD = cast(cast((*IC))->getDecl()); const auto *PrivateVD = cast(cast((*IPC))->getDecl()); const auto *CED = dyn_cast(OrigVD); d1453 1 a1453 1 if (llvm::Value *Cond = CondGen(*this)) { d1456 1 a1456 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then"); d1463 1 a1463 1 if (CED) { d1465 1 a1465 1 } else { d1472 2 a1473 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1491 8 a1498 7 /// Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } d1500 16 a1515 17 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); assert(isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive"); OMPLoopScope PreInitScope(CGF, S); // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // if (isOpenMPDistributeDirective(S.getDirectiveKind()) || isOpenMPWorksharingDirective(S.getDirectiveKind()) || isOpenMPTaskLoopDirective(S.getDirectiveKind())) { (void)EmitOMPHelperVar(CGF, cast(S.getLowerBoundVariable())); (void)EmitOMPHelperVar(CGF, cast(S.getUpperBoundVariable())); } d1517 5 a1521 16 // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); ContBlock = CGF.createBasicBlock("simd.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } d1523 8 a1530 5 // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const auto *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); d1532 1 a1532 8 // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } d1534 34 a1567 38 CGF.EmitOMPSimdInit(S); emitAlignedClause(CGF, S); (void)CGF.EmitOMPLinearClauseInit(S); { CodeGenFunction::OMPPrivateScope LoopScope(CGF); CGF.EmitOMPPrivateLoopCounters(S, LoopScope); CGF.EmitOMPLinearClause(S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit final copy of the lastprivate variables at the end of loops. if (HasLastprivateClause) CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); d1569 1 a1569 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d1579 1 a1579 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1585 1 a1585 1 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); d1588 1 a1588 1 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond"); d1590 1 a1590 1 const SourceRange R = S.getSourceRange(); d1612 1 a1612 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1616 1 a1616 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body"); d1630 1 a1630 1 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); d1672 1 a1672 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d1682 1 a1682 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1748 1 a1748 2 const std::pair DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); d1756 3 a1758 5 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d1789 1 a1789 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1800 4 a1803 4 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit); d1844 8 d1869 1 a1869 2 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc()); d1872 2 a1873 4 LS.getIterationVariable()->getType(), LS.getPrevLowerBoundVariable()->getExprLoc()); llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc()); d1876 1 a1876 2 LS.getIterationVariable()->getType(), LS.getPrevUpperBoundVariable()->getExprLoc()); d1902 4 a1905 4 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getLocStart()); llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getLocStart()); d1915 1 a1915 1 llvm::Value *LBCast = CGF.Builder.CreateIntCast( d1921 1 a1921 1 llvm::Value *UBCast = CGF.Builder.CreateIntCast( d1931 1 a1931 14 PrePostActionTy &Action) { Action.Enter(CGF); bool HasCancel = false; if (!isOpenMPSimdDirective(S.getDirectiveKind())) { if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); } CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), HasCancel); d1938 1 a1938 3 CGF, S, isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for, CGInlinedWorksharingLoop, d1948 5 a1952 2 OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1957 8 a1964 6 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1969 8 a1976 5 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); d1979 10 a1988 12 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); d1993 95 a2087 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d2107 2 a2108 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2114 1 a2114 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2120 1 a2120 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2135 1 a2135 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2142 1 RunCleanupsScope DoacrossCleanupScope(*this); d2144 1 a2144 1 if (const auto *OrderedClause = S.getSingleClause()) { d2185 1 a2185 1 if (const auto *C = S.getSingleClause()) { d2189 1 a2189 1 if (const Expr *Ch = C->getChunkSize()) { d2212 5 a2216 6 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); JumpDest LoopExit = d2233 1 a2233 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2252 1 a2252 1 [IL, &S](CodeGenFunction &CGF) { d2263 1 a2263 1 *this, S, [IL, &S](CodeGenFunction &CGF) { d2273 1 a2273 1 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { a2276 1 DoacrossCleanupScope.ForceCleanup(); d2280 1 a2280 1 EmitBlock(ContBlock, /*IsFinished=*/true); d2291 1 a2291 1 const auto &LS = cast(S); d2306 1 a2306 1 const auto &LS = cast(S); d2324 1 a2324 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2330 1 a2330 1 if (!S.getSingleClause() || HasLastprivates) d2332 1 d2344 1 a2344 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2349 1 a2349 1 if (!S.getSingleClause() || HasLastprivates) d2351 1 d2357 1 a2357 1 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); d2364 2 a2365 2 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt(); const auto *CS = dyn_cast(CapturedStmt); d2367 4 a2370 5 auto &&CodeGen = [&S, CapturedStmt, CS, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { ASTContext &C = CGF.getContext(); QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); d2374 2 a2375 3 llvm::ConstantInt *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1) : CGF.Builder.getInt32(0); d2393 2 a2394 2 S.getLocStart(), true); auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { d2406 4 a2409 4 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); llvm::SwitchInst *SwitchStmt = CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getLocStart()), ExitBB, CS == nullptr ? 1 : CS->size()); d2412 1 a2412 1 for (const Stmt *SubStmt : CS->children()) { d2421 1 a2421 1 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); d2424 1 a2424 1 CGF.EmitStmt(CapturedStmt); a2446 3 CGOpenMPRuntime::StaticRTInput StaticInit( /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2448 3 a2450 1 CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d2452 2 a2453 2 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( d2463 1 a2463 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2468 5 a2472 4 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d2503 1 a2503 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2515 1 a2515 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2517 1 a2517 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2546 1 a2546 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2549 1 a2549 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2566 1 a2566 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2568 1 a2568 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2575 1 a2575 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2577 2 a2578 2 const Expr *Hint = nullptr; if (const auto *HintClause = S.getSingleClause()) d2580 1 a2580 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2590 1 a2590 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2603 1 a2603 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2615 1 a2615 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2622 4 a2625 4 void CodeGenFunction::EmitOMPTaskBasedDirective( const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data) { d2627 4 a2630 4 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); d2635 1 a2635 1 const Expr *Cond = Clause->getCondition(); d2647 1 a2647 1 const Expr *Prio = Clause->getPriority(); d2660 2 a2661 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2674 2 a2675 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2690 2 a2691 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2710 1 a2710 1 for (const Expr *Ref : C->varlists()) { d2726 4 a2729 5 for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs, CapturedRegion](CodeGenFunction &CGF, PrePostActionTy &Action) { d2734 4 a2737 5 enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); d2742 2 a2743 2 for (const Expr *E : Data.PrivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2746 1 a2746 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2749 2 a2750 2 for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2754 1 a2754 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2757 2 a2758 2 for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2762 1 a2762 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2765 3 a2767 4 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : LastprivateDstsOrigs) { const auto *OrigVD = cast(Pair.second->getDecl()); d2777 1 a2777 1 for (const auto &Pair : PrivatePtrs) { d2784 1 a2784 1 OMPLexicalScope LexScope(CGF, S, CapturedRegion); a2791 5 // FIXME: This must removed once the runtime library is fixed. // Emit required threadprivate variables for // initilizer/combiner/finalizer. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(), RedCG, Cnt); d2799 1 a2799 1 Data.ReductionCopies[Cnt]->getExprLoc()), a2803 32 } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); SmallVector InRedVars; SmallVector InRedPrivs; SmallVector InRedOps; SmallVector TaskgroupDescriptors; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ITD = C->taskgroup_descriptors().begin(); for (const Expr *Ref : C->varlists()) { InRedVars.emplace_back(Ref); InRedPrivs.emplace_back(*IPriv); InRedOps.emplace_back(*IRed); TaskgroupDescriptors.emplace_back(*ITD); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ITD, 1); } } // Privatize in_reduction items here, because taskgroup descriptors must be // privatized earlier. OMPPrivateScope InRedScope(CGF); if (!InRedVars.empty()) { ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps); for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) { RedCG.emitSharedLValue(CGF, Cnt); RedCG.emitAggregateType(CGF, Cnt); // The taskgroup descriptor variable is always implicit firstprivate and // privatized already during procoessing of the firstprivates. a2808 14 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), TaskgroupDescriptors[Cnt]->getExprLoc()); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), [Replacement]() { return Replacement; }); d2811 1 a2811 1 (void)InRedScope.Privatize(); d2816 1 a2816 1 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( a2822 144 static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc) { auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *OrigRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *PrivateRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); QualType ElemType = C.getBaseElementType(Ty); auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType, ImplicitParamDecl::Other); auto *InitRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), InitVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue); PrivateVD->setInitStyle(VarDecl::CInit); PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue, InitRef, /*BasePath=*/nullptr, VK_RValue)); Data.FirstprivateVars.emplace_back(OrigRef); Data.FirstprivateCopies.emplace_back(PrivateRef); Data.FirstprivateInits.emplace_back(InitRef); return OrigVD; } void CodeGenFunction::EmitOMPTargetTaskBasedDirective( const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo) { // Emit outlined function for task construct. const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); OMPTaskDataTy Data; // The task is not final. Data.Final.setInt(/*IntVal=*/false); // Get list of firstprivate variables. for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto IElemInitRef = C->inits().begin(); for (auto *IInit : C->private_copies()) { Data.FirstprivateVars.push_back(*IRef); Data.FirstprivateCopies.push_back(IInit); Data.FirstprivateInits.push_back(*IElemInitRef); ++IRef; ++IElemInitRef; } } OMPPrivateScope TargetScope(*this); VarDecl *BPVD = nullptr; VarDecl *PVD = nullptr; VarDecl *SVD = nullptr; if (InputInfo.NumberOfTargetItems > 0) { auto *CD = CapturedDecl::Create( getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0); llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems); QualType BaseAndPointersType = getContext().getConstantArrayType( getContext().VoidPtrTy, ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); BPVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); PVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); QualType SizesType = getContext().getConstantArrayType( getContext().getSizeType(), ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD, S.getLocStart()); TargetScope.addPrivate( BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; }); TargetScope.addPrivate(PVD, [&InputInfo]() { return InputInfo.PointersArray; }); TargetScope.addPrivate(SVD, [&InputInfo]() { return InputInfo.SizesArray; }); } (void)TargetScope.Privatize(); // Build list of dependences. for (const auto *C : S.getClausesOfKind()) for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { // Set proper addresses for generated private copies. OMPPrivateScope Scope(CGF); if (!Data.FirstprivateVars.empty()) { enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); // Map privates. llvm::SmallVector, 16> PrivatePtrs; llvm::SmallVector CallArgs; CallArgs.push_back(PrivatesPtr); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); } CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : PrivatePtrs) { Address Replacement(CGF.Builder.CreateLoad(Pair.second), CGF.getContext().getDeclAlign(Pair.first)); Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); if (InputInfo.NumberOfTargetItems > 0) { InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize()); } Action.Enter(CGF); OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); BodyGen(CGF); }; llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true, Data.NumberOfParts); llvm::APInt TrueOrFalse(32, S.hasClausesOfKind() ? 1 : 0); IntegerLiteral IfCond(getContext(), TrueOrFalse, getContext().getIntTypeForBitwidth(32, /*Signed=*/0), SourceLocation()); CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, &IfCond, Data); } d2825 3 a2827 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d2850 1 a2850 1 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data); d2870 1 a2870 30 if (const Expr *E = S.getReductionRef()) { SmallVector LHSs; SmallVector RHSs; OMPTaskDataTy Data; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); for (const Expr *Ref : C->varlists()) { Data.ReductionVars.emplace_back(Ref); Data.ReductionCopies.emplace_back(*IPriv); Data.ReductionOps.emplace_back(*IRed); LHSs.emplace_back(*ILHS); RHSs.emplace_back(*IRHS); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ILHS, 1); std::advance(IRHS, 1); } } llvm::Value *ReductionDesc = CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(), LHSs, RHSs, Data); const auto *VD = cast(cast(E)->getDecl()); CGF.EmitVarDecl(*VD); CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), /*Volatile=*/false, E->getType()); } CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2872 1 a2872 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2877 7 a2883 9 CGM.getOpenMPRuntime().emitFlush( *this, [&S]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); return llvm::None; }(), S.getLocStart()); d2890 2 a2891 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2897 1 a2897 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2903 1 a2903 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2918 1 a2918 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2925 1 emitAlignedClause(*this, S); d2947 2 a2948 2 // Emit implicit barrier to synchronize threads and avoid data races // on initialization of firstprivate variables and post-update of d2951 2 a2952 2 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); a2954 4 if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) EmitOMPReductionClauseInit(S, LoopScope); d2962 1 a2962 1 if (const auto *C = S.getSingleClause()) { d2964 1 a2964 1 if (const Expr *Ch = C->getChunkSize()) { d2967 2 a2968 2 S.getIterationVariable()->getType(), S.getLocStart()); a2983 5 if (isOpenMPSimdDirective(S.getDirectiveKind())) EmitOMPSimdInit(S, /*IsMonotonic=*/true); CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2985 4 a2988 2 StaticInit); JumpDest LoopExit = d2999 3 a3001 4 const Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedCond() : S.getCond(); d3014 1 a3014 1 RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind()); d3024 1 a3024 30 if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) { OpenMPDirectiveKind ReductionKind = OMPD_unknown; if (isOpenMPParallelDirective(S.getDirectiveKind()) && isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for_simd; } else if (isOpenMPParallelDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for; } else if (isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_simd; } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) && S.hasClausesOfKind()) { llvm_unreachable( "No reduction clauses is allowed in distribute directive."); } EmitOMPReductionClauseFinal(S, ReductionKind); // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( *this, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } d3026 1 a3026 1 if (HasLastprivateClause) { d3029 2 a3030 2 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } d3044 1 d3047 3 a3049 2 OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d3057 2 a3058 2 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); Fn->setDoesNotRecurse(); d3063 1 a3063 3 if (S.hasClausesOfKind()) { assert(!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct."); d3068 1 a3068 1 const auto *C = S.getSingleClause(); a3070 1 const CapturedStmt *CS = S.getInnermostCapturedStmt(); d3072 1 d3075 2 a3076 3 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), OutlinedFn, CapturedVars); d3079 2 a3080 1 CGF.EmitStmt(CS->getCapturedStmt()); d3083 1 a3083 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3093 5 a3097 4 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType, Loc) : CGF.EmitComplexToScalarConversion( Val.getComplexVal(), SrcType, DestType, Loc); d3108 3 a3110 4 QualType DestElementType = DestType->castAs()->getElementType(); llvm::Value *ScalarVal = CGF.EmitScalarConversion( Val.getScalarVal(), SrcType, DestElementType, Loc); d3115 2 a3116 3 QualType SrcElementType = SrcType->castAs()->getElementType(); QualType DestElementType = DestType->castAs()->getElementType(); d3155 1 a3155 1 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, d3179 1 a3179 1 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, d3198 1 a3198 1 ASTContext &Context = CGF.getContext(); a3261 1 case BO_Cmp: d3275 1 a3275 1 llvm::Value *UpdateVal = Update.getScalarVal(); d3281 1 a3281 2 llvm::Value *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); d3288 1 a3288 1 const llvm::function_ref CommonGen) { d3309 1 a3309 1 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, d3315 1 a3315 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3325 12 a3336 12 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); return CGF.EmitAnyExpr(UE); }; d3364 1 a3364 1 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, d3375 2 a3376 3 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; d3382 1 a3382 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3389 3 a3391 3 const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; d3393 1 a3393 1 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; d3395 1 a3395 1 IsPostfixUpdate](RValue XRValue) { d3422 1 a3422 1 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) { d3445 1 a3445 1 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, d3452 1 a3452 1 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); d3455 1 a3455 1 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); d3459 1 a3459 1 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); d3462 1 a3462 1 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, a3472 1 case OMPC_in_reduction: d3517 1 a3517 1 for (const OMPClause *C : S.clauses()) { d3525 3 a3527 2 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers(); if (const auto *EWC = dyn_cast(CS)) d3529 1 d3532 2 a3533 2 for (const Stmt *C : Compound->body()) { if (const auto *EWC = dyn_cast(C)) d3535 1 d3542 1 a3542 1 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), d3546 1 a3546 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3555 1 a3555 10 // On device emit this construct as inlined code. if (CGM.getLangOpts().OpenMPIsDevice) { OMPLexicalScope Scope(CGF, S, OMPD_target); CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); }); return; } d3572 1 a3572 1 if (auto *C = S.getSingleClause()) d3574 1 d3592 1 a3592 1 if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3594 1 a3594 1 else if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3603 5 a3607 2 OMPLexicalScope Scope(CGF, S, OMPD_task); CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device); a3611 1 Action.Enter(CGF); d3617 2 a3618 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); d3647 2 a3648 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d3650 2 a3651 2 const auto *NT = S.getSingleClause(); const auto *TL = S.getSingleClause(); d3653 2 a3654 2 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr; const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr; d3669 1 a3669 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3675 1 a3675 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); d3678 3 a3680 3 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d3687 2 a3688 8 // Emit teams region as a standalone region. auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); a3689 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); a3691 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3715 286 static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeSimdRegion( CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPTeamsDistributeDirective( const OMPTeamsDistributeDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( const OMPTeamsDistributeSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( const OMPTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } static void emitTargetTeamsDistributeParallelForRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S) { // Emit SPMD target teams distribute parallel for region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeParallelForSimdRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S) { // Emit SPMD target teams distribute parallel for simd region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( const OMPTargetTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } d3743 1 a3743 3 Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for); d3753 4 a3756 4 for (const Expr *PvtVarIt : C.private_copies()) { const auto *OrigVD = cast(cast(*OrigVarIt)->getDecl()); const auto *InitVD = cast(cast(*InitIt)->getDecl()); const auto *PvtVD = cast(cast(PvtVarIt)->getDecl()); d3762 1 a3762 1 if (const auto *OED = dyn_cast(MatchingVD)) { d3765 1 a3765 1 const auto *ME = cast(OED->getInit()); d3777 1 a3777 3 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD, InitAddrIt, InitVD, PvtVD]() { d3794 1 a3794 1 // of the previous declaration, so we don't need it anymore. d3831 1 a3831 1 CodeGenFunction &CGF, PrePostActionTy &Action) { d3833 2 a3834 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d3837 1 a3837 1 // Codegen that selects whether to generate the privatization code or not. d3856 1 a3856 1 } else { a3857 1 } d3883 1 a3883 1 if (const auto *C = S.getSingleClause()) d3888 1 a3888 1 if (const auto *C = S.getSingleClause()) d3908 1 a3908 1 if (const auto *C = S.getSingleClause()) d3913 1 a3913 1 if (const auto *C = S.getSingleClause()) a3915 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3928 1 a3928 1 if (const auto *C = S.getSingleClause()) d3933 1 a3933 1 if (const auto *C = S.getSingleClause()) a3935 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3943 1 a3943 1 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); d3945 1 a3945 2 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3957 2 a3958 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3982 32 static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPCancelStackRAII CancelRegion( CGF, OMPD_target_parallel_for, S.hasCancel()); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } d3985 1 a3985 43 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d3992 3 a3994 3 const auto *VDecl = cast(Helper->getDecl()); Privates.addPrivate(VDecl, [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); d4000 3 a4002 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d4045 1 a4045 1 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); d4076 1 a4076 1 const auto *IVDecl = cast(cast(IVExpr)->getDecl()); d4083 1 a4083 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d4113 1 a4113 2 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, &Data](CodeGenFunction &CGF, PrePostActionTy &) { d4122 3 a4124 3 if (Data.Nogroup) { EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); } else { d4130 1 a4130 2 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); d4155 1 a4155 1 if (const auto *C = S.getSingleClause()) d4160 1 a4160 1 if (const auto *C = S.getSingleClause()) a4162 1 OMPLexicalScope Scope(*this, S, OMPD_task); a4164 30 void CodeGenFunction::EmitSimpleOMPExecutableDirective( const OMPExecutableDirective &D) { if (!D.hasAssociatedStmt() || !D.getAssociatedStmt()) return; auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { if (isOpenMPSimdDirective(D.getDirectiveKind())) { emitOMPSimdRegion(CGF, cast(D), Action); } else { if (const auto *LD = dyn_cast(&D)) { for (const Expr *E : LD->counters()) { if (const auto *VD = dyn_cast( cast(E)->getDecl())) { // Emit only those that were not explicitly referenced in clauses. if (!CGF.LocalDeclMap.count(VD)) CGF.EmitVarDecl(*VD); } } } CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); } }; OMPSimdLexicalScope Scope(*this, D); CGM.getOpenMPRuntime().emitInlinedDirective( *this, isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd : D.getDirectiveKind(), CodeGen); } @ 1.1.1.8.4.2 log @Mostly merge changes from HEAD upto 20200411 @ text @@ 1.1.1.8.2.1 log @Sync with HEAD @ text @d32 2 a33 3 if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { d35 1 a35 1 if (!I->hasAttr()) { d37 1 a37 1 } else { d56 2 a57 4 OMPLexicalScope( CodeGenFunction &CGF, const OMPExecutableDirective &S, const llvm::Optional CapturedRegion = llvm::None, const bool EmitPreInitStmt = true) d62 18 a79 18 if (!CapturedRegion.hasValue()) return; assert(S.hasAssociatedStmt() && "Expected associated statement for inlined directive."); const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion); for (const auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE( const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); a81 1 (void)InlinedShareds.Privatize(); d97 3 a99 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d113 3 a115 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d122 5 a126 10 CodeGenFunction::OMPMapVars PreCondVars; for (const auto *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); (void)PreCondVars.setVarAddr( CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); } (void)PreCondVars.apply(CGF); if (const auto *PreInits = cast_or_null(S.getPreInits())) { for (const auto *I : PreInits->decls()) CGF.EmitVarDecl(cast(*I)); a127 1 PreCondVars.restore(CGF); a136 66 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { CodeGenFunction::OMPPrivateScope InlinedShareds; static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || (CGF.CurCodeDecl && isa(CGF.CurCodeDecl) && cast(CGF.CurCodeDecl)->capturesVariable(VD)); } public: OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), InlinedShareds(CGF) { for (const auto *C : S.clauses()) { if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { for (const auto *I : PreInit->decls()) { if (!I->hasAttr()) { CGF.EmitVarDecl(cast(*I)); } else { CodeGenFunction::AutoVarEmission Emission = CGF.EmitAutoVarAlloca(cast(*I)); CGF.EmitAutoVarCleanups(Emission); } } } } else if (const auto *UDP = dyn_cast(C)) { for (const Expr *E : UDP->varlists()) { const Decl *D = cast(E)->getDecl(); if (const auto *OED = dyn_cast(D)) CGF.EmitVarDecl(*OED); } } } if (!isOpenMPSimdDirective(S.getDirectiveKind())) CGF.EmitOMPPrivateClause(S, InlinedShareds); if (const auto *TG = dyn_cast(&S)) { if (const Expr *E = TG->getReductionRef()) CGF.EmitVarDecl(*cast(cast(E)->getDecl())); } const auto *CS = cast_or_null(S.getAssociatedStmt()); while (CS) { for (auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE(const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); } } CS = dyn_cast(CS->getCapturedStmt()); } (void)InlinedShareds.Privatize(); } }; a138 20 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen); LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { if (const auto *OrigDRE = dyn_cast(E)) { if (const auto *OrigVD = dyn_cast(OrigDRE->getDecl())) { OrigVD = OrigVD->getCanonicalDecl(); bool IsCaptured = LambdaCaptureFields.lookup(OrigVD) || (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || (CurCodeDecl && isa(CurCodeDecl)); DeclRefExpr DRE(const_cast(OrigVD), IsCaptured, OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); return EmitLValue(&DRE); } } return EmitLValue(E); } d140 1 a140 1 ASTContext &C = getContext(); d145 4 a148 5 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) { VlaSizePair VlaSize = getVLASize(VAT); Ty = VlaSize.Type; Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts; d153 4 a156 3 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); } return CGM.getSize(SizeInChars); d168 2 a169 2 const VariableArrayType *VAT = CurField->getCapturedVLAType(); llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()]; d171 1 a171 1 } else if (CurCap->capturesThis()) { d173 3 a175 2 } else if (CurCap->capturesVariableByCopy()) { llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation()); d180 2 a181 2 ASTContext &Ctx = getContext(); Address DstAddr = CreateMemTemp( d183 1 a183 1 Twine(CurCap->getCapturedVar()->getName(), ".casted")); d186 1 a186 1 llvm::Value *SrcAddrVal = EmitScalarConversion( d188 1 a188 1 Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); d196 1 a196 1 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation()); d206 2 a207 3 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, d211 1 a211 1 llvm::Value *CastedPtr = CGF.EmitScalarConversion( d213 2 a214 2 Ctx.getPointerType(DstType), Loc); Address TmpAddr = d222 4 a225 4 llvm::Value *RefVal = TmpAddr.getPointer(); TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref")); LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true); d232 1 a232 1 if (T->isLValueReferenceType()) d236 1 a238 6 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) { if (const auto *VLA = dyn_cast(A)) return getCanonicalParamType(C, VLA->getElementType()); if (!A->isVariablyModifiedType()) return C.getCanonicalType(T); } d249 2 a250 2 const bool UIntPtrCastRequired = true; /// true if only casted arguments must be registered as local args or VLA d252 1 a252 1 const bool RegisterCastedArgsOnly = false; d254 1 a254 1 const StringRef FunctionName; d264 1 a264 1 static llvm::Function *emitOutlinedFunctionPrologue( d266 1 a266 1 llvm::MapVector> d279 1 a279 1 FunctionArgList TargetArgs; a281 3 TargetArgs.append( CD->param_begin(), std::next(CD->param_begin(), CD->getContextParamPosition())); d283 1 a283 11 FunctionDecl *DebugFunctionDecl = nullptr; if (!FO.UIntPtrCastRequired) { FunctionProtoType::ExtProtoInfo EPI; DebugFunctionDecl = FunctionDecl::Create( Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(), SourceLocation(), DeclarationName(), Ctx.VoidTy, Ctx.getTrivialTypeSourceInfo( Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)), SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false); } for (const FieldDecl *FD : RD->fields()) { d293 6 a298 4 if (FO.UIntPtrCastRequired && ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || I->capturesVariableArrayType())) ArgType = Ctx.getUIntPtrType(); d303 1 a303 1 } else if (I->capturesThis()) { d305 1 a305 1 } else { d310 4 a313 18 ArgType = getCanonicalParamType(Ctx, ArgType); VarDecl *Arg; if (DebugFunctionDecl && (CapVar || I->capturesThis())) { Arg = ParmVarDecl::Create( Ctx, DebugFunctionDecl, CapVar ? CapVar->getLocStart() : FD->getLocStart(), CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType, /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr); } else { Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), II, ArgType, ImplicitParamDecl::Other); } Args.emplace_back(Arg); // Do not cast arguments if we emit function with non-original types. TargetArgs.emplace_back( FO.UIntPtrCastRequired ? Arg : CGM.getOpenMPRuntime().translateParameter(FD, Arg)); a318 3 TargetArgs.append( std::next(CD->param_begin(), CD->getContextParamPosition() + 1), CD->param_end()); d321 1 d323 1 a323 1 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs); d326 1 a326 1 auto *F = d331 1 a331 2 F->setDoesNotThrow(); F->setDoesNotRecurse(); d334 2 a335 2 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, FO.S->getLocStart(), CD->getBody()->getLocStart()); d338 1 a338 9 for (const FieldDecl *FD : RD->fields()) { // Do not map arguments if we emit function with non-original types. Address LocalAddr(Address::invalid()); if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) { LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], TargetArgs[Cnt]); } else { LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); } d343 1 d359 3 a361 2 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), AlignmentSource::Decl); d364 9 a372 8 ArgLVal = CGF.MakeAddrLValue( castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal), FD->getType(), AlignmentSource::Decl); } llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); const VariableArrayType *VAT = FD->getCapturedVLAType(); VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg); d374 1 a374 1 const VarDecl *Var = I->getCapturedVar(); d379 3 a381 3 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { d395 1 a395 1 const VarDecl *Var = I->getCapturedVar(); d399 5 a403 5 {Var, FO.UIntPtrCastRequired ? castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType()) : ArgLVal.getAddress()}}); d407 2 a408 1 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); d415 1 a415 1 return F; d429 1 a429 1 llvm::MapVector> LocalAddrs; a430 5 SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); Out << CapturedStmtInfo->getHelperName(); if (NeedWrapperFunction) Out << "_debug__"; d432 5 a436 3 Out.str()); llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, VLASizes, CXXThisValue, FO); d448 1 a448 1 if (!NeedWrapperFunction) d453 1 a453 1 CapturedStmtInfo->getHelperName()); d455 1 a455 1 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo; d461 2 a462 1 WrapperCGF.CXXThisValue, WrapperFO); d468 3 a470 5 LValue LV = WrapperCGF.MakeAddrLValue( I->second.second, I->second.first ? I->second.first->getType() : Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d473 1 a473 1 if (EI != VLASizes.end()) { d475 1 a475 1 } else { d477 2 a478 3 Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d481 1 a481 1 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType())); d483 1 a483 2 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(), F, CallArgs); d493 1 a493 1 const llvm::function_ref CopyGen) { d498 2 a499 2 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe(); llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); d502 2 a503 2 llvm::Value *SrcBegin = SrcAddr.getPointer(); llvm::Value *DestBegin = DestAddr.getPointer(); d505 1 a505 1 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements); d507 3 a509 3 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body"); llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done"); llvm::Value *IsEmpty = d514 1 a514 1 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); d537 1 a537 1 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32( d539 1 a539 1 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32( d542 1 a542 1 llvm::Value *Done = d556 1 a556 1 const auto *BO = dyn_cast(Copy); d559 1 a559 3 LValue Dest = MakeAddrLValue(DestAddr, OriginalType); LValue Src = MakeAddrLValue(SrcAddr, OriginalType); EmitAggregateAssign(Dest, Src, OriginalType); d570 5 a574 2 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; }); Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; }); d582 2 a583 2 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; }); Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; }); d602 1 a602 6 llvm::SmallVector CaptureRegions; getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); // Force emission of the firstprivate copy if the directive does not emit // outlined function, like omp for, omp simd, omp distribute etc. bool MustEmitFirstprivateCopy = CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown; d606 2 a607 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d610 3 a612 2 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD); if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD && d622 2 a623 3 const auto *VD = cast(cast(IInit)->getDecl()); const auto *VDInit = cast(cast(*InitsRef)->getDecl()); d628 1 a628 1 LValue OriginalLVal = EmitLValue(&DRE); d634 25 a658 30 IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VD, Type, OriginalLVal, VDInit]() { AutoVarEmission Emission = EmitAutoVarAlloca(*VD); const Expr *Init = VD->getInit(); if (!isa(Init) || isTrivialInitializer(Init)) { // Perform simple memcpy. LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type); EmitAggregateAssign(Dest, OriginalLVal, Type); } else { EmitOMPAggregateAssign( Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type, [this, VDInit, Init](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the // initialization. RunCleanupsScope InitScope(*this); // Emit initialization for single element. setAddrOfLocalVar(VDInit, SrcElement); EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); }); } EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); d660 10 a669 11 Address OriginalAddr = OriginalLVal.getAddress(); IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VDInit, OriginalAddr, VD]() { // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable (for proper handling of captured global variables). setAddrOfLocalVar(VDInit, OriginalAddr); EmitDecl(*VD); LocalDeclMap.erase(VDInit); return GetAddrOfLocalVar(VD); }); d691 2 a692 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d694 7 a700 6 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); d723 2 a724 2 for (const Expr *AssignOp : C->assignment_ops()) { const auto *VD = cast(cast(*IRef)->getDecl()); d755 1 a755 2 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)), d759 2 a760 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d783 2 a784 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d792 1 a792 2 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && !getLangOpts().OpenMPSimd) d796 1 a796 1 for (const Expr *IInit : C->private_copies()) { d799 1 a799 1 const auto *OrigVD = cast(cast(*IRef)->getDecl()); d803 2 a804 3 const auto *DestVD = cast(cast(*IDestRef)->getDecl()); PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() { d816 2 a817 2 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { d855 1 a855 1 if (const auto *LoopDirective = dyn_cast(&D)) { d857 2 a858 2 for (const Expr *F : LoopDirective->finals()) { const auto *D = d871 2 a872 3 for (const Expr *AssignOp : C->assignment_ops()) { const auto *PrivateVD = cast(cast(*IRef)->getDecl()); d874 1 a874 1 const auto *CanonicalVD = PrivateVD->getCanonicalDecl(); d879 1 a879 1 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) d881 2 a882 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d887 1 a887 1 if (const auto *RefTy = PrivateVD->getType()->getAs()) d897 1 a897 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d919 1 a919 1 for (const Expr *Ref : C->varlists()) { d936 2 a937 2 for (const Expr *IRef : Shareds) { const auto *PrivateVD = cast(cast(*IPriv)->getDecl()); d941 1 a941 1 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); d952 1 a952 1 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; }); d957 3 a959 5 const auto *LHSVD = cast(cast(*ILHS)->getDecl()); const auto *RHSVD = cast(cast(*IRHS)->getDecl()); QualType Type = PrivateVD->getType(); bool isaOMPArraySectionExpr = isa(IRef); if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { d962 1 a962 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d965 4 a968 4 PrivateScope.addPrivate( RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); }); } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || isa(IRef)) { d971 1 a971 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d974 1 a974 1 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() { a988 1 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; }); d990 3 a992 1 RHSVD, [this, PrivateVD, RHSVD, IsArray]() { d1026 2 a1027 2 ReductionKind == OMPD_simd; bool SimpleReduction = ReductionKind == OMPD_simd; d1038 1 a1038 1 const llvm::function_ref CondGen) { d1043 1 a1043 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) { d1045 1 a1045 1 if (llvm::Value *Cond = CondGen(CGF)) { d1048 1 a1048 1 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); d1076 2 a1077 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d1080 2 a1081 3 llvm::Value *NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true); d1117 1 a1117 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d1132 1 a1132 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); d1137 2 a1138 2 emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d1145 3 a1147 2 for (const Expr *UE : D.updates()) EmitIgnoredExpr(UE); d1149 3 a1151 7 // In distribute directives only loop counters may be marked as linear, no // need to generate the code for them. if (!isOpenMPDistributeDirective(D.getDirectiveKind())) { for (const auto *C : D.getClausesOfKind()) { for (const Expr *UE : C->updates()) EmitIgnoredExpr(UE); } d1155 1 a1155 1 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue"); d1167 2 a1168 2 const llvm::function_ref BodyGen, const llvm::function_ref PostIncGen) { d1174 1 a1174 1 const SourceRange R = S.getSourceRange(); d1180 1 a1180 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1184 1 a1184 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body"); d1197 1 a1197 1 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); d1219 1 a1219 1 for (const Expr *Init : C->inits()) { d1221 2 a1222 3 const auto *VD = cast(cast(Init)->getDecl()); if (const auto *Ref = dyn_cast(VD->getInit()->IgnoreImpCasts())) { d1224 1 a1224 1 const auto *OrigVD = cast(Ref->getDecl()); d1233 1 a1233 1 } else { a1234 1 } d1238 2 a1239 2 if (const auto *CS = cast_or_null(C->getCalcStep())) if (const auto *SaveRef = cast(CS->getLHS())) { d1250 1 a1250 1 const llvm::function_ref CondGen) { d1257 1 a1257 1 for (const Expr *F : C->finals()) { d1259 1 a1259 1 if (llvm::Value *Cond = CondGen(*this)) { d1262 1 a1262 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu"); d1268 1 a1268 1 const auto *OrigVD = cast(cast(*IC)->getDecl()); d1274 1 a1274 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1279 1 a1279 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d1292 2 a1293 2 if (const Expr *AlignmentExpr = Clause->getAlignment()) { auto *AlignmentCI = d1297 1 a1297 1 for (const Expr *E : Clause->varlists()) { d1324 13 a1336 9 for (const Expr *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = cast(cast(*I)->getDecl()); // Emit var without initialization. AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD); EmitAutoVarCleanups(VarEmission); LocalDeclMap.erase(PrivateVD); (void)LoopScope.addPrivate(VD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); d1340 1 a1340 1 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() { a1345 4 } else { (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); }); d1361 1 a1361 1 for (const Expr *I : S.inits()) { d1375 2 a1376 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d1383 3 a1385 3 for (const Expr *E : C->varlists()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = d1388 1 a1388 1 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() { d1396 1 a1396 1 } else { a1397 1 } d1411 1 a1411 1 auto *Val = cast(Len.getScalarVal()); d1421 1 a1421 1 auto *Val = cast(Len.getScalarVal()); d1426 1 a1426 1 CGF.LoopStack.setParallel(/*Enable=*/false); d1434 1 a1434 1 LoopStack.setVectorizeEnable(); d1440 1 a1440 1 const llvm::function_ref CondGen) { d1446 4 a1449 4 for (const Expr *F : D.finals()) { const auto *OrigVD = cast(cast((*IC))->getDecl()); const auto *PrivateVD = cast(cast((*IPC))->getDecl()); const auto *CED = dyn_cast(OrigVD); d1453 1 a1453 1 if (llvm::Value *Cond = CondGen(*this)) { d1456 1 a1456 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then"); d1463 1 a1463 1 if (CED) { d1465 1 a1465 1 } else { d1472 2 a1473 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1491 8 a1498 7 /// Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } d1500 16 a1515 17 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); assert(isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive"); OMPLoopScope PreInitScope(CGF, S); // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // if (isOpenMPDistributeDirective(S.getDirectiveKind()) || isOpenMPWorksharingDirective(S.getDirectiveKind()) || isOpenMPTaskLoopDirective(S.getDirectiveKind())) { (void)EmitOMPHelperVar(CGF, cast(S.getLowerBoundVariable())); (void)EmitOMPHelperVar(CGF, cast(S.getUpperBoundVariable())); } d1517 5 a1521 16 // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); ContBlock = CGF.createBasicBlock("simd.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } d1523 8 a1530 5 // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const auto *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); d1532 1 a1532 8 // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } d1534 34 a1567 38 CGF.EmitOMPSimdInit(S); emitAlignedClause(CGF, S); (void)CGF.EmitOMPLinearClauseInit(S); { CodeGenFunction::OMPPrivateScope LoopScope(CGF); CGF.EmitOMPPrivateLoopCounters(S, LoopScope); CGF.EmitOMPLinearClause(S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit final copy of the lastprivate variables at the end of loops. if (HasLastprivateClause) CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); d1569 1 a1569 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d1579 1 a1579 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1585 1 a1585 1 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); d1588 1 a1588 1 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond"); d1590 1 a1590 1 const SourceRange R = S.getSourceRange(); d1612 1 a1612 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1616 1 a1616 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body"); d1630 1 a1630 1 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); d1672 1 a1672 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d1682 1 a1682 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1748 1 a1748 2 const std::pair DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); d1756 3 a1758 5 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d1789 1 a1789 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1800 4 a1803 4 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit); d1844 8 d1869 1 a1869 2 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc()); d1872 2 a1873 4 LS.getIterationVariable()->getType(), LS.getPrevLowerBoundVariable()->getExprLoc()); llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc()); d1876 1 a1876 2 LS.getIterationVariable()->getType(), LS.getPrevUpperBoundVariable()->getExprLoc()); d1902 4 a1905 4 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getLocStart()); llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getLocStart()); d1915 1 a1915 1 llvm::Value *LBCast = CGF.Builder.CreateIntCast( d1921 1 a1921 1 llvm::Value *UBCast = CGF.Builder.CreateIntCast( d1931 1 a1931 14 PrePostActionTy &Action) { Action.Enter(CGF); bool HasCancel = false; if (!isOpenMPSimdDirective(S.getDirectiveKind())) { if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); } CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), HasCancel); d1938 1 a1938 3 CGF, S, isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for, CGInlinedWorksharingLoop, d1948 5 a1952 2 OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1957 8 a1964 6 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1969 8 a1976 5 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); d1979 10 a1988 12 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); d1993 95 a2087 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d2107 2 a2108 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2114 1 a2114 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2120 1 a2120 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2135 1 a2135 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2142 1 RunCleanupsScope DoacrossCleanupScope(*this); d2144 1 a2144 1 if (const auto *OrderedClause = S.getSingleClause()) { d2185 1 a2185 1 if (const auto *C = S.getSingleClause()) { d2189 1 a2189 1 if (const Expr *Ch = C->getChunkSize()) { d2212 5 a2216 6 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); JumpDest LoopExit = d2233 1 a2233 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2252 1 a2252 1 [IL, &S](CodeGenFunction &CGF) { d2263 1 a2263 1 *this, S, [IL, &S](CodeGenFunction &CGF) { d2273 1 a2273 1 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { a2276 1 DoacrossCleanupScope.ForceCleanup(); d2280 1 a2280 1 EmitBlock(ContBlock, /*IsFinished=*/true); d2291 1 a2291 1 const auto &LS = cast(S); d2306 1 a2306 1 const auto &LS = cast(S); d2324 1 a2324 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2330 1 a2330 1 if (!S.getSingleClause() || HasLastprivates) d2332 1 d2344 1 a2344 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2349 1 a2349 1 if (!S.getSingleClause() || HasLastprivates) d2351 1 d2357 1 a2357 1 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); d2364 2 a2365 2 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt(); const auto *CS = dyn_cast(CapturedStmt); d2367 4 a2370 5 auto &&CodeGen = [&S, CapturedStmt, CS, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { ASTContext &C = CGF.getContext(); QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); d2374 2 a2375 3 llvm::ConstantInt *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1) : CGF.Builder.getInt32(0); d2393 2 a2394 2 S.getLocStart(), true); auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { d2406 4 a2409 4 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); llvm::SwitchInst *SwitchStmt = CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getLocStart()), ExitBB, CS == nullptr ? 1 : CS->size()); d2412 1 a2412 1 for (const Stmt *SubStmt : CS->children()) { d2421 1 a2421 1 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); d2424 1 a2424 1 CGF.EmitStmt(CapturedStmt); a2446 3 CGOpenMPRuntime::StaticRTInput StaticInit( /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2448 3 a2450 1 CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d2452 2 a2453 2 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( d2463 1 a2463 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2468 5 a2472 4 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d2503 1 a2503 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2515 1 a2515 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2517 1 a2517 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2546 1 a2546 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2549 1 a2549 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2566 1 a2566 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2568 1 a2568 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2575 1 a2575 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2577 2 a2578 2 const Expr *Hint = nullptr; if (const auto *HintClause = S.getSingleClause()) d2580 1 a2580 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2590 1 a2590 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2603 1 a2603 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2615 1 a2615 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2622 4 a2625 4 void CodeGenFunction::EmitOMPTaskBasedDirective( const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data) { d2627 4 a2630 4 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); d2635 1 a2635 1 const Expr *Cond = Clause->getCondition(); d2647 1 a2647 1 const Expr *Prio = Clause->getPriority(); d2660 2 a2661 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2674 2 a2675 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2690 2 a2691 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2710 1 a2710 1 for (const Expr *Ref : C->varlists()) { d2726 4 a2729 5 for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs, CapturedRegion](CodeGenFunction &CGF, PrePostActionTy &Action) { d2734 4 a2737 5 enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); d2742 2 a2743 2 for (const Expr *E : Data.PrivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2746 1 a2746 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2749 2 a2750 2 for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2754 1 a2754 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2757 2 a2758 2 for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2762 1 a2762 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2765 3 a2767 4 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : LastprivateDstsOrigs) { const auto *OrigVD = cast(Pair.second->getDecl()); d2777 1 a2777 1 for (const auto &Pair : PrivatePtrs) { d2784 1 a2784 1 OMPLexicalScope LexScope(CGF, S, CapturedRegion); a2791 5 // FIXME: This must removed once the runtime library is fixed. // Emit required threadprivate variables for // initilizer/combiner/finalizer. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(), RedCG, Cnt); d2799 1 a2799 1 Data.ReductionCopies[Cnt]->getExprLoc()), a2803 32 } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); SmallVector InRedVars; SmallVector InRedPrivs; SmallVector InRedOps; SmallVector TaskgroupDescriptors; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ITD = C->taskgroup_descriptors().begin(); for (const Expr *Ref : C->varlists()) { InRedVars.emplace_back(Ref); InRedPrivs.emplace_back(*IPriv); InRedOps.emplace_back(*IRed); TaskgroupDescriptors.emplace_back(*ITD); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ITD, 1); } } // Privatize in_reduction items here, because taskgroup descriptors must be // privatized earlier. OMPPrivateScope InRedScope(CGF); if (!InRedVars.empty()) { ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps); for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) { RedCG.emitSharedLValue(CGF, Cnt); RedCG.emitAggregateType(CGF, Cnt); // The taskgroup descriptor variable is always implicit firstprivate and // privatized already during procoessing of the firstprivates. a2808 14 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), TaskgroupDescriptors[Cnt]->getExprLoc()); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), [Replacement]() { return Replacement; }); d2811 1 a2811 1 (void)InRedScope.Privatize(); d2816 1 a2816 1 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( a2822 144 static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc) { auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *OrigRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *PrivateRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); QualType ElemType = C.getBaseElementType(Ty); auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType, ImplicitParamDecl::Other); auto *InitRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), InitVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue); PrivateVD->setInitStyle(VarDecl::CInit); PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue, InitRef, /*BasePath=*/nullptr, VK_RValue)); Data.FirstprivateVars.emplace_back(OrigRef); Data.FirstprivateCopies.emplace_back(PrivateRef); Data.FirstprivateInits.emplace_back(InitRef); return OrigVD; } void CodeGenFunction::EmitOMPTargetTaskBasedDirective( const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo) { // Emit outlined function for task construct. const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); OMPTaskDataTy Data; // The task is not final. Data.Final.setInt(/*IntVal=*/false); // Get list of firstprivate variables. for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto IElemInitRef = C->inits().begin(); for (auto *IInit : C->private_copies()) { Data.FirstprivateVars.push_back(*IRef); Data.FirstprivateCopies.push_back(IInit); Data.FirstprivateInits.push_back(*IElemInitRef); ++IRef; ++IElemInitRef; } } OMPPrivateScope TargetScope(*this); VarDecl *BPVD = nullptr; VarDecl *PVD = nullptr; VarDecl *SVD = nullptr; if (InputInfo.NumberOfTargetItems > 0) { auto *CD = CapturedDecl::Create( getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0); llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems); QualType BaseAndPointersType = getContext().getConstantArrayType( getContext().VoidPtrTy, ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); BPVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); PVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); QualType SizesType = getContext().getConstantArrayType( getContext().getSizeType(), ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD, S.getLocStart()); TargetScope.addPrivate( BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; }); TargetScope.addPrivate(PVD, [&InputInfo]() { return InputInfo.PointersArray; }); TargetScope.addPrivate(SVD, [&InputInfo]() { return InputInfo.SizesArray; }); } (void)TargetScope.Privatize(); // Build list of dependences. for (const auto *C : S.getClausesOfKind()) for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { // Set proper addresses for generated private copies. OMPPrivateScope Scope(CGF); if (!Data.FirstprivateVars.empty()) { enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); // Map privates. llvm::SmallVector, 16> PrivatePtrs; llvm::SmallVector CallArgs; CallArgs.push_back(PrivatesPtr); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); } CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : PrivatePtrs) { Address Replacement(CGF.Builder.CreateLoad(Pair.second), CGF.getContext().getDeclAlign(Pair.first)); Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); if (InputInfo.NumberOfTargetItems > 0) { InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize()); } Action.Enter(CGF); OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); BodyGen(CGF); }; llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true, Data.NumberOfParts); llvm::APInt TrueOrFalse(32, S.hasClausesOfKind() ? 1 : 0); IntegerLiteral IfCond(getContext(), TrueOrFalse, getContext().getIntTypeForBitwidth(32, /*Signed=*/0), SourceLocation()); CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, &IfCond, Data); } d2825 3 a2827 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d2850 1 a2850 1 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data); d2870 1 a2870 30 if (const Expr *E = S.getReductionRef()) { SmallVector LHSs; SmallVector RHSs; OMPTaskDataTy Data; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); for (const Expr *Ref : C->varlists()) { Data.ReductionVars.emplace_back(Ref); Data.ReductionCopies.emplace_back(*IPriv); Data.ReductionOps.emplace_back(*IRed); LHSs.emplace_back(*ILHS); RHSs.emplace_back(*IRHS); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ILHS, 1); std::advance(IRHS, 1); } } llvm::Value *ReductionDesc = CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(), LHSs, RHSs, Data); const auto *VD = cast(cast(E)->getDecl()); CGF.EmitVarDecl(*VD); CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), /*Volatile=*/false, E->getType()); } CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2872 1 a2872 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2877 7 a2883 9 CGM.getOpenMPRuntime().emitFlush( *this, [&S]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); return llvm::None; }(), S.getLocStart()); d2890 2 a2891 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2897 1 a2897 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2903 1 a2903 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2918 1 a2918 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2925 1 emitAlignedClause(*this, S); d2947 2 a2948 2 // Emit implicit barrier to synchronize threads and avoid data races // on initialization of firstprivate variables and post-update of d2951 2 a2952 2 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); a2954 4 if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) EmitOMPReductionClauseInit(S, LoopScope); d2962 1 a2962 1 if (const auto *C = S.getSingleClause()) { d2964 1 a2964 1 if (const Expr *Ch = C->getChunkSize()) { d2967 2 a2968 2 S.getIterationVariable()->getType(), S.getLocStart()); a2983 5 if (isOpenMPSimdDirective(S.getDirectiveKind())) EmitOMPSimdInit(S, /*IsMonotonic=*/true); CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2985 4 a2988 2 StaticInit); JumpDest LoopExit = d2999 3 a3001 4 const Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedCond() : S.getCond(); d3014 1 a3014 1 RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind()); d3024 1 a3024 30 if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) { OpenMPDirectiveKind ReductionKind = OMPD_unknown; if (isOpenMPParallelDirective(S.getDirectiveKind()) && isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for_simd; } else if (isOpenMPParallelDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for; } else if (isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_simd; } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) && S.hasClausesOfKind()) { llvm_unreachable( "No reduction clauses is allowed in distribute directive."); } EmitOMPReductionClauseFinal(S, ReductionKind); // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( *this, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } d3026 1 a3026 1 if (HasLastprivateClause) { d3029 2 a3030 2 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } d3044 1 d3047 3 a3049 2 OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d3057 2 a3058 2 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); Fn->setDoesNotRecurse(); d3063 1 a3063 3 if (S.hasClausesOfKind()) { assert(!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct."); d3068 1 a3068 1 const auto *C = S.getSingleClause(); a3070 1 const CapturedStmt *CS = S.getInnermostCapturedStmt(); d3072 1 d3075 2 a3076 3 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), OutlinedFn, CapturedVars); d3079 2 a3080 1 CGF.EmitStmt(CS->getCapturedStmt()); d3083 1 a3083 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3093 5 a3097 4 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType, Loc) : CGF.EmitComplexToScalarConversion( Val.getComplexVal(), SrcType, DestType, Loc); d3108 3 a3110 4 QualType DestElementType = DestType->castAs()->getElementType(); llvm::Value *ScalarVal = CGF.EmitScalarConversion( Val.getScalarVal(), SrcType, DestElementType, Loc); d3115 2 a3116 3 QualType SrcElementType = SrcType->castAs()->getElementType(); QualType DestElementType = DestType->castAs()->getElementType(); d3155 1 a3155 1 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, d3179 1 a3179 1 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, d3198 1 a3198 1 ASTContext &Context = CGF.getContext(); a3261 1 case BO_Cmp: d3275 1 a3275 1 llvm::Value *UpdateVal = Update.getScalarVal(); d3281 1 a3281 2 llvm::Value *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); d3288 1 a3288 1 const llvm::function_ref CommonGen) { d3309 1 a3309 1 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, d3315 1 a3315 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3325 12 a3336 12 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); return CGF.EmitAnyExpr(UE); }; d3364 1 a3364 1 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, d3375 2 a3376 3 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; d3382 1 a3382 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3389 3 a3391 3 const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; d3393 1 a3393 1 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; d3395 1 a3395 1 IsPostfixUpdate](RValue XRValue) { d3422 1 a3422 1 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) { d3445 1 a3445 1 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, d3452 1 a3452 1 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); d3455 1 a3455 1 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); d3459 1 a3459 1 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); d3462 1 a3462 1 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, a3472 1 case OMPC_in_reduction: d3517 1 a3517 1 for (const OMPClause *C : S.clauses()) { d3525 3 a3527 2 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers(); if (const auto *EWC = dyn_cast(CS)) d3529 1 d3532 2 a3533 2 for (const Stmt *C : Compound->body()) { if (const auto *EWC = dyn_cast(C)) d3535 1 d3542 1 a3542 1 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), d3546 1 a3546 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3555 1 a3555 10 // On device emit this construct as inlined code. if (CGM.getLangOpts().OpenMPIsDevice) { OMPLexicalScope Scope(CGF, S, OMPD_target); CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); }); return; } d3572 1 a3572 1 if (auto *C = S.getSingleClause()) d3574 1 d3592 1 a3592 1 if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3594 1 a3594 1 else if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3603 5 a3607 2 OMPLexicalScope Scope(CGF, S, OMPD_task); CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device); a3611 1 Action.Enter(CGF); d3617 2 a3618 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); d3647 2 a3648 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d3650 2 a3651 2 const auto *NT = S.getSingleClause(); const auto *TL = S.getSingleClause(); d3653 2 a3654 2 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr; const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr; d3669 1 a3669 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3675 1 a3675 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); d3678 3 a3680 3 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d3687 2 a3688 8 // Emit teams region as a standalone region. auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); a3689 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); a3691 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3715 286 static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeSimdRegion( CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPTeamsDistributeDirective( const OMPTeamsDistributeDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( const OMPTeamsDistributeSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( const OMPTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } static void emitTargetTeamsDistributeParallelForRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S) { // Emit SPMD target teams distribute parallel for region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeParallelForSimdRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S) { // Emit SPMD target teams distribute parallel for simd region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( const OMPTargetTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } d3743 1 a3743 3 Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for); d3753 4 a3756 4 for (const Expr *PvtVarIt : C.private_copies()) { const auto *OrigVD = cast(cast(*OrigVarIt)->getDecl()); const auto *InitVD = cast(cast(*InitIt)->getDecl()); const auto *PvtVD = cast(cast(PvtVarIt)->getDecl()); d3762 1 a3762 1 if (const auto *OED = dyn_cast(MatchingVD)) { d3765 1 a3765 1 const auto *ME = cast(OED->getInit()); d3777 1 a3777 3 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD, InitAddrIt, InitVD, PvtVD]() { d3794 1 a3794 1 // of the previous declaration, so we don't need it anymore. d3831 1 a3831 1 CodeGenFunction &CGF, PrePostActionTy &Action) { d3833 2 a3834 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d3837 1 a3837 1 // Codegen that selects whether to generate the privatization code or not. d3856 1 a3856 1 } else { a3857 1 } d3883 1 a3883 1 if (const auto *C = S.getSingleClause()) d3888 1 a3888 1 if (const auto *C = S.getSingleClause()) d3908 1 a3908 1 if (const auto *C = S.getSingleClause()) d3913 1 a3913 1 if (const auto *C = S.getSingleClause()) a3915 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3928 1 a3928 1 if (const auto *C = S.getSingleClause()) d3933 1 a3933 1 if (const auto *C = S.getSingleClause()) a3935 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3943 1 a3943 1 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); d3945 1 a3945 2 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3957 2 a3958 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3982 32 static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPCancelStackRAII CancelRegion( CGF, OMPD_target_parallel_for, S.hasCancel()); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } d3985 1 a3985 43 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d3992 3 a3994 3 const auto *VDecl = cast(Helper->getDecl()); Privates.addPrivate(VDecl, [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); d4000 3 a4002 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d4045 1 a4045 1 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); d4076 1 a4076 1 const auto *IVDecl = cast(cast(IVExpr)->getDecl()); d4083 1 a4083 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d4113 1 a4113 2 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, &Data](CodeGenFunction &CGF, PrePostActionTy &) { d4122 3 a4124 3 if (Data.Nogroup) { EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); } else { d4130 1 a4130 2 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); d4155 1 a4155 1 if (const auto *C = S.getSingleClause()) d4160 1 a4160 1 if (const auto *C = S.getSingleClause()) a4162 1 OMPLexicalScope Scope(*this, S, OMPD_task); a4164 30 void CodeGenFunction::EmitSimpleOMPExecutableDirective( const OMPExecutableDirective &D) { if (!D.hasAssociatedStmt() || !D.getAssociatedStmt()) return; auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { if (isOpenMPSimdDirective(D.getDirectiveKind())) { emitOMPSimdRegion(CGF, cast(D), Action); } else { if (const auto *LD = dyn_cast(&D)) { for (const Expr *E : LD->counters()) { if (const auto *VD = dyn_cast( cast(E)->getDecl())) { // Emit only those that were not explicitly referenced in clauses. if (!CGF.LocalDeclMap.count(VD)) CGF.EmitVarDecl(*VD); } } } CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); } }; OMPSimdLexicalScope Scope(*this, D); CGM.getOpenMPRuntime().emitInlinedDirective( *this, isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd : D.getDirectiveKind(), CodeGen); } @ 1.1.1.9 log @Import clang r337282 from trunk @ text @d32 2 a33 3 if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { d35 1 a35 1 if (!I->hasAttr()) { d37 1 a37 1 } else { d56 2 a57 4 OMPLexicalScope( CodeGenFunction &CGF, const OMPExecutableDirective &S, const llvm::Optional CapturedRegion = llvm::None, const bool EmitPreInitStmt = true) d62 18 a79 18 if (!CapturedRegion.hasValue()) return; assert(S.hasAssociatedStmt() && "Expected associated statement for inlined directive."); const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion); for (const auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE( const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); a81 1 (void)InlinedShareds.Privatize(); d97 3 a99 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d113 3 a115 2 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, EmitPreInitStmt(S)) {} d122 5 a126 10 CodeGenFunction::OMPMapVars PreCondVars; for (const auto *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); (void)PreCondVars.setVarAddr( CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); } (void)PreCondVars.apply(CGF); if (const auto *PreInits = cast_or_null(S.getPreInits())) { for (const auto *I : PreInits->decls()) CGF.EmitVarDecl(cast(*I)); a127 1 PreCondVars.restore(CGF); a136 66 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { CodeGenFunction::OMPPrivateScope InlinedShareds; static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { return CGF.LambdaCaptureFields.lookup(VD) || (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || (CGF.CurCodeDecl && isa(CGF.CurCodeDecl) && cast(CGF.CurCodeDecl)->capturesVariable(VD)); } public: OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), InlinedShareds(CGF) { for (const auto *C : S.clauses()) { if (const auto *CPI = OMPClauseWithPreInit::get(C)) { if (const auto *PreInit = cast_or_null(CPI->getPreInitStmt())) { for (const auto *I : PreInit->decls()) { if (!I->hasAttr()) { CGF.EmitVarDecl(cast(*I)); } else { CodeGenFunction::AutoVarEmission Emission = CGF.EmitAutoVarAlloca(cast(*I)); CGF.EmitAutoVarCleanups(Emission); } } } } else if (const auto *UDP = dyn_cast(C)) { for (const Expr *E : UDP->varlists()) { const Decl *D = cast(E)->getDecl(); if (const auto *OED = dyn_cast(D)) CGF.EmitVarDecl(*OED); } } } if (!isOpenMPSimdDirective(S.getDirectiveKind())) CGF.EmitOMPPrivateClause(S, InlinedShareds); if (const auto *TG = dyn_cast(&S)) { if (const Expr *E = TG->getReductionRef()) CGF.EmitVarDecl(*cast(cast(E)->getDecl())); } const auto *CS = cast_or_null(S.getAssociatedStmt()); while (CS) { for (auto &C : CS->captures()) { if (C.capturesVariable() || C.capturesVariableByCopy()) { auto *VD = C.getCapturedVar(); assert(VD == VD->getCanonicalDecl() && "Canonical decl must be captured."); DeclRefExpr DRE(const_cast(VD), isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && InlinedShareds.isGlobalVarCaptured(VD)), VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { return CGF.EmitLValue(&DRE).getAddress(); }); } } CS = dyn_cast(CS->getCapturedStmt()); } (void)InlinedShareds.Privatize(); } }; a138 20 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen); LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { if (const auto *OrigDRE = dyn_cast(E)) { if (const auto *OrigVD = dyn_cast(OrigDRE->getDecl())) { OrigVD = OrigVD->getCanonicalDecl(); bool IsCaptured = LambdaCaptureFields.lookup(OrigVD) || (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || (CurCodeDecl && isa(CurCodeDecl)); DeclRefExpr DRE(const_cast(OrigVD), IsCaptured, OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); return EmitLValue(&DRE); } } return EmitLValue(E); } d140 1 a140 1 ASTContext &C = getContext(); d145 4 a148 5 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) { VlaSizePair VlaSize = getVLASize(VAT); Ty = VlaSize.Type; Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts; d153 4 a156 3 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); } return CGM.getSize(SizeInChars); d168 2 a169 2 const VariableArrayType *VAT = CurField->getCapturedVLAType(); llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()]; d171 1 a171 1 } else if (CurCap->capturesThis()) { d173 3 a175 2 } else if (CurCap->capturesVariableByCopy()) { llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation()); d180 2 a181 2 ASTContext &Ctx = getContext(); Address DstAddr = CreateMemTemp( d183 1 a183 1 Twine(CurCap->getCapturedVar()->getName(), ".casted")); d186 1 a186 1 llvm::Value *SrcAddrVal = EmitScalarConversion( d188 1 a188 1 Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); d196 1 a196 1 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation()); d206 2 a207 3 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, d211 1 a211 1 llvm::Value *CastedPtr = CGF.EmitScalarConversion( d213 2 a214 2 Ctx.getPointerType(DstType), Loc); Address TmpAddr = d222 4 a225 4 llvm::Value *RefVal = TmpAddr.getPointer(); TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref")); LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true); d232 1 a232 1 if (T->isLValueReferenceType()) d236 1 a238 6 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) { if (const auto *VLA = dyn_cast(A)) return getCanonicalParamType(C, VLA->getElementType()); if (!A->isVariablyModifiedType()) return C.getCanonicalType(T); } d249 2 a250 2 const bool UIntPtrCastRequired = true; /// true if only casted arguments must be registered as local args or VLA d252 1 a252 1 const bool RegisterCastedArgsOnly = false; d254 1 a254 1 const StringRef FunctionName; d264 1 a264 1 static llvm::Function *emitOutlinedFunctionPrologue( d266 1 a266 1 llvm::MapVector> d279 1 a279 1 FunctionArgList TargetArgs; a281 3 TargetArgs.append( CD->param_begin(), std::next(CD->param_begin(), CD->getContextParamPosition())); d283 1 a283 11 FunctionDecl *DebugFunctionDecl = nullptr; if (!FO.UIntPtrCastRequired) { FunctionProtoType::ExtProtoInfo EPI; DebugFunctionDecl = FunctionDecl::Create( Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(), SourceLocation(), DeclarationName(), Ctx.VoidTy, Ctx.getTrivialTypeSourceInfo( Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)), SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false); } for (const FieldDecl *FD : RD->fields()) { d293 6 a298 4 if (FO.UIntPtrCastRequired && ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || I->capturesVariableArrayType())) ArgType = Ctx.getUIntPtrType(); d303 1 a303 1 } else if (I->capturesThis()) { d305 1 a305 1 } else { d310 4 a313 18 ArgType = getCanonicalParamType(Ctx, ArgType); VarDecl *Arg; if (DebugFunctionDecl && (CapVar || I->capturesThis())) { Arg = ParmVarDecl::Create( Ctx, DebugFunctionDecl, CapVar ? CapVar->getLocStart() : FD->getLocStart(), CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType, /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr); } else { Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), II, ArgType, ImplicitParamDecl::Other); } Args.emplace_back(Arg); // Do not cast arguments if we emit function with non-original types. TargetArgs.emplace_back( FO.UIntPtrCastRequired ? Arg : CGM.getOpenMPRuntime().translateParameter(FD, Arg)); a318 3 TargetArgs.append( std::next(CD->param_begin(), CD->getContextParamPosition() + 1), CD->param_end()); d321 1 d323 1 a323 1 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs); d326 1 a326 1 auto *F = d331 1 a331 2 F->setDoesNotThrow(); F->setDoesNotRecurse(); d334 2 a335 2 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, FO.S->getLocStart(), CD->getBody()->getLocStart()); d338 1 a338 9 for (const FieldDecl *FD : RD->fields()) { // Do not map arguments if we emit function with non-original types. Address LocalAddr(Address::invalid()); if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) { LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], TargetArgs[Cnt]); } else { LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); } d343 1 d359 3 a361 2 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), AlignmentSource::Decl); d364 9 a372 8 ArgLVal = CGF.MakeAddrLValue( castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal), FD->getType(), AlignmentSource::Decl); } llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); const VariableArrayType *VAT = FD->getCapturedVLAType(); VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg); d374 1 a374 1 const VarDecl *Var = I->getCapturedVar(); d379 3 a381 3 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { d395 1 a395 1 const VarDecl *Var = I->getCapturedVar(); d399 5 a403 5 {Var, FO.UIntPtrCastRequired ? castValueFromUintptr(CGF, I->getLocation(), FD->getType(), Args[Cnt]->getName(), ArgLVal, VarTy->isReferenceType()) : ArgLVal.getAddress()}}); d407 2 a408 1 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); d415 1 a415 1 return F; d429 1 a429 1 llvm::MapVector> LocalAddrs; a430 5 SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); Out << CapturedStmtInfo->getHelperName(); if (NeedWrapperFunction) Out << "_debug__"; d432 5 a436 3 Out.str()); llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, VLASizes, CXXThisValue, FO); d448 1 a448 1 if (!NeedWrapperFunction) d453 1 a453 1 CapturedStmtInfo->getHelperName()); d455 1 a455 1 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo; d461 2 a462 1 WrapperCGF.CXXThisValue, WrapperFO); d468 3 a470 5 LValue LV = WrapperCGF.MakeAddrLValue( I->second.second, I->second.first ? I->second.first->getType() : Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d473 1 a473 1 if (EI != VLASizes.end()) { d475 1 a475 1 } else { d477 2 a478 3 Arg->getType(), AlignmentSource::Decl); CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getLocStart()); d481 1 a481 1 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType())); d483 1 a483 2 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(), F, CallArgs); d493 1 a493 1 const llvm::function_ref CopyGen) { d498 2 a499 2 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe(); llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); d502 2 a503 2 llvm::Value *SrcBegin = SrcAddr.getPointer(); llvm::Value *DestBegin = DestAddr.getPointer(); d505 1 a505 1 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements); d507 3 a509 3 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body"); llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done"); llvm::Value *IsEmpty = d514 1 a514 1 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); d537 1 a537 1 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32( d539 1 a539 1 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32( d542 1 a542 1 llvm::Value *Done = d556 1 a556 1 const auto *BO = dyn_cast(Copy); d559 1 a559 3 LValue Dest = MakeAddrLValue(DestAddr, OriginalType); LValue Src = MakeAddrLValue(SrcAddr, OriginalType); EmitAggregateAssign(Dest, Src, OriginalType); d570 5 a574 2 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; }); Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; }); d582 2 a583 2 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; }); Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; }); d602 1 a602 6 llvm::SmallVector CaptureRegions; getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); // Force emission of the firstprivate copy if the directive does not emit // outlined function, like omp for, omp simd, omp distribute etc. bool MustEmitFirstprivateCopy = CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown; d606 2 a607 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d610 3 a612 2 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD); if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD && d622 2 a623 3 const auto *VD = cast(cast(IInit)->getDecl()); const auto *VDInit = cast(cast(*InitsRef)->getDecl()); d628 1 a628 1 LValue OriginalLVal = EmitLValue(&DRE); d634 25 a658 30 IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VD, Type, OriginalLVal, VDInit]() { AutoVarEmission Emission = EmitAutoVarAlloca(*VD); const Expr *Init = VD->getInit(); if (!isa(Init) || isTrivialInitializer(Init)) { // Perform simple memcpy. LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type); EmitAggregateAssign(Dest, OriginalLVal, Type); } else { EmitOMPAggregateAssign( Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type, [this, VDInit, Init](Address DestElement, Address SrcElement) { // Clean up any temporaries needed by the // initialization. RunCleanupsScope InitScope(*this); // Emit initialization for single element. setAddrOfLocalVar(VDInit, SrcElement); EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(), /*IsInitializer*/ false); LocalDeclMap.erase(VDInit); }); } EmitAutoVarCleanups(Emission); return Emission.getAllocatedAddress(); }); d660 10 a669 11 Address OriginalAddr = OriginalLVal.getAddress(); IsRegistered = PrivateScope.addPrivate( OrigVD, [this, VDInit, OriginalAddr, VD]() { // Emit private VarDecl with copy init. // Remap temp VDInit variable to the address of the original // variable (for proper handling of captured global variables). setAddrOfLocalVar(VDInit, OriginalAddr); EmitDecl(*VD); LocalDeclMap.erase(VDInit); return GetAddrOfLocalVar(VD); }); d691 2 a692 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d694 7 a700 6 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { // Emit private VarDecl with copy init. EmitDecl(*VD); return GetAddrOfLocalVar(VD); }); d723 2 a724 2 for (const Expr *AssignOp : C->assignment_ops()) { const auto *VD = cast(cast(*IRef)->getDecl()); d755 1 a755 2 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)), d759 2 a760 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d783 2 a784 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d792 1 a792 2 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && !getLangOpts().OpenMPSimd) d796 1 a796 1 for (const Expr *IInit : C->private_copies()) { d799 1 a799 1 const auto *OrigVD = cast(cast(*IRef)->getDecl()); d803 2 a804 3 const auto *DestVD = cast(cast(*IDestRef)->getDecl()); PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() { d816 2 a817 2 const auto *VD = cast(cast(IInit)->getDecl()); bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { d855 1 a855 1 if (const auto *LoopDirective = dyn_cast(&D)) { d857 2 a858 2 for (const Expr *F : LoopDirective->finals()) { const auto *D = d871 2 a872 3 for (const Expr *AssignOp : C->assignment_ops()) { const auto *PrivateVD = cast(cast(*IRef)->getDecl()); d874 1 a874 1 const auto *CanonicalVD = PrivateVD->getCanonicalDecl(); d879 1 a879 1 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) d881 2 a882 4 const auto *SrcVD = cast(cast(*ISrcRef)->getDecl()); const auto *DestVD = cast(cast(*IDestRef)->getDecl()); d887 1 a887 1 if (const auto *RefTy = PrivateVD->getType()->getAs()) d897 1 a897 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d919 1 a919 1 for (const Expr *Ref : C->varlists()) { d936 2 a937 2 for (const Expr *IRef : Shareds) { const auto *PrivateVD = cast(cast(*IPriv)->getDecl()); d941 1 a941 1 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); d952 1 a952 1 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; }); d957 3 a959 5 const auto *LHSVD = cast(cast(*ILHS)->getDecl()); const auto *RHSVD = cast(cast(*IRHS)->getDecl()); QualType Type = PrivateVD->getType(); bool isaOMPArraySectionExpr = isa(IRef); if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { d962 1 a962 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d965 4 a968 4 PrivateScope.addPrivate( RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); }); } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || isa(IRef)) { d971 1 a971 1 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() { d974 1 a974 1 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() { a988 1 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; }); d990 3 a992 1 RHSVD, [this, PrivateVD, RHSVD, IsArray]() { d1026 2 a1027 2 ReductionKind == OMPD_simd; bool SimpleReduction = ReductionKind == OMPD_simd; d1038 1 a1038 1 const llvm::function_ref CondGen) { d1043 1 a1043 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) { d1045 1 a1045 1 if (llvm::Value *Cond = CondGen(CGF)) { d1048 1 a1048 1 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); d1076 2 a1077 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d1080 2 a1081 3 llvm::Value *NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true); d1117 1 a1117 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d1132 1 a1132 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); d1137 2 a1138 2 emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d1145 3 a1147 2 for (const Expr *UE : D.updates()) EmitIgnoredExpr(UE); d1149 3 a1151 7 // In distribute directives only loop counters may be marked as linear, no // need to generate the code for them. if (!isOpenMPDistributeDirective(D.getDirectiveKind())) { for (const auto *C : D.getClausesOfKind()) { for (const Expr *UE : C->updates()) EmitIgnoredExpr(UE); } d1155 1 a1155 1 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue"); d1167 2 a1168 2 const llvm::function_ref BodyGen, const llvm::function_ref PostIncGen) { d1174 1 a1174 1 const SourceRange R = S.getSourceRange(); d1180 1 a1180 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1184 1 a1184 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body"); d1197 1 a1197 1 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); d1219 1 a1219 1 for (const Expr *Init : C->inits()) { d1221 2 a1222 3 const auto *VD = cast(cast(Init)->getDecl()); if (const auto *Ref = dyn_cast(VD->getInit()->IgnoreImpCasts())) { d1224 1 a1224 1 const auto *OrigVD = cast(Ref->getDecl()); d1233 1 a1233 1 } else { a1234 1 } d1238 2 a1239 2 if (const auto *CS = cast_or_null(C->getCalcStep())) if (const auto *SaveRef = cast(CS->getLHS())) { d1250 1 a1250 1 const llvm::function_ref CondGen) { d1257 1 a1257 1 for (const Expr *F : C->finals()) { d1259 1 a1259 1 if (llvm::Value *Cond = CondGen(*this)) { d1262 1 a1262 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu"); d1268 1 a1268 1 const auto *OrigVD = cast(cast(*IC)->getDecl()); d1274 1 a1274 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1279 1 a1279 1 if (const Expr *PostUpdate = C->getPostUpdateExpr()) d1292 2 a1293 2 if (const Expr *AlignmentExpr = Clause->getAlignment()) { auto *AlignmentCI = d1297 1 a1297 1 for (const Expr *E : Clause->varlists()) { d1324 13 a1336 9 for (const Expr *E : S.counters()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = cast(cast(*I)->getDecl()); // Emit var without initialization. AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD); EmitAutoVarCleanups(VarEmission); LocalDeclMap.erase(PrivateVD); (void)LoopScope.addPrivate(VD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); d1340 1 a1340 1 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() { a1345 4 } else { (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); }); d1361 1 a1361 1 for (const Expr *I : S.inits()) { d1375 2 a1376 2 const auto *LoopDirective = cast(&D); for (const Expr *C : LoopDirective->counters()) { d1383 3 a1385 3 for (const Expr *E : C->varlists()) { const auto *VD = cast(cast(E)->getDecl()); const auto *PrivateVD = d1388 1 a1388 1 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() { d1396 1 a1396 1 } else { a1397 1 } d1411 1 a1411 1 auto *Val = cast(Len.getScalarVal()); d1421 1 a1421 1 auto *Val = cast(Len.getScalarVal()); d1426 1 a1426 1 CGF.LoopStack.setParallel(/*Enable=*/false); d1434 1 a1434 1 LoopStack.setVectorizeEnable(); d1440 1 a1440 1 const llvm::function_ref CondGen) { d1446 4 a1449 4 for (const Expr *F : D.finals()) { const auto *OrigVD = cast(cast((*IC))->getDecl()); const auto *PrivateVD = cast(cast((*IPC))->getDecl()); const auto *CED = dyn_cast(OrigVD); d1453 1 a1453 1 if (llvm::Value *Cond = CondGen(*this)) { d1456 1 a1456 1 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then"); d1463 1 a1463 1 if (CED) { d1465 1 a1465 1 } else { d1472 2 a1473 1 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); d1491 8 a1498 7 /// Emit a helper variable and return corresponding lvalue. static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper) { auto VDecl = cast(Helper->getDecl()); CGF.EmitVarDecl(*VDecl); return CGF.EmitLValue(Helper); } d1500 16 a1515 17 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); assert(isOpenMPSimdDirective(S.getDirectiveKind()) && "Expected simd directive"); OMPLoopScope PreInitScope(CGF, S); // if (PreCond) { // for (IV in 0..LastIteration) BODY; // ; // } // if (isOpenMPDistributeDirective(S.getDirectiveKind()) || isOpenMPWorksharingDirective(S.getDirectiveKind()) || isOpenMPTaskLoopDirective(S.getDirectiveKind())) { (void)EmitOMPHelperVar(CGF, cast(S.getLowerBoundVariable())); (void)EmitOMPHelperVar(CGF, cast(S.getUpperBoundVariable())); } d1517 5 a1521 16 // Emit: if (PreCond) - begin. // If the condition constant folds and can be elided, avoid emitting the // whole loop. bool CondConstant; llvm::BasicBlock *ContBlock = nullptr; if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { if (!CondConstant) return; } else { llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); ContBlock = CGF.createBasicBlock("simd.if.end"); emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, CGF.getProfileCount(&S)); CGF.EmitBlock(ThenBlock); CGF.incrementProfileCounter(&S); } d1523 8 a1530 5 // Emit the loop iteration variable. const Expr *IVExpr = S.getIterationVariable(); const auto *IVDecl = cast(cast(IVExpr)->getDecl()); CGF.EmitVarDecl(*IVDecl); CGF.EmitIgnoredExpr(S.getInit()); d1532 1 a1532 8 // Emit the iterations count variable. // If it is not a variable, Sema decided to calculate iterations count on // each iteration (e.g., it is foldable into a constant). if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { CGF.EmitVarDecl(*cast(LIExpr->getDecl())); // Emit calculation of the iterations count. CGF.EmitIgnoredExpr(S.getCalcLastIteration()); } d1534 34 a1567 38 CGF.EmitOMPSimdInit(S); emitAlignedClause(CGF, S); (void)CGF.EmitOMPLinearClauseInit(S); { CodeGenFunction::OMPPrivateScope LoopScope(CGF); CGF.EmitOMPPrivateLoopCounters(S, LoopScope); CGF.EmitOMPLinearClause(S, LoopScope); CGF.EmitOMPPrivateClause(S, LoopScope); CGF.EmitOMPReductionClauseInit(S, LoopScope); bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); (void)LoopScope.Privatize(); CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), [&S](CodeGenFunction &CGF) { CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); CGF.EmitStopPoint(&S); }, [](CodeGenFunction &) {}); CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit final copy of the lastprivate variables at the end of loops. if (HasLastprivateClause) CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); // Emit: if (PreCond) - end. if (ContBlock) { CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); d1569 1 a1569 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d1579 1 a1579 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1585 1 a1585 1 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); d1588 1 a1588 1 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond"); d1590 1 a1590 1 const SourceRange R = S.getSourceRange(); d1612 1 a1612 1 llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); d1616 1 a1616 1 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body"); d1630 1 a1630 1 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); d1672 1 a1672 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d1682 1 a1682 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1748 1 a1748 2 const std::pair DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); d1756 3 a1758 5 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d1789 1 a1789 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d1800 4 a1803 4 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit); d1844 8 d1869 1 a1869 2 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc()); d1872 2 a1873 4 LS.getIterationVariable()->getType(), LS.getPrevLowerBoundVariable()->getExprLoc()); llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc()); d1876 1 a1876 2 LS.getIterationVariable()->getType(), LS.getPrevUpperBoundVariable()->getExprLoc()); d1902 4 a1905 4 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getLocStart()); llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getLocStart()); d1915 1 a1915 1 llvm::Value *LBCast = CGF.Builder.CreateIntCast( d1921 1 a1921 1 llvm::Value *UBCast = CGF.Builder.CreateIntCast( d1931 1 a1931 14 PrePostActionTy &Action) { Action.Enter(CGF); bool HasCancel = false; if (!isOpenMPSimdDirective(S.getDirectiveKind())) { if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); else if (const auto *D = dyn_cast(&S)) HasCancel = D->hasCancel(); } CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), HasCancel); d1938 1 a1938 3 CGF, S, isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for, CGInlinedWorksharingLoop, d1948 5 a1952 2 OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1957 8 a1964 6 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; OMPLexicalScope Scope(*this, S, OMPD_parallel); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d1969 8 a1976 5 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); d1979 10 a1988 12 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); d1993 95 a2087 4 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitOMPSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d2107 2 a2108 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2114 1 a2114 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2120 1 a2120 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2135 1 a2135 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2142 1 RunCleanupsScope DoacrossCleanupScope(*this); d2144 1 a2144 1 if (const auto *OrderedClause = S.getSingleClause()) { d2185 1 a2185 1 if (const auto *C = S.getSingleClause()) { d2189 1 a2189 1 if (const Expr *Ch = C->getChunkSize()) { d2212 5 a2216 6 CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); JumpDest LoopExit = d2233 1 a2233 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2252 1 a2252 1 [IL, &S](CodeGenFunction &CGF) { d2263 1 a2263 1 *this, S, [IL, &S](CodeGenFunction &CGF) { d2273 1 a2273 1 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { a2276 1 DoacrossCleanupScope.ForceCleanup(); d2280 1 a2280 1 EmitBlock(ContBlock, /*IsFinished=*/true); d2291 1 a2291 1 const auto &LS = cast(S); d2306 1 a2306 1 const auto &LS = cast(S); d2324 1 a2324 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2330 1 a2330 1 if (!S.getSingleClause() || HasLastprivates) d2332 1 d2344 1 a2344 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2349 1 a2349 1 if (!S.getSingleClause() || HasLastprivates) d2351 1 d2357 1 a2357 1 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); d2364 2 a2365 2 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt(); const auto *CS = dyn_cast(CapturedStmt); d2367 4 a2370 5 auto &&CodeGen = [&S, CapturedStmt, CS, &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { ASTContext &C = CGF.getContext(); QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); d2374 2 a2375 3 llvm::ConstantInt *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1) : CGF.Builder.getInt32(0); d2393 2 a2394 2 S.getLocStart(), true); auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { d2406 4 a2409 4 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); llvm::SwitchInst *SwitchStmt = CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getLocStart()), ExitBB, CS == nullptr ? 1 : CS->size()); d2412 1 a2412 1 for (const Stmt *SubStmt : CS->children()) { d2421 1 a2421 1 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); d2424 1 a2424 1 CGF.EmitStmt(CapturedStmt); a2446 3 CGOpenMPRuntime::StaticRTInput StaticInit( /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2448 3 a2450 1 CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit); d2452 2 a2453 2 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( d2463 1 a2463 2 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(), S.getDirectiveKind()); d2468 5 a2472 4 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); d2503 1 a2503 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2515 1 a2515 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2517 1 a2517 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2546 1 a2546 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2549 1 a2549 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2566 1 a2566 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2568 1 a2568 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2575 1 a2575 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2577 2 a2578 2 const Expr *Hint = nullptr; if (const auto *HintClause = S.getSingleClause()) d2580 1 a2580 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2590 1 a2590 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2603 1 a2603 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2615 1 a2615 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d2622 4 a2625 4 void CodeGenFunction::EmitOMPTaskBasedDirective( const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data) { d2627 4 a2630 4 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); d2635 1 a2635 1 const Expr *Cond = Clause->getCondition(); d2647 1 a2647 1 const Expr *Prio = Clause->getPriority(); d2660 2 a2661 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2674 2 a2675 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2690 2 a2691 2 for (const Expr *IInit : C->private_copies()) { const auto *OrigVD = cast(cast(*IRef)->getDecl()); d2710 1 a2710 1 for (const Expr *Ref : C->varlists()) { d2726 4 a2729 5 for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs, CapturedRegion](CodeGenFunction &CGF, PrePostActionTy &Action) { d2734 4 a2737 5 enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); d2742 2 a2743 2 for (const Expr *E : Data.PrivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2746 1 a2746 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2749 2 a2750 2 for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2754 1 a2754 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2757 2 a2758 2 for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast(cast(E)->getDecl()); d2762 1 a2762 1 PrivatePtrs.emplace_back(VD, PrivatePtr); d2765 3 a2767 4 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : LastprivateDstsOrigs) { const auto *OrigVD = cast(Pair.second->getDecl()); d2777 1 a2777 1 for (const auto &Pair : PrivatePtrs) { d2784 1 a2784 1 OMPLexicalScope LexScope(CGF, S, CapturedRegion); a2791 5 // FIXME: This must removed once the runtime library is fixed. // Emit required threadprivate variables for // initilizer/combiner/finalizer. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(), RedCG, Cnt); d2799 1 a2799 1 Data.ReductionCopies[Cnt]->getExprLoc()), a2803 32 } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); SmallVector InRedVars; SmallVector InRedPrivs; SmallVector InRedOps; SmallVector TaskgroupDescriptors; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ITD = C->taskgroup_descriptors().begin(); for (const Expr *Ref : C->varlists()) { InRedVars.emplace_back(Ref); InRedPrivs.emplace_back(*IPriv); InRedOps.emplace_back(*IRed); TaskgroupDescriptors.emplace_back(*ITD); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ITD, 1); } } // Privatize in_reduction items here, because taskgroup descriptors must be // privatized earlier. OMPPrivateScope InRedScope(CGF); if (!InRedVars.empty()) { ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps); for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) { RedCG.emitSharedLValue(CGF, Cnt); RedCG.emitAggregateType(CGF, Cnt); // The taskgroup descriptor variable is always implicit firstprivate and // privatized already during procoessing of the firstprivates. a2808 14 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), TaskgroupDescriptors[Cnt]->getExprLoc()); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( Replacement.getPointer(), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), [Replacement]() { return Replacement; }); d2811 1 a2811 1 (void)InRedScope.Privatize(); d2816 1 a2816 1 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( a2822 144 static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc) { auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *OrigRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, ImplicitParamDecl::Other); auto *PrivateRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); QualType ElemType = C.getBaseElementType(Ty); auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType, ImplicitParamDecl::Other); auto *InitRef = DeclRefExpr::Create( C, NestedNameSpecifierLoc(), SourceLocation(), InitVD, /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue); PrivateVD->setInitStyle(VarDecl::CInit); PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue, InitRef, /*BasePath=*/nullptr, VK_RValue)); Data.FirstprivateVars.emplace_back(OrigRef); Data.FirstprivateCopies.emplace_back(PrivateRef); Data.FirstprivateInits.emplace_back(InitRef); return OrigVD; } void CodeGenFunction::EmitOMPTargetTaskBasedDirective( const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo) { // Emit outlined function for task construct. const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); auto I = CS->getCapturedDecl()->param_begin(); auto PartId = std::next(I); auto TaskT = std::next(I, 4); OMPTaskDataTy Data; // The task is not final. Data.Final.setInt(/*IntVal=*/false); // Get list of firstprivate variables. for (const auto *C : S.getClausesOfKind()) { auto IRef = C->varlist_begin(); auto IElemInitRef = C->inits().begin(); for (auto *IInit : C->private_copies()) { Data.FirstprivateVars.push_back(*IRef); Data.FirstprivateCopies.push_back(IInit); Data.FirstprivateInits.push_back(*IElemInitRef); ++IRef; ++IElemInitRef; } } OMPPrivateScope TargetScope(*this); VarDecl *BPVD = nullptr; VarDecl *PVD = nullptr; VarDecl *SVD = nullptr; if (InputInfo.NumberOfTargetItems > 0) { auto *CD = CapturedDecl::Create( getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0); llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems); QualType BaseAndPointersType = getContext().getConstantArrayType( getContext().VoidPtrTy, ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); BPVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); PVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointersType, CD, S.getLocStart()); QualType SizesType = getContext().getConstantArrayType( getContext().getSizeType(), ArrSize, ArrayType::Normal, /*IndexTypeQuals=*/0); SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD, S.getLocStart()); TargetScope.addPrivate( BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; }); TargetScope.addPrivate(PVD, [&InputInfo]() { return InputInfo.PointersArray; }); TargetScope.addPrivate(SVD, [&InputInfo]() { return InputInfo.SizesArray; }); } (void)TargetScope.Privatize(); // Build list of dependences. for (const auto *C : S.getClausesOfKind()) for (const Expr *IRef : C->varlists()) Data.Dependences.emplace_back(C->getDependencyKind(), IRef); auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { // Set proper addresses for generated private copies. OMPPrivateScope Scope(CGF); if (!Data.FirstprivateVars.empty()) { enum { PrivatesParam = 2, CopyFnParam = 3 }; llvm::Value *CopyFn = CGF.Builder.CreateLoad( CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( CS->getCapturedDecl()->getParam(PrivatesParam))); // Map privates. llvm::SmallVector, 16> PrivatePtrs; llvm::SmallVector CallArgs; CallArgs.push_back(PrivatesPtr); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast(cast(E)->getDecl()); Address PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); } CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), CopyFn, CallArgs); for (const auto &Pair : PrivatePtrs) { Address Replacement(CGF.Builder.CreateLoad(Pair.second), CGF.getContext().getDeclAlign(Pair.first)); Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); } } // Privatize all private variables except for in_reduction items. (void)Scope.Privatize(); if (InputInfo.NumberOfTargetItems > 0) { InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize()); InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize()); } Action.Enter(CGF); OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); BodyGen(CGF); }; llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true, Data.NumberOfParts); llvm::APInt TrueOrFalse(32, S.hasClausesOfKind() ? 1 : 0); IntegerLiteral IfCond(getContext(), TrueOrFalse, getContext().getIntTypeForBitwidth(32, /*Signed=*/0), SourceLocation()); CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn, SharedsTy, CapturedStruct, &IfCond, Data); } d2825 3 a2827 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d2850 1 a2850 1 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data); d2870 1 a2870 30 if (const Expr *E = S.getReductionRef()) { SmallVector LHSs; SmallVector RHSs; OMPTaskDataTy Data; for (const auto *C : S.getClausesOfKind()) { auto IPriv = C->privates().begin(); auto IRed = C->reduction_ops().begin(); auto ILHS = C->lhs_exprs().begin(); auto IRHS = C->rhs_exprs().begin(); for (const Expr *Ref : C->varlists()) { Data.ReductionVars.emplace_back(Ref); Data.ReductionCopies.emplace_back(*IPriv); Data.ReductionOps.emplace_back(*IRed); LHSs.emplace_back(*ILHS); RHSs.emplace_back(*IRHS); std::advance(IPriv, 1); std::advance(IRed, 1); std::advance(ILHS, 1); std::advance(IRHS, 1); } } llvm::Value *ReductionDesc = CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(), LHSs, RHSs, Data); const auto *VD = cast(cast(E)->getDecl()); CGF.EmitVarDecl(*VD); CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), /*Volatile=*/false, E->getType()); } CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d2872 1 a2872 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d2877 7 a2883 9 CGM.getOpenMPRuntime().emitFlush( *this, [&S]() -> ArrayRef { if (const auto *FlushClause = S.getSingleClause()) return llvm::makeArrayRef(FlushClause->varlist_begin(), FlushClause->varlist_end()); return llvm::None; }(), S.getLocStart()); d2890 2 a2891 2 const auto *IVExpr = cast(S.getIterationVariable()); const auto *IVDecl = cast(IVExpr->getDecl()); d2897 1 a2897 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d2903 1 a2903 1 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); d2918 1 a2918 1 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); a2925 1 emitAlignedClause(*this, S); d2947 2 a2948 2 // Emit implicit barrier to synchronize threads and avoid data races // on initialization of firstprivate variables and post-update of d2951 2 a2952 2 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false, /*ForceSimpleCall=*/true); a2954 4 if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) EmitOMPReductionClauseInit(S, LoopScope); d2962 1 a2962 1 if (const auto *C = S.getSingleClause()) { d2964 1 a2964 1 if (const Expr *Ch = C->getChunkSize()) { d2967 2 a2968 2 S.getIterationVariable()->getType(), S.getLocStart()); a2983 5 if (isOpenMPSimdDirective(S.getDirectiveKind())) EmitOMPSimdInit(S, /*IsMonotonic=*/true); CGOpenMPRuntime::StaticRTInput StaticInit( IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(), LB.getAddress(), UB.getAddress(), ST.getAddress()); d2985 4 a2988 2 StaticInit); JumpDest LoopExit = d2999 3 a3001 4 const Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) ? S.getCombinedCond() : S.getCond(); d3014 1 a3014 1 RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind()); d3024 1 a3024 30 if (isOpenMPSimdDirective(S.getDirectiveKind())) { EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } if (isOpenMPSimdDirective(S.getDirectiveKind()) && !isOpenMPParallelDirective(S.getDirectiveKind()) && !isOpenMPTeamsDirective(S.getDirectiveKind())) { OpenMPDirectiveKind ReductionKind = OMPD_unknown; if (isOpenMPParallelDirective(S.getDirectiveKind()) && isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for_simd; } else if (isOpenMPParallelDirective(S.getDirectiveKind())) { ReductionKind = OMPD_parallel_for; } else if (isOpenMPSimdDirective(S.getDirectiveKind())) { ReductionKind = OMPD_simd; } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) && S.hasClausesOfKind()) { llvm_unreachable( "No reduction clauses is allowed in distribute directive."); } EmitOMPReductionClauseFinal(S, ReductionKind); // Emit post-update of the reduction variables if IsLastIter != 0. emitPostUpdateForReductionClause( *this, S, [IL, &S](CodeGenFunction &CGF) { return CGF.Builder.CreateIsNotNull( CGF.EmitLoadOfScalar(IL, S.getLocStart())); }); } d3026 1 a3026 1 if (HasLastprivateClause) { d3029 2 a3030 2 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } d3044 1 d3047 3 a3049 2 OMPLexicalScope Scope(*this, S, OMPD_unknown); CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); d3057 2 a3058 2 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); Fn->setDoesNotRecurse(); d3063 1 a3063 3 if (S.hasClausesOfKind()) { assert(!S.getAssociatedStmt() && "No associated statement must be in ordered depend construct."); d3068 1 a3068 1 const auto *C = S.getSingleClause(); a3070 1 const CapturedStmt *CS = S.getInnermostCapturedStmt(); d3072 1 d3075 2 a3076 3 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(), OutlinedFn, CapturedVars); d3079 2 a3080 1 CGF.EmitStmt(CS->getCapturedStmt()); d3083 1 a3083 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3093 5 a3097 4 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType, Loc) : CGF.EmitComplexToScalarConversion( Val.getComplexVal(), SrcType, DestType, Loc); d3108 3 a3110 4 QualType DestElementType = DestType->castAs()->getElementType(); llvm::Value *ScalarVal = CGF.EmitScalarConversion( Val.getScalarVal(), SrcType, DestElementType, Loc); d3115 2 a3116 3 QualType SrcElementType = SrcType->castAs()->getElementType(); QualType DestElementType = DestType->castAs()->getElementType(); d3155 1 a3155 1 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, d3179 1 a3179 1 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, d3198 1 a3198 1 ASTContext &Context = CGF.getContext(); a3261 1 case BO_Cmp: d3275 1 a3275 1 llvm::Value *UpdateVal = Update.getScalarVal(); d3281 1 a3281 2 llvm::Value *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); d3288 1 a3288 1 const llvm::function_ref CommonGen) { d3309 1 a3309 1 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, d3315 1 a3315 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3325 12 a3336 12 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); return CGF.EmitAnyExpr(UE); }; d3364 1 a3364 1 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, d3375 2 a3376 3 llvm::AtomicOrdering AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent : llvm::AtomicOrdering::Monotonic; d3382 1 a3382 1 const auto *BOUE = cast(UE->IgnoreImpCasts()); d3389 3 a3391 3 const auto *LHS = cast(BOUE->getLHS()->IgnoreImpCasts()); const auto *RHS = cast(BOUE->getRHS()->IgnoreImpCasts()); const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; d3393 1 a3393 1 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; d3395 1 a3395 1 IsPostfixUpdate](RValue XRValue) { d3422 1 a3422 1 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) { d3445 1 a3445 1 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, d3452 1 a3452 1 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); d3455 1 a3455 1 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); d3459 1 a3459 1 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); d3462 1 a3462 1 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, a3472 1 case OMPC_in_reduction: d3517 1 a3517 1 for (const OMPClause *C : S.clauses()) { d3525 3 a3527 2 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers(); if (const auto *EWC = dyn_cast(CS)) d3529 1 d3532 2 a3533 2 for (const Stmt *C : Compound->body()) { if (const auto *EWC = dyn_cast(C)) d3535 1 d3542 1 a3542 1 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), d3546 1 a3546 1 OMPLexicalScope Scope(*this, S, OMPD_unknown); d3555 1 a3555 10 // On device emit this construct as inlined code. if (CGM.getLangOpts().OpenMPIsDevice) { OMPLexicalScope Scope(CGF, S, OMPD_target); CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); }); return; } d3572 1 a3572 1 if (auto *C = S.getSingleClause()) d3574 1 d3592 1 a3592 1 if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3594 1 a3594 1 else if (const auto *D = dyn_cast(CGF.CurFuncDecl)) d3603 5 a3607 2 OMPLexicalScope Scope(CGF, S, OMPD_task); CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device); a3611 1 Action.Enter(CGF); d3617 2 a3618 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); d3647 2 a3648 3 llvm::Value *OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); d3650 2 a3651 2 const auto *NT = S.getSingleClause(); const auto *TL = S.getSingleClause(); d3653 2 a3654 2 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr; const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr; d3669 1 a3669 2 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3675 1 a3675 1 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); d3678 3 a3680 3 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); d3687 2 a3688 8 // Emit teams region as a standalone region. auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); CGF.EmitOMPPrivateClause(S, PrivateScope); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); a3689 1 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); a3691 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3715 286 static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( const OMPTargetTeamsDistributeDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeSimdRegion( CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( const OMPTargetTeamsDistributeSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeSimdRegion(CGF, Action, S); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPTeamsDistributeDirective( const OMPTeamsDistributeDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( const OMPTeamsDistributeSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( const OMPTeamsDistributeParallelForDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGenDistribute); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( const OMPTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); emitPostUpdateForReductionClause(*this, S, [](CodeGenFunction &) { return nullptr; }); } static void emitTargetTeamsDistributeParallelForRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S) { // Emit SPMD target teams distribute parallel for region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( const OMPTargetTeamsDistributeParallelForDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetTeamsDistributeParallelForSimdRegion( CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, S.getDistInc()); }; // Emit teams region as a standalone region. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPPrivateScope PrivateScope(CGF); CGF.EmitOMPReductionClauseInit(S, PrivateScope); (void)PrivateScope.Privatize(); CGF.CGM.getOpenMPRuntime().emitInlinedDirective( CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); }; emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, CodeGenTeams); emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S) { // Emit SPMD target teams distribute parallel for simd region as a standalone // region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( const OMPTargetTeamsDistributeParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } d3743 1 a3743 3 Kind == OMPD_target_parallel_for || Kind == OMPD_teams_distribute_parallel_for || Kind == OMPD_target_teams_distribute_parallel_for); d3753 4 a3756 4 for (const Expr *PvtVarIt : C.private_copies()) { const auto *OrigVD = cast(cast(*OrigVarIt)->getDecl()); const auto *InitVD = cast(cast(*InitIt)->getDecl()); const auto *PvtVD = cast(cast(PvtVarIt)->getDecl()); d3762 1 a3762 1 if (const auto *OED = dyn_cast(MatchingVD)) { d3765 1 a3765 1 const auto *ME = cast(OED->getInit()); d3777 1 a3777 3 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD, InitAddrIt, InitVD, PvtVD]() { d3794 1 a3794 1 // of the previous declaration, so we don't need it anymore. d3831 1 a3831 1 CodeGenFunction &CGF, PrePostActionTy &Action) { d3833 2 a3834 1 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); d3837 1 a3837 1 // Codegen that selects whether to generate the privatization code or not. d3856 1 a3856 1 } else { a3857 1 } d3883 1 a3883 1 if (const auto *C = S.getSingleClause()) d3888 1 a3888 1 if (const auto *C = S.getSingleClause()) d3908 1 a3908 1 if (const auto *C = S.getSingleClause()) d3913 1 a3913 1 if (const auto *C = S.getSingleClause()) a3915 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3928 1 a3928 1 if (const auto *C = S.getSingleClause()) d3933 1 a3933 1 if (const auto *C = S.getSingleClause()) a3935 1 OMPLexicalScope Scope(*this, S, OMPD_task); d3943 1 a3943 1 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); d3945 1 a3945 2 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); d3957 2 a3958 2 emitPostUpdateForReductionClause(CGF, S, [](CodeGenFunction &) { return nullptr; }); a3982 32 static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CodeGenFunction::OMPCancelStackRAII CancelRegion( CGF, OMPD_target_parallel_for, S.hasCancel()); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } d3985 1 a3985 43 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); } static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action) { Action.Enter(CGF); // Emit directive as a combined directive that consists of two implicit // directives: 'parallel' with 'for' directive. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { Action.Enter(CGF); CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, emitDispatchForLoopBounds); }; emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, emitEmptyBoundParameters); } void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S) { // Emit SPMD target parallel for region as a standalone region. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; llvm::Function *Fn; llvm::Constant *Addr; // Emit target region as a standalone region. CGM.getOpenMPRuntime().emitTargetOutlinedFunction( S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); assert(Fn && Addr && "Target device function emission failed."); } void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( const OMPTargetParallelForSimdDirective &S) { auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { emitTargetParallelForSimdRegion(CGF, S, Action); }; emitCommonOMPTargetDirective(*this, S, CodeGen); d3992 3 a3994 3 const auto *VDecl = cast(Helper->getDecl()); Privates.addPrivate(VDecl, [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); d4000 3 a4002 3 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop); Address CapturedStruct = GenerateCapturedStmtArgument(*CS); QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); d4045 1 a4045 1 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); d4076 1 a4076 1 const auto *IVDecl = cast(cast(IVExpr)->getDecl()); d4083 1 a4083 1 if (const auto *LIExpr = dyn_cast(S.getLastIteration())) { d4113 1 a4113 2 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, &Data](CodeGenFunction &CGF, PrePostActionTy &) { d4122 3 a4124 3 if (Data.Nogroup) { EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); } else { d4130 1 a4130 2 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); d4155 1 a4155 1 if (const auto *C = S.getSingleClause()) d4160 1 a4160 1 if (const auto *C = S.getSingleClause()) a4162 1 OMPLexicalScope Scope(*this, S, OMPD_task); a4164 30 void CodeGenFunction::EmitSimpleOMPExecutableDirective( const OMPExecutableDirective &D) { if (!D.hasAssociatedStmt() || !D.getAssociatedStmt()) return; auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { if (isOpenMPSimdDirective(D.getDirectiveKind())) { emitOMPSimdRegion(CGF, cast(D), Action); } else { if (const auto *LD = dyn_cast(&D)) { for (const Expr *E : LD->counters()) { if (const auto *VD = dyn_cast( cast(E)->getDecl())) { // Emit only those that were not explicitly referenced in clauses. if (!CGF.LocalDeclMap.count(VD)) CGF.EmitVarDecl(*VD); } } } CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); } }; OMPSimdLexicalScope Scope(*this, D); CGM.getOpenMPRuntime().emitInlinedDirective( *this, isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd : D.getDirectiveKind(), CodeGen); } @ 1.1.1.10 log @Mark old LLVM instance as dead. @ text @@ 1.1.1.2.4.1 log @file CGStmtOpenMP.cpp was added on branch tls-maxphys on 2014-08-19 23:47:27 +0000 @ text @d1 138 @ 1.1.1.2.4.2 log @Rebase to HEAD as of a few days ago. @ text @a0 138 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This contains code to emit OpenMP nodes as LLVM code. // //===----------------------------------------------------------------------===// #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "clang/AST/Stmt.h" #include "clang/AST/StmtOpenMP.h" using namespace clang; using namespace CodeGen; //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); llvm::Value *CapturedStruct = GenerateCapturedStmtArgument(*CS); llvm::Value *OutlinedFn; { CodeGenFunction CGF(CGM, true); CGCapturedStmtInfo CGInfo(*CS, CS->getCapturedRegionKind()); CGF.CapturedStmtInfo = &CGInfo; OutlinedFn = CGF.GenerateCapturedStmtFunction(*CS); } // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) llvm::Value *Args[] = { CGM.getOpenMPRuntime().EmitOpenMPUpdateLocation(*this, S.getLocStart()), Builder.getInt32(1), // Number of arguments after 'microtask' argument // (there is only one additional argument - 'context') Builder.CreateBitCast(OutlinedFn, CGM.getOpenMPRuntime().getKmpc_MicroPointerTy()), EmitCastToVoidPtr(CapturedStruct)}; llvm::Constant *RTLFn = CGM.getOpenMPRuntime().CreateRuntimeFunction( CGOpenMPRuntime::OMPRTL__kmpc_fork_call); EmitRuntimeCall(RTLFn, Args); } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); const Stmt *Body = CS->getCapturedStmt(); LoopStack.setParallel(); LoopStack.setVectorizerEnable(true); for (auto C : S.clauses()) { switch (C->getClauseKind()) { case OMPC_safelen: { RValue Len = EmitAnyExpr(cast(C)->getSafelen(), AggValueSlot::ignored(), true); llvm::ConstantInt *Val = cast(Len.getScalarVal()); LoopStack.setVectorizerWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. LoopStack.setParallel(false); break; } default: // Not handled yet ; } } EmitStmt(Body); } void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &) { llvm_unreachable("CodeGen for 'omp for' is not supported yet."); } void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &) { llvm_unreachable("CodeGen for 'omp sections' is not supported yet."); } void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &) { llvm_unreachable("CodeGen for 'omp section' is not supported yet."); } void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &) { llvm_unreachable("CodeGen for 'omp single' is not supported yet."); } void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &) { llvm_unreachable("CodeGen for 'omp master' is not supported yet."); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &) { llvm_unreachable("CodeGen for 'omp critical' is not supported yet."); } void CodeGenFunction::EmitOMPParallelForDirective(const OMPParallelForDirective &) { llvm_unreachable("CodeGen for 'omp parallel for' is not supported yet."); } void CodeGenFunction::EmitOMPParallelSectionsDirective( const OMPParallelSectionsDirective &) { llvm_unreachable("CodeGen for 'omp parallel sections' is not supported yet."); } void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &) { llvm_unreachable("CodeGen for 'omp task' is not supported yet."); } void CodeGenFunction::EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &) { llvm_unreachable("CodeGen for 'omp taskyield' is not supported yet."); } void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &) { llvm_unreachable("CodeGen for 'omp barrier' is not supported yet."); } void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) { llvm_unreachable("CodeGen for 'omp taskwait' is not supported yet."); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &) { llvm_unreachable("CodeGen for 'omp flush' is not supported yet."); } void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) { llvm_unreachable("CodeGen for 'omp ordered' is not supported yet."); } void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &) { llvm_unreachable("CodeGen for 'omp atomic' is not supported yet."); } @ 1.1.1.1.2.1 log @file CGStmtOpenMP.cpp was added on branch tls-earlyentropy on 2014-08-10 07:08:07 +0000 @ text @d1 78 @ 1.1.1.1.2.2 log @Rebase. @ text @a0 78 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This contains code to emit OpenMP nodes as LLVM code. // //===----------------------------------------------------------------------===// #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "clang/AST/Stmt.h" #include "clang/AST/StmtOpenMP.h" using namespace clang; using namespace CodeGen; //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); llvm::Value *CapturedStruct = GenerateCapturedStmtArgument(*CS); llvm::Value *OutlinedFn; { CodeGenFunction CGF(CGM, true); CGCapturedStmtInfo CGInfo(*CS, CS->getCapturedRegionKind()); CGF.CapturedStmtInfo = &CGInfo; OutlinedFn = CGF.GenerateCapturedStmtFunction( CS->getCapturedDecl(), CS->getCapturedRecordDecl(), CS->getLocStart()); } // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) llvm::Value *Args[] = { CGM.getOpenMPRuntime().EmitOpenMPUpdateLocation(*this, S.getLocStart()), Builder.getInt32(1), // Number of arguments after 'microtask' argument // (there is only one additional argument - 'context') Builder.CreateBitCast(OutlinedFn, CGM.getOpenMPRuntime().getKmpc_MicroPointerTy()), EmitCastToVoidPtr(CapturedStruct) }; llvm::Constant *RTLFn = CGM.getOpenMPRuntime().CreateRuntimeFunction( CGOpenMPRuntime::OMPRTL__kmpc_fork_call); EmitRuntimeCall(RTLFn, Args); } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { const CapturedStmt *CS = cast(S.getAssociatedStmt()); const Stmt *Body = CS->getCapturedStmt(); LoopStack.setParallel(); LoopStack.setVectorizerEnable(true); for (auto C : S.clauses()) { switch (C->getClauseKind()) { case OMPC_safelen: { RValue Len = EmitAnyExpr(cast(C)->getSafelen(), AggValueSlot::ignored(), true); llvm::ConstantInt *Val = cast(Len.getScalarVal()); LoopStack.setVectorizerWidth(Val->getZExtValue()); // In presence of finite 'safelen', it may be unsafe to mark all // the memory instructions parallel, because loop-carried // dependences of 'safelen' iterations are possible. LoopStack.setParallel(false); break; } default: // Not handled yet ; } } EmitStmt(Body); } @