Skip to content

Commit 97cb71e

Browse files
committed
[concurrency] Implement bit masking for TBI when available or in tagged pointer bits otherwise.
rdar://156525771
1 parent b472f28 commit 97cb71e

File tree

8 files changed

+391
-32
lines changed

8 files changed

+391
-32
lines changed

lib/IRGen/IRGenFunction.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ SILModule &IRGenFunction::getSILModule() const {
105105
return IGM.getSILModule();
106106
}
107107

108+
ASTContext &IRGenFunction::getASTContext() const {
109+
return getSILModule().getASTContext();
110+
}
111+
108112
Lowering::TypeConverter &IRGenFunction::getSILTypes() const {
109113
return IGM.getSILTypes();
110114
}

lib/IRGen/IRGenFunction.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ class IRGenFunction {
8484
llvm::Function *const CurFn;
8585
ModuleDecl *getSwiftModule() const;
8686
SILModule &getSILModule() const;
87+
ASTContext &getASTContext() const;
8788
Lowering::TypeConverter &getSILTypes() const;
8889
const IRGenOptions &getOptions() const;
8990

lib/IRGen/IRGenSIL.cpp

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,9 @@ class LoweredValue {
312312
bool isBoxWithAddress() const {
313313
return kind == Kind::OwnedAddress;
314314
}
315-
315+
316+
bool isExplosionVector() const { return kind == Kind::ExplosionVector; }
317+
316318
const StackAddress &getStackAddress() const {
317319
return Storage.get<StackAddress>(kind);
318320
}
@@ -3373,12 +3375,44 @@ void IRGenSILFunction::visitExistentialMetatypeInst(
33733375
setLoweredExplosion(i, result);
33743376
}
33753377

3376-
static void emitApplyArgument(IRGenSILFunction &IGF,
3377-
SILValue arg,
3378-
SILType paramType,
3379-
Explosion &out,
3380-
SILInstruction *apply = nullptr,
3381-
unsigned idx = 0) {
3378+
static llvm::Value *clearImplicitIsolatedActorBits(IRGenFunction &IGF,
3379+
llvm::Value *value) {
3380+
// Helper. We conditionally use this and just use a simple helper function
3381+
// here so we do not expose the internals of clearing implicit isolated actor
3382+
// bits into the rest of the file. The work here is very straight forward.
3383+
auto getTBIClearMask = [](IRGenFunction &IGF) -> llvm::Value * {
3384+
auto *one = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 1);
3385+
auto *three = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 3);
3386+
auto *four = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 4);
3387+
auto *valueToShiftLit = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 3);
3388+
3389+
auto *sizeOfWord =
3390+
llvm::ConstantInt::get(IGF.IGM.IntPtrTy, one->getBitWidth() >> 3);
3391+
3392+
// sizeof(Word) - 1
3393+
auto *sub = IGF.Builder.CreateSub(sizeOfWord, one);
3394+
// (sizeof(Word) - 1) << 3
3395+
auto *innerShift = IGF.Builder.CreateShl(sub, three);
3396+
// ((sizeof(Word) - 1) << 3) + 4
3397+
auto *innerShiftOffset = IGF.Builder.CreateAdd(innerShift, four);
3398+
auto *negBits = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, -1);
3399+
// (valueToShiftLit << innerShiftOffset) ^ -1.
3400+
return IGF.Builder.CreateXor(
3401+
IGF.Builder.CreateShl(valueToShiftLit, innerShiftOffset), negBits);
3402+
};
3403+
3404+
auto *cast = IGF.Builder.CreateBitOrPointerCast(value, IGF.IGM.IntPtrTy);
3405+
auto *bitMask = IGF.getASTContext().LangOpts.HasAArch64TBI
3406+
? getTBIClearMask(IGF)
3407+
: llvm::ConstantInt::get(IGF.IGM.IntPtrTy, -4);
3408+
auto *result = IGF.Builder.CreateAnd(cast, bitMask);
3409+
return IGF.Builder.CreateBitOrPointerCast(result, value->getType());
3410+
}
3411+
3412+
static void emitApplyArgument(IRGenSILFunction &IGF, SILValue arg,
3413+
SILType paramType, Explosion &out,
3414+
SILInstruction *apply = nullptr, unsigned idx = 0,
3415+
bool isImplicitIsolatedParameter = false) {
33823416
bool isSubstituted = (arg->getType() != paramType);
33833417

33843418
// For indirect arguments, we just need to pass a pointer.
@@ -3420,7 +3454,20 @@ static void emitApplyArgument(IRGenSILFunction &IGF,
34203454
}
34213455
canForwardLoadToIndirect = true;
34223456
}();
3423-
IGF.getLoweredExplosion(arg, out);
3457+
3458+
// If we are emitting a parameter for an implicit isolated parameter, then
3459+
// we need to clear the implicit isolated actor bits.
3460+
if (isImplicitIsolatedParameter) {
3461+
auto &loweredValue = IGF.getLoweredValue(arg);
3462+
assert(loweredValue.isExplosionVector() &&
3463+
"Should be an explosion of two pointers");
3464+
auto explosionVector = loweredValue.getKnownExplosionVector();
3465+
assert(explosionVector.size() == 2 && "We should have two values");
3466+
out.add(explosionVector[0]);
3467+
out.add(clearImplicitIsolatedActorBits(IGF, explosionVector[1]));
3468+
} else {
3469+
IGF.getLoweredExplosion(arg, out);
3470+
}
34243471
if (canForwardLoadToIndirect) {
34253472
IGF.setForwardableArgument(idx);
34263473
}
@@ -3838,6 +3885,20 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
38383885
}
38393886
}
38403887

3888+
// Extract the implicit isolated parameter so that we can mask it as
3889+
// appropriate.
3890+
//
3891+
// NOTE: We cannot just drop_front since we could be between the indirect
3892+
// results and the parameters.
3893+
std::optional<unsigned> implicitIsolatedParameterIndex;
3894+
if (auto actorIsolation = site.getFunction()->getActorIsolation();
3895+
actorIsolation && actorIsolation->isCallerIsolationInheriting() &&
3896+
site.isCallerIsolationInheriting()) {
3897+
auto *iso = site.getIsolatedArgumentOperandOrNullPtr();
3898+
assert(iso);
3899+
implicitIsolatedParameterIndex = site.getAppliedArgIndex(*iso);
3900+
}
3901+
38413902
// Lower the arguments and return value in the callee's generic context.
38423903
GenericContextScope scope(IGM,
38433904
origCalleeType->getInvocationGenericSignature());
@@ -3898,8 +3959,11 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
38983959
emission->setIndirectTypedErrorResultSlot(addr.getAddress());
38993960
continue;
39003961
}
3962+
39013963
emitApplyArgument(*this, args[index], emission->getParameterType(index),
3902-
llArgs, site.getInstruction(), index);
3964+
llArgs, site.getInstruction(), index,
3965+
implicitIsolatedParameterIndex &&
3966+
*implicitIsolatedParameterIndex == index);
39033967
}
39043968

39053969
// Pass the generic arguments.

lib/SIL/Utils/ConcurrencyUtils.cpp

Lines changed: 122 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,136 @@
1717

1818
using namespace swift;
1919

20+
/// We could use a higher bit. But by reusing the first bit, we just save a
21+
/// little bit of code.
22+
///
23+
/// $valueToShift << (((sizeof(Word) - 1) << 3) + 4)
24+
///
25+
/// Mathematicaly this is $valueToShift * 2**((sizeof(Word) - 1)*8 + 4)
26+
///
27+
/// On 64 bit this is 60. This works since we want to use the bottom two bits of
28+
/// the top nibble of the TBI bits.
29+
static SILValue getTBIBits(SILBuilder &b, SILLocation loc,
30+
unsigned valueToShift = 1) {
31+
auto &ctx = b.getASTContext();
32+
auto silWordType = SILType::getBuiltinWordType(ctx);
33+
34+
auto id = ctx.getIdentifier(getBuiltinName(BuiltinValueKind::Sizeof));
35+
auto *builtin = cast<FuncDecl>(getBuiltinValueDecl(ctx, id));
36+
auto wordType = BuiltinIntegerType::getWordType(ctx)->getCanonicalType();
37+
auto metatypeTy = SILType::getPrimitiveObjectType(CanMetatypeType::get(
38+
wordType->getCanonicalType(), MetatypeRepresentation::Thin));
39+
auto metatypeVal = b.createMetatype(loc, metatypeTy);
40+
41+
auto sizeOfWord =
42+
b.createBuiltin(loc, id, silWordType,
43+
SubstitutionMap::get(builtin->getGenericSignature(),
44+
ArrayRef<Type>{wordType},
45+
LookUpConformanceInModule()),
46+
{metatypeVal});
47+
auto one = b.createIntegerLiteral(loc, silWordType, 1);
48+
auto three = b.createIntegerLiteral(loc, silWordType, 3);
49+
auto four = b.createIntegerLiteral(loc, silWordType, 4);
50+
auto valueToShiftLit = b.createIntegerLiteral(loc, silWordType, valueToShift);
51+
52+
// sizeof(Word) - 1
53+
auto sub = b.createBuiltinBinaryFunction(loc, "sub", silWordType, silWordType,
54+
{sizeOfWord, one});
55+
// (sizeof(Word) - 1) << 3
56+
auto innerShift = b.createBuiltinBinaryFunction(loc, "shl", silWordType,
57+
silWordType, {sub, three});
58+
// ((sizeof(Word) - 1) << 3) + 4
59+
auto innerShiftOffset = b.createBuiltinBinaryFunction(
60+
loc, "add", silWordType, silWordType, {innerShift, four});
61+
auto outerShift =
62+
b.createBuiltinBinaryFunction(loc, "shl", silWordType, silWordType,
63+
{valueToShiftLit, innerShiftOffset});
64+
return outerShift;
65+
}
66+
67+
/// Construct the TBI mask in a platform independent way that works on all
68+
/// platforms.
69+
///
70+
/// We compute:
71+
///
72+
/// mask = (0x3 << (((sizeof(Word) - 1) << 3) + 4)) ^ -1
73+
static SILValue getTBIClearMask(SILBuilder &b, SILLocation loc) {
74+
auto &ctx = b.getASTContext();
75+
auto silWordType = SILType::getBuiltinWordType(ctx);
76+
auto negBits = b.createIntegerLiteral(loc, silWordType, -1);
77+
78+
return b.createBuiltinBinaryFunction(loc, "xor", silWordType, silWordType,
79+
{getTBIBits(b, loc, 3), negBits});
80+
}
81+
82+
static SILValue transformTupleElts(
83+
SILBuilder &b, SILLocation loc, SILValue mv, SILType finalType,
84+
llvm::function_ref<SILValue(ArrayRef<SILValue> destructureValues)> func) {
85+
auto &ctx = b.getASTContext();
86+
auto silWordType = SILType::getBuiltinWordType(ctx);
87+
auto tupleType =
88+
SILType::getTupleType(b.getASTContext(), {silWordType, silWordType});
89+
auto cast = b.emitUncheckedValueCast(loc, mv, tupleType);
90+
SmallVector<SILValue, 2> destructureValues;
91+
b.emitDestructureOperation(loc, cast, destructureValues);
92+
SILValue reformedValue = func(destructureValues);
93+
auto reformedPointer =
94+
b.emitUncheckedValueCast(loc, reformedValue, finalType);
95+
96+
// NOTE: Our reformed pointer is actually has none ownership here. Our value
97+
// gains guaranteed ownership by the merging the guaranteedness of mv (the
98+
// mark dependence base).
99+
//
100+
// DISCUSSION: This is important since we allows for us to avoid needing to
101+
// insert end_borrows on the reformed value in our caller which would be
102+
// required if we used an unchecked_ownership_conversion. The author
103+
// originally did that thinking it would be safer but hit problems in
104+
// SILGenApply since even if we inserted an end_borrow for the
105+
// unchecked_ownership_conversion, we failed to insert the end_borrow along
106+
// error paths of try_apply due to the way SILGenApply is setup. After some
107+
// thought, it became clear we could safely just rely on mark_dependence to
108+
// mixin the guaranteedness and get the semantics that we cared about (tieing
109+
// the reformed value to the lifetime of the original value in a guaranteed
110+
// way).
111+
return b.emitMarkDependence(loc, reformedPointer, mv,
112+
MarkDependenceKind::NonEscaping);
113+
}
114+
20115
SILValue swift::clearImplicitIsolationActorBits(SILBuilder &b, SILLocation loc,
21116
SILValue value,
22117
SILType finalType) {
23118
if (!finalType)
24119
finalType =
25120
SILType::getBuiltinImplicitIsolationActorType(b.getASTContext());
26-
if (value->getType() == finalType)
27-
return value;
28-
return b.emitUncheckedValueCast(loc, value, finalType);
121+
122+
auto &ctx = b.getASTContext();
123+
124+
return transformTupleElts(
125+
b, loc, value, finalType, [&](ArrayRef<SILValue> tupleElts) {
126+
auto silWordType = SILType::getBuiltinWordType(ctx);
127+
SILValue bitMask = ctx.LangOpts.HasAArch64TBI
128+
? getTBIClearMask(b, loc)
129+
: b.createIntegerLiteral(loc, silWordType, -4);
130+
131+
auto result = b.createBuiltinBinaryFunction(
132+
loc, "and", silWordType, silWordType, {tupleElts[1], bitMask});
133+
return b.createTuple(loc, {tupleElts[0], result});
134+
});
29135
}
30136

31137
SILValue swift::setImplicitIsolationActorBits(SILBuilder &b, SILLocation loc,
32138
SILValue value) {
33-
return value;
139+
auto &ctx = b.getASTContext();
140+
141+
return transformTupleElts(
142+
b, loc, value, value->getType(), [&](ArrayRef<SILValue> tupleElts) {
143+
auto silWordType = SILType::getBuiltinWordType(ctx);
144+
SILValue bitMask = ctx.LangOpts.HasAArch64TBI
145+
? getTBIBits(b, loc)
146+
: b.createIntegerLiteral(loc, silWordType, 1);
147+
148+
auto result = b.createBuiltinBinaryFunction(
149+
loc, "or", silWordType, silWordType, {tupleElts[1], bitMask});
150+
return b.createTuple(loc, {tupleElts[0], result});
151+
});
34152
}

lib/SILGen/SILGenBuilder.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1255,5 +1255,5 @@ SILValue SILGenBuilder::convertToImplicitIsolationActor(SILLocation loc,
12551255
"Builtin.ImplicitIsolationActor");
12561256
if (value->getOwnershipKind() != OwnershipKind::Guaranteed)
12571257
value = SGF.emitManagedBeginBorrow(loc, value).getValue();
1258-
return createUncheckedValueCast(loc, value, type);
1258+
return emitUncheckedValueCast(loc, value, type);
12591259
}

lib/SILGen/SILGenConcurrency.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -425,8 +425,8 @@ emitNonOptionalActorInstanceIsolation(SILGenFunction &SGF, SILLocation loc,
425425
if (actor.getType() == anyActorTy)
426426
return actor;
427427

428-
if (actor.getType() == SILType::getPrimitiveObjectType(
429-
SGF.getASTContext().TheImplicitIsolationActorType))
428+
if (actor.getType() ==
429+
SILType::getBuiltinImplicitIsolationActorType(SGF.getASTContext()))
430430
return actor;
431431

432432
CanType anyActorType = anyActorTy.getASTType();

0 commit comments

Comments
 (0)