Repository navigation
select BMI bit operations in the x86 backend - #146
Merged
Merged
Conversation
Coverage
Diff CoverageDiff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
x86gen.hppLines 954-963 954 }
955
956 bool build_bmi_and(AndInst* inst) {
957 if (!__builtin_cpu_supports("bmi")) {
! 958 return false;
! 959 }
960 for (size_t it = 0; it < 2; it++) {
961 Value* a = inst->arg(it);
962 Value* b = inst->arg(1 - it);
963 if (dynmatch(XorInst, inversion, a)) {Lines 964-972 964 for (size_t index = 0; index < 2; index++) {
965 if (dynmatch(Const, ones, inversion->arg(index))) {
966 if (ones->value() == type_mask(inst->type())) {
967 if (inst->type() == Type::Int64) {
! 968 _builder.andn64(vreg(inst), vreg(inversion->arg(1 - index)), vreg(b));
969 } else {
970 _builder.andn32(vreg(inst), vreg(inversion->arg(1 - index)), vreg(b));
971 }
972 return true;Lines 987-1017 987 }
988 if (!shift || _use_counts.at(shift) != 1 || !_vregs.at(shift).is_invalid()) {
989 continue;
990 }
! 991 if (dynmatch(Const, amount, shift->arg(1))) {
! 992 size_t width = type_width(inst->type());
! 993 uint64_t start = amount->value();
! 994 if (start == 0 || start >= width) {
! 995 continue;
! 996 }
! 997 size_t length = __builtin_popcountll(bits);
! 998 if (length > width - start) {
! 999 if (arithmetic) {
! 1000 continue;
! 1001 }
! 1002 length = width - start;
! 1003 }
! 1004 Reg control = vreg();
! 1005 _builder.mov32_imm(control, start | (length << 8));
! 1006 if (inst->type() == Type::Int64) {
! 1007 _builder.bextr64(vreg(inst), control, vreg(shift->arg(0)));
! 1008 } else {
! 1009 _builder.bextr32(vreg(inst), control, vreg(shift->arg(0)));
! 1010 }
! 1011 return true;
! 1012 }
! 1013 }
1014 }
1015 return false;
1016 }x86insts.inc.hppLines 113-123 113 imm_binop_x86_inst(Add64Imm, add64_imm, imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
114 imm_binop_x86_inst(Sub64Imm, sub64_imm, imm_usedef, true, { reg = Reg::phys(5); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
115
116 ternop_x86_inst(Andn32, andn32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf2); modrm(); })
! 117 ternop_x86_inst(Andn64, andn64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf2); modrm(); })
! 118 ternop_x86_inst(Bextr32, bextr32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf7); modrm(); })
! 119 ternop_x86_inst(Bextr64, bextr64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf7); modrm(); })
120 ternop_x86_inst(Sarx32, sarx32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 2); byte(0xf7); modrm(); })
121 ternop_x86_inst(Sarx64, sarx64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 2); byte(0xf7); modrm(); })
122
123 binop_x86_inst(And32, and32, binop_usedef, false, { rex_opt(); byte(0x23); modrm(); })Lines 235-243 235
236 imm_binop_x86_inst(Test32Imm, test32_imm, cmp_imm_usedef, false, { reg = Reg::phys(0); rex_opt(); byte(0xf7); modrm(); imm_n(4); })
237 imm_binop_x86_inst(Test64Imm, test64_imm, cmp_imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(0xf7); modrm(); imm_n(4); })
238
! 239 #undef ternop_x86_inst
240 #undef x86_inst
241 #undef binop_x86_inst
242 #undef rev_binop_x86_inst
243 #undef imm_binop_x86_inst |
Coverage
Diff CoverageDiff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
x86gen.hppLines 956-965 956 }
957
958 bool build_bmi_and(AndInst* inst) {
959 if (!__builtin_cpu_supports("bmi")) {
! 960 return false;
! 961 }
962 for (size_t it = 0; it < 2; it++) {
963 Value* a = inst->arg(it);
964 Value* b = inst->arg(1 - it);
965 if (dynmatch(XorInst, inversion, a)) {Lines 966-974 966 for (size_t index = 0; index < 2; index++) {
967 if (dynmatch(Const, ones, inversion->arg(index))) {
968 if (ones->value() == type_mask(inst->type())) {
969 if (inst->type() == Type::Int64) {
! 970 _builder.andn64(vreg(inst), vreg(inversion->arg(1 - index)), vreg(b));
971 } else {
972 _builder.andn32(vreg(inst), vreg(inversion->arg(1 - index)), vreg(b));
973 }
974 return true;Lines 989-1019 989 }
990 if (!shift || _use_counts.at(shift) != 1 || !_vregs.at(shift).is_invalid()) {
991 continue;
992 }
! 993 if (dynmatch(Const, amount, shift->arg(1))) {
! 994 size_t width = type_width(inst->type());
! 995 uint64_t start = amount->value();
! 996 if (start == 0 || start >= width) {
! 997 continue;
! 998 }
! 999 size_t length = __builtin_popcountll(bits);
! 1000 if (length > width - start) {
! 1001 if (arithmetic) {
! 1002 continue;
! 1003 }
! 1004 length = width - start;
! 1005 }
! 1006 Reg control = vreg();
! 1007 _builder.mov32_imm(control, start | (length << 8));
! 1008 if (inst->type() == Type::Int64) {
! 1009 _builder.bextr64(vreg(inst), control, vreg(shift->arg(0)));
! 1010 } else {
! 1011 _builder.bextr32(vreg(inst), control, vreg(shift->arg(0)));
! 1012 }
! 1013 return true;
! 1014 }
! 1015 }
1016 }
1017 return false;
1018 }x86insts.inc.hppLines 113-123 113 imm_binop_x86_inst(Add64Imm, add64_imm, imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
114 imm_binop_x86_inst(Sub64Imm, sub64_imm, imm_usedef, true, { reg = Reg::phys(5); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
115
116 ternop_x86_inst(Andn32, andn32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf2); modrm(); })
! 117 ternop_x86_inst(Andn64, andn64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf2); modrm(); })
! 118 ternop_x86_inst(Bextr32, bextr32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf7); modrm(); })
! 119 ternop_x86_inst(Bextr64, bextr64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf7); modrm(); })
120 ternop_x86_inst(Sarx32, sarx32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 2); byte(0xf7); modrm(); })
121 ternop_x86_inst(Sarx64, sarx64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 2); byte(0xf7); modrm(); })
122
123 binop_x86_inst(And32, and32, binop_usedef, false, { rex_opt(); byte(0x23); modrm(); })Lines 235-243 235
236 imm_binop_x86_inst(Test32Imm, test32_imm, cmp_imm_usedef, false, { reg = Reg::phys(0); rex_opt(); byte(0xf7); modrm(); imm_n(4); })
237 imm_binop_x86_inst(Test64Imm, test64_imm, cmp_imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(0xf7); modrm(); imm_n(4); })
238
! 239 #undef ternop_x86_inst
240 #undef x86_inst
241 #undef binop_x86_inst
242 #undef rev_binop_x86_inst
243 #undef imm_binop_x86_inst |
Coverage
Diff CoverageDiff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
x86gen.hppLines 1318-1332 1318 for (size_t it = 0; it < 2; it++) {
1319 Value* a = inst->arg(it);
1320 Value* b = inst->arg(1 - it);
1321 if (XorInst* inversion = is_not(a)) {
! 1322 if (inst->type() == Type::Int64) {
! 1323 _builder.andn64(vreg(inst), vreg(inversion->arg(0)), vreg(b));
! 1324 } else {
! 1325 _builder.andn32(vreg(inst), vreg(inversion->arg(0)), vreg(b));
! 1326 }
! 1327 return;
! 1328 }
1329 Inst* shift = dynamic_cast<ShrUInst*>(a);
1330 if (!shift) {
1331 shift = dynamic_cast<ShrSInst*>(a);
1332 }Lines 1332-1356 1332 }
1333 if (Const* mask = dynamic_cast<Const*>(b);
1334 mask && mask->value() && (mask->value() & (mask->value() + 1)) == 0 &&
1335 shift && _use_counts.at(shift) == 1 && _vregs.at(shift).is_invalid()) {
! 1336 if (dynmatch(Const, amount, shift->arg(1))) {
! 1337 size_t width = type_width(inst->type());
! 1338 size_t start = amount->value();
! 1339 size_t length = __builtin_popcountll(mask->value());
! 1340 if (start > 0 && start < width &&
! 1341 (dynamic_cast<ShrUInst*>(shift) || length <= width - start)) {
! 1342 Reg control = vreg();
! 1343 _builder.mov32_imm(control, start | (std::min(length, width - start) << 8));
! 1344 if (inst->type() == Type::Int64) {
! 1345 _builder.bextr64(vreg(inst), control, vreg(shift->arg(0)));
! 1346 } else {
! 1347 _builder.bextr32(vreg(inst), control, vreg(shift->arg(0)));
! 1348 }
! 1349 return;
! 1350 }
! 1351 }
! 1352 }
1353 }
1354 }
1355 auto [a, b] = commutative_args(inst);
1356 _builder.mov64(vreg(inst), vreg(a));x86insts.inc.hppLines 112-123 112
113 imm_binop_x86_inst(Add64Imm, add64_imm, imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
114 imm_binop_x86_inst(Sub64Imm, sub64_imm, imm_usedef, true, { reg = Reg::phys(5); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
115
! 116 ternop_x86_inst(Andn32, andn32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf2); modrm(); })
! 117 ternop_x86_inst(Andn64, andn64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf2); modrm(); })
! 118 ternop_x86_inst(Bextr32, bextr32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf7); modrm(); })
! 119 ternop_x86_inst(Bextr64, bextr64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf7); modrm(); })
120 ternop_x86_inst(Sarx32, sarx32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 2); byte(0xf7); modrm(); })
121 ternop_x86_inst(Sarx64, sarx64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 2); byte(0xf7); modrm(); })
122
123 binop_x86_inst(And32, and32, binop_usedef, false, { rex_opt(); byte(0x23); modrm(); })Lines 242-250 242
243 imm_binop_x86_inst(Test32Imm, test32_imm, cmp_imm_usedef, false, { reg = Reg::phys(0); rex_opt(); byte(0xf7); modrm(); imm_n(4); })
244 imm_binop_x86_inst(Test64Imm, test64_imm, cmp_imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(0xf7); modrm(); imm_n(4); })
245
! 246 #undef ternop_x86_inst
247 #undef x86_inst
248 #undef binop_x86_inst
249 #undef rev_binop_x86_inst
250 #undef imm_binop_x86_inst |
Coverage
Diff CoverageDiff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
x86gen.hppLines 1319-1333 1319 if (dynamic_cast<Const*>(a) || is_not(b)) {
1320 std::swap(a, b);
1321 }
1322 if (XorInst* inversion = is_not(a)) {
! 1323 if (inst->type() == Type::Int64) {
! 1324 _builder.andn64(vreg(inst), vreg(inversion->arg(0)), vreg(b));
! 1325 } else {
! 1326 _builder.andn32(vreg(inst), vreg(inversion->arg(0)), vreg(b));
! 1327 }
! 1328 return;
! 1329 }
1330 }
1331
1332 if (dynmatch(Const, constant_b, b)) {
1333 Inst* shift = dynamic_cast<ShrUInst*>(a);Lines 1336-1360 1336 }
1337 uint64_t mask = constant_b->value();
1338 if (__builtin_cpu_supports("bmi") && mask && (mask & (mask + 1)) == 0 &&
1339 shift && _use_counts.at(shift) == 1 && _vregs.at(shift).is_invalid()) {
! 1340 if (dynmatch(Const, amount, shift->arg(1))) {
! 1341 size_t width = type_width(inst->type());
! 1342 size_t start = amount->value();
! 1343 size_t length = __builtin_popcountll(mask);
! 1344 if (start > 0 && start < width &&
! 1345 (dynamic_cast<ShrUInst*>(shift) || length <= width - start)) {
! 1346 Reg control = vreg();
! 1347 _builder.mov32_imm(control, start | (std::min(length, width - start) << 8));
! 1348 if (inst->type() == Type::Int64) {
! 1349 _builder.bextr64(vreg(inst), control, vreg(shift->arg(0)));
! 1350 } else {
! 1351 _builder.bextr32(vreg(inst), control, vreg(shift->arg(0)));
! 1352 }
! 1353 return;
! 1354 }
! 1355 }
! 1356 }
1357 if (is_sext_imm32(constant_b)) {
1358 _builder.mov64(vreg(inst), vreg(a));
1359 if (type_size(inst->type()) == 8) {
1360 _builder.and64_imm(vreg(inst), sign_extend_imm(constant_b));x86insts.inc.hppLines 112-123 112
113 imm_binop_x86_inst(Add64Imm, add64_imm, imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
114 imm_binop_x86_inst(Sub64Imm, sub64_imm, imm_usedef, true, { reg = Reg::phys(5); rex_w(); byte(is_sext_imm8<int64_t>(imm.value()) ? 0x83 : 0x81); modrm(); imm_n(is_sext_imm8<int64_t>(imm.value()) ? 1 : 4); })
115
! 116 ternop_x86_inst(Andn32, andn32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf2); modrm(); })
! 117 ternop_x86_inst(Andn64, andn64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf2); modrm(); })
! 118 ternop_x86_inst(Bextr32, bextr32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 0); byte(0xf7); modrm(); })
! 119 ternop_x86_inst(Bextr64, bextr64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 0); byte(0xf7); modrm(); })
120 ternop_x86_inst(Sarx32, sarx32, { use(vvvv); use(rm); def(reg); }, false, { vex(false, 2); byte(0xf7); modrm(); })
121 ternop_x86_inst(Sarx64, sarx64, { use(vvvv); use(rm); def(reg); }, true, { vex(true, 2); byte(0xf7); modrm(); })
122
123 binop_x86_inst(And32, and32, binop_usedef, false, { rex_opt(); byte(0x23); modrm(); })Lines 242-250 242
243 imm_binop_x86_inst(Test32Imm, test32_imm, cmp_imm_usedef, false, { reg = Reg::phys(0); rex_opt(); byte(0xf7); modrm(); imm_n(4); })
244 imm_binop_x86_inst(Test64Imm, test64_imm, cmp_imm_usedef, true, { reg = Reg::phys(0); rex_w(); byte(0xf7); modrm(); imm_n(4); })
245
! 246 #undef ternop_x86_inst
247 #undef x86_inst
248 #undef binop_x86_inst
249 #undef rev_binop_x86_inst
250 #undef imm_binop_x86_inst |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
No description provided.