From 116edfe173d0c59ec2aa87fb91f2f31d477b61b3 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:22 +0100 Subject: [PATCH 001/331] tools/x86/kcpuid: Fix error handling Error handling in kcpuid is unreliable. On malloc() failures, the code prints an error then just goes on. The error messages are also printed to standard output instead of standard error. Use err() and errx() from to direct all error messages to standard error and automatically exit the program. Use err() to include the errno information, and errx() otherwise. Use warnx() for warnings. While at it, alphabetically reorder the header includes. [ mingo: Fix capitalization in the help text while at it. ] Fixes: c6b2f240bf8d ("tools/x86: Add a kcpuid tool to show raw CPU features") Reported-by: Remington Brasga Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-2-darwi@linutronix.de Closes: https://lkml.kernel.org/r/20240926223557.2048-1-rbrasga@uci.edu --- tools/arch/x86/kcpuid/kcpuid.c | 47 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 1b25c0a95d3f..40a9e59c2fd5 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #define _GNU_SOURCE -#include +#include +#include #include +#include #include #include -#include #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define min(a, b) (((a) < (b)) ? (a) : (b)) @@ -145,14 +146,14 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, if (!func->leafs) { func->leafs = malloc(sizeof(struct subleaf)); if (!func->leafs) - perror("malloc func leaf"); + err(EXIT_FAILURE, NULL); func->nr = 1; } else { s = func->nr; func->leafs = realloc(func->leafs, (s + 1) * sizeof(*leaf)); if (!func->leafs) - perror("realloc f->leafs"); + err(EXIT_FAILURE, NULL); func->nr++; } @@ -211,7 +212,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range = malloc(sizeof(struct cpuid_range)); if (!range) - perror("malloc range"); + err(EXIT_FAILURE, NULL); if (input_eax & 0x80000000) range->is_ext = true; @@ -220,7 +221,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); if (!range->funcs) - perror("malloc range->funcs"); + err(EXIT_FAILURE, NULL); range->nr = idx_func; memset(range->funcs, 0, sizeof(struct cpuid_func) * idx_func); @@ -395,8 +396,8 @@ static int parse_line(char *line) return 0; err_exit: - printf("Warning: wrong line format:\n"); - printf("\tline[%d]: %s\n", flines, line); + warnx("Wrong line format:\n" + "\tline[%d]: %s", flines, line); return -1; } @@ -418,10 +419,8 @@ static void parse_text(void) file = fopen("./cpuid.csv", "r"); } - if (!file) { - printf("Fail to open '%s'\n", filename); - return; - } + if (!file) + err(EXIT_FAILURE, "%s", filename); while (1) { ret = getline(&line, &len, file); @@ -530,7 +529,7 @@ static inline struct cpuid_func *index_to_func(u32 index) func_idx = index & 0xffff; if ((func_idx + 1) > (u32)range->nr) { - printf("ERR: invalid input index (0x%x)\n", index); + warnx("Invalid input index (0x%x)", index); return NULL; } return &range->funcs[func_idx]; @@ -562,7 +561,7 @@ static void show_info(void) return; } - printf("ERR: invalid input subleaf (0x%x)\n", user_sub); + warnx("Invalid input subleaf (0x%x)", user_sub); } show_func(func); @@ -593,15 +592,15 @@ static void setup_platform_cpuid(void) static void usage(void) { - printf("kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" - "\t-a|--all Show both bit flags and complex bit fields info\n" - "\t-b|--bitflags Show boolean flags only\n" - "\t-d|--detail Show details of the flag/fields (default)\n" - "\t-f|--flags Specify the cpuid csv file\n" - "\t-h|--help Show usage info\n" - "\t-l|--leaf=index Specify the leaf you want to check\n" - "\t-r|--raw Show raw cpuid data\n" - "\t-s|--subleaf=sub Specify the subleaf you want to check\n" + warnx("kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" + "\t-a|--all Show both bit flags and complex bit fields info\n" + "\t-b|--bitflags Show boolean flags only\n" + "\t-d|--detail Show details of the flag/fields (default)\n" + "\t-f|--flags Specify the CPUID CSV file\n" + "\t-h|--help Show usage info\n" + "\t-l|--leaf=index Specify the leaf you want to check\n" + "\t-r|--raw Show raw CPUID data\n" + "\t-s|--subleaf=sub Specify the subleaf you want to check" ); } @@ -652,7 +651,7 @@ static int parse_options(int argc, char *argv[]) user_sub = strtoul(optarg, NULL, 0); break; default: - printf("%s: Invalid option '%c'\n", argv[0], optopt); + warnx("Invalid option '%c'", optopt); return -1; } From a866a6775793a2554585073704b83c4691d80740 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:23 +0100 Subject: [PATCH 002/331] tools/x86/kcpuid: Exit the program on invalid parameters If the user passed an invalid CPUID index value through --leaf=index, kcpuid prints a warning, does nothing, then exits successfully. Transform the warning to an error, and exit the program with a proper error code. Similarly, if the user passed an invalid subleaf, kcpuid prints a warning, dumps the whole leaf, then exits successfully. Print a clear error message regarding the invalid subleaf and exit the program with the proper error code. Note, moving the "Invalid input index" message from index_to_func() to show_info() localizes error message handling to the latter, where it should be. It also allows index_to_func() to be refactored at further commits. Note, since after this commit and its parent kcpuid does not just "move on" on failures, remove the NULL parameter check plus silent exit at show_func() and show_leaf(). Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-3-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 40a9e59c2fd5..25b10feeb720 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -481,9 +481,6 @@ static void decode_bits(u32 value, struct reg_desc *rdesc, enum cpuid_reg reg) static void show_leaf(struct subleaf *leaf) { - if (!leaf) - return; - if (show_raw) { leaf_print_raw(leaf); } else { @@ -505,9 +502,6 @@ static void show_func(struct cpuid_func *func) { int i; - if (!func) - return; - for (i = 0; i < func->nr; i++) show_leaf(&func->leafs[i]); } @@ -528,10 +522,9 @@ static inline struct cpuid_func *index_to_func(u32 index) range = (index & 0x80000000) ? leafs_ext : leafs_basic; func_idx = index & 0xffff; - if ((func_idx + 1) > (u32)range->nr) { - warnx("Invalid input index (0x%x)", index); + if ((func_idx + 1) > (u32)range->nr) return NULL; - } + return &range->funcs[func_idx]; } @@ -550,18 +543,19 @@ static void show_info(void) /* Only show specific leaf/subleaf info */ func = index_to_func(user_index); if (!func) - return; + errx(EXIT_FAILURE, "Invalid input leaf (0x%x)", user_index); /* Dump the raw data also */ show_raw = true; if (user_sub != 0xFFFFFFFF) { - if (user_sub + 1 <= (u32)func->nr) { - show_leaf(&func->leafs[user_sub]); - return; + if (user_sub + 1 > (u32)func->nr) { + errx(EXIT_FAILURE, "Leaf 0x%x has no valid subleaf = 0x%x", + user_index, user_sub); } - warnx("Invalid input subleaf (0x%x)", user_sub); + show_leaf(&func->leafs[user_sub]); + return; } show_func(func); From 660c29fe53deeb3b3aef1d666ed3bde7608380bd Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:24 +0100 Subject: [PATCH 003/331] tools/x86/kcpuid: Simplify usage() handling Refactor usage() to accept an exit code parameter and exit the program after usage output. This streamlines its callers' code paths. Remove the "Invalid option" error message since getopt_long(3) already emits a similar message by default. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-4-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 37 +++++++++++++++------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 25b10feeb720..a90ac0b898a1 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -10,6 +10,7 @@ #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define min(a, b) (((a) < (b)) ? (a) : (b)) +#define __noreturn __attribute__((__noreturn__)) typedef unsigned int u32; typedef unsigned long long u64; @@ -584,17 +585,17 @@ static void setup_platform_cpuid(void) leafs_ext = setup_cpuid_range(0x80000000); } -static void usage(void) +static void __noreturn usage(int exit_code) { - warnx("kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" - "\t-a|--all Show both bit flags and complex bit fields info\n" - "\t-b|--bitflags Show boolean flags only\n" - "\t-d|--detail Show details of the flag/fields (default)\n" - "\t-f|--flags Specify the CPUID CSV file\n" - "\t-h|--help Show usage info\n" - "\t-l|--leaf=index Specify the leaf you want to check\n" - "\t-r|--raw Show raw CPUID data\n" - "\t-s|--subleaf=sub Specify the subleaf you want to check" + errx(exit_code, "kcpuid [-abdfhr] [-l leaf] [-s subleaf]\n" + "\t-a|--all Show both bit flags and complex bit fields info\n" + "\t-b|--bitflags Show boolean flags only\n" + "\t-d|--detail Show details of the flag/fields (default)\n" + "\t-f|--flags Specify the CPUID CSV file\n" + "\t-h|--help Show usage info\n" + "\t-l|--leaf=index Specify the leaf you want to check\n" + "\t-r|--raw Show raw CPUID data\n" + "\t-s|--subleaf=sub Specify the subleaf you want to check" ); } @@ -610,7 +611,7 @@ static struct option opts[] = { { NULL, 0, NULL, 0 } }; -static int parse_options(int argc, char *argv[]) +static void parse_options(int argc, char *argv[]) { int c; @@ -630,9 +631,7 @@ static int parse_options(int argc, char *argv[]) user_csv = optarg; break; case 'h': - usage(); - exit(1); - break; + usage(EXIT_SUCCESS); case 'l': /* main leaf */ user_index = strtoul(optarg, NULL, 0); @@ -645,11 +644,8 @@ static int parse_options(int argc, char *argv[]) user_sub = strtoul(optarg, NULL, 0); break; default: - warnx("Invalid option '%c'", optopt); - return -1; - } - - return 0; + usage(EXIT_FAILURE); + } } /* @@ -662,8 +658,7 @@ static int parse_options(int argc, char *argv[]) */ int main(int argc, char *argv[]) { - if (parse_options(argc, argv)) - return -1; + parse_options(argc, argv); /* Setup the cpuid leafs of current platform */ setup_platform_cpuid(); From 6bef74cab03ada39f16f1fb86b732f7c71a9f07a Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:25 +0100 Subject: [PATCH 004/331] tools/x86/kcpuid: Save CPUID output in an array For each CPUID leaf/subleaf query, save the output in an output[] array instead of spelling it out using EAX to EDX variables. This allows the CPUID output to be accessed programmatically instead of calling decode_bits() four times. Loop-based access also allows "kcpuid --detail" to print the correct output register names in next commit. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-5-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index a90ac0b898a1..8da0e072055b 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -51,7 +51,7 @@ static const char * const reg_names[] = { struct subleaf { u32 index; u32 sub; - u32 eax, ebx, ecx, edx; + u32 output[NR_REGS]; struct reg_desc info[NR_REGS]; }; @@ -119,11 +119,11 @@ static void leaf_print_raw(struct subleaf *leaf) if (leaf->sub == 0) printf("0x%08x: subleafs:\n", leaf->index); - printf(" %2d: EAX=0x%08x, EBX=0x%08x, ECX=0x%08x, EDX=0x%08x\n", - leaf->sub, leaf->eax, leaf->ebx, leaf->ecx, leaf->edx); + printf(" %2d: EAX=0x%08x, EBX=0x%08x, ECX=0x%08x, EDX=0x%08x\n", leaf->sub, + leaf->output[0], leaf->output[1], leaf->output[2], leaf->output[3]); } else { - printf("0x%08x: EAX=0x%08x, EBX=0x%08x, ECX=0x%08x, EDX=0x%08x\n", - leaf->index, leaf->eax, leaf->ebx, leaf->ecx, leaf->edx); + printf("0x%08x: EAX=0x%08x, EBX=0x%08x, ECX=0x%08x, EDX=0x%08x\n", leaf->index, + leaf->output[0], leaf->output[1], leaf->output[2], leaf->output[3]); } } @@ -163,10 +163,10 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, leaf->index = f; leaf->sub = subleaf; - leaf->eax = a; - leaf->ebx = b; - leaf->ecx = c; - leaf->edx = d; + leaf->output[R_EAX] = a; + leaf->output[R_EBX] = b; + leaf->output[R_ECX] = c; + leaf->output[R_EDX] = d; return false; } @@ -490,10 +490,8 @@ static void show_leaf(struct subleaf *leaf) leaf->index, leaf->sub); } - decode_bits(leaf->eax, &leaf->info[R_EAX], R_EAX); - decode_bits(leaf->ebx, &leaf->info[R_EBX], R_EBX); - decode_bits(leaf->ecx, &leaf->info[R_ECX], R_ECX); - decode_bits(leaf->edx, &leaf->info[R_EDX], R_EDX); + for (int i = R_EAX; i < NR_REGS; i++) + decode_bits(leaf->output[i], &leaf->info[i], i); if (!show_raw && show_details) printf("\n"); From ce61b6067d8cbfcd3607d942913b21c7ce2b384b Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:26 +0100 Subject: [PATCH 005/331] tools/x86/kcpuid: Print correct CPUID output register names kcpuid --all --detail claims that all bits belong to ECX, in the form of the header CPUID_${leaf}_ECX[${subleaf}]. Print the correct register name for all CPUID output. kcpuid --detail also dumps the raw register value if a leaf/subleaf is covered in the CSV file, but a certain output register within it is not covered by any CSV entry. Since register names are now properly printed, and since the CSV file has become exhaustive using x86-cpuid-db, remove that value dump as it pollutes the output. While at it, rename decode_bits() to show_reg(). This makes it match its show_range(), show_leaf() and show_reg_header() counterparts. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-6-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 8da0e072055b..aa0e03798326 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -436,20 +436,12 @@ static void parse_text(void) fclose(file); } - -/* Decode every eax/ebx/ecx/edx */ -static void decode_bits(u32 value, struct reg_desc *rdesc, enum cpuid_reg reg) +static void show_reg(const struct reg_desc *rdesc, u32 value) { - struct bits_desc *bdesc; + const struct bits_desc *bdesc; int start, end, i; u32 mask; - if (!rdesc->nr) { - if (show_details) - printf("\t %s: 0x%08x\n", reg_names[reg], value); - return; - } - for (i = 0; i < rdesc->nr; i++) { bdesc = &rdesc->descs[i]; @@ -480,18 +472,21 @@ static void decode_bits(u32 value, struct reg_desc *rdesc, enum cpuid_reg reg) } } +static void show_reg_header(bool has_entries, u32 leaf, u32 subleaf, const char *reg_name) +{ + if (show_details && has_entries) + printf("CPUID_0x%x_%s[0x%x]:\n", leaf, reg_name, subleaf); +} + static void show_leaf(struct subleaf *leaf) { - if (show_raw) { + if (show_raw) leaf_print_raw(leaf); - } else { - if (show_details) - printf("CPUID_0x%x_ECX[0x%x]:\n", - leaf->index, leaf->sub); - } - for (int i = R_EAX; i < NR_REGS; i++) - decode_bits(leaf->output[i], &leaf->info[i], i); + for (int i = R_EAX; i < NR_REGS; i++) { + show_reg_header((leaf->info[i].nr > 0), leaf->index, leaf->sub, reg_names[i]); + show_reg(&leaf->info[i], leaf->output[i]); + } if (!show_raw && show_details) printf("\n"); From c061ded035b5bdeb21adc1046c0bfbba4bcee2d2 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:27 +0100 Subject: [PATCH 006/331] tools/x86/kcpuid: Remove unused local variable The local variable "index" is written to, but is not read from anywhere. Remove it. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-7-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index aa0e03798326..f1dac5bee515 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -181,10 +181,6 @@ static void raw_dump_range(struct cpuid_range *range) for (f = 0; (int)f < range->nr; f++) { struct cpuid_func *func = &range->funcs[f]; - u32 index = f; - - if (range->is_ext) - index += 0x80000000; /* Skip leaf without valid items */ if (!func->nr) From 2b383ca0896f0ffc24fd4d93440320d2dd5daad1 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:28 +0100 Subject: [PATCH 007/331] tools/x86/kcpuid: Remove unused global variable The global variable "is_amd" is written to, but is not read from anywhere. Remove it. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-8-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index f1dac5bee515..f268c768d023 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -79,7 +79,6 @@ struct cpuid_range { */ struct cpuid_range *leafs_basic, *leafs_ext; -static bool is_amd; static bool show_details; static bool show_raw; static bool show_flags_only = true; @@ -559,16 +558,6 @@ static void show_info(void) static void setup_platform_cpuid(void) { - u32 eax, ebx, ecx, edx; - - /* Check vendor */ - eax = ebx = ecx = edx = 0; - cpuid(&eax, &ebx, &ecx, &edx); - - /* "htuA" */ - if (ebx == 0x68747541) - is_amd = true; - /* Setup leafs for the basic and extended range */ leafs_basic = setup_cpuid_range(0x0); leafs_ext = setup_cpuid_range(0x80000000); From 8984cea5c4743d7fabf5de9da142cfc5dde1a144 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:29 +0100 Subject: [PATCH 008/331] tools/x86/kcpuid: Set parse_line() return type to void parse_line() returns an integer but its caller ignored it. Change the function signature to return void. While at it, adjust some of the "Skip line" comments for readability. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-9-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index f268c768d023..4fa768bb49e1 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -277,7 +277,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) * 0, 0, EAX, 31:0, max_basic_leafs, Max input value for supported subleafs * 1, 0, ECX, 0, sse3, Streaming SIMD Extensions 3(SSE3) */ -static int parse_line(char *line) +static void parse_line(char *line) { char *str; int i; @@ -307,7 +307,7 @@ static int parse_line(char *line) /* Skip comments and NULL line */ if (line[0] == '#' || line[0] == '\n') - return 0; + return; strncpy(buffer, line, 511); buffer[511] = 0; @@ -330,16 +330,15 @@ static int parse_line(char *line) else range = leafs_basic; - index &= 0x7FFFFFFF; /* Skip line parsing for non-existing indexes */ + index &= 0x7FFFFFFF; if ((int)index >= range->nr) - return -1; + return; + /* Skip line parsing if the index CPUID output is all zero */ func = &range->funcs[index]; - - /* Return if the index has no valid item on this platform */ if (!func->nr) - return 0; + return; /* subleaf */ buf = tokens[1]; @@ -352,11 +351,11 @@ static int parse_line(char *line) subleaf_start = strtoul(start, NULL, 0); subleaf_end = min(subleaf_end, (u32)(func->nr - 1)); if (subleaf_start > subleaf_end) - return 0; + return; } else { subleaf_start = subleaf_end; if (subleaf_start > (u32)(func->nr - 1)) - return 0; + return; } /* register */ @@ -389,12 +388,11 @@ static int parse_line(char *line) strcpy(bdesc->simp, strtok(tokens[4], " \t")); strcpy(bdesc->detail, tokens[5]); } - return 0; + return; err_exit: warnx("Wrong line format:\n" "\tline[%d]: %s", flines, line); - return -1; } /* Parse csv file, and construct the array of all leafs and subleafs */ From 0a8f12ccd2e6edac89292af63c3a2050b4aac61b Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:30 +0100 Subject: [PATCH 009/331] tools/x86/kcpuid: Use C99-style for loops Since commit e8c07082a810 ("Kbuild: move to -std=gnu11") and the kernel allows C99-style variable declarations inside of a for() loop. Adjust the kcpuid code accordingly. Note, this helps readability as some of the kcpuid functions have a huge list of variable declarations on top. Note, remove the empty lines before cpuid() invocations as it is clearer to have their parameter initialization and the actual call in one block. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-10-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 52 ++++++++++++++-------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 4fa768bb49e1..413e5dad9e46 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -172,13 +172,10 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, static void raw_dump_range(struct cpuid_range *range) { - u32 f; - int i; - printf("%s Leafs :\n", range->is_ext ? "Extended" : "Basic"); printf("================\n"); - for (f = 0; (int)f < range->nr; f++) { + for (u32 f = 0; (int)f < range->nr; f++) { struct cpuid_func *func = &range->funcs[f]; /* Skip leaf without valid items */ @@ -186,7 +183,7 @@ static void raw_dump_range(struct cpuid_range *range) continue; /* First item is the main leaf, followed by all subleafs */ - for (i = 0; i < func->nr; i++) + for (int i = 0; i < func->nr; i++) leaf_print_raw(&func->leafs[i]); } } @@ -194,15 +191,14 @@ static void raw_dump_range(struct cpuid_range *range) #define MAX_SUBLEAF_NUM 64 struct cpuid_range *setup_cpuid_range(u32 input_eax) { - u32 max_func, idx_func, subleaf, max_subleaf; - u32 eax, ebx, ecx, edx, f = input_eax; struct cpuid_range *range; - bool allzero; + u32 max_func, idx_func; + u32 eax, ebx, ecx, edx; eax = input_eax; ebx = ecx = edx = 0; - cpuid(&eax, &ebx, &ecx, &edx); + max_func = eax; idx_func = (max_func & 0xffff) + 1; @@ -222,20 +218,21 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range->nr = idx_func; memset(range->funcs, 0, sizeof(struct cpuid_func) * idx_func); - for (; f <= max_func; f++) { - eax = f; - subleaf = ecx = 0; + for (u32 f = input_eax; f <= max_func; f++) { + u32 max_subleaf = MAX_SUBLEAF_NUM; + bool allzero; + eax = f; + ecx = 0; cpuid(&eax, &ebx, &ecx, &edx); - allzero = cpuid_store(range, f, subleaf, eax, ebx, ecx, edx); + + allzero = cpuid_store(range, f, 0, eax, ebx, ecx, edx); if (allzero) continue; if (!has_subleafs(f)) continue; - max_subleaf = MAX_SUBLEAF_NUM; - /* * Some can provide the exact number of subleafs, * others have to be tried (0xf) @@ -253,13 +250,12 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) if (f == 0x80000026) max_subleaf = 5; - for (subleaf = 1; subleaf < max_subleaf; subleaf++) { + for (u32 subleaf = 1; subleaf < max_subleaf; subleaf++) { eax = f; ecx = subleaf; - cpuid(&eax, &ebx, &ecx, &edx); - allzero = cpuid_store(range, f, subleaf, - eax, ebx, ecx, edx); + + allzero = cpuid_store(range, f, subleaf, eax, ebx, ecx, edx); if (allzero) continue; } @@ -280,12 +276,10 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) static void parse_line(char *line) { char *str; - int i; struct cpuid_range *range; struct cpuid_func *func; struct subleaf *leaf; u32 index; - u32 sub; char buffer[512]; char *buf; /* @@ -312,7 +306,7 @@ static void parse_line(char *line) strncpy(buffer, line, 511); buffer[511] = 0; str = buffer; - for (i = 0; i < 5; i++) { + for (int i = 0; i < 5; i++) { tokens[i] = strtok(str, ","); if (!tokens[i]) goto err_exit; @@ -378,7 +372,7 @@ static void parse_line(char *line) bit_end = strtoul(end, NULL, 0); bit_start = (start) ? strtoul(start, NULL, 0) : bit_end; - for (sub = subleaf_start; sub <= subleaf_end; sub++) { + for (u32 sub = subleaf_start; sub <= subleaf_end; sub++) { leaf = &func->leafs[sub]; reg = &leaf->info[reg_index]; bdesc = ®->descs[reg->nr++]; @@ -432,10 +426,10 @@ static void parse_text(void) static void show_reg(const struct reg_desc *rdesc, u32 value) { const struct bits_desc *bdesc; - int start, end, i; + int start, end; u32 mask; - for (i = 0; i < rdesc->nr; i++) { + for (int i = 0; i < rdesc->nr; i++) { bdesc = &rdesc->descs[i]; start = bdesc->start; @@ -487,17 +481,13 @@ static void show_leaf(struct subleaf *leaf) static void show_func(struct cpuid_func *func) { - int i; - - for (i = 0; i < func->nr; i++) + for (int i = 0; i < func->nr; i++) show_leaf(&func->leafs[i]); } static void show_range(struct cpuid_range *range) { - int i; - - for (i = 0; i < range->nr; i++) + for (int i = 0; i < range->nr; i++) show_func(&range->funcs[i]); } From c479a84488d10b3d4259186d80839f99e26b1706 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:31 +0100 Subject: [PATCH 010/331] tools/x86/kcpuid: Use intrinsics Use the __cpuid_count() intrinsic, provided by GCC and LLVM, instead of rolling a manual version. Both of the kernel's minimum required GCC version (5.1) and LLVM version (13.0.1) supports it, and it is heavily used across standard Linux user-space tooling. This also makes the CPUID call sites more readable. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-11-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 37 ++++++++++++++-------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 413e5dad9e46..25388a5d40ac 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #define _GNU_SOURCE +#include #include #include #include @@ -86,16 +87,16 @@ static u32 user_index = 0xFFFFFFFF; static u32 user_sub = 0xFFFFFFFF; static int flines; -static inline void cpuid(u32 *eax, u32 *ebx, u32 *ecx, u32 *edx) -{ - /* ecx is often an input as well as an output. */ - asm volatile("cpuid" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (*eax), "2" (*ecx)); -} +/* + * Force using __cpuid_count() instead of __cpuid(). The + * latter leaves ECX uninitialized, which can break CPUID queries. + */ + +#define cpuid(leaf, a, b, c, d) \ + __cpuid_count(leaf, 0, a, b, c, d) + +#define cpuid_count(leaf, subleaf, a, b, c, d) \ + __cpuid_count(leaf, subleaf, a, b, c, d) static inline bool has_subleafs(u32 f) { @@ -195,12 +196,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) u32 max_func, idx_func; u32 eax, ebx, ecx, edx; - eax = input_eax; - ebx = ecx = edx = 0; - cpuid(&eax, &ebx, &ecx, &edx); - - max_func = eax; - idx_func = (max_func & 0xffff) + 1; + cpuid(input_eax, max_func, ebx, ecx, edx); range = malloc(sizeof(struct cpuid_range)); if (!range) @@ -211,6 +207,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) else range->is_ext = false; + idx_func = (max_func & 0xffff) + 1; range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); if (!range->funcs) err(EXIT_FAILURE, NULL); @@ -222,9 +219,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) u32 max_subleaf = MAX_SUBLEAF_NUM; bool allzero; - eax = f; - ecx = 0; - cpuid(&eax, &ebx, &ecx, &edx); + cpuid(f, eax, ebx, ecx, edx); allzero = cpuid_store(range, f, 0, eax, ebx, ecx, edx); if (allzero) @@ -251,9 +246,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) max_subleaf = 5; for (u32 subleaf = 1; subleaf < max_subleaf; subleaf++) { - eax = f; - ecx = subleaf; - cpuid(&eax, &ebx, &ecx, &edx); + cpuid_count(f, subleaf, eax, ebx, ecx, edx); allzero = cpuid_store(range, f, subleaf, eax, ebx, ecx, edx); if (allzero) From 3151ec059dd1e71761f3beccc1e5f5c18fac4afa Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:32 +0100 Subject: [PATCH 011/331] tools/x86/kcpuid: Refactor CPUID range handling for future expansion The kcpuid code assumes only two CPUID index ranges, standard (0x0...) and extended (0x80000000...). Since additional CPUID index ranges will be added in further commits, replace the "is_ext" boolean with enumeration-based range classification. Collect all CPUID ranges in a structured array and introduce helper macros to iterate over it. Use such helpers throughout the code. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-12-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 100 +++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 25388a5d40ac..7790136f04dc 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -66,19 +66,50 @@ struct cpuid_func { int nr; }; +enum range_index { + RANGE_STD = 0, /* Standard */ + RANGE_EXT = 0x80000000, /* Extended */ +}; + +#define CPUID_INDEX_MASK 0x80000000 +#define CPUID_FUNCTION_MASK (~CPUID_INDEX_MASK) + struct cpuid_range { /* array of main leafs */ struct cpuid_func *funcs; /* number of valid leafs */ int nr; - bool is_ext; + enum range_index index; }; -/* - * basic: basic functions range: [0... ] - * ext: extended functions range: [0x80000000... ] - */ -struct cpuid_range *leafs_basic, *leafs_ext; +static struct cpuid_range ranges[] = { + { .index = RANGE_STD, }, + { .index = RANGE_EXT, }, +}; + +static char *range_to_str(struct cpuid_range *range) +{ + switch (range->index) { + case RANGE_STD: return "Standard"; + case RANGE_EXT: return "Extended"; + default: return NULL; + } +} + +#define for_each_cpuid_range(range) \ + for (unsigned int i = 0; i < ARRAY_SIZE(ranges) && ((range) = &ranges[i]); i++) + +struct cpuid_range *index_to_cpuid_range(u32 index) +{ + struct cpuid_range *range; + + for_each_cpuid_range(range) { + if (range->index == (index & CPUID_INDEX_MASK)) + return range; + } + + return NULL; +} static bool show_details; static bool show_raw; @@ -173,7 +204,7 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, static void raw_dump_range(struct cpuid_range *range) { - printf("%s Leafs :\n", range->is_ext ? "Extended" : "Basic"); + printf("%s Leafs :\n", range_to_str(range)); printf("================\n"); for (u32 f = 0; (int)f < range->nr; f++) { @@ -190,22 +221,12 @@ static void raw_dump_range(struct cpuid_range *range) } #define MAX_SUBLEAF_NUM 64 -struct cpuid_range *setup_cpuid_range(u32 input_eax) +void setup_cpuid_range(struct cpuid_range *range) { - struct cpuid_range *range; u32 max_func, idx_func; u32 eax, ebx, ecx, edx; - cpuid(input_eax, max_func, ebx, ecx, edx); - - range = malloc(sizeof(struct cpuid_range)); - if (!range) - err(EXIT_FAILURE, NULL); - - if (input_eax & 0x80000000) - range->is_ext = true; - else - range->is_ext = false; + cpuid(range->index, max_func, ebx, ecx, edx); idx_func = (max_func & 0xffff) + 1; range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); @@ -215,7 +236,7 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) range->nr = idx_func; memset(range->funcs, 0, sizeof(struct cpuid_func) * idx_func); - for (u32 f = input_eax; f <= max_func; f++) { + for (u32 f = range->index; f <= max_func; f++) { u32 max_subleaf = MAX_SUBLEAF_NUM; bool allzero; @@ -254,8 +275,6 @@ struct cpuid_range *setup_cpuid_range(u32 input_eax) } } - - return range; } /* @@ -312,13 +331,13 @@ static void parse_line(char *line) /* index/main-leaf */ index = strtoull(tokens[0], NULL, 0); - if (index & 0x80000000) - range = leafs_ext; - else - range = leafs_basic; + /* Skip line parsing if it's not covered by known ranges */ + range = index_to_cpuid_range(index); + if (!range) + return; /* Skip line parsing for non-existing indexes */ - index &= 0x7FFFFFFF; + index &= CPUID_FUNCTION_MASK; if ((int)index >= range->nr) return; @@ -489,9 +508,11 @@ static inline struct cpuid_func *index_to_func(u32 index) struct cpuid_range *range; u32 func_idx; - range = (index & 0x80000000) ? leafs_ext : leafs_basic; - func_idx = index & 0xffff; + range = index_to_cpuid_range(index); + if (!range) + return NULL; + func_idx = index & 0xffff; if ((func_idx + 1) > (u32)range->nr) return NULL; @@ -500,12 +521,13 @@ static inline struct cpuid_func *index_to_func(u32 index) static void show_info(void) { + struct cpuid_range *range; struct cpuid_func *func; if (show_raw) { /* Show all of the raw output of 'cpuid' instr */ - raw_dump_range(leafs_basic); - raw_dump_range(leafs_ext); + for_each_cpuid_range(range) + raw_dump_range(range); return; } @@ -533,15 +555,8 @@ static void show_info(void) } printf("CPU features:\n=============\n\n"); - show_range(leafs_basic); - show_range(leafs_ext); -} - -static void setup_platform_cpuid(void) -{ - /* Setup leafs for the basic and extended range */ - leafs_basic = setup_cpuid_range(0x0); - leafs_ext = setup_cpuid_range(0x80000000); + for_each_cpuid_range(range) + show_range(range); } static void __noreturn usage(int exit_code) @@ -617,10 +632,13 @@ static void parse_options(int argc, char *argv[]) */ int main(int argc, char *argv[]) { + struct cpuid_range *range; + parse_options(argc, argv); /* Setup the cpuid leafs of current platform */ - setup_platform_cpuid(); + for_each_cpuid_range(range) + setup_cpuid_range(range); /* Read and parse the 'cpuid.csv' */ parse_text(); From f2e2efe9489d883fdaac8b7b46bd669b6214b1cb Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:33 +0100 Subject: [PATCH 012/331] tools/x86/kcpuid: Extend CPUID index mask macro Extend the CPUID index mask macro from 0x80000000 to 0xffff0000. This accommodates the Transmeta (0x80860000) and Centaur (0xc0000000) index ranges which will be later added. This also automatically sets CPUID_FUNCTION_MASK to 0x0000ffff, which is the actual correct value. Use that macro, instead of the 0xffff literal where appropriate. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-13-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 7790136f04dc..1f48de343d4d 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -71,7 +71,7 @@ enum range_index { RANGE_EXT = 0x80000000, /* Extended */ }; -#define CPUID_INDEX_MASK 0x80000000 +#define CPUID_INDEX_MASK 0xffff0000 #define CPUID_FUNCTION_MASK (~CPUID_INDEX_MASK) struct cpuid_range { @@ -173,7 +173,7 @@ static bool cpuid_store(struct cpuid_range *range, u32 f, int subleaf, * Cut off vendor-prefix from CPUID function as we're using it as an * index into ->funcs. */ - func = &range->funcs[f & 0xffff]; + func = &range->funcs[f & CPUID_FUNCTION_MASK]; if (!func->leafs) { func->leafs = malloc(sizeof(struct subleaf)); @@ -228,7 +228,7 @@ void setup_cpuid_range(struct cpuid_range *range) cpuid(range->index, max_func, ebx, ecx, edx); - idx_func = (max_func & 0xffff) + 1; + idx_func = (max_func & CPUID_FUNCTION_MASK) + 1; range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); if (!range->funcs) err(EXIT_FAILURE, NULL); @@ -512,7 +512,7 @@ static inline struct cpuid_func *index_to_func(u32 index) if (!range) return NULL; - func_idx = index & 0xffff; + func_idx = index & CPUID_FUNCTION_MASK; if ((func_idx + 1) > (u32)range->nr) return NULL; From 74d29127f83042500c20c903dd67151dbdd86ec8 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:34 +0100 Subject: [PATCH 013/331] tools/x86/kcpuid: Consolidate index validity checks Let index_to_cpuid_range() return a CPUID range only if the passed index is within a CPUID range's maximum supported function on the CPU. Returning a CPUID range that is invalid on the CPU for the passed index does not make sense. This also avoids repeating the "function index is within CPUID range" checks, both at setup_cpuid_range() and index_to_func(). Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-14-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index 1f48de343d4d..ac3d36bef85f 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -101,10 +101,12 @@ static char *range_to_str(struct cpuid_range *range) struct cpuid_range *index_to_cpuid_range(u32 index) { + u32 func_idx = index & CPUID_FUNCTION_MASK; + u32 range_idx = index & CPUID_INDEX_MASK; struct cpuid_range *range; for_each_cpuid_range(range) { - if (range->index == (index & CPUID_INDEX_MASK)) + if (range->index == range_idx && (u32)range->nr > func_idx) return range; } @@ -331,17 +333,16 @@ static void parse_line(char *line) /* index/main-leaf */ index = strtoull(tokens[0], NULL, 0); - /* Skip line parsing if it's not covered by known ranges */ + /* + * Skip line parsing if the index is not covered by known-valid + * CPUID ranges on this CPU. + */ range = index_to_cpuid_range(index); if (!range) return; - /* Skip line parsing for non-existing indexes */ - index &= CPUID_FUNCTION_MASK; - if ((int)index >= range->nr) - return; - /* Skip line parsing if the index CPUID output is all zero */ + index &= CPUID_FUNCTION_MASK; func = &range->funcs[index]; if (!func->nr) return; @@ -505,17 +506,13 @@ static void show_range(struct cpuid_range *range) static inline struct cpuid_func *index_to_func(u32 index) { + u32 func_idx = index & CPUID_FUNCTION_MASK; struct cpuid_range *range; - u32 func_idx; range = index_to_cpuid_range(index); if (!range) return NULL; - func_idx = index & CPUID_FUNCTION_MASK; - if ((func_idx + 1) > (u32)range->nr) - return NULL; - return &range->funcs[func_idx]; } From 72383c8274edf0a1736973462603dcfd0a088bb9 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:35 +0100 Subject: [PATCH 014/331] tools/x86/kcpuid: Filter valid CPUID ranges Next commits will introduce vendor-specific CPUID ranges like Transmeta's 0x8086000 range and Centaur's 0xc0000000. Initially explicit vendor detection was implemented, but it turned out to be not strictly necessary. As Dave Hansen noted, even established tools like cpuid(1) just tries all ranges indices, and see if the CPU responds back with something sensible. Do something similar at setup_cpuid_range(). Query the range's index, and check the maximum range function value returned. If it's within an expected interval of [range_index, range_index + MAX_RANGE_INDEX_OFFSET], accept the range as valid and further query its leaves. Set MAX_RANGE_INDEX_OFFSET to a heuristic of 0xff. That should be sensible enough since all the ranges covered by x86-cpuid-db XML database are: 0x00000000 0x00000023 0x40000000 0x40000000 0x80000000 0x80000026 0x80860000 0x80860007 0xc0000000 0xc0000001 At setup_cpuid_range(), if the range's returned maximum function was not sane, mark it as invalid by setting its number of leaves, range->nr, to zero. Introduce the for_each_valid_cpuid_range() iterator instead of sprinkling "range->nr != 0" checks throughout the code. Suggested-by: Dave Hansen Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-15-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 37 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index ac3d36bef85f..fe3d0581afd1 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -96,8 +96,13 @@ static char *range_to_str(struct cpuid_range *range) } } -#define for_each_cpuid_range(range) \ - for (unsigned int i = 0; i < ARRAY_SIZE(ranges) && ((range) = &ranges[i]); i++) +#define __for_each_cpuid_range(range, __condition) \ + for (unsigned int i = 0; \ + i < ARRAY_SIZE(ranges) && ((range) = &ranges[i]) && (__condition); \ + i++) + +#define for_each_valid_cpuid_range(range) __for_each_cpuid_range(range, (range)->nr != 0) +#define for_each_cpuid_range(range) __for_each_cpuid_range(range, true) struct cpuid_range *index_to_cpuid_range(u32 index) { @@ -105,7 +110,7 @@ struct cpuid_range *index_to_cpuid_range(u32 index) u32 range_idx = index & CPUID_INDEX_MASK; struct cpuid_range *range; - for_each_cpuid_range(range) { + for_each_valid_cpuid_range(range) { if (range->index == range_idx && (u32)range->nr > func_idx) return range; } @@ -223,20 +228,32 @@ static void raw_dump_range(struct cpuid_range *range) } #define MAX_SUBLEAF_NUM 64 +#define MAX_RANGE_INDEX_OFFSET 0xff void setup_cpuid_range(struct cpuid_range *range) { - u32 max_func, idx_func; + u32 max_func, range_funcs_sz; u32 eax, ebx, ecx, edx; cpuid(range->index, max_func, ebx, ecx, edx); - idx_func = (max_func & CPUID_FUNCTION_MASK) + 1; - range->funcs = malloc(sizeof(struct cpuid_func) * idx_func); + /* + * If the CPUID range's maximum function value is garbage, then it + * is not recognized by this CPU. Set the range's number of valid + * leaves to zero so that for_each_valid_cpu_range() can ignore it. + */ + if (max_func < range->index || max_func > (range->index + MAX_RANGE_INDEX_OFFSET)) { + range->nr = 0; + return; + } + + range->nr = (max_func & CPUID_FUNCTION_MASK) + 1; + range_funcs_sz = range->nr * sizeof(struct cpuid_func); + + range->funcs = malloc(range_funcs_sz); if (!range->funcs) err(EXIT_FAILURE, NULL); - range->nr = idx_func; - memset(range->funcs, 0, sizeof(struct cpuid_func) * idx_func); + memset(range->funcs, 0, range_funcs_sz); for (u32 f = range->index; f <= max_func; f++) { u32 max_subleaf = MAX_SUBLEAF_NUM; @@ -523,7 +540,7 @@ static void show_info(void) if (show_raw) { /* Show all of the raw output of 'cpuid' instr */ - for_each_cpuid_range(range) + for_each_valid_cpuid_range(range) raw_dump_range(range); return; } @@ -552,7 +569,7 @@ static void show_info(void) } printf("CPU features:\n=============\n\n"); - for_each_cpuid_range(range) + for_each_valid_cpuid_range(range) show_range(range); } From 87669e74d8481a066eb8d319cef7b2ea93d4e24b Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:36 +0100 Subject: [PATCH 015/331] tools/x86/kcpuid: Define Transmeta and Centaur index ranges Explicitly define the CPUID index ranges for Transmeta (0x80860000) and Centaur/Zhaoxin (0xc0000000). Without these explicit definitions, their respective CPUID indices would be skipped during CSV bitfield parsing. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-16-darwi@linutronix.de --- tools/arch/x86/kcpuid/kcpuid.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/arch/x86/kcpuid/kcpuid.c b/tools/arch/x86/kcpuid/kcpuid.c index fe3d0581afd1..7dc6b9235d02 100644 --- a/tools/arch/x86/kcpuid/kcpuid.c +++ b/tools/arch/x86/kcpuid/kcpuid.c @@ -69,6 +69,8 @@ struct cpuid_func { enum range_index { RANGE_STD = 0, /* Standard */ RANGE_EXT = 0x80000000, /* Extended */ + RANGE_TSM = 0x80860000, /* Transmeta */ + RANGE_CTR = 0xc0000000, /* Centaur/Zhaoxin */ }; #define CPUID_INDEX_MASK 0xffff0000 @@ -85,6 +87,8 @@ struct cpuid_range { static struct cpuid_range ranges[] = { { .index = RANGE_STD, }, { .index = RANGE_EXT, }, + { .index = RANGE_TSM, }, + { .index = RANGE_CTR, }, }; static char *range_to_str(struct cpuid_range *range) @@ -92,6 +96,8 @@ static char *range_to_str(struct cpuid_range *range) switch (range->index) { case RANGE_STD: return "Standard"; case RANGE_EXT: return "Extended"; + case RANGE_TSM: return "Transmeta"; + case RANGE_CTR: return "Centaur"; default: return NULL; } } From e1dde2f5a4ef4a0e91a67adebb7037bbfbcb0c50 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:37 +0100 Subject: [PATCH 016/331] tools/x86/kcpuid: Update bitfields to x86-cpuid-db v2.0 Update kcpuid's CSV file to version v2.0, as generated by x86-cpuid-db. Summary of the v2.0 changes: * Introduce the leaves: - Leaf 0x00000003, Transmeta Processor serial number - Leaf 0x80860000, Transmeta max leaf number + CPU vendor ID - Leaf 0x80860001, Transmeta extended CPU information - Leaf 0x80860002, Transmeta Code Morphing Software (CMS) enumeration - Leaf 0x80860003 => 0x80860006, Transmeta CPU information string - Leaf 0x80860007, Transmeta "live" CPU information - Leaf 0xc0000000, Centaur/Zhaoxin's max leaf number - Leaf 0xc0000001, Centaur/Zhaoxin's extended CPU features * Add a 0x prefix for leaves 0x0 to 0x9. This maintains consistency with the rest of the CSV entries. * Add the new bitfields: - Leaf 0x7: nmi_src, NMI-source reporting - Leaf 0x80000001: e_base_type and e_mmx (Transmeta) * Update the section headers for leaves 0x80000000 and 0x80000005 to indicate that they are also valid for Transmeta. Notes: Leaf 0x3, being not unique to Transmeta, is handled at the generated CSV file v2.3 update, later in this patch queue. Leaf 0x80000001 EDX:23 bit, e_mmx, is also available on AMD. A bugfix is already merged at x86-cpuid-db's -tip for that, and it will be part of the project's upcoming v2.4 release.: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/commit/65fff25daa41 Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/blob/v2.0/CHANGELOG.rst Link: https://lore.kernel.org/r/20250324142042.29010-17-darwi@linutronix.de --- tools/arch/x86/kcpuid/cpuid.csv | 648 +++++++++++++++++++------------- 1 file changed, 382 insertions(+), 266 deletions(-) diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index d751eb8585d0..d0f7159f99ba 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v1.0 +# Generator: x86-cpuid-db v2.0 # # Auto-generated file. @@ -12,297 +12,306 @@ # Leaf 0H # Maximum standard leaf number + CPU vendor string - 0, 0, eax, 31:0, max_std_leaf , Highest cpuid standard leaf supported - 0, 0, ebx, 31:0, cpu_vendorid_0 , CPU vendor ID string bytes 0 - 3 - 0, 0, ecx, 31:0, cpu_vendorid_2 , CPU vendor ID string bytes 8 - 11 - 0, 0, edx, 31:0, cpu_vendorid_1 , CPU vendor ID string bytes 4 - 7 + 0x0, 0, eax, 31:0, max_std_leaf , Highest cpuid standard leaf supported + 0x0, 0, ebx, 31:0, cpu_vendorid_0 , CPU vendor ID string bytes 0 - 3 + 0x0, 0, ecx, 31:0, cpu_vendorid_2 , CPU vendor ID string bytes 8 - 11 + 0x0, 0, edx, 31:0, cpu_vendorid_1 , CPU vendor ID string bytes 4 - 7 # Leaf 1H # CPU FMS (Family/Model/Stepping) + standard feature flags - 1, 0, eax, 3:0, stepping , Stepping ID - 1, 0, eax, 7:4, base_model , Base CPU model ID - 1, 0, eax, 11:8, base_family_id , Base CPU family ID - 1, 0, eax, 13:12, cpu_type , CPU type - 1, 0, eax, 19:16, ext_model , Extended CPU model ID - 1, 0, eax, 27:20, ext_family , Extended CPU family ID - 1, 0, ebx, 7:0, brand_id , Brand index - 1, 0, ebx, 15:8, clflush_size , CLFLUSH instruction cache line size - 1, 0, ebx, 23:16, n_logical_cpu , Logical CPU (HW threads) count - 1, 0, ebx, 31:24, local_apic_id , Initial local APIC physical ID - 1, 0, ecx, 0, pni , Streaming SIMD Extensions 3 (SSE3) - 1, 0, ecx, 1, pclmulqdq , PCLMULQDQ instruction support - 1, 0, ecx, 2, dtes64 , 64-bit DS save area - 1, 0, ecx, 3, monitor , MONITOR/MWAIT support - 1, 0, ecx, 4, ds_cpl , CPL Qualified Debug Store - 1, 0, ecx, 5, vmx , Virtual Machine Extensions - 1, 0, ecx, 6, smx , Safer Mode Extensions - 1, 0, ecx, 7, est , Enhanced Intel SpeedStep - 1, 0, ecx, 8, tm2 , Thermal Monitor 2 - 1, 0, ecx, 9, ssse3 , Supplemental SSE3 - 1, 0, ecx, 10, cid , L1 Context ID - 1, 0, ecx, 11, sdbg , Sillicon Debug - 1, 0, ecx, 12, fma , FMA extensions using YMM state - 1, 0, ecx, 13, cx16 , CMPXCHG16B instruction support - 1, 0, ecx, 14, xtpr , xTPR Update Control - 1, 0, ecx, 15, pdcm , Perfmon and Debug Capability - 1, 0, ecx, 17, pcid , Process-context identifiers - 1, 0, ecx, 18, dca , Direct Cache Access - 1, 0, ecx, 19, sse4_1 , SSE4.1 - 1, 0, ecx, 20, sse4_2 , SSE4.2 - 1, 0, ecx, 21, x2apic , X2APIC support - 1, 0, ecx, 22, movbe , MOVBE instruction support - 1, 0, ecx, 23, popcnt , POPCNT instruction support - 1, 0, ecx, 24, tsc_deadline_timer , APIC timer one-shot operation - 1, 0, ecx, 25, aes , AES instructions - 1, 0, ecx, 26, xsave , XSAVE (and related instructions) support - 1, 0, ecx, 27, osxsave , XSAVE (and related instructions) are enabled by OS - 1, 0, ecx, 28, avx , AVX instructions support - 1, 0, ecx, 29, f16c , Half-precision floating-point conversion support - 1, 0, ecx, 30, rdrand , RDRAND instruction support - 1, 0, ecx, 31, guest_status , System is running as guest; (para-)virtualized system - 1, 0, edx, 0, fpu , Floating-Point Unit on-chip (x87) - 1, 0, edx, 1, vme , Virtual-8086 Mode Extensions - 1, 0, edx, 2, de , Debugging Extensions - 1, 0, edx, 3, pse , Page Size Extension - 1, 0, edx, 4, tsc , Time Stamp Counter - 1, 0, edx, 5, msr , Model-Specific Registers (RDMSR and WRMSR support) - 1, 0, edx, 6, pae , Physical Address Extensions - 1, 0, edx, 7, mce , Machine Check Exception - 1, 0, edx, 8, cx8 , CMPXCHG8B instruction - 1, 0, edx, 9, apic , APIC on-chip - 1, 0, edx, 11, sep , SYSENTER, SYSEXIT, and associated MSRs - 1, 0, edx, 12, mtrr , Memory Type Range Registers - 1, 0, edx, 13, pge , Page Global Extensions - 1, 0, edx, 14, mca , Machine Check Architecture - 1, 0, edx, 15, cmov , Conditional Move Instruction - 1, 0, edx, 16, pat , Page Attribute Table - 1, 0, edx, 17, pse36 , Page Size Extension (36-bit) - 1, 0, edx, 18, pn , Processor Serial Number - 1, 0, edx, 19, clflush , CLFLUSH instruction - 1, 0, edx, 21, dts , Debug Store - 1, 0, edx, 22, acpi , Thermal monitor and clock control - 1, 0, edx, 23, mmx , MMX instructions - 1, 0, edx, 24, fxsr , FXSAVE and FXRSTOR instructions - 1, 0, edx, 25, sse , SSE instructions - 1, 0, edx, 26, sse2 , SSE2 instructions - 1, 0, edx, 27, ss , Self Snoop - 1, 0, edx, 28, ht , Hyper-threading - 1, 0, edx, 29, tm , Thermal Monitor - 1, 0, edx, 30, ia64 , Legacy IA-64 (Itanium) support bit, now resreved - 1, 0, edx, 31, pbe , Pending Break Enable + 0x1, 0, eax, 3:0, stepping , Stepping ID + 0x1, 0, eax, 7:4, base_model , Base CPU model ID + 0x1, 0, eax, 11:8, base_family_id , Base CPU family ID + 0x1, 0, eax, 13:12, cpu_type , CPU type + 0x1, 0, eax, 19:16, ext_model , Extended CPU model ID + 0x1, 0, eax, 27:20, ext_family , Extended CPU family ID + 0x1, 0, ebx, 7:0, brand_id , Brand index + 0x1, 0, ebx, 15:8, clflush_size , CLFLUSH instruction cache line size + 0x1, 0, ebx, 23:16, n_logical_cpu , Logical CPU (HW threads) count + 0x1, 0, ebx, 31:24, local_apic_id , Initial local APIC physical ID + 0x1, 0, ecx, 0, pni , Streaming SIMD Extensions 3 (SSE3) + 0x1, 0, ecx, 1, pclmulqdq , PCLMULQDQ instruction support + 0x1, 0, ecx, 2, dtes64 , 64-bit DS save area + 0x1, 0, ecx, 3, monitor , MONITOR/MWAIT support + 0x1, 0, ecx, 4, ds_cpl , CPL Qualified Debug Store + 0x1, 0, ecx, 5, vmx , Virtual Machine Extensions + 0x1, 0, ecx, 6, smx , Safer Mode Extensions + 0x1, 0, ecx, 7, est , Enhanced Intel SpeedStep + 0x1, 0, ecx, 8, tm2 , Thermal Monitor 2 + 0x1, 0, ecx, 9, ssse3 , Supplemental SSE3 + 0x1, 0, ecx, 10, cid , L1 Context ID + 0x1, 0, ecx, 11, sdbg , Sillicon Debug + 0x1, 0, ecx, 12, fma , FMA extensions using YMM state + 0x1, 0, ecx, 13, cx16 , CMPXCHG16B instruction support + 0x1, 0, ecx, 14, xtpr , xTPR Update Control + 0x1, 0, ecx, 15, pdcm , Perfmon and Debug Capability + 0x1, 0, ecx, 17, pcid , Process-context identifiers + 0x1, 0, ecx, 18, dca , Direct Cache Access + 0x1, 0, ecx, 19, sse4_1 , SSE4.1 + 0x1, 0, ecx, 20, sse4_2 , SSE4.2 + 0x1, 0, ecx, 21, x2apic , X2APIC support + 0x1, 0, ecx, 22, movbe , MOVBE instruction support + 0x1, 0, ecx, 23, popcnt , POPCNT instruction support + 0x1, 0, ecx, 24, tsc_deadline_timer , APIC timer one-shot operation + 0x1, 0, ecx, 25, aes , AES instructions + 0x1, 0, ecx, 26, xsave , XSAVE (and related instructions) support + 0x1, 0, ecx, 27, osxsave , XSAVE (and related instructions) are enabled by OS + 0x1, 0, ecx, 28, avx , AVX instructions support + 0x1, 0, ecx, 29, f16c , Half-precision floating-point conversion support + 0x1, 0, ecx, 30, rdrand , RDRAND instruction support + 0x1, 0, ecx, 31, guest_status , System is running as guest; (para-)virtualized system + 0x1, 0, edx, 0, fpu , Floating-Point Unit on-chip (x87) + 0x1, 0, edx, 1, vme , Virtual-8086 Mode Extensions + 0x1, 0, edx, 2, de , Debugging Extensions + 0x1, 0, edx, 3, pse , Page Size Extension + 0x1, 0, edx, 4, tsc , Time Stamp Counter + 0x1, 0, edx, 5, msr , Model-Specific Registers (RDMSR and WRMSR support) + 0x1, 0, edx, 6, pae , Physical Address Extensions + 0x1, 0, edx, 7, mce , Machine Check Exception + 0x1, 0, edx, 8, cx8 , CMPXCHG8B instruction + 0x1, 0, edx, 9, apic , APIC on-chip + 0x1, 0, edx, 11, sep , SYSENTER, SYSEXIT, and associated MSRs + 0x1, 0, edx, 12, mtrr , Memory Type Range Registers + 0x1, 0, edx, 13, pge , Page Global Extensions + 0x1, 0, edx, 14, mca , Machine Check Architecture + 0x1, 0, edx, 15, cmov , Conditional Move Instruction + 0x1, 0, edx, 16, pat , Page Attribute Table + 0x1, 0, edx, 17, pse36 , Page Size Extension (36-bit) + 0x1, 0, edx, 18, pn , Processor Serial Number + 0x1, 0, edx, 19, clflush , CLFLUSH instruction + 0x1, 0, edx, 21, dts , Debug Store + 0x1, 0, edx, 22, acpi , Thermal monitor and clock control + 0x1, 0, edx, 23, mmx , MMX instructions + 0x1, 0, edx, 24, fxsr , FXSAVE and FXRSTOR instructions + 0x1, 0, edx, 25, sse , SSE instructions + 0x1, 0, edx, 26, sse2 , SSE2 instructions + 0x1, 0, edx, 27, ss , Self Snoop + 0x1, 0, edx, 28, ht , Hyper-threading + 0x1, 0, edx, 29, tm , Thermal Monitor + 0x1, 0, edx, 30, ia64 , Legacy IA-64 (Itanium) support bit, now resreved + 0x1, 0, edx, 31, pbe , Pending Break Enable # Leaf 2H # Intel cache and TLB information one-byte descriptors - 2, 0, eax, 7:0, iteration_count , Number of times this CPUD leaf must be queried - 2, 0, eax, 15:8, desc1 , Descriptor #1 - 2, 0, eax, 23:16, desc2 , Descriptor #2 - 2, 0, eax, 30:24, desc3 , Descriptor #3 - 2, 0, eax, 31, eax_invalid , Descriptors 1-3 are invalid if set - 2, 0, ebx, 7:0, desc4 , Descriptor #4 - 2, 0, ebx, 15:8, desc5 , Descriptor #5 - 2, 0, ebx, 23:16, desc6 , Descriptor #6 - 2, 0, ebx, 30:24, desc7 , Descriptor #7 - 2, 0, ebx, 31, ebx_invalid , Descriptors 4-7 are invalid if set - 2, 0, ecx, 7:0, desc8 , Descriptor #8 - 2, 0, ecx, 15:8, desc9 , Descriptor #9 - 2, 0, ecx, 23:16, desc10 , Descriptor #10 - 2, 0, ecx, 30:24, desc11 , Descriptor #11 - 2, 0, ecx, 31, ecx_invalid , Descriptors 8-11 are invalid if set - 2, 0, edx, 7:0, desc12 , Descriptor #12 - 2, 0, edx, 15:8, desc13 , Descriptor #13 - 2, 0, edx, 23:16, desc14 , Descriptor #14 - 2, 0, edx, 30:24, desc15 , Descriptor #15 - 2, 0, edx, 31, edx_invalid , Descriptors 12-15 are invalid if set + 0x2, 0, eax, 7:0, iteration_count , Number of times this CPUD leaf must be queried + 0x2, 0, eax, 15:8, desc1 , Descriptor #1 + 0x2, 0, eax, 23:16, desc2 , Descriptor #2 + 0x2, 0, eax, 30:24, desc3 , Descriptor #3 + 0x2, 0, eax, 31, eax_invalid , Descriptors 1-3 are invalid if set + 0x2, 0, ebx, 7:0, desc4 , Descriptor #4 + 0x2, 0, ebx, 15:8, desc5 , Descriptor #5 + 0x2, 0, ebx, 23:16, desc6 , Descriptor #6 + 0x2, 0, ebx, 30:24, desc7 , Descriptor #7 + 0x2, 0, ebx, 31, ebx_invalid , Descriptors 4-7 are invalid if set + 0x2, 0, ecx, 7:0, desc8 , Descriptor #8 + 0x2, 0, ecx, 15:8, desc9 , Descriptor #9 + 0x2, 0, ecx, 23:16, desc10 , Descriptor #10 + 0x2, 0, ecx, 30:24, desc11 , Descriptor #11 + 0x2, 0, ecx, 31, ecx_invalid , Descriptors 8-11 are invalid if set + 0x2, 0, edx, 7:0, desc12 , Descriptor #12 + 0x2, 0, edx, 15:8, desc13 , Descriptor #13 + 0x2, 0, edx, 23:16, desc14 , Descriptor #14 + 0x2, 0, edx, 30:24, desc15 , Descriptor #15 + 0x2, 0, edx, 31, edx_invalid , Descriptors 12-15 are invalid if set + +# Leaf 3H +# Transmeta Processor Serial Number (PSN) + + 0x3, 0, eax, 31:0, cpu_psn_0 , Processor Serial Number bytes 0 - 3 + 0x3, 0, ebx, 31:0, cpu_psn_1 , Processor Serial Number bytes 4 - 7 + 0x3, 0, ecx, 31:0, cpu_psn_2 , Processor Serial Number bytes 8 - 11 + 0x3, 0, edx, 31:0, cpu_psn_3 , Processor Serial Number bytes 12 - 15 # Leaf 4H # Intel deterministic cache parameters - 4, 31:0, eax, 4:0, cache_type , Cache type field - 4, 31:0, eax, 7:5, cache_level , Cache level (1-based) - 4, 31:0, eax, 8, cache_self_init , Self-initialializing cache level - 4, 31:0, eax, 9, fully_associative , Fully-associative cache - 4, 31:0, eax, 25:14, num_threads_sharing , Number logical CPUs sharing this cache - 4, 31:0, eax, 31:26, num_cores_on_die , Number of cores in the physical package - 4, 31:0, ebx, 11:0, cache_linesize , System coherency line size (0-based) - 4, 31:0, ebx, 21:12, cache_npartitions , Physical line partitions (0-based) - 4, 31:0, ebx, 31:22, cache_nways , Ways of associativity (0-based) - 4, 31:0, ecx, 30:0, cache_nsets , Cache number of sets (0-based) - 4, 31:0, edx, 0, wbinvd_rll_no_guarantee, WBINVD/INVD not guaranteed for Remote Lower-Level caches - 4, 31:0, edx, 1, ll_inclusive , Cache is inclusive of Lower-Level caches - 4, 31:0, edx, 2, complex_indexing , Not a direct-mapped cache (complex function) + 0x4, 31:0, eax, 4:0, cache_type , Cache type field + 0x4, 31:0, eax, 7:5, cache_level , Cache level (1-based) + 0x4, 31:0, eax, 8, cache_self_init , Self-initialializing cache level + 0x4, 31:0, eax, 9, fully_associative , Fully-associative cache + 0x4, 31:0, eax, 25:14, num_threads_sharing , Number logical CPUs sharing this cache + 0x4, 31:0, eax, 31:26, num_cores_on_die , Number of cores in the physical package + 0x4, 31:0, ebx, 11:0, cache_linesize , System coherency line size (0-based) + 0x4, 31:0, ebx, 21:12, cache_npartitions , Physical line partitions (0-based) + 0x4, 31:0, ebx, 31:22, cache_nways , Ways of associativity (0-based) + 0x4, 31:0, ecx, 30:0, cache_nsets , Cache number of sets (0-based) + 0x4, 31:0, edx, 0, wbinvd_rll_no_guarantee, WBINVD/INVD not guaranteed for Remote Lower-Level caches + 0x4, 31:0, edx, 1, ll_inclusive , Cache is inclusive of Lower-Level caches + 0x4, 31:0, edx, 2, complex_indexing , Not a direct-mapped cache (complex function) # Leaf 5H # MONITOR/MWAIT instructions enumeration - 5, 0, eax, 15:0, min_mon_size , Smallest monitor-line size, in bytes - 5, 0, ebx, 15:0, max_mon_size , Largest monitor-line size, in bytes - 5, 0, ecx, 0, mwait_ext , Enumeration of MONITOR/MWAIT extensions is supported - 5, 0, ecx, 1, mwait_irq_break , Interrupts as a break-event for MWAIT is supported - 5, 0, edx, 3:0, n_c0_substates , Number of C0 sub C-states supported using MWAIT - 5, 0, edx, 7:4, n_c1_substates , Number of C1 sub C-states supported using MWAIT - 5, 0, edx, 11:8, n_c2_substates , Number of C2 sub C-states supported using MWAIT - 5, 0, edx, 15:12, n_c3_substates , Number of C3 sub C-states supported using MWAIT - 5, 0, edx, 19:16, n_c4_substates , Number of C4 sub C-states supported using MWAIT - 5, 0, edx, 23:20, n_c5_substates , Number of C5 sub C-states supported using MWAIT - 5, 0, edx, 27:24, n_c6_substates , Number of C6 sub C-states supported using MWAIT - 5, 0, edx, 31:28, n_c7_substates , Number of C7 sub C-states supported using MWAIT + 0x5, 0, eax, 15:0, min_mon_size , Smallest monitor-line size, in bytes + 0x5, 0, ebx, 15:0, max_mon_size , Largest monitor-line size, in bytes + 0x5, 0, ecx, 0, mwait_ext , Enumeration of MONITOR/MWAIT extensions is supported + 0x5, 0, ecx, 1, mwait_irq_break , Interrupts as a break-event for MWAIT is supported + 0x5, 0, edx, 3:0, n_c0_substates , Number of C0 sub C-states supported using MWAIT + 0x5, 0, edx, 7:4, n_c1_substates , Number of C1 sub C-states supported using MWAIT + 0x5, 0, edx, 11:8, n_c2_substates , Number of C2 sub C-states supported using MWAIT + 0x5, 0, edx, 15:12, n_c3_substates , Number of C3 sub C-states supported using MWAIT + 0x5, 0, edx, 19:16, n_c4_substates , Number of C4 sub C-states supported using MWAIT + 0x5, 0, edx, 23:20, n_c5_substates , Number of C5 sub C-states supported using MWAIT + 0x5, 0, edx, 27:24, n_c6_substates , Number of C6 sub C-states supported using MWAIT + 0x5, 0, edx, 31:28, n_c7_substates , Number of C7 sub C-states supported using MWAIT # Leaf 6H # Thermal and Power Management enumeration - 6, 0, eax, 0, dtherm , Digital temprature sensor - 6, 0, eax, 1, turbo_boost , Intel Turbo Boost - 6, 0, eax, 2, arat , Always-Running APIC Timer (not affected by p-state) - 6, 0, eax, 4, pln , Power Limit Notification (PLN) event - 6, 0, eax, 5, ecmd , Clock modulation duty cycle extension - 6, 0, eax, 6, pts , Package thermal management - 6, 0, eax, 7, hwp , HWP (Hardware P-states) base registers are supported - 6, 0, eax, 8, hwp_notify , HWP notification (IA32_HWP_INTERRUPT MSR) - 6, 0, eax, 9, hwp_act_window , HWP activity window (IA32_HWP_REQUEST[bits 41:32]) supported - 6, 0, eax, 10, hwp_epp , HWP Energy Performance Preference - 6, 0, eax, 11, hwp_pkg_req , HWP Package Level Request - 6, 0, eax, 13, hdc_base_regs , HDC base registers are supported - 6, 0, eax, 14, turbo_boost_3_0 , Intel Turbo Boost Max 3.0 - 6, 0, eax, 15, hwp_capabilities , HWP Highest Performance change - 6, 0, eax, 16, hwp_peci_override , HWP PECI override - 6, 0, eax, 17, hwp_flexible , Flexible HWP - 6, 0, eax, 18, hwp_fast , IA32_HWP_REQUEST MSR fast access mode - 6, 0, eax, 19, hfi , HW_FEEDBACK MSRs supported - 6, 0, eax, 20, hwp_ignore_idle , Ignoring idle logical CPU HWP req is supported - 6, 0, eax, 23, thread_director , Intel thread director support - 6, 0, eax, 24, therm_interrupt_bit25 , IA32_THERM_INTERRUPT MSR bit 25 is supported - 6, 0, ebx, 3:0, n_therm_thresholds , Digital thermometer thresholds - 6, 0, ecx, 0, aperfmperf , MPERF/APERF MSRs (effective frequency interface) - 6, 0, ecx, 3, epb , IA32_ENERGY_PERF_BIAS MSR support - 6, 0, ecx, 15:8, thrd_director_nclasses , Number of classes, Intel thread director - 6, 0, edx, 0, perfcap_reporting , Performance capability reporting - 6, 0, edx, 1, encap_reporting , Energy efficiency capability reporting - 6, 0, edx, 11:8, feedback_sz , HW feedback interface struct size, in 4K pages - 6, 0, edx, 31:16, this_lcpu_hwfdbk_idx , This logical CPU index @ HW feedback struct, 0-based + 0x6, 0, eax, 0, dtherm , Digital temprature sensor + 0x6, 0, eax, 1, turbo_boost , Intel Turbo Boost + 0x6, 0, eax, 2, arat , Always-Running APIC Timer (not affected by p-state) + 0x6, 0, eax, 4, pln , Power Limit Notification (PLN) event + 0x6, 0, eax, 5, ecmd , Clock modulation duty cycle extension + 0x6, 0, eax, 6, pts , Package thermal management + 0x6, 0, eax, 7, hwp , HWP (Hardware P-states) base registers are supported + 0x6, 0, eax, 8, hwp_notify , HWP notification (IA32_HWP_INTERRUPT MSR) + 0x6, 0, eax, 9, hwp_act_window , HWP activity window (IA32_HWP_REQUEST[bits 41:32]) supported + 0x6, 0, eax, 10, hwp_epp , HWP Energy Performance Preference + 0x6, 0, eax, 11, hwp_pkg_req , HWP Package Level Request + 0x6, 0, eax, 13, hdc_base_regs , HDC base registers are supported + 0x6, 0, eax, 14, turbo_boost_3_0 , Intel Turbo Boost Max 3.0 + 0x6, 0, eax, 15, hwp_capabilities , HWP Highest Performance change + 0x6, 0, eax, 16, hwp_peci_override , HWP PECI override + 0x6, 0, eax, 17, hwp_flexible , Flexible HWP + 0x6, 0, eax, 18, hwp_fast , IA32_HWP_REQUEST MSR fast access mode + 0x6, 0, eax, 19, hfi , HW_FEEDBACK MSRs supported + 0x6, 0, eax, 20, hwp_ignore_idle , Ignoring idle logical CPU HWP req is supported + 0x6, 0, eax, 23, thread_director , Intel thread director support + 0x6, 0, eax, 24, therm_interrupt_bit25 , IA32_THERM_INTERRUPT MSR bit 25 is supported + 0x6, 0, ebx, 3:0, n_therm_thresholds , Digital thermometer thresholds + 0x6, 0, ecx, 0, aperfmperf , MPERF/APERF MSRs (effective frequency interface) + 0x6, 0, ecx, 3, epb , IA32_ENERGY_PERF_BIAS MSR support + 0x6, 0, ecx, 15:8, thrd_director_nclasses , Number of classes, Intel thread director + 0x6, 0, edx, 0, perfcap_reporting , Performance capability reporting + 0x6, 0, edx, 1, encap_reporting , Energy efficiency capability reporting + 0x6, 0, edx, 11:8, feedback_sz , HW feedback interface struct size, in 4K pages + 0x6, 0, edx, 31:16, this_lcpu_hwfdbk_idx , This logical CPU index @ HW feedback struct, 0-based # Leaf 7H # Extended CPU features enumeration - 7, 0, eax, 31:0, leaf7_n_subleaves , Number of cpuid 0x7 subleaves - 7, 0, ebx, 0, fsgsbase , FSBASE/GSBASE read/write support - 7, 0, ebx, 1, tsc_adjust , IA32_TSC_ADJUST MSR supported - 7, 0, ebx, 2, sgx , Intel SGX (Software Guard Extensions) - 7, 0, ebx, 3, bmi1 , Bit manipulation extensions group 1 - 7, 0, ebx, 4, hle , Hardware Lock Elision - 7, 0, ebx, 5, avx2 , AVX2 instruction set - 7, 0, ebx, 6, fdp_excptn_only , FPU Data Pointer updated only on x87 exceptions - 7, 0, ebx, 7, smep , Supervisor Mode Execution Protection - 7, 0, ebx, 8, bmi2 , Bit manipulation extensions group 2 - 7, 0, ebx, 9, erms , Enhanced REP MOVSB/STOSB - 7, 0, ebx, 10, invpcid , INVPCID instruction (Invalidate Processor Context ID) - 7, 0, ebx, 11, rtm , Intel restricted transactional memory - 7, 0, ebx, 12, cqm , Intel RDT-CMT / AMD Platform-QoS cache monitoring - 7, 0, ebx, 13, zero_fcs_fds , Deprecated FPU CS/DS (stored as zero) - 7, 0, ebx, 14, mpx , Intel memory protection extensions - 7, 0, ebx, 15, rdt_a , Intel RDT / AMD Platform-QoS Enforcemeent - 7, 0, ebx, 16, avx512f , AVX-512 foundation instructions - 7, 0, ebx, 17, avx512dq , AVX-512 double/quadword instructions - 7, 0, ebx, 18, rdseed , RDSEED instruction - 7, 0, ebx, 19, adx , ADCX/ADOX instructions - 7, 0, ebx, 20, smap , Supervisor mode access prevention - 7, 0, ebx, 21, avx512ifma , AVX-512 integer fused multiply add - 7, 0, ebx, 23, clflushopt , CLFLUSHOPT instruction - 7, 0, ebx, 24, clwb , CLWB instruction - 7, 0, ebx, 25, intel_pt , Intel processor trace - 7, 0, ebx, 26, avx512pf , AVX-512 prefetch instructions - 7, 0, ebx, 27, avx512er , AVX-512 exponent/reciprocal instrs - 7, 0, ebx, 28, avx512cd , AVX-512 conflict detection instrs - 7, 0, ebx, 29, sha_ni , SHA/SHA256 instructions - 7, 0, ebx, 30, avx512bw , AVX-512 BW (byte/word granular) instructions - 7, 0, ebx, 31, avx512vl , AVX-512 VL (128/256 vector length) extensions - 7, 0, ecx, 0, prefetchwt1 , PREFETCHWT1 (Intel Xeon Phi only) - 7, 0, ecx, 1, avx512vbmi , AVX-512 Vector byte manipulation instrs - 7, 0, ecx, 2, umip , User mode instruction protection - 7, 0, ecx, 3, pku , Protection keys for user-space - 7, 0, ecx, 4, ospke , OS protection keys enable - 7, 0, ecx, 5, waitpkg , WAITPKG instructions - 7, 0, ecx, 6, avx512_vbmi2 , AVX-512 vector byte manipulation instrs group 2 - 7, 0, ecx, 7, cet_ss , CET shadow stack features - 7, 0, ecx, 8, gfni , Galois field new instructions - 7, 0, ecx, 9, vaes , Vector AES instrs - 7, 0, ecx, 10, vpclmulqdq , VPCLMULQDQ 256-bit instruction support - 7, 0, ecx, 11, avx512_vnni , Vector neural network instructions - 7, 0, ecx, 12, avx512_bitalg , AVX-512 bit count/shiffle - 7, 0, ecx, 13, tme , Intel total memory encryption - 7, 0, ecx, 14, avx512_vpopcntdq , AVX-512: POPCNT for vectors of DW/QW - 7, 0, ecx, 16, la57 , 57-bit linear addreses (five-level paging) - 7, 0, ecx, 21:17, mawau_val_lm , BNDLDX/BNDSTX MAWAU value in 64-bit mode - 7, 0, ecx, 22, rdpid , RDPID instruction - 7, 0, ecx, 23, key_locker , Intel key locker support - 7, 0, ecx, 24, bus_lock_detect , OS bus-lock detection - 7, 0, ecx, 25, cldemote , CLDEMOTE instruction - 7, 0, ecx, 27, movdiri , MOVDIRI instruction - 7, 0, ecx, 28, movdir64b , MOVDIR64B instruction - 7, 0, ecx, 29, enqcmd , Enqueue stores supported (ENQCMD{,S}) - 7, 0, ecx, 30, sgx_lc , Intel SGX launch configuration - 7, 0, ecx, 31, pks , Protection keys for supervisor-mode pages - 7, 0, edx, 1, sgx_keys , Intel SGX attestation services - 7, 0, edx, 2, avx512_4vnniw , AVX-512 neural network instructions - 7, 0, edx, 3, avx512_4fmaps , AVX-512 multiply accumulation single precision - 7, 0, edx, 4, fsrm , Fast short REP MOV - 7, 0, edx, 5, uintr , CPU supports user interrupts - 7, 0, edx, 8, avx512_vp2intersect , VP2INTERSECT{D,Q} instructions - 7, 0, edx, 9, srdbs_ctrl , SRBDS mitigation MSR available - 7, 0, edx, 10, md_clear , VERW MD_CLEAR microcode support - 7, 0, edx, 11, rtm_always_abort , XBEGIN (RTM transaction) always aborts - 7, 0, edx, 13, tsx_force_abort , MSR TSX_FORCE_ABORT, RTM_ABORT bit, supported - 7, 0, edx, 14, serialize , SERIALIZE instruction - 7, 0, edx, 15, hybrid_cpu , The CPU is identified as a 'hybrid part' - 7, 0, edx, 16, tsxldtrk , TSX suspend/resume load address tracking - 7, 0, edx, 18, pconfig , PCONFIG instruction - 7, 0, edx, 19, arch_lbr , Intel architectural LBRs - 7, 0, edx, 20, ibt , CET indirect branch tracking - 7, 0, edx, 22, amx_bf16 , AMX-BF16: tile bfloat16 support - 7, 0, edx, 23, avx512_fp16 , AVX-512 FP16 instructions - 7, 0, edx, 24, amx_tile , AMX-TILE: tile architecture support - 7, 0, edx, 25, amx_int8 , AMX-INT8: tile 8-bit integer support - 7, 0, edx, 26, spec_ctrl , Speculation Control (IBRS/IBPB: indirect branch restrictions) - 7, 0, edx, 27, intel_stibp , Single thread indirect branch predictors - 7, 0, edx, 28, flush_l1d , FLUSH L1D cache: IA32_FLUSH_CMD MSR - 7, 0, edx, 29, arch_capabilities , Intel IA32_ARCH_CAPABILITIES MSR - 7, 0, edx, 30, core_capabilities , IA32_CORE_CAPABILITIES MSR - 7, 0, edx, 31, spec_ctrl_ssbd , Speculative store bypass disable - 7, 1, eax, 4, avx_vnni , AVX-VNNI instructions - 7, 1, eax, 5, avx512_bf16 , AVX-512 bFloat16 instructions - 7, 1, eax, 6, lass , Linear address space separation - 7, 1, eax, 7, cmpccxadd , CMPccXADD instructions - 7, 1, eax, 8, arch_perfmon_ext , ArchPerfmonExt: CPUID leaf 0x23 is supported - 7, 1, eax, 10, fzrm , Fast zero-length REP MOVSB - 7, 1, eax, 11, fsrs , Fast short REP STOSB - 7, 1, eax, 12, fsrc , Fast Short REP CMPSB/SCASB - 7, 1, eax, 17, fred , FRED: Flexible return and event delivery transitions - 7, 1, eax, 18, lkgs , LKGS: Load 'kernel' (userspace) GS - 7, 1, eax, 19, wrmsrns , WRMSRNS instr (WRMSR-non-serializing) - 7, 1, eax, 21, amx_fp16 , AMX-FP16: FP16 tile operations - 7, 1, eax, 22, hreset , History reset support - 7, 1, eax, 23, avx_ifma , Integer fused multiply add - 7, 1, eax, 26, lam , Linear address masking - 7, 1, eax, 27, rd_wr_msrlist , RDMSRLIST/WRMSRLIST instructions - 7, 1, ebx, 0, intel_ppin , Protected processor inventory number (PPIN{,_CTL} MSRs) - 7, 1, edx, 4, avx_vnni_int8 , AVX-VNNI-INT8 instructions - 7, 1, edx, 5, avx_ne_convert , AVX-NE-CONVERT instructions - 7, 1, edx, 8, amx_complex , AMX-COMPLEX instructions (starting from Granite Rapids) - 7, 1, edx, 14, prefetchit_0_1 , PREFETCHIT0/1 instructions - 7, 1, edx, 18, cet_sss , CET supervisor shadow stacks safe to use - 7, 2, edx, 0, intel_psfd , Intel predictive store forward disable - 7, 2, edx, 1, ipred_ctrl , MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S} - 7, 2, edx, 2, rrsba_ctrl , MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S} - 7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U - 7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S - 7, 2, edx, 5, mcdt_no , MCDT mitigation not needed - 7, 2, edx, 6, uclock_disable , UC-lock disable is supported + 0x7, 0, eax, 31:0, leaf7_n_subleaves , Number of cpuid 0x7 subleaves + 0x7, 0, ebx, 0, fsgsbase , FSBASE/GSBASE read/write support + 0x7, 0, ebx, 1, tsc_adjust , IA32_TSC_ADJUST MSR supported + 0x7, 0, ebx, 2, sgx , Intel SGX (Software Guard Extensions) + 0x7, 0, ebx, 3, bmi1 , Bit manipulation extensions group 1 + 0x7, 0, ebx, 4, hle , Hardware Lock Elision + 0x7, 0, ebx, 5, avx2 , AVX2 instruction set + 0x7, 0, ebx, 6, fdp_excptn_only , FPU Data Pointer updated only on x87 exceptions + 0x7, 0, ebx, 7, smep , Supervisor Mode Execution Protection + 0x7, 0, ebx, 8, bmi2 , Bit manipulation extensions group 2 + 0x7, 0, ebx, 9, erms , Enhanced REP MOVSB/STOSB + 0x7, 0, ebx, 10, invpcid , INVPCID instruction (Invalidate Processor Context ID) + 0x7, 0, ebx, 11, rtm , Intel restricted transactional memory + 0x7, 0, ebx, 12, cqm , Intel RDT-CMT / AMD Platform-QoS cache monitoring + 0x7, 0, ebx, 13, zero_fcs_fds , Deprecated FPU CS/DS (stored as zero) + 0x7, 0, ebx, 14, mpx , Intel memory protection extensions + 0x7, 0, ebx, 15, rdt_a , Intel RDT / AMD Platform-QoS Enforcemeent + 0x7, 0, ebx, 16, avx512f , AVX-512 foundation instructions + 0x7, 0, ebx, 17, avx512dq , AVX-512 double/quadword instructions + 0x7, 0, ebx, 18, rdseed , RDSEED instruction + 0x7, 0, ebx, 19, adx , ADCX/ADOX instructions + 0x7, 0, ebx, 20, smap , Supervisor mode access prevention + 0x7, 0, ebx, 21, avx512ifma , AVX-512 integer fused multiply add + 0x7, 0, ebx, 23, clflushopt , CLFLUSHOPT instruction + 0x7, 0, ebx, 24, clwb , CLWB instruction + 0x7, 0, ebx, 25, intel_pt , Intel processor trace + 0x7, 0, ebx, 26, avx512pf , AVX-512 prefetch instructions + 0x7, 0, ebx, 27, avx512er , AVX-512 exponent/reciprocal instrs + 0x7, 0, ebx, 28, avx512cd , AVX-512 conflict detection instrs + 0x7, 0, ebx, 29, sha_ni , SHA/SHA256 instructions + 0x7, 0, ebx, 30, avx512bw , AVX-512 BW (byte/word granular) instructions + 0x7, 0, ebx, 31, avx512vl , AVX-512 VL (128/256 vector length) extensions + 0x7, 0, ecx, 0, prefetchwt1 , PREFETCHWT1 (Intel Xeon Phi only) + 0x7, 0, ecx, 1, avx512vbmi , AVX-512 Vector byte manipulation instrs + 0x7, 0, ecx, 2, umip , User mode instruction protection + 0x7, 0, ecx, 3, pku , Protection keys for user-space + 0x7, 0, ecx, 4, ospke , OS protection keys enable + 0x7, 0, ecx, 5, waitpkg , WAITPKG instructions + 0x7, 0, ecx, 6, avx512_vbmi2 , AVX-512 vector byte manipulation instrs group 2 + 0x7, 0, ecx, 7, cet_ss , CET shadow stack features + 0x7, 0, ecx, 8, gfni , Galois field new instructions + 0x7, 0, ecx, 9, vaes , Vector AES instrs + 0x7, 0, ecx, 10, vpclmulqdq , VPCLMULQDQ 256-bit instruction support + 0x7, 0, ecx, 11, avx512_vnni , Vector neural network instructions + 0x7, 0, ecx, 12, avx512_bitalg , AVX-512 bit count/shiffle + 0x7, 0, ecx, 13, tme , Intel total memory encryption + 0x7, 0, ecx, 14, avx512_vpopcntdq , AVX-512: POPCNT for vectors of DW/QW + 0x7, 0, ecx, 16, la57 , 57-bit linear addreses (five-level paging) + 0x7, 0, ecx, 21:17, mawau_val_lm , BNDLDX/BNDSTX MAWAU value in 64-bit mode + 0x7, 0, ecx, 22, rdpid , RDPID instruction + 0x7, 0, ecx, 23, key_locker , Intel key locker support + 0x7, 0, ecx, 24, bus_lock_detect , OS bus-lock detection + 0x7, 0, ecx, 25, cldemote , CLDEMOTE instruction + 0x7, 0, ecx, 27, movdiri , MOVDIRI instruction + 0x7, 0, ecx, 28, movdir64b , MOVDIR64B instruction + 0x7, 0, ecx, 29, enqcmd , Enqueue stores supported (ENQCMD{,S}) + 0x7, 0, ecx, 30, sgx_lc , Intel SGX launch configuration + 0x7, 0, ecx, 31, pks , Protection keys for supervisor-mode pages + 0x7, 0, edx, 1, sgx_keys , Intel SGX attestation services + 0x7, 0, edx, 2, avx512_4vnniw , AVX-512 neural network instructions + 0x7, 0, edx, 3, avx512_4fmaps , AVX-512 multiply accumulation single precision + 0x7, 0, edx, 4, fsrm , Fast short REP MOV + 0x7, 0, edx, 5, uintr , CPU supports user interrupts + 0x7, 0, edx, 8, avx512_vp2intersect , VP2INTERSECT{D,Q} instructions + 0x7, 0, edx, 9, srdbs_ctrl , SRBDS mitigation MSR available + 0x7, 0, edx, 10, md_clear , VERW MD_CLEAR microcode support + 0x7, 0, edx, 11, rtm_always_abort , XBEGIN (RTM transaction) always aborts + 0x7, 0, edx, 13, tsx_force_abort , MSR TSX_FORCE_ABORT, RTM_ABORT bit, supported + 0x7, 0, edx, 14, serialize , SERIALIZE instruction + 0x7, 0, edx, 15, hybrid_cpu , The CPU is identified as a 'hybrid part' + 0x7, 0, edx, 16, tsxldtrk , TSX suspend/resume load address tracking + 0x7, 0, edx, 18, pconfig , PCONFIG instruction + 0x7, 0, edx, 19, arch_lbr , Intel architectural LBRs + 0x7, 0, edx, 20, ibt , CET indirect branch tracking + 0x7, 0, edx, 22, amx_bf16 , AMX-BF16: tile bfloat16 support + 0x7, 0, edx, 23, avx512_fp16 , AVX-512 FP16 instructions + 0x7, 0, edx, 24, amx_tile , AMX-TILE: tile architecture support + 0x7, 0, edx, 25, amx_int8 , AMX-INT8: tile 8-bit integer support + 0x7, 0, edx, 26, spec_ctrl , Speculation Control (IBRS/IBPB: indirect branch restrictions) + 0x7, 0, edx, 27, intel_stibp , Single thread indirect branch predictors + 0x7, 0, edx, 28, flush_l1d , FLUSH L1D cache: IA32_FLUSH_CMD MSR + 0x7, 0, edx, 29, arch_capabilities , Intel IA32_ARCH_CAPABILITIES MSR + 0x7, 0, edx, 30, core_capabilities , IA32_CORE_CAPABILITIES MSR + 0x7, 0, edx, 31, spec_ctrl_ssbd , Speculative store bypass disable + 0x7, 1, eax, 4, avx_vnni , AVX-VNNI instructions + 0x7, 1, eax, 5, avx512_bf16 , AVX-512 bFloat16 instructions + 0x7, 1, eax, 6, lass , Linear address space separation + 0x7, 1, eax, 7, cmpccxadd , CMPccXADD instructions + 0x7, 1, eax, 8, arch_perfmon_ext , ArchPerfmonExt: CPUID leaf 0x23 is supported + 0x7, 1, eax, 10, fzrm , Fast zero-length REP MOVSB + 0x7, 1, eax, 11, fsrs , Fast short REP STOSB + 0x7, 1, eax, 12, fsrc , Fast Short REP CMPSB/SCASB + 0x7, 1, eax, 17, fred , FRED: Flexible return and event delivery transitions + 0x7, 1, eax, 18, lkgs , LKGS: Load 'kernel' (userspace) GS + 0x7, 1, eax, 19, wrmsrns , WRMSRNS instr (WRMSR-non-serializing) + 0x7, 1, eax, 20, nmi_src , NMI-source reporting with FRED event data + 0x7, 1, eax, 21, amx_fp16 , AMX-FP16: FP16 tile operations + 0x7, 1, eax, 22, hreset , History reset support + 0x7, 1, eax, 23, avx_ifma , Integer fused multiply add + 0x7, 1, eax, 26, lam , Linear address masking + 0x7, 1, eax, 27, rd_wr_msrlist , RDMSRLIST/WRMSRLIST instructions + 0x7, 1, ebx, 0, intel_ppin , Protected processor inventory number (PPIN{,_CTL} MSRs) + 0x7, 1, edx, 4, avx_vnni_int8 , AVX-VNNI-INT8 instructions + 0x7, 1, edx, 5, avx_ne_convert , AVX-NE-CONVERT instructions + 0x7, 1, edx, 8, amx_complex , AMX-COMPLEX instructions (starting from Granite Rapids) + 0x7, 1, edx, 14, prefetchit_0_1 , PREFETCHIT0/1 instructions + 0x7, 1, edx, 18, cet_sss , CET supervisor shadow stacks safe to use + 0x7, 2, edx, 0, intel_psfd , Intel predictive store forward disable + 0x7, 2, edx, 1, ipred_ctrl , MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S} + 0x7, 2, edx, 2, rrsba_ctrl , MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S} + 0x7, 2, edx, 3, ddp_ctrl , MSR bit IA32_SPEC_CTRL.DDPD_U + 0x7, 2, edx, 4, bhi_ctrl , MSR bit IA32_SPEC_CTRL.BHI_DIS_S + 0x7, 2, edx, 5, mcdt_no , MCDT mitigation not needed + 0x7, 2, edx, 6, uclock_disable , UC-lock disable is supported # Leaf 9H # Intel DCA (Direct Cache Access) enumeration - 9, 0, eax, 0, dca_enabled_in_bios , DCA is enabled in BIOS + 0x9, 0, eax, 0, dca_enabled_in_bios , DCA is enabled in BIOS # Leaf AH # Intel PMU (Performance Monitoring Unit) enumeration @@ -623,7 +632,7 @@ 0x40000000, 0, edx, 31:0, hypervisor_id_2 , Hypervisor ID string bytes 8 - 11 # Leaf 80000000H -# Maximum extended leaf number + CPU vendor string (AMD) +# Maximum extended leaf number + AMD/Transmeta CPU vendor string 0x80000000, 0, eax, 31:0, max_ext_leaf , Maximum extended cpuid leaf supported 0x80000000, 0, ebx, 31:0, cpu_vendorid_0 , Vendor ID string bytes 0 - 3 @@ -636,6 +645,7 @@ 0x80000001, 0, eax, 3:0, e_stepping_id , Stepping ID 0x80000001, 0, eax, 7:4, e_base_model , Base processor model 0x80000001, 0, eax, 11:8, e_base_family , Base processor family +0x80000001, 0, eax, 13:12, e_base_type , Base processor type (Transmeta) 0x80000001, 0, eax, 19:16, e_ext_model , Extended processor model 0x80000001, 0, eax, 27:20, e_ext_family , Extended processor family 0x80000001, 0, ebx, 15:0, brand_id , Brand ID @@ -687,6 +697,7 @@ 0x80000001, 0, edx, 19, mp , Out-of-spec AMD Multiprocessing bit 0x80000001, 0, edx, 20, nx , No-execute page protection 0x80000001, 0, edx, 22, mmxext , AMD MMX extensions +0x80000001, 0, edx, 23, e_mmx , MMX instructions (Transmeta) 0x80000001, 0, edx, 24, e_fxsr , FXSAVE and FXRSTOR instructions 0x80000001, 0, edx, 25, fxsr_opt , FXSAVE and FXRSTOR optimizations 0x80000001, 0, edx, 26, pdpe1gb , 1-GB large page support @@ -720,7 +731,7 @@ 0x80000004, 0, edx, 31:0, cpu_brandid_11 , CPU brand ID string, bytes 44 - 47 # Leaf 80000005H -# AMD L1 cache and L1 TLB enumeration +# AMD/Transmeta L1 cache and L1 TLB enumeration 0x80000005, 0, eax, 7:0, l1_itlb_2m_4m_nentries , L1 ITLB #entires, 2M and 4M pages 0x80000005, 0, eax, 15:8, l1_itlb_2m_4m_assoc , L1 ITLB associativity, 2M and 4M pages @@ -1051,3 +1062,108 @@ 0x80000026, 3:0, ecx, 7:0, domain_level , This domain level (subleaf ID) 0x80000026, 3:0, ecx, 15:8, domain_type , This domain type 0x80000026, 3:0, edx, 31:0, x2apic_id , x2APIC ID of current logical CPU + +# Leaf 80860000H +# Maximum Transmeta leaf number + CPU vendor ID string + +0x80860000, 0, eax, 31:0, max_tra_leaf , Maximum supported Transmeta leaf number +0x80860000, 0, ebx, 31:0, cpu_vendorid_0 , Transmeta Vendor ID string bytes 0 - 3 +0x80860000, 0, ecx, 31:0, cpu_vendorid_2 , Transmeta Vendor ID string bytes 8 - 11 +0x80860000, 0, edx, 31:0, cpu_vendorid_1 , Transmeta Vendor ID string bytes 4 - 7 + +# Leaf 80860001H +# Transmeta extended CPU information + +0x80860001, 0, eax, 3:0, stepping , Stepping ID +0x80860001, 0, eax, 7:4, base_model , Base CPU model ID +0x80860001, 0, eax, 11:8, base_family_id , Base CPU family ID +0x80860001, 0, eax, 13:12, cpu_type , CPU type +0x80860001, 0, ebx, 7:0, cpu_rev_mask_minor , CPU revision ID, mask minor +0x80860001, 0, ebx, 15:8, cpu_rev_mask_major , CPU revision ID, mask major +0x80860001, 0, ebx, 23:16, cpu_rev_minor , CPU revision ID, minor +0x80860001, 0, ebx, 31:24, cpu_rev_major , CPU revision ID, major +0x80860001, 0, ecx, 31:0, cpu_base_mhz , CPU nominal frequency, in MHz +0x80860001, 0, edx, 0, recovery , Recovery CMS is active (after bad flush) +0x80860001, 0, edx, 1, longrun , LongRun power management capabilities +0x80860001, 0, edx, 3, lrti , LongRun Table Interface + +# Leaf 80860002H +# Transmeta Code Morphing Software (CMS) enumeration + +0x80860002, 0, eax, 31:0, cpu_rev_id , CPU revision ID +0x80860002, 0, ebx, 7:0, cms_rev_mask_2 , CMS revision ID, mask component 2 +0x80860002, 0, ebx, 15:8, cms_rev_mask_1 , CMS revision ID, mask component 1 +0x80860002, 0, ebx, 23:16, cms_rev_minor , CMS revision ID, minor +0x80860002, 0, ebx, 31:24, cms_rev_major , CMS revision ID, major +0x80860002, 0, ecx, 31:0, cms_rev_mask_3 , CMS revision ID, mask component 3 + +# Leaf 80860003H +# Transmeta CPU information string, bytes 0 - 15 + +0x80860003, 0, eax, 31:0, cpu_info_0 , CPU info string bytes 0 - 3 +0x80860003, 0, ebx, 31:0, cpu_info_1 , CPU info string bytes 4 - 7 +0x80860003, 0, ecx, 31:0, cpu_info_2 , CPU info string bytes 8 - 11 +0x80860003, 0, edx, 31:0, cpu_info_3 , CPU info string bytes 12 - 15 + +# Leaf 80860004H +# Transmeta CPU information string, bytes 16 - 31 + +0x80860004, 0, eax, 31:0, cpu_info_4 , CPU info string bytes 16 - 19 +0x80860004, 0, ebx, 31:0, cpu_info_5 , CPU info string bytes 20 - 23 +0x80860004, 0, ecx, 31:0, cpu_info_6 , CPU info string bytes 24 - 27 +0x80860004, 0, edx, 31:0, cpu_info_7 , CPU info string bytes 28 - 31 + +# Leaf 80860005H +# Transmeta CPU information string, bytes 32 - 47 + +0x80860005, 0, eax, 31:0, cpu_info_8 , CPU info string bytes 32 - 35 +0x80860005, 0, ebx, 31:0, cpu_info_9 , CPU info string bytes 36 - 39 +0x80860005, 0, ecx, 31:0, cpu_info_10 , CPU info string bytes 40 - 43 +0x80860005, 0, edx, 31:0, cpu_info_11 , CPU info string bytes 44 - 47 + +# Leaf 80860006H +# Transmeta CPU information string, bytes 48 - 63 + +0x80860006, 0, eax, 31:0, cpu_info_12 , CPU info string bytes 48 - 51 +0x80860006, 0, ebx, 31:0, cpu_info_13 , CPU info string bytes 52 - 55 +0x80860006, 0, ecx, 31:0, cpu_info_14 , CPU info string bytes 56 - 59 +0x80860006, 0, edx, 31:0, cpu_info_15 , CPU info string bytes 60 - 63 + +# Leaf 80860007H +# Transmeta live CPU information + +0x80860007, 0, eax, 31:0, cpu_cur_mhz , Current CPU frequency, in MHz +0x80860007, 0, ebx, 31:0, cpu_cur_voltage , Current CPU voltage, in millivolts +0x80860007, 0, ecx, 31:0, cpu_cur_perf_pctg , Current CPU performance percentage, 0 - 100d +0x80860007, 0, edx, 31:0, cpu_cur_gate_delay , Current CPU gate delay, in femtoseconds + +# Leaf C0000000H +# Maximum Centaur/Zhaoxin leaf number + +0xc0000000, 0, eax, 31:0, max_cntr_leaf , Maximum Centaur/Zhaoxin leaf number + +# Leaf C0000001H +# Centaur/Zhaoxin extended CPU features + +0xc0000001, 0, edx, 0, ccs_sm2 , CCS SM2 instructions +0xc0000001, 0, edx, 1, ccs_sm2_en , CCS SM2 enabled +0xc0000001, 0, edx, 2, xstore , Random Number Generator +0xc0000001, 0, edx, 3, xstore_en , RNG enabled +0xc0000001, 0, edx, 4, ccs_sm3_sm4 , CCS SM3 and SM4 instructions +0xc0000001, 0, edx, 5, ccs_sm3_sm4_en , CCS SM3/SM4 enabled +0xc0000001, 0, edx, 6, ace , Advanced Cryptography Engine +0xc0000001, 0, edx, 7, ace_en , ACE enabled +0xc0000001, 0, edx, 8, ace2 , Advanced Cryptography Engine v2 +0xc0000001, 0, edx, 9, ace2_en , ACE v2 enabled +0xc0000001, 0, edx, 10, phe , PadLock Hash Engine +0xc0000001, 0, edx, 11, phe_en , PHE enabled +0xc0000001, 0, edx, 12, pmm , PadLock Montgomery Multiplier +0xc0000001, 0, edx, 13, pmm_en , PMM enabled +0xc0000001, 0, edx, 16, parallax , Parallax auto adjust processor voltage +0xc0000001, 0, edx, 17, parallax_en , Parallax enabled +0xc0000001, 0, edx, 20, tm3 , Thermal Monitor v3 +0xc0000001, 0, edx, 21, tm3_en , TM v3 enabled +0xc0000001, 0, edx, 25, phe2 , PadLock Hash Engine v2 (SHA384/SHA512) +0xc0000001, 0, edx, 26, phe2_en , PHE v2 enabled +0xc0000001, 0, edx, 27, rsa , RSA instructions (XMODEXP/MONTMUL2) +0xc0000001, 0, edx, 28, rsa_en , RSA instructions enabled From f5e7fd6857963e697600d881242bbb39b0b9ac37 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:38 +0100 Subject: [PATCH 017/331] tools/x86/kcpuid: Update bitfields to x86-cpuid-db v2.1 Update kcpuid's CSV file to version 2.1, as generated by x86-cpuid-db. Summary of the v2.1 changes: * Use a standardized style for all x86 trademarks, registers, opcodes, byte units, hexadecimal digits, and x86 technical terms. This was enforced by a number of x86-specific hunspell(5) dictionary and affix files at the x86-cpuid-db project's CI pipeline. * Expand abbreviated terms that might be OK in code but not in official listings (e.g., "addr", "instr", "reg", "virt", etc.) * Add new Zen5 SoC bits to leaf 0x80000020 and leaf 0x80000021. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/blob/v2.1/CHANGELOG.rst Link: https://lore.kernel.org/r/20250324142042.29010-18-darwi@linutronix.de --- tools/arch/x86/kcpuid/cpuid.csv | 171 +++++++++++++++++--------------- 1 file changed, 91 insertions(+), 80 deletions(-) diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index d0f7159f99ba..0f9c11208674 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v2.0 +# Generator: x86-cpuid-db v2.1 # # Auto-generated file. @@ -28,7 +28,7 @@ 0x1, 0, eax, 27:20, ext_family , Extended CPU family ID 0x1, 0, ebx, 7:0, brand_id , Brand index 0x1, 0, ebx, 15:8, clflush_size , CLFLUSH instruction cache line size - 0x1, 0, ebx, 23:16, n_logical_cpu , Logical CPU (HW threads) count + 0x1, 0, ebx, 23:16, n_logical_cpu , Logical CPU count 0x1, 0, ebx, 31:24, local_apic_id , Initial local APIC physical ID 0x1, 0, ecx, 0, pni , Streaming SIMD Extensions 3 (SSE3) 0x1, 0, ecx, 1, pclmulqdq , PCLMULQDQ instruction support @@ -41,7 +41,7 @@ 0x1, 0, ecx, 8, tm2 , Thermal Monitor 2 0x1, 0, ecx, 9, ssse3 , Supplemental SSE3 0x1, 0, ecx, 10, cid , L1 Context ID - 0x1, 0, ecx, 11, sdbg , Sillicon Debug + 0x1, 0, ecx, 11, sdbg , Silicon Debug 0x1, 0, ecx, 12, fma , FMA extensions using YMM state 0x1, 0, ecx, 13, cx16 , CMPXCHG16B instruction support 0x1, 0, ecx, 14, xtpr , xTPR Update Control @@ -89,13 +89,13 @@ 0x1, 0, edx, 27, ss , Self Snoop 0x1, 0, edx, 28, ht , Hyper-threading 0x1, 0, edx, 29, tm , Thermal Monitor - 0x1, 0, edx, 30, ia64 , Legacy IA-64 (Itanium) support bit, now resreved + 0x1, 0, edx, 30, ia64 , Legacy IA-64 (Itanium) support bit, now reserved 0x1, 0, edx, 31, pbe , Pending Break Enable # Leaf 2H # Intel cache and TLB information one-byte descriptors - 0x2, 0, eax, 7:0, iteration_count , Number of times this CPUD leaf must be queried + 0x2, 0, eax, 7:0, iteration_count , Number of times this leaf must be queried 0x2, 0, eax, 15:8, desc1 , Descriptor #1 0x2, 0, eax, 23:16, desc2 , Descriptor #2 0x2, 0, eax, 30:24, desc3 , Descriptor #3 @@ -129,7 +129,7 @@ 0x4, 31:0, eax, 4:0, cache_type , Cache type field 0x4, 31:0, eax, 7:5, cache_level , Cache level (1-based) - 0x4, 31:0, eax, 8, cache_self_init , Self-initialializing cache level + 0x4, 31:0, eax, 8, cache_self_init , Self-initializing cache level 0x4, 31:0, eax, 9, fully_associative , Fully-associative cache 0x4, 31:0, eax, 25:14, num_threads_sharing , Number logical CPUs sharing this cache 0x4, 31:0, eax, 31:26, num_cores_on_die , Number of cores in the physical package @@ -160,7 +160,7 @@ # Leaf 6H # Thermal and Power Management enumeration - 0x6, 0, eax, 0, dtherm , Digital temprature sensor + 0x6, 0, eax, 0, dtherm , Digital temperature sensor 0x6, 0, eax, 1, turbo_boost , Intel Turbo Boost 0x6, 0, eax, 2, arat , Always-Running APIC Timer (not affected by p-state) 0x6, 0, eax, 4, pln , Power Limit Notification (PLN) event @@ -187,8 +187,8 @@ 0x6, 0, ecx, 15:8, thrd_director_nclasses , Number of classes, Intel thread director 0x6, 0, edx, 0, perfcap_reporting , Performance capability reporting 0x6, 0, edx, 1, encap_reporting , Energy efficiency capability reporting - 0x6, 0, edx, 11:8, feedback_sz , HW feedback interface struct size, in 4K pages - 0x6, 0, edx, 31:16, this_lcpu_hwfdbk_idx , This logical CPU index @ HW feedback struct, 0-based + 0x6, 0, edx, 11:8, feedback_sz , Feedback interface structure size, in 4K pages + 0x6, 0, edx, 31:16, this_lcpu_hwfdbk_idx , This logical CPU hardware feedback interface index # Leaf 7H # Extended CPU features enumeration @@ -209,7 +209,7 @@ 0x7, 0, ebx, 12, cqm , Intel RDT-CMT / AMD Platform-QoS cache monitoring 0x7, 0, ebx, 13, zero_fcs_fds , Deprecated FPU CS/DS (stored as zero) 0x7, 0, ebx, 14, mpx , Intel memory protection extensions - 0x7, 0, ebx, 15, rdt_a , Intel RDT / AMD Platform-QoS Enforcemeent + 0x7, 0, ebx, 15, rdt_a , Intel RDT / AMD Platform-QoS Enforcement 0x7, 0, ebx, 16, avx512f , AVX-512 foundation instructions 0x7, 0, ebx, 17, avx512dq , AVX-512 double/quadword instructions 0x7, 0, ebx, 18, rdseed , RDSEED instruction @@ -220,27 +220,27 @@ 0x7, 0, ebx, 24, clwb , CLWB instruction 0x7, 0, ebx, 25, intel_pt , Intel processor trace 0x7, 0, ebx, 26, avx512pf , AVX-512 prefetch instructions - 0x7, 0, ebx, 27, avx512er , AVX-512 exponent/reciprocal instrs - 0x7, 0, ebx, 28, avx512cd , AVX-512 conflict detection instrs + 0x7, 0, ebx, 27, avx512er , AVX-512 exponent/reciprocal instructions + 0x7, 0, ebx, 28, avx512cd , AVX-512 conflict detection instructions 0x7, 0, ebx, 29, sha_ni , SHA/SHA256 instructions - 0x7, 0, ebx, 30, avx512bw , AVX-512 BW (byte/word granular) instructions + 0x7, 0, ebx, 30, avx512bw , AVX-512 byte/word instructions 0x7, 0, ebx, 31, avx512vl , AVX-512 VL (128/256 vector length) extensions 0x7, 0, ecx, 0, prefetchwt1 , PREFETCHWT1 (Intel Xeon Phi only) - 0x7, 0, ecx, 1, avx512vbmi , AVX-512 Vector byte manipulation instrs + 0x7, 0, ecx, 1, avx512vbmi , AVX-512 Vector byte manipulation instructions 0x7, 0, ecx, 2, umip , User mode instruction protection 0x7, 0, ecx, 3, pku , Protection keys for user-space 0x7, 0, ecx, 4, ospke , OS protection keys enable 0x7, 0, ecx, 5, waitpkg , WAITPKG instructions - 0x7, 0, ecx, 6, avx512_vbmi2 , AVX-512 vector byte manipulation instrs group 2 + 0x7, 0, ecx, 6, avx512_vbmi2 , AVX-512 vector byte manipulation instructions group 2 0x7, 0, ecx, 7, cet_ss , CET shadow stack features 0x7, 0, ecx, 8, gfni , Galois field new instructions - 0x7, 0, ecx, 9, vaes , Vector AES instrs + 0x7, 0, ecx, 9, vaes , Vector AES instructions 0x7, 0, ecx, 10, vpclmulqdq , VPCLMULQDQ 256-bit instruction support 0x7, 0, ecx, 11, avx512_vnni , Vector neural network instructions - 0x7, 0, ecx, 12, avx512_bitalg , AVX-512 bit count/shiffle + 0x7, 0, ecx, 12, avx512_bitalg , AVX-512 bitwise algorithms 0x7, 0, ecx, 13, tme , Intel total memory encryption - 0x7, 0, ecx, 14, avx512_vpopcntdq , AVX-512: POPCNT for vectors of DW/QW - 0x7, 0, ecx, 16, la57 , 57-bit linear addreses (five-level paging) + 0x7, 0, ecx, 14, avx512_vpopcntdq , AVX-512: POPCNT for vectors of DWORD/QWORD + 0x7, 0, ecx, 16, la57 , 57-bit linear addresses (five-level paging) 0x7, 0, ecx, 21:17, mawau_val_lm , BNDLDX/BNDSTX MAWAU value in 64-bit mode 0x7, 0, ecx, 22, rdpid , RDPID instruction 0x7, 0, ecx, 23, key_locker , Intel key locker support @@ -278,7 +278,7 @@ 0x7, 0, edx, 30, core_capabilities , IA32_CORE_CAPABILITIES MSR 0x7, 0, edx, 31, spec_ctrl_ssbd , Speculative store bypass disable 0x7, 1, eax, 4, avx_vnni , AVX-VNNI instructions - 0x7, 1, eax, 5, avx512_bf16 , AVX-512 bFloat16 instructions + 0x7, 1, eax, 5, avx512_bf16 , AVX-512 bfloat16 instructions 0x7, 1, eax, 6, lass , Linear address space separation 0x7, 1, eax, 7, cmpccxadd , CMPccXADD instructions 0x7, 1, eax, 8, arch_perfmon_ext , ArchPerfmonExt: CPUID leaf 0x23 is supported @@ -287,7 +287,7 @@ 0x7, 1, eax, 12, fsrc , Fast Short REP CMPSB/SCASB 0x7, 1, eax, 17, fred , FRED: Flexible return and event delivery transitions 0x7, 1, eax, 18, lkgs , LKGS: Load 'kernel' (userspace) GS - 0x7, 1, eax, 19, wrmsrns , WRMSRNS instr (WRMSR-non-serializing) + 0x7, 1, eax, 19, wrmsrns , WRMSRNS instruction (WRMSR-non-serializing) 0x7, 1, eax, 20, nmi_src , NMI-source reporting with FRED event data 0x7, 1, eax, 21, amx_fp16 , AMX-FP16: FP16 tile operations 0x7, 1, eax, 22, hreset , History reset support @@ -348,18 +348,18 @@ 0xd, 0, eax, 0, xcr0_x87 , XCR0.X87 (bit 0) supported 0xd, 0, eax, 1, xcr0_sse , XCR0.SEE (bit 1) supported 0xd, 0, eax, 2, xcr0_avx , XCR0.AVX (bit 2) supported - 0xd, 0, eax, 3, xcr0_mpx_bndregs , XCR0.BNDREGS (bit 3) supported (MPX BND0-BND3 regs) - 0xd, 0, eax, 4, xcr0_mpx_bndcsr , XCR0.BNDCSR (bit 4) supported (MPX BNDCFGU/BNDSTATUS regs) - 0xd, 0, eax, 5, xcr0_avx512_opmask , XCR0.OPMASK (bit 5) supported (AVX-512 k0-k7 regs) - 0xd, 0, eax, 6, xcr0_avx512_zmm_hi256 , XCR0.ZMM_Hi256 (bit 6) supported (AVX-512 ZMM0->ZMM7/15 regs) - 0xd, 0, eax, 7, xcr0_avx512_hi16_zmm , XCR0.HI16_ZMM (bit 7) supported (AVX-512 ZMM16->ZMM31 regs) - 0xd, 0, eax, 9, xcr0_pkru , XCR0.PKRU (bit 9) supported (XSAVE PKRU reg) - 0xd, 0, eax, 11, xcr0_cet_u , AMD XCR0.CET_U (bit 11) supported (CET supervisor state) - 0xd, 0, eax, 12, xcr0_cet_s , AMD XCR0.CET_S (bit 12) support (CET user state) + 0xd, 0, eax, 3, xcr0_mpx_bndregs , XCR0.BNDREGS (bit 3) supported (MPX BND0-BND3 registers) + 0xd, 0, eax, 4, xcr0_mpx_bndcsr , XCR0.BNDCSR (bit 4) supported (MPX BNDCFGU/BNDSTATUS registers) + 0xd, 0, eax, 5, xcr0_avx512_opmask , XCR0.OPMASK (bit 5) supported (AVX-512 k0-k7 registers) + 0xd, 0, eax, 6, xcr0_avx512_zmm_hi256 , XCR0.ZMM_Hi256 (bit 6) supported (AVX-512 ZMM0->ZMM7/15 registers) + 0xd, 0, eax, 7, xcr0_avx512_hi16_zmm , XCR0.HI16_ZMM (bit 7) supported (AVX-512 ZMM16->ZMM31 registers) + 0xd, 0, eax, 9, xcr0_pkru , XCR0.PKRU (bit 9) supported (XSAVE PKRU registers) + 0xd, 0, eax, 11, xcr0_cet_u , XCR0.CET_U (bit 11) supported (CET user state) + 0xd, 0, eax, 12, xcr0_cet_s , XCR0.CET_S (bit 12) supported (CET supervisor state) 0xd, 0, eax, 17, xcr0_tileconfig , XCR0.TILECONFIG (bit 17) supported (AMX can manage TILECONFIG) 0xd, 0, eax, 18, xcr0_tiledata , XCR0.TILEDATA (bit 18) supported (AMX can manage TILEDATA) - 0xd, 0, ebx, 31:0, xsave_sz_xcr0_enabled , XSAVE/XRSTR area byte size, for XCR0 enabled features - 0xd, 0, ecx, 31:0, xsave_sz_max , XSAVE/XRSTR area max byte size, all CPU features + 0xd, 0, ebx, 31:0, xsave_sz_xcr0_enabled , XSAVE/XRSTOR area byte size, for XCR0 enabled features + 0xd, 0, ecx, 31:0, xsave_sz_max , XSAVE/XRSTOR area max byte size, all CPU features 0xd, 0, edx, 30, xcr0_lwp , AMD XCR0.LWP (bit 62) supported (Light-weight Profiling) 0xd, 1, eax, 0, xsaveopt , XSAVEOPT instruction 0xd, 1, eax, 1, xsavec , XSAVEC instruction @@ -378,7 +378,7 @@ 0xd, 63:2, eax, 31:0, xsave_sz , Size of save area for subleaf-N feature, in bytes 0xd, 63:2, ebx, 31:0, xsave_offset , Offset of save area for subleaf-N feature, in bytes 0xd, 63:2, ecx, 0, is_xss_bit , Subleaf N describes an XSS bit, otherwise XCR0 bit - 0xd, 63:2, ecx, 1, compacted_xsave_64byte_aligned, When compacted, subleaf-N feature xsave area is 64-byte aligned + 0xd, 63:2, ecx, 1, compacted_xsave_64byte_aligned, When compacted, subleaf-N feature XSAVE area is 64-byte aligned # Leaf FH # Intel RDT / AMD PQoS resource monitoring @@ -435,17 +435,17 @@ 0x12, 1, ecx, 0, xfrm_x87 , Enclave XFRM.X87 (bit 0) supported 0x12, 1, ecx, 1, xfrm_sse , Enclave XFRM.SEE (bit 1) supported 0x12, 1, ecx, 2, xfrm_avx , Enclave XFRM.AVX (bit 2) supported - 0x12, 1, ecx, 3, xfrm_mpx_bndregs , Enclave XFRM.BNDREGS (bit 3) supported (MPX BND0-BND3 regs) - 0x12, 1, ecx, 4, xfrm_mpx_bndcsr , Enclave XFRM.BNDCSR (bit 4) supported (MPX BNDCFGU/BNDSTATUS regs) - 0x12, 1, ecx, 5, xfrm_avx512_opmask , Enclave XFRM.OPMASK (bit 5) supported (AVX-512 k0-k7 regs) - 0x12, 1, ecx, 6, xfrm_avx512_zmm_hi256 , Enclave XFRM.ZMM_Hi256 (bit 6) supported (AVX-512 ZMM0->ZMM7/15 regs) - 0x12, 1, ecx, 7, xfrm_avx512_hi16_zmm , Enclave XFRM.HI16_ZMM (bit 7) supported (AVX-512 ZMM16->ZMM31 regs) - 0x12, 1, ecx, 9, xfrm_pkru , Enclave XFRM.PKRU (bit 9) supported (XSAVE PKRU reg) + 0x12, 1, ecx, 3, xfrm_mpx_bndregs , Enclave XFRM.BNDREGS (bit 3) supported (MPX BND0-BND3 registers) + 0x12, 1, ecx, 4, xfrm_mpx_bndcsr , Enclave XFRM.BNDCSR (bit 4) supported (MPX BNDCFGU/BNDSTATUS registers) + 0x12, 1, ecx, 5, xfrm_avx512_opmask , Enclave XFRM.OPMASK (bit 5) supported (AVX-512 k0-k7 registers) + 0x12, 1, ecx, 6, xfrm_avx512_zmm_hi256 , Enclave XFRM.ZMM_Hi256 (bit 6) supported (AVX-512 ZMM0->ZMM7/15 registers) + 0x12, 1, ecx, 7, xfrm_avx512_hi16_zmm , Enclave XFRM.HI16_ZMM (bit 7) supported (AVX-512 ZMM16->ZMM31 registers) + 0x12, 1, ecx, 9, xfrm_pkru , Enclave XFRM.PKRU (bit 9) supported (XSAVE PKRU registers) 0x12, 1, ecx, 17, xfrm_tileconfig , Enclave XFRM.TILECONFIG (bit 17) supported (AMX can manage TILECONFIG) 0x12, 1, ecx, 18, xfrm_tiledata , Enclave XFRM.TILEDATA (bit 18) supported (AMX can manage TILEDATA) 0x12, 31:2, eax, 3:0, subleaf_type , Subleaf type (dictates output layout) - 0x12, 31:2, eax, 31:12, epc_sec_base_addr_0 , EPC section base addr, bits[12:31] - 0x12, 31:2, ebx, 19:0, epc_sec_base_addr_1 , EPC section base addr, bits[32:51] + 0x12, 31:2, eax, 31:12, epc_sec_base_addr_0 , EPC section base address, bits[12:31] + 0x12, 31:2, ebx, 19:0, epc_sec_base_addr_1 , EPC section base address, bits[32:51] 0x12, 31:2, ecx, 3:0, epc_sec_type , EPC section type / property encoding 0x12, 31:2, ecx, 31:12, epc_sec_size_0 , EPC section size, bits[12:31] 0x12, 31:2, edx, 19:0, epc_sec_size_1 , EPC section size, bits[32:51] @@ -481,7 +481,7 @@ 0x15, 0, ecx, 31:0, cpu_crystal_hz , Core crystal clock nominal frequency, in Hz # Leaf 16H -# Intel processor fequency enumeration +# Intel processor frequency enumeration 0x16, 0, eax, 15:0, cpu_base_mhz , Processor base frequency, in MHz 0x16, 0, ebx, 15:0, cpu_max_mhz , Processor max frequency, in MHz @@ -492,7 +492,7 @@ 0x17, 0, eax, 31:0, soc_max_subleaf , Max cpuid leaf 0x17 subleaf 0x17, 0, ebx, 15:0, soc_vendor_id , SoC vendor ID - 0x17, 0, ebx, 16, is_vendor_scheme , Assigned by industry enumaeratoion scheme (not Intel) + 0x17, 0, ebx, 16, is_vendor_scheme , Assigned by industry enumeration scheme (not Intel) 0x17, 0, ecx, 31:0, soc_proj_id , SoC project ID, assigned by vendor 0x17, 0, edx, 31:0, soc_stepping_id , Soc project stepping ID, assigned by vendor 0x17, 3:1, eax, 31:0, vendor_brand_a , Vendor Brand ID string, bytes subleaf_nr * (0 -> 3) @@ -508,13 +508,13 @@ 0x18, 31:0, ebx, 1, tlb_2m_page , TLB 2MB-page entries supported 0x18, 31:0, ebx, 2, tlb_4m_page , TLB 4MB-page entries supported 0x18, 31:0, ebx, 3, tlb_1g_page , TLB 1GB-page entries supported - 0x18, 31:0, ebx, 10:8, hard_partitioning , (Hard/Soft) partitioning between logical CPUs sharing this struct + 0x18, 31:0, ebx, 10:8, hard_partitioning , (Hard/Soft) partitioning between logical CPUs sharing this structure 0x18, 31:0, ebx, 31:16, n_way_associative , Ways of associativity 0x18, 31:0, ecx, 31:0, n_sets , Number of sets 0x18, 31:0, edx, 4:0, tlb_type , Translation cache type (TLB type) 0x18, 31:0, edx, 7:5, tlb_cache_level , Translation cache level (1-based) 0x18, 31:0, edx, 8, is_fully_associative , Fully-associative structure - 0x18, 31:0, edx, 25:14, tlb_max_addressible_ids, Max num of addressible IDs for logical CPUs sharing this TLB - 1 + 0x18, 31:0, edx, 25:14, tlb_max_addressible_ids, Max number of addressable IDs for logical CPUs sharing this TLB - 1 # Leaf 19H # Intel Key Locker enumeration @@ -577,7 +577,7 @@ # Intel AMX, TMUL (Tile-matrix MULtiply) accelerator unit enumeration 0x1e, 0, ebx, 7:0, tmul_maxk , TMUL unit maximum height, K (rows or columns) - 0x1e, 0, ebx, 23:8, tmul_maxn , TMUL unit maxiumum SIMD dimension, N (column bytes) + 0x1e, 0, ebx, 23:8, tmul_maxn , TMUL unit maximum SIMD dimension, N (column bytes) # Leaf 1FH # Intel extended topology enumeration v2 @@ -733,9 +733,9 @@ # Leaf 80000005H # AMD/Transmeta L1 cache and L1 TLB enumeration -0x80000005, 0, eax, 7:0, l1_itlb_2m_4m_nentries , L1 ITLB #entires, 2M and 4M pages +0x80000005, 0, eax, 7:0, l1_itlb_2m_4m_nentries , L1 ITLB #entries, 2M and 4M pages 0x80000005, 0, eax, 15:8, l1_itlb_2m_4m_assoc , L1 ITLB associativity, 2M and 4M pages -0x80000005, 0, eax, 23:16, l1_dtlb_2m_4m_nentries , L1 DTLB #entires, 2M and 4M pages +0x80000005, 0, eax, 23:16, l1_dtlb_2m_4m_nentries , L1 DTLB #entries, 2M and 4M pages 0x80000005, 0, eax, 31:24, l1_dtlb_2m_4m_assoc , L1 DTLB associativity, 2M and 4M pages 0x80000005, 0, ebx, 7:0, l1_itlb_4k_nentries , L1 ITLB #entries, 4K pages 0x80000005, 0, ebx, 15:8, l1_itlb_4k_assoc , L1 ITLB associativity, 4K pages @@ -774,11 +774,11 @@ # CPU power management (mostly AMD) and AMD RAS enumeration 0x80000007, 0, ebx, 0, overflow_recov , MCA overflow conditions not fatal -0x80000007, 0, ebx, 1, succor , Software containment of UnCORRectable errors +0x80000007, 0, ebx, 1, succor , Software containment of uncorrectable errors 0x80000007, 0, ebx, 2, hw_assert , Hardware assert MSRs 0x80000007, 0, ebx, 3, smca , Scalable MCA (MCAX MSRs) 0x80000007, 0, ecx, 31:0, cpu_pwr_sample_ratio , CPU power sample time ratio -0x80000007, 0, edx, 0, digital_temp , Digital temprature sensor +0x80000007, 0, edx, 0, digital_temp , Digital temperature sensor 0x80000007, 0, edx, 1, powernow_freq_id , PowerNOW! frequency scaling 0x80000007, 0, edx, 2, powernow_volt_id , PowerNOW! voltage scaling 0x80000007, 0, edx, 3, thermal_trip , THERMTRIP (Thermal Trip) @@ -821,7 +821,7 @@ 0x80000008, 0, ebx, 23, amd_ppin , Protected Processor Inventory Number 0x80000008, 0, ebx, 24, amd_ssbd , Speculative Store Bypass Disable 0x80000008, 0, ebx, 25, virt_ssbd , virtualized SSBD (Speculative Store Bypass Disable) -0x80000008, 0, ebx, 26, amd_ssb_no , SSBD not needed (fixed in HW) +0x80000008, 0, ebx, 26, amd_ssb_no , SSBD is not needed (fixed in hardware) 0x80000008, 0, ebx, 27, cppc , Collaborative Processor Performance Control 0x80000008, 0, ebx, 28, amd_psfd , Predictive Store Forward Disable 0x80000008, 0, ebx, 29, btc_no , CPU not affected by Branch Type Confusion @@ -849,7 +849,7 @@ 0x8000000a, 0, edx, 10, pausefilter , Pause intercept filter 0x8000000a, 0, edx, 12, pfthreshold , Pause filter threshold 0x8000000a, 0, edx, 13, avic , Advanced virtual interrupt controller -0x8000000a, 0, edx, 15, v_vmsave_vmload , Virtual VMSAVE/VMLOAD (nested virt) +0x8000000a, 0, edx, 15, v_vmsave_vmload , Virtual VMSAVE/VMLOAD (nested virtualization) 0x8000000a, 0, edx, 16, vgif , Virtualize the Global Interrupt Flag 0x8000000a, 0, edx, 17, gmet , Guest mode execution trap 0x8000000a, 0, edx, 18, x2avic , Virtual x2APIC @@ -861,7 +861,7 @@ 0x8000000a, 0, edx, 25, vnmi , NMI virtualization 0x8000000a, 0, edx, 26, ibs_virt , IBS Virtualization 0x8000000a, 0, edx, 27, ext_lvt_off_chg , Extended LVT offset fault change -0x8000000a, 0, edx, 28, svme_addr_chk , Guest SVME addr check +0x8000000a, 0, edx, 28, svme_addr_chk , Guest SVME address check # Leaf 80000019H # AMD TLB 1G-pages enumeration @@ -902,20 +902,20 @@ # AMD LWP (Lightweight Profiling) 0x8000001c, 0, eax, 0, os_lwp_avail , LWP is available to application programs (supported by OS) -0x8000001c, 0, eax, 1, os_lpwval , LWPVAL instruction (EventId=1) is supported by OS -0x8000001c, 0, eax, 2, os_lwp_ire , Instructions Retired Event (EventId=2) is supported by OS -0x8000001c, 0, eax, 3, os_lwp_bre , Branch Retired Event (EventId=3) is supported by OS -0x8000001c, 0, eax, 4, os_lwp_dme , DCache Miss Event (EventId=4) is supported by OS -0x8000001c, 0, eax, 5, os_lwp_cnh , CPU Clocks Not Halted event (EventId=5) is supported by OS -0x8000001c, 0, eax, 6, os_lwp_rnh , CPU Reference clocks Not Halted event (EventId=6) is supported by OS +0x8000001c, 0, eax, 1, os_lpwval , LWPVAL instruction is supported by OS +0x8000001c, 0, eax, 2, os_lwp_ire , Instructions Retired Event is supported by OS +0x8000001c, 0, eax, 3, os_lwp_bre , Branch Retired Event is supported by OS +0x8000001c, 0, eax, 4, os_lwp_dme , Dcache Miss Event is supported by OS +0x8000001c, 0, eax, 5, os_lwp_cnh , CPU Clocks Not Halted event is supported by OS +0x8000001c, 0, eax, 6, os_lwp_rnh , CPU Reference clocks Not Halted event is supported by OS 0x8000001c, 0, eax, 29, os_lwp_cont , LWP sampling in continuous mode is supported by OS 0x8000001c, 0, eax, 30, os_lwp_ptsc , Performance Time Stamp Counter in event records is supported by OS 0x8000001c, 0, eax, 31, os_lwp_int , Interrupt on threshold overflow is supported by OS 0x8000001c, 0, ebx, 7:0, lwp_lwpcb_sz , LWP Control Block size, in quadwords 0x8000001c, 0, ebx, 15:8, lwp_event_sz , LWP event record size, in bytes -0x8000001c, 0, ebx, 23:16, lwp_max_events , LWP max supported EventId value (EventID 255 not included) +0x8000001c, 0, ebx, 23:16, lwp_max_events , LWP max supported EventID value (EventID 255 not included) 0x8000001c, 0, ebx, 31:24, lwp_event_offset , LWP events area offset in the LWP Control Block -0x8000001c, 0, ecx, 4:0, lwp_latency_max , Num of bits in cache latency counters (10 to 31) +0x8000001c, 0, ecx, 4:0, lwp_latency_max , Number of bits in cache latency counters (10 to 31) 0x8000001c, 0, ecx, 5, lwp_data_adddr , Cache miss events report the data address of the reference 0x8000001c, 0, ecx, 8:6, lwp_latency_rnd , Amount by which cache latency is rounded 0x8000001c, 0, ecx, 15:9, lwp_version , LWP implementation version @@ -924,16 +924,16 @@ 0x8000001c, 0, ecx, 29, lwp_ip_filtering , IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP) supported 0x8000001c, 0, ecx, 30, lwp_cache_levels , Cache-related events can be filtered by cache level 0x8000001c, 0, ecx, 31, lwp_cache_latency , Cache-related events can be filtered by latency -0x8000001c, 0, edx, 0, hw_lwp_avail , LWP is available in Hardware -0x8000001c, 0, edx, 1, hw_lpwval , LWPVAL instruction (EventId=1) is available in HW -0x8000001c, 0, edx, 2, hw_lwp_ire , Instructions Retired Event (EventId=2) is available in HW -0x8000001c, 0, edx, 3, hw_lwp_bre , Branch Retired Event (EventId=3) is available in HW -0x8000001c, 0, edx, 4, hw_lwp_dme , DCache Miss Event (EventId=4) is available in HW -0x8000001c, 0, edx, 5, hw_lwp_cnh , CPU Clocks Not Halted event (EventId=5) is available in HW -0x8000001c, 0, edx, 6, hw_lwp_rnh , CPU Reference clocks Not Halted event (EventId=6) is available in HW -0x8000001c, 0, edx, 29, hw_lwp_cont , LWP sampling in continuous mode is available in HW -0x8000001c, 0, edx, 30, hw_lwp_ptsc , Performance Time Stamp Counter in event records is available in HW -0x8000001c, 0, edx, 31, hw_lwp_int , Interrupt on threshold overflow is available in HW +0x8000001c, 0, edx, 0, hw_lwp_avail , LWP is available in hardware +0x8000001c, 0, edx, 1, hw_lpwval , LWPVAL instruction is available in hardware +0x8000001c, 0, edx, 2, hw_lwp_ire , Instructions Retired Event is available in hardware +0x8000001c, 0, edx, 3, hw_lwp_bre , Branch Retired Event is available in hardware +0x8000001c, 0, edx, 4, hw_lwp_dme , Dcache Miss Event is available in hardware +0x8000001c, 0, edx, 5, hw_lwp_cnh , Clocks Not Halted event is available in hardware +0x8000001c, 0, edx, 6, hw_lwp_rnh , Reference clocks Not Halted event is available in hardware +0x8000001c, 0, edx, 29, hw_lwp_cont , LWP sampling in continuous mode is available in hardware +0x8000001c, 0, edx, 30, hw_lwp_ptsc , Performance Time Stamp Counter in event records is available in hardware +0x8000001c, 0, edx, 31, hw_lwp_int , Interrupt on threshold overflow is available in hardware # Leaf 8000001DH # AMD deterministic cache parameters @@ -969,10 +969,10 @@ 0x8000001f, 0, eax, 4, sev_nested_paging , SEV secure nested paging supported 0x8000001f, 0, eax, 5, vm_permission_levels , VMPL supported 0x8000001f, 0, eax, 6, rpmquery , RPMQUERY instruction supported -0x8000001f, 0, eax, 7, vmpl_sss , VMPL supervisor shadwo stack supported +0x8000001f, 0, eax, 7, vmpl_sss , VMPL supervisor shadow stack supported 0x8000001f, 0, eax, 8, secure_tsc , Secure TSC supported 0x8000001f, 0, eax, 9, v_tsc_aux , Hardware virtualizes TSC_AUX -0x8000001f, 0, eax, 10, sme_coherent , HW enforces cache coherency across encryption domains +0x8000001f, 0, eax, 10, sme_coherent , Cache coherency is enforced across encryption domains 0x8000001f, 0, eax, 11, req_64bit_hypervisor , SEV guest mandates 64-bit hypervisor 0x8000001f, 0, eax, 12, restricted_injection , Restricted Injection supported 0x8000001f, 0, eax, 13, alternate_injection , Alternate Injection supported @@ -984,13 +984,13 @@ 0x8000001f, 0, eax, 19, virt_ibs , IBS state virtualization is supported for SEV-ES guests 0x8000001f, 0, eax, 24, vmsa_reg_protection , VMSA register protection is supported 0x8000001f, 0, eax, 25, smt_protection , SMT protection is supported -0x8000001f, 0, eax, 28, svsm_page_msr , SVSM communication page MSR (0xc001f000h) is supported +0x8000001f, 0, eax, 28, svsm_page_msr , SVSM communication page MSR (0xc001f000) is supported 0x8000001f, 0, eax, 29, nested_virt_snp_msr , VIRT_RMPUPDATE/VIRT_PSMASH MSRs are supported 0x8000001f, 0, ebx, 5:0, pte_cbit_pos , PTE bit number used to enable memory encryption 0x8000001f, 0, ebx, 11:6, phys_addr_reduction_nbits, Reduction of phys address space when encryption is enabled, in bits 0x8000001f, 0, ebx, 15:12, vmpl_count , Number of VM permission levels (VMPL) supported 0x8000001f, 0, ecx, 31:0, enc_guests_max , Max supported number of simultaneous encrypted guests -0x8000001f, 0, edx, 31:0, min_sev_asid_no_sev_es , Mininum ASID for SEV-enabled SEV-ES-disabled guest +0x8000001f, 0, edx, 31:0, min_sev_asid_no_sev_es , Minimum ASID for SEV-enabled SEV-ES-disabled guest # Leaf 80000020H # AMD Platform QoS extended feature IDs @@ -999,6 +999,8 @@ 0x80000020, 0, ebx, 2, smba , Slow Memory Bandwidth Allocation support 0x80000020, 0, ebx, 3, bmec , Bandwidth Monitoring Event Configuration support 0x80000020, 0, ebx, 4, l3rr , L3 Range Reservation support +0x80000020, 0, ebx, 5, abmc , Assignable Bandwidth Monitoring Counters +0x80000020, 0, ebx, 6, sdciae , Smart Data Cache Injection (SDCI) Allocation Enforcement 0x80000020, 1, eax, 31:0, mba_limit_len , MBA enforcement limit size 0x80000020, 1, edx, 31:0, mba_cos_max , MBA max Class of Service number (zero-based) 0x80000020, 2, eax, 31:0, smba_limit_len , SMBA enforcement limit size @@ -1023,12 +1025,21 @@ 0x80000021, 0, eax, 7, upper_addr_ignore , EFER MSR Upper Address Ignore Enable bit supported 0x80000021, 0, eax, 8, autoibrs , EFER MSR Automatic IBRS enable bit supported 0x80000021, 0, eax, 9, no_smm_ctl_msr , SMM_CTL MSR (0xc0010116) is not present -0x80000021, 0, eax, 10, fsrs_supported , Fast Short Rep Stosb (FSRS) is supported -0x80000021, 0, eax, 11, fsrc_supported , Fast Short Repe Cmpsb (FSRC) is supported +0x80000021, 0, eax, 10, fsrs_supported , Fast Short Rep STOSB (FSRS) is supported +0x80000021, 0, eax, 11, fsrc_supported , Fast Short Rep CMPSB (FSRC) is supported 0x80000021, 0, eax, 13, prefetch_ctl_msr , Prefetch control MSR is supported +0x80000021, 0, eax, 16, opcode_reclaim , Reserves opcode space 0x80000021, 0, eax, 17, user_cpuid_disable , #GP when executing CPUID at CPL > 0 is supported 0x80000021, 0, eax, 18, epsf_supported , Enhanced Predictive Store Forwarding (EPSF) is supported -0x80000021, 0, ebx, 11:0, microcode_patch_size , Size of microcode patch, in 16-byte units +0x80000021, 0, eax, 22, wl_feedback , Workload-based heuristic feedback to OS +0x80000021, 0, eax, 24, eraps_support , Enhanced Return Address Predictor Security +0x80000021, 0, eax, 27, sbpb , Support for the Selective Branch Predictor Barrier +0x80000021, 0, eax, 28, ibpb_brtype , Branch predictions flushed from CPU branch predictor +0x80000021, 0, eax, 29, srso_no , CPU is not subject to the SRSO vulnerability +0x80000021, 0, eax, 30, srso_uk_no , CPU is not vulnerable to SRSO at user-kernel boundary +0x80000021, 0, eax, 31, srso_msr_fix , Software may use MSR BP_CFG[BpSpecReduce] to mitigate SRSO +0x80000021, 0, ebx, 15:0, microcode_patch_size , Size of microcode patch, in 16-byte units +0x80000021, 0, ebx, 23:16, rap_size , Return Address Predictor size # Leaf 80000022H # AMD Performance Monitoring v2 enumeration @@ -1036,7 +1047,7 @@ 0x80000022, 0, eax, 0, perfmon_v2 , Performance monitoring v2 supported 0x80000022, 0, eax, 1, lbr_v2 , Last Branch Record v2 extensions (LBR Stack) 0x80000022, 0, eax, 2, lbr_pmc_freeze , Freezing core performance counters / LBR Stack supported -0x80000022, 0, ebx, 3:0, n_pmc_core , Number of core perfomance counters +0x80000022, 0, ebx, 3:0, n_pmc_core , Number of core performance counters 0x80000022, 0, ebx, 9:4, lbr_v2_stack_size , Number of available LBR stack entries 0x80000022, 0, ebx, 15:10, n_pmc_northbridge , Number of available northbridge (data fabric) performance counters 0x80000022, 0, ebx, 21:16, n_pmc_umc , Number of available UMC performance counters @@ -1046,7 +1057,7 @@ # AMD Secure Multi-key Encryption enumeration 0x80000023, 0, eax, 0, mem_hmk_mode , MEM-HMK encryption mode is supported -0x80000023, 0, ebx, 15:0, mem_hmk_avail_keys , MEM-HMK mode: total num of available encryption keys +0x80000023, 0, ebx, 15:0, mem_hmk_avail_keys , MEM-HMK mode: total number of available encryption keys # Leaf 80000026H # AMD extended topology enumeration v2 @@ -1134,7 +1145,7 @@ 0x80860007, 0, eax, 31:0, cpu_cur_mhz , Current CPU frequency, in MHz 0x80860007, 0, ebx, 31:0, cpu_cur_voltage , Current CPU voltage, in millivolts -0x80860007, 0, ecx, 31:0, cpu_cur_perf_pctg , Current CPU performance percentage, 0 - 100d +0x80860007, 0, ecx, 31:0, cpu_cur_perf_pctg , Current CPU performance percentage, 0 - 100 0x80860007, 0, edx, 31:0, cpu_cur_gate_delay , Current CPU gate delay, in femtoseconds # Leaf C0000000H From 5e0c3c5e95f09236e5f7a73615bfa2666c58f182 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:39 +0100 Subject: [PATCH 018/331] tools/x86/kcpuid: Update bitfields to x86-cpuid-db v2.2 Update kcpuid's CSV file to version 2.2, as generated by x86-cpuid-db. Per Ingo Molnar's feedback, it is desired to always use CPUID in its capitalized form. The v2.2 release fixed all instances of small case "cpuid" at the project's XML database, and thus all of its generated files. Reported-by: Ingo Molnar Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/blob/v2.2/CHANGELOG.rst Link: https://lore.kernel.org/r/20250324142042.29010-19-darwi@linutronix.de Closes: https://lkml.kernel.org/r/Z8bHK391zKE4gUEW@gmail.com --- tools/arch/x86/kcpuid/cpuid.csv | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index 0f9c11208674..9613e09cbfb3 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v2.1 +# Generator: x86-cpuid-db v2.2 # # Auto-generated file. @@ -12,7 +12,7 @@ # Leaf 0H # Maximum standard leaf number + CPU vendor string - 0x0, 0, eax, 31:0, max_std_leaf , Highest cpuid standard leaf supported + 0x0, 0, eax, 31:0, max_std_leaf , Highest standard CPUID leaf supported 0x0, 0, ebx, 31:0, cpu_vendorid_0 , CPU vendor ID string bytes 0 - 3 0x0, 0, ecx, 31:0, cpu_vendorid_2 , CPU vendor ID string bytes 8 - 11 0x0, 0, edx, 31:0, cpu_vendorid_1 , CPU vendor ID string bytes 4 - 7 @@ -193,7 +193,7 @@ # Leaf 7H # Extended CPU features enumeration - 0x7, 0, eax, 31:0, leaf7_n_subleaves , Number of cpuid 0x7 subleaves + 0x7, 0, eax, 31:0, leaf7_n_subleaves , Number of leaf 0x7 subleaves 0x7, 0, ebx, 0, fsgsbase , FSBASE/GSBASE read/write support 0x7, 0, ebx, 1, tsc_adjust , IA32_TSC_ADJUST MSR supported 0x7, 0, ebx, 2, sgx , Intel SGX (Software Guard Extensions) @@ -281,7 +281,7 @@ 0x7, 1, eax, 5, avx512_bf16 , AVX-512 bfloat16 instructions 0x7, 1, eax, 6, lass , Linear address space separation 0x7, 1, eax, 7, cmpccxadd , CMPccXADD instructions - 0x7, 1, eax, 8, arch_perfmon_ext , ArchPerfmonExt: CPUID leaf 0x23 is supported + 0x7, 1, eax, 8, arch_perfmon_ext , ArchPerfmonExt: leaf 0x23 is supported 0x7, 1, eax, 10, fzrm , Fast zero-length REP MOVSB 0x7, 1, eax, 11, fsrs , Fast short REP STOSB 0x7, 1, eax, 12, fsrc , Fast Short REP CMPSB/SCASB @@ -319,7 +319,7 @@ 0xa, 0, eax, 7:0, pmu_version , Performance monitoring unit version ID 0xa, 0, eax, 15:8, pmu_n_gcounters , Number of general PMU counters per logical CPU 0xa, 0, eax, 23:16, pmu_gcounters_nbits , Bitwidth of PMU general counters - 0xa, 0, eax, 31:24, pmu_cpuid_ebx_bits , Length of cpuid leaf 0xa EBX bit vector + 0xa, 0, eax, 31:24, pmu_cpuid_ebx_bits , Length of leaf 0xa EBX bit vector 0xa, 0, ebx, 0, no_core_cycle_evt , Core cycle event not available 0xa, 0, ebx, 1, no_insn_retired_evt , Instruction retired event not available 0xa, 0, ebx, 2, no_refcycle_evt , Reference cycles event not available @@ -453,7 +453,7 @@ # Leaf 14H # Intel Processor Trace enumeration - 0x14, 0, eax, 31:0, pt_max_subleaf , Max cpuid 0x14 subleaf + 0x14, 0, eax, 31:0, pt_max_subleaf , Maximum leaf 0x14 subleaf 0x14, 0, ebx, 0, cr3_filtering , IA32_RTIT_CR3_MATCH is accessible 0x14, 0, ebx, 1, psb_cyc , Configurable PSB and cycle-accurate mode 0x14, 0, ebx, 2, ip_filtering , IP/TraceStop filtering; Warm-reset PT MSRs preservation @@ -490,7 +490,7 @@ # Leaf 17H # Intel SoC vendor attributes enumeration - 0x17, 0, eax, 31:0, soc_max_subleaf , Max cpuid leaf 0x17 subleaf + 0x17, 0, eax, 31:0, soc_max_subleaf , Maximum leaf 0x17 subleaf 0x17, 0, ebx, 15:0, soc_vendor_id , SoC vendor ID 0x17, 0, ebx, 16, is_vendor_scheme , Assigned by industry enumeration scheme (not Intel) 0x17, 0, ecx, 31:0, soc_proj_id , SoC project ID, assigned by vendor @@ -503,7 +503,7 @@ # Leaf 18H # Intel determenestic address translation (TLB) parameters - 0x18, 31:0, eax, 31:0, tlb_max_subleaf , Max cpuid 0x18 subleaf + 0x18, 31:0, eax, 31:0, tlb_max_subleaf , Maximum leaf 0x18 subleaf 0x18, 31:0, ebx, 0, tlb_4k_page , TLB 4KB-page entries supported 0x18, 31:0, ebx, 1, tlb_2m_page , TLB 2MB-page entries supported 0x18, 31:0, ebx, 2, tlb_4m_page , TLB 4MB-page entries supported @@ -634,7 +634,7 @@ # Leaf 80000000H # Maximum extended leaf number + AMD/Transmeta CPU vendor string -0x80000000, 0, eax, 31:0, max_ext_leaf , Maximum extended cpuid leaf supported +0x80000000, 0, eax, 31:0, max_ext_leaf , Maximum extended CPUID leaf supported 0x80000000, 0, ebx, 31:0, cpu_vendorid_0 , Vendor ID string bytes 0 - 3 0x80000000, 0, ecx, 31:0, cpu_vendorid_2 , Vendor ID string bytes 8 - 11 0x80000000, 0, edx, 31:0, cpu_vendorid_1 , Vendor ID string bytes 4 - 7 @@ -669,7 +669,7 @@ 0x80000001, 0, ecx, 17, tce , Translation cache extension 0x80000001, 0, ecx, 19, nodeid_msr , NodeId MSR (0xc001100c) 0x80000001, 0, ecx, 21, tbm , Trailing bit manipulations -0x80000001, 0, ecx, 22, topoext , Topology Extensions (cpuid leaf 0x8000001d) +0x80000001, 0, ecx, 22, topoext , Topology Extensions (leaf 0x8000001d) 0x80000001, 0, ecx, 23, perfctr_core , Core performance counter extensions 0x80000001, 0, ecx, 24, perfctr_nb , NB/DF performance counter extensions 0x80000001, 0, ecx, 26, bpext , Data access breakpoint extension From 300ba891418ae72df8d16fc26886f1d049c70907 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:40 +0100 Subject: [PATCH 019/331] tools/x86/kcpuid: Update bitfields to x86-cpuid-db v2.3 Update kcpuid's CSV file to version 2.3, as generated by x86-cpuid-db. Summary of the v2.3 changes: * Per H. Peter Anvin's feedback, leaf 0x3 is not unique to Transmeta as the CSV file earlier claimed. Since leaf 0x3's format differs between Intel and Transmeta, and the project does not yet support having the same CPUID bitfield with varying interpretations across vendors, leaf 0x3 is removed for now. Given that Intel discontinued support for PSN from Pentium 4 onward, and Linux force disables it on early boot for privacy concerns, this should have minimal impact. * Leaf 0x80000021: Make bitfield IDs and descriptions coherent with each other. Remove "_support" from bitfield IDs, as no other leaf has such convention. Reported-by: "H. Peter Anvin" Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/blob/v2.3/CHANGELOG.rst Link: https://lore.kernel.org/r/20250324142042.29010-20-darwi@linutronix.de Closes: https://lkml.kernel.org/r/C7684E03-36E0-4D58-B6F0-78F4DB82D737@zytor.com --- tools/arch/x86/kcpuid/cpuid.csv | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index 9613e09cbfb3..8d25b0b49f3b 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v2.2 +# Generator: x86-cpuid-db v2.3 # # Auto-generated file. @@ -116,14 +116,6 @@ 0x2, 0, edx, 30:24, desc15 , Descriptor #15 0x2, 0, edx, 31, edx_invalid , Descriptors 12-15 are invalid if set -# Leaf 3H -# Transmeta Processor Serial Number (PSN) - - 0x3, 0, eax, 31:0, cpu_psn_0 , Processor Serial Number bytes 0 - 3 - 0x3, 0, ebx, 31:0, cpu_psn_1 , Processor Serial Number bytes 4 - 7 - 0x3, 0, ecx, 31:0, cpu_psn_2 , Processor Serial Number bytes 8 - 11 - 0x3, 0, edx, 31:0, cpu_psn_3 , Processor Serial Number bytes 12 - 15 - # Leaf 4H # Intel deterministic cache parameters @@ -1020,20 +1012,20 @@ 0x80000021, 0, eax, 0, no_nested_data_bp , No nested data breakpoints 0x80000021, 0, eax, 1, fsgs_non_serializing , WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing 0x80000021, 0, eax, 2, lfence_rdtsc , LFENCE always serializing / synchronizes RDTSC -0x80000021, 0, eax, 3, smm_page_cfg_lock , SMM paging configuration lock is supported +0x80000021, 0, eax, 3, smm_page_cfg_lock , SMM paging configuration lock 0x80000021, 0, eax, 6, null_sel_clr_base , Null selector clears base -0x80000021, 0, eax, 7, upper_addr_ignore , EFER MSR Upper Address Ignore Enable bit supported -0x80000021, 0, eax, 8, autoibrs , EFER MSR Automatic IBRS enable bit supported -0x80000021, 0, eax, 9, no_smm_ctl_msr , SMM_CTL MSR (0xc0010116) is not present -0x80000021, 0, eax, 10, fsrs_supported , Fast Short Rep STOSB (FSRS) is supported -0x80000021, 0, eax, 11, fsrc_supported , Fast Short Rep CMPSB (FSRC) is supported -0x80000021, 0, eax, 13, prefetch_ctl_msr , Prefetch control MSR is supported +0x80000021, 0, eax, 7, upper_addr_ignore , EFER MSR Upper Address Ignore +0x80000021, 0, eax, 8, autoibrs , EFER MSR Automatic IBRS +0x80000021, 0, eax, 9, no_smm_ctl_msr , SMM_CTL MSR (0xc0010116) is not available +0x80000021, 0, eax, 10, fsrs , Fast Short Rep STOSB +0x80000021, 0, eax, 11, fsrc , Fast Short Rep CMPSB +0x80000021, 0, eax, 13, prefetch_ctl_msr , Prefetch control MSR is available 0x80000021, 0, eax, 16, opcode_reclaim , Reserves opcode space 0x80000021, 0, eax, 17, user_cpuid_disable , #GP when executing CPUID at CPL > 0 is supported -0x80000021, 0, eax, 18, epsf_supported , Enhanced Predictive Store Forwarding (EPSF) is supported +0x80000021, 0, eax, 18, epsf , Enhanced Predictive Store Forwarding 0x80000021, 0, eax, 22, wl_feedback , Workload-based heuristic feedback to OS -0x80000021, 0, eax, 24, eraps_support , Enhanced Return Address Predictor Security -0x80000021, 0, eax, 27, sbpb , Support for the Selective Branch Predictor Barrier +0x80000021, 0, eax, 24, eraps , Enhanced Return Address Predictor Security +0x80000021, 0, eax, 27, sbpb , Selective Branch Predictor Barrier 0x80000021, 0, eax, 28, ibpb_brtype , Branch predictions flushed from CPU branch predictor 0x80000021, 0, eax, 29, srso_no , CPU is not subject to the SRSO vulnerability 0x80000021, 0, eax, 30, srso_uk_no , CPU is not vulnerable to SRSO at user-kernel boundary From 0efb4dc3b084579efe7f5f88370b76d4f1c3bcee Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 15:20:41 +0100 Subject: [PATCH 020/331] MAINTAINERS: Include the entire kcpuid/ directory under the X86 CPUID DATABASE entry kcpuid's CSV file is covered by the "x86 CPUID database" MAINTAINERS entry. Recent patches have shown that changes to that file may require updates to the kcpuid code, so include the whole of tools/x86/kcpuid/ under the same entry. This also ensures that myself and the x86-cpuid mailing list are CCed on future kcpuid patches. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250324142042.29010-21-darwi@linutronix.de --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3b5fa8436987..47dbac5517bb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -25727,7 +25727,7 @@ R: Ahmed S. Darwish L: x86-cpuid@lists.linux.dev S: Maintained W: https://x86-cpuid.org -F: tools/arch/x86/kcpuid/cpuid.csv +F: tools/arch/x86/kcpuid/ X86 ENTRY CODE M: Andy Lutomirski From b5969494c8d8a2384c0636707a538efad15d8660 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:32:56 +0100 Subject: [PATCH 021/331] x86/cpu: Remove CPUID leaf 0x2 parsing loop Leaf 0x2 output includes a "query count" byte where it was supposed to specify the number of repeated CPUID leaf 0x2 subleaf 0 queries needed to extract all of the CPU's cache and TLB descriptors. Per current Intel manuals, all CPUs supporting this leaf "will always" return an iteration count of 1. Remove the leaf 0x2 query loop and just query the hardware once. Note, as previously done in: aec28d852ed2 ("x86/cpuid: Standardize on u32 in ") standardize on using 'u32' and 'u8' types. Suggested-by: Ingo Molnar Suggested-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-2-darwi@linutronix.de --- arch/x86/kernel/cpu/intel.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 4cbb2e69bea1..0570d4d86006 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -6,6 +6,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 #include @@ -777,28 +778,27 @@ static void intel_tlb_lookup(const unsigned char desc) static void intel_detect_tlb(struct cpuinfo_x86 *c) { - int i, j, n; - unsigned int regs[4]; - unsigned char *desc = (unsigned char *)regs; + u32 regs[4]; + u8 *desc = (u8 *)regs; if (c->cpuid_level < 2) return; - /* Number of times to iterate */ - n = cpuid_eax(2) & 0xFF; + cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); - for (i = 0 ; i < n ; i++) { - cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); + /* Intel CPUs must report an iteration count of 1 */ + if (desc[0] != 0x01) + return; - /* If bit 31 is set, this is an unknown format */ - for (j = 0 ; j < 4 ; j++) - if (regs[j] & (1 << 31)) - regs[j] = 0; - - /* Byte 0 is level count, not a descriptor */ - for (j = 1 ; j < 16 ; j++) - intel_tlb_lookup(desc[j]); + /* If a register's bit 31 is set, it is an unknown format */ + for (int i = 0; i < 4; i++) { + if (regs[i] & (1 << 31)) + regs[i] = 0; } + + /* Skip the first byte as it is not a descriptor */ + for (int i = 1; i < 16; i++) + intel_tlb_lookup(desc[i]); } static const struct cpu_dev intel_cpu_dev = { From 09a1da4beb310420b720994f7b6599e8d0bce3e1 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:32:57 +0100 Subject: [PATCH 022/331] x86/cacheinfo: Remove CPUID leaf 0x2 parsing loop Leaf 0x2 output includes a "query count" byte where it was supposed to specify the number of repeated CPUID leaf 0x2 subleaf 0 queries needed to extract all of the CPU's cache and TLB descriptors. Per current Intel manuals, all CPUs supporting this leaf "will always" return an iteration count of 1. Remove the leaf 0x2 query loop and just query the hardware once. Note, as previously done at commit aec28d852ed2 ("x86/cpuid: Standardize on u32 in "), standardize on using 'u32' and 'u8' types. Suggested-by: Ingo Molnar Suggested-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-3-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 69 ++++++++++++++++----------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index b3a520959b51..36782fd017b3 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -42,7 +42,7 @@ static cpumask_var_t cpu_cacheinfo_mask; unsigned int memory_caching_control __ro_after_init; struct _cache_table { - unsigned char descriptor; + u8 descriptor; char cache_type; short size; }; @@ -783,50 +783,47 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) /* Don't use CPUID(2) if CPUID(4) is supported. */ if (!ci->num_leaves && c->cpuid_level > 1) { - /* supports eax=2 call */ - int j, n; - unsigned int regs[4]; - unsigned char *dp = (unsigned char *)regs; + u32 regs[4]; + u8 *desc = (u8 *)regs; - /* Number of times to iterate */ - n = cpuid_eax(2) & 0xFF; + cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); - for (i = 0 ; i < n ; i++) { - cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); + /* Intel CPUs must report an iteration count of 1 */ + if (desc[0] != 0x01) + return; - /* If bit 31 is set, this is an unknown format */ - for (j = 0 ; j < 4 ; j++) - if (regs[j] & (1 << 31)) - regs[j] = 0; + /* If a register's bit 31 is set, it is an unknown format */ + for (int i = 0; i < 4; i++) { + if (regs[i] & (1 << 31)) + regs[i] = 0; + } - /* Byte 0 is level count, not a descriptor */ - for (j = 1 ; j < 16 ; j++) { - unsigned char des = dp[j]; - unsigned char k = 0; - - /* look up this descriptor in the table */ - while (cache_table[k].descriptor != 0) { - if (cache_table[k].descriptor == des) { - switch (cache_table[k].cache_type) { - case LVL_1_INST: - l1i += cache_table[k].size; - break; - case LVL_1_DATA: - l1d += cache_table[k].size; - break; - case LVL_2: - l2 += cache_table[k].size; - break; - case LVL_3: - l3 += cache_table[k].size; - break; - } + /* Skip the first byte as it is not a descriptor */ + for (int i = 1; i < 16; i++) { + u8 des = desc[i]; + u8 k = 0; + /* look up this descriptor in the table */ + while (cache_table[k].descriptor != 0) { + if (cache_table[k].descriptor == des) { + switch (cache_table[k].cache_type) { + case LVL_1_INST: + l1i += cache_table[k].size; + break; + case LVL_1_DATA: + l1d += cache_table[k].size; + break; + case LVL_2: + l2 += cache_table[k].size; + break; + case LVL_3: + l3 += cache_table[k].size; break; } - k++; + break; } + k++; } } } From fe78079ec07fd48a19abcfeac74bc97e07171fb6 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:32:58 +0100 Subject: [PATCH 023/331] x86/cpu: Introduce and use CPUID leaf 0x2 parsing helpers Introduce CPUID leaf 0x2 parsing helpers at . This allows sharing the leaf 0x2's output validation and iteration logic across both x86/cpu intel.c and cacheinfo.c. Start by converting intel.c to the new API. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-4-darwi@linutronix.de --- arch/x86/include/asm/cpuid.h | 1 + arch/x86/include/asm/cpuid/leaf_0x2_api.h | 65 +++++++++++++++++++++++ arch/x86/include/asm/cpuid/types.h | 16 ++++++ arch/x86/kernel/cpu/intel.c | 23 +++----- 4 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 arch/x86/include/asm/cpuid/leaf_0x2_api.h diff --git a/arch/x86/include/asm/cpuid.h b/arch/x86/include/asm/cpuid.h index d5749b25fa10..585819331dc6 100644 --- a/arch/x86/include/asm/cpuid.h +++ b/arch/x86/include/asm/cpuid.h @@ -4,5 +4,6 @@ #define _ASM_X86_CPUID_H #include +#include #endif /* _ASM_X86_CPUID_H */ diff --git a/arch/x86/include/asm/cpuid/leaf_0x2_api.h b/arch/x86/include/asm/cpuid/leaf_0x2_api.h new file mode 100644 index 000000000000..4c845fc96716 --- /dev/null +++ b/arch/x86/include/asm/cpuid/leaf_0x2_api.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_CPUID_LEAF_0x2_API_H +#define _ASM_X86_CPUID_LEAF_0x2_API_H + +#include +#include + +/** + * cpuid_get_leaf_0x2_regs() - Return sanitized leaf 0x2 register output + * @regs: Output parameter + * + * Query CPUID leaf 0x2 and store its output in @regs. Force set any + * invalid 1-byte descriptor returned by the hardware to zero (the NULL + * cache/TLB descriptor) before returning it to the caller. + * + * Use for_each_leaf_0x2_desc() to iterate over the returned output. + */ +static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) +{ + cpuid_leaf(0x2, regs); + + /* + * All Intel CPUs must report an iteration count of 1. In case + * of bogus hardware, treat all returned descriptors as NULL. + */ + if (regs->desc[0] != 0x01) { + for (int i = 0; i < 4; i++) + regs->regv[i] = 0; + return; + } + + /* + * The most significant bit (MSB) of each register must be clear. + * If a register is invalid, replace its descriptors with NULL. + */ + for (int i = 0; i < 4; i++) { + if (regs->reg[i].invalid) + regs->regv[i] = 0; + } +} + +/** + * for_each_leaf_0x2_desc() - Iterator for CPUID leaf 0x2 descriptors + * @regs: Leaf 0x2 output, as returned by cpuid_get_leaf_0x2_regs() + * @desc: Pointer to the returned descriptor for each iteration + * + * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers + * @regs. Provide each descriptor through @desc. + * + * Note that the first byte is skipped as it is not a descriptor. + * + * Sample usage:: + * + * union leaf_0x2_regs regs; + * u8 *desc; + * + * cpuid_get_leaf_0x2_regs(®s); + * for_each_leaf_0x2_desc(regs, desc) { + * // Handle *desc value + * } + */ +#define for_each_leaf_0x2_desc(regs, desc) \ + for (desc = &(regs).desc[1]; desc < &(regs).desc[16]; desc++) + +#endif /* _ASM_X86_CPUID_LEAF_0x2_API_H */ diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index 8582e27e836d..753f6c4514f4 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -29,4 +29,20 @@ enum cpuid_regs_idx { #define CPUID_LEAF_FREQ 0x16 #define CPUID_LEAF_TILE 0x1d +/* + * Types for CPUID(0x2) parsing + * Check + */ + +struct leaf_0x2_reg { + u32 : 31, + invalid : 1; +}; + +union leaf_0x2_regs { + struct leaf_0x2_reg reg[4]; + u32 regv[4]; + u8 desc[16]; +}; + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 0570d4d86006..aeb7d6d48379 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -778,27 +779,15 @@ static void intel_tlb_lookup(const unsigned char desc) static void intel_detect_tlb(struct cpuinfo_x86 *c) { - u32 regs[4]; - u8 *desc = (u8 *)regs; + union leaf_0x2_regs regs; + u8 *desc; if (c->cpuid_level < 2) return; - cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); - - /* Intel CPUs must report an iteration count of 1 */ - if (desc[0] != 0x01) - return; - - /* If a register's bit 31 is set, it is an unknown format */ - for (int i = 0; i < 4; i++) { - if (regs[i] & (1 << 31)) - regs[i] = 0; - } - - /* Skip the first byte as it is not a descriptor */ - for (int i = 1; i < 16; i++) - intel_tlb_lookup(desc[i]); + cpuid_get_leaf_0x2_regs(®s); + for_each_leaf_0x2_desc(regs, desc) + intel_tlb_lookup(*desc); } static const struct cpu_dev intel_cpu_dev = { From a078aaa38a23a2595addb14ac286d8b8ae793d39 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:32:59 +0100 Subject: [PATCH 024/331] x86/cacheinfo: Use CPUID leaf 0x2 parsing helpers Parent commit introduced CPUID leaf 0x2 parsing helpers at . The new API allows sharing leaf 0x2's output validation and iteration logic across both intel.c and cacheinfo.c. Convert cacheinfo.c to that new API. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-5-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 36782fd017b3..6c610805e356 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -783,29 +784,16 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) /* Don't use CPUID(2) if CPUID(4) is supported. */ if (!ci->num_leaves && c->cpuid_level > 1) { - u32 regs[4]; - u8 *desc = (u8 *)regs; + union leaf_0x2_regs regs; + u8 *desc; - cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); - - /* Intel CPUs must report an iteration count of 1 */ - if (desc[0] != 0x01) - return; - - /* If a register's bit 31 is set, it is an unknown format */ - for (int i = 0; i < 4; i++) { - if (regs[i] & (1 << 31)) - regs[i] = 0; - } - - /* Skip the first byte as it is not a descriptor */ - for (int i = 1; i < 16; i++) { - u8 des = desc[i]; + cpuid_get_leaf_0x2_regs(®s); + for_each_leaf_0x2_desc(regs, desc) { u8 k = 0; /* look up this descriptor in the table */ while (cache_table[k].descriptor != 0) { - if (cache_table[k].descriptor == des) { + if (cache_table[k].descriptor == *desc) { switch (cache_table[k].cache_type) { case LVL_1_INST: l1i += cache_table[k].size; From ee159792a4db06e359a5e2e231c6f1242ec76c31 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 24 Mar 2025 14:33:00 +0100 Subject: [PATCH 025/331] x86/cacheinfo: Refactor CPUID leaf 0x2 cache descriptor lookup Extract the cache descriptor lookup logic out of the leaf 0x2 parsing code and into a dedicated function. This disentangles such lookup from the deeply nested leaf 0x2 parsing loop. Remove the cache table termination entry, as it is no longer needed after the ARRAY_SIZE()-based lookup. [ darwi: Move refactoring logic into this separate commit + commit log. Remove the cache table termination entry. ] Signed-off-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-6-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 42 +++++++++++++++------------------ 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 6c610805e356..d0bfdb85b96a 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -123,7 +123,6 @@ static const struct _cache_table cache_table[] = { 0xea, LVL_3, MB(12) }, /* 24-way set assoc, 64 byte line size */ { 0xeb, LVL_3, MB(18) }, /* 24-way set assoc, 64 byte line size */ { 0xec, LVL_3, MB(24) }, /* 24-way set assoc, 64 byte line size */ - { 0x00, 0, 0} }; @@ -728,6 +727,16 @@ void init_hygon_cacheinfo(struct cpuinfo_x86 *c) ci->num_leaves = find_num_cache_leaves(c); } +static const struct _cache_table *cache_table_get(u8 desc) +{ + for (int i = 0; i < ARRAY_SIZE(cache_table); i++) { + if (cache_table[i].descriptor == desc) + return &cache_table[i]; + } + + return NULL; +} + void init_intel_cacheinfo(struct cpuinfo_x86 *c) { /* Cache sizes */ @@ -784,34 +793,21 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) /* Don't use CPUID(2) if CPUID(4) is supported. */ if (!ci->num_leaves && c->cpuid_level > 1) { + const struct _cache_table *entry; union leaf_0x2_regs regs; u8 *desc; cpuid_get_leaf_0x2_regs(®s); for_each_leaf_0x2_desc(regs, desc) { - u8 k = 0; + entry = cache_table_get(*desc); + if (!entry) + continue; - /* look up this descriptor in the table */ - while (cache_table[k].descriptor != 0) { - if (cache_table[k].descriptor == *desc) { - switch (cache_table[k].cache_type) { - case LVL_1_INST: - l1i += cache_table[k].size; - break; - case LVL_1_DATA: - l1d += cache_table[k].size; - break; - case LVL_2: - l2 += cache_table[k].size; - break; - case LVL_3: - l3 += cache_table[k].size; - break; - } - - break; - } - k++; + switch (entry->cache_type) { + case LVL_1_INST: l1i += entry->size; break; + case LVL_1_DATA: l1d += entry->size; break; + case LVL_2: l2 += entry->size; break; + case LVL_3: l3 += entry->size; break; } } } From 21e2a452dca34538db0731b45c113cfd31aa45e6 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 24 Mar 2025 14:33:01 +0100 Subject: [PATCH 026/331] x86/cacheinfo: Properly name amd_cpuid4()'s first parameter amd_cpuid4()'s first parameter, "leaf", is not a CPUID leaf as the name implies. Rather, it's an index emulating CPUID(4)'s subleaf semantics; i.e. an ID for the cache object currently enumerated. Rename that parameter to "index". Apply minor coding style fixes to the rest of the function as well. [ darwi: Move into a separate commit and write commit log. Use "index" instead of "subleaf" for amd_cpuid4() first param, as that's the name typically used at the whole of cacheinfo.c. ] Signed-off-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-7-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index d0bfdb85b96a..0fd4e9673665 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -233,12 +233,10 @@ static const enum cache_type cache_type_map[] = { }; static void -amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, - union _cpuid4_leaf_ebx *ebx, - union _cpuid4_leaf_ecx *ecx) +amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, + union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) { - unsigned dummy; - unsigned line_size, lines_per_tag, assoc, size_in_kb; + unsigned int dummy, line_size, lines_per_tag, assoc, size_in_kb; union l1_cache l1i, l1d; union l2_cache l2; union l3_cache l3; @@ -251,7 +249,7 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val); cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val); - switch (leaf) { + switch (index) { case 1: l1 = &l1i; fallthrough; @@ -289,12 +287,11 @@ amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, } eax->split.is_self_initializing = 1; - eax->split.type = types[leaf]; - eax->split.level = levels[leaf]; + eax->split.type = types[index]; + eax->split.level = levels[index]; eax->split.num_threads_sharing = 0; eax->split.num_cores_on_die = topology_num_cores_per_package(); - if (assoc == 0xffff) eax->split.is_fully_associative = 1; ebx->split.coherency_line_size = line_size - 1; From cf8758205264707b4521163c0a37bf587c4425c4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 24 Mar 2025 14:33:02 +0100 Subject: [PATCH 027/331] x86/cacheinfo: Use proper name for cacheinfo instances The cacheinfo structure defined at is a generic cache info object representation. Calling its instances at x86 cacheinfo.c "leaf" confuses it with a CPUID leaf -- especially that multiple CPUID calls are already sprinkled across that file. Most of such instances also have a redundant "this_" prefix. Rename all of the cacheinfo "this_leaf" instances to just "ci". [ darwi: Move into separate commit and write commit log ] Signed-off-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-8-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 89 ++++++++++++++++----------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 0fd4e9673665..be9be5e56b44 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -349,11 +349,10 @@ static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot) return -1; } -static ssize_t show_cache_disable(struct cacheinfo *this_leaf, char *buf, - unsigned int slot) +static ssize_t show_cache_disable(struct cacheinfo *ci, char *buf, unsigned int slot) { int index; - struct amd_northbridge *nb = this_leaf->priv; + struct amd_northbridge *nb = ci->priv; index = amd_get_l3_disable_slot(nb, slot); if (index >= 0) @@ -367,8 +366,8 @@ static ssize_t \ cache_disable_##slot##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ { \ - struct cacheinfo *this_leaf = dev_get_drvdata(dev); \ - return show_cache_disable(this_leaf, buf, slot); \ + struct cacheinfo *ci = dev_get_drvdata(dev); \ + return show_cache_disable(ci, buf, slot); \ } SHOW_CACHE_DISABLE(0) SHOW_CACHE_DISABLE(1) @@ -435,18 +434,17 @@ static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu, return 0; } -static ssize_t store_cache_disable(struct cacheinfo *this_leaf, - const char *buf, size_t count, - unsigned int slot) +static ssize_t store_cache_disable(struct cacheinfo *ci, const char *buf, + size_t count, unsigned int slot) { unsigned long val = 0; int cpu, err = 0; - struct amd_northbridge *nb = this_leaf->priv; + struct amd_northbridge *nb = ci->priv; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - cpu = cpumask_first(&this_leaf->shared_cpu_map); + cpu = cpumask_first(&ci->shared_cpu_map); if (kstrtoul(buf, 10, &val) < 0) return -EINVAL; @@ -467,8 +465,8 @@ cache_disable_##slot##_store(struct device *dev, \ struct device_attribute *attr, \ const char *buf, size_t count) \ { \ - struct cacheinfo *this_leaf = dev_get_drvdata(dev); \ - return store_cache_disable(this_leaf, buf, count, slot); \ + struct cacheinfo *ci = dev_get_drvdata(dev); \ + return store_cache_disable(ci, buf, count, slot); \ } STORE_CACHE_DISABLE(0) STORE_CACHE_DISABLE(1) @@ -476,8 +474,8 @@ STORE_CACHE_DISABLE(1) static ssize_t subcaches_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct cacheinfo *this_leaf = dev_get_drvdata(dev); - int cpu = cpumask_first(&this_leaf->shared_cpu_map); + struct cacheinfo *ci = dev_get_drvdata(dev); + int cpu = cpumask_first(&ci->shared_cpu_map); return sprintf(buf, "%x\n", amd_get_subcaches(cpu)); } @@ -486,8 +484,8 @@ static ssize_t subcaches_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct cacheinfo *this_leaf = dev_get_drvdata(dev); - int cpu = cpumask_first(&this_leaf->shared_cpu_map); + struct cacheinfo *ci = dev_get_drvdata(dev); + int cpu = cpumask_first(&ci->shared_cpu_map); unsigned long val; if (!capable(CAP_SYS_ADMIN)) @@ -511,10 +509,10 @@ cache_private_attrs_is_visible(struct kobject *kobj, struct attribute *attr, int unused) { struct device *dev = kobj_to_dev(kobj); - struct cacheinfo *this_leaf = dev_get_drvdata(dev); + struct cacheinfo *ci = dev_get_drvdata(dev); umode_t mode = attr->mode; - if (!this_leaf->priv) + if (!ci->priv) return 0; if ((attr == &dev_attr_subcaches.attr) && @@ -562,11 +560,11 @@ static void init_amd_l3_attrs(void) } const struct attribute_group * -cache_get_priv_group(struct cacheinfo *this_leaf) +cache_get_priv_group(struct cacheinfo *ci) { - struct amd_northbridge *nb = this_leaf->priv; + struct amd_northbridge *nb = ci->priv; - if (this_leaf->level < 3 || !nb) + if (ci->level < 3 || !nb) return NULL; if (nb && nb->l3_cache.indices) @@ -846,7 +844,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, struct _cpuid4_info_regs *base) { struct cpu_cacheinfo *this_cpu_ci; - struct cacheinfo *this_leaf; + struct cacheinfo *ci; int i, sibling; /* @@ -858,12 +856,12 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, this_cpu_ci = get_cpu_cacheinfo(i); if (!this_cpu_ci->info_list) continue; - this_leaf = this_cpu_ci->info_list + index; + ci = this_cpu_ci->info_list + index; for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) { if (!cpu_online(sibling)) continue; cpumask_set_cpu(sibling, - &this_leaf->shared_cpu_map); + &ci->shared_cpu_map); } } } else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { @@ -883,14 +881,14 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, if ((apicid < first) || (apicid > last)) continue; - this_leaf = this_cpu_ci->info_list + index; + ci = this_cpu_ci->info_list + index; for_each_online_cpu(sibling) { apicid = cpu_data(sibling).topo.apicid; if ((apicid < first) || (apicid > last)) continue; cpumask_set_cpu(sibling, - &this_leaf->shared_cpu_map); + &ci->shared_cpu_map); } } } else @@ -903,7 +901,7 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, struct _cpuid4_info_regs *base) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - struct cacheinfo *this_leaf, *sibling_leaf; + struct cacheinfo *ci, *sibling_ci; unsigned long num_threads_sharing; int index_msb, i; struct cpuinfo_x86 *c = &cpu_data(cpu); @@ -914,10 +912,10 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, return; } - this_leaf = this_cpu_ci->info_list + index; + ci = this_cpu_ci->info_list + index; num_threads_sharing = 1 + base->eax.split.num_threads_sharing; - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + cpumask_set_cpu(cpu, &ci->shared_cpu_map); if (num_threads_sharing == 1) return; @@ -929,28 +927,27 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, if (i == cpu || !sib_cpu_ci->info_list) continue;/* skip if itself or no cacheinfo */ - sibling_leaf = sib_cpu_ci->info_list + index; - cpumask_set_cpu(i, &this_leaf->shared_cpu_map); - cpumask_set_cpu(cpu, &sibling_leaf->shared_cpu_map); + sibling_ci = sib_cpu_ci->info_list + index; + cpumask_set_cpu(i, &ci->shared_cpu_map); + cpumask_set_cpu(cpu, &sibling_ci->shared_cpu_map); } } -static void ci_leaf_init(struct cacheinfo *this_leaf, - struct _cpuid4_info_regs *base) +static void ci_info_init(struct cacheinfo *ci, struct _cpuid4_info_regs *base) { - this_leaf->id = base->id; - this_leaf->attributes = CACHE_ID; - this_leaf->level = base->eax.split.level; - this_leaf->type = cache_type_map[base->eax.split.type]; - this_leaf->coherency_line_size = + ci->id = base->id; + ci->attributes = CACHE_ID; + ci->level = base->eax.split.level; + ci->type = cache_type_map[base->eax.split.type]; + ci->coherency_line_size = base->ebx.split.coherency_line_size + 1; - this_leaf->ways_of_associativity = + ci->ways_of_associativity = base->ebx.split.ways_of_associativity + 1; - this_leaf->size = base->size; - this_leaf->number_of_sets = base->ecx.split.number_of_sets + 1; - this_leaf->physical_line_partition = + ci->size = base->size; + ci->number_of_sets = base->ecx.split.number_of_sets + 1; + ci->physical_line_partition = base->ebx.split.physical_line_partition + 1; - this_leaf->priv = base->nb; + ci->priv = base->nb; } int init_cache_level(unsigned int cpu) @@ -984,7 +981,7 @@ int populate_cache_leaves(unsigned int cpu) { unsigned int idx, ret; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - struct cacheinfo *this_leaf = this_cpu_ci->info_list; + struct cacheinfo *ci = this_cpu_ci->info_list; struct _cpuid4_info_regs id4_regs = {}; for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { @@ -992,7 +989,7 @@ int populate_cache_leaves(unsigned int cpu) if (ret) return ret; get_cache_id(cpu, &id4_regs); - ci_leaf_init(this_leaf++, &id4_regs); + ci_info_init(ci++, &id4_regs); __cache_cpumap_setup(cpu, idx, &id4_regs); } this_cpu_ci->cpu_map_populated = true; From 7b83e0d2b20b38a025d15ba88516c1b8fe9c6f3d Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:03 +0100 Subject: [PATCH 028/331] x86/cacheinfo: Constify _cpuid4_info_regs instances _cpuid4_info_regs instances are passed through a large number of functions at cacheinfo.c. For clarity, constify the instance parameters where _cpuid4_info_regs is only read from. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-9-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index be9be5e56b44..fc4b49ec42df 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -841,7 +841,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) } static int __cache_amd_cpumap_setup(unsigned int cpu, int index, - struct _cpuid4_info_regs *base) + const struct _cpuid4_info_regs *base) { struct cpu_cacheinfo *this_cpu_ci; struct cacheinfo *ci; @@ -898,7 +898,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, } static void __cache_cpumap_setup(unsigned int cpu, int index, - struct _cpuid4_info_regs *base) + const struct _cpuid4_info_regs *base) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci, *sibling_ci; @@ -933,7 +933,8 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, } } -static void ci_info_init(struct cacheinfo *ci, struct _cpuid4_info_regs *base) +static void ci_info_init(struct cacheinfo *ci, + const struct _cpuid4_info_regs *base) { ci->id = base->id; ci->attributes = CACHE_ID; From 036a73b5174477b72d881b4c75740d3dbfd7ec49 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:04 +0100 Subject: [PATCH 029/331] x86/cacheinfo: Align ci_info_init() assignment expressions The ci_info_init() function initializes 10 members of a 'struct cacheinfo' instance using passed data from CPUID leaf 0x4. Such assignment expressions are difficult to read in their current form. Align them for clarity. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-10-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index fc4b49ec42df..b273ecf3f538 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -936,19 +936,16 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, static void ci_info_init(struct cacheinfo *ci, const struct _cpuid4_info_regs *base) { - ci->id = base->id; - ci->attributes = CACHE_ID; - ci->level = base->eax.split.level; - ci->type = cache_type_map[base->eax.split.type]; - ci->coherency_line_size = - base->ebx.split.coherency_line_size + 1; - ci->ways_of_associativity = - base->ebx.split.ways_of_associativity + 1; - ci->size = base->size; - ci->number_of_sets = base->ecx.split.number_of_sets + 1; - ci->physical_line_partition = - base->ebx.split.physical_line_partition + 1; - ci->priv = base->nb; + ci->id = base->id; + ci->attributes = CACHE_ID; + ci->level = base->eax.split.level; + ci->type = cache_type_map[base->eax.split.type]; + ci->coherency_line_size = base->ebx.split.coherency_line_size + 1; + ci->ways_of_associativity = base->ebx.split.ways_of_associativity + 1; + ci->size = base->size; + ci->number_of_sets = base->ecx.split.number_of_sets + 1; + ci->physical_line_partition = base->ebx.split.physical_line_partition + 1; + ci->priv = base->nb; } int init_cache_level(unsigned int cpu) From 1374ff60ed0d4cedd55f653edd3f59f9f19c4171 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:05 +0100 Subject: [PATCH 030/331] x86/cacheinfo: Standardize _cpuid4_info_regs instance naming The cacheinfo code frequently uses the output registers from CPUID leaf 0x4. Such registers are cached in 'struct _cpuid4_info_regs', augmented with related information, and are then passed across functions. The naming of these _cpuid4_info_regs instances is confusing at best. Some instances are called "this_leaf", which is vague as "this" lacks context and "leaf" is overly generic given that other CPUID leaves are also processed within cacheinfo. Other _cpuid4_info_regs instances are just called "base", adding further ambiguity. Standardize on id4 for all instances. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-11-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 97 +++++++++++++++++---------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index b273ecf3f538..1b2a2bf97d7f 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -573,7 +573,7 @@ cache_get_priv_group(struct cacheinfo *ci) return &cache_private_group; } -static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) +static void amd_init_l3_cache(struct _cpuid4_info_regs *id4, int index) { int node; @@ -582,16 +582,16 @@ static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) return; node = topology_amd_node_id(smp_processor_id()); - this_leaf->nb = node_to_amd_nb(node); - if (this_leaf->nb && !this_leaf->nb->l3_cache.indices) - amd_calc_l3_indices(this_leaf->nb); + id4->nb = node_to_amd_nb(node); + if (id4->nb && !id4->nb->l3_cache.indices) + amd_calc_l3_indices(id4->nb); } #else #define amd_init_l3_cache(x, y) #endif /* CONFIG_AMD_NB && CONFIG_SYSFS */ static int -cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf) +cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4) { union _cpuid4_leaf_eax eax; union _cpuid4_leaf_ebx ebx; @@ -604,11 +604,11 @@ cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf) &ebx.full, &ecx.full, &edx); else amd_cpuid4(index, &eax, &ebx, &ecx); - amd_init_l3_cache(this_leaf, index); + amd_init_l3_cache(id4, index); } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { cpuid_count(0x8000001d, index, &eax.full, &ebx.full, &ecx.full, &edx); - amd_init_l3_cache(this_leaf, index); + amd_init_l3_cache(id4, index); } else { cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx); } @@ -616,13 +616,14 @@ cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf) if (eax.split.type == CTYPE_NULL) return -EIO; /* better error ? */ - this_leaf->eax = eax; - this_leaf->ebx = ebx; - this_leaf->ecx = ecx; - this_leaf->size = (ecx.split.number_of_sets + 1) * - (ebx.split.coherency_line_size + 1) * - (ebx.split.physical_line_partition + 1) * - (ebx.split.ways_of_associativity + 1); + id4->eax = eax; + id4->ebx = ebx; + id4->ecx = ecx; + id4->size = (ecx.split.number_of_sets + 1) * + (ebx.split.coherency_line_size + 1) * + (ebx.split.physical_line_partition + 1) * + (ebx.split.ways_of_associativity + 1); + return 0; } @@ -754,29 +755,29 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) * parameters cpuid leaf to find the cache details */ for (i = 0; i < ci->num_leaves; i++) { - struct _cpuid4_info_regs this_leaf = {}; + struct _cpuid4_info_regs id4 = {}; int retval; - retval = cpuid4_cache_lookup_regs(i, &this_leaf); + retval = cpuid4_cache_lookup_regs(i, &id4); if (retval < 0) continue; - switch (this_leaf.eax.split.level) { + switch (id4.eax.split.level) { case 1: - if (this_leaf.eax.split.type == CTYPE_DATA) - new_l1d = this_leaf.size/1024; - else if (this_leaf.eax.split.type == CTYPE_INST) - new_l1i = this_leaf.size/1024; + if (id4.eax.split.type == CTYPE_DATA) + new_l1d = id4.size/1024; + else if (id4.eax.split.type == CTYPE_INST) + new_l1i = id4.size/1024; break; case 2: - new_l2 = this_leaf.size/1024; - num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing; + new_l2 = id4.size/1024; + num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; index_msb = get_count_order(num_threads_sharing); l2_id = c->topo.apicid & ~((1 << index_msb) - 1); break; case 3: - new_l3 = this_leaf.size/1024; - num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing; + new_l3 = id4.size/1024; + num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; index_msb = get_count_order(num_threads_sharing); l3_id = c->topo.apicid & ~((1 << index_msb) - 1); break; @@ -841,7 +842,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) } static int __cache_amd_cpumap_setup(unsigned int cpu, int index, - const struct _cpuid4_info_regs *base) + const struct _cpuid4_info_regs *id4) { struct cpu_cacheinfo *this_cpu_ci; struct cacheinfo *ci; @@ -867,7 +868,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, } else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { unsigned int apicid, nshared, first, last; - nshared = base->eax.split.num_threads_sharing + 1; + nshared = id4->eax.split.num_threads_sharing + 1; apicid = cpu_data(cpu).topo.apicid; first = apicid - (apicid % nshared); last = first + nshared - 1; @@ -898,7 +899,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, } static void __cache_cpumap_setup(unsigned int cpu, int index, - const struct _cpuid4_info_regs *base) + const struct _cpuid4_info_regs *id4) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci, *sibling_ci; @@ -908,12 +909,12 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { - if (__cache_amd_cpumap_setup(cpu, index, base)) + if (__cache_amd_cpumap_setup(cpu, index, id4)) return; } ci = this_cpu_ci->info_list + index; - num_threads_sharing = 1 + base->eax.split.num_threads_sharing; + num_threads_sharing = 1 + id4->eax.split.num_threads_sharing; cpumask_set_cpu(cpu, &ci->shared_cpu_map); if (num_threads_sharing == 1) @@ -934,18 +935,18 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, } static void ci_info_init(struct cacheinfo *ci, - const struct _cpuid4_info_regs *base) + const struct _cpuid4_info_regs *id4) { - ci->id = base->id; + ci->id = id4->id; ci->attributes = CACHE_ID; - ci->level = base->eax.split.level; - ci->type = cache_type_map[base->eax.split.type]; - ci->coherency_line_size = base->ebx.split.coherency_line_size + 1; - ci->ways_of_associativity = base->ebx.split.ways_of_associativity + 1; - ci->size = base->size; - ci->number_of_sets = base->ecx.split.number_of_sets + 1; - ci->physical_line_partition = base->ebx.split.physical_line_partition + 1; - ci->priv = base->nb; + ci->level = id4->eax.split.level; + ci->type = cache_type_map[id4->eax.split.type]; + ci->coherency_line_size = id4->ebx.split.coherency_line_size + 1; + ci->ways_of_associativity = id4->ebx.split.ways_of_associativity + 1; + ci->size = id4->size; + ci->number_of_sets = id4->ecx.split.number_of_sets + 1; + ci->physical_line_partition = id4->ebx.split.physical_line_partition + 1; + ci->priv = id4->nb; } int init_cache_level(unsigned int cpu) @@ -964,15 +965,15 @@ int init_cache_level(unsigned int cpu) * ECX as cache index. Then right shift apicid by the number's order to get * cache id for this cache node. */ -static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4_regs) +static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4) { struct cpuinfo_x86 *c = &cpu_data(cpu); unsigned long num_threads_sharing; int index_msb; - num_threads_sharing = 1 + id4_regs->eax.split.num_threads_sharing; + num_threads_sharing = 1 + id4->eax.split.num_threads_sharing; index_msb = get_count_order(num_threads_sharing); - id4_regs->id = c->topo.apicid >> index_msb; + id4->id = c->topo.apicid >> index_msb; } int populate_cache_leaves(unsigned int cpu) @@ -980,15 +981,15 @@ int populate_cache_leaves(unsigned int cpu) unsigned int idx, ret; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci = this_cpu_ci->info_list; - struct _cpuid4_info_regs id4_regs = {}; + struct _cpuid4_info_regs id4 = {}; for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { - ret = cpuid4_cache_lookup_regs(idx, &id4_regs); + ret = cpuid4_cache_lookup_regs(idx, &id4); if (ret) return ret; - get_cache_id(cpu, &id4_regs); - ci_info_init(ci++, &id4_regs); - __cache_cpumap_setup(cpu, idx, &id4_regs); + get_cache_id(cpu, &id4); + ci_info_init(ci++, &id4); + __cache_cpumap_setup(cpu, idx, &id4); } this_cpu_ci->cpu_map_populated = true; From 77676e6802a10ffa5a0ad6367e8f6e14cbd88781 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:06 +0100 Subject: [PATCH 031/331] x86/cacheinfo: Consolidate AMD/Hygon leaf 0x8000001d calls While gathering CPU cache info, CPUID leaf 0x8000001d is invoked in two separate if blocks: one for Hygon CPUs and one for AMDs with topology extensions. After each invocation, amd_init_l3_cache() is called. Merge the two if blocks into a single condition, thus removing the duplicated code. Future commits will expand these if blocks, so combining them now is both cleaner and more maintainable. Note, while at it, remove a useless "better error?" comment that was within the same function since the 2005 commit e2cac78935ff ("[PATCH] x86_64: When running cpuid4 need to run on the correct CPU"). Note, as previously done at commit aec28d852ed2 ("x86/cpuid: Standardize on u32 in "), standardize on using 'u32' and 'u8' types. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-12-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 1b2a2bf97d7f..f1055e806c9f 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -593,28 +593,28 @@ static void amd_init_l3_cache(struct _cpuid4_info_regs *id4, int index) static int cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4) { - union _cpuid4_leaf_eax eax; - union _cpuid4_leaf_ebx ebx; - union _cpuid4_leaf_ecx ecx; - unsigned edx; + u8 cpu_vendor = boot_cpu_data.x86_vendor; + union _cpuid4_leaf_eax eax; + union _cpuid4_leaf_ebx ebx; + union _cpuid4_leaf_ecx ecx; + u32 edx; - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { - if (boot_cpu_has(X86_FEATURE_TOPOEXT)) - cpuid_count(0x8000001d, index, &eax.full, - &ebx.full, &ecx.full, &edx); - else + if (cpu_vendor == X86_VENDOR_AMD || cpu_vendor == X86_VENDOR_HYGON) { + if (boot_cpu_has(X86_FEATURE_TOPOEXT) || cpu_vendor == X86_VENDOR_HYGON) { + /* AMD with TOPOEXT, or HYGON */ + cpuid_count(0x8000001d, index, &eax.full, &ebx.full, &ecx.full, &edx); + } else { + /* Legacy AMD fallback */ amd_cpuid4(index, &eax, &ebx, &ecx); - amd_init_l3_cache(id4, index); - } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - cpuid_count(0x8000001d, index, &eax.full, - &ebx.full, &ecx.full, &edx); + } amd_init_l3_cache(id4, index); } else { + /* Intel */ cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx); } if (eax.split.type == CTYPE_NULL) - return -EIO; /* better error ? */ + return -EIO; id4->eax = eax; id4->ebx = ebx; From c58ed2d4da8dced3fa4505f498bd393f565b471a Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:07 +0100 Subject: [PATCH 032/331] x86/cacheinfo: Separate amd_northbridge from _cpuid4_info_regs 'struct _cpuid4_info_regs' is meant to hold the CPUID leaf 0x4 output registers (EAX, EBX, and ECX), as well as derived information such as the cache node ID and size. It also contains a reference to amd_northbridge, which is there only to be "parked" until ci_info_init() can store it in the priv pointer of the API. That priv pointer is then used by AMD-specific L3 cache_disable_0/1 sysfs attributes. Decouple amd_northbridge from _cpuid4_info_regs and pass it explicitly through the functions at x86/cacheinfo. Doing so clarifies when amd_northbridge is actually needed (AMD-only code) and when it is not (Intel-specific code). It also prepares for moving the AMD-specific L3 cache_disable_0/1 sysfs code into its own file in next commit. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-13-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 45 +++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index f1055e806c9f..8c2b51bb032a 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -168,7 +168,6 @@ struct _cpuid4_info_regs { union _cpuid4_leaf_ecx ecx; unsigned int id; unsigned long size; - struct amd_northbridge *nb; }; /* AMD doesn't have CPUID4. Emulate it here to report the same @@ -573,25 +572,36 @@ cache_get_priv_group(struct cacheinfo *ci) return &cache_private_group; } -static void amd_init_l3_cache(struct _cpuid4_info_regs *id4, int index) +static struct amd_northbridge *amd_init_l3_cache(int index) { + struct amd_northbridge *nb; int node; /* only for L3, and not in virtualized environments */ if (index < 3) - return; + return NULL; node = topology_amd_node_id(smp_processor_id()); - id4->nb = node_to_amd_nb(node); - if (id4->nb && !id4->nb->l3_cache.indices) - amd_calc_l3_indices(id4->nb); + nb = node_to_amd_nb(node); + if (nb && !nb->l3_cache.indices) + amd_calc_l3_indices(nb); + + return nb; } #else -#define amd_init_l3_cache(x, y) +static struct amd_northbridge *amd_init_l3_cache(int index) +{ + return NULL; +} #endif /* CONFIG_AMD_NB && CONFIG_SYSFS */ -static int -cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4) +/* + * Fill passed _cpuid4_info_regs structure. + * Intel-only code paths should pass NULL for the amd_northbridge + * return pointer. + */ +static int cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4, + struct amd_northbridge **nb) { u8 cpu_vendor = boot_cpu_data.x86_vendor; union _cpuid4_leaf_eax eax; @@ -607,7 +617,9 @@ cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4) /* Legacy AMD fallback */ amd_cpuid4(index, &eax, &ebx, &ecx); } - amd_init_l3_cache(id4, index); + + if (nb) + *nb = amd_init_l3_cache(index); } else { /* Intel */ cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx); @@ -758,7 +770,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) struct _cpuid4_info_regs id4 = {}; int retval; - retval = cpuid4_cache_lookup_regs(i, &id4); + retval = cpuid4_cache_lookup_regs(i, &id4, NULL); if (retval < 0) continue; @@ -934,8 +946,8 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, } } -static void ci_info_init(struct cacheinfo *ci, - const struct _cpuid4_info_regs *id4) +static void ci_info_init(struct cacheinfo *ci, const struct _cpuid4_info_regs *id4, + struct amd_northbridge *nb) { ci->id = id4->id; ci->attributes = CACHE_ID; @@ -946,7 +958,7 @@ static void ci_info_init(struct cacheinfo *ci, ci->size = id4->size; ci->number_of_sets = id4->ecx.split.number_of_sets + 1; ci->physical_line_partition = id4->ebx.split.physical_line_partition + 1; - ci->priv = id4->nb; + ci->priv = nb; } int init_cache_level(unsigned int cpu) @@ -982,13 +994,14 @@ int populate_cache_leaves(unsigned int cpu) struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci = this_cpu_ci->info_list; struct _cpuid4_info_regs id4 = {}; + struct amd_northbridge *nb; for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { - ret = cpuid4_cache_lookup_regs(idx, &id4); + ret = cpuid4_cache_lookup_regs(idx, &id4, &nb); if (ret) return ret; get_cache_id(cpu, &id4); - ci_info_init(ci++, &id4); + ci_info_init(ci++, &id4, nb); __cache_cpumap_setup(cpu, idx, &id4); } this_cpu_ci->cpu_map_populated = true; From 365e960d296ef9f36b20971aa4854ce07817a9bb Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:08 +0100 Subject: [PATCH 033/331] x86/cacheinfo: Move AMD cache_disable_0/1 handling to separate file Parent commit decoupled amd_northbridge out of _cpuid4_info_regs, where it was merely "parked" there until ci_info_init() can store it in the private pointer of the API. Given that decoupling, move the AMD-specific L3 cache_disable_0/1 sysfs code from the generic (and already extremely convoluted) x86/cacheinfo code into its own file. Compile the file only if CONFIG_AMD_NB and CONFIG_SYSFS are both enabled, which mirrors the existing logic. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-14-darwi@linutronix.de --- arch/x86/kernel/cpu/Makefile | 3 + arch/x86/kernel/cpu/amd_cache_disable.c | 301 ++++++++++++++++++++++++ arch/x86/kernel/cpu/cacheinfo.c | 298 ----------------------- arch/x86/kernel/cpu/cpu.h | 9 + 4 files changed, 313 insertions(+), 298 deletions(-) create mode 100644 arch/x86/kernel/cpu/amd_cache_disable.c diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 4efdf5c2efc8..3a39396d422d 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -38,6 +38,9 @@ obj-y += intel.o tsx.o obj-$(CONFIG_PM) += intel_epb.o endif obj-$(CONFIG_CPU_SUP_AMD) += amd.o +ifeq ($(CONFIG_AMD_NB)$(CONFIG_SYSFS),yy) +obj-y += amd_cache_disable.o +endif obj-$(CONFIG_CPU_SUP_HYGON) += hygon.o obj-$(CONFIG_CPU_SUP_CYRIX_32) += cyrix.o obj-$(CONFIG_CPU_SUP_CENTAUR) += centaur.o diff --git a/arch/x86/kernel/cpu/amd_cache_disable.c b/arch/x86/kernel/cpu/amd_cache_disable.c new file mode 100644 index 000000000000..6d53aee0d869 --- /dev/null +++ b/arch/x86/kernel/cpu/amd_cache_disable.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD L3 cache_disable_{0,1} sysfs handling + * Documentation/ABI/testing/sysfs-devices-system-cpu + */ + +#include +#include +#include +#include + +#include + +#include "cpu.h" + +/* + * L3 cache descriptors + */ +static void amd_calc_l3_indices(struct amd_northbridge *nb) +{ + struct amd_l3_cache *l3 = &nb->l3_cache; + unsigned int sc0, sc1, sc2, sc3; + u32 val = 0; + + pci_read_config_dword(nb->misc, 0x1C4, &val); + + /* calculate subcache sizes */ + l3->subcaches[0] = sc0 = !(val & BIT(0)); + l3->subcaches[1] = sc1 = !(val & BIT(4)); + + if (boot_cpu_data.x86 == 0x15) { + l3->subcaches[0] = sc0 += !(val & BIT(1)); + l3->subcaches[1] = sc1 += !(val & BIT(5)); + } + + l3->subcaches[2] = sc2 = !(val & BIT(8)) + !(val & BIT(9)); + l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13)); + + l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1; +} + +/* + * check whether a slot used for disabling an L3 index is occupied. + * @l3: L3 cache descriptor + * @slot: slot number (0..1) + * + * @returns: the disabled index if used or negative value if slot free. + */ +static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned int slot) +{ + unsigned int reg = 0; + + pci_read_config_dword(nb->misc, 0x1BC + slot * 4, ®); + + /* check whether this slot is activated already */ + if (reg & (3UL << 30)) + return reg & 0xfff; + + return -1; +} + +static ssize_t show_cache_disable(struct cacheinfo *ci, char *buf, unsigned int slot) +{ + int index; + struct amd_northbridge *nb = ci->priv; + + index = amd_get_l3_disable_slot(nb, slot); + if (index >= 0) + return sprintf(buf, "%d\n", index); + + return sprintf(buf, "FREE\n"); +} + +#define SHOW_CACHE_DISABLE(slot) \ +static ssize_t \ +cache_disable_##slot##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct cacheinfo *ci = dev_get_drvdata(dev); \ + return show_cache_disable(ci, buf, slot); \ +} + +SHOW_CACHE_DISABLE(0) +SHOW_CACHE_DISABLE(1) + +static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu, + unsigned int slot, unsigned long idx) +{ + int i; + + idx |= BIT(30); + + /* + * disable index in all 4 subcaches + */ + for (i = 0; i < 4; i++) { + u32 reg = idx | (i << 20); + + if (!nb->l3_cache.subcaches[i]) + continue; + + pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg); + + /* + * We need to WBINVD on a core on the node containing the L3 + * cache which indices we disable therefore a simple wbinvd() + * is not sufficient. + */ + wbinvd_on_cpu(cpu); + + reg |= BIT(31); + pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg); + } +} + +/* + * disable a L3 cache index by using a disable-slot + * + * @l3: L3 cache descriptor + * @cpu: A CPU on the node containing the L3 cache + * @slot: slot number (0..1) + * @index: index to disable + * + * @return: 0 on success, error status on failure + */ +static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu, + unsigned int slot, unsigned long index) +{ + int ret = 0; + + /* check if @slot is already used or the index is already disabled */ + ret = amd_get_l3_disable_slot(nb, slot); + if (ret >= 0) + return -EEXIST; + + if (index > nb->l3_cache.indices) + return -EINVAL; + + /* check whether the other slot has disabled the same index already */ + if (index == amd_get_l3_disable_slot(nb, !slot)) + return -EEXIST; + + amd_l3_disable_index(nb, cpu, slot, index); + + return 0; +} + +static ssize_t store_cache_disable(struct cacheinfo *ci, const char *buf, + size_t count, unsigned int slot) +{ + struct amd_northbridge *nb = ci->priv; + unsigned long val = 0; + int cpu, err = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + cpu = cpumask_first(&ci->shared_cpu_map); + + if (kstrtoul(buf, 10, &val) < 0) + return -EINVAL; + + err = amd_set_l3_disable_slot(nb, cpu, slot, val); + if (err) { + if (err == -EEXIST) + pr_warn("L3 slot %d in use/index already disabled!\n", + slot); + return err; + } + return count; +} + +#define STORE_CACHE_DISABLE(slot) \ +static ssize_t \ +cache_disable_##slot##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ +{ \ + struct cacheinfo *ci = dev_get_drvdata(dev); \ + return store_cache_disable(ci, buf, count, slot); \ +} + +STORE_CACHE_DISABLE(0) +STORE_CACHE_DISABLE(1) + +static ssize_t subcaches_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cacheinfo *ci = dev_get_drvdata(dev); + int cpu = cpumask_first(&ci->shared_cpu_map); + + return sprintf(buf, "%x\n", amd_get_subcaches(cpu)); +} + +static ssize_t subcaches_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cacheinfo *ci = dev_get_drvdata(dev); + int cpu = cpumask_first(&ci->shared_cpu_map); + unsigned long val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (kstrtoul(buf, 16, &val) < 0) + return -EINVAL; + + if (amd_set_subcaches(cpu, val)) + return -EINVAL; + + return count; +} + +static DEVICE_ATTR_RW(cache_disable_0); +static DEVICE_ATTR_RW(cache_disable_1); +static DEVICE_ATTR_RW(subcaches); + +static umode_t cache_private_attrs_is_visible(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct device *dev = kobj_to_dev(kobj); + struct cacheinfo *ci = dev_get_drvdata(dev); + umode_t mode = attr->mode; + + if (!ci->priv) + return 0; + + if ((attr == &dev_attr_subcaches.attr) && + amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) + return mode; + + if ((attr == &dev_attr_cache_disable_0.attr || + attr == &dev_attr_cache_disable_1.attr) && + amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) + return mode; + + return 0; +} + +static struct attribute_group cache_private_group = { + .is_visible = cache_private_attrs_is_visible, +}; + +static void init_amd_l3_attrs(void) +{ + static struct attribute **amd_l3_attrs; + int n = 1; + + if (amd_l3_attrs) /* already initialized */ + return; + + if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) + n += 2; + if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) + n += 1; + + amd_l3_attrs = kcalloc(n, sizeof(*amd_l3_attrs), GFP_KERNEL); + if (!amd_l3_attrs) + return; + + n = 0; + if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) { + amd_l3_attrs[n++] = &dev_attr_cache_disable_0.attr; + amd_l3_attrs[n++] = &dev_attr_cache_disable_1.attr; + } + if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) + amd_l3_attrs[n++] = &dev_attr_subcaches.attr; + + cache_private_group.attrs = amd_l3_attrs; +} + +const struct attribute_group *cache_get_priv_group(struct cacheinfo *ci) +{ + struct amd_northbridge *nb = ci->priv; + + if (ci->level < 3 || !nb) + return NULL; + + if (nb && nb->l3_cache.indices) + init_amd_l3_attrs(); + + return &cache_private_group; +} + +struct amd_northbridge *amd_init_l3_cache(int index) +{ + struct amd_northbridge *nb; + int node; + + /* only for L3, and not in virtualized environments */ + if (index < 3) + return NULL; + + node = topology_amd_node_id(smp_processor_id()); + nb = node_to_amd_nb(node); + if (nb && !nb->l3_cache.indices) + amd_calc_l3_indices(nb); + + return nb; +} diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 8c2b51bb032a..ea6fba942d27 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -9,12 +9,9 @@ */ #include -#include #include #include -#include #include -#include #include #include @@ -300,301 +297,6 @@ amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, (ebx->split.ways_of_associativity + 1) - 1; } -#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS) - -/* - * L3 cache descriptors - */ -static void amd_calc_l3_indices(struct amd_northbridge *nb) -{ - struct amd_l3_cache *l3 = &nb->l3_cache; - unsigned int sc0, sc1, sc2, sc3; - u32 val = 0; - - pci_read_config_dword(nb->misc, 0x1C4, &val); - - /* calculate subcache sizes */ - l3->subcaches[0] = sc0 = !(val & BIT(0)); - l3->subcaches[1] = sc1 = !(val & BIT(4)); - - if (boot_cpu_data.x86 == 0x15) { - l3->subcaches[0] = sc0 += !(val & BIT(1)); - l3->subcaches[1] = sc1 += !(val & BIT(5)); - } - - l3->subcaches[2] = sc2 = !(val & BIT(8)) + !(val & BIT(9)); - l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13)); - - l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1; -} - -/* - * check whether a slot used for disabling an L3 index is occupied. - * @l3: L3 cache descriptor - * @slot: slot number (0..1) - * - * @returns: the disabled index if used or negative value if slot free. - */ -static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot) -{ - unsigned int reg = 0; - - pci_read_config_dword(nb->misc, 0x1BC + slot * 4, ®); - - /* check whether this slot is activated already */ - if (reg & (3UL << 30)) - return reg & 0xfff; - - return -1; -} - -static ssize_t show_cache_disable(struct cacheinfo *ci, char *buf, unsigned int slot) -{ - int index; - struct amd_northbridge *nb = ci->priv; - - index = amd_get_l3_disable_slot(nb, slot); - if (index >= 0) - return sprintf(buf, "%d\n", index); - - return sprintf(buf, "FREE\n"); -} - -#define SHOW_CACHE_DISABLE(slot) \ -static ssize_t \ -cache_disable_##slot##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct cacheinfo *ci = dev_get_drvdata(dev); \ - return show_cache_disable(ci, buf, slot); \ -} -SHOW_CACHE_DISABLE(0) -SHOW_CACHE_DISABLE(1) - -static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu, - unsigned slot, unsigned long idx) -{ - int i; - - idx |= BIT(30); - - /* - * disable index in all 4 subcaches - */ - for (i = 0; i < 4; i++) { - u32 reg = idx | (i << 20); - - if (!nb->l3_cache.subcaches[i]) - continue; - - pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg); - - /* - * We need to WBINVD on a core on the node containing the L3 - * cache which indices we disable therefore a simple wbinvd() - * is not sufficient. - */ - wbinvd_on_cpu(cpu); - - reg |= BIT(31); - pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg); - } -} - -/* - * disable a L3 cache index by using a disable-slot - * - * @l3: L3 cache descriptor - * @cpu: A CPU on the node containing the L3 cache - * @slot: slot number (0..1) - * @index: index to disable - * - * @return: 0 on success, error status on failure - */ -static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu, - unsigned slot, unsigned long index) -{ - int ret = 0; - - /* check if @slot is already used or the index is already disabled */ - ret = amd_get_l3_disable_slot(nb, slot); - if (ret >= 0) - return -EEXIST; - - if (index > nb->l3_cache.indices) - return -EINVAL; - - /* check whether the other slot has disabled the same index already */ - if (index == amd_get_l3_disable_slot(nb, !slot)) - return -EEXIST; - - amd_l3_disable_index(nb, cpu, slot, index); - - return 0; -} - -static ssize_t store_cache_disable(struct cacheinfo *ci, const char *buf, - size_t count, unsigned int slot) -{ - unsigned long val = 0; - int cpu, err = 0; - struct amd_northbridge *nb = ci->priv; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - cpu = cpumask_first(&ci->shared_cpu_map); - - if (kstrtoul(buf, 10, &val) < 0) - return -EINVAL; - - err = amd_set_l3_disable_slot(nb, cpu, slot, val); - if (err) { - if (err == -EEXIST) - pr_warn("L3 slot %d in use/index already disabled!\n", - slot); - return err; - } - return count; -} - -#define STORE_CACHE_DISABLE(slot) \ -static ssize_t \ -cache_disable_##slot##_store(struct device *dev, \ - struct device_attribute *attr, \ - const char *buf, size_t count) \ -{ \ - struct cacheinfo *ci = dev_get_drvdata(dev); \ - return store_cache_disable(ci, buf, count, slot); \ -} -STORE_CACHE_DISABLE(0) -STORE_CACHE_DISABLE(1) - -static ssize_t subcaches_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct cacheinfo *ci = dev_get_drvdata(dev); - int cpu = cpumask_first(&ci->shared_cpu_map); - - return sprintf(buf, "%x\n", amd_get_subcaches(cpu)); -} - -static ssize_t subcaches_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct cacheinfo *ci = dev_get_drvdata(dev); - int cpu = cpumask_first(&ci->shared_cpu_map); - unsigned long val; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - if (kstrtoul(buf, 16, &val) < 0) - return -EINVAL; - - if (amd_set_subcaches(cpu, val)) - return -EINVAL; - - return count; -} - -static DEVICE_ATTR_RW(cache_disable_0); -static DEVICE_ATTR_RW(cache_disable_1); -static DEVICE_ATTR_RW(subcaches); - -static umode_t -cache_private_attrs_is_visible(struct kobject *kobj, - struct attribute *attr, int unused) -{ - struct device *dev = kobj_to_dev(kobj); - struct cacheinfo *ci = dev_get_drvdata(dev); - umode_t mode = attr->mode; - - if (!ci->priv) - return 0; - - if ((attr == &dev_attr_subcaches.attr) && - amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) - return mode; - - if ((attr == &dev_attr_cache_disable_0.attr || - attr == &dev_attr_cache_disable_1.attr) && - amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) - return mode; - - return 0; -} - -static struct attribute_group cache_private_group = { - .is_visible = cache_private_attrs_is_visible, -}; - -static void init_amd_l3_attrs(void) -{ - int n = 1; - static struct attribute **amd_l3_attrs; - - if (amd_l3_attrs) /* already initialized */ - return; - - if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) - n += 2; - if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) - n += 1; - - amd_l3_attrs = kcalloc(n, sizeof(*amd_l3_attrs), GFP_KERNEL); - if (!amd_l3_attrs) - return; - - n = 0; - if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) { - amd_l3_attrs[n++] = &dev_attr_cache_disable_0.attr; - amd_l3_attrs[n++] = &dev_attr_cache_disable_1.attr; - } - if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) - amd_l3_attrs[n++] = &dev_attr_subcaches.attr; - - cache_private_group.attrs = amd_l3_attrs; -} - -const struct attribute_group * -cache_get_priv_group(struct cacheinfo *ci) -{ - struct amd_northbridge *nb = ci->priv; - - if (ci->level < 3 || !nb) - return NULL; - - if (nb && nb->l3_cache.indices) - init_amd_l3_attrs(); - - return &cache_private_group; -} - -static struct amd_northbridge *amd_init_l3_cache(int index) -{ - struct amd_northbridge *nb; - int node; - - /* only for L3, and not in virtualized environments */ - if (index < 3) - return NULL; - - node = topology_amd_node_id(smp_processor_id()); - nb = node_to_amd_nb(node); - if (nb && !nb->l3_cache.indices) - amd_calc_l3_indices(nb); - - return nb; -} -#else -static struct amd_northbridge *amd_init_l3_cache(int index) -{ - return NULL; -} -#endif /* CONFIG_AMD_NB && CONFIG_SYSFS */ - /* * Fill passed _cpuid4_info_regs structure. * Intel-only code paths should pass NULL for the amd_northbridge diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 51deb60a9d26..bc38b2d56f26 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -75,6 +75,15 @@ extern void check_null_seg_clears_base(struct cpuinfo_x86 *c); void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, u16 die_id); void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c); +#if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS) +struct amd_northbridge *amd_init_l3_cache(int index); +#else +static inline struct amd_northbridge *amd_init_l3_cache(int index) +{ + return NULL; +} +#endif + unsigned int aperfmperf_get_khz(int cpu); void cpu_select_mitigations(void); From 071f4ad6494aff76589ca30a2d13e74bc1e33e0f Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:09 +0100 Subject: [PATCH 034/331] x86/cacheinfo: Use sysfs_emit() for sysfs attributes show() Per Documentation/filesystems/sysfs.rst, a sysfs attribute's show() method should only use sysfs_emit() or sysfs_emit_at() when returning values to user space. Use sysfs_emit() for the AMD L3 cache sysfs attributes cache_disable_0, cache_disable_1, and subcaches. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-15-darwi@linutronix.de --- arch/x86/kernel/cpu/amd_cache_disable.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/amd_cache_disable.c b/arch/x86/kernel/cpu/amd_cache_disable.c index 6d53aee0d869..d860ad3f8a5a 100644 --- a/arch/x86/kernel/cpu/amd_cache_disable.c +++ b/arch/x86/kernel/cpu/amd_cache_disable.c @@ -66,9 +66,9 @@ static ssize_t show_cache_disable(struct cacheinfo *ci, char *buf, unsigned int index = amd_get_l3_disable_slot(nb, slot); if (index >= 0) - return sprintf(buf, "%d\n", index); + return sysfs_emit(buf, "%d\n", index); - return sprintf(buf, "FREE\n"); + return sysfs_emit(buf, "FREE\n"); } #define SHOW_CACHE_DISABLE(slot) \ @@ -189,7 +189,7 @@ static ssize_t subcaches_show(struct device *dev, struct device_attribute *attr, struct cacheinfo *ci = dev_get_drvdata(dev); int cpu = cpumask_first(&ci->shared_cpu_map); - return sprintf(buf, "%x\n", amd_get_subcaches(cpu)); + return sysfs_emit(buf, "%x\n", amd_get_subcaches(cpu)); } static ssize_t subcaches_store(struct device *dev, From 2d56cc87225035919dfceb40e676081d833dab13 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:10 +0100 Subject: [PATCH 035/331] x86/cacheinfo: Separate Intel and AMD CPUID leaf 0x4 code paths The CPUID leaf 0x4 parsing code at cpuid4_cache_lookup_regs() is ugly and convoluted. It is tangled with multiple nested conditions to handle: * AMD with TOPEXT, or Hygon CPUs via leaf 0x8000001d * Legacy AMD fallback via leaf 0x4 emulation * Intel CPUs via the actual CPUID leaf 0x4 Moreover, AMD L3 northbridge initialization is also awkwardly placed alongside the CPUID calls of the first two scenarios above. Refactor all of that as follows: * Update AMD's leaf 0x4 emulation comment to represent current state * Clearly label the AMD leaf 0x4 emulation function as a fallback * Split AMD/Hygon and Intel code paths into separate functions * Move AMD L3 northbridge initialization out of CPUID leaf 0x4 code, and into populate_cache_leaves() where it belongs. There, ci_info_init() can directly store the initialized object in the private pointer of the API. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-16-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 95 +++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index ea6fba942d27..10a79d87ce86 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -167,12 +167,11 @@ struct _cpuid4_info_regs { unsigned long size; }; -/* AMD doesn't have CPUID4. Emulate it here to report the same - information to the user. This makes some assumptions about the machine: - L2 not shared, no SMT etc. that is currently true on AMD CPUs. +/* + * Fallback AMD CPUID(4) emulation + * AMD CPUs with TOPOEXT can just use CPUID(0x8000001d) + */ - In theory the TLBs could be reported as fake type (they are in "dummy"). - Maybe later */ union l1_cache { struct { unsigned line_size:8; @@ -228,9 +227,8 @@ static const enum cache_type cache_type_map[] = { [CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED, }; -static void -amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, - union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) +static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, + union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) { unsigned int dummy, line_size, lines_per_tag, assoc, size_in_kb; union l1_cache l1i, l1d; @@ -297,36 +295,9 @@ amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, (ebx->split.ways_of_associativity + 1) - 1; } -/* - * Fill passed _cpuid4_info_regs structure. - * Intel-only code paths should pass NULL for the amd_northbridge - * return pointer. - */ -static int cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4, - struct amd_northbridge **nb) +static int cpuid4_info_fill_done(struct _cpuid4_info_regs *id4, union _cpuid4_leaf_eax eax, + union _cpuid4_leaf_ebx ebx, union _cpuid4_leaf_ecx ecx) { - u8 cpu_vendor = boot_cpu_data.x86_vendor; - union _cpuid4_leaf_eax eax; - union _cpuid4_leaf_ebx ebx; - union _cpuid4_leaf_ecx ecx; - u32 edx; - - if (cpu_vendor == X86_VENDOR_AMD || cpu_vendor == X86_VENDOR_HYGON) { - if (boot_cpu_has(X86_FEATURE_TOPOEXT) || cpu_vendor == X86_VENDOR_HYGON) { - /* AMD with TOPOEXT, or HYGON */ - cpuid_count(0x8000001d, index, &eax.full, &ebx.full, &ecx.full, &edx); - } else { - /* Legacy AMD fallback */ - amd_cpuid4(index, &eax, &ebx, &ecx); - } - - if (nb) - *nb = amd_init_l3_cache(index); - } else { - /* Intel */ - cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx); - } - if (eax.split.type == CTYPE_NULL) return -EIO; @@ -341,6 +312,42 @@ static int cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *id4, return 0; } +static int amd_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +{ + union _cpuid4_leaf_eax eax; + union _cpuid4_leaf_ebx ebx; + union _cpuid4_leaf_ecx ecx; + u32 ignored; + + if (boot_cpu_has(X86_FEATURE_TOPOEXT) || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + cpuid_count(0x8000001d, index, &eax.full, &ebx.full, &ecx.full, &ignored); + else + legacy_amd_cpuid4(index, &eax, &ebx, &ecx); + + return cpuid4_info_fill_done(id4, eax, ebx, ecx); +} + +static int intel_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +{ + union _cpuid4_leaf_eax eax; + union _cpuid4_leaf_ebx ebx; + union _cpuid4_leaf_ecx ecx; + u32 ignored; + + cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &ignored); + + return cpuid4_info_fill_done(id4, eax, ebx, ecx); +} + +static int fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +{ + u8 cpu_vendor = boot_cpu_data.x86_vendor; + + return (cpu_vendor == X86_VENDOR_AMD || cpu_vendor == X86_VENDOR_HYGON) ? + amd_fill_cpuid4_info(index, id4) : + intel_fill_cpuid4_info(index, id4); +} + static int find_num_cache_leaves(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx, op; @@ -472,7 +479,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) struct _cpuid4_info_regs id4 = {}; int retval; - retval = cpuid4_cache_lookup_regs(i, &id4, NULL); + retval = intel_fill_cpuid4_info(i, &id4); if (retval < 0) continue; @@ -692,17 +699,23 @@ static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4) int populate_cache_leaves(unsigned int cpu) { - unsigned int idx, ret; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci = this_cpu_ci->info_list; + u8 cpu_vendor = boot_cpu_data.x86_vendor; struct _cpuid4_info_regs id4 = {}; - struct amd_northbridge *nb; + struct amd_northbridge *nb = NULL; + int idx, ret; for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { - ret = cpuid4_cache_lookup_regs(idx, &id4, &nb); + ret = fill_cpuid4_info(idx, &id4); if (ret) return ret; + get_cache_id(cpu, &id4); + + if (cpu_vendor == X86_VENDOR_AMD || cpu_vendor == X86_VENDOR_HYGON) + nb = amd_init_l3_cache(idx); + ci_info_init(ci++, &id4, nb); __cache_cpumap_setup(cpu, idx, &id4); } From eb1c7c08c5a8155d0d2d0a6c7fd741039b7287f1 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:11 +0100 Subject: [PATCH 036/331] x86/cacheinfo: Rename 'struct _cpuid4_info_regs' to 'struct _cpuid4_info' Parent commits decoupled amd_northbridge from _cpuid4_info_regs, moved AMD L3 northbridge cache_disable_0/1 sysfs code to its own file, and splitted AMD vs. Intel leaf 0x4 handling into: amd_fill_cpuid4_info() intel_fill_cpuid4_info() fill_cpuid4_info() After doing all that, the "_cpuid4_info_regs" name becomes a mouthful. It is also not totally accurate, as the structure holds cpuid4 derived information like cache node ID and size -- not just regs. Rename struct _cpuid4_info_regs to _cpuid4_info. That new name also better matches the AMD/Intel leaf 0x4 functions mentioned above. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-17-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 10a79d87ce86..2aaa0f830ec3 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -159,7 +159,7 @@ union _cpuid4_leaf_ecx { u32 full; }; -struct _cpuid4_info_regs { +struct _cpuid4_info { union _cpuid4_leaf_eax eax; union _cpuid4_leaf_ebx ebx; union _cpuid4_leaf_ecx ecx; @@ -295,7 +295,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, (ebx->split.ways_of_associativity + 1) - 1; } -static int cpuid4_info_fill_done(struct _cpuid4_info_regs *id4, union _cpuid4_leaf_eax eax, +static int cpuid4_info_fill_done(struct _cpuid4_info *id4, union _cpuid4_leaf_eax eax, union _cpuid4_leaf_ebx ebx, union _cpuid4_leaf_ecx ecx) { if (eax.split.type == CTYPE_NULL) @@ -312,7 +312,7 @@ static int cpuid4_info_fill_done(struct _cpuid4_info_regs *id4, union _cpuid4_le return 0; } -static int amd_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +static int amd_fill_cpuid4_info(int index, struct _cpuid4_info *id4) { union _cpuid4_leaf_eax eax; union _cpuid4_leaf_ebx ebx; @@ -327,7 +327,7 @@ static int amd_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) return cpuid4_info_fill_done(id4, eax, ebx, ecx); } -static int intel_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +static int intel_fill_cpuid4_info(int index, struct _cpuid4_info *id4) { union _cpuid4_leaf_eax eax; union _cpuid4_leaf_ebx ebx; @@ -339,7 +339,7 @@ static int intel_fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) return cpuid4_info_fill_done(id4, eax, ebx, ecx); } -static int fill_cpuid4_info(int index, struct _cpuid4_info_regs *id4) +static int fill_cpuid4_info(int index, struct _cpuid4_info *id4) { u8 cpu_vendor = boot_cpu_data.x86_vendor; @@ -476,7 +476,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) * parameters cpuid leaf to find the cache details */ for (i = 0; i < ci->num_leaves; i++) { - struct _cpuid4_info_regs id4 = {}; + struct _cpuid4_info id4 = {}; int retval; retval = intel_fill_cpuid4_info(i, &id4); @@ -563,7 +563,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) } static int __cache_amd_cpumap_setup(unsigned int cpu, int index, - const struct _cpuid4_info_regs *id4) + const struct _cpuid4_info *id4) { struct cpu_cacheinfo *this_cpu_ci; struct cacheinfo *ci; @@ -620,7 +620,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, } static void __cache_cpumap_setup(unsigned int cpu, int index, - const struct _cpuid4_info_regs *id4) + const struct _cpuid4_info *id4) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci, *sibling_ci; @@ -655,7 +655,7 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, } } -static void ci_info_init(struct cacheinfo *ci, const struct _cpuid4_info_regs *id4, +static void ci_info_init(struct cacheinfo *ci, const struct _cpuid4_info *id4, struct amd_northbridge *nb) { ci->id = id4->id; @@ -686,7 +686,7 @@ int init_cache_level(unsigned int cpu) * ECX as cache index. Then right shift apicid by the number's order to get * cache id for this cache node. */ -static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4) +static void get_cache_id(int cpu, struct _cpuid4_info *id4) { struct cpuinfo_x86 *c = &cpu_data(cpu); unsigned long num_threads_sharing; @@ -702,8 +702,8 @@ int populate_cache_leaves(unsigned int cpu) struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *ci = this_cpu_ci->info_list; u8 cpu_vendor = boot_cpu_data.x86_vendor; - struct _cpuid4_info_regs id4 = {}; struct amd_northbridge *nb = NULL; + struct _cpuid4_info id4 = {}; int idx, ret; for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { From 7596ab7a107bf70d571d75ad6ae078127f5ea11b Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:12 +0100 Subject: [PATCH 037/331] x86/cacheinfo: Clarify type markers for CPUID leaf 0x2 cache descriptors CPUID leaf 0x2 output is a stream of one-byte descriptors, each implying certain details about the CPU's cache and TLB entries. Two separate tables exist for interpreting these descriptors: one for TLBs at intel.c and one for caches at cacheinfo.c. These mapping tables will be merged in further commits, among other improvements to their model. In preparation for this, use more descriptive type names for the leaf 0x2 descriptors associated with cpu caches. Namely: LVL_1_INST => CACHE_L1_INST LVL_1_DATA => CACHE_L1_DATA LVL_2 => CACHE_L2 LVL_3 => CACHE_L3 After the TLB and cache descriptors mapping tables are merged, this will make it clear that such descriptors correspond to cpu caches. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-18-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 152 ++++++++++++++++---------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 2aaa0f830ec3..626f55f960dc 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -23,10 +23,10 @@ #include "cpu.h" -#define LVL_1_INST 1 -#define LVL_1_DATA 2 -#define LVL_2 3 -#define LVL_3 4 +#define CACHE_L1_INST 1 +#define CACHE_L1_DATA 2 +#define CACHE_L2 3 +#define CACHE_L3 4 /* Shared last level cache maps */ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); @@ -52,74 +52,74 @@ struct _cache_table { static const struct _cache_table cache_table[] = { - { 0x06, LVL_1_INST, 8 }, /* 4-way set assoc, 32 byte line size */ - { 0x08, LVL_1_INST, 16 }, /* 4-way set assoc, 32 byte line size */ - { 0x09, LVL_1_INST, 32 }, /* 4-way set assoc, 64 byte line size */ - { 0x0a, LVL_1_DATA, 8 }, /* 2 way set assoc, 32 byte line size */ - { 0x0c, LVL_1_DATA, 16 }, /* 4-way set assoc, 32 byte line size */ - { 0x0d, LVL_1_DATA, 16 }, /* 4-way set assoc, 64 byte line size */ - { 0x0e, LVL_1_DATA, 24 }, /* 6-way set assoc, 64 byte line size */ - { 0x21, LVL_2, 256 }, /* 8-way set assoc, 64 byte line size */ - { 0x22, LVL_3, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x23, LVL_3, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x25, LVL_3, MB(2) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x29, LVL_3, MB(4) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x2c, LVL_1_DATA, 32 }, /* 8-way set assoc, 64 byte line size */ - { 0x30, LVL_1_INST, 32 }, /* 8-way set assoc, 64 byte line size */ - { 0x39, LVL_2, 128 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3a, LVL_2, 192 }, /* 6-way set assoc, sectored cache, 64 byte line size */ - { 0x3b, LVL_2, 128 }, /* 2-way set assoc, sectored cache, 64 byte line size */ - { 0x3c, LVL_2, 256 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3d, LVL_2, 384 }, /* 6-way set assoc, sectored cache, 64 byte line size */ - { 0x3e, LVL_2, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3f, LVL_2, 256 }, /* 2-way set assoc, 64 byte line size */ - { 0x41, LVL_2, 128 }, /* 4-way set assoc, 32 byte line size */ - { 0x42, LVL_2, 256 }, /* 4-way set assoc, 32 byte line size */ - { 0x43, LVL_2, 512 }, /* 4-way set assoc, 32 byte line size */ - { 0x44, LVL_2, MB(1) }, /* 4-way set assoc, 32 byte line size */ - { 0x45, LVL_2, MB(2) }, /* 4-way set assoc, 32 byte line size */ - { 0x46, LVL_3, MB(4) }, /* 4-way set assoc, 64 byte line size */ - { 0x47, LVL_3, MB(8) }, /* 8-way set assoc, 64 byte line size */ - { 0x48, LVL_2, MB(3) }, /* 12-way set assoc, 64 byte line size */ - { 0x49, LVL_3, MB(4) }, /* 16-way set assoc, 64 byte line size */ - { 0x4a, LVL_3, MB(6) }, /* 12-way set assoc, 64 byte line size */ - { 0x4b, LVL_3, MB(8) }, /* 16-way set assoc, 64 byte line size */ - { 0x4c, LVL_3, MB(12) }, /* 12-way set assoc, 64 byte line size */ - { 0x4d, LVL_3, MB(16) }, /* 16-way set assoc, 64 byte line size */ - { 0x4e, LVL_2, MB(6) }, /* 24-way set assoc, 64 byte line size */ - { 0x60, LVL_1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x66, LVL_1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x67, LVL_1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x68, LVL_1_DATA, 32 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x78, LVL_2, MB(1) }, /* 4-way set assoc, 64 byte line size */ - { 0x79, LVL_2, 128 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7a, LVL_2, 256 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7b, LVL_2, 512 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7c, LVL_2, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7d, LVL_2, MB(2) }, /* 8-way set assoc, 64 byte line size */ - { 0x7f, LVL_2, 512 }, /* 2-way set assoc, 64 byte line size */ - { 0x80, LVL_2, 512 }, /* 8-way set assoc, 64 byte line size */ - { 0x82, LVL_2, 256 }, /* 8-way set assoc, 32 byte line size */ - { 0x83, LVL_2, 512 }, /* 8-way set assoc, 32 byte line size */ - { 0x84, LVL_2, MB(1) }, /* 8-way set assoc, 32 byte line size */ - { 0x85, LVL_2, MB(2) }, /* 8-way set assoc, 32 byte line size */ - { 0x86, LVL_2, 512 }, /* 4-way set assoc, 64 byte line size */ - { 0x87, LVL_2, MB(1) }, /* 8-way set assoc, 64 byte line size */ - { 0xd0, LVL_3, 512 }, /* 4-way set assoc, 64 byte line size */ - { 0xd1, LVL_3, MB(1) }, /* 4-way set assoc, 64 byte line size */ - { 0xd2, LVL_3, MB(2) }, /* 4-way set assoc, 64 byte line size */ - { 0xd6, LVL_3, MB(1) }, /* 8-way set assoc, 64 byte line size */ - { 0xd7, LVL_3, MB(2) }, /* 8-way set assoc, 64 byte line size */ - { 0xd8, LVL_3, MB(4) }, /* 12-way set assoc, 64 byte line size */ - { 0xdc, LVL_3, MB(2) }, /* 12-way set assoc, 64 byte line size */ - { 0xdd, LVL_3, MB(4) }, /* 12-way set assoc, 64 byte line size */ - { 0xde, LVL_3, MB(8) }, /* 12-way set assoc, 64 byte line size */ - { 0xe2, LVL_3, MB(2) }, /* 16-way set assoc, 64 byte line size */ - { 0xe3, LVL_3, MB(4) }, /* 16-way set assoc, 64 byte line size */ - { 0xe4, LVL_3, MB(8) }, /* 16-way set assoc, 64 byte line size */ - { 0xea, LVL_3, MB(12) }, /* 24-way set assoc, 64 byte line size */ - { 0xeb, LVL_3, MB(18) }, /* 24-way set assoc, 64 byte line size */ - { 0xec, LVL_3, MB(24) }, /* 24-way set assoc, 64 byte line size */ + { 0x06, CACHE_L1_INST, 8 }, /* 4-way set assoc, 32 byte line size */ + { 0x08, CACHE_L1_INST, 16 }, /* 4-way set assoc, 32 byte line size */ + { 0x09, CACHE_L1_INST, 32 }, /* 4-way set assoc, 64 byte line size */ + { 0x0a, CACHE_L1_DATA, 8 }, /* 2 way set assoc, 32 byte line size */ + { 0x0c, CACHE_L1_DATA, 16 }, /* 4-way set assoc, 32 byte line size */ + { 0x0d, CACHE_L1_DATA, 16 }, /* 4-way set assoc, 64 byte line size */ + { 0x0e, CACHE_L1_DATA, 24 }, /* 6-way set assoc, 64 byte line size */ + { 0x21, CACHE_L2, 256 }, /* 8-way set assoc, 64 byte line size */ + { 0x22, CACHE_L3, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x23, CACHE_L3, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x25, CACHE_L3, MB(2) }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x29, CACHE_L3, MB(4) }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x2c, CACHE_L1_DATA, 32 }, /* 8-way set assoc, 64 byte line size */ + { 0x30, CACHE_L1_INST, 32 }, /* 8-way set assoc, 64 byte line size */ + { 0x39, CACHE_L2, 128 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x3a, CACHE_L2, 192 }, /* 6-way set assoc, sectored cache, 64 byte line size */ + { 0x3b, CACHE_L2, 128 }, /* 2-way set assoc, sectored cache, 64 byte line size */ + { 0x3c, CACHE_L2, 256 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x3d, CACHE_L2, 384 }, /* 6-way set assoc, sectored cache, 64 byte line size */ + { 0x3e, CACHE_L2, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x3f, CACHE_L2, 256 }, /* 2-way set assoc, 64 byte line size */ + { 0x41, CACHE_L2, 128 }, /* 4-way set assoc, 32 byte line size */ + { 0x42, CACHE_L2, 256 }, /* 4-way set assoc, 32 byte line size */ + { 0x43, CACHE_L2, 512 }, /* 4-way set assoc, 32 byte line size */ + { 0x44, CACHE_L2, MB(1) }, /* 4-way set assoc, 32 byte line size */ + { 0x45, CACHE_L2, MB(2) }, /* 4-way set assoc, 32 byte line size */ + { 0x46, CACHE_L3, MB(4) }, /* 4-way set assoc, 64 byte line size */ + { 0x47, CACHE_L3, MB(8) }, /* 8-way set assoc, 64 byte line size */ + { 0x48, CACHE_L2, MB(3) }, /* 12-way set assoc, 64 byte line size */ + { 0x49, CACHE_L3, MB(4) }, /* 16-way set assoc, 64 byte line size */ + { 0x4a, CACHE_L3, MB(6) }, /* 12-way set assoc, 64 byte line size */ + { 0x4b, CACHE_L3, MB(8) }, /* 16-way set assoc, 64 byte line size */ + { 0x4c, CACHE_L3, MB(12) }, /* 12-way set assoc, 64 byte line size */ + { 0x4d, CACHE_L3, MB(16) }, /* 16-way set assoc, 64 byte line size */ + { 0x4e, CACHE_L2, MB(6) }, /* 24-way set assoc, 64 byte line size */ + { 0x60, CACHE_L1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x66, CACHE_L1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x67, CACHE_L1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x68, CACHE_L1_DATA, 32 }, /* 4-way set assoc, sectored cache, 64 byte line size */ + { 0x78, CACHE_L2, MB(1) }, /* 4-way set assoc, 64 byte line size */ + { 0x79, CACHE_L2, 128 }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x7a, CACHE_L2, 256 }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x7b, CACHE_L2, 512 }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x7c, CACHE_L2, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ + { 0x7d, CACHE_L2, MB(2) }, /* 8-way set assoc, 64 byte line size */ + { 0x7f, CACHE_L2, 512 }, /* 2-way set assoc, 64 byte line size */ + { 0x80, CACHE_L2, 512 }, /* 8-way set assoc, 64 byte line size */ + { 0x82, CACHE_L2, 256 }, /* 8-way set assoc, 32 byte line size */ + { 0x83, CACHE_L2, 512 }, /* 8-way set assoc, 32 byte line size */ + { 0x84, CACHE_L2, MB(1) }, /* 8-way set assoc, 32 byte line size */ + { 0x85, CACHE_L2, MB(2) }, /* 8-way set assoc, 32 byte line size */ + { 0x86, CACHE_L2, 512 }, /* 4-way set assoc, 64 byte line size */ + { 0x87, CACHE_L2, MB(1) }, /* 8-way set assoc, 64 byte line size */ + { 0xd0, CACHE_L3, 512 }, /* 4-way set assoc, 64 byte line size */ + { 0xd1, CACHE_L3, MB(1) }, /* 4-way set assoc, 64 byte line size */ + { 0xd2, CACHE_L3, MB(2) }, /* 4-way set assoc, 64 byte line size */ + { 0xd6, CACHE_L3, MB(1) }, /* 8-way set assoc, 64 byte line size */ + { 0xd7, CACHE_L3, MB(2) }, /* 8-way set assoc, 64 byte line size */ + { 0xd8, CACHE_L3, MB(4) }, /* 12-way set assoc, 64 byte line size */ + { 0xdc, CACHE_L3, MB(2) }, /* 12-way set assoc, 64 byte line size */ + { 0xdd, CACHE_L3, MB(4) }, /* 12-way set assoc, 64 byte line size */ + { 0xde, CACHE_L3, MB(8) }, /* 12-way set assoc, 64 byte line size */ + { 0xe2, CACHE_L3, MB(2) }, /* 16-way set assoc, 64 byte line size */ + { 0xe3, CACHE_L3, MB(4) }, /* 16-way set assoc, 64 byte line size */ + { 0xe4, CACHE_L3, MB(8) }, /* 16-way set assoc, 64 byte line size */ + { 0xea, CACHE_L3, MB(12) }, /* 24-way set assoc, 64 byte line size */ + { 0xeb, CACHE_L3, MB(18) }, /* 24-way set assoc, 64 byte line size */ + { 0xec, CACHE_L3, MB(24) }, /* 24-way set assoc, 64 byte line size */ }; @@ -521,10 +521,10 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) continue; switch (entry->cache_type) { - case LVL_1_INST: l1i += entry->size; break; - case LVL_1_DATA: l1d += entry->size; break; - case LVL_2: l2 += entry->size; break; - case LVL_3: l3 += entry->size; break; + case CACHE_L1_INST: l1i += entry->size; break; + case CACHE_L1_DATA: l1d += entry->size; break; + case CACHE_L2: l2 += entry->size; break; + case CACHE_L3: l3 += entry->size; break; } } } From e1e6b57146554a321d0ed1e76d2839ac24117f26 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:13 +0100 Subject: [PATCH 038/331] x86/cacheinfo: Use enums for cache descriptor types The leaf 0x2 one-byte cache descriptor types: CACHE_L1_INST CACHE_L1_DATA CACHE_L2 CACHE_L3 are just discriminators to be used within the cache_table[] mapping. Their specific values are irrelevant. Use enums for such types. Make the enum packed and static assert that its values remain within a single byte so that the cache_table[] array size do not go out of hand. Use a __CHECKER__ guard for the static_assert(sizeof(enum) == 1) line as sparse ignores the __packed annotation on enums. This is similar to: fe3944fb245a ("fs: Move enum rw_hint into a new header file") for the core SCSI code. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/Z9rsTirs9lLfEPD9@lx-t490 Link: https://lore.kernel.org/r/20250324133324.23458-19-darwi@linutronix.de --- arch/x86/include/asm/cpuid/types.h | 15 +++++++++++++++ arch/x86/kernel/cpu/cacheinfo.c | 9 ++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index 753f6c4514f4..39c3c79c081b 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -2,6 +2,7 @@ #ifndef _ASM_X86_CPUID_TYPES_H #define _ASM_X86_CPUID_TYPES_H +#include #include /* @@ -45,4 +46,18 @@ union leaf_0x2_regs { u8 desc[16]; }; +/* + * Leaf 0x2 1-byte descriptors' cache types + * To be used for their mappings at cache_table[] + */ +enum _cache_table_type { + CACHE_L1_INST, + CACHE_L1_DATA, + CACHE_L2, + CACHE_L3, +} __packed; +#ifndef __CHECKER__ +static_assert(sizeof(enum _cache_table_type) == 1); +#endif + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 626f55f960dc..09c5aa9af973 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -23,11 +23,6 @@ #include "cpu.h" -#define CACHE_L1_INST 1 -#define CACHE_L1_DATA 2 -#define CACHE_L2 3 -#define CACHE_L3 4 - /* Shared last level cache maps */ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); @@ -41,7 +36,7 @@ unsigned int memory_caching_control __ro_after_init; struct _cache_table { u8 descriptor; - char cache_type; + enum _cache_table_type type; short size; }; @@ -520,7 +515,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) if (!entry) continue; - switch (entry->cache_type) { + switch (entry->type) { case CACHE_L1_INST: l1i += entry->size; break; case CACHE_L1_DATA: l1d += entry->size; break; case CACHE_L2: l2 += entry->size; break; From 543904cdfe1eb53ff4267f561a4d59cb1fe6ceb7 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:14 +0100 Subject: [PATCH 039/331] x86/cpu: Use enums for TLB descriptor types The leaf 0x2 one-byte TLB descriptor types: TLB_INST_4K TLB_INST_4M TLB_INST_2M_4M ... are just discriminators to be used within the intel_tlb_table[] mapping. Their specific values are irrelevant. Use enums for such types. Make the enum packed and static assert that its values remain within a single byte so that the intel_tlb_table[] size do not go out of hand. Use a __CHECKER__ guard for the static_assert(sizeof(enum) == 1) line as sparse ignores the __packed annotation on enums. This is similar to: fe3944fb245a ("fs: Move enum rw_hint into a new header file") for the core SCSI code. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/Z9rsTirs9lLfEPD9@lx-t490 Link: https://lore.kernel.org/r/20250324133324.23458-20-darwi@linutronix.de --- arch/x86/include/asm/cpuid/types.h | 31 ++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/intel.c | 28 +++------------------------ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index 39c3c79c081b..e756327f8e02 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -60,4 +60,35 @@ enum _cache_table_type { static_assert(sizeof(enum _cache_table_type) == 1); #endif +/* + * Leaf 0x2 1-byte descriptors' TLB types + * To be used for their mappings at intel_tlb_table[] + * + * Start at 1 since type 0 is reserved for HW byte descriptors which are + * not recognized by the kernel; i.e., those without an explicit mapping. + */ +enum _tlb_table_type { + TLB_INST_4K = 1, + TLB_INST_4M, + TLB_INST_2M_4M, + TLB_INST_ALL, + + TLB_DATA_4K, + TLB_DATA_4M, + TLB_DATA_2M_4M, + TLB_DATA_4K_4M, + TLB_DATA_1G, + TLB_DATA_1G_2M_4M, + + TLB_DATA0_4K, + TLB_DATA0_4M, + TLB_DATA0_2M_4M, + + STLB_4K, + STLB_4K_2M, +} __packed; +#ifndef __CHECKER__ +static_assert(sizeof(enum _tlb_table_type) == 1); +#endif + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index aeb7d6d48379..def433e0081f 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -626,28 +626,6 @@ static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) } #endif -#define TLB_INST_4K 0x01 -#define TLB_INST_4M 0x02 -#define TLB_INST_2M_4M 0x03 - -#define TLB_INST_ALL 0x05 -#define TLB_INST_1G 0x06 - -#define TLB_DATA_4K 0x11 -#define TLB_DATA_4M 0x12 -#define TLB_DATA_2M_4M 0x13 -#define TLB_DATA_4K_4M 0x14 - -#define TLB_DATA_1G 0x16 -#define TLB_DATA_1G_2M_4M 0x17 - -#define TLB_DATA0_4K 0x21 -#define TLB_DATA0_4M 0x22 -#define TLB_DATA0_2M_4M 0x23 - -#define STLB_4K 0x41 -#define STLB_4K_2M 0x42 - /* * All of leaf 0x2's one-byte TLB descriptors implies the same number of * entries for their respective TLB types. The 0x63 descriptor is an @@ -660,7 +638,7 @@ static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) struct _tlb_table { unsigned char descriptor; - char tlb_type; + enum _tlb_table_type type; unsigned int entries; }; @@ -718,11 +696,11 @@ static void intel_tlb_lookup(const unsigned char desc) intel_tlb_table[k].descriptor != 0; k++) ; - if (intel_tlb_table[k].tlb_type == 0) + if (intel_tlb_table[k].type == 0) return; entries = intel_tlb_table[k].entries; - switch (intel_tlb_table[k].tlb_type) { + switch (intel_tlb_table[k].type) { case STLB_4K: tlb_lli_4k = max(tlb_lli_4k, entries); tlb_lld_4k = max(tlb_lld_4k, entries); From 37aedb806bc68ff23bd0d90dce1564bfb0dca911 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 24 Mar 2025 14:33:15 +0100 Subject: [PATCH 040/331] x86/cpu: Consolidate CPUID leaf 0x2 tables CPUID leaf 0x2 describes TLBs and caches. So there are two tables with the respective descriptor constants in intel.c and cacheinfo.c. The tables occupy almost 600 byte and require a loop based lookup for each variant. Combining them into one table occupies exactly 1k rodata and allows to get rid of the loop based lookup by just using the descriptor byte provided by CPUID leaf 0x2 as index into the table, which simplifies the code and reduces text size. The conversion of the intel.c and cacheinfo.c code is done separately. [ darwi: Actually define struct leaf_0x2_table. Tab-align all of cpuid_0x2_table[] mapping entries. Define needed SZ_* macros at instead (merged commit.) Use CACHE_L1_{INST,DATA} as names for L1 cache descriptor types. Set descriptor 0x63 type as TLB_DATA_1G_2M_4M and explain why. Use enums for cache and TLB descriptor types (parent commits.) Start enum types at 1 since type 0 is reserved for unknown descriptors. Ensure that cache and TLB enum type values do not intersect. Add leaf 0x2 table accessor for_each_leaf_0x2_entry() + documentation. ] Co-developed-by: Ahmed S. Darwish Signed-off-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-21-darwi@linutronix.de --- arch/x86/include/asm/cpuid/leaf_0x2_api.h | 33 +++++- arch/x86/include/asm/cpuid/types.h | 40 +++++-- arch/x86/kernel/cpu/Makefile | 2 +- arch/x86/kernel/cpu/cpuid_0x2_table.c | 128 ++++++++++++++++++++++ 4 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 arch/x86/kernel/cpu/cpuid_0x2_table.c diff --git a/arch/x86/include/asm/cpuid/leaf_0x2_api.h b/arch/x86/include/asm/cpuid/leaf_0x2_api.h index 4c845fc96716..46ecb15e92d9 100644 --- a/arch/x86/include/asm/cpuid/leaf_0x2_api.h +++ b/arch/x86/include/asm/cpuid/leaf_0x2_api.h @@ -13,7 +13,8 @@ * invalid 1-byte descriptor returned by the hardware to zero (the NULL * cache/TLB descriptor) before returning it to the caller. * - * Use for_each_leaf_0x2_desc() to iterate over the returned output. + * Use for_each_leaf_0x2_entry() to iterate over the register output in + * parsed form. */ static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) { @@ -62,4 +63,34 @@ static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) #define for_each_leaf_0x2_desc(regs, desc) \ for (desc = &(regs).desc[1]; desc < &(regs).desc[16]; desc++) +/** + * for_each_leaf_0x2_entry() - Iterator for parsed leaf 0x2 descriptors + * @regs: Leaf 0x2 register output, returned by cpuid_get_leaf_0x2_regs() + * @__ptr: u8 pointer, for macro internal use only + * @entry: Pointer to parsed descriptor information at each iteration + * + * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers + * @regs. Provide the parsed information for each descriptor through @entry. + * + * To handle cache-specific descriptors, switch on @entry->c_type. For TLB + * descriptors, switch on @entry->t_type. + * + * Example usage for cache descriptors:: + * + * const struct leaf_0x2_table *entry; + * union leaf_0x2_regs regs; + * u8 *ptr; + * + * cpuid_get_leaf_0x2_regs(®s); + * for_each_leaf_0x2_entry(regs, ptr, entry) { + * switch (entry->c_type) { + * ... + * } + * } + */ +#define for_each_leaf_0x2_entry(regs, __ptr, entry) \ + for (__ptr = &(regs).desc[1]; \ + __ptr < &(regs).desc[16] && (entry = &cpuid_0x2_table[*__ptr]); \ + __ptr++) + #endif /* _ASM_X86_CPUID_LEAF_0x2_API_H */ diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index e756327f8e02..24f643f2a2ad 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -48,27 +48,34 @@ union leaf_0x2_regs { /* * Leaf 0x2 1-byte descriptors' cache types - * To be used for their mappings at cache_table[] + * To be used for their mappings at cpuid_0x2_table[] + * + * Start at 1 since type 0 is reserved for HW byte descriptors which are + * not recognized by the kernel; i.e., those without an explicit mapping. */ enum _cache_table_type { - CACHE_L1_INST, + CACHE_L1_INST = 1, CACHE_L1_DATA, CACHE_L2, - CACHE_L3, + CACHE_L3 + /* Adjust __TLB_TABLE_TYPE_BEGIN before adding more types */ } __packed; #ifndef __CHECKER__ static_assert(sizeof(enum _cache_table_type) == 1); #endif +/* + * Ensure that leaf 0x2 cache and TLB type values do not intersect, + * since they share the same type field at struct cpuid_0x2_table. + */ +#define __TLB_TABLE_TYPE_BEGIN (CACHE_L3 + 1) + /* * Leaf 0x2 1-byte descriptors' TLB types - * To be used for their mappings at intel_tlb_table[] - * - * Start at 1 since type 0 is reserved for HW byte descriptors which are - * not recognized by the kernel; i.e., those without an explicit mapping. + * To be used for their mappings at cpuid_0x2_table[] */ enum _tlb_table_type { - TLB_INST_4K = 1, + TLB_INST_4K = __TLB_TABLE_TYPE_BEGIN, TLB_INST_4M, TLB_INST_2M_4M, TLB_INST_ALL, @@ -91,4 +98,21 @@ enum _tlb_table_type { static_assert(sizeof(enum _tlb_table_type) == 1); #endif +/* + * Combined parsing table for leaf 0x2 cache and TLB descriptors. + */ + +struct leaf_0x2_table { + union { + enum _cache_table_type c_type; + enum _tlb_table_type t_type; + }; + union { + short c_size; + short entries; + }; +}; + +extern const struct leaf_0x2_table cpuid_0x2_table[256]; + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 3a39396d422d..1e26179ff18c 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -24,7 +24,7 @@ obj-y += rdrand.o obj-y += match.o obj-y += bugs.o obj-y += aperfmperf.o -obj-y += cpuid-deps.o +obj-y += cpuid-deps.o cpuid_0x2_table.o obj-y += umwait.o obj-y += capflags.o powerflags.o diff --git a/arch/x86/kernel/cpu/cpuid_0x2_table.c b/arch/x86/kernel/cpu/cpuid_0x2_table.c new file mode 100644 index 000000000000..89bc8db5e9c6 --- /dev/null +++ b/arch/x86/kernel/cpu/cpuid_0x2_table.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include + +#include "cpu.h" + +#define CACHE_ENTRY(_desc, _type, _size) \ + [_desc] = { \ + .c_type = (_type), \ + .c_size = (_size) / SZ_1K, \ + } + +#define TLB_ENTRY(_desc, _type, _entries) \ + [_desc] = { \ + .t_type = (_type), \ + .entries = (_entries), \ + } + +const struct leaf_0x2_table cpuid_0x2_table[256] = { + CACHE_ENTRY(0x06, CACHE_L1_INST, SZ_8K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x08, CACHE_L1_INST, SZ_16K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x09, CACHE_L1_INST, SZ_32K ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x0a, CACHE_L1_DATA, SZ_8K ), /* 2 way set assoc, 32 byte line size */ + CACHE_ENTRY(0x0c, CACHE_L1_DATA, SZ_16K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x0d, CACHE_L1_DATA, SZ_16K ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x0e, CACHE_L1_DATA, SZ_24K ), /* 6-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x21, CACHE_L2, SZ_256K ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x22, CACHE_L3, SZ_512K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x23, CACHE_L3, SZ_1M ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x25, CACHE_L3, SZ_2M ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x29, CACHE_L3, SZ_4M ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x2c, CACHE_L1_DATA, SZ_32K ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x30, CACHE_L1_INST, SZ_32K ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x39, CACHE_L2, SZ_128K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3a, CACHE_L2, SZ_192K ), /* 6-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3b, CACHE_L2, SZ_128K ), /* 2-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3c, CACHE_L2, SZ_256K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3d, CACHE_L2, SZ_384K ), /* 6-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3e, CACHE_L2, SZ_512K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x3f, CACHE_L2, SZ_256K ), /* 2-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x41, CACHE_L2, SZ_128K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x42, CACHE_L2, SZ_256K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x43, CACHE_L2, SZ_512K ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x44, CACHE_L2, SZ_1M ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x45, CACHE_L2, SZ_2M ), /* 4-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x46, CACHE_L3, SZ_4M ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x47, CACHE_L3, SZ_8M ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x48, CACHE_L2, SZ_3M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x49, CACHE_L3, SZ_4M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x4a, CACHE_L3, SZ_6M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x4b, CACHE_L3, SZ_8M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x4c, CACHE_L3, SZ_12M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x4d, CACHE_L3, SZ_16M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x4e, CACHE_L2, SZ_6M ), /* 24-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x60, CACHE_L1_DATA, SZ_16K ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x66, CACHE_L1_DATA, SZ_8K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x67, CACHE_L1_DATA, SZ_16K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x68, CACHE_L1_DATA, SZ_32K ), /* 4-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x78, CACHE_L2, SZ_1M ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x79, CACHE_L2, SZ_128K ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x7a, CACHE_L2, SZ_256K ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x7b, CACHE_L2, SZ_512K ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x7c, CACHE_L2, SZ_1M ), /* 8-way set assoc, sectored cache, 64 byte line size */ + CACHE_ENTRY(0x7d, CACHE_L2, SZ_2M ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x7f, CACHE_L2, SZ_512K ), /* 2-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x80, CACHE_L2, SZ_512K ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x82, CACHE_L2, SZ_256K ), /* 8-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x83, CACHE_L2, SZ_512K ), /* 8-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x84, CACHE_L2, SZ_1M ), /* 8-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x85, CACHE_L2, SZ_2M ), /* 8-way set assoc, 32 byte line size */ + CACHE_ENTRY(0x86, CACHE_L2, SZ_512K ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0x87, CACHE_L2, SZ_1M ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd0, CACHE_L3, SZ_512K ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd1, CACHE_L3, SZ_1M ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd2, CACHE_L3, SZ_2M ), /* 4-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd6, CACHE_L3, SZ_1M ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd7, CACHE_L3, SZ_2M ), /* 8-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xd8, CACHE_L3, SZ_4M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xdc, CACHE_L3, SZ_2M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xdd, CACHE_L3, SZ_4M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xde, CACHE_L3, SZ_8M ), /* 12-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xe2, CACHE_L3, SZ_2M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xe3, CACHE_L3, SZ_4M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xe4, CACHE_L3, SZ_8M ), /* 16-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xea, CACHE_L3, SZ_12M ), /* 24-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xeb, CACHE_L3, SZ_18M ), /* 24-way set assoc, 64 byte line size */ + CACHE_ENTRY(0xec, CACHE_L3, SZ_24M ), /* 24-way set assoc, 64 byte line size */ + + TLB_ENTRY( 0x01, TLB_INST_4K, 32 ), /* TLB_INST 4 KByte pages, 4-way set associative */ + TLB_ENTRY( 0x02, TLB_INST_4M, 2 ), /* TLB_INST 4 MByte pages, full associative */ + TLB_ENTRY( 0x03, TLB_DATA_4K, 64 ), /* TLB_DATA 4 KByte pages, 4-way set associative */ + TLB_ENTRY( 0x04, TLB_DATA_4M, 8 ), /* TLB_DATA 4 MByte pages, 4-way set associative */ + TLB_ENTRY( 0x05, TLB_DATA_4M, 32 ), /* TLB_DATA 4 MByte pages, 4-way set associative */ + TLB_ENTRY( 0x0b, TLB_INST_4M, 4 ), /* TLB_INST 4 MByte pages, 4-way set associative */ + TLB_ENTRY( 0x4f, TLB_INST_4K, 32 ), /* TLB_INST 4 KByte pages */ + TLB_ENTRY( 0x50, TLB_INST_ALL, 64 ), /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ + TLB_ENTRY( 0x51, TLB_INST_ALL, 128 ), /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ + TLB_ENTRY( 0x52, TLB_INST_ALL, 256 ), /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ + TLB_ENTRY( 0x55, TLB_INST_2M_4M, 7 ), /* TLB_INST 2-MByte or 4-MByte pages, fully associative */ + TLB_ENTRY( 0x56, TLB_DATA0_4M, 16 ), /* TLB_DATA0 4 MByte pages, 4-way set associative */ + TLB_ENTRY( 0x57, TLB_DATA0_4K, 16 ), /* TLB_DATA0 4 KByte pages, 4-way associative */ + TLB_ENTRY( 0x59, TLB_DATA0_4K, 16 ), /* TLB_DATA0 4 KByte pages, fully associative */ + TLB_ENTRY( 0x5a, TLB_DATA0_2M_4M, 32 ), /* TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative */ + TLB_ENTRY( 0x5b, TLB_DATA_4K_4M, 64 ), /* TLB_DATA 4 KByte and 4 MByte pages */ + TLB_ENTRY( 0x5c, TLB_DATA_4K_4M, 128 ), /* TLB_DATA 4 KByte and 4 MByte pages */ + TLB_ENTRY( 0x5d, TLB_DATA_4K_4M, 256 ), /* TLB_DATA 4 KByte and 4 MByte pages */ + TLB_ENTRY( 0x61, TLB_INST_4K, 48 ), /* TLB_INST 4 KByte pages, full associative */ + TLB_ENTRY( 0x63, TLB_DATA_1G_2M_4M, 4 ), /* TLB_DATA 1 GByte pages, 4-way set associative + * (plus 32 entries TLB_DATA 2 MByte or 4 MByte pages, not encoded here) */ + TLB_ENTRY( 0x6b, TLB_DATA_4K, 256 ), /* TLB_DATA 4 KByte pages, 8-way associative */ + TLB_ENTRY( 0x6c, TLB_DATA_2M_4M, 128 ), /* TLB_DATA 2 MByte or 4 MByte pages, 8-way associative */ + TLB_ENTRY( 0x6d, TLB_DATA_1G, 16 ), /* TLB_DATA 1 GByte pages, fully associative */ + TLB_ENTRY( 0x76, TLB_INST_2M_4M, 8 ), /* TLB_INST 2-MByte or 4-MByte pages, fully associative */ + TLB_ENTRY( 0xb0, TLB_INST_4K, 128 ), /* TLB_INST 4 KByte pages, 4-way set associative */ + TLB_ENTRY( 0xb1, TLB_INST_2M_4M, 4 ), /* TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries */ + TLB_ENTRY( 0xb2, TLB_INST_4K, 64 ), /* TLB_INST 4KByte pages, 4-way set associative */ + TLB_ENTRY( 0xb3, TLB_DATA_4K, 128 ), /* TLB_DATA 4 KByte pages, 4-way set associative */ + TLB_ENTRY( 0xb4, TLB_DATA_4K, 256 ), /* TLB_DATA 4 KByte pages, 4-way associative */ + TLB_ENTRY( 0xb5, TLB_INST_4K, 64 ), /* TLB_INST 4 KByte pages, 8-way set associative */ + TLB_ENTRY( 0xb6, TLB_INST_4K, 128 ), /* TLB_INST 4 KByte pages, 8-way set associative */ + TLB_ENTRY( 0xba, TLB_DATA_4K, 64 ), /* TLB_DATA 4 KByte pages, 4-way associative */ + TLB_ENTRY( 0xc0, TLB_DATA_4K_4M, 8 ), /* TLB_DATA 4 KByte and 4 MByte pages, 4-way associative */ + TLB_ENTRY( 0xc1, STLB_4K_2M, 1024 ), /* STLB 4 KByte and 2 MByte pages, 8-way associative */ + TLB_ENTRY( 0xc2, TLB_DATA_2M_4M, 16 ), /* TLB_DATA 2 MByte/4MByte pages, 4-way associative */ + TLB_ENTRY( 0xca, STLB_4K, 512 ), /* STLB 4 KByte pages, 4-way associative */ +}; From da23a6259844b576d98ad5c633eb437d3a2d90d3 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:16 +0100 Subject: [PATCH 041/331] x86/cacheinfo: Use consolidated CPUID leaf 0x2 descriptor table CPUID leaf 0x2 output is a stream of one-byte descriptors, each implying certain details about the CPU's cache and TLB entries. At previous commits, the mapping tables for such descriptors were merged into one consolidated table. The mapping was also transformed into a hash lookup instead of a loop-based lookup for each descriptor. Use the new consolidated table and its hash-based lookup through the for_each_leaf_0x2_tlb_entry() accessor. Remove the old cache-specific mapping, cache_table[], as it is no longer used. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-22-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 114 +++----------------------------- 1 file changed, 8 insertions(+), 106 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 09c5aa9af973..e399bf2a701d 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -34,90 +34,6 @@ static cpumask_var_t cpu_cacheinfo_mask; /* Kernel controls MTRR and/or PAT MSRs. */ unsigned int memory_caching_control __ro_after_init; -struct _cache_table { - u8 descriptor; - enum _cache_table_type type; - short size; -}; - -#define MB(x) ((x) * 1024) - -/* All the cache descriptor types we care about (no TLB or - trace cache entries) */ - -static const struct _cache_table cache_table[] = -{ - { 0x06, CACHE_L1_INST, 8 }, /* 4-way set assoc, 32 byte line size */ - { 0x08, CACHE_L1_INST, 16 }, /* 4-way set assoc, 32 byte line size */ - { 0x09, CACHE_L1_INST, 32 }, /* 4-way set assoc, 64 byte line size */ - { 0x0a, CACHE_L1_DATA, 8 }, /* 2 way set assoc, 32 byte line size */ - { 0x0c, CACHE_L1_DATA, 16 }, /* 4-way set assoc, 32 byte line size */ - { 0x0d, CACHE_L1_DATA, 16 }, /* 4-way set assoc, 64 byte line size */ - { 0x0e, CACHE_L1_DATA, 24 }, /* 6-way set assoc, 64 byte line size */ - { 0x21, CACHE_L2, 256 }, /* 8-way set assoc, 64 byte line size */ - { 0x22, CACHE_L3, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x23, CACHE_L3, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x25, CACHE_L3, MB(2) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x29, CACHE_L3, MB(4) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x2c, CACHE_L1_DATA, 32 }, /* 8-way set assoc, 64 byte line size */ - { 0x30, CACHE_L1_INST, 32 }, /* 8-way set assoc, 64 byte line size */ - { 0x39, CACHE_L2, 128 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3a, CACHE_L2, 192 }, /* 6-way set assoc, sectored cache, 64 byte line size */ - { 0x3b, CACHE_L2, 128 }, /* 2-way set assoc, sectored cache, 64 byte line size */ - { 0x3c, CACHE_L2, 256 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3d, CACHE_L2, 384 }, /* 6-way set assoc, sectored cache, 64 byte line size */ - { 0x3e, CACHE_L2, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x3f, CACHE_L2, 256 }, /* 2-way set assoc, 64 byte line size */ - { 0x41, CACHE_L2, 128 }, /* 4-way set assoc, 32 byte line size */ - { 0x42, CACHE_L2, 256 }, /* 4-way set assoc, 32 byte line size */ - { 0x43, CACHE_L2, 512 }, /* 4-way set assoc, 32 byte line size */ - { 0x44, CACHE_L2, MB(1) }, /* 4-way set assoc, 32 byte line size */ - { 0x45, CACHE_L2, MB(2) }, /* 4-way set assoc, 32 byte line size */ - { 0x46, CACHE_L3, MB(4) }, /* 4-way set assoc, 64 byte line size */ - { 0x47, CACHE_L3, MB(8) }, /* 8-way set assoc, 64 byte line size */ - { 0x48, CACHE_L2, MB(3) }, /* 12-way set assoc, 64 byte line size */ - { 0x49, CACHE_L3, MB(4) }, /* 16-way set assoc, 64 byte line size */ - { 0x4a, CACHE_L3, MB(6) }, /* 12-way set assoc, 64 byte line size */ - { 0x4b, CACHE_L3, MB(8) }, /* 16-way set assoc, 64 byte line size */ - { 0x4c, CACHE_L3, MB(12) }, /* 12-way set assoc, 64 byte line size */ - { 0x4d, CACHE_L3, MB(16) }, /* 16-way set assoc, 64 byte line size */ - { 0x4e, CACHE_L2, MB(6) }, /* 24-way set assoc, 64 byte line size */ - { 0x60, CACHE_L1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x66, CACHE_L1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x67, CACHE_L1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x68, CACHE_L1_DATA, 32 }, /* 4-way set assoc, sectored cache, 64 byte line size */ - { 0x78, CACHE_L2, MB(1) }, /* 4-way set assoc, 64 byte line size */ - { 0x79, CACHE_L2, 128 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7a, CACHE_L2, 256 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7b, CACHE_L2, 512 }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7c, CACHE_L2, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */ - { 0x7d, CACHE_L2, MB(2) }, /* 8-way set assoc, 64 byte line size */ - { 0x7f, CACHE_L2, 512 }, /* 2-way set assoc, 64 byte line size */ - { 0x80, CACHE_L2, 512 }, /* 8-way set assoc, 64 byte line size */ - { 0x82, CACHE_L2, 256 }, /* 8-way set assoc, 32 byte line size */ - { 0x83, CACHE_L2, 512 }, /* 8-way set assoc, 32 byte line size */ - { 0x84, CACHE_L2, MB(1) }, /* 8-way set assoc, 32 byte line size */ - { 0x85, CACHE_L2, MB(2) }, /* 8-way set assoc, 32 byte line size */ - { 0x86, CACHE_L2, 512 }, /* 4-way set assoc, 64 byte line size */ - { 0x87, CACHE_L2, MB(1) }, /* 8-way set assoc, 64 byte line size */ - { 0xd0, CACHE_L3, 512 }, /* 4-way set assoc, 64 byte line size */ - { 0xd1, CACHE_L3, MB(1) }, /* 4-way set assoc, 64 byte line size */ - { 0xd2, CACHE_L3, MB(2) }, /* 4-way set assoc, 64 byte line size */ - { 0xd6, CACHE_L3, MB(1) }, /* 8-way set assoc, 64 byte line size */ - { 0xd7, CACHE_L3, MB(2) }, /* 8-way set assoc, 64 byte line size */ - { 0xd8, CACHE_L3, MB(4) }, /* 12-way set assoc, 64 byte line size */ - { 0xdc, CACHE_L3, MB(2) }, /* 12-way set assoc, 64 byte line size */ - { 0xdd, CACHE_L3, MB(4) }, /* 12-way set assoc, 64 byte line size */ - { 0xde, CACHE_L3, MB(8) }, /* 12-way set assoc, 64 byte line size */ - { 0xe2, CACHE_L3, MB(2) }, /* 16-way set assoc, 64 byte line size */ - { 0xe3, CACHE_L3, MB(4) }, /* 16-way set assoc, 64 byte line size */ - { 0xe4, CACHE_L3, MB(8) }, /* 16-way set assoc, 64 byte line size */ - { 0xea, CACHE_L3, MB(12) }, /* 24-way set assoc, 64 byte line size */ - { 0xeb, CACHE_L3, MB(18) }, /* 24-way set assoc, 64 byte line size */ - { 0xec, CACHE_L3, MB(24) }, /* 24-way set assoc, 64 byte line size */ -}; - - enum _cache_type { CTYPE_NULL = 0, CTYPE_DATA = 1, @@ -439,16 +355,6 @@ void init_hygon_cacheinfo(struct cpuinfo_x86 *c) ci->num_leaves = find_num_cache_leaves(c); } -static const struct _cache_table *cache_table_get(u8 desc) -{ - for (int i = 0; i < ARRAY_SIZE(cache_table); i++) { - if (cache_table[i].descriptor == desc) - return &cache_table[i]; - } - - return NULL; -} - void init_intel_cacheinfo(struct cpuinfo_x86 *c) { /* Cache sizes */ @@ -505,21 +411,17 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) /* Don't use CPUID(2) if CPUID(4) is supported. */ if (!ci->num_leaves && c->cpuid_level > 1) { - const struct _cache_table *entry; + const struct leaf_0x2_table *entry; union leaf_0x2_regs regs; - u8 *desc; + u8 *ptr; cpuid_get_leaf_0x2_regs(®s); - for_each_leaf_0x2_desc(regs, desc) { - entry = cache_table_get(*desc); - if (!entry) - continue; - - switch (entry->type) { - case CACHE_L1_INST: l1i += entry->size; break; - case CACHE_L1_DATA: l1d += entry->size; break; - case CACHE_L2: l2 += entry->size; break; - case CACHE_L3: l3 += entry->size; break; + for_each_leaf_0x2_entry(regs, ptr, entry) { + switch (entry->c_type) { + case CACHE_L1_INST: l1i += entry->c_size; break; + case CACHE_L1_DATA: l1d += entry->c_size; break; + case CACHE_L2: l2 += entry->c_size; break; + case CACHE_L3: l3 += entry->c_size; break; } } } From 4772304ee651b952fd098ec80a8298af9905743f Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:17 +0100 Subject: [PATCH 042/331] x86/cpu: Use consolidated CPUID leaf 0x2 descriptor table CPUID leaf 0x2 output is a stream of one-byte descriptors, each implying certain details about the CPU's cache and TLB entries. At previous commits, the mapping tables for such descriptors were merged into one consolidated table. The mapping was also transformed into a hash lookup instead of a loop-based lookup for each descriptor. Use the new consolidated table and its hash-based lookup through the for_each_leaf_0x2_tlb_entry() accessor. Remove the TLB-specific mapping, intel_tlb_table[], as it is now no longer used. Remove the macro, for_each_leaf_0x2_desc(), since the converted code was its last user. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-23-darwi@linutronix.de --- arch/x86/include/asm/cpuid/types.h | 10 ++++ arch/x86/kernel/cpu/intel.c | 83 +++--------------------------- 2 files changed, 17 insertions(+), 76 deletions(-) diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index 24f643f2a2ad..c95fee66e148 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -115,4 +115,14 @@ struct leaf_0x2_table { extern const struct leaf_0x2_table cpuid_0x2_table[256]; +/* + * All of leaf 0x2's one-byte TLB descriptors implies the same number of entries + * for their respective TLB types. TLB descriptor 0x63 is an exception: it + * implies 4 dTLB entries for 1GB pages and 32 dTLB entries for 2MB or 4MB pages. + * + * Encode that descriptor's dTLB entry count for 2MB/4MB pages here, as the entry + * count for dTLB 1GB pages is already encoded at the cpuid_0x2_table[]'s mapping. + */ +#define TLB_0x63_2M_4M_ENTRIES 32 + #endif /* _ASM_X86_CPUID_TYPES_H */ diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index def433e0081f..e5d814703406 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -626,81 +626,11 @@ static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) } #endif -/* - * All of leaf 0x2's one-byte TLB descriptors implies the same number of - * entries for their respective TLB types. The 0x63 descriptor is an - * exception: it implies 4 dTLB entries for 1GB pages 32 dTLB entries - * for 2MB or 4MB pages. Encode descriptor 0x63 dTLB entry count for - * 2MB/4MB pages here, as its count for dTLB 1GB pages is already at the - * intel_tlb_table[] mapping. - */ -#define TLB_0x63_2M_4M_ENTRIES 32 - -struct _tlb_table { - unsigned char descriptor; - enum _tlb_table_type type; - unsigned int entries; -}; - -static const struct _tlb_table intel_tlb_table[] = { - { 0x01, TLB_INST_4K, 32}, /* TLB_INST 4 KByte pages, 4-way set associative */ - { 0x02, TLB_INST_4M, 2}, /* TLB_INST 4 MByte pages, full associative */ - { 0x03, TLB_DATA_4K, 64}, /* TLB_DATA 4 KByte pages, 4-way set associative */ - { 0x04, TLB_DATA_4M, 8}, /* TLB_DATA 4 MByte pages, 4-way set associative */ - { 0x05, TLB_DATA_4M, 32}, /* TLB_DATA 4 MByte pages, 4-way set associative */ - { 0x0b, TLB_INST_4M, 4}, /* TLB_INST 4 MByte pages, 4-way set associative */ - { 0x4f, TLB_INST_4K, 32}, /* TLB_INST 4 KByte pages */ - { 0x50, TLB_INST_ALL, 64}, /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ - { 0x51, TLB_INST_ALL, 128}, /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ - { 0x52, TLB_INST_ALL, 256}, /* TLB_INST 4 KByte and 2-MByte or 4-MByte pages */ - { 0x55, TLB_INST_2M_4M, 7}, /* TLB_INST 2-MByte or 4-MByte pages, fully associative */ - { 0x56, TLB_DATA0_4M, 16}, /* TLB_DATA0 4 MByte pages, 4-way set associative */ - { 0x57, TLB_DATA0_4K, 16}, /* TLB_DATA0 4 KByte pages, 4-way associative */ - { 0x59, TLB_DATA0_4K, 16}, /* TLB_DATA0 4 KByte pages, fully associative */ - { 0x5a, TLB_DATA0_2M_4M, 32}, /* TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative */ - { 0x5b, TLB_DATA_4K_4M, 64}, /* TLB_DATA 4 KByte and 4 MByte pages */ - { 0x5c, TLB_DATA_4K_4M, 128}, /* TLB_DATA 4 KByte and 4 MByte pages */ - { 0x5d, TLB_DATA_4K_4M, 256}, /* TLB_DATA 4 KByte and 4 MByte pages */ - { 0x61, TLB_INST_4K, 48}, /* TLB_INST 4 KByte pages, full associative */ - { 0x63, TLB_DATA_1G_2M_4M, 4}, /* TLB_DATA 1 GByte pages, 4-way set associative - * (plus 32 entries TLB_DATA 2 MByte or 4 MByte pages, not encoded here) */ - { 0x6b, TLB_DATA_4K, 256}, /* TLB_DATA 4 KByte pages, 8-way associative */ - { 0x6c, TLB_DATA_2M_4M, 128}, /* TLB_DATA 2 MByte or 4 MByte pages, 8-way associative */ - { 0x6d, TLB_DATA_1G, 16}, /* TLB_DATA 1 GByte pages, fully associative */ - { 0x76, TLB_INST_2M_4M, 8}, /* TLB_INST 2-MByte or 4-MByte pages, fully associative */ - { 0xb0, TLB_INST_4K, 128}, /* TLB_INST 4 KByte pages, 4-way set associative */ - { 0xb1, TLB_INST_2M_4M, 4}, /* TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries */ - { 0xb2, TLB_INST_4K, 64}, /* TLB_INST 4KByte pages, 4-way set associative */ - { 0xb3, TLB_DATA_4K, 128}, /* TLB_DATA 4 KByte pages, 4-way set associative */ - { 0xb4, TLB_DATA_4K, 256}, /* TLB_DATA 4 KByte pages, 4-way associative */ - { 0xb5, TLB_INST_4K, 64}, /* TLB_INST 4 KByte pages, 8-way set associative */ - { 0xb6, TLB_INST_4K, 128}, /* TLB_INST 4 KByte pages, 8-way set associative */ - { 0xba, TLB_DATA_4K, 64}, /* TLB_DATA 4 KByte pages, 4-way associative */ - { 0xc0, TLB_DATA_4K_4M, 8}, /* TLB_DATA 4 KByte and 4 MByte pages, 4-way associative */ - { 0xc1, STLB_4K_2M, 1024}, /* STLB 4 KByte and 2 MByte pages, 8-way associative */ - { 0xc2, TLB_DATA_2M_4M, 16}, /* TLB_DATA 2 MByte/4MByte pages, 4-way associative */ - { 0xca, STLB_4K, 512}, /* STLB 4 KByte pages, 4-way associative */ - { 0x00, 0, 0 } -}; - -static void intel_tlb_lookup(const unsigned char desc) +static void intel_tlb_lookup(const struct leaf_0x2_table *entry) { - unsigned int entries; - unsigned char k; + short entries = entry->entries; - if (desc == 0) - return; - - /* look up this descriptor in the table */ - for (k = 0; intel_tlb_table[k].descriptor != desc && - intel_tlb_table[k].descriptor != 0; k++) - ; - - if (intel_tlb_table[k].type == 0) - return; - - entries = intel_tlb_table[k].entries; - switch (intel_tlb_table[k].type) { + switch (entry->t_type) { case STLB_4K: tlb_lli_4k = max(tlb_lli_4k, entries); tlb_lld_4k = max(tlb_lld_4k, entries); @@ -757,15 +687,16 @@ static void intel_tlb_lookup(const unsigned char desc) static void intel_detect_tlb(struct cpuinfo_x86 *c) { + const struct leaf_0x2_table *entry; union leaf_0x2_regs regs; - u8 *desc; + u8 *ptr; if (c->cpuid_level < 2) return; cpuid_get_leaf_0x2_regs(®s); - for_each_leaf_0x2_desc(regs, desc) - intel_tlb_lookup(*desc); + for_each_leaf_0x2_entry(regs, ptr, entry) + intel_tlb_lookup(entry); } static const struct cpu_dev intel_cpu_dev = { From 5adfd367589cf9bb984aabfab74107bcf4402fde Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:18 +0100 Subject: [PATCH 043/331] x86/cacheinfo: Separate CPUID leaf 0x2 handling and post-processing logic The logic of init_intel_cacheinfo() is quite convoluted: it mixes leaf 0x4 parsing, leaf 0x2 parsing, plus some post-processing, in a single place. Begin simplifying its logic by extracting the leaf 0x2 parsing code, and the post-processing logic, into their own functions. While at it, rework the SMT LLC topology ID comment for clarity. Suggested-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-24-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 106 +++++++++++++++++--------------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index e399bf2a701d..b39aad1ecf9c 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -355,14 +355,56 @@ void init_hygon_cacheinfo(struct cpuinfo_x86 *c) ci->num_leaves = find_num_cache_leaves(c); } +static void intel_cacheinfo_done(struct cpuinfo_x86 *c, unsigned int l3, + unsigned int l2, unsigned int l1i, unsigned int l1d) +{ + /* + * If llc_id is still unset, then cpuid_level < 4, which implies + * that the only possibility left is SMT. Since CPUID(2) doesn't + * specify any shared caches and SMT shares all caches, we can + * unconditionally set LLC ID to the package ID so that all + * threads share it. + */ + if (c->topo.llc_id == BAD_APICID) + c->topo.llc_id = c->topo.pkg_id; + + c->x86_cache_size = l3 ? l3 : (l2 ? l2 : l1i + l1d); + + if (!l2) + cpu_detect_cache_sizes(c); +} + +/* + * Legacy Intel CPUID(2) path if CPUID(4) is not available. + */ +static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) +{ + unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; + const struct leaf_0x2_table *entry; + union leaf_0x2_regs regs; + u8 *ptr; + + if (c->cpuid_level < 2) + return; + + cpuid_get_leaf_0x2_regs(®s); + for_each_leaf_0x2_entry(regs, ptr, entry) { + switch (entry->c_type) { + case CACHE_L1_INST: l1i += entry->c_size; break; + case CACHE_L1_DATA: l1d += entry->c_size; break; + case CACHE_L2: l2 += entry->c_size; break; + case CACHE_L3: l3 += entry->c_size; break; + } + } + + intel_cacheinfo_done(c, l3, l2, l1i, l1d); +} + void init_intel_cacheinfo(struct cpuinfo_x86 *c) { - /* Cache sizes */ - unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; - unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */ - unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */ - unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb; struct cpu_cacheinfo *ci = get_cpu_cacheinfo(c->cpu_index); + unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; + unsigned int l2_id = 0, l3_id = 0; if (c->cpuid_level > 3) { /* @@ -376,7 +418,8 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) * Whenever possible use cpuid(4), deterministic cache * parameters cpuid leaf to find the cache details */ - for (i = 0; i < ci->num_leaves; i++) { + for (int i = 0; i < ci->num_leaves; i++) { + unsigned int num_threads_sharing, index_msb; struct _cpuid4_info id4 = {}; int retval; @@ -387,18 +430,18 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) switch (id4.eax.split.level) { case 1: if (id4.eax.split.type == CTYPE_DATA) - new_l1d = id4.size/1024; + l1d = id4.size / 1024; else if (id4.eax.split.type == CTYPE_INST) - new_l1i = id4.size/1024; + l1i = id4.size / 1024; break; case 2: - new_l2 = id4.size/1024; + l2 = id4.size / 1024; num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; index_msb = get_count_order(num_threads_sharing); l2_id = c->topo.apicid & ~((1 << index_msb) - 1); break; case 3: - new_l3 = id4.size/1024; + l3 = id4.size / 1024; num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; index_msb = get_count_order(num_threads_sharing); l3_id = c->topo.apicid & ~((1 << index_msb) - 1); @@ -411,52 +454,19 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) /* Don't use CPUID(2) if CPUID(4) is supported. */ if (!ci->num_leaves && c->cpuid_level > 1) { - const struct leaf_0x2_table *entry; - union leaf_0x2_regs regs; - u8 *ptr; - - cpuid_get_leaf_0x2_regs(®s); - for_each_leaf_0x2_entry(regs, ptr, entry) { - switch (entry->c_type) { - case CACHE_L1_INST: l1i += entry->c_size; break; - case CACHE_L1_DATA: l1d += entry->c_size; break; - case CACHE_L2: l2 += entry->c_size; break; - case CACHE_L3: l3 += entry->c_size; break; - } - } + intel_cacheinfo_0x2(c); + return; } - if (new_l1d) - l1d = new_l1d; - - if (new_l1i) - l1i = new_l1i; - - if (new_l2) { - l2 = new_l2; + if (l2) { c->topo.llc_id = l2_id; c->topo.l2c_id = l2_id; } - if (new_l3) { - l3 = new_l3; + if (l3) c->topo.llc_id = l3_id; - } - /* - * If llc_id is not yet set, this means cpuid_level < 4 which in - * turns means that the only possibility is SMT (as indicated in - * cpuid1). Since cpuid2 doesn't specify shared caches, and we know - * that SMT shares all caches, we can unconditionally set cpu_llc_id to - * c->topo.pkg_id. - */ - if (c->topo.llc_id == BAD_APICID) - c->topo.llc_id = c->topo.pkg_id; - - c->x86_cache_size = l3 ? l3 : (l2 ? l2 : (l1i+l1d)); - - if (!l2) - cpu_detect_cache_sizes(c); + intel_cacheinfo_done(c, l3, l2, l1i, l1d); } static int __cache_amd_cpumap_setup(unsigned int cpu, int index, From 66122616e212f1b9fee7d03582a5fdab2e8ed0e4 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:19 +0100 Subject: [PATCH 044/331] x86/cacheinfo: Separate Intel CPUID leaf 0x4 handling init_intel_cacheinfo() was overly complex. It parsed leaf 0x4 data, leaf 0x2 data, and performed post-processing, all within one function. Parent commit moved leaf 0x2 parsing and the post-processing logic into their own functions. Continue the refactoring by extracting leaf 0x4 parsing into its own function. Initialize local L2/L3 topology ID variables to BAD_APICID by default, thus ensuring they can be used unconditionally. Suggested-by: Thomas Gleixner Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-25-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 112 ++++++++++++++++---------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index b39aad1ecf9c..72cc32d22c4d 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -400,73 +400,71 @@ static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) intel_cacheinfo_done(c, l3, l2, l1i, l1d); } -void init_intel_cacheinfo(struct cpuinfo_x86 *c) +static bool intel_cacheinfo_0x4(struct cpuinfo_x86 *c) { struct cpu_cacheinfo *ci = get_cpu_cacheinfo(c->cpu_index); - unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; - unsigned int l2_id = 0, l3_id = 0; + unsigned int l2_id = BAD_APICID, l3_id = BAD_APICID; + unsigned int l1d = 0, l1i = 0, l2 = 0, l3 = 0; - if (c->cpuid_level > 3) { - /* - * There should be at least one leaf. A non-zero value means - * that the number of leaves has been initialized. - */ - if (!ci->num_leaves) - ci->num_leaves = find_num_cache_leaves(c); + if (c->cpuid_level < 4) + return false; - /* - * Whenever possible use cpuid(4), deterministic cache - * parameters cpuid leaf to find the cache details - */ - for (int i = 0; i < ci->num_leaves; i++) { - unsigned int num_threads_sharing, index_msb; - struct _cpuid4_info id4 = {}; - int retval; + /* + * There should be at least one leaf. A non-zero value means + * that the number of leaves has been previously initialized. + */ + if (!ci->num_leaves) + ci->num_leaves = find_num_cache_leaves(c); - retval = intel_fill_cpuid4_info(i, &id4); - if (retval < 0) - continue; + if (!ci->num_leaves) + return false; - switch (id4.eax.split.level) { - case 1: - if (id4.eax.split.type == CTYPE_DATA) - l1d = id4.size / 1024; - else if (id4.eax.split.type == CTYPE_INST) - l1i = id4.size / 1024; - break; - case 2: - l2 = id4.size / 1024; - num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; - index_msb = get_count_order(num_threads_sharing); - l2_id = c->topo.apicid & ~((1 << index_msb) - 1); - break; - case 3: - l3 = id4.size / 1024; - num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; - index_msb = get_count_order(num_threads_sharing); - l3_id = c->topo.apicid & ~((1 << index_msb) - 1); - break; - default: - break; - } + for (int i = 0; i < ci->num_leaves; i++) { + unsigned int num_threads_sharing, index_msb; + struct _cpuid4_info id4 = {}; + int ret; + + ret = intel_fill_cpuid4_info(i, &id4); + if (ret < 0) + continue; + + switch (id4.eax.split.level) { + case 1: + if (id4.eax.split.type == CTYPE_DATA) + l1d = id4.size / 1024; + else if (id4.eax.split.type == CTYPE_INST) + l1i = id4.size / 1024; + break; + case 2: + l2 = id4.size / 1024; + num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; + index_msb = get_count_order(num_threads_sharing); + l2_id = c->topo.apicid & ~((1 << index_msb) - 1); + break; + case 3: + l3 = id4.size / 1024; + num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; + index_msb = get_count_order(num_threads_sharing); + l3_id = c->topo.apicid & ~((1 << index_msb) - 1); + break; + default: + break; } } - /* Don't use CPUID(2) if CPUID(4) is supported. */ - if (!ci->num_leaves && c->cpuid_level > 1) { - intel_cacheinfo_0x2(c); - return; - } - - if (l2) { - c->topo.llc_id = l2_id; - c->topo.l2c_id = l2_id; - } - - if (l3) - c->topo.llc_id = l3_id; - + c->topo.l2c_id = l2_id; + c->topo.llc_id = (l3_id == BAD_APICID) ? l2_id : l3_id; intel_cacheinfo_done(c, l3, l2, l1i, l1d); + return true; +} + +void init_intel_cacheinfo(struct cpuinfo_x86 *c) +{ + /* Don't use CPUID(2) if CPUID(4) is supported. */ + if (intel_cacheinfo_0x4(c)) + return; + + intel_cacheinfo_0x2(c); } static int __cache_amd_cpumap_setup(unsigned int cpu, int index, From fda5f817ae41035db9e36854600525145688b35e Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:20 +0100 Subject: [PATCH 045/331] x86/cacheinfo: Extract out cache level topology ID calculation For Intel CPUID leaf 0x4 parsing, refactor the cache level topology ID calculation code into its own method instead of repeating the same logic twice for L2 and L3. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-26-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 72cc32d22c4d..7b274da7c725 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -400,6 +400,16 @@ static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) intel_cacheinfo_done(c, l3, l2, l1i, l1d); } +static unsigned int calc_cache_topo_id(struct cpuinfo_x86 *c, const struct _cpuid4_info *id4) +{ + unsigned int num_threads_sharing; + int index_msb; + + num_threads_sharing = 1 + id4->eax.split.num_threads_sharing; + index_msb = get_count_order(num_threads_sharing); + return c->topo.apicid & ~((1 << index_msb) - 1); +} + static bool intel_cacheinfo_0x4(struct cpuinfo_x86 *c) { struct cpu_cacheinfo *ci = get_cpu_cacheinfo(c->cpu_index); @@ -420,7 +430,6 @@ static bool intel_cacheinfo_0x4(struct cpuinfo_x86 *c) return false; for (int i = 0; i < ci->num_leaves; i++) { - unsigned int num_threads_sharing, index_msb; struct _cpuid4_info id4 = {}; int ret; @@ -437,15 +446,11 @@ static bool intel_cacheinfo_0x4(struct cpuinfo_x86 *c) break; case 2: l2 = id4.size / 1024; - num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; - index_msb = get_count_order(num_threads_sharing); - l2_id = c->topo.apicid & ~((1 << index_msb) - 1); + l2_id = calc_cache_topo_id(c, &id4); break; case 3: l3 = id4.size / 1024; - num_threads_sharing = 1 + id4.eax.split.num_threads_sharing; - index_msb = get_count_order(num_threads_sharing); - l3_id = c->topo.apicid & ~((1 << index_msb) - 1); + l3_id = calc_cache_topo_id(c, &id4); break; default: break; From 05d48035e5f69cfe3c125b2bac47e4003b4acccf Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:21 +0100 Subject: [PATCH 046/331] x86/cacheinfo: Extract out cache self-snoop checks The logic of not doing a cache flush if the CPU declares cache self snooping support is repeated across the x86/cacheinfo code. Extract it into its own function. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-27-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 7b274da7c725..231470cdc1da 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -646,6 +646,17 @@ int populate_cache_leaves(unsigned int cpu) static unsigned long saved_cr4; static DEFINE_RAW_SPINLOCK(cache_disable_lock); +/* + * Cache flushing is the most time-consuming step when programming the + * MTRRs. On many Intel CPUs without known erratas, it can be skipped + * if the CPU declares cache self-snooping support. + */ +static void maybe_flush_caches(void) +{ + if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) + wbinvd(); +} + void cache_disable(void) __acquires(cache_disable_lock) { unsigned long cr0; @@ -663,14 +674,7 @@ void cache_disable(void) __acquires(cache_disable_lock) cr0 = read_cr0() | X86_CR0_CD; write_cr0(cr0); - /* - * Cache flushing is the most time-consuming step when programming - * the MTRRs. Fortunately, as per the Intel Software Development - * Manual, we can skip it if the processor supports cache self- - * snooping. - */ - if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) - wbinvd(); + maybe_flush_caches(); /* Save value of CR4 and clear Page Global Enable (bit 7) */ if (cpu_feature_enabled(X86_FEATURE_PGE)) { @@ -685,9 +689,7 @@ void cache_disable(void) __acquires(cache_disable_lock) if (cpu_feature_enabled(X86_FEATURE_MTRR)) mtrr_disable(); - /* Again, only flush caches if we have to. */ - if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) - wbinvd(); + maybe_flush_caches(); } void cache_enable(void) __releases(cache_disable_lock) From eeeebc4fc642f492522e4be92dcaf22119123581 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:22 +0100 Subject: [PATCH 047/331] x86/cacheinfo: Relocate CPUID leaf 0x4 cache_type mapping The cache_type_map[] array is used to map Intel leaf 0x4 cache_type values to their corresponding types at . Move that array's definition after the actual CPUID leaf 0x4 structures, instead of having it in the middle of AMD leaf 0x4 emulation code. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-28-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 231470cdc1da..e0d531e27ff3 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -78,6 +78,14 @@ struct _cpuid4_info { unsigned long size; }; +/* Map CPUID(4) EAX.cache_type to linux/cacheinfo.h types */ +static const enum cache_type cache_type_map[] = { + [CTYPE_NULL] = CACHE_TYPE_NOCACHE, + [CTYPE_DATA] = CACHE_TYPE_DATA, + [CTYPE_INST] = CACHE_TYPE_INST, + [CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED, +}; + /* * Fallback AMD CPUID(4) emulation * AMD CPUs with TOPOEXT can just use CPUID(0x8000001d) @@ -131,13 +139,6 @@ static const unsigned short assocs[] = { static const unsigned char levels[] = { 1, 1, 2, 3 }; static const unsigned char types[] = { 1, 2, 3, 3 }; -static const enum cache_type cache_type_map[] = { - [CTYPE_NULL] = CACHE_TYPE_NOCACHE, - [CTYPE_DATA] = CACHE_TYPE_DATA, - [CTYPE_INST] = CACHE_TYPE_INST, - [CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED, -}; - static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) { From 6c963c42fc19d7c9ae9582ab75c3476d1752d979 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:23 +0100 Subject: [PATCH 048/331] x86/cacheinfo: Introduce cpuid_amd_hygon_has_l3_cache() Multiple code paths at cacheinfo.c and amd_nb.c check for AMD/Hygon CPUs L3 cache presensce by directly checking leaf 0x80000006 EDX output. Extract that logic into its own function. While at it, rework the AMD/Hygon LLC topology ID caclculation comments for clarity. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-29-darwi@linutronix.de --- arch/x86/include/asm/cpuid/api.h | 9 +++++++++ arch/x86/kernel/amd_nb.c | 7 +++---- arch/x86/kernel/cpu/cacheinfo.c | 32 ++++++++++++++------------------ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index 9c180c9cc58e..bf75c6267311 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -207,4 +207,13 @@ static inline u32 hypervisor_cpuid_base(const char *sig, u32 leaves) return 0; } +/* + * CPUID(0x80000006) parsing helpers + */ + +static inline bool cpuid_amd_hygon_has_l3_cache(void) +{ + return cpuid_edx(0x80000006); +} + #endif /* _ASM_X86_CPUID_API_H */ diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 6d12a9b69432..5a8cc48f80be 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -13,7 +13,9 @@ #include #include #include + #include +#include static u32 *flush_words; @@ -91,10 +93,7 @@ static int amd_cache_northbridges(void) if (amd_gart_present()) amd_northbridges.flags |= AMD_NB_GART; - /* - * Check for L3 cache presence. - */ - if (!cpuid_edx(0x80000006)) + if (!cpuid_amd_hygon_has_l3_cache()) return 0; /* diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index e0d531e27ff3..71587570705f 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -281,29 +281,29 @@ static int find_num_cache_leaves(struct cpuinfo_x86 *c) return i; } +/* + * AMD/Hygon CPUs may have multiple LLCs if L3 caches exist. + */ + void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, u16 die_id) { - /* - * We may have multiple LLCs if L3 caches exist, so check if we - * have an L3 cache by looking at the L3 cache CPUID leaf. - */ - if (!cpuid_edx(0x80000006)) + if (!cpuid_amd_hygon_has_l3_cache()) return; if (c->x86 < 0x17) { - /* LLC is at the node level. */ + /* Pre-Zen: LLC is at the node level */ c->topo.llc_id = die_id; } else if (c->x86 == 0x17 && c->x86_model <= 0x1F) { /* - * LLC is at the core complex level. - * Core complex ID is ApicId[3] for these processors. + * Family 17h up to 1F models: LLC is at the core + * complex level. Core complex ID is ApicId[3]. */ c->topo.llc_id = c->topo.apicid >> 3; } else { /* - * LLC ID is calculated from the number of threads sharing the - * cache. - * */ + * Newer families: LLC ID is calculated from the number + * of threads sharing the L3 cache. + */ u32 eax, ebx, ecx, edx, num_sharing_cache = 0; u32 llc_index = find_num_cache_leaves(c) - 1; @@ -321,16 +321,12 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, u16 die_id) void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c) { - /* - * We may have multiple LLCs if L3 caches exist, so check if we - * have an L3 cache by looking at the L3 cache CPUID leaf. - */ - if (!cpuid_edx(0x80000006)) + if (!cpuid_amd_hygon_has_l3_cache()) return; /* - * LLC is at the core complex level. - * Core complex ID is ApicId[3] for these processors. + * Hygons are similar to AMD Family 17h up to 1F models: LLC is + * at the core complex level. Core complex ID is ApicId[3]. */ c->topo.llc_id = c->topo.apicid >> 3; } From 0dd09e215a391805130338688586805fe8bf2b32 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Mon, 24 Mar 2025 14:33:24 +0100 Subject: [PATCH 049/331] x86/cacheinfo: Apply maintainer-tip coding style fixes The x86/cacheinfo code has been heavily refactored and fleshed out at parent commits, where any necessary coding style fixes were also done in place. Apply Documentation/process/maintainer-tip.rst coding style fixes to the rest of the code, and align its assignment expressions for readability. Standardize on CPUID(n) when mentioning leaf queries. Avoid breaking long lines when doing so helps readability. At cacheinfo_amd_init_llc_id(), rename variable 'msb' to 'index_msb' as this is how it's called at the rest of cacheinfo.c code. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250324133324.23458-30-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 215 ++++++++++++++++---------------- 1 file changed, 107 insertions(+), 108 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 71587570705f..cd48d34ac04b 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Routines to identify caches on Intel CPU. + * x86 CPU caches detection and configuration * - * Changes: - * Venkatesh Pallipadi : Adding cache identification through cpuid(4) - * Ashok Raj : Work with CPU hotplug infrastructure. - * Andi Kleen / Andreas Herrmann : CPUID4 emulation on AMD. + * Previous changes + * - Venkatesh Pallipadi: Cache identification through CPUID(4) + * - Ashok Raj : Work with CPU hotplug infrastructure + * - Andi Kleen / Andreas Herrmann: CPUID(4) emulation on AMD */ #include @@ -35,37 +35,37 @@ static cpumask_var_t cpu_cacheinfo_mask; unsigned int memory_caching_control __ro_after_init; enum _cache_type { - CTYPE_NULL = 0, - CTYPE_DATA = 1, - CTYPE_INST = 2, - CTYPE_UNIFIED = 3 + CTYPE_NULL = 0, + CTYPE_DATA = 1, + CTYPE_INST = 2, + CTYPE_UNIFIED = 3 }; union _cpuid4_leaf_eax { struct { - enum _cache_type type:5; - unsigned int level:3; - unsigned int is_self_initializing:1; - unsigned int is_fully_associative:1; - unsigned int reserved:4; - unsigned int num_threads_sharing:12; - unsigned int num_cores_on_die:6; + enum _cache_type type :5; + unsigned int level :3; + unsigned int is_self_initializing :1; + unsigned int is_fully_associative :1; + unsigned int reserved :4; + unsigned int num_threads_sharing :12; + unsigned int num_cores_on_die :6; } split; u32 full; }; union _cpuid4_leaf_ebx { struct { - unsigned int coherency_line_size:12; - unsigned int physical_line_partition:10; - unsigned int ways_of_associativity:10; + unsigned int coherency_line_size :12; + unsigned int physical_line_partition :10; + unsigned int ways_of_associativity :10; } split; u32 full; }; union _cpuid4_leaf_ecx { struct { - unsigned int number_of_sets:32; + unsigned int number_of_sets :32; } split; u32 full; }; @@ -93,60 +93,59 @@ static const enum cache_type cache_type_map[] = { union l1_cache { struct { - unsigned line_size:8; - unsigned lines_per_tag:8; - unsigned assoc:8; - unsigned size_in_kb:8; + unsigned line_size :8; + unsigned lines_per_tag :8; + unsigned assoc :8; + unsigned size_in_kb :8; }; - unsigned val; + unsigned int val; }; union l2_cache { struct { - unsigned line_size:8; - unsigned lines_per_tag:4; - unsigned assoc:4; - unsigned size_in_kb:16; + unsigned line_size :8; + unsigned lines_per_tag :4; + unsigned assoc :4; + unsigned size_in_kb :16; }; - unsigned val; + unsigned int val; }; union l3_cache { struct { - unsigned line_size:8; - unsigned lines_per_tag:4; - unsigned assoc:4; - unsigned res:2; - unsigned size_encoded:14; + unsigned line_size :8; + unsigned lines_per_tag :4; + unsigned assoc :4; + unsigned res :2; + unsigned size_encoded :14; }; - unsigned val; + unsigned int val; }; static const unsigned short assocs[] = { - [1] = 1, - [2] = 2, - [4] = 4, - [6] = 8, - [8] = 16, - [0xa] = 32, - [0xb] = 48, - [0xc] = 64, - [0xd] = 96, - [0xe] = 128, - [0xf] = 0xffff /* fully associative - no way to show this currently */ + [1] = 1, + [2] = 2, + [4] = 4, + [6] = 8, + [8] = 16, + [0xa] = 32, + [0xb] = 48, + [0xc] = 64, + [0xd] = 96, + [0xe] = 128, + [0xf] = 0xffff /* Fully associative */ }; static const unsigned char levels[] = { 1, 1, 2, 3 }; -static const unsigned char types[] = { 1, 2, 3, 3 }; +static const unsigned char types[] = { 1, 2, 3, 3 }; static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) { unsigned int dummy, line_size, lines_per_tag, assoc, size_in_kb; - union l1_cache l1i, l1d; + union l1_cache l1i, l1d, *l1; union l2_cache l2; union l3_cache l3; - union l1_cache *l1 = &l1d; eax->full = 0; ebx->full = 0; @@ -155,6 +154,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val); cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val); + l1 = &l1d; switch (index) { case 1: l1 = &l1i; @@ -162,48 +162,52 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, case 0: if (!l1->val) return; - assoc = assocs[l1->assoc]; - line_size = l1->line_size; - lines_per_tag = l1->lines_per_tag; - size_in_kb = l1->size_in_kb; + + assoc = assocs[l1->assoc]; + line_size = l1->line_size; + lines_per_tag = l1->lines_per_tag; + size_in_kb = l1->size_in_kb; break; case 2: if (!l2.val) return; - assoc = assocs[l2.assoc]; - line_size = l2.line_size; - lines_per_tag = l2.lines_per_tag; - /* cpu_data has errata corrections for K7 applied */ - size_in_kb = __this_cpu_read(cpu_info.x86_cache_size); + + /* Use x86_cache_size as it might have K7 errata fixes */ + assoc = assocs[l2.assoc]; + line_size = l2.line_size; + lines_per_tag = l2.lines_per_tag; + size_in_kb = __this_cpu_read(cpu_info.x86_cache_size); break; case 3: if (!l3.val) return; - assoc = assocs[l3.assoc]; - line_size = l3.line_size; - lines_per_tag = l3.lines_per_tag; - size_in_kb = l3.size_encoded * 512; + + assoc = assocs[l3.assoc]; + line_size = l3.line_size; + lines_per_tag = l3.lines_per_tag; + size_in_kb = l3.size_encoded * 512; if (boot_cpu_has(X86_FEATURE_AMD_DCM)) { - size_in_kb = size_in_kb >> 1; - assoc = assoc >> 1; + size_in_kb = size_in_kb >> 1; + assoc = assoc >> 1; } break; default: return; } - eax->split.is_self_initializing = 1; - eax->split.type = types[index]; - eax->split.level = levels[index]; - eax->split.num_threads_sharing = 0; - eax->split.num_cores_on_die = topology_num_cores_per_package(); + eax->split.is_self_initializing = 1; + eax->split.type = types[index]; + eax->split.level = levels[index]; + eax->split.num_threads_sharing = 0; + eax->split.num_cores_on_die = topology_num_cores_per_package(); if (assoc == 0xffff) eax->split.is_fully_associative = 1; - ebx->split.coherency_line_size = line_size - 1; - ebx->split.ways_of_associativity = assoc - 1; - ebx->split.physical_line_partition = lines_per_tag - 1; - ecx->split.number_of_sets = (size_in_kb * 1024) / line_size / + + ebx->split.coherency_line_size = line_size - 1; + ebx->split.ways_of_associativity = assoc - 1; + ebx->split.physical_line_partition = lines_per_tag - 1; + ecx->split.number_of_sets = (size_in_kb * 1024) / line_size / (ebx->split.ways_of_associativity + 1) - 1; } @@ -262,19 +266,14 @@ static int fill_cpuid4_info(int index, struct _cpuid4_info *id4) static int find_num_cache_leaves(struct cpuinfo_x86 *c) { - unsigned int eax, ebx, ecx, edx, op; - union _cpuid4_leaf_eax cache_eax; - int i = -1; - - if (c->x86_vendor == X86_VENDOR_AMD || - c->x86_vendor == X86_VENDOR_HYGON) - op = 0x8000001d; - else - op = 4; + unsigned int eax, ebx, ecx, edx, op; + union _cpuid4_leaf_eax cache_eax; + int i = -1; + /* Do a CPUID(op) loop to calculate num_cache_leaves */ + op = (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) ? 0x8000001d : 4; do { ++i; - /* Do cpuid(op) loop to find out num_cache_leaves */ cpuid_count(op, i, &eax, &ebx, &ecx, &edx); cache_eax.full = eax; } while (cache_eax.split.type != CTYPE_NULL); @@ -312,9 +311,9 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, u16 die_id) num_sharing_cache = ((eax >> 14) & 0xfff) + 1; if (num_sharing_cache) { - int bits = get_count_order(num_sharing_cache); + int index_msb = get_count_order(num_sharing_cache); - c->topo.llc_id = c->topo.apicid >> bits; + c->topo.llc_id = c->topo.apicid >> index_msb; } } } @@ -335,14 +334,10 @@ void init_amd_cacheinfo(struct cpuinfo_x86 *c) { struct cpu_cacheinfo *ci = get_cpu_cacheinfo(c->cpu_index); - if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { + if (boot_cpu_has(X86_FEATURE_TOPOEXT)) ci->num_leaves = find_num_cache_leaves(c); - } else if (c->extended_cpuid_level >= 0x80000006) { - if (cpuid_edx(0x80000006) & 0xf000) - ci->num_leaves = 4; - else - ci->num_leaves = 3; - } + else if (c->extended_cpuid_level >= 0x80000006) + ci->num_leaves = (cpuid_edx(0x80000006) & 0xf000) ? 4 : 3; } void init_hygon_cacheinfo(struct cpuinfo_x86 *c) @@ -469,6 +464,9 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) intel_cacheinfo_0x2(c); } +/* + * linux/cacheinfo.h shared_cpu_map setup, AMD/Hygon + */ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, const struct _cpuid4_info *id4) { @@ -485,12 +483,12 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, this_cpu_ci = get_cpu_cacheinfo(i); if (!this_cpu_ci->info_list) continue; + ci = this_cpu_ci->info_list + index; for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) { if (!cpu_online(sibling)) continue; - cpumask_set_cpu(sibling, - &ci->shared_cpu_map); + cpumask_set_cpu(sibling, &ci->shared_cpu_map); } } } else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { @@ -516,8 +514,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, apicid = cpu_data(sibling).topo.apicid; if ((apicid < first) || (apicid > last)) continue; - cpumask_set_cpu(sibling, - &ci->shared_cpu_map); + cpumask_set_cpu(sibling, &ci->shared_cpu_map); } } } else @@ -526,17 +523,19 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, return 1; } +/* + * linux/cacheinfo.h shared_cpu_map setup, Intel + fallback AMD/Hygon + */ static void __cache_cpumap_setup(unsigned int cpu, int index, const struct _cpuid4_info *id4) { struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cpuinfo_x86 *c = &cpu_data(cpu); struct cacheinfo *ci, *sibling_ci; unsigned long num_threads_sharing; int index_msb, i; - struct cpuinfo_x86 *c = &cpu_data(cpu); - if (c->x86_vendor == X86_VENDOR_AMD || - c->x86_vendor == X86_VENDOR_HYGON) { + if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { if (__cache_amd_cpumap_setup(cpu, index, id4)) return; } @@ -554,8 +553,10 @@ static void __cache_cpumap_setup(unsigned int cpu, int index, if (cpu_data(i).topo.apicid >> index_msb == c->topo.apicid >> index_msb) { struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i); + /* Skip if itself or no cacheinfo */ if (i == cpu || !sib_cpu_ci->info_list) - continue;/* skip if itself or no cacheinfo */ + continue; + sibling_ci = sib_cpu_ci->info_list + index; cpumask_set_cpu(i, &ci->shared_cpu_map); cpumask_set_cpu(cpu, &sibling_ci->shared_cpu_map); @@ -589,7 +590,7 @@ int init_cache_level(unsigned int cpu) } /* - * The max shared threads number comes from CPUID.4:EAX[25-14] with input + * The max shared threads number comes from CPUID(4) EAX[25-14] with input * ECX as cache index. Then right shift apicid by the number's order to get * cache id for this cache node. */ @@ -626,8 +627,8 @@ int populate_cache_leaves(unsigned int cpu) ci_info_init(ci++, &id4, nb); __cache_cpumap_setup(cpu, idx, &id4); } - this_cpu_ci->cpu_map_populated = true; + this_cpu_ci->cpu_map_populated = true; return 0; } @@ -659,12 +660,10 @@ void cache_disable(void) __acquires(cache_disable_lock) unsigned long cr0; /* - * Note that this is not ideal - * since the cache is only flushed/disabled for this CPU while the - * MTRRs are changed, but changing this requires more invasive - * changes to the way the kernel boots + * This is not ideal since the cache is only flushed/disabled + * for this CPU while the MTRRs are changed, but changing this + * requires more invasive changes to the way the kernel boots. */ - raw_spin_lock(&cache_disable_lock); /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */ From 8df505af7fefe34573fdd1eacdd2b3381349c45c Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 14 Mar 2025 17:27:33 +0000 Subject: [PATCH 050/331] x86/kexec: Debugging support: Load an IDT and basic exception entry points [ mingo: Minor readability edits ] Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Link: https://lore.kernel.org/r/20250314173226.3062535-2-dwmw2@infradead.org --- arch/x86/include/asm/kexec.h | 5 ++ arch/x86/kernel/machine_kexec_64.c | 21 ++++++++ arch/x86/kernel/relocate_kernel_64.S | 77 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 5432457d2338..fb4537c58b54 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -8,6 +8,9 @@ # define PA_PGD 2 # define PA_SWAP_PAGE 3 # define PAGES_NR 4 +#else +/* Size of each exception handler referenced by the IDT */ +# define KEXEC_DEBUG_EXC_HANDLER_SIZE 6 /* PUSHI, PUSHI, 2-byte JMP */ #endif # define KEXEC_CONTROL_PAGE_SIZE 4096 @@ -59,6 +62,8 @@ struct kimage; extern unsigned long kexec_va_control_page; extern unsigned long kexec_pa_table_page; extern unsigned long kexec_pa_swap_page; +extern gate_desc kexec_debug_idt[]; +extern unsigned char kexec_debug_exc_vectors[]; #endif /* diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index a68f5a0a9f37..cc73f9708464 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -304,6 +304,24 @@ static void load_segments(void) ); } +static void prepare_debug_idt(unsigned long control_page, unsigned long vec_ofs) +{ + gate_desc idtentry = { 0 }; + int i; + + idtentry.bits.p = 1; + idtentry.bits.type = GATE_TRAP; + idtentry.segment = __KERNEL_CS; + idtentry.offset_low = (control_page & 0xFFFF) + vec_ofs; + idtentry.offset_middle = (control_page >> 16) & 0xFFFF; + idtentry.offset_high = control_page >> 32; + + for (i = 0; i < 16; i++) { + kexec_debug_idt[i] = idtentry; + idtentry.offset_low += KEXEC_DEBUG_EXC_HANDLER_SIZE; + } +} + int machine_kexec_prepare(struct kimage *image) { void *control_page = page_address(image->control_code_page); @@ -321,6 +339,9 @@ int machine_kexec_prepare(struct kimage *image) if (image->type == KEXEC_TYPE_DEFAULT) kexec_pa_swap_page = page_to_pfn(image->swap_page) << PAGE_SHIFT; + prepare_debug_idt((unsigned long)__pa(control_page), + (unsigned long)kexec_debug_exc_vectors - reloc_start); + __memcpy(control_page, __relocate_kernel_start, reloc_end - reloc_start); set_memory_rox((unsigned long)control_page, 1); diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index ac058971a382..8f26ffd10849 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -50,6 +50,11 @@ SYM_DATA_START_LOCAL(kexec_debug_gdt) .quad 0x00cf92000000ffff /* __KERNEL_DS */ SYM_DATA_END_LABEL(kexec_debug_gdt, SYM_L_LOCAL, kexec_debug_gdt_end) + .balign 8 +SYM_DATA_START(kexec_debug_idt) + .skip 0x100, 0x00 +SYM_DATA_END(kexec_debug_idt) + .section .text..relocate_kernel,"ax"; .code64 SYM_CODE_START_NOALIGN(relocate_kernel) @@ -139,6 +144,15 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) movq %ds, %rax movq %rax, %ds + /* Now an IDTR on the stack to load the IDT the kernel created */ + leaq kexec_debug_idt(%rip), %rsi + pushq %rsi + pushw $0xff + lidt (%rsp) + addq $10, %rsp + + //int3 + /* * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP * below. @@ -364,3 +378,66 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) ret int3 SYM_CODE_END(swap_pages) + +SYM_CODE_START_NOALIGN(kexec_debug_exc_vectors) + /* Each of these is 6 bytes. */ +.macro vec_err exc + UNWIND_HINT_ENTRY + . = kexec_debug_exc_vectors + (\exc * KEXEC_DEBUG_EXC_HANDLER_SIZE) + nop + nop + pushq $\exc + jmp exc_handler +.endm + +.macro vec_noerr exc + UNWIND_HINT_ENTRY + . = kexec_debug_exc_vectors + (\exc * KEXEC_DEBUG_EXC_HANDLER_SIZE) + pushq $0 + pushq $\exc + jmp exc_handler +.endm + + ANNOTATE_NOENDBR + vec_noerr 0 // #DE + vec_noerr 1 // #DB + vec_noerr 2 // #NMI + vec_noerr 3 // #BP + vec_noerr 4 // #OF + vec_noerr 5 // #BR + vec_noerr 6 // #UD + vec_noerr 7 // #NM + vec_err 8 // #DF + vec_noerr 9 + vec_err 10 // #TS + vec_err 11 // #NP + vec_err 12 // #SS + vec_err 13 // #GP + vec_err 14 // #PF + vec_noerr 15 +SYM_CODE_END(kexec_debug_exc_vectors) + +SYM_CODE_START_LOCAL_NOALIGN(exc_handler) + /* No need for RET mitigations during kexec */ + VALIDATE_UNRET_END + + pushq %rax + pushq %rdx + movw $0x3f8, %dx + movb $'A', %al + outb %al, %dx + popq %rdx + popq %rax + + /* Only return from INT3 */ + cmpq $3, (%rsp) + jne .Ldie + + addq $16, %rsp + iretq + +.Ldie: + hlt + jmp .Ldie + +SYM_CODE_END(exc_handler) From 3d66af75b08609281cbd7b71d40bbb9829c88764 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 14 Mar 2025 17:27:34 +0000 Subject: [PATCH 051/331] x86/kexec: Debugging support: Dump registers on exception The actual serial output function is a no-op for now. Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Link: https://lore.kernel.org/r/20250314173226.3062535-3-dwmw2@infradead.org --- arch/x86/kernel/relocate_kernel_64.S | 121 +++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 8f26ffd10849..29cb399a34d4 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -379,6 +379,69 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) int3 SYM_CODE_END(swap_pages) +/* + * Generic 'print character' routine (as yet unimplemented) + * - %al: Character to be printed (may clobber %rax) + * - %rdx: MMIO address or port. + */ +SYM_CODE_START_LOCAL_NOALIGN(pr_char) + UNWIND_HINT_FUNC + ANNOTATE_NOENDBR + ANNOTATE_UNRET_SAFE + ret +SYM_CODE_END(pr_char) + +/* + * Load pr_char function pointer into %rsi and load %rdx with whatever + * that function wants to see there (typically port/MMIO address). + */ +.macro pr_setup + /* No output; pr_char just returns */ + leaq pr_char(%rip), %rsi +.endm + +/* Print the nybble in %bl, clobber %rax */ +SYM_CODE_START_LOCAL_NOALIGN(pr_nybble) + UNWIND_HINT_FUNC + movb %bl, %al + nop + andb $0x0f, %al + addb $0x30, %al + cmpb $0x3a, %al + jb 1f + addb $('a' - '0' - 10), %al + ANNOTATE_RETPOLINE_SAFE +1: jmp *%rsi +SYM_CODE_END(pr_nybble) + +SYM_CODE_START_LOCAL_NOALIGN(pr_qword) + UNWIND_HINT_FUNC + movq $16, %rcx +1: rolq $4, %rbx + call pr_nybble + loop 1b + movb $'\n', %al + ANNOTATE_RETPOLINE_SAFE + jmp *%rsi +SYM_CODE_END(pr_qword) + +.macro print_reg a, b, c, d, r + movb $\a, %al + ANNOTATE_RETPOLINE_SAFE + call *%rsi + movb $\b, %al + ANNOTATE_RETPOLINE_SAFE + call *%rsi + movb $\c, %al + ANNOTATE_RETPOLINE_SAFE + call *%rsi + movb $\d, %al + ANNOTATE_RETPOLINE_SAFE + call *%rsi + movq \r, %rbx + call pr_qword +.endm + SYM_CODE_START_NOALIGN(kexec_debug_exc_vectors) /* Each of these is 6 bytes. */ .macro vec_err exc @@ -422,17 +485,63 @@ SYM_CODE_START_LOCAL_NOALIGN(exc_handler) VALIDATE_UNRET_END pushq %rax + pushq %rbx + pushq %rcx pushq %rdx - movw $0x3f8, %dx - movb $'A', %al - outb %al, %dx - popq %rdx - popq %rax + pushq %rsi + + /* Stack frame */ +#define EXC_SS 0x58 /* Architectural... */ +#define EXC_RSP 0x50 +#define EXC_EFLAGS 0x48 +#define EXC_CS 0x40 +#define EXC_RIP 0x38 +#define EXC_ERRORCODE 0x30 /* Either architectural or zero pushed by handler */ +#define EXC_EXCEPTION 0x28 /* Pushed by handler entry point */ +#define EXC_RAX 0x20 /* Pushed just above in exc_handler */ +#define EXC_RBX 0x18 +#define EXC_RCX 0x10 +#define EXC_RDX 0x08 +#define EXC_RSI 0x00 + + /* Set up %rdx/%rsi for debug output */ + pr_setup + + /* rip and exception info */ + print_reg 'E', 'x', 'c', ':', EXC_EXCEPTION(%rsp) + print_reg 'E', 'r', 'r', ':', EXC_ERRORCODE(%rsp) + print_reg 'r', 'i', 'p', ':', EXC_RIP(%rsp) + print_reg 'r', 's', 'p', ':', EXC_RSP(%rsp) + + /* We spilled these to the stack */ + print_reg 'r', 'a', 'x', ':', EXC_RAX(%rsp) + print_reg 'r', 'b', 'x', ':', EXC_RBX(%rsp) + print_reg 'r', 'c', 'x', ':', EXC_RCX(%rsp) + print_reg 'r', 'd', 'x', ':', EXC_RDX(%rsp) + print_reg 'r', 's', 'i', ':', EXC_RSI(%rsp) + + /* Other registers untouched */ + print_reg 'r', 'd', 'i', ':', %rdi + print_reg 'r', '8', ' ', ':', %r8 + print_reg 'r', '9', ' ', ':', %r9 + print_reg 'r', '1', '0', ':', %r10 + print_reg 'r', '1', '1', ':', %r11 + print_reg 'r', '1', '2', ':', %r12 + print_reg 'r', '1', '3', ':', %r13 + print_reg 'r', '1', '4', ':', %r14 + print_reg 'r', '1', '5', ':', %r15 + print_reg 'c', 'r', '2', ':', %cr2 /* Only return from INT3 */ - cmpq $3, (%rsp) + cmpq $3, EXC_EXCEPTION(%rsp) jne .Ldie + popq %rsi + popq %rdx + popq %rcx + popq %rbx + popq %rax + addq $16, %rsp iretq From 0717b1392dc7e3f350e5a5d25ea794aa92210684 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 25 Mar 2025 18:52:01 +0100 Subject: [PATCH 052/331] x86/bitops: Use TZCNT mnemonic in Current minimum required version of binutils is 2.25, which supports TZCNT instruction mnemonic. Replace "REP; BSF" in variable__{ffs,ffz}() function with this proper mnemonic. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250325175215.330659-1-ubizjak@gmail.com --- arch/x86/include/asm/bitops.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 100413aff640..bbaf75ea6703 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -248,7 +248,7 @@ arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) static __always_inline unsigned long variable__ffs(unsigned long word) { - asm("rep; bsf %1,%0" + asm("tzcnt %1,%0" : "=r" (word) : ASM_INPUT_RM (word)); return word; @@ -267,7 +267,7 @@ static __always_inline unsigned long variable__ffs(unsigned long word) static __always_inline unsigned long variable_ffz(unsigned long word) { - asm("rep; bsf %1,%0" + asm("tzcnt %1,%0" : "=r" (word) : "r" (~word)); return word; From e29c5d0e5dc35ed8b8920b573925d8aa2f8bafb0 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 27 Mar 2025 10:56:28 +0100 Subject: [PATCH 053/331] x86/bitops: Simplify variable_ffz() as variable__ffs(~word) Find first zero (FFZ) can be implemented by negating the input and using find first set (FFS). Before/after code generation comparison on ffz()-using kernel code shows that code generation has not changed: # kernel/signal.o: text data bss dec hex filename 42121 3472 8 45601 b221 signal.o.before 42121 3472 8 45601 b221 signal.o.after md5: ce4c31e1bce96af19b62a5f9659842f1 signal.o.before.asm ce4c31e1bce96af19b62a5f9659842f1 signal.o.after.asm [ mingo: Added code generation check. ] Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250327095641.131483-1-ubizjak@gmail.com --- arch/x86/include/asm/bitops.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index bbaf75ea6703..eebbc8889e70 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -267,10 +267,7 @@ static __always_inline unsigned long variable__ffs(unsigned long word) static __always_inline unsigned long variable_ffz(unsigned long word) { - asm("tzcnt %1,%0" - : "=r" (word) - : "r" (~word)); - return word; + return variable__ffs(~word); } /** From 2e016da1cbbd013368095270c040e065678c38f7 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:21 +0000 Subject: [PATCH 054/331] x86/nmi: Simplify unknown NMI panic handling The unknown_nmi_panic variable is used to control whether the kernel should panic on unknown NMIs. There is a sysctl entry under: /proc/sys/kernel/unknown_nmi_panic which can be used to change the behavior at runtime. However, it seems that in some places, the option unnecessarily depends on CONFIG_X86_LOCAL_APIC. Other code in nmi.c uses unknown_nmi_panic without such a dependency. This results in a few messy #ifdefs splattered across the code. The dependency was likely introduce due to a potential build bug reported a long time ago: https://lore.kernel.org/lkml/40BC67F9.3000609@myrealbox.com/ This build bug no longer exists. Also, similar NMI panic options, such as panic_on_unrecovered_nmi and panic_on_io_nmi, do not have an explicit dependency on the local APIC either. Though, it's hard to imagine a production system without the local APIC configuration, making a specific NMI sysctl option dependent on it doesn't make sense. Remove the explicit dependency between unknown NMI handling and the local APIC to make the code cleaner and more consistent. While at it, reorder the header includes to maintain alphabetical order. [ mingo: Cleaned up the changelog a bit, truly ordered the headers ... ] Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-2-sohil.mehta@intel.com --- arch/x86/include/asm/nmi.h | 4 ++-- arch/x86/kernel/setup.c | 35 +++++++++++++++-------------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h index f677382093f3..9cf96cce02fc 100644 --- a/arch/x86/include/asm/nmi.h +++ b/arch/x86/include/asm/nmi.h @@ -14,10 +14,10 @@ extern void release_perfctr_nmi(unsigned int); extern int reserve_evntsel_nmi(unsigned int); extern void release_evntsel_nmi(unsigned int); -extern int unknown_nmi_panic; - #endif /* CONFIG_X86_LOCAL_APIC */ +extern int unknown_nmi_panic; + #define NMI_FLAG_FIRST 1 enum { diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index c7164a8de983..ecf4c133f060 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,21 +19,19 @@ #include #include #include +#include #include -#include -#include -#include #include #include -#include +#include +#include +#include #include #include #include -#include -#include #include #include #include @@ -47,18 +46,16 @@ #include #include #include -#include +#include +#include #include #include #include #include +#include #include #include #include -#include -#if defined(CONFIG_X86_LOCAL_APIC) -#include -#endif /* * max_low_pfn_mapped: highest directly mapped pfn < 4 GB @@ -150,6 +147,13 @@ static size_t ima_kexec_buffer_size; int bootloader_type, bootloader_version; static const struct ctl_table x86_sysctl_table[] = { + { + .procname = "unknown_nmi_panic", + .data = &unknown_nmi_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "panic_on_unrecovered_nmi", .data = &panic_on_unrecovered_nmi, @@ -185,15 +189,6 @@ static const struct ctl_table x86_sysctl_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, -#if defined(CONFIG_X86_LOCAL_APIC) - { - .procname = "unknown_nmi_panic", - .data = &unknown_nmi_panic, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif #if defined(CONFIG_ACPI_SLEEP) { .procname = "acpi_video_flags", From 78a0323506f01e8017a5826cd7e91951c13184fa Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:22 +0000 Subject: [PATCH 055/331] x86/nmi: Consolidate NMI panic variables Commit: c305a4e98378 ("x86: Move sysctls into arch/x86") recently moved the sysctl handling of panic_on_unrecovered_nmi and panic_on_io_nmi to x86-specific code. These variables no longer need to be declared in the generic header file. Relocate the variable definitions and declarations closer to where they are used. This makes all the NMI panic options consistent and easier to track. [ mingo: Fixed up the SHA1 of the commit reference. ] Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Reviewed-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Cc: Joel Granados Link: https://lore.kernel.org/r/20250327234629.3953536-3-sohil.mehta@intel.com --- arch/x86/include/asm/nmi.h | 2 ++ arch/x86/kernel/dumpstack.c | 2 -- arch/x86/kernel/nmi.c | 3 +++ include/linux/panic.h | 2 -- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h index 9cf96cce02fc..f85aea7bf7f1 100644 --- a/arch/x86/include/asm/nmi.h +++ b/arch/x86/include/asm/nmi.h @@ -17,6 +17,8 @@ extern void release_evntsel_nmi(unsigned int); #endif /* CONFIG_X86_LOCAL_APIC */ extern int unknown_nmi_panic; +extern int panic_on_unrecovered_nmi; +extern int panic_on_io_nmi; #define NMI_FLAG_FIRST 1 diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index c6fefd4585f8..71ee20102a8a 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -23,8 +23,6 @@ #include #include -int panic_on_unrecovered_nmi; -int panic_on_io_nmi; static int die_counter; static struct pt_regs exec_summary_regs; diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 9a95d00f1423..671d846ed620 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -91,6 +91,9 @@ static DEFINE_PER_CPU(struct nmi_stats, nmi_stats); static int ignore_nmis __read_mostly; int unknown_nmi_panic; +int panic_on_unrecovered_nmi; +int panic_on_io_nmi; + /* * Prevent NMI reason port (0x61) being accessed simultaneously, can * only be used in NMI handler. diff --git a/include/linux/panic.h b/include/linux/panic.h index 2494d51707ef..4adc65766935 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -20,8 +20,6 @@ extern bool panic_triggering_all_cpu_backtrace; extern int panic_timeout; extern unsigned long panic_print; extern int panic_on_oops; -extern int panic_on_unrecovered_nmi; -extern int panic_on_io_nmi; extern int panic_on_warn; extern unsigned long panic_on_taint; From 4a8fba4be879251fa010d248c179efbd42ec667d Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:23 +0000 Subject: [PATCH 056/331] x86/nmi: Use a macro to initialize NMI descriptors The NMI descriptors for each NMI type are stored in an array. However, they are currently initialized using raw numbers, which makes it difficult to understand the code. Introduce a macro to initialize the NMI descriptors using the NMI type enum values to make the code more readable. No functional change intended. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Reviewed-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-4-sohil.mehta@intel.com --- arch/x86/kernel/nmi.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 671d846ed620..6a5dc35522c8 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -49,27 +49,20 @@ struct nmi_desc { struct list_head head; }; -static struct nmi_desc nmi_desc[NMI_MAX] = -{ - { - .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock), - .head = LIST_HEAD_INIT(nmi_desc[0].head), - }, - { - .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock), - .head = LIST_HEAD_INIT(nmi_desc[1].head), - }, - { - .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock), - .head = LIST_HEAD_INIT(nmi_desc[2].head), - }, - { - .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock), - .head = LIST_HEAD_INIT(nmi_desc[3].head), - }, +#define NMI_DESC_INIT(type) { \ + .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[type].lock), \ + .head = LIST_HEAD_INIT(nmi_desc[type].head), \ +} +static struct nmi_desc nmi_desc[NMI_MAX] = { + NMI_DESC_INIT(NMI_LOCAL), + NMI_DESC_INIT(NMI_UNKNOWN), + NMI_DESC_INIT(NMI_SERR), + NMI_DESC_INIT(NMI_IO_CHECK), }; +#define nmi_to_desc(type) (&nmi_desc[type]) + struct nmi_stats { unsigned int normal; unsigned int unknown; @@ -107,8 +100,6 @@ static int __init setup_unknown_nmi_panic(char *str) } __setup("unknown_nmi_panic", setup_unknown_nmi_panic); -#define nmi_to_desc(type) (&nmi_desc[type]) - static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC; static int __init nmi_warning_debugfs(void) From 6325f947014644ff39e5902afa48238af8f92e0e Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:24 +0000 Subject: [PATCH 057/331] x86/nmi: Remove export of local_touch_nmi() Commit: feb6cd6a0f9f ("thermal/intel_powerclamp: stop sched tick in forced idle") got rid of the last exported user of local_touch_nmi() a while back. Remove the unnecessary export. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Reviewed-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-5-sohil.mehta@intel.com --- arch/x86/kernel/nmi.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 6a5dc35522c8..cdfb3864d59a 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -745,4 +745,3 @@ void local_touch_nmi(void) { __this_cpu_write(last_nmi_rip, 0); } -EXPORT_SYMBOL_GPL(local_touch_nmi); From b4bc3144c1eca9107f45018000a1e68bfd308d8c Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:25 +0000 Subject: [PATCH 058/331] x86/nmi: Fix comment in unknown_nmi_error() The comment in unknown_nmi_error() is incorrect and misleading. There is no longer a restriction on having a single Unknown NMI handler. Also, nmi_handle() never used the 'b2b' parameter. The commits that made the comment outdated are: 0d443b70cc92 ("x86/platform: Remove warning message for duplicate NMI handlers") bf9f2ee28d47 ("x86/nmi: Remove the 'b2b' parameter from nmi_handle()") Remove the old comment and update it to reflect the current logic. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Reviewed-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-6-sohil.mehta@intel.com --- arch/x86/kernel/nmi.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index cdfb3864d59a..2a07c9adc6a6 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -327,10 +327,9 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) int handled; /* - * Use 'false' as back-to-back NMIs are dealt with one level up. - * Of course this makes having multiple 'unknown' handlers useless - * as only the first one is ever run (unless it can actually determine - * if it caused the NMI) + * As a last resort, let the "unknown" handlers make a + * best-effort attempt to figure out if they can claim + * responsibility for this Unknown NMI. */ handled = nmi_handle(NMI_UNKNOWN, regs); if (handled) { From 59cddd397accfa92fc36c223e3d532b47fdab841 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:26 +0000 Subject: [PATCH 059/331] x86/nmi: Improve and relocate NMI handler comments Some of the comments in the default NMI handling code are out of place or inadequate. Move them to the appropriate locations and update them as needed. Move the comment related to CPU-specific NMIs closer to the actual code. Also, add more details about how back-to-back NMIs are detected since that isn't immediately obvious. Opportunistically, replace an #ifdef section in the vicinity with an IS_ENABLED() check to make the code easier to read. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Acked-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-7-sohil.mehta@intel.com --- arch/x86/kernel/nmi.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 2a07c9adc6a6..59ed74ec010e 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -359,17 +359,18 @@ static noinstr void default_do_nmi(struct pt_regs *regs) bool b2b = false; /* - * CPU-specific NMI must be processed before non-CPU-specific - * NMI, otherwise we may lose it, because the CPU-specific - * NMI can not be detected/processed on other CPUs. - */ - - /* - * Back-to-back NMIs are interesting because they can either - * be two NMI or more than two NMIs (any thing over two is dropped - * due to NMI being edge-triggered). If this is the second half - * of the back-to-back NMI, assume we dropped things and process - * more handlers. Otherwise reset the 'swallow' NMI behaviour + * Back-to-back NMIs are detected by comparing the RIP of the + * current NMI with that of the previous NMI. If it is the same, + * it is assumed that the CPU did not have a chance to jump back + * into a non-NMI context and execute code in between the two + * NMIs. + * + * They are interesting because even if there are more than two, + * only a maximum of two can be detected (anything over two is + * dropped due to NMI being edge-triggered). If this is the + * second half of the back-to-back NMI, assume we dropped things + * and process more handlers. Otherwise, reset the 'swallow' NMI + * behavior. */ if (regs->ip == __this_cpu_read(last_nmi_rip)) b2b = true; @@ -383,6 +384,11 @@ static noinstr void default_do_nmi(struct pt_regs *regs) if (microcode_nmi_handler_enabled() && microcode_nmi_handler()) goto out; + /* + * CPU-specific NMI must be processed before non-CPU-specific + * NMI, otherwise we may lose it, because the CPU-specific + * NMI can not be detected/processed on other CPUs. + */ handled = nmi_handle(NMI_LOCAL, regs); __this_cpu_add(nmi_stats.normal, handled); if (handled) { @@ -419,13 +425,14 @@ static noinstr void default_do_nmi(struct pt_regs *regs) pci_serr_error(reason, regs); else if (reason & NMI_REASON_IOCHK) io_check_error(reason, regs); -#ifdef CONFIG_X86_32 + /* * Reassert NMI in case it became active * meanwhile as it's edge-triggered: */ - reassert_nmi(); -#endif + if (IS_ENABLED(CONFIG_X86_32)) + reassert_nmi(); + __this_cpu_add(nmi_stats.external, 1); raw_spin_unlock(&nmi_reason_lock); goto out; From 3b1292706305e5949d2aa739eb0ae009a9e6f824 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:27 +0000 Subject: [PATCH 060/331] x86/nmi: Improve documentation NMI handlers can be registered by various subsystems, including drivers. However, the interface for registering and unregistering such handlers is not clearly documented. In the future, the interface may need to be extended to identify the source of the NMI. Add documentation to make the current API more understandable and easier to use. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-8-sohil.mehta@intel.com --- arch/x86/include/asm/nmi.h | 43 +++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h index f85aea7bf7f1..79d88d12c8fb 100644 --- a/arch/x86/include/asm/nmi.h +++ b/arch/x86/include/asm/nmi.h @@ -20,8 +20,20 @@ extern int unknown_nmi_panic; extern int panic_on_unrecovered_nmi; extern int panic_on_io_nmi; +/* NMI handler flags */ #define NMI_FLAG_FIRST 1 +/** + * enum - NMI types. + * @NMI_LOCAL: Local NMI, CPU-specific NMI generated by the Local APIC. + * @NMI_UNKNOWN: Unknown NMI, the source of the NMI may not be identified. + * @NMI_SERR: System Error NMI, typically triggered by PCI errors. + * @NMI_IO_CHECK: I/O Check NMI, related to I/O errors. + * @NMI_MAX: Maximum value for NMI types. + * + * NMI types are used to categorize NMIs and to dispatch them to the + * appropriate handler. + */ enum { NMI_LOCAL=0, NMI_UNKNOWN, @@ -30,6 +42,7 @@ enum { NMI_MAX }; +/* NMI handler return values */ #define NMI_DONE 0 #define NMI_HANDLED 1 @@ -43,6 +56,25 @@ struct nmiaction { const char *name; }; +/** + * register_nmi_handler - Register a handler for a specific NMI type + * @t: NMI type (e.g. NMI_LOCAL) + * @fn: The NMI handler + * @fg: Flags associated with the NMI handler + * @n: Name of the NMI handler + * @init: Optional __init* attributes for struct nmiaction + * + * Adds the provided handler to the list of handlers for the specified + * NMI type. Handlers flagged with NMI_FLAG_FIRST would be executed first. + * + * Sometimes the source of an NMI can't be reliably determined which + * results in an NMI being tagged as "unknown". Register an additional + * handler using the NMI type - NMI_UNKNOWN to handle such cases. The + * caller would get one last chance to assume responsibility for the + * NMI. + * + * Return: 0 on success, or an error code on failure. + */ #define register_nmi_handler(t, fn, fg, n, init...) \ ({ \ static struct nmiaction init fn##_na = { \ @@ -56,7 +88,16 @@ struct nmiaction { int __register_nmi_handler(unsigned int, struct nmiaction *); -void unregister_nmi_handler(unsigned int, const char *); +/** + * unregister_nmi_handler - Unregister a handler for a specific NMI type + * @type: NMI type (e.g. NMI_LOCAL) + * @name: Name of the NMI handler used during registration + * + * Removes the handler associated with the specified NMI type from the + * NMI handler list. The "name" is used as a lookup key to identify the + * handler. + */ +void unregister_nmi_handler(unsigned int type, const char *name); void set_emergency_nmi_handler(unsigned int type, nmi_handler_t handler); From 7324d7de7740fdbb29c79f5e2a001524370fa732 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:27 +0000 Subject: [PATCH 061/331] x86/nmi: Add missing description x86_platform_ops::get_nmi_reason to [ mingo: Split off from another patch. ] Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-8-sohil.mehta@intel.com --- arch/x86/include/asm/x86_init.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 213cf5379a5a..36698cc9fb44 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -292,6 +292,7 @@ struct x86_hyper_runtime { * @set_wallclock: set time back to HW clock * @is_untracked_pat_range exclude from PAT logic * @nmi_init enable NMI on cpus + * @get_nmi_reason get the reason an NMI was received * @save_sched_clock_state: save state for sched_clock() on suspend * @restore_sched_clock_state: restore state for sched_clock() on resume * @apic_post_init: adjust apic if needed From 05279a2863ddba6ed993aad655b9f3e57d9a94ce Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:28 +0000 Subject: [PATCH 062/331] x86/nmi: Clean up NMI selftest The expected_testcase_failures variable in the NMI selftest has never been set since its introduction. Remove this unused variable along with the related checks to simplify the code. While at it, replace printk() with the corresponding pr_{cont,info}() calls. Also, get rid of the superfluous testname wrapper and the redundant file path comment. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-9-sohil.mehta@intel.com --- arch/x86/kernel/nmi_selftest.c | 52 +++++++++++----------------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/arch/x86/kernel/nmi_selftest.c b/arch/x86/kernel/nmi_selftest.c index e93a8545c74d..a010e9d062bf 100644 --- a/arch/x86/kernel/nmi_selftest.c +++ b/arch/x86/kernel/nmi_selftest.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 /* - * arch/x86/kernel/nmi-selftest.c - * * Testsuite for NMI: IPIs * * Started by Don Zickus: @@ -30,7 +28,6 @@ static DECLARE_BITMAP(nmi_ipi_mask, NR_CPUS) __initdata; static int __initdata testcase_total; static int __initdata testcase_successes; -static int __initdata expected_testcase_failures; static int __initdata unexpected_testcase_failures; static int __initdata unexpected_testcase_unknowns; @@ -120,26 +117,22 @@ static void __init dotest(void (*testcase_fn)(void), int expected) unexpected_testcase_failures++; if (nmi_fail == FAILURE) - printk(KERN_CONT "FAILED |"); + pr_cont("FAILED |"); else if (nmi_fail == TIMEOUT) - printk(KERN_CONT "TIMEOUT|"); + pr_cont("TIMEOUT|"); else - printk(KERN_CONT "ERROR |"); + pr_cont("ERROR |"); dump_stack(); } else { testcase_successes++; - printk(KERN_CONT " ok |"); + pr_cont(" ok |"); } + pr_cont("\n"); + testcase_total++; - reset_nmi(); } -static inline void __init print_testname(const char *testname) -{ - printk("%12s:", testname); -} - void __init nmi_selftest(void) { init_nmi_testsuite(); @@ -147,38 +140,25 @@ void __init nmi_selftest(void) /* * Run the testsuite: */ - printk("----------------\n"); - printk("| NMI testsuite:\n"); - printk("--------------------\n"); + pr_info("----------------\n"); + pr_info("| NMI testsuite:\n"); + pr_info("--------------------\n"); - print_testname("remote IPI"); + pr_info("%12s:", "remote IPI"); dotest(remote_ipi, SUCCESS); - printk(KERN_CONT "\n"); - print_testname("local IPI"); + + pr_info("%12s:", "local IPI"); dotest(local_ipi, SUCCESS); - printk(KERN_CONT "\n"); cleanup_nmi_testsuite(); + pr_info("--------------------\n"); if (unexpected_testcase_failures) { - printk("--------------------\n"); - printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n", + pr_info("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n", unexpected_testcase_failures, testcase_total); - printk("-----------------------------------------------------------------\n"); - } else if (expected_testcase_failures && testcase_successes) { - printk("--------------------\n"); - printk("%3d out of %3d testcases failed, as expected. |\n", - expected_testcase_failures, testcase_total); - printk("----------------------------------------------------\n"); - } else if (expected_testcase_failures && !testcase_successes) { - printk("--------------------\n"); - printk("All %3d testcases failed, as expected. |\n", - expected_testcase_failures); - printk("----------------------------------------\n"); } else { - printk("--------------------\n"); - printk("Good, all %3d testcases passed! |\n", + pr_info("Good, all %3d testcases passed! |\n", testcase_successes); - printk("---------------------------------\n"); } + pr_info("-----------------------------------------------------------------\n"); } From f2e01dcf6df2d12e86c363ea9c37d53994d89dd6 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Thu, 27 Mar 2025 23:46:29 +0000 Subject: [PATCH 063/331] x86/nmi: Improve NMI duration console printouts Convert the last remaining printk() in nmi.c to pr_info(). Along with it, use timespec macros to calculate the NMI handler duration. Signed-off-by: Sohil Mehta Signed-off-by: Ingo Molnar Reviewed-by: Kai Huang Acked-by: Peter Zijlstra (Intel) Reviewed-by: Nikolay Borisov Link: https://lore.kernel.org/r/20250327234629.3953536-10-sohil.mehta@intel.com --- arch/x86/kernel/nmi.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 59ed74ec010e..be93ec7255bf 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -119,12 +119,12 @@ static void nmi_check_duration(struct nmiaction *action, u64 duration) action->max_duration = duration; - remainder_ns = do_div(duration, (1000 * 1000)); - decimal_msecs = remainder_ns / 1000; + /* Convert duration from nsec to msec */ + remainder_ns = do_div(duration, NSEC_PER_MSEC); + decimal_msecs = remainder_ns / NSEC_PER_USEC; - printk_ratelimited(KERN_INFO - "INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n", - action->handler, duration, decimal_msecs); + pr_info_ratelimited("INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n", + action->handler, duration, decimal_msecs); } static int nmi_handle(unsigned int type, struct pt_regs *regs) From b0510ac74e189442dde8799c1b212bd106f2300c Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 31 Mar 2025 16:13:25 +0800 Subject: [PATCH 064/331] x86/mm: Remove the arch-specific pgd_leaf() definition PGD huge pages are not supported yet, let's use the generic definition in . [ mingo: Cleaned up the changelog. ] Signed-off-by: Baoquan He Signed-off-by: Ingo Molnar Reviewed-by: Oscar Salvador Link: https://lore.kernel.org/r/20250331081327.256412-6-bhe@redhat.com --- arch/x86/include/asm/pgtable.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 7bd6bd6df4a1..5f4fcc0eea17 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1472,9 +1472,6 @@ static inline bool pgdp_maps_userspace(void *__ptr) return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START); } -#define pgd_leaf pgd_leaf -static inline bool pgd_leaf(pgd_t pgd) { return false; } - #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION /* * All top-level MITIGATION_PAGE_TABLE_ISOLATION page tables are order-1 pages From c083eff324edd73eb23f4bd3f40f388a3e7c2cd2 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 31 Mar 2025 16:13:26 +0800 Subject: [PATCH 065/331] x86/mm: Remove the arch-specific p4d_leaf() definition P4D huge pages are not supported yet, let's use the generic definition in . [ mingo: Cleaned up the changelog. ] Signed-off-by: Baoquan He Signed-off-by: Ingo Molnar Reviewed-by: Oscar Salvador Link: https://lore.kernel.org/r/20250331081327.256412-7-bhe@redhat.com --- arch/x86/include/asm/pgtable.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 5f4fcc0eea17..5ddba366d3b4 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -292,13 +292,6 @@ static inline unsigned long pgd_pfn(pgd_t pgd) return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT; } -#define p4d_leaf p4d_leaf -static inline bool p4d_leaf(p4d_t p4d) -{ - /* No 512 GiB pages yet */ - return 0; -} - #define pte_page(pte) pfn_to_page(pte_pfn(pte)) #define pmd_leaf pmd_leaf From 2b00d9031e42eabc8d32847d231ef48b8be0373d Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 31 Mar 2025 16:13:25 +0800 Subject: [PATCH 066/331] x86/mm: Simplify the pgd_leaf() and p4d_leaf() checks a bit The functions return bool, simplify the checks. [ mingo: Split off from two other patches. ] Signed-off-by: Baoquan He Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20250331081327.256412-6-bhe@redhat.com --- arch/x86/mm/pti.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 5f0d579932c6..190299834011 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -185,7 +185,7 @@ static p4d_t *pti_user_pagetable_walk_p4d(unsigned long address) set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); } - BUILD_BUG_ON(pgd_leaf(*pgd) != 0); + BUILD_BUG_ON(pgd_leaf(*pgd)); return p4d_offset(pgd, address); } @@ -206,7 +206,7 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) if (!p4d) return NULL; - BUILD_BUG_ON(p4d_leaf(*p4d) != 0); + BUILD_BUG_ON(p4d_leaf(*p4d)); if (p4d_none(*p4d)) { unsigned long new_pud_page = __get_free_page(gfp); if (WARN_ON_ONCE(!new_pud_page)) From 1701771d3069fbee154ca48e882e227fdcfbb583 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Tue, 1 Apr 2025 16:35:20 +0200 Subject: [PATCH 067/331] x86/mm: Stop prefetching current->mm->mmap_lock on page faults The prefetchw() dates back decades and the fundamental notion of doing something like this on a lock is shady. Moreover, for a few years now in the fast path faults are handled with RCU + per-vma locking, hopefully not even looking at the lock to begin with. As such just remove it. I did not see a point benchmarking this. Given that it is not expected to be looked at by default justifies not doing the prefetch. Signed-off-by: Mateusz Guzik Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250401143520.1113572-1-mjguzik@gmail.com --- arch/x86/mm/fault.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 296d294142c8..697432f63c59 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -13,7 +13,6 @@ #include /* kmmio_handler, ... */ #include /* perf_sw_event */ #include /* hstate_index_to_shift */ -#include /* prefetchw */ #include /* exception_enter(), ... */ #include /* faulthandler_disabled() */ #include /* efi_crash_gracefully_on_page_fault()*/ @@ -1496,8 +1495,6 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) address = cpu_feature_enabled(X86_FEATURE_FRED) ? fred_event_data(regs) : read_cr2(); - prefetchw(¤t->mm->mmap_lock); - /* * KVM uses #PF vector to deliver 'page not present' events to guests * (asynchronous page fault mechanism). The event happens when a From 1f13c60d84e880df6698441026e64f84c7110c49 Mon Sep 17 00:00:00 2001 From: Andrew Cooper Date: Wed, 2 Apr 2025 18:24:58 +0100 Subject: [PATCH 068/331] x86/idle: Remove MFENCEs for X86_BUG_CLFLUSH_MONITOR in mwait_idle_with_hints() and prefer_mwait_c1_over_halt() The following commit, 12 years ago: 7e98b7192046 ("x86, idle: Use static_cpu_has() for CLFLUSH workaround, add barriers") added barriers around the CLFLUSH in mwait_idle_with_hints(), justified with: ... and add memory barriers around it since the documentation is explicit that CLFLUSH is only ordered with respect to MFENCE. This also triggered, 11 years ago, the same adjustment in: f8e617f45829 ("sched/idle/x86: Optimize unnecessary mwait_idle() resched IPIs") during development, although it failed to get the static_cpu_has_bug() treatment. X86_BUG_CLFLUSH_MONITOR (a.k.a the AAI65 errata) is specific to Intel CPUs, and the SDM currently states: Executions of the CLFLUSH instruction are ordered with respect to each other and with respect to writes, locked read-modify-write instructions, and fence instructions[1]. With footnote 1 reading: Earlier versions of this manual specified that executions of the CLFLUSH instruction were ordered only by the MFENCE instruction. All processors implementing the CLFLUSH instruction also order it relative to the other operations enumerated above. i.e. The SDM was incorrect at the time, and barriers should not have been inserted. Double checking the original AAI65 errata (not available from intel.com any more) shows no mention of barriers either. Note: If this were a general codepath, the MFENCEs would be needed, because AMD CPUs of the same vintage do sport otherwise-unordered CLFLUSHs. Remove the unnecessary barriers. Furthermore, use a plain alternative(), rather than static_cpu_has_bug() and/or no optimisation. The workaround is a single instruction. Use an explicit %rax pointer rather than a general memory operand, because MONITOR takes the pointer implicitly in the same way. [ mingo: Cleaned up the commit a bit. ] Fixes: 7e98b7192046 ("x86, idle: Use static_cpu_has() for CLFLUSH workaround, add barriers") Signed-off-by: Andrew Cooper Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Borislav Petkov (AMD) Cc: "H. Peter Anvin" Cc: Peter Zijlstra Cc: Rik van Riel Cc: Linus Torvalds Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Rafael J. Wysocki Link: https://lore.kernel.org/r/20250402172458.1378112-1-andrew.cooper3@citrix.com --- arch/x86/include/asm/mwait.h | 9 +++------ arch/x86/kernel/process.c | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index ce857ef54cf1..54dc313bcdf0 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -116,13 +116,10 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx) static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) { if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) { - if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) { - mb(); - clflush((void *)¤t_thread_info()->flags); - mb(); - } + const void *addr = ¤t_thread_info()->flags; - __monitor((void *)¤t_thread_info()->flags, 0, 0); + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); + __monitor(addr, 0, 0); if (!need_resched()) { if (ecx & 1) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 91f6ff618852..bda47d93c8f9 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -907,13 +907,10 @@ static __init bool prefer_mwait_c1_over_halt(void) static __cpuidle void mwait_idle(void) { if (!current_set_polling_and_test()) { - if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) { - mb(); /* quirk */ - clflush((void *)¤t_thread_info()->flags); - mb(); /* quirk */ - } + const void *addr = ¤t_thread_info()->flags; - __monitor((void *)¤t_thread_info()->flags, 0, 0); + alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr)); + __monitor(addr, 0, 0); if (!need_resched()) { __sti_mwait(0, 0); raw_local_irq_disable(); From 1ae899e413105aa81068d0282ab6e22974891d74 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Wed, 2 Apr 2025 20:08:05 +0200 Subject: [PATCH 069/331] x86/idle: Standardize argument types for MONITOR{,X} and MWAIT{,X} instruction wrappers on 'u32' MONITOR and MONITORX expect 32-bit unsigned integer arguments in the %ecx and %edx registers. MWAIT and MWAITX expect 32-bit usigned int argument in %eax and %ecx registers. Some of the helpers around these instructions in are using too wide types (long), standardize on u32 instead that makes it clear that this is a hardware ABI. [ mingo: Cleaned up the changelog. ] Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Andrew Cooper Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250402180827.3762-1-ubizjak@gmail.com --- arch/x86/include/asm/mwait.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 54dc313bcdf0..3377869ff2e8 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -25,23 +25,21 @@ #define TPAUSE_C01_STATE 1 #define TPAUSE_C02_STATE 0 -static __always_inline void __monitor(const void *eax, unsigned long ecx, - unsigned long edx) +static __always_inline void __monitor(const void *eax, u32 ecx, u32 edx) { /* "monitor %eax, %ecx, %edx;" */ asm volatile(".byte 0x0f, 0x01, 0xc8;" :: "a" (eax), "c" (ecx), "d"(edx)); } -static __always_inline void __monitorx(const void *eax, unsigned long ecx, - unsigned long edx) +static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx) { /* "monitorx %eax, %ecx, %edx;" */ asm volatile(".byte 0x0f, 0x01, 0xfa;" :: "a" (eax), "c" (ecx), "d"(edx)); } -static __always_inline void __mwait(unsigned long eax, unsigned long ecx) +static __always_inline void __mwait(u32 eax, u32 ecx) { mds_idle_clear_cpu_buffers(); @@ -76,8 +74,7 @@ static __always_inline void __mwait(unsigned long eax, unsigned long ecx) * EAX (logical) address to monitor * ECX #GP if not zero */ -static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx, - unsigned long ecx) +static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx) { /* No MDS buffer clear as this is AMD/HYGON only */ @@ -95,7 +92,7 @@ static __always_inline void __mwaitx(unsigned long eax, unsigned long ebx, * executing mwait, it would otherwise go unnoticed and the next tick * would not be reprogrammed accordingly before mwait ever wakes up. */ -static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx) +static __always_inline void __sti_mwait(u32 eax, u32 ecx) { mds_idle_clear_cpu_buffers(); /* "mwait %eax, %ecx;" */ From 19c3dcd953bc8961ab486cf05f5dd45455393c42 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Wed, 2 Apr 2025 20:08:07 +0200 Subject: [PATCH 070/331] x86/idle: Remove .s output beautifying delimiters from simpler asm() templates Delimiters in asm() templates such as ';', '\t' or '\n' are not required syntactically, they were used historically in the Linux kernel to prettify the compiler's .s output for people who were looking at compiler generated .s output. Most x86 developers these days are primarily looking at: 1) objdump --disassemble-all .o 2) perf top's live kernel function annotation and disassembler feature that uses /dev/mem. ... because: - this kind of assembler output is standardized regardless of compiler used, - it's generally less messy looking, - it gives ground-truth instead of being some intermediate layer in the toolchain that might or might not be the real deal, - and on a live kernel it also sees through the kernel's various layers of runtime patching code obfuscation facilities, also known as: alternative-instructions, tracepoints and jump labels. There are some cases where the .s output is the most useful tool, such as alternatives() code generation, but other than that these delimiters used in simple asm() statements mostly add noise to the source code side, which isn't desirable for assembly code that is fragile enough already. Remove the delimiters for , which also happens to make the GCC inliner's asm() instruction length heuristics more accurate... [ mingo: Wrote a new changelog to give historic context and to give people a chance to object. :-) ] Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Andrew Cooper Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250402180827.3762-3-ubizjak@gmail.com --- arch/x86/include/asm/mwait.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 3377869ff2e8..0e020a69b431 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -34,8 +34,8 @@ static __always_inline void __monitor(const void *eax, u32 ecx, u32 edx) static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx) { - /* "monitorx %eax, %ecx, %edx;" */ - asm volatile(".byte 0x0f, 0x01, 0xfa;" + /* "monitorx %eax, %ecx, %edx" */ + asm volatile(".byte 0x0f, 0x01, 0xfa" :: "a" (eax), "c" (ecx), "d"(edx)); } @@ -78,8 +78,8 @@ static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx) { /* No MDS buffer clear as this is AMD/HYGON only */ - /* "mwaitx %eax, %ebx, %ecx;" */ - asm volatile(".byte 0x0f, 0x01, 0xfb;" + /* "mwaitx %eax, %ebx, %ecx" */ + asm volatile(".byte 0x0f, 0x01, 0xfb" :: "a" (eax), "b" (ebx), "c" (ecx)); } @@ -138,13 +138,13 @@ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned lo */ static inline void __tpause(u32 ecx, u32 edx, u32 eax) { - /* "tpause %ecx, %edx, %eax;" */ + /* "tpause %ecx, %edx, %eax" */ #ifdef CONFIG_AS_TPAUSE - asm volatile("tpause %%ecx\n" + asm volatile("tpause %%ecx" : : "c"(ecx), "d"(edx), "a"(eax)); #else - asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n" + asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1" : : "c"(ecx), "d"(edx), "a"(eax)); #endif From a72d55dc3bd6555cc1f97459b42b7f62ae480f13 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Wed, 2 Apr 2025 20:08:08 +0200 Subject: [PATCH 071/331] x86/idle: Remove CONFIG_AS_TPAUSE There is not much point in CONFIG_AS_TPAUSE at all when the emitted assembly is always the same - it only obfuscates the __tpause() code in essence. Remove the TPAUSE insn mnemonic from __tpause() and leave only the equivalent byte-wise definition. This can then be changed back to insn mnemonic once binutils 2.31.1 is the minimum version to build the kernel. (Right now it's 2.25.) Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Andrew Cooper Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250402180827.3762-4-ubizjak@gmail.com --- arch/x86/Kconfig.assembler | 4 ---- arch/x86/include/asm/mwait.h | 11 ++--------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler index 6d20a6ce0507..fa8858546d5e 100644 --- a/arch/x86/Kconfig.assembler +++ b/arch/x86/Kconfig.assembler @@ -15,10 +15,6 @@ config AS_SHA256_NI def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) help Supported by binutils >= 2.24 and LLVM integrated assembler -config AS_TPAUSE - def_bool $(as-instr,tpause %ecx) - help - Supported by binutils >= 2.31.1 and LLVM integrated assembler >= V7 config AS_GFNI def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 0e020a69b431..6a2ec2083043 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -138,16 +138,9 @@ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned lo */ static inline void __tpause(u32 ecx, u32 edx, u32 eax) { - /* "tpause %ecx, %edx, %eax" */ - #ifdef CONFIG_AS_TPAUSE - asm volatile("tpause %%ecx" - : - : "c"(ecx), "d"(edx), "a"(eax)); - #else + /* "tpause %ecx" */ asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1" - : - : "c"(ecx), "d"(edx), "a"(eax)); - #endif + :: "c" (ecx), "d" (edx), "a" (eax)); } #endif /* _ASM_X86_MWAIT_H */ From 2fb34b1566a386913b291d04f91ba6f6e6a5bb99 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Thu, 3 Apr 2025 10:56:23 +0200 Subject: [PATCH 072/331] x86/tlb: Simplify choose_new_asid() and generate better code Have it return the two things it does return: - a new ASID and - the need to flush the TLB or not, in a struct which fits in a single 32-bit register and whack the IO parameters. Beyond being easier to read, this also helps the compiler generate better, more compact code: # arch/x86/mm/tlb.o: text data bss dec hex filename 9341 753 516 10610 2972 tlb.o.before 9213 753 516 10482 28f2 tlb.o.after No functional changes. Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Andrew Cooper Cc: Uros Bizjak Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Peter Zijlstra Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250403085623.20824-1-bp@kernel.org --- arch/x86/mm/tlb.c | 63 +++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index e459d97ef397..d00ae21d0ee2 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -215,16 +215,20 @@ static void clear_asid_other(void) atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1); +struct new_asid { + unsigned int asid : 16; + unsigned int need_flush : 1; +}; -static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, - u16 *new_asid, bool *need_flush) +static struct new_asid choose_new_asid(struct mm_struct *next, u64 next_tlb_gen) { + struct new_asid ns; u16 asid; if (!static_cpu_has(X86_FEATURE_PCID)) { - *new_asid = 0; - *need_flush = true; - return; + ns.asid = 0; + ns.need_flush = 1; + return ns; } /* @@ -235,9 +239,9 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, u16 global_asid = mm_global_asid(next); if (global_asid) { - *new_asid = global_asid; - *need_flush = false; - return; + ns.asid = global_asid; + ns.need_flush = 0; + return ns; } } @@ -249,22 +253,23 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, next->context.ctx_id) continue; - *new_asid = asid; - *need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) < - next_tlb_gen); - return; + ns.asid = asid; + ns.need_flush = (this_cpu_read(cpu_tlbstate.ctxs[asid].tlb_gen) < next_tlb_gen); + return ns; } /* * We don't currently own an ASID slot on this CPU. * Allocate a slot. */ - *new_asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1; - if (*new_asid >= TLB_NR_DYN_ASIDS) { - *new_asid = 0; + ns.asid = this_cpu_add_return(cpu_tlbstate.next_asid, 1) - 1; + if (ns.asid >= TLB_NR_DYN_ASIDS) { + ns.asid = 0; this_cpu_write(cpu_tlbstate.next_asid, 1); } - *need_flush = true; + ns.need_flush = true; + + return ns; } /* @@ -781,9 +786,9 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, bool was_lazy = this_cpu_read(cpu_tlbstate_shared.is_lazy); unsigned cpu = smp_processor_id(); unsigned long new_lam; + struct new_asid ns; u64 next_tlb_gen; - bool need_flush; - u16 new_asid; + /* We don't want flush_tlb_func() to run concurrently with us. */ if (IS_ENABLED(CONFIG_PROVE_LOCKING)) @@ -854,7 +859,7 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, /* Check if the current mm is transitioning to a global ASID */ if (mm_needs_global_asid(next, prev_asid)) { next_tlb_gen = atomic64_read(&next->context.tlb_gen); - choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); + ns = choose_new_asid(next, next_tlb_gen); goto reload_tlb; } @@ -889,8 +894,8 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, * TLB contents went out of date while we were in lazy * mode. Fall through to the TLB switching code below. */ - new_asid = prev_asid; - need_flush = true; + ns.asid = prev_asid; + ns.need_flush = true; } else { /* * Apply process to process speculation vulnerability @@ -918,21 +923,21 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, cpumask_set_cpu(cpu, mm_cpumask(next)); next_tlb_gen = atomic64_read(&next->context.tlb_gen); - choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); + ns = choose_new_asid(next, next_tlb_gen); } reload_tlb: new_lam = mm_lam_cr3_mask(next); - if (need_flush) { - VM_WARN_ON_ONCE(is_global_asid(new_asid)); - this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); - this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); - load_new_mm_cr3(next->pgd, new_asid, new_lam, true); + if (ns.need_flush) { + VM_WARN_ON_ONCE(is_global_asid(ns.asid)); + this_cpu_write(cpu_tlbstate.ctxs[ns.asid].ctx_id, next->context.ctx_id); + this_cpu_write(cpu_tlbstate.ctxs[ns.asid].tlb_gen, next_tlb_gen); + load_new_mm_cr3(next->pgd, ns.asid, new_lam, true); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); } else { /* The new ASID is already up to date. */ - load_new_mm_cr3(next->pgd, new_asid, new_lam, false); + load_new_mm_cr3(next->pgd, ns.asid, new_lam, false); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0); } @@ -941,7 +946,7 @@ reload_tlb: barrier(); this_cpu_write(cpu_tlbstate.loaded_mm, next); - this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid); + this_cpu_write(cpu_tlbstate.loaded_mm_asid, ns.asid); cpu_tlbstate_update_lam(new_lam, mm_untag_mask(next)); if (next != prev) { From a17b37a3f416c9e385bbd2b5fc603d337eab76eb Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 3 Apr 2025 09:30:49 +0200 Subject: [PATCH 073/331] x86/idle: Change arguments of mwait_idle_with_hints() to u32 All functions in mwait_idle_with_hints() cast eax and ecx arguments to u32. Propagate argument type to the enclosing function. Suggested-by: Andrew Cooper Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20250403073105.245987-1-ubizjak@gmail.com --- arch/x86/include/asm/mwait.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 6a2ec2083043..44d3bb2a259d 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -110,7 +110,7 @@ static __always_inline void __sti_mwait(u32 eax, u32 ecx) * New with Core Duo processors, MWAIT can take some hints based on CPU * capability. */ -static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) +static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx) { if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) { const void *addr = ¤t_thread_info()->flags; From fc1cd60042b3df1d162278461c7a87f0362502b8 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 3 Apr 2025 14:50:45 +0200 Subject: [PATCH 074/331] x86/idle: Use MONITOR and MWAIT mnemonics in Current minimum required version of binutils is 2.25, which supports MONITOR and MWAIT instruction mnemonics. Replace the byte-wise specification of MONITOR and MWAIT with these proper mnemonics. No functional change intended. Note: LLVM assembler is not able to assemble correct forms of MONITOR and MWAIT instructions with explicit operands and reports: error: invalid operand for instruction monitor %rax,%ecx,%edx ^~~~ # https://lore.kernel.org/oe-kbuild-all/202504030802.2lEVBSpN-lkp@intel.com/ Use instruction mnemonics with implicit operands to work around this issue. Reported-by: kernel test robot Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20250403125111.429805-1-ubizjak@gmail.com --- arch/x86/include/asm/mwait.h | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index 44d3bb2a259d..dd2b129b0418 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -27,9 +27,11 @@ static __always_inline void __monitor(const void *eax, u32 ecx, u32 edx) { - /* "monitor %eax, %ecx, %edx;" */ - asm volatile(".byte 0x0f, 0x01, 0xc8;" - :: "a" (eax), "c" (ecx), "d"(edx)); + /* + * Use the instruction mnemonic with implicit operands, as the LLVM + * assembler fails to assemble the mnemonic with explicit operands: + */ + asm volatile("monitor" :: "a" (eax), "c" (ecx), "d" (edx)); } static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx) @@ -43,9 +45,11 @@ static __always_inline void __mwait(u32 eax, u32 ecx) { mds_idle_clear_cpu_buffers(); - /* "mwait %eax, %ecx;" */ - asm volatile(".byte 0x0f, 0x01, 0xc9;" - :: "a" (eax), "c" (ecx)); + /* + * Use the instruction mnemonic with implicit operands, as the LLVM + * assembler fails to assemble the mnemonic with explicit operands: + */ + asm volatile("mwait" :: "a" (eax), "c" (ecx)); } /* @@ -95,9 +99,8 @@ static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx) static __always_inline void __sti_mwait(u32 eax, u32 ecx) { mds_idle_clear_cpu_buffers(); - /* "mwait %eax, %ecx;" */ - asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" - :: "a" (eax), "c" (ecx)); + + asm volatile("sti; mwait" :: "a" (eax), "c" (ecx)); } /* From e37aa1211fbfeb9d5e79f4a4b0da898e6d0d53bb Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 24 Sep 2024 10:01:28 -0700 Subject: [PATCH 075/331] x86/cpuid: Add AMX and SPEC_CTRL dependencies Add some missing dependencies to the CPUID dependency table: - All the AMX features depend on AMX_TILE - All the SPEC_CTRL features depend on SPEC_CTRL [ mingo: Keep the AMX part of the table grouped ... ] Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Cc: "H. Peter Anvin" Cc: Sohil Mehta Cc: Tony Luck Cc: Ahmed S. Darwish Link: https://lore.kernel.org/r/20240924170128.2611854-1-ak@linux.intel.com --- arch/x86/kernel/cpu/cpuid-deps.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index a2fbea0be535..94c062cddfa4 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -82,8 +82,12 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_XFD, X86_FEATURE_XSAVES }, { X86_FEATURE_XFD, X86_FEATURE_XGETBV1 }, { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD }, + { X86_FEATURE_AMX_FP16, X86_FEATURE_AMX_TILE }, + { X86_FEATURE_AMX_BF16, X86_FEATURE_AMX_TILE }, + { X86_FEATURE_AMX_INT8, X86_FEATURE_AMX_TILE }, { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES }, { X86_FEATURE_FRED, X86_FEATURE_LKGS }, + { X86_FEATURE_SPEC_CTRL_SSBD, X86_FEATURE_SPEC_CTRL }, {} }; From 0ee07a07920285d519152974872ca6e43f8742e8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 4 Apr 2025 19:53:03 +0300 Subject: [PATCH 076/331] x86/boot: Use __ALIGN_KERNEL_MASK() instead of open coded analogue LOAD_PHYSICAL_ADDR is calculated as an aligned (up) CONFIG_PHYSICAL_START with the respective alignment value CONFIG_PHYSICAL_ALIGN. However, the code is written openly while we have __ALIGN_KERNEL_MASK() macro that does the same. This macro has nothing special, that's why it may be used in assembler code or linker scripts (on the contrary __ALIGN_KERNEL() may not). Do it so. Signed-off-by: Andy Shevchenko Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20250404165303.3657139-1-andriy.shevchenko@linux.intel.com --- arch/x86/include/asm/page_types.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h index 9f77bf03d747..018a8d906ca3 100644 --- a/arch/x86/include/asm/page_types.h +++ b/arch/x86/include/asm/page_types.h @@ -29,9 +29,7 @@ #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC /* Physical address where kernel should be loaded. */ -#define LOAD_PHYSICAL_ADDR ((CONFIG_PHYSICAL_START \ - + (CONFIG_PHYSICAL_ALIGN - 1)) \ - & ~(CONFIG_PHYSICAL_ALIGN - 1)) +#define LOAD_PHYSICAL_ADDR __ALIGN_KERNEL_MASK(CONFIG_PHYSICAL_START, CONFIG_PHYSICAL_ALIGN - 1) #define __START_KERNEL (__START_KERNEL_map + LOAD_PHYSICAL_ADDR) From 5d4456fc88f7aa9bd139b7c5bd4f1c03f552b973 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 1 Apr 2025 15:34:18 +0200 Subject: [PATCH 077/331] x86/boot/compressed: Merge the local pgtable.h include into Merge the local include "pgtable.h" -which declares the API of the 5-level paging trampoline- into so that its implementation in la57toggle.S as well as the calling code can be decoupled from the traditional decompressor. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: David Woodhouse Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250401133416.1436741-9-ardb+git@google.com --- arch/x86/boot/compressed/head_64.S | 1 - arch/x86/boot/compressed/la57toggle.S | 1 - arch/x86/boot/compressed/misc.c | 1 - arch/x86/boot/compressed/pgtable.h | 18 ------------------ arch/x86/boot/compressed/pgtable_64.c | 1 - arch/x86/include/asm/boot.h | 10 ++++++++++ 6 files changed, 10 insertions(+), 22 deletions(-) delete mode 100644 arch/x86/boot/compressed/pgtable.h diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index eafd4f185e77..d9dab940ff62 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -35,7 +35,6 @@ #include #include #include -#include "pgtable.h" /* * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S index 9ee002387eb1..370075b4d95b 100644 --- a/arch/x86/boot/compressed/la57toggle.S +++ b/arch/x86/boot/compressed/la57toggle.S @@ -5,7 +5,6 @@ #include #include #include -#include "pgtable.h" /* * This is the 32-bit trampoline that will be copied over to low memory. It diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 1cdcd4aaf395..94b5991da001 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -14,7 +14,6 @@ #include "misc.h" #include "error.h" -#include "pgtable.h" #include "../string.h" #include "../voffset.h" #include diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h deleted file mode 100644 index 6d595abe06b3..000000000000 --- a/arch/x86/boot/compressed/pgtable.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef BOOT_COMPRESSED_PAGETABLE_H -#define BOOT_COMPRESSED_PAGETABLE_H - -#define TRAMPOLINE_32BIT_SIZE (2 * PAGE_SIZE) - -#define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE -#define TRAMPOLINE_32BIT_CODE_SIZE 0xA0 - -#ifndef __ASSEMBLER__ - -extern unsigned long *trampoline_32bit; - -extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl); - -extern const u16 trampoline_ljmp_imm_offset; - -#endif /* __ASSEMBLER__ */ -#endif /* BOOT_COMPRESSED_PAGETABLE_H */ diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index d8c5de40669d..5a6c7a190e5b 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -4,7 +4,6 @@ #include #include #include -#include "pgtable.h" #include "../string.h" #include "efi.h" diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h index 3f02ff6d333d..02b23aa78955 100644 --- a/arch/x86/include/asm/boot.h +++ b/arch/x86/include/asm/boot.h @@ -74,6 +74,11 @@ # define BOOT_STACK_SIZE 0x1000 #endif +#define TRAMPOLINE_32BIT_SIZE (2 * PAGE_SIZE) + +#define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE +#define TRAMPOLINE_32BIT_CODE_SIZE 0xA0 + #ifndef __ASSEMBLER__ extern unsigned int output_len; extern const unsigned long kernel_text_size; @@ -83,6 +88,11 @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr, void (*error)(char *x)); extern struct boot_params *boot_params_ptr; +extern unsigned long *trampoline_32bit; +extern const u16 trampoline_ljmp_imm_offset; + +void trampoline_32bit_src(void *trampoline, bool enable_5lvl); + #endif #endif /* _ASM_X86_BOOT_H */ From 5a67da1f49cf4f16ec9966446885131dad0eb245 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 1 Apr 2025 15:34:19 +0200 Subject: [PATCH 078/331] x86/boot: Move the 5-level paging trampoline into /startup The 5-level paging trampoline is used by both the EFI stub and the traditional decompressor. Move it out of the decompressor sources into the newly minted arch/x86/boot/startup/ sub-directory which will hold startup code that may be shared between the decompressor, the EFI stub and the kernel proper, and needs to tolerate being called during early boot, before the kernel virtual mapping has been created. This will allow the 5-level paging trampoline to be used by EFI boot images such as zboot that omit the traditional decompressor entirely. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: David Woodhouse Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250401133416.1436741-10-ardb+git@google.com --- arch/x86/Makefile | 1 + arch/x86/boot/compressed/Makefile | 2 +- arch/x86/boot/startup/Makefile | 3 +++ arch/x86/boot/{compressed => startup}/la57toggle.S | 0 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 arch/x86/boot/startup/Makefile rename arch/x86/boot/{compressed => startup}/la57toggle.S (100%) diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 594723005d95..53daf4654f6c 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -281,6 +281,7 @@ archprepare: $(cpufeaturemasks.hdr) ### # Kernel objects +core-y += arch/x86/boot/startup/ libs-y += arch/x86/lib/ # drivers-y are linked after core-y diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index fdbce022db55..37b85ce9b2a3 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -97,7 +97,6 @@ ifdef CONFIG_X86_64 vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o - vmlinux-objs-y += $(obj)/la57toggle.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o @@ -106,6 +105,7 @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a +vmlinux-libs-$(CONFIG_X86_64) += $(objtree)/arch/x86/boot/startup/lib.a $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE $(call if_changed,ld) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile new file mode 100644 index 000000000000..03519ef4869d --- /dev/null +++ b/arch/x86/boot/startup/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +lib-$(CONFIG_X86_64) += la57toggle.o diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S similarity index 100% rename from arch/x86/boot/compressed/la57toggle.S rename to arch/x86/boot/startup/la57toggle.S From 4f2d1bbc2c92a32fd612e6c3b51832d5c1c3678e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 1 Apr 2025 15:34:20 +0200 Subject: [PATCH 079/331] x86/boot: Move the EFI mixed mode startup code back under arch/x86, into startup/ Linus expressed a strong preference for arch-specific asm code (i.e., virtually all of it) to reside under arch/ rather than anywhere else. So move the EFI mixed mode startup code back, and put it under arch/x86/boot/startup/ where all shared x86 startup code is going to live. Suggested-by: Linus Torvalds Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: David Woodhouse Cc: H. Peter Anvin Cc: Kees Cook Link: https://lore.kernel.org/r/20250401133416.1436741-11-ardb+git@google.com --- arch/x86/boot/startup/Makefile | 3 +++ .../libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0 drivers/firmware/efi/libstub/Makefile | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) rename drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S (100%) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 03519ef4869d..73946a3f6b3b 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +KBUILD_AFLAGS += -D__DISABLE_EXPORTS + lib-$(CONFIG_X86_64) += la57toggle.o +lib-$(CONFIG_EFI_MIXED) += efi-mixed.o diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S similarity index 100% rename from drivers/firmware/efi/libstub/x86-mixed.S rename to arch/x86/boot/startup/efi-mixed.S diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index d23a1b9fed75..2f173391b63d 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -85,7 +85,6 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o lib-$(CONFIG_X86) += x86-stub.o smbios.o -lib-$(CONFIG_EFI_MIXED) += x86-mixed.o lib-$(CONFIG_X86_64) += x86-5lvl.o lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o From 321550859f3bd64f547d0b4e9fbd97bd539ef47c Mon Sep 17 00:00:00 2001 From: Boris Ostrovsky Date: Thu, 27 Mar 2025 19:05:03 -0400 Subject: [PATCH 080/331] x86/microcode/AMD: Clean the cache if update did not load microcode If microcode did not get loaded there is no reason to keep it in the cache. Moreover, if loading failed it will not be possible to load an earlier version of microcode since the failed revision will always be selected from the cache on the next reload attempt. Since the failed revisions is not easily available at this point just clean the whole cache. It will be rebuilt later if needed. Signed-off-by: Boris Ostrovsky Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20250327230503.1850368-3-boris.ostrovsky@oracle.com --- arch/x86/kernel/cpu/microcode/amd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index b61028cf5c8a..57bd61f9c69b 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -1171,11 +1171,18 @@ static void microcode_fini_cpu_amd(int cpu) uci->mc = NULL; } +static void finalize_late_load_amd(int result) +{ + if (result) + cleanup(); +} + static struct microcode_ops microcode_amd_ops = { .request_microcode_fw = request_microcode_amd, .collect_cpu_info = collect_cpu_info_amd, .apply_microcode = apply_microcode_amd, .microcode_fini_cpu = microcode_fini_cpu_amd, + .finalize_late_load = finalize_late_load_amd, .nmi_safe = true, }; From d9fa398fe82728ee703ad2bd9cf5247df9626470 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 8 Apr 2025 10:52:56 +0200 Subject: [PATCH 081/331] x86/boot/startup: Disable objtool validation for library code The library code built under arch/x86/boot/startup is not intended to be linked into vmlinux but only into the decompressor and/or the EFI stub. This means objtool validation is not needed here, and may result in false positive errors for things like missing retpolines. So disable it for all objects added to lib-y Tested-by: Chaitanya Kumar Borah Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Cc: David Woodhouse Cc: Rafael J. Wysocki Cc: Len Brown Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250408085254.836788-10-ardb+git@google.com --- arch/x86/boot/startup/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 73946a3f6b3b..8919a1cbcb5a 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -4,3 +4,9 @@ KBUILD_AFLAGS += -D__DISABLE_EXPORTS lib-$(CONFIG_X86_64) += la57toggle.o lib-$(CONFIG_EFI_MIXED) += efi-mixed.o + +# +# Disable objtool validation for all library code, which is intended +# to be linked into the decompressor or the EFI stub but not vmlinux +# +$(patsubst %.o,$(obj)/%.o,$(lib-y)): OBJECT_FILES_NON_STANDARD := y From d274cde0dbe0217ee2f2ddbb1a3c545dedf81a06 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Wed, 9 Apr 2025 14:22:30 +0200 Subject: [PATCH 082/331] x86/cacheinfo: Properly parse CPUID(0x80000005) L1d/L1i associativity For the AMD CPUID(4) emulation cache info logic, the same associativity mapping array, assocs[], is used for both CPUID(0x80000005) and CPUID(0x80000006). This is incorrect since per the AMD manuals, the mappings for CPUID(0x80000005) L1d/L1i associativity is: n = 0x1 -> 0xfe n n = 0xff fully associative while assocs[] maps these values to: n = 0x1, 0x2, 0x4 n n = 0x3, 0x7, 0x9 0 n = 0x6 8 n = 0x8 16 n = 0xa 32 n = 0xb 48 n = 0xc 64 n = 0xd 96 n = 0xe 128 n = 0xf fully associative which is only valid for CPUID(0x80000006). Parse CPUID(0x80000005) L1d/L1i associativity values as shown in the AMD manuals. Since the 0xffff literal is used to denote full associativity at the AMD CPUID(4)-emulation logic, define AMD_CPUID4_FULLY_ASSOCIATIVE for it instead of spreading that literal in more places. Mark the assocs[] mapping array as only valid for CPUID(0x80000006) L2/L3 cache information. Fixes: a326e948c538 ("x86, cacheinfo: Fixup L3 cache information for AMD multi-node processors") Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: "H. Peter Anvin" Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250409122233.1058601-2-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index cd48d34ac04b..f4817cd50cfb 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -91,6 +91,8 @@ static const enum cache_type cache_type_map[] = { * AMD CPUs with TOPOEXT can just use CPUID(0x8000001d) */ +#define AMD_CPUID4_FULLY_ASSOCIATIVE 0xffff + union l1_cache { struct { unsigned line_size :8; @@ -122,6 +124,7 @@ union l3_cache { unsigned int val; }; +/* L2/L3 associativity mapping */ static const unsigned short assocs[] = { [1] = 1, [2] = 2, @@ -133,7 +136,7 @@ static const unsigned short assocs[] = { [0xc] = 64, [0xd] = 96, [0xe] = 128, - [0xf] = 0xffff /* Fully associative */ + [0xf] = AMD_CPUID4_FULLY_ASSOCIATIVE }; static const unsigned char levels[] = { 1, 1, 2, 3 }; @@ -163,7 +166,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, if (!l1->val) return; - assoc = assocs[l1->assoc]; + assoc = (l1->assoc == 0xff) ? AMD_CPUID4_FULLY_ASSOCIATIVE : l1->assoc; line_size = l1->line_size; lines_per_tag = l1->lines_per_tag; size_in_kb = l1->size_in_kb; @@ -201,7 +204,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, eax->split.num_threads_sharing = 0; eax->split.num_cores_on_die = topology_num_cores_per_package(); - if (assoc == 0xffff) + if (assoc == AMD_CPUID4_FULLY_ASSOCIATIVE) eax->split.is_fully_associative = 1; ebx->split.coherency_line_size = line_size - 1; From d02c83d75f9f76dda046edbd9f39b911427677c9 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Wed, 9 Apr 2025 14:22:31 +0200 Subject: [PATCH 083/331] x86/cacheinfo: Properly parse CPUID(0x80000006) L2/L3 associativity Complete the AMD CPUID(4) emulation logic, which uses CPUID(0x80000006) for L2/L3 cache info and an assocs[] associativity mapping array, by adding entries for 3-way caches and 6-way caches. Properly handle the case where CPUID(0x80000006) returns an L2/L3 associativity of 9. This is not real associativity, but a marker to indicate that the respective L2/L3 cache information should be retrieved from CPUID(0x8000001d) instead. If such a marker is encountered, return early from legacy_amd_cpuid4(), thus effectively emulating an "invalid index" CPUID(4) response with a cache type of zero. When checking if CPUID(0x80000006) L2/L3 cache info output is valid, and given the associtivity marker 9 above, do not just check if the whole ECX/EDX register is zero. Rather, check if the associativity is zero or 9. An associativity of zero implies no L2/L3 cache, which make it the more correct check anyway vs. a zero check of the whole output register. Fixes: a326e948c538 ("x86, cacheinfo: Fixup L3 cache information for AMD multi-node processors") Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: "H. Peter Anvin" Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250409122233.1058601-3-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index f4817cd50cfb..52727f8c0006 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -89,9 +89,13 @@ static const enum cache_type cache_type_map[] = { /* * Fallback AMD CPUID(4) emulation * AMD CPUs with TOPOEXT can just use CPUID(0x8000001d) + * + * @AMD_L2_L3_INVALID_ASSOC: cache info for the respective L2/L3 cache should + * be determined from CPUID(0x8000001d) instead of CPUID(0x80000006). */ #define AMD_CPUID4_FULLY_ASSOCIATIVE 0xffff +#define AMD_L2_L3_INVALID_ASSOC 0x9 union l1_cache { struct { @@ -128,7 +132,9 @@ union l3_cache { static const unsigned short assocs[] = { [1] = 1, [2] = 2, + [3] = 3, [4] = 4, + [5] = 6, [6] = 8, [8] = 16, [0xa] = 32, @@ -172,7 +178,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, size_in_kb = l1->size_in_kb; break; case 2: - if (!l2.val) + if (!l2.assoc || l2.assoc == AMD_L2_L3_INVALID_ASSOC) return; /* Use x86_cache_size as it might have K7 errata fixes */ @@ -182,7 +188,7 @@ static void legacy_amd_cpuid4(int index, union _cpuid4_leaf_eax *eax, size_in_kb = __this_cpu_read(cpu_info.x86_cache_size); break; case 3: - if (!l3.val) + if (!l3.assoc || l3.assoc == AMD_L2_L3_INVALID_ASSOC) return; assoc = assocs[l3.assoc]; From 60567e93c05d7064c93830cf4bf0d2c58f11b2f2 Mon Sep 17 00:00:00 2001 From: Malaya Kumar Rout Date: Wed, 9 Apr 2025 19:23:37 +0530 Subject: [PATCH 084/331] selftests/x86/lam: Fix clean up fds in do_uring() and allocate_dsa_pasid() Resolve minor fd leaks reported by cppcheck in lam.c. Specifically, the 'file_fd' and 'fd' were not closed in do_uring() and allocate_dsa_pasid() functions, respectively. cppcheck output before this patch: tools/testing/selftests/x86/lam.c:685:3: error: Resource leak: file_fd [resourceLeak] tools/testing/selftests/x86/lam.c:693:3: error: Resource leak: file_fd [resourceLeak] tools/testing/selftests/x86/lam.c:1195:2: error: Resource leak: fd [resourceLeak] cppcheck output after this patch: No resource leaks found While this is a standalone test tool that doesn't really leak anything in practice, as exit() cleans it up all, clean up resources nevertheless. [ mingo: Updated the changelog. ] Signed-off-by: Malaya Kumar Rout Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20250409135341.28987-1-malayarout91@gmail.com --- tools/testing/selftests/x86/lam.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/x86/lam.c b/tools/testing/selftests/x86/lam.c index 18d736640ece..0873b0e5f48b 100644 --- a/tools/testing/selftests/x86/lam.c +++ b/tools/testing/selftests/x86/lam.c @@ -682,7 +682,7 @@ int do_uring(unsigned long lam) return 1; if (fstat(file_fd, &st) < 0) - return 1; + goto cleanup; off_t file_sz = st.st_size; @@ -690,7 +690,7 @@ int do_uring(unsigned long lam) fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks); if (!fi) - return 1; + goto cleanup; fi->file_sz = file_sz; fi->file_fd = file_fd; @@ -698,7 +698,7 @@ int do_uring(unsigned long lam) ring = malloc(sizeof(*ring)); if (!ring) { free(fi); - return 1; + goto cleanup; } memset(ring, 0, sizeof(struct io_ring)); @@ -729,6 +729,8 @@ out: } free(fi); +cleanup: + close(file_fd); return ret; } @@ -1189,6 +1191,7 @@ void *allocate_dsa_pasid(void) wq = mmap(NULL, 0x1000, PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); + close(fd); if (wq == MAP_FAILED) perror("mmap"); From 6f9bd8ae0340326a7c711bc681321bf9a4e15fb2 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Tue, 1 Apr 2025 22:30:29 +0200 Subject: [PATCH 085/331] x86/uaccess: Predict valid_user_address() returning true This works around what seems to be an optimization bug in GCC (at least 13.3.0), where it predicts access_ok() to fail despite the hint to the contrary. _copy_to_user() contains: if (access_ok(to, n)) { instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } Where access_ok() is likely(__access_ok(addr, size)), yet the compiler emits conditional jumps forward for the case where it succeeds: <+0>: endbr64 <+4>: mov %rdx,%rcx <+7>: mov %rdx,%rax <+10>: xor %edx,%edx <+12>: add %rdi,%rcx <+15>: setb %dl <+18>: movabs $0x123456789abcdef,%r8 <+28>: test %rdx,%rdx <+31>: jne 0xffffffff81b3b7c6 <_copy_to_user+38> <+33>: cmp %rcx,%r8 <+36>: jae 0xffffffff81b3b7cb <_copy_to_user+43> <+38>: jmp 0xffffffff822673e0 <__x86_return_thunk> <+43>: nop <+44>: nop <+45>: nop <+46>: mov %rax,%rcx <+49>: rep movsb %ds:(%rsi),%es:(%rdi) <+51>: nop <+52>: nop <+53>: nop <+54>: mov %rcx,%rax <+57>: nop <+58>: nop <+59>: nop <+60>: jmp 0xffffffff822673e0 <__x86_return_thunk> Patching _copy_to_user() to likely() around the access_ok() use does not change the asm. However, spelling out the prediction *within* valid_user_address() does the trick: <+0>: endbr64 <+4>: xor %eax,%eax <+6>: mov %rdx,%rcx <+9>: add %rdi,%rdx <+12>: setb %al <+15>: movabs $0x123456789abcdef,%r8 <+25>: test %rax,%rax <+28>: jne 0xffffffff81b315e6 <_copy_to_user+54> <+30>: cmp %rdx,%r8 <+33>: jb 0xffffffff81b315e6 <_copy_to_user+54> <+35>: nop <+36>: nop <+37>: nop <+38>: rep movsb %ds:(%rsi),%es:(%rdi) <+40>: nop <+41>: nop <+42>: nop <+43>: nop <+44>: nop <+45>: nop <+46>: mov %rcx,%rax <+49>: jmp 0xffffffff82255ba0 <__x86_return_thunk> <+54>: mov %rcx,%rax <+57>: jmp 0xffffffff82255ba0 <__x86_return_thunk> Since we kinda expect valid_user_address() to be likely anyway, add the likely() annotation that also happens to work around this compiler bug. [ mingo: Moved the unlikely() branch into valid_user_address() & updated the changelog ] Signed-off-by: Mateusz Guzik Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250401203029.1132135-1-mjguzik@gmail.com --- arch/x86/include/asm/uaccess_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index c52f0133425b..4c13883371aa 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -54,7 +54,7 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, #endif #define valid_user_address(x) \ - ((__force unsigned long)(x) <= runtime_const_ptr(USER_PTR_MAX)) + likely((__force unsigned long)(x) <= runtime_const_ptr(USER_PTR_MAX)) /* * Masking the user address is an alternative to a conditional From 35c3151a98a6e6f56552cff8dc7d59e8ef7aca50 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 9 Apr 2025 15:28:15 +0300 Subject: [PATCH 086/331] x86/mm: Consolidate initmem_init() There are 4 wariants of initmem_init(), for 32 and 64 bits and for CONFIG_NUMA enabled and disabled. After commit bbeb69ce3013 ("x86/mm: Remove CONFIG_HIGHMEM64G support") NUMA is not supported on 32 bit kernels anymore, and arch/x86/mm/numa_32.c can be just deleted and setup_bootmem_allocator() with completely misleading name can be folded into initmem_init(). For 64 bits the NUMA variant calls x86_numa_init() and !NUMA variant sets all memory to node 0. The later can be split out into inline helper called x86_numa_init() and then both initmem_init() functions become the same. Split out memblock_set_node() from initmem_init() for !NUMA on 64 bit into x86_numa_init() helper and remove arch/x86/mm/numa_*.c that only contained initmem_init() variants for NUMA configs. Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Ingo Molnar Cc: Ard Biesheuvel Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Cc: David Woodhouse Cc: Rafael J. Wysocki Cc: Len Brown Link: https://lore.kernel.org/r/20250409122815.420041-1-rppt@kernel.org --- arch/x86/include/asm/page_32_types.h | 1 - arch/x86/mm/Makefile | 2 +- arch/x86/mm/init_32.c | 7 ---- arch/x86/mm/init_64.c | 7 +++- arch/x86/mm/mm_internal.h | 4 ++ arch/x86/mm/numa.c | 3 +- arch/x86/mm/numa_32.c | 61 ---------------------------- arch/x86/mm/numa_64.c | 13 ------ arch/x86/mm/numa_internal.h | 10 ----- 9 files changed, 13 insertions(+), 95 deletions(-) delete mode 100644 arch/x86/mm/numa_32.c delete mode 100644 arch/x86/mm/numa_64.c delete mode 100644 arch/x86/mm/numa_internal.h diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h index a9b62e0e6f79..623f1e9f493e 100644 --- a/arch/x86/include/asm/page_32_types.h +++ b/arch/x86/include/asm/page_32_types.h @@ -73,7 +73,6 @@ extern unsigned int __VMALLOC_RESERVE; extern int sysctl_legacy_va_layout; extern void find_low_pfn_range(void); -extern void setup_bootmem_allocator(void); #endif /* !__ASSEMBLER__ */ diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 32035d5be5a0..1e72f06b6ba5 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -52,7 +52,7 @@ obj-$(CONFIG_MMIOTRACE) += mmiotrace.o mmiotrace-y := kmmio.o pf_in.o mmio-mod.o obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o -obj-$(CONFIG_NUMA) += numa.o numa_$(BITS).o +obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_AMD_NUMA) += amdtopology.o obj-$(CONFIG_ACPI_NUMA) += srat.o diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index ad662cc4605c..d467f89191cd 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -612,7 +612,6 @@ void __init find_low_pfn_range(void) highmem_pfn_init(); } -#ifndef CONFIG_NUMA void __init initmem_init(void) { #ifdef CONFIG_HIGHMEM @@ -633,12 +632,6 @@ void __init initmem_init(void) printk(KERN_NOTICE "%ldMB LOWMEM available.\n", pages_to_mb(max_low_pfn)); - setup_bootmem_allocator(); -} -#endif /* !CONFIG_NUMA */ - -void __init setup_bootmem_allocator(void) -{ printk(KERN_INFO " mapped low ram: 0 - %08lx\n", max_pfn_mapped< #include #include +#include -#include "numa_internal.h" +#include "mm_internal.h" int numa_off; diff --git a/arch/x86/mm/numa_32.c b/arch/x86/mm/numa_32.c deleted file mode 100644 index 65fda406e6f2..000000000000 --- a/arch/x86/mm/numa_32.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Written by: Patricia Gaughen , IBM Corporation - * August 2002: added remote node KVA remap - Martin J. Bligh - * - * Copyright (C) 2002, IBM Corp. - * - * All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or - * NON INFRINGEMENT. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include -#include - -#include "numa_internal.h" - -extern unsigned long highend_pfn, highstart_pfn; - -void __init initmem_init(void) -{ - x86_numa_init(); - -#ifdef CONFIG_HIGHMEM - highstart_pfn = highend_pfn = max_pfn; - if (max_pfn > max_low_pfn) - highstart_pfn = max_low_pfn; - printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", - pages_to_mb(highend_pfn - highstart_pfn)); - high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; -#else - high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; -#endif - printk(KERN_NOTICE "%ldMB LOWMEM available.\n", - pages_to_mb(max_low_pfn)); - printk(KERN_DEBUG "max_low_pfn = %lx, highstart_pfn = %lx\n", - max_low_pfn, highstart_pfn); - - printk(KERN_DEBUG "Low memory ends at vaddr %08lx\n", - (ulong) pfn_to_kaddr(max_low_pfn)); - - printk(KERN_DEBUG "High memory starts at vaddr %08lx\n", - (ulong) pfn_to_kaddr(highstart_pfn)); - - __vmalloc_start_set = true; - setup_bootmem_allocator(); -} diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c deleted file mode 100644 index 59d80160fa5a..000000000000 --- a/arch/x86/mm/numa_64.c +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Generic VM initialization for x86-64 NUMA setups. - * Copyright 2002,2003 Andi Kleen, SuSE Labs. - */ -#include - -#include "numa_internal.h" - -void __init initmem_init(void) -{ - x86_numa_init(); -} diff --git a/arch/x86/mm/numa_internal.h b/arch/x86/mm/numa_internal.h deleted file mode 100644 index 11e1ff370c10..000000000000 --- a/arch/x86/mm/numa_internal.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __X86_MM_NUMA_INTERNAL_H -#define __X86_MM_NUMA_INTERNAL_H - -#include -#include - -void __init x86_numa_init(void); - -#endif /* __X86_MM_NUMA_INTERNAL_H */ From a23be6ccd8b966ae2483bfc873720b2868ad63c3 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 7 Apr 2025 13:23:01 +0200 Subject: [PATCH 087/331] x86: Remove __FORCE_ORDER workaround GCC PR82602 that caused invalid scheduling of volatile asms: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82602 was fixed for gcc-8.1.0, the current minimum version of the compiler required to compile the kernel. Remove workaround that prevented invalid scheduling for compilers, affected by PR82602. There were no differences between old and new kernel object file when compiled for x86_64 defconfig with gcc-8.1.0. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Peter Zijlstra Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250407112316.378347-1-ubizjak@gmail.com --- arch/x86/include/asm/debugreg.h | 12 ++++++------ arch/x86/include/asm/special_insns.h | 21 +++++---------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h index fdbbbfec745a..719d95f1ab5e 100644 --- a/arch/x86/include/asm/debugreg.h +++ b/arch/x86/include/asm/debugreg.h @@ -23,7 +23,7 @@ DECLARE_PER_CPU(unsigned long, cpu_dr7); static __always_inline unsigned long native_get_debugreg(int regno) { - unsigned long val = 0; /* Damn you, gcc! */ + unsigned long val; switch (regno) { case 0: @@ -43,7 +43,7 @@ static __always_inline unsigned long native_get_debugreg(int regno) break; case 7: /* - * Apply __FORCE_ORDER to DR7 reads to forbid re-ordering them + * Use "asm volatile" for DR7 reads to forbid re-ordering them * with other code. * * This is needed because a DR7 access can cause a #VC exception @@ -55,7 +55,7 @@ static __always_inline unsigned long native_get_debugreg(int regno) * re-ordered to happen before the call to sev_es_ist_enter(), * causing stack recursion. */ - asm volatile("mov %%db7, %0" : "=r" (val) : __FORCE_ORDER); + asm volatile("mov %%db7, %0" : "=r" (val)); break; default: BUG(); @@ -83,15 +83,15 @@ static __always_inline void native_set_debugreg(int regno, unsigned long value) break; case 7: /* - * Apply __FORCE_ORDER to DR7 writes to forbid re-ordering them + * Use "asm volatile" for DR7 writes to forbid re-ordering them * with other code. * * While is didn't happen with a DR7 write (see the DR7 read * comment above which explains where it happened), add the - * __FORCE_ORDER here too to avoid similar problems in the + * "asm volatile" here too to avoid similar problems in the * future. */ - asm volatile("mov %0, %%db7" ::"r" (value), __FORCE_ORDER); + asm volatile("mov %0, %%db7" ::"r" (value)); break; default: BUG(); diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 6266d6b9e0b8..ecda17efa042 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -10,30 +10,19 @@ #include #include -/* - * The compiler should not reorder volatile asm statements with respect to each - * other: they should execute in program order. However GCC 4.9.x and 5.x have - * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder - * volatile asm. The write functions are not affected since they have memory - * clobbers preventing reordering. To prevent reads from being reordered with - * respect to writes, use a dummy memory operand. - */ - -#define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL) - void native_write_cr0(unsigned long val); static inline unsigned long native_read_cr0(void) { unsigned long val; - asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER); + asm volatile("mov %%cr0,%0" : "=r" (val)); return val; } static __always_inline unsigned long native_read_cr2(void) { unsigned long val; - asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER); + asm volatile("mov %%cr2,%0" : "=r" (val)); return val; } @@ -45,7 +34,7 @@ static __always_inline void native_write_cr2(unsigned long val) static __always_inline unsigned long __native_read_cr3(void) { unsigned long val; - asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER); + asm volatile("mov %%cr3,%0" : "=r" (val)); return val; } @@ -66,10 +55,10 @@ static inline unsigned long native_read_cr4(void) asm volatile("1: mov %%cr4, %0\n" "2:\n" _ASM_EXTABLE(1b, 2b) - : "=r" (val) : "0" (0), __FORCE_ORDER); + : "=r" (val) : "0" (0)); #else /* CR4 always exists on x86_64. */ - asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER); + asm volatile("mov %%cr4,%0" : "=r" (val)); #endif return val; } From dfe2574ce87e031c0c37d49b9bee7e1f3c95bff9 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:48 +0200 Subject: [PATCH 088/331] x86/msr: Standardize on u64 in There's 9 uses of 'unsigned long long' in , which is really the same as 'u64', which is used 34 times. Standardize on u64. Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: H. Peter Anvin Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/msr.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 9397a319d165..8ee6fc633476 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -48,7 +48,7 @@ struct saved_msrs { #define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) #else -#define DECLARE_ARGS(val, low, high) unsigned long long val +#define DECLARE_ARGS(val, low, high) u64 val #define EAX_EDX_VAL(val, low, high) (val) #define EAX_EDX_RET(val, low, high) "=A" (val) #endif @@ -79,7 +79,7 @@ static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {} * think of extending them - you will be slapped with a stinking trout or a frozen * shark will reach you, wherever you are! You've been warned. */ -static __always_inline unsigned long long __rdmsr(unsigned int msr) +static __always_inline u64 __rdmsr(unsigned int msr) { DECLARE_ARGS(val, low, high); @@ -113,9 +113,9 @@ do { \ __wrmsr((msr), (u32)((u64)(val)), \ (u32)((u64)(val) >> 32)) -static inline unsigned long long native_read_msr(unsigned int msr) +static inline u64 native_read_msr(unsigned int msr) { - unsigned long long val; + u64 val; val = __rdmsr(msr); @@ -125,7 +125,7 @@ static inline unsigned long long native_read_msr(unsigned int msr) return val; } -static inline unsigned long long native_read_msr_safe(unsigned int msr, +static inline u64 native_read_msr_safe(unsigned int msr, int *err) { DECLARE_ARGS(val, low, high); @@ -179,7 +179,7 @@ extern int wrmsr_safe_regs(u32 regs[8]); * CPU can and will speculatively execute that RDTSC, though, so the * results can be non-monotonic if compared on different CPUs. */ -static __always_inline unsigned long long rdtsc(void) +static __always_inline u64 rdtsc(void) { DECLARE_ARGS(val, low, high); @@ -196,7 +196,7 @@ static __always_inline unsigned long long rdtsc(void) * be impossible to observe non-monotonic rdtsc_unordered() behavior * across multiple CPUs as long as the TSC is synced. */ -static __always_inline unsigned long long rdtsc_ordered(void) +static __always_inline u64 rdtsc_ordered(void) { DECLARE_ARGS(val, low, high); @@ -224,7 +224,7 @@ static __always_inline unsigned long long rdtsc_ordered(void) return EAX_EDX_VAL(val, low, high); } -static inline unsigned long long native_read_pmc(int counter) +static inline u64 native_read_pmc(int counter) { DECLARE_ARGS(val, low, high); @@ -280,7 +280,7 @@ static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high) __err; \ }) -static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p) +static inline int rdmsrl_safe(unsigned int msr, u64 *p) { int err; From f4138de5e41fae1a0b406f0d354a3028dc46bf1f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:49 +0200 Subject: [PATCH 089/331] x86/msr: Standardize on u64 in Also fix some nearby whitespace damage while at it. Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: H. Peter Anvin Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/msr-index.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index e6134ef2263d..e0d60805b869 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -525,7 +525,7 @@ #define MSR_HWP_CAPABILITIES 0x00000771 #define MSR_HWP_REQUEST_PKG 0x00000772 #define MSR_HWP_INTERRUPT 0x00000773 -#define MSR_HWP_REQUEST 0x00000774 +#define MSR_HWP_REQUEST 0x00000774 #define MSR_HWP_STATUS 0x00000777 /* CPUID.6.EAX */ @@ -542,16 +542,16 @@ #define HWP_LOWEST_PERF(x) (((x) >> 24) & 0xff) /* IA32_HWP_REQUEST */ -#define HWP_MIN_PERF(x) (x & 0xff) -#define HWP_MAX_PERF(x) ((x & 0xff) << 8) +#define HWP_MIN_PERF(x) (x & 0xff) +#define HWP_MAX_PERF(x) ((x & 0xff) << 8) #define HWP_DESIRED_PERF(x) ((x & 0xff) << 16) -#define HWP_ENERGY_PERF_PREFERENCE(x) (((unsigned long long) x & 0xff) << 24) +#define HWP_ENERGY_PERF_PREFERENCE(x) (((u64)x & 0xff) << 24) #define HWP_EPP_PERFORMANCE 0x00 #define HWP_EPP_BALANCE_PERFORMANCE 0x80 #define HWP_EPP_BALANCE_POWERSAVE 0xC0 #define HWP_EPP_POWERSAVE 0xFF -#define HWP_ACTIVITY_WINDOW(x) ((unsigned long long)(x & 0xff3) << 32) -#define HWP_PACKAGE_CONTROL(x) ((unsigned long long)(x & 0x1) << 42) +#define HWP_ACTIVITY_WINDOW(x) ((u64)(x & 0xff3) << 32) +#define HWP_PACKAGE_CONTROL(x) ((u64)(x & 0x1) << 42) /* IA32_HWP_STATUS */ #define HWP_GUARANTEED_CHANGE(x) (x & 0x1) From 73bd1e01e98e71715aa060d40b8273ff6434e8d7 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:50 +0200 Subject: [PATCH 090/331] x86/msr: Use u64 in rdmsrl_amd_safe() and wrmsrl_amd_safe() Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: H. Peter Anvin Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/amd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 79569f72b8ee..b2ab2369c9cb 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -31,7 +31,7 @@ u16 invlpgb_count_max __ro_after_init; -static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) +static inline int rdmsrl_amd_safe(unsigned msr, u64 *p) { u32 gprs[8] = { 0 }; int err; @@ -49,7 +49,7 @@ static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) return err; } -static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val) +static inline int wrmsrl_amd_safe(unsigned msr, u64 val) { u32 gprs[8] = { 0 }; From cd905826cbc833b7494573998bd1c407dfa7924f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:51 +0200 Subject: [PATCH 091/331] x86/msr: Use u64 in rdmsrl_safe() and paravirt_read_pmc() The paravirt_read_pmc() result is in fact only loaded into an u64 variable. Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: H. Peter Anvin Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/paravirt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index c4c23190925c..c270ca050781 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -231,7 +231,7 @@ static inline void wrmsrl(unsigned msr, u64 val) _err; \ }) -static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) +static inline int rdmsrl_safe(unsigned msr, u64 *p) { int err; @@ -239,7 +239,7 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) return err; } -static inline unsigned long long paravirt_read_pmc(int counter) +static inline u64 paravirt_read_pmc(int counter) { return PVOP_CALL1(u64, cpu.read_pmc, counter); } From d8f8aad698b85f197c877ec51f8585e2b2abb195 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:52 +0200 Subject: [PATCH 092/331] x86/msr: Harmonize the prototype and definition of do_trace_rdpmc() In the first parameter of do_trace_rdpmc() is named 'msr': extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed); But in the definition it's 'counter': void do_trace_rdpmc(unsigned counter, u64 val, int failed) Use 'msr' in both cases, and change the type to u32. Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: H. Peter Anvin Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/lib/msr.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 8ee6fc633476..ec5c873a83a1 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -65,11 +65,11 @@ DECLARE_TRACEPOINT(write_msr); DECLARE_TRACEPOINT(rdpmc); extern void do_trace_write_msr(unsigned int msr, u64 val, int failed); extern void do_trace_read_msr(unsigned int msr, u64 val, int failed); -extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed); +extern void do_trace_rdpmc(u32 msr, u64 val, int failed); #else static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {} static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {} -static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {} +static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {} #endif /* diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c index 5a18ecc04a6c..20f5c36e5fde 100644 --- a/arch/x86/lib/msr.c +++ b/arch/x86/lib/msr.c @@ -136,9 +136,9 @@ void do_trace_read_msr(unsigned int msr, u64 val, int failed) EXPORT_SYMBOL(do_trace_read_msr); EXPORT_TRACEPOINT_SYMBOL(read_msr); -void do_trace_rdpmc(unsigned counter, u64 val, int failed) +void do_trace_rdpmc(u32 msr, u64 val, int failed) { - trace_rdpmc(counter, val, failed); + trace_rdpmc(msr, val, failed); } EXPORT_SYMBOL(do_trace_rdpmc); EXPORT_TRACEPOINT_SYMBOL(rdpmc); From d58c04cf1d702fd6e133e89130f11a2ccd9269fa Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:53 +0200 Subject: [PATCH 093/331] x86/msr: Standardize on 'u32' MSR indices in This is the customary type used for hardware ABIs. Suggested-by: Xin Li Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Linus Torvalds --- arch/x86/include/asm/msr.h | 29 ++++++++++++++--------------- arch/x86/lib/msr.c | 4 ++-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index ec5c873a83a1..2f5a661719a4 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -63,12 +63,12 @@ struct saved_msrs { DECLARE_TRACEPOINT(read_msr); DECLARE_TRACEPOINT(write_msr); DECLARE_TRACEPOINT(rdpmc); -extern void do_trace_write_msr(unsigned int msr, u64 val, int failed); -extern void do_trace_read_msr(unsigned int msr, u64 val, int failed); +extern void do_trace_write_msr(u32 msr, u64 val, int failed); +extern void do_trace_read_msr(u32 msr, u64 val, int failed); extern void do_trace_rdpmc(u32 msr, u64 val, int failed); #else -static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {} -static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {} +static inline void do_trace_write_msr(u32 msr, u64 val, int failed) {} +static inline void do_trace_read_msr(u32 msr, u64 val, int failed) {} static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {} #endif @@ -79,7 +79,7 @@ static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {} * think of extending them - you will be slapped with a stinking trout or a frozen * shark will reach you, wherever you are! You've been warned. */ -static __always_inline u64 __rdmsr(unsigned int msr) +static __always_inline u64 __rdmsr(u32 msr) { DECLARE_ARGS(val, low, high); @@ -91,7 +91,7 @@ static __always_inline u64 __rdmsr(unsigned int msr) return EAX_EDX_VAL(val, low, high); } -static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high) +static __always_inline void __wrmsr(u32 msr, u32 low, u32 high) { asm volatile("1: wrmsr\n" "2:\n" @@ -113,7 +113,7 @@ do { \ __wrmsr((msr), (u32)((u64)(val)), \ (u32)((u64)(val) >> 32)) -static inline u64 native_read_msr(unsigned int msr) +static inline u64 native_read_msr(u32 msr) { u64 val; @@ -125,8 +125,7 @@ static inline u64 native_read_msr(unsigned int msr) return val; } -static inline u64 native_read_msr_safe(unsigned int msr, - int *err) +static inline u64 native_read_msr_safe(u32 msr, int *err) { DECLARE_ARGS(val, low, high); @@ -142,7 +141,7 @@ static inline u64 native_read_msr_safe(unsigned int msr, /* Can be uninlined because referenced by paravirt */ static inline void notrace -native_write_msr(unsigned int msr, u32 low, u32 high) +native_write_msr(u32 msr, u32 low, u32 high) { __wrmsr(msr, low, high); @@ -152,7 +151,7 @@ native_write_msr(unsigned int msr, u32 low, u32 high) /* Can be uninlined because referenced by paravirt */ static inline int notrace -native_write_msr_safe(unsigned int msr, u32 low, u32 high) +native_write_msr_safe(u32 msr, u32 low, u32 high) { int err; @@ -251,7 +250,7 @@ do { \ (void)((high) = (u32)(__val >> 32)); \ } while (0) -static inline void wrmsr(unsigned int msr, u32 low, u32 high) +static inline void wrmsr(u32 msr, u32 low, u32 high) { native_write_msr(msr, low, high); } @@ -259,13 +258,13 @@ static inline void wrmsr(unsigned int msr, u32 low, u32 high) #define rdmsrl(msr, val) \ ((val) = native_read_msr((msr))) -static inline void wrmsrl(unsigned int msr, u64 val) +static inline void wrmsrl(u32 msr, u64 val) { native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); } /* wrmsr with exception handling */ -static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high) +static inline int wrmsr_safe(u32 msr, u32 low, u32 high) { return native_write_msr_safe(msr, low, high); } @@ -280,7 +279,7 @@ static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high) __err; \ }) -static inline int rdmsrl_safe(unsigned int msr, u64 *p) +static inline int rdmsrl_safe(u32 msr, u64 *p) { int err; diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c index 20f5c36e5fde..7b90f547b109 100644 --- a/arch/x86/lib/msr.c +++ b/arch/x86/lib/msr.c @@ -122,14 +122,14 @@ int msr_clear_bit(u32 msr, u8 bit) EXPORT_SYMBOL_GPL(msr_clear_bit); #ifdef CONFIG_TRACEPOINTS -void do_trace_write_msr(unsigned int msr, u64 val, int failed) +void do_trace_write_msr(u32 msr, u64 val, int failed) { trace_write_msr(msr, val, failed); } EXPORT_SYMBOL(do_trace_write_msr); EXPORT_TRACEPOINT_SYMBOL(write_msr); -void do_trace_read_msr(unsigned int msr, u64 val, int failed) +void do_trace_read_msr(u32 msr, u64 val, int failed) { trace_read_msr(msr, val, failed); } From c435e608cf59ffab815aa2571182dc8c50fe4112 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:54 +0200 Subject: [PATCH 094/331] x86/msr: Rename 'rdmsrl()' to 'rdmsrq()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/coco/sev/core.c | 2 +- arch/x86/events/amd/brs.c | 4 +-- arch/x86/events/amd/core.c | 4 +-- arch/x86/events/amd/ibs.c | 18 +++++------ arch/x86/events/amd/lbr.c | 8 ++--- arch/x86/events/amd/power.c | 8 ++--- arch/x86/events/amd/uncore.c | 2 +- arch/x86/events/core.c | 20 ++++++------ arch/x86/events/intel/core.c | 10 +++--- arch/x86/events/intel/cstate.c | 2 +- arch/x86/events/intel/knc.c | 6 ++-- arch/x86/events/intel/lbr.c | 14 ++++---- arch/x86/events/intel/p4.c | 4 +-- arch/x86/events/intel/p6.c | 4 +-- arch/x86/events/intel/pt.c | 12 +++---- arch/x86/events/intel/uncore.c | 2 +- arch/x86/events/intel/uncore_nhmex.c | 4 +-- arch/x86/events/intel/uncore_snb.c | 2 +- arch/x86/events/intel/uncore_snbep.c | 6 ++-- arch/x86/events/msr.c | 2 +- arch/x86/events/perf_event.h | 6 ++-- arch/x86/events/rapl.c | 6 ++-- arch/x86/events/zhaoxin/core.c | 6 ++-- arch/x86/hyperv/hv_apic.c | 2 +- arch/x86/hyperv/hv_init.c | 26 +++++++-------- arch/x86/hyperv/hv_spinlock.c | 6 ++-- arch/x86/include/asm/apic.h | 4 +-- arch/x86/include/asm/debugreg.h | 2 +- arch/x86/include/asm/fsgsbase.h | 2 +- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/include/asm/msr.h | 4 +-- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/kernel/apic/apic.c | 6 ++-- arch/x86/kernel/apic/apic_numachip.c | 6 ++-- arch/x86/kernel/cet.c | 2 +- arch/x86/kernel/cpu/amd.c | 8 ++--- arch/x86/kernel/cpu/aperfmperf.c | 8 ++--- arch/x86/kernel/cpu/bugs.c | 12 +++---- arch/x86/kernel/cpu/bus_lock.c | 8 ++--- arch/x86/kernel/cpu/common.c | 10 +++--- arch/x86/kernel/cpu/hygon.c | 4 +-- arch/x86/kernel/cpu/intel.c | 4 +-- arch/x86/kernel/cpu/intel_epb.c | 4 +-- arch/x86/kernel/cpu/mce/amd.c | 14 ++++---- arch/x86/kernel/cpu/mce/core.c | 6 ++-- arch/x86/kernel/cpu/mce/inject.c | 2 +- arch/x86/kernel/cpu/mce/intel.c | 18 +++++------ arch/x86/kernel/cpu/mshyperv.c | 6 ++-- arch/x86/kernel/cpu/resctrl/core.c | 2 +- arch/x86/kernel/cpu/resctrl/monitor.c | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 2 +- arch/x86/kernel/cpu/topology.c | 2 +- arch/x86/kernel/cpu/topology_amd.c | 4 +-- arch/x86/kernel/cpu/tsx.c | 10 +++--- arch/x86/kernel/cpu/umwait.c | 2 +- arch/x86/kernel/fpu/core.c | 2 +- arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/kvm.c | 2 +- arch/x86/kernel/mmconf-fam10h_64.c | 6 ++-- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/process_64.c | 14 ++++---- arch/x86/kernel/shstk.c | 4 +-- arch/x86/kernel/traps.c | 4 +-- arch/x86/kernel/tsc.c | 2 +- arch/x86/kernel/tsc_sync.c | 6 ++-- arch/x86/kvm/svm/svm.c | 6 ++-- arch/x86/kvm/vmx/nested.c | 4 +-- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/vmx/sgx.c | 6 ++-- arch/x86/kvm/vmx/vmx.c | 32 +++++++++---------- arch/x86/kvm/x86.c | 4 +-- arch/x86/lib/insn-eval.c | 6 ++-- arch/x86/mm/pat/memtype.c | 2 +- arch/x86/pci/amd_bus.c | 8 ++--- arch/x86/platform/olpc/olpc-xo1-rtc.c | 6 ++-- arch/x86/power/cpu.c | 10 +++--- arch/x86/realmode/init.c | 2 +- arch/x86/virt/svm/sev.c | 16 +++++----- arch/x86/xen/suspend.c | 2 +- drivers/cpufreq/acpi-cpufreq.c | 2 +- drivers/cpufreq/amd-pstate.c | 4 +-- drivers/cpufreq/e_powersaver.c | 4 +-- drivers/cpufreq/intel_pstate.c | 28 ++++++++-------- drivers/cpufreq/longhaul.c | 8 ++--- drivers/cpufreq/powernow-k7.c | 10 +++--- drivers/edac/amd64_edac.c | 6 ++-- drivers/idle/intel_idle.c | 24 +++++++------- drivers/mtd/nand/raw/cs553x_nand.c | 6 ++-- drivers/platform/x86/intel/ifs/load.c | 10 +++--- drivers/platform/x86/intel/ifs/runtest.c | 8 ++--- drivers/platform/x86/intel/pmc/cnp.c | 2 +- .../intel/speed_select_if/isst_if_mbox_msr.c | 6 ++-- .../intel/speed_select_if/isst_tpmi_core.c | 2 +- drivers/platform/x86/intel_ips.c | 20 ++++++------ drivers/thermal/intel/intel_hfi.c | 8 ++--- drivers/thermal/intel/therm_throt.c | 6 ++-- drivers/video/fbdev/geode/gxfb_core.c | 2 +- drivers/video/fbdev/geode/lxfb_ops.c | 12 +++---- drivers/video/fbdev/geode/suspend_gx.c | 8 ++--- drivers/video/fbdev/geode/video_gx.c | 8 ++--- include/hyperv/hvgdk_mini.h | 2 +- 101 files changed, 340 insertions(+), 340 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index b0c1a7a57497..b18a33fe8dd3 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -3278,7 +3278,7 @@ void __init snp_secure_tsc_init(void) return; setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); - rdmsrl(MSR_AMD64_GUEST_TSC_FREQ, tsc_freq_mhz); + rdmsrq(MSR_AMD64_GUEST_TSC_FREQ, tsc_freq_mhz); snp_tsc_freq_khz = (unsigned long)(tsc_freq_mhz * 1000); x86_platform.calibrate_cpu = securetsc_get_tsc_khz; diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index ec3427463382..86ef0fe8f435 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -325,7 +325,7 @@ void amd_brs_drain(void) u32 brs_idx = tos - i; u64 from, to; - rdmsrl(brs_to(brs_idx), to); + rdmsrq(brs_to(brs_idx), to); /* Entry does not belong to us (as marked by kernel) */ if (to == BRS_POISON) @@ -341,7 +341,7 @@ void amd_brs_drain(void) if (!amd_brs_match_plm(event, to)) continue; - rdmsrl(brs_from(brs_idx), from); + rdmsrq(brs_from(brs_idx), from); perf_clear_branch_entry_bitfields(br+nr); diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 30d6ceb4c8ad..94d6d2428f2a 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -659,7 +659,7 @@ static inline u64 amd_pmu_get_global_status(void) u64 status; /* PerfCntrGlobalStatus is read-only */ - rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status); + rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status); return status; } @@ -679,7 +679,7 @@ static bool amd_pmu_test_overflow_topbit(int idx) { u64 counter; - rdmsrl(x86_pmu_event_addr(idx), counter); + rdmsrq(x86_pmu_event_addr(idx), counter); return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1)); } diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 0252b7ea8bca..ca6cc90f4719 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -424,7 +424,7 @@ perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, * prev count manually on overflow. */ while (!perf_event_try_update(event, count, 64)) { - rdmsrl(event->hw.config_base, *config); + rdmsrq(event->hw.config_base, *config); count = perf_ibs->get_count(*config); } } @@ -513,7 +513,7 @@ static void perf_ibs_stop(struct perf_event *event, int flags) if (!stopping && (hwc->state & PERF_HES_UPTODATE)) return; - rdmsrl(hwc->config_base, config); + rdmsrq(hwc->config_base, config); if (stopping) { /* @@ -1256,7 +1256,7 @@ fail: hwc = &event->hw; msr = hwc->config_base; buf = ibs_data.regs; - rdmsrl(msr, *buf); + rdmsrq(msr, *buf); if (!(*buf++ & perf_ibs->valid_mask)) goto fail; @@ -1274,7 +1274,7 @@ fail: offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip); do { - rdmsrl(msr + offset, *buf++); + rdmsrq(msr + offset, *buf++); size++; offset = find_next_bit(perf_ibs->offset_mask, perf_ibs->offset_max, @@ -1304,17 +1304,17 @@ fail: if (event->attr.sample_type & PERF_SAMPLE_RAW) { if (perf_ibs == &perf_ibs_op) { if (ibs_caps & IBS_CAPS_BRNTRGT) { - rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); + rdmsrq(MSR_AMD64_IBSBRTARGET, *buf++); br_target_idx = size; size++; } if (ibs_caps & IBS_CAPS_OPDATA4) { - rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); + rdmsrq(MSR_AMD64_IBSOPDATA4, *buf++); size++; } } if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) { - rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++); + rdmsrq(MSR_AMD64_ICIBSEXTDCTL, *buf++); size++; } } @@ -1565,7 +1565,7 @@ static inline int ibs_eilvt_valid(void) preempt_disable(); - rdmsrl(MSR_AMD64_IBSCTL, val); + rdmsrq(MSR_AMD64_IBSCTL, val); offset = val & IBSCTL_LVT_OFFSET_MASK; if (!(val & IBSCTL_LVT_OFFSET_VALID)) { @@ -1680,7 +1680,7 @@ static inline int get_ibs_lvt_offset(void) { u64 val; - rdmsrl(MSR_AMD64_IBSCTL, val); + rdmsrq(MSR_AMD64_IBSCTL, val); if (!(val & IBSCTL_LVT_OFFSET_VALID)) return -EINVAL; diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index c06ccca96851..a1aecf5c04a0 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -73,7 +73,7 @@ static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx) { u64 val; - rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); + rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val); return val; } @@ -82,7 +82,7 @@ static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx) { u64 val; - rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); + rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); return val; } @@ -400,11 +400,11 @@ void amd_pmu_lbr_enable_all(void) } if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { - rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); } - rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); + rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); } diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c index 37d5b380516e..7e96f4652230 100644 --- a/arch/x86/events/amd/power.c +++ b/arch/x86/events/amd/power.c @@ -48,8 +48,8 @@ static void event_update(struct perf_event *event) prev_pwr_acc = hwc->pwr_acc; prev_ptsc = hwc->ptsc; - rdmsrl(MSR_F15H_CU_PWR_ACCUMULATOR, new_pwr_acc); - rdmsrl(MSR_F15H_PTSC, new_ptsc); + rdmsrq(MSR_F15H_CU_PWR_ACCUMULATOR, new_pwr_acc); + rdmsrq(MSR_F15H_PTSC, new_ptsc); /* * Calculate the CU power consumption over a time period, the unit of @@ -75,8 +75,8 @@ static void __pmu_event_start(struct perf_event *event) event->hw.state = 0; - rdmsrl(MSR_F15H_PTSC, event->hw.ptsc); - rdmsrl(MSR_F15H_CU_PWR_ACCUMULATOR, event->hw.pwr_acc); + rdmsrq(MSR_F15H_PTSC, event->hw.ptsc); + rdmsrq(MSR_F15H_CU_PWR_ACCUMULATOR, event->hw.pwr_acc); } static void pmu_event_start(struct perf_event *event, int mode) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 49c26ce2b115..4ddf5429a063 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -106,7 +106,7 @@ static void amd_uncore_read(struct perf_event *event) * read counts directly from the corresponding PERF_CTR. */ if (hwc->event_base_rdpmc < 0) - rdmsrl(hwc->event_base, new); + rdmsrq(hwc->event_base, new); else rdpmcl(hwc->event_base_rdpmc, new); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 6866cc5acb0b..a27f2e680070 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -693,7 +693,7 @@ void x86_pmu_disable_all(void) if (!test_bit(idx, cpuc->active_mask)) continue; - rdmsrl(x86_pmu_config_addr(idx), val); + rdmsrq(x86_pmu_config_addr(idx), val); if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) continue; val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; @@ -1550,10 +1550,10 @@ void perf_event_print_debug(void) return; if (x86_pmu.version >= 2) { - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); - rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); - rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); - rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); + rdmsrq(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); + rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status); + rdmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); + rdmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); pr_info("\n"); pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); @@ -1561,19 +1561,19 @@ void perf_event_print_debug(void) pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); if (pebs_constraints) { - rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); + rdmsrq(MSR_IA32_PEBS_ENABLE, pebs); pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); } if (x86_pmu.lbr_nr) { - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); } } pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) { - rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); - rdmsrl(x86_pmu_event_addr(idx), pmc_count); + rdmsrq(x86_pmu_config_addr(idx), pmc_ctrl); + rdmsrq(x86_pmu_event_addr(idx), pmc_count); prev_left = per_cpu(pmc_prev_left[idx], cpu); @@ -1587,7 +1587,7 @@ void perf_event_print_debug(void) for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) { if (fixed_counter_disabled(idx, cpuc->pmu)) continue; - rdmsrl(x86_pmu_fixed_ctr_addr(idx), pmc_count); + rdmsrq(x86_pmu_fixed_ctr_addr(idx), pmc_count); pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", cpu, idx, pmc_count); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 09d2d66c9f21..852b1ea9bc17 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2489,7 +2489,7 @@ static inline u64 intel_pmu_get_status(void) { u64 status; - rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); + rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status); return status; } @@ -5054,7 +5054,7 @@ static void update_pmu_cap(struct x86_hybrid_pmu *pmu) if (!intel_pmu_broken_perf_cap()) { /* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */ - rdmsrl(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities); + rdmsrq(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities); } } @@ -5202,7 +5202,7 @@ static void intel_pmu_cpu_starting(int cpu) if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics) { union perf_capabilities perf_cap; - rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities); + rdmsrq(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities); if (!perf_cap.perf_metrics) { x86_pmu.intel_cap.perf_metrics = 0; x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS); @@ -5627,7 +5627,7 @@ static bool check_msr(unsigned long msr, u64 mask) /* * Quirk only affects validation in wrmsr(), so wrmsrl()'s value - * should equal rdmsrl()'s even with the quirk. + * should equal rdmsrq()'s even with the quirk. */ if (val_new != val_tmp) return false; @@ -6642,7 +6642,7 @@ __init int intel_pmu_init(void) if (boot_cpu_has(X86_FEATURE_PDCM)) { u64 capabilities; - rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities); + rdmsrq(MSR_IA32_PERF_CAPABILITIES, capabilities); x86_pmu.intel_cap.capabilities = capabilities; } diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index ae4ec16156bb..56b1c391ccc7 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -320,7 +320,7 @@ static inline u64 cstate_pmu_read_counter(struct perf_event *event) { u64 val; - rdmsrl(event->hw.event_base, val); + rdmsrq(event->hw.event_base, val); return val; } diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index 034a1f6a457c..0ee32a0b5bfd 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -159,7 +159,7 @@ static void knc_pmu_disable_all(void) { u64 val; - rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); + rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); val &= ~(KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1); wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); } @@ -168,7 +168,7 @@ static void knc_pmu_enable_all(int added) { u64 val; - rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); + rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); val |= (KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1); wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); } @@ -200,7 +200,7 @@ static inline u64 knc_pmu_get_status(void) { u64 status; - rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_STATUS, status); + rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_STATUS, status); return status; } diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index f44c3d866f24..2b33aacb5983 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -139,7 +139,7 @@ static void __intel_pmu_lbr_enable(bool pmi) if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) wrmsrl(MSR_LBR_SELECT, lbr_select); - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); orig_debugctl = debugctl; if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) @@ -209,7 +209,7 @@ static inline u64 intel_pmu_lbr_tos(void) { u64 tos; - rdmsrl(x86_pmu.lbr_tos, tos); + rdmsrq(x86_pmu.lbr_tos, tos); return tos; } @@ -302,7 +302,7 @@ static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) if (lbr) return lbr->from; - rdmsrl(x86_pmu.lbr_from + idx, val); + rdmsrq(x86_pmu.lbr_from + idx, val); return lbr_from_signext_quirk_rd(val); } @@ -314,7 +314,7 @@ static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr) if (lbr) return lbr->to; - rdmsrl(x86_pmu.lbr_to + idx, val); + rdmsrq(x86_pmu.lbr_to + idx, val); return val; } @@ -326,7 +326,7 @@ static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr) if (lbr) return lbr->info; - rdmsrl(x86_pmu.lbr_info + idx, val); + rdmsrq(x86_pmu.lbr_info + idx, val); return val; } @@ -475,7 +475,7 @@ void intel_pmu_lbr_save(void *ctx) task_ctx->tos = tos; if (cpuc->lbr_select) - rdmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel); + rdmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); } static void intel_pmu_arch_lbr_save(void *ctx) @@ -752,7 +752,7 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) u64 lbr; } msr_lastbranch; - rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); + rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); perf_clear_branch_entry_bitfields(br); diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index c85a9fc44355..a0eb63b6c859 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -859,7 +859,7 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc) u64 v; /* an official way for overflow indication */ - rdmsrl(hwc->config_base, v); + rdmsrq(hwc->config_base, v); if (v & P4_CCCR_OVF) { wrmsrl(hwc->config_base, v & ~P4_CCCR_OVF); return 1; @@ -872,7 +872,7 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc) * the counter has reached zero value and continued counting before * real NMI signal was received: */ - rdmsrl(hwc->event_base, v); + rdmsrq(hwc->event_base, v); if (!(v & ARCH_P4_UNFLAGGED_BIT)) return 1; diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c index 65b45e9d7016..701f8a8ab0ba 100644 --- a/arch/x86/events/intel/p6.c +++ b/arch/x86/events/intel/p6.c @@ -142,7 +142,7 @@ static void p6_pmu_disable_all(void) u64 val; /* p6 only has one enable register */ - rdmsrl(MSR_P6_EVNTSEL0, val); + rdmsrq(MSR_P6_EVNTSEL0, val); val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; wrmsrl(MSR_P6_EVNTSEL0, val); } @@ -152,7 +152,7 @@ static void p6_pmu_enable_all(int added) unsigned long val; /* p6 only has one enable register */ - rdmsrl(MSR_P6_EVNTSEL0, val); + rdmsrq(MSR_P6_EVNTSEL0, val); val |= ARCH_PERFMON_EVENTSEL_ENABLE; wrmsrl(MSR_P6_EVNTSEL0, val); } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index fa37565f6418..b8212d8d0174 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -194,7 +194,7 @@ static int __init pt_pmu_hw_init(void) int ret; long i; - rdmsrl(MSR_PLATFORM_INFO, reg); + rdmsrq(MSR_PLATFORM_INFO, reg); pt_pmu.max_nonturbo_ratio = (reg & 0xff00) >> 8; /* @@ -230,7 +230,7 @@ static int __init pt_pmu_hw_init(void) * "IA32_VMX_MISC[bit 14]" being 1 means PT can trace * post-VMXON. */ - rdmsrl(MSR_IA32_VMX_MISC, reg); + rdmsrq(MSR_IA32_VMX_MISC, reg); if (reg & BIT(14)) pt_pmu.vmx = true; } @@ -926,7 +926,7 @@ static void pt_handle_status(struct pt *pt) int advance = 0; u64 status; - rdmsrl(MSR_IA32_RTIT_STATUS, status); + rdmsrq(MSR_IA32_RTIT_STATUS, status); if (status & RTIT_STATUS_ERROR) { pr_err_ratelimited("ToPA ERROR encountered, trying to recover\n"); @@ -985,12 +985,12 @@ static void pt_read_offset(struct pt_buffer *buf) struct topa_page *tp; if (!buf->single) { - rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base); + rdmsrq(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base); tp = phys_to_virt(pt->output_base); buf->cur = &tp->topa; } - rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask); + rdmsrq(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask); /* offset within current output region */ buf->output_off = pt->output_mask >> 32; /* index of current output region within this table */ @@ -1611,7 +1611,7 @@ static void pt_event_start(struct perf_event *event, int mode) * PMI might have just cleared these, so resume_allowed * must be checked again also. */ - rdmsrl(MSR_IA32_RTIT_STATUS, status); + rdmsrq(MSR_IA32_RTIT_STATUS, status); if (!(status & (RTIT_STATUS_TRIGGEREN | RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED)) && diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index a34e50fc4a8f..d6070529c58e 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -150,7 +150,7 @@ u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *eve { u64 count; - rdmsrl(event->hw.event_base, count); + rdmsrq(event->hw.event_base, count); return count; } diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index 466833478e81..e6fccad9708d 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -214,7 +214,7 @@ static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box) u64 config; if (msr) { - rdmsrl(msr, config); + rdmsrq(msr, config); config &= ~((1ULL << uncore_num_counters(box)) - 1); /* WBox has a fixed counter */ if (uncore_msr_fixed_ctl(box)) @@ -229,7 +229,7 @@ static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box) u64 config; if (msr) { - rdmsrl(msr, config); + rdmsrq(msr, config); config |= (1ULL << uncore_num_counters(box)) - 1; /* WBox has a fixed counter */ if (uncore_msr_fixed_ctl(box)) diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index edb7fd50efe0..66dadf58799b 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -504,7 +504,7 @@ static int icl_get_cbox_num(void) { u64 num_boxes; - rdmsrl(ICL_UNC_CBO_CONFIG, num_boxes); + rdmsrq(ICL_UNC_CBO_CONFIG, num_boxes); return num_boxes & ICL_UNC_NUM_CBO_MASK; } diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 60973c209c0e..e498485e0ae9 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -618,7 +618,7 @@ static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) msr = uncore_msr_box_ctl(box); if (msr) { - rdmsrl(msr, config); + rdmsrq(msr, config); config |= SNBEP_PMON_BOX_CTL_FRZ; wrmsrl(msr, config); } @@ -631,7 +631,7 @@ static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box) msr = uncore_msr_box_ctl(box); if (msr) { - rdmsrl(msr, config); + rdmsrq(msr, config); config &= ~SNBEP_PMON_BOX_CTL_FRZ; wrmsrl(msr, config); } @@ -6572,7 +6572,7 @@ void spr_uncore_cpu_init(void) * of UNCORE_SPR_CHA) is incorrect on some SPR variants because of a * firmware bug. Using the value from SPR_MSR_UNC_CBO_CONFIG to replace it. */ - rdmsrl(SPR_MSR_UNC_CBO_CONFIG, num_cbo); + rdmsrq(SPR_MSR_UNC_CBO_CONFIG, num_cbo); /* * The MSR doesn't work on the EMR XCC, but the firmware bug doesn't impact * the EMR XCC. Don't let the value from the MSR replace the existing value. diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 45b1866ff051..8970ecef87c5 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -231,7 +231,7 @@ static inline u64 msr_read_counter(struct perf_event *event) u64 now; if (event->hw.event_base) - rdmsrl(event->hw.event_base, now); + rdmsrq(event->hw.event_base, now); else now = rdtsc_ordered(); diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 2c0ce0e9545e..34b6d35e76e8 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1394,11 +1394,11 @@ static __always_inline void __amd_pmu_lbr_disable(void) { u64 dbg_ctl, dbg_extn_cfg; - rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); + rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { - rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); } } @@ -1543,7 +1543,7 @@ static __always_inline void __intel_pmu_lbr_disable(void) { u64 debugctl; - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); } diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index 8ddace8cea96..b7c256a2e201 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -192,7 +192,7 @@ static inline unsigned int get_rapl_pmu_idx(int cpu, int scope) static inline u64 rapl_read_counter(struct perf_event *event) { u64 raw; - rdmsrl(event->hw.event_base, raw); + rdmsrq(event->hw.event_base, raw); return raw; } @@ -221,7 +221,7 @@ static u64 rapl_event_update(struct perf_event *event) prev_raw_count = local64_read(&hwc->prev_count); do { - rdmsrl(event->hw.event_base, new_raw_count); + rdmsrq(event->hw.event_base, new_raw_count); } while (!local64_try_cmpxchg(&hwc->prev_count, &prev_raw_count, new_raw_count)); @@ -610,7 +610,7 @@ static int rapl_check_hw_unit(void) u64 msr_rapl_power_unit_bits; int i; - /* protect rdmsrl() to handle virtualization */ + /* protect rdmsrq() to handle virtualization */ if (rdmsrl_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits)) return -1; for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index 2fd9b0cf9a5e..67ff23f6bcba 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -266,7 +266,7 @@ static inline u64 zhaoxin_pmu_get_status(void) { u64 status; - rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); + rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status); return status; } @@ -293,7 +293,7 @@ static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc) mask = 0xfULL << (idx * 4); - rdmsrl(hwc->config_base, ctrl_val); + rdmsrq(hwc->config_base, ctrl_val); ctrl_val &= ~mask; wrmsrl(hwc->config_base, ctrl_val); } @@ -329,7 +329,7 @@ static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc) bits <<= (idx * 4); mask = 0xfULL << (idx * 4); - rdmsrl(hwc->config_base, ctrl_val); + rdmsrq(hwc->config_base, ctrl_val); ctrl_val &= ~mask; ctrl_val |= bits; wrmsrl(hwc->config_base, ctrl_val); diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index 6d91ac5f9836..d1d6aa5fb17b 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -37,7 +37,7 @@ static u64 hv_apic_icr_read(void) { u64 reg_val; - rdmsrl(HV_X64_MSR_ICR, reg_val); + rdmsrq(HV_X64_MSR_ICR, reg_val); return reg_val; } diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index ddeb40930bc8..7c58affa129b 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -62,7 +62,7 @@ static int hyperv_init_ghcb(void) * returned by MSR_AMD64_SEV_ES_GHCB is above shared * memory boundary and map it here. */ - rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa); + rdmsrq(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa); /* Mask out vTOM bit. ioremap_cache() maps decrypted */ ghcb_gpa &= ~ms_hyperv.shared_gpa_boundary; @@ -95,7 +95,7 @@ static int hv_cpu_init(unsigned int cpu) * For root partition we get the hypervisor provided VP assist * page, instead of allocating a new page. */ - rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); + rdmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); *hvp = memremap(msr.pfn << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT, PAGE_SIZE, MEMREMAP_WB); } else { @@ -140,7 +140,7 @@ static void hv_reenlightenment_notify(struct work_struct *dummy) { struct hv_tsc_emulation_status emu_status; - rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); + rdmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); /* Don't issue the callback if TSC accesses are not emulated */ if (hv_reenlightenment_cb && emu_status.inprogress) @@ -153,11 +153,11 @@ void hyperv_stop_tsc_emulation(void) u64 freq; struct hv_tsc_emulation_status emu_status; - rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); + rdmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); emu_status.inprogress = 0; wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); - rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); + rdmsrq(HV_X64_MSR_TSC_FREQUENCY, freq); tsc_khz = div64_u64(freq, 1000); } EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation); @@ -217,7 +217,7 @@ void clear_hv_tscchange_cb(void) if (!hv_reenlightenment_available()) return; - rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); + rdmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); re_ctrl.enabled = 0; wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); @@ -251,7 +251,7 @@ static int hv_cpu_die(unsigned int cpu) */ memunmap(hv_vp_assist_page[cpu]); hv_vp_assist_page[cpu] = NULL; - rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); + rdmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); msr.enable = 0; } wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); @@ -260,7 +260,7 @@ static int hv_cpu_die(unsigned int cpu) if (hv_reenlightenment_cb == NULL) return 0; - rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); + rdmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); if (re_ctrl.target_vp == hv_vp_index[cpu]) { /* * Reassign reenlightenment notifications to some other online @@ -331,7 +331,7 @@ static int hv_suspend(void) hv_hypercall_pg = NULL; /* Disable the hypercall page in the hypervisor */ - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hypercall_msr.enable = 0; wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); @@ -348,7 +348,7 @@ static void hv_resume(void) WARN_ON(ret); /* Re-enable the hypercall page */ - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hypercall_msr.enable = 1; hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg_saved); @@ -515,7 +515,7 @@ void __init hyperv_init(void) if (hv_hypercall_pg == NULL) goto clean_guest_os_id; - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hypercall_msr.enable = 1; if (hv_root_partition()) { @@ -667,7 +667,7 @@ void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die) return; panic_reported = true; - rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); + rdmsrq(HV_X64_MSR_GUEST_OS_ID, guest_id); wrmsrl(HV_X64_MSR_CRASH_P0, err); wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); @@ -701,7 +701,7 @@ bool hv_is_hyperv_initialized(void) * that the hypercall page is setup */ hypercall_msr.as_uint64 = 0; - rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); return hypercall_msr.enable; } diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 151e851bef09..626f6d4d6253 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -39,18 +39,18 @@ static void hv_qlock_wait(u8 *byte, u8 val) * To prevent a race against the unlock path it is required to * disable interrupts before accessing the HV_X64_MSR_GUEST_IDLE * MSR. Otherwise, if the IPI from hv_qlock_kick() arrives between - * the lock value check and the rdmsrl() then the vCPU might be put + * the lock value check and the rdmsrq() then the vCPU might be put * into 'idle' state by the hypervisor and kept in that state for * an unspecified amount of time. */ local_irq_save(flags); /* - * Only issue the rdmsrl() when the lock state has not changed. + * Only issue the rdmsrq() when the lock state has not changed. */ if (READ_ONCE(*byte) == val) { unsigned long msr_val; - rdmsrl(HV_X64_MSR_GUEST_IDLE, msr_val); + rdmsrq(HV_X64_MSR_GUEST_IDLE, msr_val); (void)msr_val; } diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index c903d358405d..45819cfc043b 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -224,7 +224,7 @@ static inline u32 native_apic_msr_read(u32 reg) if (reg == APIC_DFR) return -1; - rdmsrl(APIC_BASE_MSR + (reg >> 4), msr); + rdmsrq(APIC_BASE_MSR + (reg >> 4), msr); return (u32)msr; } @@ -237,7 +237,7 @@ static inline u64 native_x2apic_icr_read(void) { unsigned long val; - rdmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), val); + rdmsrq(APIC_BASE_MSR + (APIC_ICR >> 4), val); return val; } diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h index fdbbbfec745a..0e703c0ef192 100644 --- a/arch/x86/include/asm/debugreg.h +++ b/arch/x86/include/asm/debugreg.h @@ -169,7 +169,7 @@ static inline unsigned long get_debugctlmsr(void) if (boot_cpu_data.x86 < 6) return 0; #endif - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctlmsr); return debugctlmsr; } diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h index 02f239569b93..d7708f2fd68f 100644 --- a/arch/x86/include/asm/fsgsbase.h +++ b/arch/x86/include/asm/fsgsbase.h @@ -60,7 +60,7 @@ static inline unsigned long x86_fsbase_read_cpu(void) if (boot_cpu_has(X86_FEATURE_FSGSBASE)) fsbase = rdfsbase(); else - rdmsrl(MSR_FS_BASE, fsbase); + rdmsrq(MSR_FS_BASE, fsbase); return fsbase; } diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a884ab544335..9af20b3c0f1d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -2272,7 +2272,7 @@ static inline unsigned long read_msr(unsigned long msr) { u64 value; - rdmsrl(msr, value); + rdmsrq(msr, value); return value; } #endif diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 2f5a661719a4..b35513a81b24 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -255,7 +255,7 @@ static inline void wrmsr(u32 msr, u32 low, u32 high) native_write_msr(msr, low, high); } -#define rdmsrl(msr, val) \ +#define rdmsrq(msr, val) \ ((val) = native_read_msr((msr))) static inline void wrmsrl(u32 msr, u64 val) @@ -352,7 +352,7 @@ static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) } static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { - rdmsrl(msr_no, *q); + rdmsrq(msr_no, *q); return 0; } static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index c270ca050781..1cda83bbd5b7 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -209,7 +209,7 @@ do { \ paravirt_write_msr(msr, val1, val2); \ } while (0) -#define rdmsrl(msr, val) \ +#define rdmsrq(msr, val) \ do { \ val = paravirt_read_msr(msr); \ } while (0) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 62584a347931..dc9af058e5a9 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1694,7 +1694,7 @@ static bool x2apic_hw_locked(void) x86_arch_cap_msr = x86_read_arch_cap_msr(); if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) { - rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr); + rdmsrq(MSR_IA32_XAPIC_DISABLE_STATUS, msr); return (msr & LEGACY_XAPIC_DISABLED); } return false; @@ -1707,7 +1707,7 @@ static void __x2apic_disable(void) if (!boot_cpu_has(X86_FEATURE_APIC)) return; - rdmsrl(MSR_IA32_APICBASE, msr); + rdmsrq(MSR_IA32_APICBASE, msr); if (!(msr & X2APIC_ENABLE)) return; /* Disable xapic and x2apic first and then reenable xapic mode */ @@ -1720,7 +1720,7 @@ static void __x2apic_enable(void) { u64 msr; - rdmsrl(MSR_IA32_APICBASE, msr); + rdmsrq(MSR_IA32_APICBASE, msr); if (msr & X2APIC_ENABLE) return; wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 16410f087b7a..fcfef701c17a 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -31,7 +31,7 @@ static u32 numachip1_get_apic_id(u32 x) unsigned int id = (x >> 24) & 0xff; if (static_cpu_has(X86_FEATURE_NODEID_MSR)) { - rdmsrl(MSR_FAM10H_NODE_ID, value); + rdmsrq(MSR_FAM10H_NODE_ID, value); id |= (value << 2) & 0xff00; } @@ -42,7 +42,7 @@ static u32 numachip2_get_apic_id(u32 x) { u64 mcfg; - rdmsrl(MSR_FAM10H_MMIO_CONF_BASE, mcfg); + rdmsrq(MSR_FAM10H_MMIO_CONF_BASE, mcfg); return ((mcfg >> (28 - 8)) & 0xfff00) | (x >> 24); } @@ -150,7 +150,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node) /* Account for nodes per socket in multi-core-module processors */ if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) { - rdmsrl(MSR_FAM10H_NODE_ID, val); + rdmsrq(MSR_FAM10H_NODE_ID, val); nodes = ((val >> 3) & 7) + 1; } diff --git a/arch/x86/kernel/cet.c b/arch/x86/kernel/cet.c index 303bf74d175b..d897aadd1d44 100644 --- a/arch/x86/kernel/cet.c +++ b/arch/x86/kernel/cet.c @@ -55,7 +55,7 @@ static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code) * will be whatever is live in userspace. So read the SSP before enabling * interrupts so locking the fpregs to do it later is not required. */ - rdmsrl(MSR_IA32_PL3_SSP, ssp); + rdmsrq(MSR_IA32_PL3_SSP, ssp); cond_local_irq_enable(regs); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index b2ab2369c9cb..7c7eca7451b6 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -383,7 +383,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) (c->x86 == 0x10 && c->x86_model >= 0x2)) { u64 val; - rdmsrl(MSR_K7_HWCR, val); + rdmsrq(MSR_K7_HWCR, val); if (!(val & BIT(24))) pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n"); } @@ -508,7 +508,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) */ if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) { /* Check if memory encryption is enabled */ - rdmsrl(MSR_AMD64_SYSCFG, msr); + rdmsrq(MSR_AMD64_SYSCFG, msr); if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) goto clear_all; @@ -525,7 +525,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c) if (!sme_me_mask) setup_clear_cpu_cap(X86_FEATURE_SME); - rdmsrl(MSR_K7_HWCR, msr); + rdmsrq(MSR_K7_HWCR, msr); if (!(msr & MSR_K7_HWCR_SMMLOCK)) goto clear_sev; @@ -1014,7 +1014,7 @@ static void init_amd(struct cpuinfo_x86 *c) init_amd_cacheinfo(c); if (cpu_has(c, X86_FEATURE_SVM)) { - rdmsrl(MSR_VM_CR, vm_cr); + rdmsrq(MSR_VM_CR, vm_cr); if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) { pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n"); clear_cpu_cap(c, X86_FEATURE_SVM); diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index 6cf31a1649c4..dca78650d99e 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -40,8 +40,8 @@ static void init_counter_refs(void) { u64 aperf, mperf; - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); this_cpu_write(cpu_samples.aperf, aperf); this_cpu_write(cpu_samples.mperf, mperf); @@ -474,8 +474,8 @@ void arch_scale_freq_tick(void) if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF)) return; - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); acnt = aperf - s->aperf; mcnt = mperf - s->mperf; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 4386aa6c69e1..78ffe0ebdff2 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -140,7 +140,7 @@ void __init cpu_select_mitigations(void) * init code as it is not enumerated and depends on the family. */ if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { - rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); + rdmsrq(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); /* * Previously running kernel (kexec), may have some controls @@ -656,7 +656,7 @@ void update_srbds_msr(void) if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) return; - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); switch (srbds_mitigation) { case SRBDS_MITIGATION_OFF: @@ -776,7 +776,7 @@ void update_gds_msr(void) switch (gds_mitigation) { case GDS_MITIGATION_OFF: - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); mcu_ctrl |= GDS_MITG_DIS; break; case GDS_MITIGATION_FULL_LOCKED: @@ -786,7 +786,7 @@ void update_gds_msr(void) * CPUs. */ case GDS_MITIGATION_FULL: - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); mcu_ctrl &= ~GDS_MITG_DIS; break; case GDS_MITIGATION_FORCE: @@ -802,7 +802,7 @@ void update_gds_msr(void) * GDS_MITG_DIS will be ignored if this processor is locked but the boot * processor was not. */ - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); } @@ -841,7 +841,7 @@ static void __init gds_select_mitigation(void) if (gds_mitigation == GDS_MITIGATION_FORCE) gds_mitigation = GDS_MITIGATION_FULL; - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); if (mcu_ctrl & GDS_MITG_LOCKED) { if (gds_mitigation == GDS_MITIGATION_OFF) pr_warn("Mitigation locked. Disable failed.\n"); diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index 237faf7e700c..725072b82816 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -103,7 +103,7 @@ static bool split_lock_verify_msr(bool on) ctrl &= ~MSR_TEST_CTRL_SPLIT_LOCK_DETECT; if (wrmsrl_safe(MSR_TEST_CTRL, ctrl)) return false; - rdmsrl(MSR_TEST_CTRL, tmp); + rdmsrq(MSR_TEST_CTRL, tmp); return ctrl == tmp; } @@ -137,7 +137,7 @@ static void __init __split_lock_setup(void) return; } - rdmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache); + rdmsrq(MSR_TEST_CTRL, msr_test_ctrl_cache); if (!split_lock_verify_msr(true)) { pr_info("MSR access failed: Disabled\n"); @@ -297,7 +297,7 @@ void bus_lock_init(void) if (!boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) return; - rdmsrl(MSR_IA32_DEBUGCTLMSR, val); + rdmsrq(MSR_IA32_DEBUGCTLMSR, val); if ((boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT) && (sld_state == sld_warn || sld_state == sld_fatal)) || @@ -375,7 +375,7 @@ static void __init split_lock_setup(struct cpuinfo_x86 *c) * MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT is. All CPUs that set * it have split lock detection. */ - rdmsrl(MSR_IA32_CORE_CAPS, ia32_core_caps); + rdmsrq(MSR_IA32_CORE_CAPS, ia32_core_caps); if (ia32_core_caps & MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT) goto supported; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 12126adbc3a9..c3c0ba239828 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -562,7 +562,7 @@ __noendbr u64 ibt_save(bool disable) u64 msr = 0; if (cpu_feature_enabled(X86_FEATURE_IBT)) { - rdmsrl(MSR_IA32_S_CET, msr); + rdmsrq(MSR_IA32_S_CET, msr); if (disable) wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN); } @@ -575,7 +575,7 @@ __noendbr void ibt_restore(u64 save) u64 msr; if (cpu_feature_enabled(X86_FEATURE_IBT)) { - rdmsrl(MSR_IA32_S_CET, msr); + rdmsrq(MSR_IA32_S_CET, msr); msr &= ~CET_ENDBR_EN; msr |= (save & CET_ENDBR_EN); wrmsrl(MSR_IA32_S_CET, msr); @@ -1288,7 +1288,7 @@ u64 x86_read_arch_cap_msr(void) u64 x86_arch_cap_msr = 0; if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) - rdmsrl(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr); + rdmsrq(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr); return x86_arch_cap_msr; } @@ -1749,10 +1749,10 @@ static bool detect_null_seg_behavior(void) */ unsigned long old_base, tmp; - rdmsrl(MSR_FS_BASE, old_base); + rdmsrq(MSR_FS_BASE, old_base); wrmsrl(MSR_FS_BASE, 1); loadsegment(fs, 0); - rdmsrl(MSR_FS_BASE, tmp); + rdmsrq(MSR_FS_BASE, tmp); wrmsrl(MSR_FS_BASE, old_base); return tmp == 0; } diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 6af4a4a90a52..10eeda3ba6ae 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -96,7 +96,7 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { u64 val; - rdmsrl(MSR_K7_HWCR, val); + rdmsrq(MSR_K7_HWCR, val); if (!(val & BIT(24))) pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n"); } @@ -194,7 +194,7 @@ static void init_hygon(struct cpuinfo_x86 *c) init_hygon_cacheinfo(c); if (cpu_has(c, X86_FEATURE_SVM)) { - rdmsrl(MSR_VM_CR, vm_cr); + rdmsrq(MSR_VM_CR, vm_cr); if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) { pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n"); clear_cpu_cap(c, X86_FEATURE_SVM); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index cdc9813871ef..e9a6e0d00819 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -157,7 +157,7 @@ static void detect_tme_early(struct cpuinfo_x86 *c) u64 tme_activate; int keyid_bits; - rdmsrl(MSR_IA32_TME_ACTIVATE, tme_activate); + rdmsrq(MSR_IA32_TME_ACTIVATE, tme_activate); if (!TME_ACTIVATE_LOCKED(tme_activate) || !TME_ACTIVATE_ENABLED(tme_activate)) { pr_info_once("x86/tme: not enabled by BIOS\n"); @@ -299,7 +299,7 @@ static void early_init_intel(struct cpuinfo_x86 *c) * string flag and enhanced fast string capabilities accordingly. */ if (c->x86_vfm >= INTEL_PENTIUM_M_DOTHAN) { - rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable); + rdmsrq(MSR_IA32_MISC_ENABLE, misc_enable); if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) { /* X86_FEATURE_ERMS is set based on CPUID */ set_cpu_cap(c, X86_FEATURE_REP_GOOD); diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index 30b1d63b97f3..6e6af07fc61f 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -79,7 +79,7 @@ static int intel_epb_save(void) { u64 epb; - rdmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); + rdmsrq(MSR_IA32_ENERGY_PERF_BIAS, epb); /* * Ensure that saved_epb will always be nonzero after this write even if * the EPB value read from the MSR is 0. @@ -94,7 +94,7 @@ static void intel_epb_restore(void) u64 val = this_cpu_read(saved_epb); u64 epb; - rdmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); + rdmsrq(MSR_IA32_ENERGY_PERF_BIAS, epb); if (val) { val &= EPB_MASK; } else { diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 1075a90141da..c29165f41bb6 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -662,7 +662,7 @@ static void disable_err_thresholding(struct cpuinfo_x86 *c, unsigned int bank) return; } - rdmsrl(MSR_K7_HWCR, hwcr); + rdmsrq(MSR_K7_HWCR, hwcr); /* McStatusWrEn has to be set */ need_toggle = !(hwcr & BIT(18)); @@ -805,12 +805,12 @@ static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc) } if (mce_flags.smca) { - rdmsrl(MSR_AMD64_SMCA_MCx_IPID(bank), m->ipid); + rdmsrq(MSR_AMD64_SMCA_MCx_IPID(bank), m->ipid); if (m->status & MCI_STATUS_SYNDV) { - rdmsrl(MSR_AMD64_SMCA_MCx_SYND(bank), m->synd); - rdmsrl(MSR_AMD64_SMCA_MCx_SYND1(bank), err.vendor.amd.synd1); - rdmsrl(MSR_AMD64_SMCA_MCx_SYND2(bank), err.vendor.amd.synd2); + rdmsrq(MSR_AMD64_SMCA_MCx_SYND(bank), m->synd); + rdmsrq(MSR_AMD64_SMCA_MCx_SYND1(bank), err.vendor.amd.synd1); + rdmsrq(MSR_AMD64_SMCA_MCx_SYND2(bank), err.vendor.amd.synd2); } } @@ -834,12 +834,12 @@ _log_error_bank(unsigned int bank, u32 msr_stat, u32 msr_addr, u64 misc) { u64 status, addr = 0; - rdmsrl(msr_stat, status); + rdmsrq(msr_stat, status); if (!(status & MCI_STATUS_VAL)) return false; if (status & MCI_STATUS_ADDRV) - rdmsrl(msr_addr, addr); + rdmsrq(msr_addr, addr); __log_error(bank, status, addr, misc); diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index f6fd71b64b66..a71228d9a4f4 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1822,7 +1822,7 @@ static void __mcheck_cpu_cap_init(void) u64 cap; u8 b; - rdmsrl(MSR_IA32_MCG_CAP, cap); + rdmsrq(MSR_IA32_MCG_CAP, cap); b = cap & MCG_BANKCNT_MASK; @@ -1863,7 +1863,7 @@ static void __mcheck_cpu_init_generic(void) cr4_set_bits(X86_CR4_MCE); - rdmsrl(MSR_IA32_MCG_CAP, cap); + rdmsrq(MSR_IA32_MCG_CAP, cap); if (cap & MCG_CTL_P) wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); } @@ -1905,7 +1905,7 @@ static void __mcheck_cpu_check_banks(void) if (!b->init) continue; - rdmsrl(mca_msr_reg(i, MCA_CTL), msrval); + rdmsrq(mca_msr_reg(i, MCA_CTL), msrval); b->init = !!msrval; } } diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 06e3cf7229ce..08d52af7a8ad 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -741,7 +741,7 @@ static void check_hw_inj_possible(void) u64 status = MCI_STATUS_VAL, ipid; /* Check whether bank is populated */ - rdmsrl(MSR_AMD64_SMCA_MCx_IPID(bank), ipid); + rdmsrq(MSR_AMD64_SMCA_MCx_IPID(bank), ipid); if (!ipid) continue; diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index f863df0ff42c..9fda956d4e48 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -94,7 +94,7 @@ static bool cmci_supported(int *banks) if (!boot_cpu_has(X86_FEATURE_APIC) || lapic_get_maxlvt() < 6) return false; - rdmsrl(MSR_IA32_MCG_CAP, cap); + rdmsrq(MSR_IA32_MCG_CAP, cap); *banks = min_t(unsigned, MAX_NR_BANKS, cap & MCG_BANKCNT_MASK); return !!(cap & MCG_CMCI_P); } @@ -106,7 +106,7 @@ static bool lmce_supported(void) if (mca_cfg.lmce_disabled) return false; - rdmsrl(MSR_IA32_MCG_CAP, tmp); + rdmsrq(MSR_IA32_MCG_CAP, tmp); /* * LMCE depends on recovery support in the processor. Hence both @@ -123,7 +123,7 @@ static bool lmce_supported(void) * WARN if the MSR isn't locked as init_ia32_feat_ctl() unconditionally * locks the MSR in the event that it wasn't already locked by BIOS. */ - rdmsrl(MSR_IA32_FEAT_CTL, tmp); + rdmsrq(MSR_IA32_FEAT_CTL, tmp); if (WARN_ON_ONCE(!(tmp & FEAT_CTL_LOCKED))) return false; @@ -141,7 +141,7 @@ static void cmci_set_threshold(int bank, int thresh) u64 val; raw_spin_lock_irqsave(&cmci_discover_lock, flags); - rdmsrl(MSR_IA32_MCx_CTL2(bank), val); + rdmsrq(MSR_IA32_MCx_CTL2(bank), val); val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK; wrmsrl(MSR_IA32_MCx_CTL2(bank), val | thresh); raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); @@ -184,7 +184,7 @@ static bool cmci_skip_bank(int bank, u64 *val) if (test_bit(bank, mce_banks_ce_disabled)) return true; - rdmsrl(MSR_IA32_MCx_CTL2(bank), *val); + rdmsrq(MSR_IA32_MCx_CTL2(bank), *val); /* Already owned by someone else? */ if (*val & MCI_CTL2_CMCI_EN) { @@ -233,7 +233,7 @@ static void cmci_claim_bank(int bank, u64 val, int bios_zero_thresh, int *bios_w val |= MCI_CTL2_CMCI_EN; wrmsrl(MSR_IA32_MCx_CTL2(bank), val); - rdmsrl(MSR_IA32_MCx_CTL2(bank), val); + rdmsrq(MSR_IA32_MCx_CTL2(bank), val); /* If the enable bit did not stick, this bank should be polled. */ if (!(val & MCI_CTL2_CMCI_EN)) { @@ -324,7 +324,7 @@ static void __cmci_disable_bank(int bank) if (!test_bit(bank, this_cpu_ptr(mce_banks_owned))) return; - rdmsrl(MSR_IA32_MCx_CTL2(bank), val); + rdmsrq(MSR_IA32_MCx_CTL2(bank), val); val &= ~MCI_CTL2_CMCI_EN; wrmsrl(MSR_IA32_MCx_CTL2(bank), val); __clear_bit(bank, this_cpu_ptr(mce_banks_owned)); @@ -430,7 +430,7 @@ void intel_init_lmce(void) if (!lmce_supported()) return; - rdmsrl(MSR_IA32_MCG_EXT_CTL, val); + rdmsrq(MSR_IA32_MCG_EXT_CTL, val); if (!(val & MCG_EXT_CTL_LMCE_EN)) wrmsrl(MSR_IA32_MCG_EXT_CTL, val | MCG_EXT_CTL_LMCE_EN); @@ -443,7 +443,7 @@ void intel_clear_lmce(void) if (!lmce_supported()) return; - rdmsrl(MSR_IA32_MCG_EXT_CTL, val); + rdmsrq(MSR_IA32_MCG_EXT_CTL, val); val &= ~MCG_EXT_CTL_LMCE_EN; wrmsrl(MSR_IA32_MCG_EXT_CTL, val); } diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 3e2533954675..b1b9b4c4997e 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -70,7 +70,7 @@ u64 hv_get_non_nested_msr(unsigned int reg) if (hv_is_synic_msr(reg) && ms_hyperv.paravisor_present) hv_ivm_msr_read(reg, &value); else - rdmsrl(reg, value); + rdmsrq(reg, value); return value; } EXPORT_SYMBOL_GPL(hv_get_non_nested_msr); @@ -345,7 +345,7 @@ static unsigned long hv_get_tsc_khz(void) { unsigned long freq; - rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq); + rdmsrq(HV_X64_MSR_TSC_FREQUENCY, freq); return freq / 1000; } @@ -541,7 +541,7 @@ static void __init ms_hyperv_init_platform(void) */ u64 hv_lapic_frequency; - rdmsrl(HV_X64_MSR_APIC_FREQUENCY, hv_lapic_frequency); + rdmsrq(HV_X64_MSR_APIC_FREQUENCY, hv_lapic_frequency); hv_lapic_frequency = div_u64(hv_lapic_frequency, HZ); lapic_timer_period = hv_lapic_frequency; pr_info("Hyper-V: LAPIC Timer Frequency: %#x\n", diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index cf29681d01e0..76c887ba9e9c 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -148,7 +148,7 @@ static inline void cache_alloc_hsw_probe(void) if (wrmsrl_safe(MSR_IA32_L3_CBM_BASE, max_cbm)) return; - rdmsrl(MSR_IA32_L3_CBM_BASE, l3_cbm_0); + rdmsrq(MSR_IA32_L3_CBM_BASE, l3_cbm_0); /* If all the bits were set in MSR, return success */ if (l3_cbm_0 != max_cbm) diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index a93ed7d2a160..f73a74945ffa 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -238,7 +238,7 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) * are error bits. */ wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid); - rdmsrl(MSR_IA32_QM_CTR, msr_val); + rdmsrq(MSR_IA32_QM_CTR, msr_val); if (msr_val & RMID_VAL_ERROR) return -EIO; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 93ec829015f1..3be9866be0e7 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1635,7 +1635,7 @@ void resctrl_arch_mon_event_config_read(void *_config_info) pr_warn_once("Invalid event id %d\n", config_info->evtid); return; } - rdmsrl(MSR_IA32_EVT_CFG_BASE + index, msrval); + rdmsrq(MSR_IA32_EVT_CFG_BASE + index, msrval); /* Report only the valid event configuration bits */ config_info->mon_config = msrval & MAX_EVT_CONFIG_BITS; diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 01456236a6dd..6e1885dece0f 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -154,7 +154,7 @@ static __init bool check_for_real_bsp(u32 apic_id) * kernel must rely on the firmware enumeration order. */ if (has_apic_base) { - rdmsrl(MSR_IA32_APICBASE, msr); + rdmsrq(MSR_IA32_APICBASE, msr); is_bsp = !!(msr & MSR_IA32_APICBASE_BSP); } diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index 03b3c9c3a45e..535dcf511096 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -133,7 +133,7 @@ static void parse_fam10h_node_id(struct topo_scan *tscan) if (!boot_cpu_has(X86_FEATURE_NODEID_MSR)) return; - rdmsrl(MSR_FAM10H_NODE_ID, nid.msr); + rdmsrq(MSR_FAM10H_NODE_ID, nid.msr); store_node(tscan, nid.nodes_per_pkg + 1, nid.node_id); tscan->c->topo.llc_id = nid.node_id; } @@ -160,7 +160,7 @@ static void topoext_fixup(struct topo_scan *tscan) if (msr_set_bit(0xc0011005, 54) <= 0) return; - rdmsrl(0xc0011005, msrval); + rdmsrq(0xc0011005, msrval); if (msrval & BIT_64(54)) { set_cpu_cap(c, X86_FEATURE_TOPOEXT); pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n"); diff --git a/arch/x86/kernel/cpu/tsx.c b/arch/x86/kernel/cpu/tsx.c index b31ee4f1657a..c3aaa6815c0b 100644 --- a/arch/x86/kernel/cpu/tsx.c +++ b/arch/x86/kernel/cpu/tsx.c @@ -24,7 +24,7 @@ static void tsx_disable(void) { u64 tsx; - rdmsrl(MSR_IA32_TSX_CTRL, tsx); + rdmsrq(MSR_IA32_TSX_CTRL, tsx); /* Force all transactions to immediately abort */ tsx |= TSX_CTRL_RTM_DISABLE; @@ -44,7 +44,7 @@ static void tsx_enable(void) { u64 tsx; - rdmsrl(MSR_IA32_TSX_CTRL, tsx); + rdmsrq(MSR_IA32_TSX_CTRL, tsx); /* Enable the RTM feature in the cpu */ tsx &= ~TSX_CTRL_RTM_DISABLE; @@ -115,11 +115,11 @@ static void tsx_clear_cpuid(void) */ if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { - rdmsrl(MSR_TSX_FORCE_ABORT, msr); + rdmsrq(MSR_TSX_FORCE_ABORT, msr); msr |= MSR_TFA_TSX_CPUID_CLEAR; wrmsrl(MSR_TSX_FORCE_ABORT, msr); } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { - rdmsrl(MSR_IA32_TSX_CTRL, msr); + rdmsrq(MSR_IA32_TSX_CTRL, msr); msr |= TSX_CTRL_CPUID_CLEAR; wrmsrl(MSR_IA32_TSX_CTRL, msr); } @@ -146,7 +146,7 @@ static void tsx_dev_mode_disable(void) !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) return; - rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); + rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); if (mcu_opt_ctrl & RTM_ALLOW) { mcu_opt_ctrl &= ~RTM_ALLOW; diff --git a/arch/x86/kernel/cpu/umwait.c b/arch/x86/kernel/cpu/umwait.c index 2293efd6ffa6..0050eae153bb 100644 --- a/arch/x86/kernel/cpu/umwait.c +++ b/arch/x86/kernel/cpu/umwait.c @@ -214,7 +214,7 @@ static int __init umwait_init(void) * changed. This is the only place where orig_umwait_control_cached * is modified. */ - rdmsrl(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached); + rdmsrq(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached); ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online", umwait_cpu_online, umwait_cpu_offline); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 91d6341f281f..985dfffe28c1 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -327,7 +327,7 @@ void fpu_sync_guest_vmexit_xfd_state(void) lockdep_assert_irqs_disabled(); if (fpu_state_size_dynamic()) { - rdmsrl(MSR_IA32_XFD, fps->xfd); + rdmsrq(MSR_IA32_XFD, fps->xfd); __this_cpu_write(xfd_state, fps->xfd); } } diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 7f4b2966e15c..cc5d12232216 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -970,7 +970,7 @@ static bool __init hpet_is_pc10_damaged(void) return false; /* Check whether PC10 is enabled in PKG C-state limit */ - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, pcfg); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, pcfg); if ((pcfg & 0xF) < 8) return false; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 3be9b3342c67..f7aa39cd56ec 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -728,7 +728,7 @@ static int kvm_suspend(void) #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) - rdmsrl(MSR_KVM_POLL_CONTROL, val); + rdmsrq(MSR_KVM_POLL_CONTROL, val); has_guest_poll = !(val & 1); #endif return 0; diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c index 1f54eedc3015..5c6a55f8a7fe 100644 --- a/arch/x86/kernel/mmconf-fam10h_64.c +++ b/arch/x86/kernel/mmconf-fam10h_64.c @@ -97,7 +97,7 @@ static void get_fam10h_pci_mmconf_base(void) /* SYS_CFG */ address = MSR_AMD64_SYSCFG; - rdmsrl(address, val); + rdmsrq(address, val); /* TOP_MEM2 is not enabled? */ if (!(val & (1<<21))) { @@ -105,7 +105,7 @@ static void get_fam10h_pci_mmconf_base(void) } else { /* TOP_MEM2 */ address = MSR_K8_TOP_MEM2; - rdmsrl(address, val); + rdmsrq(address, val); tom2 = max(val & 0xffffff800000ULL, 1ULL << 32); } @@ -177,7 +177,7 @@ void fam10h_check_enable_mmcfg(void) return; address = MSR_FAM10H_MMIO_CONF_BASE; - rdmsrl(address, val); + rdmsrq(address, val); /* try to make sure that AP's setting is identical to BSP setting */ if (val & FAM10H_MMIO_CONF_ENABLE) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 962c3ce39323..197be718e964 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -710,7 +710,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) arch_has_block_step()) { unsigned long debugctl, msk; - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl &= ~DEBUGCTLMSR_BTF; msk = tifn & _TIF_BLOCKSTEP; debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT; diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 7196ca7048be..422c5dad1046 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -95,8 +95,8 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode, return; if (mode == SHOW_REGS_USER) { - rdmsrl(MSR_FS_BASE, fs); - rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); + rdmsrq(MSR_FS_BASE, fs); + rdmsrq(MSR_KERNEL_GS_BASE, shadowgs); printk("%sFS: %016lx GS: %016lx\n", log_lvl, fs, shadowgs); return; @@ -107,9 +107,9 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode, asm("movl %%fs,%0" : "=r" (fsindex)); asm("movl %%gs,%0" : "=r" (gsindex)); - rdmsrl(MSR_FS_BASE, fs); - rdmsrl(MSR_GS_BASE, gs); - rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); + rdmsrq(MSR_FS_BASE, fs); + rdmsrq(MSR_GS_BASE, gs); + rdmsrq(MSR_KERNEL_GS_BASE, shadowgs); cr0 = read_cr0(); cr2 = read_cr2(); @@ -195,7 +195,7 @@ static noinstr unsigned long __rdgsbase_inactive(void) native_swapgs(); } else { instrumentation_begin(); - rdmsrl(MSR_KERNEL_GS_BASE, gsbase); + rdmsrq(MSR_KERNEL_GS_BASE, gsbase); instrumentation_end(); } @@ -463,7 +463,7 @@ unsigned long x86_gsbase_read_cpu_inactive(void) gsbase = __rdgsbase_inactive(); local_irq_restore(flags); } else { - rdmsrl(MSR_KERNEL_GS_BASE, gsbase); + rdmsrq(MSR_KERNEL_GS_BASE, gsbase); } return gsbase; diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index 059685612362..3d1ba20f840f 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -239,7 +239,7 @@ static unsigned long get_user_shstk_addr(void) fpregs_lock_and_load(); - rdmsrl(MSR_IA32_PL3_SSP, ssp); + rdmsrq(MSR_IA32_PL3_SSP, ssp); fpregs_unlock(); @@ -460,7 +460,7 @@ static int wrss_control(bool enable) return 0; fpregs_lock_and_load(); - rdmsrl(MSR_IA32_U_CET, msrval); + rdmsrq(MSR_IA32_U_CET, msrval); if (enable) { features_set(ARCH_SHSTK_WRSS); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 9f88b8a78e50..71b14675e50e 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -1120,7 +1120,7 @@ static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6) */ unsigned long debugctl; - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl |= DEBUGCTLMSR_BTF; wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); } @@ -1386,7 +1386,7 @@ static bool handle_xfd_event(struct pt_regs *regs) if (!IS_ENABLED(CONFIG_X86_64) || !cpu_feature_enabled(X86_FEATURE_XFD)) return false; - rdmsrl(MSR_IA32_XFD_ERR, xfd_err); + rdmsrq(MSR_IA32_XFD_ERR, xfd_err); if (!xfd_err) return false; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 88e5a4ed9db3..160fef71b9a3 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1098,7 +1098,7 @@ static void __init detect_art(void) if (art_base_clk.denominator < ART_MIN_DENOMINATOR) return; - rdmsrl(MSR_IA32_TSC_ADJUST, art_base_clk.offset); + rdmsrq(MSR_IA32_TSC_ADJUST, art_base_clk.offset); /* Make this sticky over multiple CPU init calls */ setup_force_cpu_cap(X86_FEATURE_ART); diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index 4334033658ed..826039199828 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -65,7 +65,7 @@ void tsc_verify_tsc_adjust(bool resume) adj->nextcheck = jiffies + HZ; - rdmsrl(MSR_IA32_TSC_ADJUST, curval); + rdmsrq(MSR_IA32_TSC_ADJUST, curval); if (adj->adjusted == curval) return; @@ -165,7 +165,7 @@ bool __init tsc_store_and_check_tsc_adjust(bool bootcpu) if (check_tsc_unstable()) return false; - rdmsrl(MSR_IA32_TSC_ADJUST, bootval); + rdmsrq(MSR_IA32_TSC_ADJUST, bootval); cur->bootval = bootval; cur->nextcheck = jiffies + HZ; tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu); @@ -187,7 +187,7 @@ bool tsc_store_and_check_tsc_adjust(bool bootcpu) if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) return false; - rdmsrl(MSR_IA32_TSC_ADJUST, bootval); + rdmsrq(MSR_IA32_TSC_ADJUST, bootval); cur->bootval = bootval; cur->nextcheck = jiffies + HZ; cur->warned = false; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index d5d0c5c3300b..21c3563547f2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -580,7 +580,7 @@ static inline void kvm_cpu_svm_disable(void) uint64_t efer; wrmsrl(MSR_VM_HSAVE_PA, 0); - rdmsrl(MSR_EFER, efer); + rdmsrq(MSR_EFER, efer); if (efer & EFER_SVME) { /* * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and @@ -619,7 +619,7 @@ static int svm_enable_virtualization_cpu(void) uint64_t efer; int me = raw_smp_processor_id(); - rdmsrl(MSR_EFER, efer); + rdmsrq(MSR_EFER, efer); if (efer & EFER_SVME) return -EBUSY; @@ -5232,7 +5232,7 @@ static __init void svm_adjust_mmio_mask(void) return; /* If memory encryption is not enabled, use existing mask */ - rdmsrl(MSR_AMD64_SYSCFG, msr); + rdmsrq(MSR_AMD64_SYSCFG, msr); if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) return; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 5504d9e9fd32..a7fea622e204 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -7202,8 +7202,8 @@ static void nested_vmx_setup_cr_fixed(struct nested_vmx_msrs *msrs) msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; /* These MSRs specify bits which the guest must keep fixed off. */ - rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); - rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); + rdmsrq(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); + rdmsrq(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); if (vmx_umip_emulated()) msrs->cr4_fixed1 |= X86_CR4_UMIP; diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 77012b2eca0e..f0f02eee48cb 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -279,7 +279,7 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, local_irq_disable(); if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) { if (read) - rdmsrl(index, msr_info->data); + rdmsrq(index, msr_info->data); else wrmsrl(index, msr_info->data); __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 9961e07cf071..3cf4ef659dbd 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -418,9 +418,9 @@ void setup_default_sgx_lepubkeyhash(void) sgx_pubkey_hash[3] = 0xd4f8c05909f9bb3bULL; } else { /* MSR_IA32_SGXLEPUBKEYHASH0 is read above */ - rdmsrl(MSR_IA32_SGXLEPUBKEYHASH1, sgx_pubkey_hash[1]); - rdmsrl(MSR_IA32_SGXLEPUBKEYHASH2, sgx_pubkey_hash[2]); - rdmsrl(MSR_IA32_SGXLEPUBKEYHASH3, sgx_pubkey_hash[3]); + rdmsrq(MSR_IA32_SGXLEPUBKEYHASH1, sgx_pubkey_hash[1]); + rdmsrq(MSR_IA32_SGXLEPUBKEYHASH2, sgx_pubkey_hash[2]); + rdmsrq(MSR_IA32_SGXLEPUBKEYHASH3, sgx_pubkey_hash[3]); } } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 5c5766467a61..3e0762a6939b 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1206,13 +1206,13 @@ static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) { u32 i; - rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); - rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); - rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); - rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); + rdmsrq(MSR_IA32_RTIT_STATUS, ctx->status); + rdmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); + rdmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); + rdmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); for (i = 0; i < addr_range; i++) { - rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); - rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); + rdmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); + rdmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); } } @@ -1225,7 +1225,7 @@ static void pt_guest_enter(struct vcpu_vmx *vmx) * GUEST_IA32_RTIT_CTL is already set in the VMCS. * Save host state before VM entry. */ - rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); + rdmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { wrmsrl(MSR_IA32_RTIT_CTL, 0); pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); @@ -1362,7 +1362,7 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) ++vmx->vcpu.stat.host_state_reload; #ifdef CONFIG_X86_64 - rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); + rdmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); #endif if (host_state->ldt_sel || (host_state->gs_sel & 7)) { kvm_load_ldt(host_state->ldt_sel); @@ -1394,7 +1394,7 @@ static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) { preempt_disable(); if (vmx->guest_state_loaded) - rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); + rdmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); preempt_enable(); return vmx->msr_guest_kernel_gs_base; } @@ -2574,7 +2574,7 @@ static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) { u64 allowed; - rdmsrl(msr, allowed); + rdmsrq(msr, allowed); return ctl_opt & allowed; } @@ -2746,7 +2746,7 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, break; } - rdmsrl(MSR_IA32_VMX_BASIC, basic_msr); + rdmsrq(MSR_IA32_VMX_BASIC, basic_msr); /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ if (vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE) @@ -2766,7 +2766,7 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (vmx_basic_vmcs_mem_type(basic_msr) != X86_MEMTYPE_WB) return -EIO; - rdmsrl(MSR_IA32_VMX_MISC, misc_msr); + rdmsrq(MSR_IA32_VMX_MISC, misc_msr); vmcs_conf->basic = basic_msr; vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; @@ -4391,7 +4391,7 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx) if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); - rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); + rdmsrq(MSR_IA32_SYSENTER_EIP, tmpl); vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { @@ -7052,7 +7052,7 @@ static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) * the #NM exception. */ if (is_xfd_nm_fault(vcpu)) - rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); } static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) @@ -7959,7 +7959,7 @@ static __init u64 vmx_get_perf_capabilities(void) return 0; if (boot_cpu_has(X86_FEATURE_PDCM)) - rdmsrl(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); + rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { x86_perf_get_lbr(&vmx_lbr_caps); @@ -8508,7 +8508,7 @@ __init int vmx_hardware_setup(void) kvm_enable_efer_bits(EFER_NX); if (boot_cpu_has(X86_FEATURE_MPX)) { - rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); + rdmsrq(MSR_IA32_BNDCFGS, host_bndcfgs); WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost"); } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c841817a914a..c96c2ca7bb61 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9773,12 +9773,12 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) rdmsrl_safe(MSR_EFER, &kvm_host.efer); if (boot_cpu_has(X86_FEATURE_XSAVES)) - rdmsrl(MSR_IA32_XSS, kvm_host.xss); + rdmsrq(MSR_IA32_XSS, kvm_host.xss); kvm_init_pmu_capability(ops->pmu_ops); if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) - rdmsrl(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities); + rdmsrq(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities); r = ops->hardware_setup(); if (r != 0) diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 98631c0e7a11..da5af3cc25b1 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -702,16 +702,16 @@ unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) unsigned long base; if (seg_reg_idx == INAT_SEG_REG_FS) { - rdmsrl(MSR_FS_BASE, base); + rdmsrq(MSR_FS_BASE, base); } else if (seg_reg_idx == INAT_SEG_REG_GS) { /* * swapgs was called at the kernel entry point. Thus, * MSR_KERNEL_GS_BASE will have the user-space GS base. */ if (user_mode(regs)) - rdmsrl(MSR_KERNEL_GS_BASE, base); + rdmsrq(MSR_KERNEL_GS_BASE, base); else - rdmsrl(MSR_GS_BASE, base); + rdmsrq(MSR_GS_BASE, base); } else { base = 0; } diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 72d8cbc61158..ce6105e64944 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -256,7 +256,7 @@ void __init pat_bp_init(void) if (!cpu_feature_enabled(X86_FEATURE_PAT)) pat_disable("PAT not supported by the CPU."); else - rdmsrl(MSR_IA32_CR_PAT, pat_msr_val); + rdmsrq(MSR_IA32_CR_PAT, pat_msr_val); if (!pat_msr_val) { pat_disable("PAT support disabled by the firmware."); diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index 631512f7ec85..b92917c052f3 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -202,7 +202,7 @@ static int __init early_root_info_init(void) /* need to take out [0, TOM) for RAM*/ address = MSR_K8_TOP_MEM1; - rdmsrl(address, val); + rdmsrq(address, val); end = (val & 0xffffff800000ULL); printk(KERN_INFO "TOM: %016llx aka %lldM\n", end, end>>20); if (end < (1ULL<<32)) @@ -293,12 +293,12 @@ static int __init early_root_info_init(void) /* need to take out [4G, TOM2) for RAM*/ /* SYS_CFG */ address = MSR_AMD64_SYSCFG; - rdmsrl(address, val); + rdmsrq(address, val); /* TOP_MEM2 is enabled? */ if (val & (1<<21)) { /* TOP_MEM2 */ address = MSR_K8_TOP_MEM2; - rdmsrl(address, val); + rdmsrq(address, val); end = (val & 0xffffff800000ULL); printk(KERN_INFO "TOM2: %016llx aka %lldM\n", end, end>>20); subtract_range(range, RANGE_NUM, 1ULL<<32, end); @@ -341,7 +341,7 @@ static int amd_bus_cpu_online(unsigned int cpu) { u64 reg; - rdmsrl(MSR_AMD64_NB_CFG, reg); + rdmsrq(MSR_AMD64_NB_CFG, reg); if (!(reg & ENABLE_CF8_EXT_CFG)) { reg |= ENABLE_CF8_EXT_CFG; wrmsrl(MSR_AMD64_NB_CFG, reg); diff --git a/arch/x86/platform/olpc/olpc-xo1-rtc.c b/arch/x86/platform/olpc/olpc-xo1-rtc.c index 57f210cda761..ee77d57bcab7 100644 --- a/arch/x86/platform/olpc/olpc-xo1-rtc.c +++ b/arch/x86/platform/olpc/olpc-xo1-rtc.c @@ -64,9 +64,9 @@ static int __init xo1_rtc_init(void) of_node_put(node); pr_info("olpc-xo1-rtc: Initializing OLPC XO-1 RTC\n"); - rdmsrl(MSR_RTC_DOMA_OFFSET, rtc_info.rtc_day_alarm); - rdmsrl(MSR_RTC_MONA_OFFSET, rtc_info.rtc_mon_alarm); - rdmsrl(MSR_RTC_CEN_OFFSET, rtc_info.rtc_century); + rdmsrq(MSR_RTC_DOMA_OFFSET, rtc_info.rtc_day_alarm); + rdmsrq(MSR_RTC_MONA_OFFSET, rtc_info.rtc_mon_alarm); + rdmsrq(MSR_RTC_CEN_OFFSET, rtc_info.rtc_century); r = platform_device_register(&xo1_rtc_device); if (r) diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 08e76a5ca155..874fe980c4bd 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -44,7 +44,7 @@ static void msr_save_context(struct saved_context *ctxt) while (msr < end) { if (msr->valid) - rdmsrl(msr->info.msr_no, msr->info.reg.q); + rdmsrq(msr->info.msr_no, msr->info.reg.q); msr++; } } @@ -110,12 +110,12 @@ static void __save_processor_state(struct saved_context *ctxt) savesegment(ds, ctxt->ds); savesegment(es, ctxt->es); - rdmsrl(MSR_FS_BASE, ctxt->fs_base); - rdmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base); - rdmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base); + rdmsrq(MSR_FS_BASE, ctxt->fs_base); + rdmsrq(MSR_GS_BASE, ctxt->kernelmode_gs_base); + rdmsrq(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base); mtrr_save_fixed_ranges(NULL); - rdmsrl(MSR_EFER, ctxt->efer); + rdmsrq(MSR_EFER, ctxt->efer); #endif /* diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index f9bc444a3064..263787b4800c 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -145,7 +145,7 @@ static void __init setup_real_mode(void) * Some AMD processors will #GP(0) if EFER.LMA is set in WRMSR * so we need to mask it out. */ - rdmsrl(MSR_EFER, efer); + rdmsrq(MSR_EFER, efer); trampoline_header->efer = efer & ~EFER_LMA; trampoline_header->start = (u64) secondary_startup_64; diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index fc473ca12c44..9ddd10265de7 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -136,7 +136,7 @@ static int __mfd_enable(unsigned int cpu) if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return 0; - rdmsrl(MSR_AMD64_SYSCFG, val); + rdmsrq(MSR_AMD64_SYSCFG, val); val |= MSR_AMD64_SYSCFG_MFDM; @@ -157,7 +157,7 @@ static int __snp_enable(unsigned int cpu) if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return 0; - rdmsrl(MSR_AMD64_SYSCFG, val); + rdmsrq(MSR_AMD64_SYSCFG, val); val |= MSR_AMD64_SYSCFG_SNP_EN; val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; @@ -522,7 +522,7 @@ int __init snp_rmptable_init(void) * Check if SEV-SNP is already enabled, this can happen in case of * kexec boot. */ - rdmsrl(MSR_AMD64_SYSCFG, val); + rdmsrq(MSR_AMD64_SYSCFG, val); if (val & MSR_AMD64_SYSCFG_SNP_EN) goto skip_enable; @@ -576,8 +576,8 @@ static bool probe_contiguous_rmptable_info(void) { u64 rmp_sz, rmp_base, rmp_end; - rdmsrl(MSR_AMD64_RMP_BASE, rmp_base); - rdmsrl(MSR_AMD64_RMP_END, rmp_end); + rdmsrq(MSR_AMD64_RMP_BASE, rmp_base); + rdmsrq(MSR_AMD64_RMP_END, rmp_end); if (!(rmp_base & RMP_ADDR_MASK) || !(rmp_end & RMP_ADDR_MASK)) { pr_err("Memory for the RMP table has not been reserved by BIOS\n"); @@ -610,13 +610,13 @@ static bool probe_segmented_rmptable_info(void) unsigned int eax, ebx, segment_shift, segment_shift_min, segment_shift_max; u64 rmp_base, rmp_end; - rdmsrl(MSR_AMD64_RMP_BASE, rmp_base); + rdmsrq(MSR_AMD64_RMP_BASE, rmp_base); if (!(rmp_base & RMP_ADDR_MASK)) { pr_err("Memory for the RMP table has not been reserved by BIOS\n"); return false; } - rdmsrl(MSR_AMD64_RMP_END, rmp_end); + rdmsrq(MSR_AMD64_RMP_END, rmp_end); WARN_ONCE(rmp_end & RMP_ADDR_MASK, "Segmented RMP enabled but RMP_END MSR is non-zero\n"); @@ -652,7 +652,7 @@ static bool probe_segmented_rmptable_info(void) bool snp_probe_rmptable_info(void) { if (cpu_feature_enabled(X86_FEATURE_SEGMENTED_RMP)) - rdmsrl(MSR_AMD64_RMP_CFG, rmp_cfg); + rdmsrq(MSR_AMD64_RMP_CFG, rmp_cfg); if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) return probe_segmented_rmptable_info(); diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c index 77a6ea1c60e4..cacac3d75f32 100644 --- a/arch/x86/xen/suspend.c +++ b/arch/x86/xen/suspend.c @@ -55,7 +55,7 @@ static void xen_vcpu_notify_suspend(void *data) tick_suspend_local(); if (xen_pv_domain() && boot_cpu_has(X86_FEATURE_SPEC_CTRL)) { - rdmsrl(MSR_IA32_SPEC_CTRL, tmp); + rdmsrq(MSR_IA32_SPEC_CTRL, tmp); this_cpu_write(spec_ctrl, tmp); wrmsrl(MSR_IA32_SPEC_CTRL, 0); } diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 924314cdeebc..8e31c25c0905 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -110,7 +110,7 @@ static int boost_set_msr(bool enable) return -EINVAL; } - rdmsrl(msr_addr, val); + rdmsrq(msr_addr, val); if (enable) val &= ~msr_mask; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6789eed1bb5b..2b673ac0781f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -518,8 +518,8 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) unsigned long flags; local_irq_save(flags); - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); tsc = rdtsc(); if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c index d23a97ba6478..6d6afcda3681 100644 --- a/drivers/cpufreq/e_powersaver.c +++ b/drivers/cpufreq/e_powersaver.c @@ -225,12 +225,12 @@ static int eps_cpu_init(struct cpufreq_policy *policy) return -ENODEV; } /* Enable Enhanced PowerSaver */ - rdmsrl(MSR_IA32_MISC_ENABLE, val); + rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { val |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP; wrmsrl(MSR_IA32_MISC_ENABLE, val); /* Can be locked at 0 */ - rdmsrl(MSR_IA32_MISC_ENABLE, val); + rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { pr_info("Can't enable Enhanced PowerSaver\n"); return -ENODEV; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 4aad79d26c64..d66a4741fe9f 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -598,7 +598,7 @@ static bool turbo_is_disabled(void) { u64 misc_en; - rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); + rdmsrq(MSR_IA32_MISC_ENABLE, misc_en); return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); } @@ -1285,7 +1285,7 @@ static void set_power_ctl_ee_state(bool input) u64 power_ctl; mutex_lock(&intel_pstate_driver_lock); - rdmsrl(MSR_IA32_POWER_CTL, power_ctl); + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); if (input) { power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE); power_ctl_ee_state = POWER_CTL_EE_ENABLE; @@ -1703,7 +1703,7 @@ static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribut u64 power_ctl; int enable; - rdmsrl(MSR_IA32_POWER_CTL, power_ctl); + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE)); return sprintf(buf, "%d\n", !enable); } @@ -1990,7 +1990,7 @@ static int atom_get_min_pstate(int not_used) { u64 value; - rdmsrl(MSR_ATOM_CORE_RATIOS, value); + rdmsrq(MSR_ATOM_CORE_RATIOS, value); return (value >> 8) & 0x7F; } @@ -1998,7 +1998,7 @@ static int atom_get_max_pstate(int not_used) { u64 value; - rdmsrl(MSR_ATOM_CORE_RATIOS, value); + rdmsrq(MSR_ATOM_CORE_RATIOS, value); return (value >> 16) & 0x7F; } @@ -2006,7 +2006,7 @@ static int atom_get_turbo_pstate(int not_used) { u64 value; - rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value); + rdmsrq(MSR_ATOM_CORE_TURBO_RATIOS, value); return value & 0x7F; } @@ -2041,7 +2041,7 @@ static int silvermont_get_scaling(void) static int silvermont_freq_table[] = { 83300, 100000, 133300, 116700, 80000}; - rdmsrl(MSR_FSB_FREQ, value); + rdmsrq(MSR_FSB_FREQ, value); i = value & 0x7; WARN_ON(i > 4); @@ -2057,7 +2057,7 @@ static int airmont_get_scaling(void) 83300, 100000, 133300, 116700, 80000, 93300, 90000, 88900, 87500}; - rdmsrl(MSR_FSB_FREQ, value); + rdmsrq(MSR_FSB_FREQ, value); i = value & 0xF; WARN_ON(i > 8); @@ -2068,7 +2068,7 @@ static void atom_get_vid(struct cpudata *cpudata) { u64 value; - rdmsrl(MSR_ATOM_CORE_VIDS, value); + rdmsrq(MSR_ATOM_CORE_VIDS, value); cpudata->vid.min = int_tofp((value >> 8) & 0x7f); cpudata->vid.max = int_tofp((value >> 16) & 0x7f); cpudata->vid.ratio = div_fp( @@ -2076,7 +2076,7 @@ static void atom_get_vid(struct cpudata *cpudata) int_tofp(cpudata->pstate.max_pstate - cpudata->pstate.min_pstate)); - rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value); + rdmsrq(MSR_ATOM_CORE_TURBO_VIDS, value); cpudata->vid.turbo = value & 0x7f; } @@ -2425,8 +2425,8 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) u64 tsc; local_irq_save(flags); - rdmsrl(MSR_IA32_APERF, aperf); - rdmsrl(MSR_IA32_MPERF, mperf); + rdmsrq(MSR_IA32_APERF, aperf); + rdmsrq(MSR_IA32_MPERF, mperf); tsc = rdtsc(); if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) { local_irq_restore(flags); @@ -3573,7 +3573,7 @@ static bool __init intel_pstate_platform_pwr_mgmt_exists(void) id = x86_match_cpu(intel_pstate_cpu_oob_ids); if (id) { - rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr); + rdmsrq(MSR_MISC_PWR_MGMT, misc_pwr); if (misc_pwr & BITMASK_OOB) { pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n"); pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n"); @@ -3629,7 +3629,7 @@ static bool intel_pstate_hwp_is_enabled(void) { u64 value; - rdmsrl(MSR_PM_ENABLE, value); + rdmsrq(MSR_PM_ENABLE, value); return !!(value & 0x1); } diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c index 68ccd73c8129..d3146f09fe80 100644 --- a/drivers/cpufreq/longhaul.c +++ b/drivers/cpufreq/longhaul.c @@ -136,7 +136,7 @@ static void do_longhaul1(unsigned int mults_index) { union msr_bcr2 bcr2; - rdmsrl(MSR_VIA_BCR2, bcr2.val); + rdmsrq(MSR_VIA_BCR2, bcr2.val); /* Enable software clock multiplier */ bcr2.bits.ESOFTBF = 1; bcr2.bits.CLOCKMUL = mults_index & 0xff; @@ -151,7 +151,7 @@ static void do_longhaul1(unsigned int mults_index) /* Disable software clock multiplier */ local_irq_disable(); - rdmsrl(MSR_VIA_BCR2, bcr2.val); + rdmsrq(MSR_VIA_BCR2, bcr2.val); bcr2.bits.ESOFTBF = 0; wrmsrl(MSR_VIA_BCR2, bcr2.val); } @@ -164,7 +164,7 @@ static void do_powersaver(int cx_address, unsigned int mults_index, union msr_longhaul longhaul; u32 t; - rdmsrl(MSR_VIA_LONGHAUL, longhaul.val); + rdmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Setup new frequency */ if (!revid_errata) longhaul.bits.RevisionKey = longhaul.bits.RevisionID; @@ -534,7 +534,7 @@ static void longhaul_setup_voltagescaling(void) unsigned int j, speed, pos, kHz_step, numvscales; int min_vid_speed; - rdmsrl(MSR_VIA_LONGHAUL, longhaul.val); + rdmsrq(MSR_VIA_LONGHAUL, longhaul.val); if (!(longhaul.bits.RevisionID & 1)) { pr_info("Voltage scaling not supported by CPU\n"); return; diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index fb2197dc170f..dbbc27cb9ce2 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -219,7 +219,7 @@ static void change_FID(int fid) { union msr_fidvidctl fidvidctl; - rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + rdmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); if (fidvidctl.bits.FID != fid) { fidvidctl.bits.SGTC = latency; fidvidctl.bits.FID = fid; @@ -234,7 +234,7 @@ static void change_VID(int vid) { union msr_fidvidctl fidvidctl; - rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + rdmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); if (fidvidctl.bits.VID != vid) { fidvidctl.bits.SGTC = latency; fidvidctl.bits.VID = vid; @@ -260,7 +260,7 @@ static int powernow_target(struct cpufreq_policy *policy, unsigned int index) fid = powernow_table[index].driver_data & 0xFF; vid = (powernow_table[index].driver_data & 0xFF00) >> 8; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); cfid = fidvidstatus.bits.CFID; freqs.old = fsb * fid_codes[cfid] / 10; @@ -557,7 +557,7 @@ static unsigned int powernow_get(unsigned int cpu) if (cpu) return 0; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); cfid = fidvidstatus.bits.CFID; return fsb * fid_codes[cfid] / 10; @@ -598,7 +598,7 @@ static int powernow_cpu_init(struct cpufreq_policy *policy) if (policy->cpu != 0) return -ENODEV; - rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); + rdmsrq(MSR_K7_FID_VID_STATUS, fidvidstatus.val); recalibrate_cpu_khz(); diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 90f0eb7cc5b9..db758aa900b0 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2942,13 +2942,13 @@ static void dct_read_mc_regs(struct amd64_pvt *pvt) * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since * those are Read-As-Zero. */ - rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); + rdmsrq(MSR_K8_TOP_MEM1, pvt->top_mem); edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); /* Check first whether TOP_MEM2 is enabled: */ - rdmsrl(MSR_AMD64_SYSCFG, msr_val); + rdmsrq(MSR_AMD64_SYSCFG, msr_val); if (msr_val & BIT(21)) { - rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); + rdmsrq(MSR_K8_TOP_MEM2, pvt->top_mem2); edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); } else { edac_dbg(0, " TOP_MEM2 disabled\n"); diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 976f5be54e36..6e4b5b33ba81 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1928,35 +1928,35 @@ static void __init bxt_idle_state_table_update(void) unsigned long long msr; unsigned int usec; - rdmsrl(MSR_PKGC6_IRTL, msr); + rdmsrq(MSR_PKGC6_IRTL, msr); usec = irtl_2_usec(msr); if (usec) { bxt_cstates[2].exit_latency = usec; bxt_cstates[2].target_residency = usec; } - rdmsrl(MSR_PKGC7_IRTL, msr); + rdmsrq(MSR_PKGC7_IRTL, msr); usec = irtl_2_usec(msr); if (usec) { bxt_cstates[3].exit_latency = usec; bxt_cstates[3].target_residency = usec; } - rdmsrl(MSR_PKGC8_IRTL, msr); + rdmsrq(MSR_PKGC8_IRTL, msr); usec = irtl_2_usec(msr); if (usec) { bxt_cstates[4].exit_latency = usec; bxt_cstates[4].target_residency = usec; } - rdmsrl(MSR_PKGC9_IRTL, msr); + rdmsrq(MSR_PKGC9_IRTL, msr); usec = irtl_2_usec(msr); if (usec) { bxt_cstates[5].exit_latency = usec; bxt_cstates[5].target_residency = usec; } - rdmsrl(MSR_PKGC10_IRTL, msr); + rdmsrq(MSR_PKGC10_IRTL, msr); usec = irtl_2_usec(msr); if (usec) { bxt_cstates[6].exit_latency = usec; @@ -1984,7 +1984,7 @@ static void __init sklh_idle_state_table_update(void) if ((mwait_substates & (0xF << 28)) == 0) return; - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); /* PC10 is not enabled in PKG C-state limit */ if ((msr & 0xF) != 8) @@ -1996,7 +1996,7 @@ static void __init sklh_idle_state_table_update(void) /* if SGX is present */ if (ebx & (1 << 2)) { - rdmsrl(MSR_IA32_FEAT_CTL, msr); + rdmsrq(MSR_IA32_FEAT_CTL, msr); /* if SGX is enabled */ if (msr & (1 << 18)) @@ -2015,7 +2015,7 @@ static void __init skx_idle_state_table_update(void) { unsigned long long msr; - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); /* * 000b: C0/C1 (no package C-state support) @@ -2068,7 +2068,7 @@ static void __init spr_idle_state_table_update(void) * C6. However, if PC6 is disabled, we update the numbers to match * core C6. */ - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); /* Limit value 2 and above allow for PC6. */ if ((msr & 0x7) < 2) { @@ -2241,7 +2241,7 @@ static void auto_demotion_disable(void) { unsigned long long msr_bits; - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); msr_bits &= ~auto_demotion_disable_flags; wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); } @@ -2250,7 +2250,7 @@ static void c1e_promotion_enable(void) { unsigned long long msr_bits; - rdmsrl(MSR_IA32_POWER_CTL, msr_bits); + rdmsrq(MSR_IA32_POWER_CTL, msr_bits); msr_bits |= 0x2; wrmsrl(MSR_IA32_POWER_CTL, msr_bits); } @@ -2259,7 +2259,7 @@ static void c1e_promotion_disable(void) { unsigned long long msr_bits; - rdmsrl(MSR_IA32_POWER_CTL, msr_bits); + rdmsrq(MSR_IA32_POWER_CTL, msr_bits); msr_bits &= ~0x2; wrmsrl(MSR_IA32_POWER_CTL, msr_bits); } diff --git a/drivers/mtd/nand/raw/cs553x_nand.c b/drivers/mtd/nand/raw/cs553x_nand.c index 341318024a19..ec95d787001b 100644 --- a/drivers/mtd/nand/raw/cs553x_nand.c +++ b/drivers/mtd/nand/raw/cs553x_nand.c @@ -351,20 +351,20 @@ static int __init cs553x_init(void) return -ENXIO; /* If it doesn't have the CS553[56], abort */ - rdmsrl(MSR_DIVIL_GLD_CAP, val); + rdmsrq(MSR_DIVIL_GLD_CAP, val); val &= ~0xFFULL; if (val != CAP_CS5535 && val != CAP_CS5536) return -ENXIO; /* If it doesn't have the NAND controller enabled, abort */ - rdmsrl(MSR_DIVIL_BALL_OPTS, val); + rdmsrq(MSR_DIVIL_BALL_OPTS, val); if (val & PIN_OPT_IDE) { pr_info("CS553x NAND controller: Flash I/O not enabled in MSR_DIVIL_BALL_OPTS.\n"); return -ENXIO; } for (i = 0; i < NR_CS553X_CONTROLLERS; i++) { - rdmsrl(MSR_DIVIL_LBAR_FLSH0 + i, val); + rdmsrq(MSR_DIVIL_LBAR_FLSH0 + i, val); if ((val & (FLSH_LBAR_EN|FLSH_NOR_NAND)) == (FLSH_LBAR_EN|FLSH_NOR_NAND)) err = cs553x_init_one(i, !!(val & FLSH_MEM_IO), val & 0xFFFFFFFF); diff --git a/drivers/platform/x86/intel/ifs/load.c b/drivers/platform/x86/intel/ifs/load.c index de54bd1a5970..d3e48669a01d 100644 --- a/drivers/platform/x86/intel/ifs/load.c +++ b/drivers/platform/x86/intel/ifs/load.c @@ -128,7 +128,7 @@ static void copy_hashes_authenticate_chunks(struct work_struct *work) msrs = ifs_get_test_msrs(dev); /* run scan hash copy */ wrmsrl(msrs->copy_hashes, ifs_hash_ptr); - rdmsrl(msrs->copy_hashes_status, hashes_status.data); + rdmsrq(msrs->copy_hashes_status, hashes_status.data); /* enumerate the scan image information */ num_chunks = hashes_status.num_chunks; @@ -150,7 +150,7 @@ static void copy_hashes_authenticate_chunks(struct work_struct *work) linear_addr |= i; wrmsrl(msrs->copy_chunks, linear_addr); - rdmsrl(msrs->copy_chunks_status, chunk_status.data); + rdmsrq(msrs->copy_chunks_status, chunk_status.data); ifsd->valid_chunks = chunk_status.valid_chunks; err_code = chunk_status.error_code; @@ -196,7 +196,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) if (need_copy_scan_hashes(ifsd)) { wrmsrl(msrs->copy_hashes, ifs_hash_ptr); - rdmsrl(msrs->copy_hashes_status, hashes_status.data); + rdmsrq(msrs->copy_hashes_status, hashes_status.data); /* enumerate the scan image information */ chunk_size = hashes_status.chunk_size * SZ_1K; @@ -217,7 +217,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) if (ifsd->generation >= IFS_GEN_STRIDE_AWARE) { wrmsrl(msrs->test_ctrl, INVALIDATE_STRIDE); - rdmsrl(msrs->copy_chunks_status, chunk_status.data); + rdmsrq(msrs->copy_chunks_status, chunk_status.data); if (chunk_status.valid_chunks != 0) { dev_err(dev, "Couldn't invalidate installed stride - %d\n", chunk_status.valid_chunks); @@ -240,7 +240,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) local_irq_disable(); wrmsrl(msrs->copy_chunks, (u64)chunk_table); local_irq_enable(); - rdmsrl(msrs->copy_chunks_status, chunk_status.data); + rdmsrq(msrs->copy_chunks_status, chunk_status.data); err_code = chunk_status.error_code; } while (err_code == AUTH_INTERRUPTED_ERROR && --retry_count); diff --git a/drivers/platform/x86/intel/ifs/runtest.c b/drivers/platform/x86/intel/ifs/runtest.c index f978dd05d4d8..8e669540a0a7 100644 --- a/drivers/platform/x86/intel/ifs/runtest.c +++ b/drivers/platform/x86/intel/ifs/runtest.c @@ -210,7 +210,7 @@ static int doscan(void *data) * are processed in a single pass) before it retires. */ wrmsrl(MSR_ACTIVATE_SCAN, params->activate->data); - rdmsrl(MSR_SCAN_STATUS, status.data); + rdmsrq(MSR_SCAN_STATUS, status.data); trace_ifs_status(ifsd->cur_batch, start, stop, status.data); @@ -323,7 +323,7 @@ static int do_array_test(void *data) if (cpu == first) { wrmsrl(MSR_ARRAY_BIST, command->data); /* Pass back the result of the test */ - rdmsrl(MSR_ARRAY_BIST, command->data); + rdmsrq(MSR_ARRAY_BIST, command->data); } return 0; @@ -375,7 +375,7 @@ static int do_array_test_gen1(void *status) if (cpu == first) { wrmsrl(MSR_ARRAY_TRIGGER, ARRAY_GEN1_TEST_ALL_ARRAYS); - rdmsrl(MSR_ARRAY_STATUS, *((u64 *)status)); + rdmsrq(MSR_ARRAY_STATUS, *((u64 *)status)); } return 0; @@ -527,7 +527,7 @@ static int dosbaf(void *data) * during the "execution" of the WRMSR. */ wrmsrl(MSR_ACTIVATE_SBAF, run_params->activate->data); - rdmsrl(MSR_SBAF_STATUS, status.data); + rdmsrq(MSR_SBAF_STATUS, status.data); trace_ifs_sbaf(ifsd->cur_batch, *run_params->activate, status); /* Pass back the result of the test */ diff --git a/drivers/platform/x86/intel/pmc/cnp.c b/drivers/platform/x86/intel/pmc/cnp.c index 2c5af158bbe2..aa9aa1cc244a 100644 --- a/drivers/platform/x86/intel/pmc/cnp.c +++ b/drivers/platform/x86/intel/pmc/cnp.c @@ -227,7 +227,7 @@ static void disable_c1_auto_demote(void *unused) int cpunum = smp_processor_id(); u64 val; - rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, val); + rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, val); per_cpu(pkg_cst_config, cpunum) = val; val &= ~NHM_C1_AUTO_DEMOTE; wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, val); diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c index c4b7af00352b..20624fcb9675 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c @@ -39,7 +39,7 @@ static int isst_if_send_mbox_cmd(u8 command, u8 sub_command, u32 parameter, /* Poll for rb bit == 0 */ retries = OS_MAILBOX_RETRY_COUNT; do { - rdmsrl(MSR_OS_MAILBOX_INTERFACE, data); + rdmsrq(MSR_OS_MAILBOX_INTERFACE, data); if (data & BIT_ULL(MSR_OS_MAILBOX_BUSY_BIT)) { ret = -EBUSY; continue; @@ -64,7 +64,7 @@ static int isst_if_send_mbox_cmd(u8 command, u8 sub_command, u32 parameter, /* Poll for rb bit == 0 */ retries = OS_MAILBOX_RETRY_COUNT; do { - rdmsrl(MSR_OS_MAILBOX_INTERFACE, data); + rdmsrq(MSR_OS_MAILBOX_INTERFACE, data); if (data & BIT_ULL(MSR_OS_MAILBOX_BUSY_BIT)) { ret = -EBUSY; continue; @@ -74,7 +74,7 @@ static int isst_if_send_mbox_cmd(u8 command, u8 sub_command, u32 parameter, return -ENXIO; if (response_data) { - rdmsrl(MSR_OS_MAILBOX_DATA, data); + rdmsrq(MSR_OS_MAILBOX_DATA, data); *response_data = data; } ret = 0; diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 9978cdd19851..0b8ef0cfaf80 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -556,7 +556,7 @@ static bool disable_dynamic_sst_features(void) { u64 value; - rdmsrl(MSR_PM_ENABLE, value); + rdmsrq(MSR_PM_ENABLE, value); return !(value & 0x1); } diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c index 5d717b1c23cf..74b2a8700022 100644 --- a/drivers/platform/x86/intel_ips.c +++ b/drivers/platform/x86/intel_ips.c @@ -370,7 +370,7 @@ static void ips_cpu_raise(struct ips_driver *ips) if (!ips->cpu_turbo_enabled) return; - rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); cur_tdp_limit = turbo_override & TURBO_TDP_MASK; new_tdp_limit = cur_tdp_limit + 8; /* 1W increase */ @@ -405,7 +405,7 @@ static void ips_cpu_lower(struct ips_driver *ips) u64 turbo_override; u16 cur_limit, new_limit; - rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); cur_limit = turbo_override & TURBO_TDP_MASK; new_limit = cur_limit - 8; /* 1W decrease */ @@ -437,7 +437,7 @@ static void do_enable_cpu_turbo(void *data) { u64 perf_ctl; - rdmsrl(IA32_PERF_CTL, perf_ctl); + rdmsrq(IA32_PERF_CTL, perf_ctl); if (perf_ctl & IA32_PERF_TURBO_DIS) { perf_ctl &= ~IA32_PERF_TURBO_DIS; wrmsrl(IA32_PERF_CTL, perf_ctl); @@ -475,7 +475,7 @@ static void do_disable_cpu_turbo(void *data) { u64 perf_ctl; - rdmsrl(IA32_PERF_CTL, perf_ctl); + rdmsrq(IA32_PERF_CTL, perf_ctl); if (!(perf_ctl & IA32_PERF_TURBO_DIS)) { perf_ctl |= IA32_PERF_TURBO_DIS; wrmsrl(IA32_PERF_CTL, perf_ctl); @@ -1215,7 +1215,7 @@ static int cpu_clamp_show(struct seq_file *m, void *data) u64 turbo_override; int tdp, tdc; - rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); tdp = (int)(turbo_override & TURBO_TDP_MASK); tdc = (int)((turbo_override & TURBO_TDC_MASK) >> TURBO_TDC_SHIFT); @@ -1290,7 +1290,7 @@ static struct ips_mcp_limits *ips_detect_cpu(struct ips_driver *ips) return NULL; } - rdmsrl(IA32_MISC_ENABLE, misc_en); + rdmsrq(IA32_MISC_ENABLE, misc_en); /* * If the turbo enable bit isn't set, we shouldn't try to enable/disable * turbo manually or we'll get an illegal MSR access, even though @@ -1312,7 +1312,7 @@ static struct ips_mcp_limits *ips_detect_cpu(struct ips_driver *ips) return NULL; } - rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_power); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_power); tdp = turbo_power & TURBO_TDP_MASK; /* Sanity check TDP against CPU */ @@ -1496,7 +1496,7 @@ static int ips_probe(struct pci_dev *dev, const struct pci_device_id *id) * Check PLATFORM_INFO MSR to make sure this chip is * turbo capable. */ - rdmsrl(PLATFORM_INFO, platform_info); + rdmsrq(PLATFORM_INFO, platform_info); if (!(platform_info & PLATFORM_TDP)) { dev_err(&dev->dev, "platform indicates TDP override unavailable, aborting\n"); return -ENODEV; @@ -1529,7 +1529,7 @@ static int ips_probe(struct pci_dev *dev, const struct pci_device_id *id) ips->mgta_val = thm_readw(THM_MGTA); /* Save turbo limits & ratios */ - rdmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit); ips_disable_cpu_turbo(ips); ips->cpu_turbo_enabled = false; @@ -1596,7 +1596,7 @@ static void ips_remove(struct pci_dev *dev) if (ips->gpu_turbo_disable) symbol_put(i915_gpu_turbo_disable); - rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); turbo_override &= ~(TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN); wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit); diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c index 5b18a46a10b0..2aaac15748a5 100644 --- a/drivers/thermal/intel/intel_hfi.c +++ b/drivers/thermal/intel/intel_hfi.c @@ -284,7 +284,7 @@ void intel_hfi_process_event(__u64 pkg_therm_status_msr_val) if (!raw_spin_trylock(&hfi_instance->event_lock)) return; - rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr); + rdmsrq(MSR_IA32_PACKAGE_THERM_STATUS, msr); hfi = msr & PACKAGE_THERM_STATUS_HFI_UPDATED; if (!hfi) { raw_spin_unlock(&hfi_instance->event_lock); @@ -356,7 +356,7 @@ static void hfi_enable(void) { u64 msr_val; - rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); + rdmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); } @@ -377,7 +377,7 @@ static void hfi_disable(void) u64 msr_val; int i; - rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); + rdmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); msr_val &= ~HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); @@ -388,7 +388,7 @@ static void hfi_disable(void) * memory. */ for (i = 0; i < 2000; i++) { - rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); + rdmsrq(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); if (msr_val & PACKAGE_THERM_STATUS_HFI_UPDATED) break; diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c index e69868e868eb..f296f35ef7d4 100644 --- a/drivers/thermal/intel/therm_throt.c +++ b/drivers/thermal/intel/therm_throt.c @@ -287,7 +287,7 @@ static void get_therm_status(int level, bool *proc_hot, u8 *temp) else msr = MSR_IA32_PACKAGE_THERM_STATUS; - rdmsrl(msr, msr_val); + rdmsrq(msr, msr_val); if (msr_val & THERM_STATUS_PROCHOT_LOG) *proc_hot = true; else @@ -654,7 +654,7 @@ void intel_thermal_interrupt(void) if (static_cpu_has(X86_FEATURE_HWP)) notify_hwp_interrupt(); - rdmsrl(MSR_IA32_THERM_STATUS, msr_val); + rdmsrq(MSR_IA32_THERM_STATUS, msr_val); /* Check for violation of core thermal thresholds*/ notify_thresholds(msr_val); @@ -669,7 +669,7 @@ void intel_thermal_interrupt(void) CORE_LEVEL); if (this_cpu_has(X86_FEATURE_PTS)) { - rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); + rdmsrq(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); /* check violations of package thermal thresholds */ notify_package_thresholds(msr_val); therm_throt_process(msr_val & PACKAGE_THERM_STATUS_PROCHOT, diff --git a/drivers/video/fbdev/geode/gxfb_core.c b/drivers/video/fbdev/geode/gxfb_core.c index af996634c1a9..2b27d6540805 100644 --- a/drivers/video/fbdev/geode/gxfb_core.c +++ b/drivers/video/fbdev/geode/gxfb_core.c @@ -377,7 +377,7 @@ static int gxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* Figure out if this is a TFT or CRT part */ - rdmsrl(MSR_GX_GLD_MSR_CONFIG, val); + rdmsrq(MSR_GX_GLD_MSR_CONFIG, val); if ((val & MSR_GX_GLD_MSR_CONFIG_FP) == MSR_GX_GLD_MSR_CONFIG_FP) par->enable_crt = 0; diff --git a/drivers/video/fbdev/geode/lxfb_ops.c b/drivers/video/fbdev/geode/lxfb_ops.c index 32baaf59fcf7..5402c000ce8e 100644 --- a/drivers/video/fbdev/geode/lxfb_ops.c +++ b/drivers/video/fbdev/geode/lxfb_ops.c @@ -358,7 +358,7 @@ void lx_set_mode(struct fb_info *info) /* Set output mode */ - rdmsrl(MSR_LX_GLD_MSR_CONFIG, msrval); + rdmsrq(MSR_LX_GLD_MSR_CONFIG, msrval); msrval &= ~MSR_LX_GLD_MSR_CONFIG_FMT; if (par->output & OUTPUT_PANEL) { @@ -419,7 +419,7 @@ void lx_set_mode(struct fb_info *info) /* Set default watermark values */ - rdmsrl(MSR_LX_SPARE_MSR, msrval); + rdmsrq(MSR_LX_SPARE_MSR, msrval); msrval &= ~(MSR_LX_SPARE_MSR_DIS_CFIFO_HGO | MSR_LX_SPARE_MSR_VFIFO_ARB_SEL @@ -591,10 +591,10 @@ static void lx_save_regs(struct lxfb_par *par) } while ((i & GP_BLT_STATUS_PB) || !(i & GP_BLT_STATUS_CE)); /* save MSRs */ - rdmsrl(MSR_LX_MSR_PADSEL, par->msr.padsel); - rdmsrl(MSR_GLCP_DOTPLL, par->msr.dotpll); - rdmsrl(MSR_LX_GLD_MSR_CONFIG, par->msr.dfglcfg); - rdmsrl(MSR_LX_SPARE_MSR, par->msr.dcspare); + rdmsrq(MSR_LX_MSR_PADSEL, par->msr.padsel); + rdmsrq(MSR_GLCP_DOTPLL, par->msr.dotpll); + rdmsrq(MSR_LX_GLD_MSR_CONFIG, par->msr.dfglcfg); + rdmsrq(MSR_LX_SPARE_MSR, par->msr.dcspare); write_dc(par, DC_UNLOCK, DC_UNLOCK_UNLOCK); diff --git a/drivers/video/fbdev/geode/suspend_gx.c b/drivers/video/fbdev/geode/suspend_gx.c index 8c49d4e98772..ed4997724063 100644 --- a/drivers/video/fbdev/geode/suspend_gx.c +++ b/drivers/video/fbdev/geode/suspend_gx.c @@ -21,8 +21,8 @@ static void gx_save_regs(struct gxfb_par *par) } while (i & (GP_BLT_STATUS_BLT_PENDING | GP_BLT_STATUS_BLT_BUSY)); /* save MSRs */ - rdmsrl(MSR_GX_MSR_PADSEL, par->msr.padsel); - rdmsrl(MSR_GLCP_DOTPLL, par->msr.dotpll); + rdmsrq(MSR_GX_MSR_PADSEL, par->msr.padsel); + rdmsrq(MSR_GLCP_DOTPLL, par->msr.dotpll); write_dc(par, DC_UNLOCK, DC_UNLOCK_UNLOCK); @@ -43,14 +43,14 @@ static void gx_set_dotpll(uint32_t dotpll_hi) uint32_t dotpll_lo; int i; - rdmsrl(MSR_GLCP_DOTPLL, dotpll_lo); + rdmsrq(MSR_GLCP_DOTPLL, dotpll_lo); dotpll_lo |= MSR_GLCP_DOTPLL_DOTRESET; dotpll_lo &= ~MSR_GLCP_DOTPLL_BYPASS; wrmsr(MSR_GLCP_DOTPLL, dotpll_lo, dotpll_hi); /* wait for the PLL to lock */ for (i = 0; i < 200; i++) { - rdmsrl(MSR_GLCP_DOTPLL, dotpll_lo); + rdmsrq(MSR_GLCP_DOTPLL, dotpll_lo); if (dotpll_lo & MSR_GLCP_DOTPLL_LOCK) break; udelay(1); diff --git a/drivers/video/fbdev/geode/video_gx.c b/drivers/video/fbdev/geode/video_gx.c index 91dac2aa247c..9bee01751f47 100644 --- a/drivers/video/fbdev/geode/video_gx.c +++ b/drivers/video/fbdev/geode/video_gx.c @@ -142,8 +142,8 @@ void gx_set_dclk_frequency(struct fb_info *info) } } - rdmsrl(MSR_GLCP_SYS_RSTPLL, sys_rstpll); - rdmsrl(MSR_GLCP_DOTPLL, dotpll); + rdmsrq(MSR_GLCP_SYS_RSTPLL, sys_rstpll); + rdmsrq(MSR_GLCP_DOTPLL, dotpll); /* Program new M, N and P. */ dotpll &= 0x00000000ffffffffull; @@ -167,7 +167,7 @@ void gx_set_dclk_frequency(struct fb_info *info) /* Wait for LOCK bit. */ do { - rdmsrl(MSR_GLCP_DOTPLL, dotpll); + rdmsrq(MSR_GLCP_DOTPLL, dotpll); } while (timeout-- && !(dotpll & MSR_GLCP_DOTPLL_LOCK)); } @@ -180,7 +180,7 @@ gx_configure_tft(struct fb_info *info) /* Set up the DF pad select MSR */ - rdmsrl(MSR_GX_MSR_PADSEL, val); + rdmsrq(MSR_GX_MSR_PADSEL, val); val &= ~MSR_GX_MSR_PADSEL_MASK; val |= MSR_GX_MSR_PADSEL_TFT; wrmsrl(MSR_GX_MSR_PADSEL, val); diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index abf0bd76e370..53699750db86 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h @@ -1013,7 +1013,7 @@ enum hv_register_name { /* * To support arch-generic code calling hv_set/get_register: - * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrl/wrmsrl + * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrq/wrmsrl * - On ARM, HV_MSR_ indicates a VP register accessed via hypercall */ #define HV_MSR_CRASH_P0 (HV_X64_MSR_CRASH_P0) From 78255eb23973323633432d9ec40b65c15e41888a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:55 +0200 Subject: [PATCH 095/331] x86/msr: Rename 'wrmsrl()' to 'wrmsrq()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/events/amd/brs.c | 4 +- arch/x86/events/amd/core.c | 8 +-- arch/x86/events/amd/ibs.c | 8 +-- arch/x86/events/amd/lbr.c | 12 ++-- arch/x86/events/amd/uncore.c | 10 +-- arch/x86/events/core.c | 12 ++-- arch/x86/events/intel/core.c | 44 ++++++------- arch/x86/events/intel/ds.c | 10 +-- arch/x86/events/intel/knc.c | 6 +- arch/x86/events/intel/lbr.c | 28 ++++---- arch/x86/events/intel/p4.c | 4 +- arch/x86/events/intel/p6.c | 4 +- arch/x86/events/intel/pt.c | 18 ++--- arch/x86/events/intel/uncore_discovery.c | 10 +-- arch/x86/events/intel/uncore_nhmex.c | 66 +++++++++---------- arch/x86/events/intel/uncore_snb.c | 40 +++++------ arch/x86/events/intel/uncore_snbep.c | 42 ++++++------ arch/x86/events/perf_event.h | 20 +++--- arch/x86/events/zhaoxin/core.c | 10 +-- arch/x86/hyperv/hv_apic.c | 2 +- arch/x86/hyperv/hv_init.c | 40 +++++------ arch/x86/include/asm/apic.h | 2 +- arch/x86/include/asm/debugreg.h | 2 +- arch/x86/include/asm/fsgsbase.h | 2 +- arch/x86/include/asm/msr.h | 4 +- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/kernel/apic/apic.c | 10 +-- arch/x86/kernel/cpu/bugs.c | 12 ++-- arch/x86/kernel/cpu/bus_lock.c | 6 +- arch/x86/kernel/cpu/common.c | 34 +++++----- arch/x86/kernel/cpu/feat_ctl.c | 2 +- arch/x86/kernel/cpu/intel.c | 2 +- arch/x86/kernel/cpu/intel_epb.c | 2 +- arch/x86/kernel/cpu/mce/amd.c | 8 +-- arch/x86/kernel/cpu/mce/core.c | 8 +-- arch/x86/kernel/cpu/mce/inject.c | 20 +++--- arch/x86/kernel/cpu/mce/intel.c | 10 +-- arch/x86/kernel/cpu/mshyperv.c | 6 +- arch/x86/kernel/cpu/resctrl/core.c | 6 +- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 4 +- arch/x86/kernel/cpu/sgx/main.c | 2 +- arch/x86/kernel/cpu/tsx.c | 10 +-- arch/x86/kernel/fpu/xstate.c | 10 +-- arch/x86/kernel/fpu/xstate.h | 2 +- arch/x86/kernel/fred.c | 20 +++--- arch/x86/kernel/kvm.c | 26 ++++---- arch/x86/kernel/kvmclock.c | 4 +- arch/x86/kernel/mmconf-fam10h_64.c | 2 +- arch/x86/kernel/process.c | 14 ++-- arch/x86/kernel/process_64.c | 6 +- arch/x86/kernel/reboot_fixups_32.c | 2 +- arch/x86/kernel/shstk.c | 14 ++-- arch/x86/kernel/traps.c | 6 +- arch/x86/kernel/tsc_sync.c | 8 +-- arch/x86/kvm/svm/avic.c | 2 +- arch/x86/kvm/svm/svm.c | 10 +-- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/vmx/vmx.c | 24 +++---- arch/x86/kvm/x86.c | 16 ++--- arch/x86/mm/pat/memtype.c | 2 +- arch/x86/mm/tlb.c | 2 +- arch/x86/pci/amd_bus.c | 2 +- arch/x86/platform/olpc/olpc-xo1-sci.c | 2 +- arch/x86/power/cpu.c | 12 ++-- arch/x86/virt/svm/sev.c | 4 +- arch/x86/xen/suspend.c | 4 +- drivers/cpufreq/acpi-cpufreq.c | 2 +- drivers/cpufreq/amd-pstate.c | 2 +- drivers/cpufreq/e_powersaver.c | 2 +- drivers/cpufreq/intel_pstate.c | 12 ++-- drivers/cpufreq/longhaul.c | 16 ++--- drivers/cpufreq/powernow-k7.c | 4 +- drivers/crypto/ccp/sev-dev.c | 2 +- drivers/idle/intel_idle.c | 10 +-- drivers/platform/x86/intel/ifs/load.c | 10 +-- drivers/platform/x86/intel/ifs/runtest.c | 8 +-- drivers/platform/x86/intel/pmc/cnp.c | 4 +- .../intel/speed_select_if/isst_if_mbox_msr.c | 4 +- drivers/platform/x86/intel_ips.c | 16 ++--- drivers/thermal/intel/intel_hfi.c | 6 +- drivers/thermal/intel/therm_throt.c | 2 +- drivers/video/fbdev/geode/lxfb_ops.c | 10 +-- drivers/video/fbdev/geode/suspend_gx.c | 2 +- drivers/video/fbdev/geode/video_gx.c | 8 +-- include/hyperv/hvgdk_mini.h | 2 +- 86 files changed, 436 insertions(+), 436 deletions(-) diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index 86ef0fe8f435..ec4e8a4cace4 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -187,7 +187,7 @@ void amd_brs_reset(void) /* * Mark first entry as poisoned */ - wrmsrl(brs_to(0), BRS_POISON); + wrmsrq(brs_to(0), BRS_POISON); } int __init amd_brs_init(void) @@ -371,7 +371,7 @@ static void amd_brs_poison_buffer(void) idx = amd_brs_get_tos(&cfg); /* Poison target of entry */ - wrmsrl(brs_to(idx), BRS_POISON); + wrmsrq(brs_to(idx), BRS_POISON); } /* diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 94d6d2428f2a..cb62b6d12691 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -563,13 +563,13 @@ static void amd_pmu_cpu_reset(int cpu) return; /* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */ - wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0); + wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0); /* * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze * and PerfCntrGLobalStatus.PerfCntrOvfl */ - wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, + wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask); } @@ -651,7 +651,7 @@ static void amd_pmu_cpu_dead(int cpu) static __always_inline void amd_pmu_set_global_ctl(u64 ctl) { - wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl); + wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl); } static inline u64 amd_pmu_get_global_status(void) @@ -672,7 +672,7 @@ static inline void amd_pmu_ack_global_status(u64 status) * clears the same bit in PerfCntrGlobalStatus */ - wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status); + wrmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status); } static bool amd_pmu_test_overflow_topbit(int idx) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index ca6cc90f4719..82fa755d1b12 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -435,9 +435,9 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, u64 tmp = hwc->config | config; if (perf_ibs->fetch_count_reset_broken) - wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask); + wrmsrq(hwc->config_base, tmp & ~perf_ibs->enable_mask); - wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask); + wrmsrq(hwc->config_base, tmp | perf_ibs->enable_mask); } /* @@ -452,9 +452,9 @@ static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, { config &= ~perf_ibs->cnt_mask; if (boot_cpu_data.x86 == 0x10) - wrmsrl(hwc->config_base, config); + wrmsrq(hwc->config_base, config); config &= ~perf_ibs->enable_mask; - wrmsrl(hwc->config_base, config); + wrmsrq(hwc->config_base, config); } /* diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index a1aecf5c04a0..198851985bb7 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -61,12 +61,12 @@ struct branch_entry { static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val) { - wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); + wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val); } static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val) { - wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); + wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); } static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx) @@ -333,7 +333,7 @@ void amd_pmu_lbr_reset(void) cpuc->last_task_ctx = NULL; cpuc->last_log_id = 0; - wrmsrl(MSR_AMD64_LBR_SELECT, 0); + wrmsrq(MSR_AMD64_LBR_SELECT, 0); } void amd_pmu_lbr_add(struct perf_event *event) @@ -396,16 +396,16 @@ void amd_pmu_lbr_enable_all(void) /* Set hardware branch filter */ if (cpuc->lbr_select) { lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK; - wrmsrl(MSR_AMD64_LBR_SELECT, lbr_select); + wrmsrq(MSR_AMD64_LBR_SELECT, lbr_select); } if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); - wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); + wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); } rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); - wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); + wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); } void amd_pmu_lbr_disable_all(void) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 4ddf5429a063..2a3259df619a 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -121,10 +121,10 @@ static void amd_uncore_start(struct perf_event *event, int flags) struct hw_perf_event *hwc = &event->hw; if (flags & PERF_EF_RELOAD) - wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); + wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count)); hwc->state = 0; - wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); + wrmsrq(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); perf_event_update_userpage(event); } @@ -132,7 +132,7 @@ static void amd_uncore_stop(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config); + wrmsrq(hwc->config_base, hwc->config); hwc->state |= PERF_HES_STOPPED; if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { @@ -883,10 +883,10 @@ static void amd_uncore_umc_start(struct perf_event *event, int flags) struct hw_perf_event *hwc = &event->hw; if (flags & PERF_EF_RELOAD) - wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); + wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count)); hwc->state = 0; - wrmsrl(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC)); + wrmsrq(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC)); perf_event_update_userpage(event); } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index a27f2e680070..0a16d6bc11fa 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -697,9 +697,9 @@ void x86_pmu_disable_all(void) if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) continue; val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; - wrmsrl(x86_pmu_config_addr(idx), val); + wrmsrq(x86_pmu_config_addr(idx), val); if (is_counter_pair(hwc)) - wrmsrl(x86_pmu_config_addr(idx + 1), 0); + wrmsrq(x86_pmu_config_addr(idx + 1), 0); } } @@ -1420,14 +1420,14 @@ int x86_perf_event_set_period(struct perf_event *event) */ local64_set(&hwc->prev_count, (u64)-left); - wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); + wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); /* * Sign extend the Merge event counter's upper 16 bits since * we currently declare a 48-bit counter width */ if (is_counter_pair(hwc)) - wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff); + wrmsrq(x86_pmu_event_addr(idx + 1), 0xffff); perf_event_update_userpage(event); @@ -2496,9 +2496,9 @@ void perf_clear_dirty_counters(void) if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask))) continue; - wrmsrl(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0); + wrmsrq(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0); } else { - wrmsrl(x86_pmu_event_addr(i), 0); + wrmsrq(x86_pmu_event_addr(i), 0); } } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 852b1ea9bc17..3f4673708184 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2285,7 +2285,7 @@ static __always_inline void __intel_pmu_disable_all(bool bts) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0); if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) intel_pmu_disable_bts(); @@ -2306,11 +2306,11 @@ static void __intel_pmu_enable_all(int added, bool pmi) intel_pmu_lbr_enable_all(pmi); if (cpuc->fixed_ctrl_val != cpuc->active_fixed_ctrl_val) { - wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val); + wrmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val); cpuc->active_fixed_ctrl_val = cpuc->fixed_ctrl_val; } - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, intel_ctrl & ~cpuc->intel_ctrl_guest_mask); if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { @@ -2426,12 +2426,12 @@ static void intel_pmu_nhm_workaround(void) } for (i = 0; i < 4; i++) { - wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]); - wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0); + wrmsrq(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]); + wrmsrq(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0); } - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf); - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0xf); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0x0); for (i = 0; i < 4; i++) { event = cpuc->events[i]; @@ -2441,7 +2441,7 @@ static void intel_pmu_nhm_workaround(void) __x86_pmu_enable_event(&event->hw, ARCH_PERFMON_EVENTSEL_ENABLE); } else - wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0); + wrmsrq(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0); } } @@ -2458,7 +2458,7 @@ static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on) if (cpuc->tfa_shadow != val) { cpuc->tfa_shadow = val; - wrmsrl(MSR_TSX_FORCE_ABORT, val); + wrmsrq(MSR_TSX_FORCE_ABORT, val); } } @@ -2496,7 +2496,7 @@ static inline u64 intel_pmu_get_status(void) static inline void intel_pmu_ack_status(u64 ack) { - wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack); + wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack); } static inline bool event_is_checkpointed(struct perf_event *event) @@ -2619,15 +2619,15 @@ static int icl_set_topdown_event_period(struct perf_event *event) * Don't need to clear them again. */ if (left == x86_pmu.max_period) { - wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0); - wrmsrl(MSR_PERF_METRICS, 0); + wrmsrq(MSR_CORE_PERF_FIXED_CTR3, 0); + wrmsrq(MSR_PERF_METRICS, 0); hwc->saved_slots = 0; hwc->saved_metric = 0; } if ((hwc->saved_slots) && is_slots_event(event)) { - wrmsrl(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots); - wrmsrl(MSR_PERF_METRICS, hwc->saved_metric); + wrmsrq(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots); + wrmsrq(MSR_PERF_METRICS, hwc->saved_metric); } perf_event_update_userpage(event); @@ -2773,8 +2773,8 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end, if (reset) { /* The fixed counter 3 has to be written before the PERF_METRICS. */ - wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0); - wrmsrl(MSR_PERF_METRICS, 0); + wrmsrq(MSR_CORE_PERF_FIXED_CTR3, 0); + wrmsrq(MSR_PERF_METRICS, 0); if (event) update_saved_topdown_regs(event, 0, 0, metric_end); } @@ -2937,7 +2937,7 @@ int intel_pmu_save_and_restart(struct perf_event *event) */ if (unlikely(event_is_checkpointed(event))) { /* No race with NMIs because the counter should not be armed */ - wrmsrl(event->hw.event_base, 0); + wrmsrq(event->hw.event_base, 0); local64_set(&event->hw.prev_count, 0); } return static_call(x86_pmu_set_period)(event); @@ -2991,7 +2991,7 @@ static void intel_pmu_reset(void) /* Ack all overflows and disable fixed counters */ if (x86_pmu.version >= 2) { intel_pmu_ack_status(intel_pmu_get_status()); - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0); } /* Reset LBRs and LBR freezing */ @@ -3103,7 +3103,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) * Update the MSR if pebs_enabled is changed. */ if (pebs_enabled != cpuc->pebs_enabled) - wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); + wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); } /* @@ -5614,7 +5614,7 @@ static bool check_msr(unsigned long msr, u64 mask) return false; /* - * Only change the bits which can be updated by wrmsrl. + * Only change the bits which can be updated by wrmsrq. */ val_tmp = val_old ^ mask; @@ -5626,7 +5626,7 @@ static bool check_msr(unsigned long msr, u64 mask) return false; /* - * Quirk only affects validation in wrmsr(), so wrmsrl()'s value + * Quirk only affects validation in wrmsr(), so wrmsrq()'s value * should equal rdmsrq()'s even with the quirk. */ if (val_new != val_tmp) @@ -5638,7 +5638,7 @@ static bool check_msr(unsigned long msr, u64 mask) /* Here it's sure that the MSR can be safely accessed. * Restore the old value and return. */ - wrmsrl(msr, val_old); + wrmsrq(msr, val_old); return true; } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 1f7e1a692a7a..410a8975d1b9 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1515,7 +1515,7 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) else value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx]; } - wrmsrl(base + idx, value); + wrmsrq(base + idx, value); } static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc) @@ -1552,7 +1552,7 @@ void intel_pmu_pebs_enable(struct perf_event *event) */ intel_pmu_drain_pebs_buffer(); adaptive_pebs_record_size_update(); - wrmsrl(MSR_PEBS_DATA_CFG, pebs_data_cfg); + wrmsrq(MSR_PEBS_DATA_CFG, pebs_data_cfg); cpuc->active_pebs_data_cfg = pebs_data_cfg; } } @@ -1615,7 +1615,7 @@ void intel_pmu_pebs_disable(struct perf_event *event) intel_pmu_pebs_via_pt_disable(event); if (cpuc->enabled) - wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); + wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); hwc->config |= ARCH_PERFMON_EVENTSEL_INT; } @@ -1625,7 +1625,7 @@ void intel_pmu_pebs_enable_all(void) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); if (cpuc->pebs_enabled) - wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); + wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); } void intel_pmu_pebs_disable_all(void) @@ -2771,5 +2771,5 @@ void perf_restore_debug_store(void) if (!x86_pmu.bts && !x86_pmu.pebs) return; - wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds); + wrmsrq(MSR_IA32_DS_AREA, (unsigned long)ds); } diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index 0ee32a0b5bfd..afb7801bdece 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -161,7 +161,7 @@ static void knc_pmu_disable_all(void) rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); val &= ~(KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1); - wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); + wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); } static void knc_pmu_enable_all(int added) @@ -170,7 +170,7 @@ static void knc_pmu_enable_all(int added) rdmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); val |= (KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1); - wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); + wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val); } static inline void @@ -207,7 +207,7 @@ static inline u64 knc_pmu_get_status(void) static inline void knc_pmu_ack_status(u64 ack) { - wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL, ack); + wrmsrq(MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL, ack); } static int knc_pmu_handle_irq(struct pt_regs *regs) diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 2b33aacb5983..b55c59ab53d2 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -137,7 +137,7 @@ static void __intel_pmu_lbr_enable(bool pmi) if (cpuc->lbr_sel) lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) - wrmsrl(MSR_LBR_SELECT, lbr_select); + wrmsrq(MSR_LBR_SELECT, lbr_select); rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); orig_debugctl = debugctl; @@ -155,10 +155,10 @@ static void __intel_pmu_lbr_enable(bool pmi) debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; if (orig_debugctl != debugctl) - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); if (static_cpu_has(X86_FEATURE_ARCH_LBR)) - wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); + wrmsrq(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); } void intel_pmu_lbr_reset_32(void) @@ -166,7 +166,7 @@ void intel_pmu_lbr_reset_32(void) int i; for (i = 0; i < x86_pmu.lbr_nr; i++) - wrmsrl(x86_pmu.lbr_from + i, 0); + wrmsrq(x86_pmu.lbr_from + i, 0); } void intel_pmu_lbr_reset_64(void) @@ -174,17 +174,17 @@ void intel_pmu_lbr_reset_64(void) int i; for (i = 0; i < x86_pmu.lbr_nr; i++) { - wrmsrl(x86_pmu.lbr_from + i, 0); - wrmsrl(x86_pmu.lbr_to + i, 0); + wrmsrq(x86_pmu.lbr_from + i, 0); + wrmsrq(x86_pmu.lbr_to + i, 0); if (x86_pmu.lbr_has_info) - wrmsrl(x86_pmu.lbr_info + i, 0); + wrmsrq(x86_pmu.lbr_info + i, 0); } } static void intel_pmu_arch_lbr_reset(void) { /* Write to ARCH_LBR_DEPTH MSR, all LBR entries are reset to 0 */ - wrmsrl(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr); + wrmsrq(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr); } void intel_pmu_lbr_reset(void) @@ -199,7 +199,7 @@ void intel_pmu_lbr_reset(void) cpuc->last_task_ctx = NULL; cpuc->last_log_id = 0; if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) - wrmsrl(MSR_LBR_SELECT, 0); + wrmsrq(MSR_LBR_SELECT, 0); } /* @@ -282,17 +282,17 @@ static u64 lbr_from_signext_quirk_rd(u64 val) static __always_inline void wrlbr_from(unsigned int idx, u64 val) { val = lbr_from_signext_quirk_wr(val); - wrmsrl(x86_pmu.lbr_from + idx, val); + wrmsrq(x86_pmu.lbr_from + idx, val); } static __always_inline void wrlbr_to(unsigned int idx, u64 val) { - wrmsrl(x86_pmu.lbr_to + idx, val); + wrmsrq(x86_pmu.lbr_to + idx, val); } static __always_inline void wrlbr_info(unsigned int idx, u64 val) { - wrmsrl(x86_pmu.lbr_info + idx, val); + wrmsrq(x86_pmu.lbr_info + idx, val); } static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) @@ -380,10 +380,10 @@ void intel_pmu_lbr_restore(void *ctx) wrlbr_info(lbr_idx, 0); } - wrmsrl(x86_pmu.lbr_tos, tos); + wrmsrq(x86_pmu.lbr_tos, tos); if (cpuc->lbr_select) - wrmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel); + wrmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); } static void intel_pmu_arch_lbr_restore(void *ctx) diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index a0eb63b6c859..36d4cd23182c 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -861,7 +861,7 @@ static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc) /* an official way for overflow indication */ rdmsrq(hwc->config_base, v); if (v & P4_CCCR_OVF) { - wrmsrl(hwc->config_base, v & ~P4_CCCR_OVF); + wrmsrq(hwc->config_base, v & ~P4_CCCR_OVF); return 1; } @@ -1024,7 +1024,7 @@ static int p4_pmu_set_period(struct perf_event *event) * * the former idea is taken from OProfile code */ - wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); + wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); } return ret; diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c index 701f8a8ab0ba..6a1fdbc6d20f 100644 --- a/arch/x86/events/intel/p6.c +++ b/arch/x86/events/intel/p6.c @@ -144,7 +144,7 @@ static void p6_pmu_disable_all(void) /* p6 only has one enable register */ rdmsrq(MSR_P6_EVNTSEL0, val); val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; - wrmsrl(MSR_P6_EVNTSEL0, val); + wrmsrq(MSR_P6_EVNTSEL0, val); } static void p6_pmu_enable_all(int added) @@ -154,7 +154,7 @@ static void p6_pmu_enable_all(int added) /* p6 only has one enable register */ rdmsrq(MSR_P6_EVNTSEL0, val); val |= ARCH_PERFMON_EVENTSEL_ENABLE; - wrmsrl(MSR_P6_EVNTSEL0, val); + wrmsrq(MSR_P6_EVNTSEL0, val); } static inline void diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index b8212d8d0174..a645b0993d83 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -426,7 +426,7 @@ static void pt_config_start(struct perf_event *event) if (READ_ONCE(pt->vmx_on)) perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL); else - wrmsrl(MSR_IA32_RTIT_CTL, ctl); + wrmsrq(MSR_IA32_RTIT_CTL, ctl); WRITE_ONCE(event->hw.aux_config, ctl); } @@ -485,12 +485,12 @@ static u64 pt_config_filters(struct perf_event *event) /* avoid redundant msr writes */ if (pt->filters.filter[range].msr_a != filter->msr_a) { - wrmsrl(pt_address_ranges[range].msr_a, filter->msr_a); + wrmsrq(pt_address_ranges[range].msr_a, filter->msr_a); pt->filters.filter[range].msr_a = filter->msr_a; } if (pt->filters.filter[range].msr_b != filter->msr_b) { - wrmsrl(pt_address_ranges[range].msr_b, filter->msr_b); + wrmsrq(pt_address_ranges[range].msr_b, filter->msr_b); pt->filters.filter[range].msr_b = filter->msr_b; } @@ -509,7 +509,7 @@ static void pt_config(struct perf_event *event) /* First round: clear STATUS, in particular the PSB byte counter. */ if (!event->hw.aux_config) { perf_event_itrace_started(event); - wrmsrl(MSR_IA32_RTIT_STATUS, 0); + wrmsrq(MSR_IA32_RTIT_STATUS, 0); } reg = pt_config_filters(event); @@ -569,7 +569,7 @@ static void pt_config_stop(struct perf_event *event) ctl &= ~RTIT_CTL_TRACEEN; if (!READ_ONCE(pt->vmx_on)) - wrmsrl(MSR_IA32_RTIT_CTL, ctl); + wrmsrq(MSR_IA32_RTIT_CTL, ctl); WRITE_ONCE(event->hw.aux_config, ctl); @@ -658,13 +658,13 @@ static void pt_config_buffer(struct pt_buffer *buf) reg = virt_to_phys(base); if (pt->output_base != reg) { pt->output_base = reg; - wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, reg); + wrmsrq(MSR_IA32_RTIT_OUTPUT_BASE, reg); } reg = 0x7f | (mask << 7) | ((u64)buf->output_off << 32); if (pt->output_mask != reg) { pt->output_mask = reg; - wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, reg); + wrmsrq(MSR_IA32_RTIT_OUTPUT_MASK, reg); } } @@ -970,7 +970,7 @@ static void pt_handle_status(struct pt *pt) if (advance) pt_buffer_advance(buf); - wrmsrl(MSR_IA32_RTIT_STATUS, status); + wrmsrq(MSR_IA32_RTIT_STATUS, status); } /** @@ -1585,7 +1585,7 @@ void intel_pt_handle_vmx(int on) /* Turn PTs back on */ if (!on && event) - wrmsrl(MSR_IA32_RTIT_CTL, event->hw.aux_config); + wrmsrq(MSR_IA32_RTIT_CTL, event->hw.aux_config); local_irq_restore(flags); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 571e44b49691..4fc3eec325f6 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -441,17 +441,17 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { - wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); + wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) { - wrmsrl(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); + wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); } void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrl(intel_generic_uncore_box_ctl(box), 0); + wrmsrq(intel_generic_uncore_box_ctl(box), 0); } static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, @@ -459,7 +459,7 @@ static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config); + wrmsrq(hwc->config_base, hwc->config); } static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box, @@ -467,7 +467,7 @@ static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box, { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, 0); + wrmsrq(hwc->config_base, 0); } static struct intel_uncore_ops generic_uncore_msr_ops = { diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index e6fccad9708d..bef9c782c781 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -200,12 +200,12 @@ DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63"); static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box) { - wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); + wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); } static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box) { - wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, 0); + wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, 0); } static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -219,7 +219,7 @@ static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box) /* WBox has a fixed counter */ if (uncore_msr_fixed_ctl(box)) config &= ~NHMEX_W_PMON_GLOBAL_FIXED_EN; - wrmsrl(msr, config); + wrmsrq(msr, config); } } @@ -234,13 +234,13 @@ static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box) /* WBox has a fixed counter */ if (uncore_msr_fixed_ctl(box)) config |= NHMEX_W_PMON_GLOBAL_FIXED_EN; - wrmsrl(msr, config); + wrmsrq(msr, config); } } static void nhmex_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event) { - wrmsrl(event->hw.config_base, 0); + wrmsrq(event->hw.config_base, 0); } static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event) @@ -248,11 +248,11 @@ static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct p struct hw_perf_event *hwc = &event->hw; if (hwc->idx == UNCORE_PMC_IDX_FIXED) - wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0); + wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0); else if (box->pmu->type->event_mask & NHMEX_PMON_CTL_EN_BIT0) - wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22); + wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22); else - wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0); + wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0); } #define NHMEX_UNCORE_OPS_COMMON_INIT() \ @@ -382,10 +382,10 @@ static void nhmex_bbox_msr_enable_event(struct intel_uncore_box *box, struct per struct hw_perf_event_extra *reg2 = &hwc->branch_reg; if (reg1->idx != EXTRA_REG_NONE) { - wrmsrl(reg1->reg, reg1->config); - wrmsrl(reg1->reg + 1, reg2->config); + wrmsrq(reg1->reg, reg1->config); + wrmsrq(reg1->reg + 1, reg2->config); } - wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 | + wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 | (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK)); } @@ -467,12 +467,12 @@ static void nhmex_sbox_msr_enable_event(struct intel_uncore_box *box, struct per struct hw_perf_event_extra *reg2 = &hwc->branch_reg; if (reg1->idx != EXTRA_REG_NONE) { - wrmsrl(reg1->reg, 0); - wrmsrl(reg1->reg + 1, reg1->config); - wrmsrl(reg1->reg + 2, reg2->config); - wrmsrl(reg1->reg, NHMEX_S_PMON_MM_CFG_EN); + wrmsrq(reg1->reg, 0); + wrmsrq(reg1->reg + 1, reg1->config); + wrmsrq(reg1->reg + 2, reg2->config); + wrmsrq(reg1->reg, NHMEX_S_PMON_MM_CFG_EN); } - wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22); + wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22); } static struct attribute *nhmex_uncore_sbox_formats_attr[] = { @@ -842,25 +842,25 @@ static void nhmex_mbox_msr_enable_event(struct intel_uncore_box *box, struct per idx = __BITS_VALUE(reg1->idx, 0, 8); if (idx != 0xff) - wrmsrl(__BITS_VALUE(reg1->reg, 0, 16), + wrmsrq(__BITS_VALUE(reg1->reg, 0, 16), nhmex_mbox_shared_reg_config(box, idx)); idx = __BITS_VALUE(reg1->idx, 1, 8); if (idx != 0xff) - wrmsrl(__BITS_VALUE(reg1->reg, 1, 16), + wrmsrq(__BITS_VALUE(reg1->reg, 1, 16), nhmex_mbox_shared_reg_config(box, idx)); if (reg2->idx != EXTRA_REG_NONE) { - wrmsrl(reg2->reg, 0); + wrmsrq(reg2->reg, 0); if (reg2->config != ~0ULL) { - wrmsrl(reg2->reg + 1, + wrmsrq(reg2->reg + 1, reg2->config & NHMEX_M_PMON_ADDR_MATCH_MASK); - wrmsrl(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK & + wrmsrq(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK & (reg2->config >> NHMEX_M_PMON_ADDR_MASK_SHIFT)); - wrmsrl(reg2->reg, NHMEX_M_PMON_MM_CFG_EN); + wrmsrq(reg2->reg, NHMEX_M_PMON_MM_CFG_EN); } } - wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0); + wrmsrq(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0); } DEFINE_UNCORE_FORMAT_ATTR(count_mode, count_mode, "config:2-3"); @@ -1121,31 +1121,31 @@ static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct per switch (idx % 6) { case 0: - wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config); + wrmsrq(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config); break; case 1: - wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config); + wrmsrq(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config); break; case 2: case 3: - wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port), + wrmsrq(NHMEX_R_MSR_PORTN_QLX_CFG(port), uncore_shared_reg_config(box, 2 + (idx / 6) * 5)); break; case 4: - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port), + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port), hwc->config >> 32); - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config); - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config); + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config); + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config); break; case 5: - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port), + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port), hwc->config >> 32); - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config); - wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config); + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config); + wrmsrq(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config); break; } - wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 | + wrmsrq(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 | (hwc->config & NHMEX_R_PMON_CTL_EV_SEL_MASK)); } diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 66dadf58799b..afc8ef02a7a9 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -260,34 +260,34 @@ static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct per struct hw_perf_event *hwc = &event->hw; if (hwc->idx < UNCORE_PMC_IDX_FIXED) - wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNB_UNC_CTL_EN); else - wrmsrl(hwc->config_base, SNB_UNC_CTL_EN); + wrmsrq(hwc->config_base, SNB_UNC_CTL_EN); } static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event) { - wrmsrl(event->hw.config_base, 0); + wrmsrq(event->hw.config_base, 0); } static void snb_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { - wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, + wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } } static void snb_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, + wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } static void snb_uncore_msr_exit_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, 0); + wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, 0); } static struct uncore_event_desc snb_uncore_events[] = { @@ -372,7 +372,7 @@ void snb_uncore_cpu_init(void) static void skl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { - wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, + wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL); } @@ -383,14 +383,14 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box) static void skl_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, + wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL); } static void skl_uncore_msr_exit_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, 0); + wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, 0); } static struct intel_uncore_ops skl_uncore_msr_ops = { @@ -525,7 +525,7 @@ static struct intel_uncore_type *tgl_msr_uncores[] = { static void rkl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); } void tgl_uncore_cpu_init(void) @@ -541,24 +541,24 @@ void tgl_uncore_cpu_init(void) static void adl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); } static void adl_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); } static void adl_uncore_msr_disable_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, 0); + wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, 0); } static void adl_uncore_msr_exit_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(ADL_UNC_PERF_GLOBAL_CTL, 0); + wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, 0); } static struct intel_uncore_ops adl_uncore_msr_ops = { @@ -691,7 +691,7 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = { static void mtl_uncore_msr_init_box(struct intel_uncore_box *box) { - wrmsrl(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); + wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); } static struct intel_uncore_ops mtl_uncore_msr_ops = { @@ -758,7 +758,7 @@ static struct intel_uncore_type *lnl_msr_uncores[] = { static void lnl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) - wrmsrl(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); } static struct intel_uncore_ops lnl_uncore_msr_ops = { @@ -1306,12 +1306,12 @@ int skl_uncore_pci_init(void) /* Nehalem uncore support */ static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box) { - wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0); + wrmsrq(NHM_UNC_PERF_GLOBAL_CTL, 0); } static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC); + wrmsrq(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC); } static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event) @@ -1319,9 +1319,9 @@ static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct per struct hw_perf_event *hwc = &event->hw; if (hwc->idx < UNCORE_PMC_IDX_FIXED) - wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNB_UNC_CTL_EN); else - wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN); + wrmsrq(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN); } static struct attribute *nhm_uncore_formats_attr[] = { diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index e498485e0ae9..756dd11742ad 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -620,7 +620,7 @@ static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) if (msr) { rdmsrq(msr, config); config |= SNBEP_PMON_BOX_CTL_FRZ; - wrmsrl(msr, config); + wrmsrq(msr, config); } } @@ -633,7 +633,7 @@ static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box) if (msr) { rdmsrq(msr, config); config &= ~SNBEP_PMON_BOX_CTL_FRZ; - wrmsrl(msr, config); + wrmsrq(msr, config); } } @@ -643,9 +643,9 @@ static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct p struct hw_perf_event_extra *reg1 = &hwc->extra_reg; if (reg1->idx != EXTRA_REG_NONE) - wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0)); + wrmsrq(reg1->reg, uncore_shared_reg_config(box, 0)); - wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); } static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, @@ -653,7 +653,7 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config); + wrmsrq(hwc->config_base, hwc->config); } static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) @@ -661,7 +661,7 @@ static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) unsigned msr = uncore_msr_box_ctl(box); if (msr) - wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT); + wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT); } static struct attribute *snbep_uncore_formats_attr[] = { @@ -1532,7 +1532,7 @@ static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) - wrmsrl(msr, IVBEP_PMON_BOX_CTL_INT); + wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT); } static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) @@ -1783,11 +1783,11 @@ static void ivbep_cbox_enable_event(struct intel_uncore_box *box, struct perf_ev if (reg1->idx != EXTRA_REG_NONE) { u64 filter = uncore_shared_reg_config(box, 0); - wrmsrl(reg1->reg, filter & 0xffffffff); - wrmsrl(reg1->reg + 6, filter >> 32); + wrmsrq(reg1->reg, filter & 0xffffffff); + wrmsrq(reg1->reg + 6, filter >> 32); } - wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); } static struct intel_uncore_ops ivbep_uncore_cbox_ops = { @@ -2767,11 +2767,11 @@ static void hswep_cbox_enable_event(struct intel_uncore_box *box, if (reg1->idx != EXTRA_REG_NONE) { u64 filter = uncore_shared_reg_config(box, 0); - wrmsrl(reg1->reg, filter & 0xffffffff); - wrmsrl(reg1->reg + 1, filter >> 32); + wrmsrq(reg1->reg, filter & 0xffffffff); + wrmsrq(reg1->reg + 1, filter >> 32); } - wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); } static struct intel_uncore_ops hswep_uncore_cbox_ops = { @@ -2816,7 +2816,7 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) for_each_set_bit(i, (unsigned long *)&init, 64) { flags |= (1ULL << i); - wrmsrl(msr, flags); + wrmsrq(msr, flags); } } } @@ -3708,7 +3708,7 @@ static void skx_iio_enable_event(struct intel_uncore_box *box, { struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); } static struct intel_uncore_ops skx_uncore_iio_ops = { @@ -4655,9 +4655,9 @@ static void snr_cha_enable_event(struct intel_uncore_box *box, struct hw_perf_event_extra *reg1 = &hwc->extra_reg; if (reg1->idx != EXTRA_REG_NONE) - wrmsrl(reg1->reg, reg1->config); + wrmsrq(reg1->reg, reg1->config); - wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); + wrmsrq(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); } static struct intel_uncore_ops snr_uncore_chabox_ops = { @@ -5913,9 +5913,9 @@ static void spr_uncore_msr_enable_event(struct intel_uncore_box *box, struct hw_perf_event_extra *reg1 = &hwc->extra_reg; if (reg1->idx != EXTRA_REG_NONE) - wrmsrl(reg1->reg, reg1->config); + wrmsrq(reg1->reg, reg1->config); - wrmsrl(hwc->config_base, hwc->config); + wrmsrq(hwc->config_base, hwc->config); } static void spr_uncore_msr_disable_event(struct intel_uncore_box *box, @@ -5925,9 +5925,9 @@ static void spr_uncore_msr_disable_event(struct intel_uncore_box *box, struct hw_perf_event_extra *reg1 = &hwc->extra_reg; if (reg1->idx != EXTRA_REG_NONE) - wrmsrl(reg1->reg, 0); + wrmsrq(reg1->reg, 0); - wrmsrl(hwc->config_base, 0); + wrmsrq(hwc->config_base, 0); } static int spr_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 34b6d35e76e8..a5166fa9339b 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1198,16 +1198,16 @@ static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask); if (hwc->extra_reg.reg) - wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config); + wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config); /* * Add enabled Merge event on next counter * if large increment event being enabled on this counter */ if (is_counter_pair(hwc)) - wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en); + wrmsrq(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en); - wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask); + wrmsrq(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask); } void x86_pmu_enable_all(int added); @@ -1223,10 +1223,10 @@ static inline void x86_pmu_disable_event(struct perf_event *event) u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask); struct hw_perf_event *hwc = &event->hw; - wrmsrl(hwc->config_base, hwc->config & ~disable_mask); + wrmsrq(hwc->config_base, hwc->config & ~disable_mask); if (is_counter_pair(hwc)) - wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0); + wrmsrq(x86_pmu_config_addr(hwc->idx + 1), 0); } void x86_pmu_enable_event(struct perf_event *event); @@ -1395,11 +1395,11 @@ static __always_inline void __amd_pmu_lbr_disable(void) u64 dbg_ctl, dbg_extn_cfg; rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); - wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); + wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); - wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); + wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); } } @@ -1531,12 +1531,12 @@ static inline bool intel_pmu_has_bts(struct perf_event *event) static __always_inline void __intel_pmu_pebs_disable_all(void) { - wrmsrl(MSR_IA32_PEBS_ENABLE, 0); + wrmsrq(MSR_IA32_PEBS_ENABLE, 0); } static __always_inline void __intel_pmu_arch_lbr_disable(void) { - wrmsrl(MSR_ARCH_LBR_CTL, 0); + wrmsrq(MSR_ARCH_LBR_CTL, 0); } static __always_inline void __intel_pmu_lbr_disable(void) @@ -1545,7 +1545,7 @@ static __always_inline void __intel_pmu_lbr_disable(void) rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } int intel_pmu_save_and_restart(struct perf_event *event); diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index 67ff23f6bcba..e299364eb889 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -254,12 +254,12 @@ static __initconst const u64 zxe_hw_cache_event_ids static void zhaoxin_pmu_disable_all(void) { - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, 0); } static void zhaoxin_pmu_enable_all(int added) { - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl); + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl); } static inline u64 zhaoxin_pmu_get_status(void) @@ -273,7 +273,7 @@ static inline u64 zhaoxin_pmu_get_status(void) static inline void zhaoxin_pmu_ack_status(u64 ack) { - wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack); + wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack); } static inline void zxc_pmu_ack_status(u64 ack) @@ -295,7 +295,7 @@ static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc) rdmsrq(hwc->config_base, ctrl_val); ctrl_val &= ~mask; - wrmsrl(hwc->config_base, ctrl_val); + wrmsrq(hwc->config_base, ctrl_val); } static void zhaoxin_pmu_disable_event(struct perf_event *event) @@ -332,7 +332,7 @@ static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc) rdmsrq(hwc->config_base, ctrl_val); ctrl_val &= ~mask; ctrl_val |= bits; - wrmsrl(hwc->config_base, ctrl_val); + wrmsrq(hwc->config_base, ctrl_val); } static void zhaoxin_pmu_enable_event(struct perf_event *event) diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index d1d6aa5fb17b..c450e67cb0a4 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -49,7 +49,7 @@ static void hv_apic_icr_write(u32 low, u32 id) reg_val = reg_val << 32; reg_val |= low; - wrmsrl(HV_X64_MSR_ICR, reg_val); + wrmsrq(HV_X64_MSR_ICR, reg_val); } static u32 hv_apic_read(u32 reg) diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 7c58affa129b..ed89867b6fd0 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -128,7 +128,7 @@ static int hv_cpu_init(unsigned int cpu) } if (!WARN_ON(!(*hvp))) { msr.enable = 1; - wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); + wrmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); } return hyperv_init_ghcb(); @@ -155,7 +155,7 @@ void hyperv_stop_tsc_emulation(void) rdmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); emu_status.inprogress = 0; - wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); + wrmsrq(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status); rdmsrq(HV_X64_MSR_TSC_FREQUENCY, freq); tsc_khz = div64_u64(freq, 1000); @@ -203,8 +203,8 @@ void set_hv_tscchange_cb(void (*cb)(void)) re_ctrl.target_vp = hv_vp_index[get_cpu()]; - wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); - wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); + wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); + wrmsrq(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl)); put_cpu(); } @@ -219,7 +219,7 @@ void clear_hv_tscchange_cb(void) rdmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); re_ctrl.enabled = 0; - wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); + wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl); hv_reenlightenment_cb = NULL; } @@ -254,7 +254,7 @@ static int hv_cpu_die(unsigned int cpu) rdmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); msr.enable = 0; } - wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); + wrmsrq(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64); } if (hv_reenlightenment_cb == NULL) @@ -274,7 +274,7 @@ static int hv_cpu_die(unsigned int cpu) else re_ctrl.enabled = 0; - wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); + wrmsrq(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl)); } return 0; @@ -333,7 +333,7 @@ static int hv_suspend(void) /* Disable the hypercall page in the hypervisor */ rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hypercall_msr.enable = 0; - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); ret = hv_cpu_die(0); return ret; @@ -352,7 +352,7 @@ static void hv_resume(void) hypercall_msr.enable = 1; hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg_saved); - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hv_hypercall_pg = hv_hypercall_pg_saved; hv_hypercall_pg_saved = NULL; @@ -499,7 +499,7 @@ void __init hyperv_init(void) * in such a VM and is only used in such a VM. */ guest_id = hv_generate_guest_id(LINUX_VERSION_CODE); - wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); + wrmsrq(HV_X64_MSR_GUEST_OS_ID, guest_id); /* With the paravisor, the VM must also write the ID via GHCB/GHCI */ hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id); @@ -532,7 +532,7 @@ void __init hyperv_init(void) * so it is populated with code, then copy the code to an * executable page. */ - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); pg = vmalloc_to_page(hv_hypercall_pg); src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE, @@ -544,7 +544,7 @@ void __init hyperv_init(void) hv_remap_tsc_clocksource(); } else { hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); } skip_hypercall_pg_init: @@ -608,7 +608,7 @@ skip_hypercall_pg_init: return; clean_guest_os_id: - wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); + wrmsrq(HV_X64_MSR_GUEST_OS_ID, 0); hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); cpuhp_remove_state(CPUHP_AP_HYPERV_ONLINE); free_ghcb_page: @@ -629,7 +629,7 @@ void hyperv_cleanup(void) union hv_reference_tsc_msr tsc_msr; /* Reset our OS id */ - wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); + wrmsrq(HV_X64_MSR_GUEST_OS_ID, 0); hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); /* @@ -669,16 +669,16 @@ void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die) rdmsrq(HV_X64_MSR_GUEST_OS_ID, guest_id); - wrmsrl(HV_X64_MSR_CRASH_P0, err); - wrmsrl(HV_X64_MSR_CRASH_P1, guest_id); - wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip); - wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax); - wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp); + wrmsrq(HV_X64_MSR_CRASH_P0, err); + wrmsrq(HV_X64_MSR_CRASH_P1, guest_id); + wrmsrq(HV_X64_MSR_CRASH_P2, regs->ip); + wrmsrq(HV_X64_MSR_CRASH_P3, regs->ax); + wrmsrq(HV_X64_MSR_CRASH_P4, regs->sp); /* * Let Hyper-V know there is crash data available */ - wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); + wrmsrq(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY); } EXPORT_SYMBOL_GPL(hyperv_report_panic); diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 45819cfc043b..36e17988b7ba 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -230,7 +230,7 @@ static inline u32 native_apic_msr_read(u32 reg) static inline void native_x2apic_icr_write(u32 low, u32 id) { - wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low); + wrmsrq(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low); } static inline u64 native_x2apic_icr_read(void) diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h index 0e703c0ef192..da6aedbadd64 100644 --- a/arch/x86/include/asm/debugreg.h +++ b/arch/x86/include/asm/debugreg.h @@ -180,7 +180,7 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr) if (boot_cpu_data.x86 < 6) return; #endif - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); + wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctlmsr); } #endif /* _ASM_X86_DEBUGREG_H */ diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h index d7708f2fd68f..ab2547f97c2c 100644 --- a/arch/x86/include/asm/fsgsbase.h +++ b/arch/x86/include/asm/fsgsbase.h @@ -70,7 +70,7 @@ static inline void x86_fsbase_write_cpu(unsigned long fsbase) if (boot_cpu_has(X86_FEATURE_FSGSBASE)) wrfsbase(fsbase); else - wrmsrl(MSR_FS_BASE, fsbase); + wrmsrq(MSR_FS_BASE, fsbase); } extern unsigned long x86_gsbase_read_cpu_inactive(void); diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index b35513a81b24..2e456cd2c97e 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -258,7 +258,7 @@ static inline void wrmsr(u32 msr, u32 low, u32 high) #define rdmsrq(msr, val) \ ((val) = native_read_msr((msr))) -static inline void wrmsrl(u32 msr, u64 val) +static inline void wrmsrq(u32 msr, u64 val) { native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); } @@ -357,7 +357,7 @@ static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) } static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { - wrmsrl(msr_no, q); + wrmsrq(msr_no, q); return 0; } static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 1cda83bbd5b7..87c2597c68f2 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -214,7 +214,7 @@ do { \ val = paravirt_read_msr(msr); \ } while (0) -static inline void wrmsrl(unsigned msr, u64 val) +static inline void wrmsrq(unsigned msr, u64 val) { wrmsr(msr, (u32)val, (u32)(val>>32)); } diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index dc9af058e5a9..a05871c85183 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -425,7 +425,7 @@ static int lapic_next_deadline(unsigned long delta, weak_wrmsr_fence(); tsc = rdtsc(); - wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); + wrmsrq(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR)); return 0; } @@ -449,7 +449,7 @@ static int lapic_timer_shutdown(struct clock_event_device *evt) * the timer _and_ zero the counter registers: */ if (v & APIC_LVT_TIMER_TSCDEADLINE) - wrmsrl(MSR_IA32_TSC_DEADLINE, 0); + wrmsrq(MSR_IA32_TSC_DEADLINE, 0); else apic_write(APIC_TMICT, 0); @@ -1711,8 +1711,8 @@ static void __x2apic_disable(void) if (!(msr & X2APIC_ENABLE)) return; /* Disable xapic and x2apic first and then reenable xapic mode */ - wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); - wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); + wrmsrq(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE)); + wrmsrq(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE); printk_once(KERN_INFO "x2apic disabled\n"); } @@ -1723,7 +1723,7 @@ static void __x2apic_enable(void) rdmsrq(MSR_IA32_APICBASE, msr); if (msr & X2APIC_ENABLE) return; - wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); + wrmsrq(MSR_IA32_APICBASE, msr | X2APIC_ENABLE); printk_once(KERN_INFO "x2apic enabled\n"); } diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 78ffe0ebdff2..86c1f0ce4eae 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -71,7 +71,7 @@ void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; static void update_spec_ctrl(u64 val) { this_cpu_write(x86_spec_ctrl_current, val); - wrmsrl(MSR_IA32_SPEC_CTRL, val); + wrmsrq(MSR_IA32_SPEC_CTRL, val); } /* @@ -90,7 +90,7 @@ void update_spec_ctrl_cond(u64 val) * forced the update can be delayed until that time. */ if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) - wrmsrl(MSR_IA32_SPEC_CTRL, val); + wrmsrq(MSR_IA32_SPEC_CTRL, val); } noinstr u64 spec_ctrl_current(void) @@ -228,9 +228,9 @@ static void x86_amd_ssb_disable(void) u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) - wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); + wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) - wrmsrl(MSR_AMD64_LS_CFG, msrval); + wrmsrq(MSR_AMD64_LS_CFG, msrval); } #undef pr_fmt @@ -670,7 +670,7 @@ void update_srbds_msr(void) break; } - wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); } static void __init srbds_select_mitigation(void) @@ -795,7 +795,7 @@ void update_gds_msr(void) return; } - wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); + wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); /* * Check to make sure that the WRMSR value was not ignored. Writes to diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index 725072b82816..a96cfdcd94a0 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -145,7 +145,7 @@ static void __init __split_lock_setup(void) } /* Restore the MSR to its cached value. */ - wrmsrl(MSR_TEST_CTRL, msr_test_ctrl_cache); + wrmsrq(MSR_TEST_CTRL, msr_test_ctrl_cache); setup_force_cpu_cap(X86_FEATURE_SPLIT_LOCK_DETECT); } @@ -162,7 +162,7 @@ static void sld_update_msr(bool on) if (on) test_ctrl_val |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; - wrmsrl(MSR_TEST_CTRL, test_ctrl_val); + wrmsrq(MSR_TEST_CTRL, test_ctrl_val); } void split_lock_init(void) @@ -311,7 +311,7 @@ void bus_lock_init(void) val |= DEBUGCTLMSR_BUS_LOCK_DETECT; } - wrmsrl(MSR_IA32_DEBUGCTLMSR, val); + wrmsrq(MSR_IA32_DEBUGCTLMSR, val); } bool handle_user_split_lock(struct pt_regs *regs, long error_code) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index c3c0ba239828..a1f1be84a55f 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -564,7 +564,7 @@ __noendbr u64 ibt_save(bool disable) if (cpu_feature_enabled(X86_FEATURE_IBT)) { rdmsrq(MSR_IA32_S_CET, msr); if (disable) - wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN); + wrmsrq(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN); } return msr; @@ -578,7 +578,7 @@ __noendbr void ibt_restore(u64 save) rdmsrq(MSR_IA32_S_CET, msr); msr &= ~CET_ENDBR_EN; msr |= (save & CET_ENDBR_EN); - wrmsrl(MSR_IA32_S_CET, msr); + wrmsrq(MSR_IA32_S_CET, msr); } } @@ -602,15 +602,15 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_USER_SHSTK); if (kernel_ibt) - wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN); + wrmsrq(MSR_IA32_S_CET, CET_ENDBR_EN); else - wrmsrl(MSR_IA32_S_CET, 0); + wrmsrq(MSR_IA32_S_CET, 0); cr4_set_bits(X86_CR4_CET); if (kernel_ibt && ibt_selftest()) { pr_err("IBT selftest: Failed!\n"); - wrmsrl(MSR_IA32_S_CET, 0); + wrmsrq(MSR_IA32_S_CET, 0); setup_clear_cpu_cap(X86_FEATURE_IBT); } } @@ -621,8 +621,8 @@ __noendbr void cet_disable(void) cpu_feature_enabled(X86_FEATURE_SHSTK))) return; - wrmsrl(MSR_IA32_S_CET, 0); - wrmsrl(MSR_IA32_U_CET, 0); + wrmsrq(MSR_IA32_S_CET, 0); + wrmsrq(MSR_IA32_U_CET, 0); } /* @@ -751,9 +751,9 @@ void __init switch_gdt_and_percpu_base(int cpu) * No need to load %gs. It is already correct. * * Writing %gs on 64bit would zero GSBASE which would make any per - * CPU operation up to the point of the wrmsrl() fault. + * CPU operation up to the point of the wrmsrq() fault. * - * Set GSBASE to the new offset. Until the wrmsrl() happens the + * Set GSBASE to the new offset. Until the wrmsrq() happens the * early mapping is still valid. That means the GSBASE update will * lose any prior per CPU data which was not copied over in * setup_per_cpu_areas(). @@ -761,7 +761,7 @@ void __init switch_gdt_and_percpu_base(int cpu) * This works even with stackprotector enabled because the * per CPU stack canary is 0 in both per CPU areas. */ - wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu)); + wrmsrq(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu)); #else /* * %fs is already set to __KERNEL_PERCPU, but after switching GDT @@ -1750,10 +1750,10 @@ static bool detect_null_seg_behavior(void) unsigned long old_base, tmp; rdmsrq(MSR_FS_BASE, old_base); - wrmsrl(MSR_FS_BASE, 1); + wrmsrq(MSR_FS_BASE, 1); loadsegment(fs, 0); rdmsrq(MSR_FS_BASE, tmp); - wrmsrl(MSR_FS_BASE, old_base); + wrmsrq(MSR_FS_BASE, old_base); return tmp == 0; } @@ -2099,12 +2099,12 @@ static void wrmsrl_cstar(unsigned long val) * guest. Avoid the pointless write on all Intel CPUs. */ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) - wrmsrl(MSR_CSTAR, val); + wrmsrq(MSR_CSTAR, val); } static inline void idt_syscall_init(void) { - wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64); + wrmsrq(MSR_LSTAR, (unsigned long)entry_SYSCALL_64); if (ia32_enabled()) { wrmsrl_cstar((unsigned long)entry_SYSCALL_compat); @@ -2129,7 +2129,7 @@ static inline void idt_syscall_init(void) * Flags to clear on syscall; clear as much as possible * to minimize user space-kernel interference. */ - wrmsrl(MSR_SYSCALL_MASK, + wrmsrq(MSR_SYSCALL_MASK, X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF| X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF| X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF| @@ -2313,8 +2313,8 @@ void cpu_init(void) memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8); syscall_init(); - wrmsrl(MSR_FS_BASE, 0); - wrmsrl(MSR_KERNEL_GS_BASE, 0); + wrmsrq(MSR_FS_BASE, 0); + wrmsrq(MSR_KERNEL_GS_BASE, 0); barrier(); x2apic_setup(); diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 4a4118784c13..edd41950a959 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -165,7 +165,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c) msr |= FEAT_CTL_SGX_LC_ENABLED; } - wrmsrl(MSR_IA32_FEAT_CTL, msr); + wrmsrq(MSR_IA32_FEAT_CTL, msr); update_caps: set_cpu_cap(c, X86_FEATURE_MSR_IA32_FEAT_CTL); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index e9a6e0d00819..72a11040e2e1 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -509,7 +509,7 @@ static void init_intel_misc_features(struct cpuinfo_x86 *c) probe_xeon_phi_r3mwait(c); msr = this_cpu_read(msr_misc_features_shadow); - wrmsrl(MSR_MISC_FEATURES_ENABLES, msr); + wrmsrq(MSR_MISC_FEATURES_ENABLES, msr); } /* diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index 6e6af07fc61f..01d81b78d09e 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -111,7 +111,7 @@ static void intel_epb_restore(void) pr_warn_once("ENERGY_PERF_BIAS: Set to 'normal', was 'performance'\n"); } } - wrmsrl(MSR_IA32_ENERGY_PERF_BIAS, (epb & ~EPB_MASK) | val); + wrmsrq(MSR_IA32_ENERGY_PERF_BIAS, (epb & ~EPB_MASK) | val); } static struct syscore_ops intel_epb_syscore_ops = { diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index c29165f41bb6..9d852c3b2cb5 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -667,7 +667,7 @@ static void disable_err_thresholding(struct cpuinfo_x86 *c, unsigned int bank) /* McStatusWrEn has to be set */ need_toggle = !(hwcr & BIT(18)); if (need_toggle) - wrmsrl(MSR_K7_HWCR, hwcr | BIT(18)); + wrmsrq(MSR_K7_HWCR, hwcr | BIT(18)); /* Clear CntP bit safely */ for (i = 0; i < num_msrs; i++) @@ -675,7 +675,7 @@ static void disable_err_thresholding(struct cpuinfo_x86 *c, unsigned int bank) /* restore old settings */ if (need_toggle) - wrmsrl(MSR_K7_HWCR, hwcr); + wrmsrq(MSR_K7_HWCR, hwcr); } /* cpu init entry point, called from mce.c with preempt off */ @@ -843,7 +843,7 @@ _log_error_bank(unsigned int bank, u32 msr_stat, u32 msr_addr, u64 misc) __log_error(bank, status, addr, misc); - wrmsrl(msr_stat, 0); + wrmsrq(msr_stat, 0); return status & MCI_STATUS_DEFERRED; } @@ -862,7 +862,7 @@ static bool _log_error_deferred(unsigned int bank, u32 misc) return true; /* Clear MCA_DESTAT if the deferred error was logged from MCA_STATUS. */ - wrmsrl(MSR_AMD64_SMCA_MCx_DESTAT(bank), 0); + wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(bank), 0); return true; } diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index a71228d9a4f4..c2740248b852 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1878,8 +1878,8 @@ static void __mcheck_cpu_init_clear_banks(void) if (!b->init) continue; - wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); - wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); + wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl); + wrmsrq(mca_msr_reg(i, MCA_STATUS), 0); } } @@ -2436,7 +2436,7 @@ static void mce_disable_error_reporting(void) struct mce_bank *b = &mce_banks[i]; if (b->init) - wrmsrl(mca_msr_reg(i, MCA_CTL), 0); + wrmsrq(mca_msr_reg(i, MCA_CTL), 0); } return; } @@ -2786,7 +2786,7 @@ static void mce_reenable_cpu(void) struct mce_bank *b = &mce_banks[i]; if (b->init) - wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); + wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl); } } diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 08d52af7a8ad..75df35fca6a8 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -475,27 +475,27 @@ static void prepare_msrs(void *info) struct mce m = *(struct mce *)info; u8 b = m.bank; - wrmsrl(MSR_IA32_MCG_STATUS, m.mcgstatus); + wrmsrq(MSR_IA32_MCG_STATUS, m.mcgstatus); if (boot_cpu_has(X86_FEATURE_SMCA)) { if (m.inject_flags == DFR_INT_INJ) { - wrmsrl(MSR_AMD64_SMCA_MCx_DESTAT(b), m.status); - wrmsrl(MSR_AMD64_SMCA_MCx_DEADDR(b), m.addr); + wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(b), m.status); + wrmsrq(MSR_AMD64_SMCA_MCx_DEADDR(b), m.addr); } else { - wrmsrl(MSR_AMD64_SMCA_MCx_STATUS(b), m.status); - wrmsrl(MSR_AMD64_SMCA_MCx_ADDR(b), m.addr); + wrmsrq(MSR_AMD64_SMCA_MCx_STATUS(b), m.status); + wrmsrq(MSR_AMD64_SMCA_MCx_ADDR(b), m.addr); } - wrmsrl(MSR_AMD64_SMCA_MCx_SYND(b), m.synd); + wrmsrq(MSR_AMD64_SMCA_MCx_SYND(b), m.synd); if (m.misc) - wrmsrl(MSR_AMD64_SMCA_MCx_MISC(b), m.misc); + wrmsrq(MSR_AMD64_SMCA_MCx_MISC(b), m.misc); } else { - wrmsrl(MSR_IA32_MCx_STATUS(b), m.status); - wrmsrl(MSR_IA32_MCx_ADDR(b), m.addr); + wrmsrq(MSR_IA32_MCx_STATUS(b), m.status); + wrmsrq(MSR_IA32_MCx_ADDR(b), m.addr); if (m.misc) - wrmsrl(MSR_IA32_MCx_MISC(b), m.misc); + wrmsrq(MSR_IA32_MCx_MISC(b), m.misc); } } diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index 9fda956d4e48..28f04c663c95 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -143,7 +143,7 @@ static void cmci_set_threshold(int bank, int thresh) raw_spin_lock_irqsave(&cmci_discover_lock, flags); rdmsrq(MSR_IA32_MCx_CTL2(bank), val); val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK; - wrmsrl(MSR_IA32_MCx_CTL2(bank), val | thresh); + wrmsrq(MSR_IA32_MCx_CTL2(bank), val | thresh); raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } @@ -232,7 +232,7 @@ static void cmci_claim_bank(int bank, u64 val, int bios_zero_thresh, int *bios_w struct mca_storm_desc *storm = this_cpu_ptr(&storm_desc); val |= MCI_CTL2_CMCI_EN; - wrmsrl(MSR_IA32_MCx_CTL2(bank), val); + wrmsrq(MSR_IA32_MCx_CTL2(bank), val); rdmsrq(MSR_IA32_MCx_CTL2(bank), val); /* If the enable bit did not stick, this bank should be polled. */ @@ -326,7 +326,7 @@ static void __cmci_disable_bank(int bank) return; rdmsrq(MSR_IA32_MCx_CTL2(bank), val); val &= ~MCI_CTL2_CMCI_EN; - wrmsrl(MSR_IA32_MCx_CTL2(bank), val); + wrmsrq(MSR_IA32_MCx_CTL2(bank), val); __clear_bit(bank, this_cpu_ptr(mce_banks_owned)); if ((val & MCI_CTL2_CMCI_THRESHOLD_MASK) == CMCI_STORM_THRESHOLD) @@ -433,7 +433,7 @@ void intel_init_lmce(void) rdmsrq(MSR_IA32_MCG_EXT_CTL, val); if (!(val & MCG_EXT_CTL_LMCE_EN)) - wrmsrl(MSR_IA32_MCG_EXT_CTL, val | MCG_EXT_CTL_LMCE_EN); + wrmsrq(MSR_IA32_MCG_EXT_CTL, val | MCG_EXT_CTL_LMCE_EN); } void intel_clear_lmce(void) @@ -445,7 +445,7 @@ void intel_clear_lmce(void) rdmsrq(MSR_IA32_MCG_EXT_CTL, val); val &= ~MCG_EXT_CTL_LMCE_EN; - wrmsrl(MSR_IA32_MCG_EXT_CTL, val); + wrmsrq(MSR_IA32_MCG_EXT_CTL, val); } /* diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index b1b9b4c4997e..b924befe8d6e 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -82,9 +82,9 @@ void hv_set_non_nested_msr(unsigned int reg, u64 value) /* Write proxy bit via wrmsl instruction */ if (hv_is_sint_msr(reg)) - wrmsrl(reg, value | 1 << 20); + wrmsrq(reg, value | 1 << 20); } else { - wrmsrl(reg, value); + wrmsrq(reg, value); } } EXPORT_SYMBOL_GPL(hv_set_non_nested_msr); @@ -574,7 +574,7 @@ static void __init ms_hyperv_init_platform(void) * setting of this MSR bit should happen before init_intel() * is called. */ - wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, HV_EXPOSE_INVARIANT_TSC); + wrmsrq(HV_X64_MSR_TSC_INVARIANT_CONTROL, HV_EXPOSE_INVARIANT_TSC); setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); } diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 76c887ba9e9c..6d408f78873c 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -309,7 +309,7 @@ static void mba_wrmsr_amd(struct msr_param *m) unsigned int i; for (i = m->low; i < m->high; i++) - wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]); + wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); } /* @@ -334,7 +334,7 @@ static void mba_wrmsr_intel(struct msr_param *m) /* Write the delay values for mba. */ for (i = m->low; i < m->high; i++) - wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res)); + wrmsrq(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res)); } static void cat_wrmsr(struct msr_param *m) @@ -344,7 +344,7 @@ static void cat_wrmsr(struct msr_param *m) unsigned int i; for (i = m->low; i < m->high; i++) - wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]); + wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]); } u32 resctrl_arch_get_num_closid(struct rdt_resource *r) diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 92ea1472bde9..2a82eb6a0376 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -534,7 +534,7 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) __wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, closid_p); /* Re-enable the hardware prefetcher(s) */ - wrmsrl(MSR_MISC_FEATURE_CONTROL, saved_msr); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved_msr); local_irq_enable(); plr->thread_done = 1; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 3be9866be0e7..26f4d820ee6e 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -2326,14 +2326,14 @@ static void l3_qos_cfg_update(void *arg) { bool *enable = arg; - wrmsrl(MSR_IA32_L3_QOS_CFG, *enable ? L3_QOS_CDP_ENABLE : 0ULL); + wrmsrq(MSR_IA32_L3_QOS_CFG, *enable ? L3_QOS_CDP_ENABLE : 0ULL); } static void l2_qos_cfg_update(void *arg) { bool *enable = arg; - wrmsrl(MSR_IA32_L2_QOS_CFG, *enable ? L2_QOS_CDP_ENABLE : 0ULL); + wrmsrq(MSR_IA32_L2_QOS_CFG, *enable ? L2_QOS_CDP_ENABLE : 0ULL); } static inline bool is_mba_linear(void) diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 8ce352fc72ac..40967d8f995a 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -871,7 +871,7 @@ void sgx_update_lepubkeyhash(u64 *lepubkeyhash) WARN_ON_ONCE(preemptible()); for (i = 0; i < 4; i++) - wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]); + wrmsrq(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]); } const struct file_operations sgx_provision_fops = { diff --git a/arch/x86/kernel/cpu/tsx.c b/arch/x86/kernel/cpu/tsx.c index c3aaa6815c0b..b0a9c9e9d029 100644 --- a/arch/x86/kernel/cpu/tsx.c +++ b/arch/x86/kernel/cpu/tsx.c @@ -37,7 +37,7 @@ static void tsx_disable(void) */ tsx |= TSX_CTRL_CPUID_CLEAR; - wrmsrl(MSR_IA32_TSX_CTRL, tsx); + wrmsrq(MSR_IA32_TSX_CTRL, tsx); } static void tsx_enable(void) @@ -56,7 +56,7 @@ static void tsx_enable(void) */ tsx &= ~TSX_CTRL_CPUID_CLEAR; - wrmsrl(MSR_IA32_TSX_CTRL, tsx); + wrmsrq(MSR_IA32_TSX_CTRL, tsx); } static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) @@ -117,11 +117,11 @@ static void tsx_clear_cpuid(void) boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { rdmsrq(MSR_TSX_FORCE_ABORT, msr); msr |= MSR_TFA_TSX_CPUID_CLEAR; - wrmsrl(MSR_TSX_FORCE_ABORT, msr); + wrmsrq(MSR_TSX_FORCE_ABORT, msr); } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { rdmsrq(MSR_IA32_TSX_CTRL, msr); msr |= TSX_CTRL_CPUID_CLEAR; - wrmsrl(MSR_IA32_TSX_CTRL, msr); + wrmsrq(MSR_IA32_TSX_CTRL, msr); } } @@ -150,7 +150,7 @@ static void tsx_dev_mode_disable(void) if (mcu_opt_ctrl & RTM_ALLOW) { mcu_opt_ctrl &= ~RTM_ALLOW; - wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); + wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); } } diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 6a41d1610d8b..2bd87b788630 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -199,7 +199,7 @@ void fpu__init_cpu_xstate(void) * MSR_IA32_XSS sets supervisor states managed by XSAVES. */ if (boot_cpu_has(X86_FEATURE_XSAVES)) { - wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | + wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | xfeatures_mask_independent()); } } @@ -639,7 +639,7 @@ static unsigned int __init get_xsave_compacted_size(void) return get_compacted_size(); /* Disable independent features. */ - wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()); + wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor()); /* * Ask the hardware what size is required of the buffer. @@ -648,7 +648,7 @@ static unsigned int __init get_xsave_compacted_size(void) size = get_compacted_size(); /* Re-enable independent features so XSAVES will work on them again. */ - wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); + wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); return size; } @@ -904,12 +904,12 @@ void fpu__resume_cpu(void) * of XSAVES and MSR_IA32_XSS. */ if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { - wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | + wrmsrq(MSR_IA32_XSS, xfeatures_mask_supervisor() | xfeatures_mask_independent()); } if (fpu_state_size_dynamic()) - wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd); + wrmsrq(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd); } /* diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h index 0fd34f53f025..5e5d35027f13 100644 --- a/arch/x86/kernel/fpu/xstate.h +++ b/arch/x86/kernel/fpu/xstate.h @@ -171,7 +171,7 @@ static inline void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rs #ifdef CONFIG_X86_64 static inline void xfd_set_state(u64 xfd) { - wrmsrl(MSR_IA32_XFD, xfd); + wrmsrq(MSR_IA32_XFD, xfd); __this_cpu_write(xfd_state, xfd); } diff --git a/arch/x86/kernel/fred.c b/arch/x86/kernel/fred.c index 5e2cd1004980..10b0169f3fc1 100644 --- a/arch/x86/kernel/fred.c +++ b/arch/x86/kernel/fred.c @@ -43,23 +43,23 @@ void cpu_init_fred_exceptions(void) */ loadsegment(ss, __KERNEL_DS); - wrmsrl(MSR_IA32_FRED_CONFIG, + wrmsrq(MSR_IA32_FRED_CONFIG, /* Reserve for CALL emulation */ FRED_CONFIG_REDZONE | FRED_CONFIG_INT_STKLVL(0) | FRED_CONFIG_ENTRYPOINT(asm_fred_entrypoint_user)); - wrmsrl(MSR_IA32_FRED_STKLVLS, 0); + wrmsrq(MSR_IA32_FRED_STKLVLS, 0); /* * Ater a CPU offline/online cycle, the FRED RSP0 MSR should be * resynchronized with its per-CPU cache. */ - wrmsrl(MSR_IA32_FRED_RSP0, __this_cpu_read(fred_rsp0)); + wrmsrq(MSR_IA32_FRED_RSP0, __this_cpu_read(fred_rsp0)); - wrmsrl(MSR_IA32_FRED_RSP1, 0); - wrmsrl(MSR_IA32_FRED_RSP2, 0); - wrmsrl(MSR_IA32_FRED_RSP3, 0); + wrmsrq(MSR_IA32_FRED_RSP1, 0); + wrmsrq(MSR_IA32_FRED_RSP2, 0); + wrmsrq(MSR_IA32_FRED_RSP3, 0); /* Enable FRED */ cr4_set_bits(X86_CR4_FRED); @@ -79,14 +79,14 @@ void cpu_init_fred_rsps(void) * (remember that user space faults are always taken on stack level 0) * is to avoid overflowing the kernel stack. */ - wrmsrl(MSR_IA32_FRED_STKLVLS, + wrmsrq(MSR_IA32_FRED_STKLVLS, FRED_STKLVL(X86_TRAP_DB, FRED_DB_STACK_LEVEL) | FRED_STKLVL(X86_TRAP_NMI, FRED_NMI_STACK_LEVEL) | FRED_STKLVL(X86_TRAP_MC, FRED_MC_STACK_LEVEL) | FRED_STKLVL(X86_TRAP_DF, FRED_DF_STACK_LEVEL)); /* The FRED equivalents to IST stacks... */ - wrmsrl(MSR_IA32_FRED_RSP1, __this_cpu_ist_top_va(DB)); - wrmsrl(MSR_IA32_FRED_RSP2, __this_cpu_ist_top_va(NMI)); - wrmsrl(MSR_IA32_FRED_RSP3, __this_cpu_ist_top_va(DF)); + wrmsrq(MSR_IA32_FRED_RSP1, __this_cpu_ist_top_va(DB)); + wrmsrq(MSR_IA32_FRED_RSP2, __this_cpu_ist_top_va(NMI)); + wrmsrq(MSR_IA32_FRED_RSP3, __this_cpu_ist_top_va(DF)); } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index f7aa39cd56ec..44a45df7200a 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -301,7 +301,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) token = __this_cpu_read(apf_reason.token); kvm_async_pf_task_wake(token); __this_cpu_write(apf_reason.token, 0); - wrmsrl(MSR_KVM_ASYNC_PF_ACK, 1); + wrmsrq(MSR_KVM_ASYNC_PF_ACK, 1); } set_irq_regs(old_regs); @@ -327,7 +327,7 @@ static void kvm_register_steal_time(void) if (!has_steal_clock) return; - wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED)); + wrmsrq(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED)); pr_debug("stealtime: cpu %d, msr %llx\n", cpu, (unsigned long long) slow_virt_to_phys(st)); } @@ -361,9 +361,9 @@ static void kvm_guest_cpu_init(void) if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT)) pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; - wrmsrl(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR); + wrmsrq(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR); - wrmsrl(MSR_KVM_ASYNC_PF_EN, pa); + wrmsrq(MSR_KVM_ASYNC_PF_EN, pa); __this_cpu_write(async_pf_enabled, true); pr_debug("setup async PF for cpu %d\n", smp_processor_id()); } @@ -376,7 +376,7 @@ static void kvm_guest_cpu_init(void) __this_cpu_write(kvm_apic_eoi, 0); pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi)) | KVM_MSR_ENABLED; - wrmsrl(MSR_KVM_PV_EOI_EN, pa); + wrmsrq(MSR_KVM_PV_EOI_EN, pa); } if (has_steal_clock) @@ -388,7 +388,7 @@ static void kvm_pv_disable_apf(void) if (!__this_cpu_read(async_pf_enabled)) return; - wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); + wrmsrq(MSR_KVM_ASYNC_PF_EN, 0); __this_cpu_write(async_pf_enabled, false); pr_debug("disable async PF for cpu %d\n", smp_processor_id()); @@ -451,9 +451,9 @@ static void kvm_guest_cpu_offline(bool shutdown) { kvm_disable_steal_time(); if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) - wrmsrl(MSR_KVM_PV_EOI_EN, 0); + wrmsrq(MSR_KVM_PV_EOI_EN, 0); if (kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) - wrmsrl(MSR_KVM_MIGRATION_CONTROL, 0); + wrmsrq(MSR_KVM_MIGRATION_CONTROL, 0); kvm_pv_disable_apf(); if (!shutdown) apf_task_wake_all(); @@ -615,7 +615,7 @@ static int __init setup_efi_kvm_sev_migration(void) } pr_info("%s : live migration enabled in EFI\n", __func__); - wrmsrl(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY); + wrmsrq(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY); return 1; } @@ -740,7 +740,7 @@ static void kvm_resume(void) #ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll) - wrmsrl(MSR_KVM_POLL_CONTROL, 0); + wrmsrq(MSR_KVM_POLL_CONTROL, 0); #endif } @@ -975,7 +975,7 @@ static void __init kvm_init_platform(void) * If not booted using EFI, enable Live migration support. */ if (!efi_enabled(EFI_BOOT)) - wrmsrl(MSR_KVM_MIGRATION_CONTROL, + wrmsrq(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY); } kvmclock_init(); @@ -1124,12 +1124,12 @@ out: static void kvm_disable_host_haltpoll(void *i) { - wrmsrl(MSR_KVM_POLL_CONTROL, 0); + wrmsrq(MSR_KVM_POLL_CONTROL, 0); } static void kvm_enable_host_haltpoll(void *i) { - wrmsrl(MSR_KVM_POLL_CONTROL, 1); + wrmsrq(MSR_KVM_POLL_CONTROL, 1); } void arch_haltpoll_enable(unsigned int cpu) diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 5b2c15214a6b..0af797930ccb 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -60,7 +60,7 @@ EXPORT_PER_CPU_SYMBOL_GPL(hv_clock_per_cpu); */ static void kvm_get_wallclock(struct timespec64 *now) { - wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(&wall_clock)); + wrmsrq(msr_kvm_wall_clock, slow_virt_to_phys(&wall_clock)); preempt_disable(); pvclock_read_wallclock(&wall_clock, this_cpu_pvti(), now); preempt_enable(); @@ -173,7 +173,7 @@ static void kvm_register_clock(char *txt) return; pa = slow_virt_to_phys(&src->pvti) | 0x01ULL; - wrmsrl(msr_kvm_system_time, pa); + wrmsrq(msr_kvm_system_time, pa); pr_debug("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt); } diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c index 5c6a55f8a7fe..ef6104e7cc72 100644 --- a/arch/x86/kernel/mmconf-fam10h_64.c +++ b/arch/x86/kernel/mmconf-fam10h_64.c @@ -212,7 +212,7 @@ void fam10h_check_enable_mmcfg(void) (FAM10H_MMIO_CONF_BUSRANGE_MASK<shared_state->lock); /* First sibling enables SSBD: */ if (!st->shared_state->disable_state) - wrmsrl(MSR_AMD64_LS_CFG, msr); + wrmsrq(MSR_AMD64_LS_CFG, msr); st->shared_state->disable_state++; raw_spin_unlock(&st->shared_state->lock); } else { @@ -588,7 +588,7 @@ static __always_inline void amd_set_core_ssb_state(unsigned long tifn) raw_spin_lock(&st->shared_state->lock); st->shared_state->disable_state--; if (!st->shared_state->disable_state) - wrmsrl(MSR_AMD64_LS_CFG, msr); + wrmsrq(MSR_AMD64_LS_CFG, msr); raw_spin_unlock(&st->shared_state->lock); } } @@ -597,7 +597,7 @@ static __always_inline void amd_set_core_ssb_state(unsigned long tifn) { u64 msr = x86_amd_ls_cfg_base | ssbd_tif_to_amd_ls_cfg(tifn); - wrmsrl(MSR_AMD64_LS_CFG, msr); + wrmsrq(MSR_AMD64_LS_CFG, msr); } #endif @@ -607,7 +607,7 @@ static __always_inline void amd_set_ssb_virt_state(unsigned long tifn) * SSBD has the same definition in SPEC_CTRL and VIRT_SPEC_CTRL, * so ssbd_tif_to_spec_ctrl() just works. */ - wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn)); + wrmsrq(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn)); } /* @@ -714,7 +714,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) debugctl &= ~DEBUGCTLMSR_BTF; msk = tifn & _TIF_BLOCKSTEP; debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT; - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } if ((tifp ^ tifn) & _TIF_NOTSC) diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 422c5dad1046..24e1ccf22912 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -221,7 +221,7 @@ static noinstr void __wrgsbase_inactive(unsigned long gsbase) native_swapgs(); } else { instrumentation_begin(); - wrmsrl(MSR_KERNEL_GS_BASE, gsbase); + wrmsrq(MSR_KERNEL_GS_BASE, gsbase); instrumentation_end(); } } @@ -353,7 +353,7 @@ static __always_inline void load_seg_legacy(unsigned short prev_index, } else { if (prev_index != next_index) loadseg(which, next_index); - wrmsrl(which == FS ? MSR_FS_BASE : MSR_KERNEL_GS_BASE, + wrmsrq(which == FS ? MSR_FS_BASE : MSR_KERNEL_GS_BASE, next_base); } } else { @@ -478,7 +478,7 @@ void x86_gsbase_write_cpu_inactive(unsigned long gsbase) __wrgsbase_inactive(gsbase); local_irq_restore(flags); } else { - wrmsrl(MSR_KERNEL_GS_BASE, gsbase); + wrmsrq(MSR_KERNEL_GS_BASE, gsbase); } } diff --git a/arch/x86/kernel/reboot_fixups_32.c b/arch/x86/kernel/reboot_fixups_32.c index b7c0f142d026..4679ac0a03eb 100644 --- a/arch/x86/kernel/reboot_fixups_32.c +++ b/arch/x86/kernel/reboot_fixups_32.c @@ -27,7 +27,7 @@ static void cs5530a_warm_reset(struct pci_dev *dev) static void cs5536_warm_reset(struct pci_dev *dev) { /* writing 1 to the LSB of this MSR causes a hard reset */ - wrmsrl(MSR_DIVIL_SOFT_RESET, 1ULL); + wrmsrq(MSR_DIVIL_SOFT_RESET, 1ULL); udelay(50); /* shouldn't get here but be safe and spin a while */ } diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index 3d1ba20f840f..2ddf23387c7e 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -173,8 +173,8 @@ static int shstk_setup(void) return PTR_ERR((void *)addr); fpregs_lock_and_load(); - wrmsrl(MSR_IA32_PL3_SSP, addr + size); - wrmsrl(MSR_IA32_U_CET, CET_SHSTK_EN); + wrmsrq(MSR_IA32_PL3_SSP, addr + size); + wrmsrq(MSR_IA32_U_CET, CET_SHSTK_EN); fpregs_unlock(); shstk->base = addr; @@ -372,7 +372,7 @@ int setup_signal_shadow_stack(struct ksignal *ksig) return -EFAULT; fpregs_lock_and_load(); - wrmsrl(MSR_IA32_PL3_SSP, ssp); + wrmsrq(MSR_IA32_PL3_SSP, ssp); fpregs_unlock(); return 0; @@ -396,7 +396,7 @@ int restore_signal_shadow_stack(void) return err; fpregs_lock_and_load(); - wrmsrl(MSR_IA32_PL3_SSP, ssp); + wrmsrq(MSR_IA32_PL3_SSP, ssp); fpregs_unlock(); return 0; @@ -473,7 +473,7 @@ static int wrss_control(bool enable) msrval &= ~CET_WRSS_EN; } - wrmsrl(MSR_IA32_U_CET, msrval); + wrmsrq(MSR_IA32_U_CET, msrval); unlock: fpregs_unlock(); @@ -492,8 +492,8 @@ static int shstk_disable(void) fpregs_lock_and_load(); /* Disable WRSS too when disabling shadow stack */ - wrmsrl(MSR_IA32_U_CET, 0); - wrmsrl(MSR_IA32_PL3_SSP, 0); + wrmsrq(MSR_IA32_U_CET, 0); + wrmsrq(MSR_IA32_PL3_SSP, 0); fpregs_unlock(); shstk_free(current); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 71b14675e50e..823410aaf429 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -749,7 +749,7 @@ static bool try_fixup_enqcmd_gp(void) if (current->pasid_activated) return false; - wrmsrl(MSR_IA32_PASID, pasid | MSR_IA32_PASID_VALID); + wrmsrq(MSR_IA32_PASID, pasid | MSR_IA32_PASID_VALID); current->pasid_activated = 1; return true; @@ -1122,7 +1122,7 @@ static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6) rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl |= DEBUGCTLMSR_BTF; - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } /* @@ -1390,7 +1390,7 @@ static bool handle_xfd_event(struct pt_regs *regs) if (!xfd_err) return false; - wrmsrl(MSR_IA32_XFD_ERR, 0); + wrmsrq(MSR_IA32_XFD_ERR, 0); /* Die if that happens in kernel space */ if (WARN_ON(!user_mode(regs))) diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index 826039199828..f1c7a86dbf49 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -70,7 +70,7 @@ void tsc_verify_tsc_adjust(bool resume) return; /* Restore the original value */ - wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted); + wrmsrq(MSR_IA32_TSC_ADJUST, adj->adjusted); if (!adj->warned || resume) { pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n", @@ -142,7 +142,7 @@ static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval, if (likely(!tsc_async_resets)) { pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n", cpu, bootval); - wrmsrl(MSR_IA32_TSC_ADJUST, 0); + wrmsrq(MSR_IA32_TSC_ADJUST, 0); bootval = 0; } else { pr_info("TSC ADJUST: CPU%u: %lld NOT forced to 0\n", @@ -229,7 +229,7 @@ bool tsc_store_and_check_tsc_adjust(bool bootcpu) */ if (bootval != ref->adjusted) { cur->adjusted = ref->adjusted; - wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); + wrmsrq(MSR_IA32_TSC_ADJUST, ref->adjusted); } /* * We have the TSCs forced to be in sync on this package. Skip sync @@ -518,7 +518,7 @@ retry: pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", cpu, cur_max_warp, cur->adjusted); - wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); + wrmsrq(MSR_IA32_TSC_ADJUST, cur->adjusted); goto retry; } diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 65fd245a9953..51842100f6d2 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -330,7 +330,7 @@ void avic_ring_doorbell(struct kvm_vcpu *vcpu) int cpu = READ_ONCE(vcpu->cpu); if (cpu != get_cpu()) { - wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); + wrmsrq(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu)); } put_cpu(); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 21c3563547f2..67657b3a36ce 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -566,7 +566,7 @@ static void __svm_write_tsc_multiplier(u64 multiplier) if (multiplier == __this_cpu_read(current_tsc_ratio)) return; - wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); + wrmsrq(MSR_AMD64_TSC_RATIO, multiplier); __this_cpu_write(current_tsc_ratio, multiplier); } @@ -579,7 +579,7 @@ static inline void kvm_cpu_svm_disable(void) { uint64_t efer; - wrmsrl(MSR_VM_HSAVE_PA, 0); + wrmsrq(MSR_VM_HSAVE_PA, 0); rdmsrq(MSR_EFER, efer); if (efer & EFER_SVME) { /* @@ -587,7 +587,7 @@ static inline void kvm_cpu_svm_disable(void) * NMI aren't blocked. */ stgi(); - wrmsrl(MSR_EFER, efer & ~EFER_SVME); + wrmsrq(MSR_EFER, efer & ~EFER_SVME); } } @@ -629,9 +629,9 @@ static int svm_enable_virtualization_cpu(void) sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; - wrmsrl(MSR_EFER, efer | EFER_SVME); + wrmsrq(MSR_EFER, efer | EFER_SVME); - wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); + wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa); if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { /* diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index f0f02eee48cb..5e0bb821c7bc 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -281,7 +281,7 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, if (read) rdmsrq(index, msr_info->data); else - wrmsrl(index, msr_info->data); + wrmsrq(index, msr_info->data); __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); local_irq_enable(); return true; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3e0762a6939b..0e12a3a97439 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1063,7 +1063,7 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, * provide that period, so a CPU could write host's record into * guest's memory. */ - wrmsrl(MSR_IA32_PEBS_ENABLE, 0); + wrmsrq(MSR_IA32_PEBS_ENABLE, 0); } i = vmx_find_loadstore_msr_slot(&m->guest, msr); @@ -1192,13 +1192,13 @@ static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) { u32 i; - wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); - wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); - wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); - wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); + wrmsrq(MSR_IA32_RTIT_STATUS, ctx->status); + wrmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); + wrmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); + wrmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); for (i = 0; i < addr_range; i++) { - wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); - wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); + wrmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); + wrmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); } } @@ -1227,7 +1227,7 @@ static void pt_guest_enter(struct vcpu_vmx *vmx) */ rdmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { - wrmsrl(MSR_IA32_RTIT_CTL, 0); + wrmsrq(MSR_IA32_RTIT_CTL, 0); pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); } @@ -1248,7 +1248,7 @@ static void pt_guest_exit(struct vcpu_vmx *vmx) * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. */ if (vmx->pt_desc.host.ctl) - wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); + wrmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); } void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, @@ -1338,7 +1338,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); } - wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); + wrmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); #else savesegment(fs, fs_sel); savesegment(gs, gs_sel); @@ -1382,7 +1382,7 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) #endif invalidate_tss_limit(); #ifdef CONFIG_X86_64 - wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); + wrmsrq(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); #endif load_fixmap_gdt(raw_smp_processor_id()); vmx->guest_state_loaded = false; @@ -1403,7 +1403,7 @@ static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) { preempt_disable(); if (vmx->guest_state_loaded) - wrmsrl(MSR_KERNEL_GS_BASE, data); + wrmsrq(MSR_KERNEL_GS_BASE, data); preempt_enable(); vmx->msr_guest_kernel_gs_base = data; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c96c2ca7bb61..36504a170efc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -578,7 +578,7 @@ static void kvm_on_user_return(struct user_return_notifier *urn) for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) { values = &msrs->values[slot]; if (values->host != values->curr) { - wrmsrl(kvm_uret_msrs_list[slot], values->host); + wrmsrq(kvm_uret_msrs_list[slot], values->host); values->curr = values->host; } } @@ -1174,7 +1174,7 @@ void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) && vcpu->arch.ia32_xss != kvm_host.xss) - wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss); + wrmsrq(MSR_IA32_XSS, vcpu->arch.ia32_xss); } if (cpu_feature_enabled(X86_FEATURE_PKU) && @@ -1205,7 +1205,7 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) && vcpu->arch.ia32_xss != kvm_host.xss) - wrmsrl(MSR_IA32_XSS, kvm_host.xss); + wrmsrq(MSR_IA32_XSS, kvm_host.xss); } } @@ -3827,7 +3827,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!data) break; - wrmsrl(MSR_IA32_PRED_CMD, data); + wrmsrq(MSR_IA32_PRED_CMD, data); break; } case MSR_IA32_FLUSH_CMD: @@ -3840,7 +3840,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!data) break; - wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); + wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); break; case MSR_EFER: return set_efer(vcpu, msr_info); @@ -10974,7 +10974,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) switch_fpu_return(); if (vcpu->arch.guest_fpu.xfd_err) - wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); + wrmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); if (unlikely(vcpu->arch.switch_db_regs)) { set_debugreg(0, 7); @@ -11060,7 +11060,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(handle_exit_irqoff)(vcpu); if (vcpu->arch.guest_fpu.xfd_err) - wrmsrl(MSR_IA32_XFD_ERR, 0); + wrmsrq(MSR_IA32_XFD_ERR, 0); /* * Consume any pending interrupts, including the possible source of @@ -13657,7 +13657,7 @@ int kvm_spec_ctrl_test_value(u64 value) else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value)) ret = 1; else - wrmsrl(MSR_IA32_SPEC_CTRL, saved_value); + wrmsrq(MSR_IA32_SPEC_CTRL, saved_value); local_irq_restore(flags); diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index ce6105e64944..72e87eee0b13 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -232,7 +232,7 @@ void pat_cpu_init(void) panic("x86/PAT: PAT enabled, but not supported by secondary CPU\n"); } - wrmsrl(MSR_IA32_CR_PAT, pat_msr_val); + wrmsrq(MSR_IA32_CR_PAT, pat_msr_val); __flush_tlb_all(); } diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index e459d97ef397..b1d521201e0b 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -623,7 +623,7 @@ static void l1d_flush_evaluate(unsigned long prev_mm, unsigned long next_mm, { /* Flush L1D if the outgoing task requests it */ if (prev_mm & LAST_USER_MM_L1D_FLUSH) - wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); + wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); /* Check whether the incoming task opted in for L1D flush */ if (likely(!(next_mm & LAST_USER_MM_L1D_FLUSH))) diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index b92917c052f3..6158f652a7cd 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -344,7 +344,7 @@ static int amd_bus_cpu_online(unsigned int cpu) rdmsrq(MSR_AMD64_NB_CFG, reg); if (!(reg & ENABLE_CF8_EXT_CFG)) { reg |= ENABLE_CF8_EXT_CFG; - wrmsrl(MSR_AMD64_NB_CFG, reg); + wrmsrq(MSR_AMD64_NB_CFG, reg); } return 0; } diff --git a/arch/x86/platform/olpc/olpc-xo1-sci.c b/arch/x86/platform/olpc/olpc-xo1-sci.c index 63066e7c8517..30751b42d54e 100644 --- a/arch/x86/platform/olpc/olpc-xo1-sci.c +++ b/arch/x86/platform/olpc/olpc-xo1-sci.c @@ -325,7 +325,7 @@ static int setup_sci_interrupt(struct platform_device *pdev) dev_info(&pdev->dev, "SCI unmapped. Mapping to IRQ 3\n"); sci_irq = 3; lo |= 0x00300000; - wrmsrl(0x51400020, lo); + wrmsrq(0x51400020, lo); } /* Select level triggered in PIC */ diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 874fe980c4bd..78903d330bec 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -56,7 +56,7 @@ static void msr_restore_context(struct saved_context *ctxt) while (msr < end) { if (msr->valid) - wrmsrl(msr->info.msr_no, msr->info.reg.q); + wrmsrq(msr->info.msr_no, msr->info.reg.q); msr++; } } @@ -198,7 +198,7 @@ static void notrace __restore_processor_state(struct saved_context *ctxt) struct cpuinfo_x86 *c; if (ctxt->misc_enable_saved) - wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable); + wrmsrq(MSR_IA32_MISC_ENABLE, ctxt->misc_enable); /* * control registers */ @@ -208,7 +208,7 @@ static void notrace __restore_processor_state(struct saved_context *ctxt) __write_cr4(ctxt->cr4); #else /* CONFIG X86_64 */ - wrmsrl(MSR_EFER, ctxt->efer); + wrmsrq(MSR_EFER, ctxt->efer); __write_cr4(ctxt->cr4); #endif write_cr3(ctxt->cr3); @@ -231,7 +231,7 @@ static void notrace __restore_processor_state(struct saved_context *ctxt) * handlers or in complicated helpers like load_gs_index(). */ #ifdef CONFIG_X86_64 - wrmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base); + wrmsrq(MSR_GS_BASE, ctxt->kernelmode_gs_base); /* * Reinitialize FRED to ensure the FRED MSRs contain the same values @@ -267,8 +267,8 @@ static void notrace __restore_processor_state(struct saved_context *ctxt) * restoring the selectors clobbers the bases. Keep in mind * that MSR_KERNEL_GS_BASE is horribly misnamed. */ - wrmsrl(MSR_FS_BASE, ctxt->fs_base); - wrmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base); + wrmsrq(MSR_FS_BASE, ctxt->fs_base); + wrmsrq(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base); #else loadsegment(gs, ctxt->gs); #endif diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 9ddd10265de7..334177c89df0 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -140,7 +140,7 @@ static int __mfd_enable(unsigned int cpu) val |= MSR_AMD64_SYSCFG_MFDM; - wrmsrl(MSR_AMD64_SYSCFG, val); + wrmsrq(MSR_AMD64_SYSCFG, val); return 0; } @@ -162,7 +162,7 @@ static int __snp_enable(unsigned int cpu) val |= MSR_AMD64_SYSCFG_SNP_EN; val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; - wrmsrl(MSR_AMD64_SYSCFG, val); + wrmsrq(MSR_AMD64_SYSCFG, val); return 0; } diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c index cacac3d75f32..7bb3ac2d5ac8 100644 --- a/arch/x86/xen/suspend.c +++ b/arch/x86/xen/suspend.c @@ -39,7 +39,7 @@ void xen_arch_post_suspend(int cancelled) static void xen_vcpu_notify_restore(void *data) { if (xen_pv_domain() && boot_cpu_has(X86_FEATURE_SPEC_CTRL)) - wrmsrl(MSR_IA32_SPEC_CTRL, this_cpu_read(spec_ctrl)); + wrmsrq(MSR_IA32_SPEC_CTRL, this_cpu_read(spec_ctrl)); /* Boot processor notified via generic timekeeping_resume() */ if (smp_processor_id() == 0) @@ -57,7 +57,7 @@ static void xen_vcpu_notify_suspend(void *data) if (xen_pv_domain() && boot_cpu_has(X86_FEATURE_SPEC_CTRL)) { rdmsrq(MSR_IA32_SPEC_CTRL, tmp); this_cpu_write(spec_ctrl, tmp); - wrmsrl(MSR_IA32_SPEC_CTRL, 0); + wrmsrq(MSR_IA32_SPEC_CTRL, 0); } } diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 8e31c25c0905..d19867cf816a 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -117,7 +117,7 @@ static int boost_set_msr(bool enable) else val |= msr_mask; - wrmsrl(msr_addr, val); + wrmsrq(msr_addr, val); return 0; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 2b673ac0781f..b7caaa83a649 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -258,7 +258,7 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, return 0; if (fast_switch) { - wrmsrl(MSR_AMD_CPPC_REQ, value); + wrmsrq(MSR_AMD_CPPC_REQ, value); return 0; } else { int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c index 6d6afcda3681..320a0af2266a 100644 --- a/drivers/cpufreq/e_powersaver.c +++ b/drivers/cpufreq/e_powersaver.c @@ -228,7 +228,7 @@ static int eps_cpu_init(struct cpufreq_policy *policy) rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { val |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP; - wrmsrl(MSR_IA32_MISC_ENABLE, val); + wrmsrq(MSR_IA32_MISC_ENABLE, val); /* Can be locked at 0 */ rdmsrq(MSR_IA32_MISC_ENABLE, val); if (!(val & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) { diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index d66a4741fe9f..3883a2b3e599 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1293,7 +1293,7 @@ static void set_power_ctl_ee_state(bool input) power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE); power_ctl_ee_state = POWER_CTL_EE_DISABLE; } - wrmsrl(MSR_IA32_POWER_CTL, power_ctl); + wrmsrq(MSR_IA32_POWER_CTL, power_ctl); mutex_unlock(&intel_pstate_driver_lock); } @@ -2351,7 +2351,7 @@ static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu) return; hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min; - wrmsrl(MSR_HWP_REQUEST, hwp_req); + wrmsrq(MSR_HWP_REQUEST, hwp_req); cpu->last_update = cpu->sample.time; } @@ -2364,7 +2364,7 @@ static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu) expired = time_after64(cpu->sample.time, cpu->last_update + hwp_boost_hold_time_ns); if (expired) { - wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached); + wrmsrq(MSR_HWP_REQUEST, cpu->hwp_req_cached); cpu->hwp_boost_min = 0; } } @@ -2520,7 +2520,7 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) return; cpu->pstate.current_pstate = pstate; - wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); + wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } static void intel_pstate_adjust_pstate(struct cpudata *cpu) @@ -3100,7 +3100,7 @@ static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max, WRITE_ONCE(cpu->hwp_req_cached, value); if (fast_switch) - wrmsrl(MSR_HWP_REQUEST, value); + wrmsrq(MSR_HWP_REQUEST, value); else wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); } @@ -3109,7 +3109,7 @@ static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu, u32 target_pstate, bool fast_switch) { if (fast_switch) - wrmsrl(MSR_IA32_PERF_CTL, + wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, target_pstate)); else wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c index d3146f09fe80..ba0e08c8486a 100644 --- a/drivers/cpufreq/longhaul.c +++ b/drivers/cpufreq/longhaul.c @@ -144,7 +144,7 @@ static void do_longhaul1(unsigned int mults_index) /* Sync to timer tick */ safe_halt(); /* Change frequency on next halt or sleep */ - wrmsrl(MSR_VIA_BCR2, bcr2.val); + wrmsrq(MSR_VIA_BCR2, bcr2.val); /* Invoke transition */ ACPI_FLUSH_CPU_CACHE(); halt(); @@ -153,7 +153,7 @@ static void do_longhaul1(unsigned int mults_index) local_irq_disable(); rdmsrq(MSR_VIA_BCR2, bcr2.val); bcr2.bits.ESOFTBF = 0; - wrmsrl(MSR_VIA_BCR2, bcr2.val); + wrmsrq(MSR_VIA_BCR2, bcr2.val); } /* For processor with Longhaul MSR */ @@ -180,7 +180,7 @@ static void do_powersaver(int cx_address, unsigned int mults_index, /* Raise voltage if necessary */ if (can_scale_voltage && dir) { longhaul.bits.EnableSoftVID = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Change voltage */ if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); @@ -194,12 +194,12 @@ static void do_powersaver(int cx_address, unsigned int mults_index, t = inl(acpi_gbl_FADT.xpm_timer_block.address); } longhaul.bits.EnableSoftVID = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); } /* Change frequency on next halt or sleep */ longhaul.bits.EnableSoftBusRatio = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); halt(); @@ -212,12 +212,12 @@ static void do_powersaver(int cx_address, unsigned int mults_index, } /* Disable bus ratio bit */ longhaul.bits.EnableSoftBusRatio = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Reduce voltage if necessary */ if (can_scale_voltage && !dir) { longhaul.bits.EnableSoftVID = 1; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); /* Change voltage */ if (!cx_address) { ACPI_FLUSH_CPU_CACHE(); @@ -231,7 +231,7 @@ static void do_powersaver(int cx_address, unsigned int mults_index, t = inl(acpi_gbl_FADT.xpm_timer_block.address); } longhaul.bits.EnableSoftVID = 0; - wrmsrl(MSR_VIA_LONGHAUL, longhaul.val); + wrmsrq(MSR_VIA_LONGHAUL, longhaul.val); } } diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index dbbc27cb9ce2..31039330a3ba 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -225,7 +225,7 @@ static void change_FID(int fid) fidvidctl.bits.FID = fid; fidvidctl.bits.VIDC = 0; fidvidctl.bits.FIDC = 1; - wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + wrmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); } } @@ -240,7 +240,7 @@ static void change_VID(int vid) fidvidctl.bits.VID = vid; fidvidctl.bits.FIDC = 0; fidvidctl.bits.VIDC = 1; - wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); + wrmsrq(MSR_K7_FID_VID_CTL, fidvidctl.val); } } diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 2e87ca0e292a..bb8a25ef5b43 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1060,7 +1060,7 @@ static inline int __sev_do_init_locked(int *psp_ret) static void snp_set_hsave_pa(void *arg) { - wrmsrl(MSR_VM_HSAVE_PA, 0); + wrmsrq(MSR_VM_HSAVE_PA, 0); } static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 6e4b5b33ba81..517b28a85560 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -2082,8 +2082,8 @@ static void __init spr_idle_state_table_update(void) */ static void __init byt_cht_auto_demotion_disable(void) { - wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); - wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); + wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); + wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); } static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) @@ -2243,7 +2243,7 @@ static void auto_demotion_disable(void) rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); msr_bits &= ~auto_demotion_disable_flags; - wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); + wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); } static void c1e_promotion_enable(void) @@ -2252,7 +2252,7 @@ static void c1e_promotion_enable(void) rdmsrq(MSR_IA32_POWER_CTL, msr_bits); msr_bits |= 0x2; - wrmsrl(MSR_IA32_POWER_CTL, msr_bits); + wrmsrq(MSR_IA32_POWER_CTL, msr_bits); } static void c1e_promotion_disable(void) @@ -2261,7 +2261,7 @@ static void c1e_promotion_disable(void) rdmsrq(MSR_IA32_POWER_CTL, msr_bits); msr_bits &= ~0x2; - wrmsrl(MSR_IA32_POWER_CTL, msr_bits); + wrmsrq(MSR_IA32_POWER_CTL, msr_bits); } /** diff --git a/drivers/platform/x86/intel/ifs/load.c b/drivers/platform/x86/intel/ifs/load.c index d3e48669a01d..0289391eccde 100644 --- a/drivers/platform/x86/intel/ifs/load.c +++ b/drivers/platform/x86/intel/ifs/load.c @@ -127,7 +127,7 @@ static void copy_hashes_authenticate_chunks(struct work_struct *work) ifsd = ifs_get_data(dev); msrs = ifs_get_test_msrs(dev); /* run scan hash copy */ - wrmsrl(msrs->copy_hashes, ifs_hash_ptr); + wrmsrq(msrs->copy_hashes, ifs_hash_ptr); rdmsrq(msrs->copy_hashes_status, hashes_status.data); /* enumerate the scan image information */ @@ -149,7 +149,7 @@ static void copy_hashes_authenticate_chunks(struct work_struct *work) linear_addr = base + i * chunk_size; linear_addr |= i; - wrmsrl(msrs->copy_chunks, linear_addr); + wrmsrq(msrs->copy_chunks, linear_addr); rdmsrq(msrs->copy_chunks_status, chunk_status.data); ifsd->valid_chunks = chunk_status.valid_chunks; @@ -195,7 +195,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) msrs = ifs_get_test_msrs(dev); if (need_copy_scan_hashes(ifsd)) { - wrmsrl(msrs->copy_hashes, ifs_hash_ptr); + wrmsrq(msrs->copy_hashes, ifs_hash_ptr); rdmsrq(msrs->copy_hashes_status, hashes_status.data); /* enumerate the scan image information */ @@ -216,7 +216,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) } if (ifsd->generation >= IFS_GEN_STRIDE_AWARE) { - wrmsrl(msrs->test_ctrl, INVALIDATE_STRIDE); + wrmsrq(msrs->test_ctrl, INVALIDATE_STRIDE); rdmsrq(msrs->copy_chunks_status, chunk_status.data); if (chunk_status.valid_chunks != 0) { dev_err(dev, "Couldn't invalidate installed stride - %d\n", @@ -238,7 +238,7 @@ static int copy_hashes_authenticate_chunks_gen2(struct device *dev) chunk_table[1] = linear_addr; do { local_irq_disable(); - wrmsrl(msrs->copy_chunks, (u64)chunk_table); + wrmsrq(msrs->copy_chunks, (u64)chunk_table); local_irq_enable(); rdmsrq(msrs->copy_chunks_status, chunk_status.data); err_code = chunk_status.error_code; diff --git a/drivers/platform/x86/intel/ifs/runtest.c b/drivers/platform/x86/intel/ifs/runtest.c index 8e669540a0a7..6b6ed7be461a 100644 --- a/drivers/platform/x86/intel/ifs/runtest.c +++ b/drivers/platform/x86/intel/ifs/runtest.c @@ -209,7 +209,7 @@ static int doscan(void *data) * take up to 200 milliseconds (in the case where all chunks * are processed in a single pass) before it retires. */ - wrmsrl(MSR_ACTIVATE_SCAN, params->activate->data); + wrmsrq(MSR_ACTIVATE_SCAN, params->activate->data); rdmsrq(MSR_SCAN_STATUS, status.data); trace_ifs_status(ifsd->cur_batch, start, stop, status.data); @@ -321,7 +321,7 @@ static int do_array_test(void *data) first = cpumask_first(cpu_smt_mask(cpu)); if (cpu == first) { - wrmsrl(MSR_ARRAY_BIST, command->data); + wrmsrq(MSR_ARRAY_BIST, command->data); /* Pass back the result of the test */ rdmsrq(MSR_ARRAY_BIST, command->data); } @@ -374,7 +374,7 @@ static int do_array_test_gen1(void *status) first = cpumask_first(cpu_smt_mask(cpu)); if (cpu == first) { - wrmsrl(MSR_ARRAY_TRIGGER, ARRAY_GEN1_TEST_ALL_ARRAYS); + wrmsrq(MSR_ARRAY_TRIGGER, ARRAY_GEN1_TEST_ALL_ARRAYS); rdmsrq(MSR_ARRAY_STATUS, *((u64 *)status)); } @@ -526,7 +526,7 @@ static int dosbaf(void *data) * starts scan of each requested bundle. The core test happens * during the "execution" of the WRMSR. */ - wrmsrl(MSR_ACTIVATE_SBAF, run_params->activate->data); + wrmsrq(MSR_ACTIVATE_SBAF, run_params->activate->data); rdmsrq(MSR_SBAF_STATUS, status.data); trace_ifs_sbaf(ifsd->cur_batch, *run_params->activate, status); diff --git a/drivers/platform/x86/intel/pmc/cnp.c b/drivers/platform/x86/intel/pmc/cnp.c index aa9aa1cc244a..547bdf1ab02d 100644 --- a/drivers/platform/x86/intel/pmc/cnp.c +++ b/drivers/platform/x86/intel/pmc/cnp.c @@ -230,7 +230,7 @@ static void disable_c1_auto_demote(void *unused) rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, val); per_cpu(pkg_cst_config, cpunum) = val; val &= ~NHM_C1_AUTO_DEMOTE; - wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, val); + wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, val); pr_debug("%s: cpu:%d cst %llx\n", __func__, cpunum, val); } @@ -239,7 +239,7 @@ static void restore_c1_auto_demote(void *unused) { int cpunum = smp_processor_id(); - wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, per_cpu(pkg_cst_config, cpunum)); + wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, per_cpu(pkg_cst_config, cpunum)); pr_debug("%s: cpu:%d cst %llx\n", __func__, cpunum, per_cpu(pkg_cst_config, cpunum)); diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c index 20624fcb9675..0d4d67771afa 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c @@ -52,14 +52,14 @@ static int isst_if_send_mbox_cmd(u8 command, u8 sub_command, u32 parameter, return ret; /* Write DATA register */ - wrmsrl(MSR_OS_MAILBOX_DATA, command_data); + wrmsrq(MSR_OS_MAILBOX_DATA, command_data); /* Write command register */ data = BIT_ULL(MSR_OS_MAILBOX_BUSY_BIT) | (parameter & GENMASK_ULL(13, 0)) << 16 | (sub_command << 8) | command; - wrmsrl(MSR_OS_MAILBOX_INTERFACE, data); + wrmsrq(MSR_OS_MAILBOX_INTERFACE, data); /* Poll for rb bit == 0 */ retries = OS_MAILBOX_RETRY_COUNT; diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c index 74b2a8700022..9506f28fb7d8 100644 --- a/drivers/platform/x86/intel_ips.c +++ b/drivers/platform/x86/intel_ips.c @@ -382,12 +382,12 @@ static void ips_cpu_raise(struct ips_driver *ips) thm_writew(THM_MPCPC, (new_tdp_limit * 10) / 8); turbo_override |= TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN; - wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); turbo_override &= ~TURBO_TDP_MASK; turbo_override |= new_tdp_limit; - wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); } /** @@ -417,12 +417,12 @@ static void ips_cpu_lower(struct ips_driver *ips) thm_writew(THM_MPCPC, (new_limit * 10) / 8); turbo_override |= TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN; - wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); turbo_override &= ~TURBO_TDP_MASK; turbo_override |= new_limit; - wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); } /** @@ -440,7 +440,7 @@ static void do_enable_cpu_turbo(void *data) rdmsrq(IA32_PERF_CTL, perf_ctl); if (perf_ctl & IA32_PERF_TURBO_DIS) { perf_ctl &= ~IA32_PERF_TURBO_DIS; - wrmsrl(IA32_PERF_CTL, perf_ctl); + wrmsrq(IA32_PERF_CTL, perf_ctl); } } @@ -478,7 +478,7 @@ static void do_disable_cpu_turbo(void *data) rdmsrq(IA32_PERF_CTL, perf_ctl); if (!(perf_ctl & IA32_PERF_TURBO_DIS)) { perf_ctl |= IA32_PERF_TURBO_DIS; - wrmsrl(IA32_PERF_CTL, perf_ctl); + wrmsrq(IA32_PERF_CTL, perf_ctl); } } @@ -1598,8 +1598,8 @@ static void ips_remove(struct pci_dev *dev) rdmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); turbo_override &= ~(TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN); - wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override); - wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, turbo_override); + wrmsrq(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit); free_irq(ips->irq, ips); pci_free_irq_vectors(dev); diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c index 2aaac15748a5..bd2fca7dc017 100644 --- a/drivers/thermal/intel/intel_hfi.c +++ b/drivers/thermal/intel/intel_hfi.c @@ -358,7 +358,7 @@ static void hfi_enable(void) rdmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; - wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); + wrmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); } static void hfi_set_hw_table(struct hfi_instance *hfi_instance) @@ -368,7 +368,7 @@ static void hfi_set_hw_table(struct hfi_instance *hfi_instance) hw_table_pa = virt_to_phys(hfi_instance->hw_table); msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT; - wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val); + wrmsrq(MSR_IA32_HW_FEEDBACK_PTR, msr_val); } /* Caller must hold hfi_instance_lock. */ @@ -379,7 +379,7 @@ static void hfi_disable(void) rdmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); msr_val &= ~HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT; - wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); + wrmsrq(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val); /* * Wait for hardware to acknowledge the disabling of HFI. Some diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c index f296f35ef7d4..c4f3f13cd9e8 100644 --- a/drivers/thermal/intel/therm_throt.c +++ b/drivers/thermal/intel/therm_throt.c @@ -273,7 +273,7 @@ void thermal_clear_package_intr_status(int level, u64 bit_mask) } msr_val &= ~bit_mask; - wrmsrl(msr, msr_val); + wrmsrq(msr, msr_val); } EXPORT_SYMBOL_GPL(thermal_clear_package_intr_status); diff --git a/drivers/video/fbdev/geode/lxfb_ops.c b/drivers/video/fbdev/geode/lxfb_ops.c index 5402c000ce8e..a27531b5de11 100644 --- a/drivers/video/fbdev/geode/lxfb_ops.c +++ b/drivers/video/fbdev/geode/lxfb_ops.c @@ -371,7 +371,7 @@ void lx_set_mode(struct fb_info *info) } else msrval |= MSR_LX_GLD_MSR_CONFIG_FMT_CRT; - wrmsrl(MSR_LX_GLD_MSR_CONFIG, msrval); + wrmsrq(MSR_LX_GLD_MSR_CONFIG, msrval); /* Clear the various buffers */ /* FIXME: Adjust for panning here */ @@ -427,7 +427,7 @@ void lx_set_mode(struct fb_info *info) | MSR_LX_SPARE_MSR_WM_LPEN_OVRD); msrval |= MSR_LX_SPARE_MSR_DIS_VIFO_WM | MSR_LX_SPARE_MSR_DIS_INIT_V_PRI; - wrmsrl(MSR_LX_SPARE_MSR, msrval); + wrmsrq(MSR_LX_SPARE_MSR, msrval); gcfg = DC_GENERAL_CFG_DFLE; /* Display fifo enable */ gcfg |= (0x6 << DC_GENERAL_CFG_DFHPSL_SHIFT) | /* default priority */ @@ -664,7 +664,7 @@ static void lx_restore_display_ctlr(struct lxfb_par *par) uint32_t filt; int i; - wrmsrl(MSR_LX_SPARE_MSR, par->msr.dcspare); + wrmsrq(MSR_LX_SPARE_MSR, par->msr.dcspare); for (i = 0; i < ARRAY_SIZE(par->dc); i++) { switch (i) { @@ -729,8 +729,8 @@ static void lx_restore_video_proc(struct lxfb_par *par) { int i; - wrmsrl(MSR_LX_GLD_MSR_CONFIG, par->msr.dfglcfg); - wrmsrl(MSR_LX_MSR_PADSEL, par->msr.padsel); + wrmsrq(MSR_LX_GLD_MSR_CONFIG, par->msr.dfglcfg); + wrmsrq(MSR_LX_MSR_PADSEL, par->msr.padsel); for (i = 0; i < ARRAY_SIZE(par->vp); i++) { switch (i) { diff --git a/drivers/video/fbdev/geode/suspend_gx.c b/drivers/video/fbdev/geode/suspend_gx.c index ed4997724063..73307f42e343 100644 --- a/drivers/video/fbdev/geode/suspend_gx.c +++ b/drivers/video/fbdev/geode/suspend_gx.c @@ -133,7 +133,7 @@ static void gx_restore_video_proc(struct gxfb_par *par) { int i; - wrmsrl(MSR_GX_MSR_PADSEL, par->msr.padsel); + wrmsrq(MSR_GX_MSR_PADSEL, par->msr.padsel); for (i = 0; i < ARRAY_SIZE(par->vp); i++) { switch (i) { diff --git a/drivers/video/fbdev/geode/video_gx.c b/drivers/video/fbdev/geode/video_gx.c index 9bee01751f47..5717c3356949 100644 --- a/drivers/video/fbdev/geode/video_gx.c +++ b/drivers/video/fbdev/geode/video_gx.c @@ -151,7 +151,7 @@ void gx_set_dclk_frequency(struct fb_info *info) dotpll |= MSR_GLCP_DOTPLL_DOTRESET; dotpll &= ~MSR_GLCP_DOTPLL_BYPASS; - wrmsrl(MSR_GLCP_DOTPLL, dotpll); + wrmsrq(MSR_GLCP_DOTPLL, dotpll); /* Program dividers. */ sys_rstpll &= ~( MSR_GLCP_SYS_RSTPLL_DOTPREDIV2 @@ -159,11 +159,11 @@ void gx_set_dclk_frequency(struct fb_info *info) | MSR_GLCP_SYS_RSTPLL_DOTPOSTDIV3 ); sys_rstpll |= pll_table[best_i].sys_rstpll_bits; - wrmsrl(MSR_GLCP_SYS_RSTPLL, sys_rstpll); + wrmsrq(MSR_GLCP_SYS_RSTPLL, sys_rstpll); /* Clear reset bit to start PLL. */ dotpll &= ~(MSR_GLCP_DOTPLL_DOTRESET); - wrmsrl(MSR_GLCP_DOTPLL, dotpll); + wrmsrq(MSR_GLCP_DOTPLL, dotpll); /* Wait for LOCK bit. */ do { @@ -183,7 +183,7 @@ gx_configure_tft(struct fb_info *info) rdmsrq(MSR_GX_MSR_PADSEL, val); val &= ~MSR_GX_MSR_PADSEL_MASK; val |= MSR_GX_MSR_PADSEL_TFT; - wrmsrl(MSR_GX_MSR_PADSEL, val); + wrmsrq(MSR_GX_MSR_PADSEL, val); /* Turn off the panel */ diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h index 53699750db86..68606fa5fe73 100644 --- a/include/hyperv/hvgdk_mini.h +++ b/include/hyperv/hvgdk_mini.h @@ -1013,7 +1013,7 @@ enum hv_register_name { /* * To support arch-generic code calling hv_set/get_register: - * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrq/wrmsrl + * - On x86, HV_MSR_ indicates an MSR accessed via rdmsrq/wrmsrq * - On ARM, HV_MSR_ indicates a VP register accessed via hypercall */ #define HV_MSR_CRASH_P0 (HV_X64_MSR_CRASH_P0) From 6fe22abacd40e259fffec744a02d5ca3febccd68 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:56 +0200 Subject: [PATCH 096/331] x86/msr: Rename 'rdmsrl_safe()' to 'rdmsrq_safe()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/events/amd/power.c | 2 +- arch/x86/events/core.c | 8 ++++---- arch/x86/events/intel/core.c | 4 ++-- arch/x86/events/probe.c | 2 +- arch/x86/events/rapl.c | 2 +- arch/x86/include/asm/apic.h | 2 +- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/kernel/amd_nb.c | 2 +- arch/x86/kernel/cpu/amd.c | 6 +++--- arch/x86/kernel/cpu/aperfmperf.c | 20 +++++++++---------- arch/x86/kernel/cpu/bus_lock.c | 2 +- arch/x86/kernel/cpu/common.c | 4 ++-- arch/x86/kernel/cpu/feat_ctl.c | 2 +- arch/x86/kernel/cpu/hygon.c | 2 +- arch/x86/kernel/cpu/intel.c | 4 ++-- arch/x86/kernel/cpu/mce/inject.c | 2 +- arch/x86/kernel/cpu/mce/intel.c | 2 +- arch/x86/kvm/vmx/sgx.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 12 +++++------ arch/x86/lib/msr.c | 2 +- arch/x86/power/cpu.c | 4 ++-- drivers/acpi/acpi_extlog.c | 2 +- drivers/acpi/acpi_lpit.c | 2 +- drivers/cpufreq/amd-pstate-ut.c | 4 ++-- drivers/cpufreq/amd_freq_sensitivity.c | 2 +- drivers/cpufreq/intel_pstate.c | 2 +- drivers/gpu/drm/i915/selftests/librapl.c | 4 ++-- drivers/hwmon/fam15h_power.c | 6 +++--- drivers/platform/x86/intel/ifs/core.c | 4 ++-- drivers/platform/x86/intel/pmc/core.c | 8 ++++---- .../intel/speed_select_if/isst_if_common.c | 10 +++++----- .../intel/speed_select_if/isst_if_mbox_msr.c | 4 ++-- .../platform/x86/intel/tpmi_power_domains.c | 4 ++-- drivers/platform/x86/intel/turbo_max_3.c | 2 +- drivers/powercap/intel_rapl_msr.c | 2 +- .../processor_thermal_device.c | 2 +- drivers/thermal/intel/intel_powerclamp.c | 4 ++-- drivers/thermal/intel/intel_tcc_cooling.c | 4 ++-- 40 files changed, 80 insertions(+), 80 deletions(-) diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c index 7e96f4652230..598a727d823a 100644 --- a/arch/x86/events/amd/power.c +++ b/arch/x86/events/amd/power.c @@ -272,7 +272,7 @@ static int __init amd_power_pmu_init(void) cpu_pwr_sample_ratio = cpuid_ecx(0x80000007); - if (rdmsrl_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &max_cu_acc_power)) { + if (rdmsrq_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &max_cu_acc_power)) { pr_err("Failed to read max compute unit power accumulator MSR\n"); return -ENODEV; } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 0a16d6bc11fa..5a2eda65db0a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -269,7 +269,7 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, */ for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) { reg = x86_pmu_config_addr(i); - ret = rdmsrl_safe(reg, &val); + ret = rdmsrq_safe(reg, &val); if (ret) goto msr_fail; if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { @@ -283,7 +283,7 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, if (*(u64 *)fixed_cntr_mask) { reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; - ret = rdmsrl_safe(reg, &val); + ret = rdmsrq_safe(reg, &val); if (ret) goto msr_fail; for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) { @@ -314,11 +314,11 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, * (qemu/kvm) that don't trap on the MSR access and always return 0s. */ reg = x86_pmu_event_addr(reg_safe); - if (rdmsrl_safe(reg, &val)) + if (rdmsrq_safe(reg, &val)) goto msr_fail; val ^= 0xffffUL; ret = wrmsrl_safe(reg, val); - ret |= rdmsrl_safe(reg, &val_new); + ret |= rdmsrq_safe(reg, &val_new); if (ret || val != val_new) goto msr_fail; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 3f4673708184..1e8c89f5f6ed 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5610,7 +5610,7 @@ static bool check_msr(unsigned long msr, u64 mask) * matches, this is needed to detect certain hardware emulators * (qemu/kvm) that don't trap on the MSR access and always return 0s. */ - if (rdmsrl_safe(msr, &val_old)) + if (rdmsrq_safe(msr, &val_old)) return false; /* @@ -5622,7 +5622,7 @@ static bool check_msr(unsigned long msr, u64 mask) val_tmp = lbr_from_signext_quirk_wr(val_tmp); if (wrmsrl_safe(msr, val_tmp) || - rdmsrl_safe(msr, &val_new)) + rdmsrq_safe(msr, &val_new)) return false; /* diff --git a/arch/x86/events/probe.c b/arch/x86/events/probe.c index 600bf8d15c0c..fda35cf25528 100644 --- a/arch/x86/events/probe.c +++ b/arch/x86/events/probe.c @@ -43,7 +43,7 @@ perf_msr_probe(struct perf_msr *msr, int cnt, bool zero, void *data) if (msr[bit].test && !msr[bit].test(bit, data)) continue; /* Virt sucks; you cannot tell if a R/O MSR is present :/ */ - if (rdmsrl_safe(msr[bit].msr, &val)) + if (rdmsrq_safe(msr[bit].msr, &val)) continue; mask = msr[bit].mask; diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index b7c256a2e201..7ff52c23d7a1 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -611,7 +611,7 @@ static int rapl_check_hw_unit(void) int i; /* protect rdmsrq() to handle virtualization */ - if (rdmsrl_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits)) + if (rdmsrq_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits)) return -1; for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) rapl_pkg_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL; diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 36e17988b7ba..1c136f54651c 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -120,7 +120,7 @@ static inline bool apic_is_x2apic_enabled(void) { u64 msr; - if (rdmsrl_safe(MSR_IA32_APICBASE, &msr)) + if (rdmsrq_safe(MSR_IA32_APICBASE, &msr)) return false; return msr & X2APIC_ENABLE; } diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 2e456cd2c97e..58554b5e7911 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -279,7 +279,7 @@ static inline int wrmsr_safe(u32 msr, u32 low, u32 high) __err; \ }) -static inline int rdmsrl_safe(u32 msr, u64 *p) +static inline int rdmsrq_safe(u32 msr, u64 *p) { int err; @@ -381,7 +381,7 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) } static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { - return rdmsrl_safe(msr_no, q); + return rdmsrq_safe(msr_no, q); } static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 87c2597c68f2..86a77528792d 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -231,7 +231,7 @@ static inline void wrmsrq(unsigned msr, u64 val) _err; \ }) -static inline int rdmsrl_safe(unsigned msr, u64 *p) +static inline int rdmsrq_safe(unsigned msr, u64 *p) { int err; diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 6d12a9b69432..dc389ca052b7 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -151,7 +151,7 @@ struct resource *amd_get_mmconfig_range(struct resource *res) /* Assume CPUs from Fam10h have mmconfig, although not all VMs do */ if (boot_cpu_data.x86 < 0x10 || - rdmsrl_safe(MSR_FAM10H_MMIO_CONF_BASE, &msr)) + rdmsrq_safe(MSR_FAM10H_MMIO_CONF_BASE, &msr)) return NULL; /* mmconfig is not enabled */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 7c7eca7451b6..3c49a92931e8 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -422,7 +422,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) * Try to cache the base value so further operations can * avoid RMW. If that faults, do not enable SSBD. */ - if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) { + if (!rdmsrq_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) { setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD); setup_force_cpu_cap(X86_FEATURE_SSBD); x86_amd_ls_cfg_ssbd_mask = 1ULL << bit; @@ -788,7 +788,7 @@ static void init_amd_bd(struct cpuinfo_x86 *c) * Disable it on the affected CPUs. */ if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) { - if (!rdmsrl_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) { + if (!rdmsrq_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) { value |= 0x1E; wrmsrl_safe(MSR_F15H_IC_CFG, value); } @@ -838,7 +838,7 @@ void init_spectral_chicken(struct cpuinfo_x86 *c) * suppresses non-branch predictions. */ if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { - if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) { + if (!rdmsrq_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) { value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT; wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value); } diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index dca78650d99e..e99892aad628 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -99,7 +99,7 @@ static bool __init turbo_disabled(void) u64 misc_en; int err; - err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en); + err = rdmsrq_safe(MSR_IA32_MISC_ENABLE, &misc_en); if (err) return false; @@ -110,11 +110,11 @@ static bool __init slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq) { int err; - err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq); + err = rdmsrq_safe(MSR_ATOM_CORE_RATIOS, base_freq); if (err) return false; - err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq); + err = rdmsrq_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq); if (err) return false; @@ -152,13 +152,13 @@ static bool __init knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int err, i; u64 msr; - err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + err = rdmsrq_safe(MSR_PLATFORM_INFO, base_freq); if (err) return false; *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */ - err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr); + err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, &msr); if (err) return false; @@ -190,17 +190,17 @@ static bool __init skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int s u32 group_size; int err, i; - err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + err = rdmsrq_safe(MSR_PLATFORM_INFO, base_freq); if (err) return false; *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */ - err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios); + err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, &ratios); if (err) return false; - err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts); + err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT1, &counts); if (err) return false; @@ -220,11 +220,11 @@ static bool __init core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq) u64 msr; int err; - err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq); + err = rdmsrq_safe(MSR_PLATFORM_INFO, base_freq); if (err) return false; - err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr); + err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, &msr); if (err) return false; diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index a96cfdcd94a0..18bd8a876bb8 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -95,7 +95,7 @@ static bool split_lock_verify_msr(bool on) { u64 ctrl, tmp; - if (rdmsrl_safe(MSR_TEST_CTRL, &ctrl)) + if (rdmsrq_safe(MSR_TEST_CTRL, &ctrl)) return false; if (on) ctrl |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a1f1be84a55f..dfccea1f120b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -148,7 +148,7 @@ static void ppin_init(struct cpuinfo_x86 *c) */ info = (struct ppin_info *)id->driver_data; - if (rdmsrl_safe(info->msr_ppin_ctl, &val)) + if (rdmsrq_safe(info->msr_ppin_ctl, &val)) goto clear_ppin; if ((val & 3UL) == 1UL) { @@ -159,7 +159,7 @@ static void ppin_init(struct cpuinfo_x86 *c) /* If PPIN is disabled, try to enable */ if (!(val & 2UL)) { wrmsrl_safe(info->msr_ppin_ctl, val | 2UL); - rdmsrl_safe(info->msr_ppin_ctl, &val); + rdmsrq_safe(info->msr_ppin_ctl, &val); } /* Is the enable bit set? */ diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index edd41950a959..441174844e01 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -118,7 +118,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c) bool enable_vmx; u64 msr; - if (rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr)) { + if (rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr)) { clear_cpu_cap(c, X86_FEATURE_VMX); clear_cpu_cap(c, X86_FEATURE_SGX); return; diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 10eeda3ba6ae..21541e310c2c 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -110,7 +110,7 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) * Try to cache the base value so further operations can * avoid RMW. If that faults, do not enable SSBD. */ - if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) { + if (!rdmsrq_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) { setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD); setup_force_cpu_cap(X86_FEATURE_SSBD); x86_amd_ls_cfg_ssbd_mask = 1ULL << 10; diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 72a11040e2e1..86bdda022009 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -488,7 +488,7 @@ static void init_cpuid_fault(struct cpuinfo_x86 *c) { u64 msr; - if (!rdmsrl_safe(MSR_PLATFORM_INFO, &msr)) { + if (!rdmsrq_safe(MSR_PLATFORM_INFO, &msr)) { if (msr & MSR_PLATFORM_INFO_CPUID_FAULT) set_cpu_cap(c, X86_FEATURE_CPUID_FAULT); } @@ -498,7 +498,7 @@ static void init_intel_misc_features(struct cpuinfo_x86 *c) { u64 msr; - if (rdmsrl_safe(MSR_MISC_FEATURES_ENABLES, &msr)) + if (rdmsrq_safe(MSR_MISC_FEATURES_ENABLES, &msr)) return; /* Clear all MISC features */ diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 75df35fca6a8..75ff605c8acc 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -748,7 +748,7 @@ static void check_hw_inj_possible(void) toggle_hw_mce_inject(cpu, true); wrmsrl_safe(mca_msr_reg(bank, MCA_STATUS), status); - rdmsrl_safe(mca_msr_reg(bank, MCA_STATUS), &status); + rdmsrq_safe(mca_msr_reg(bank, MCA_STATUS), &status); wrmsrl_safe(mca_msr_reg(bank, MCA_STATUS), 0); if (!status) { diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index 28f04c663c95..9b9ef7d421dc 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -460,7 +460,7 @@ static void intel_imc_init(struct cpuinfo_x86 *c) case INTEL_SANDYBRIDGE_X: case INTEL_IVYBRIDGE_X: case INTEL_HASWELL_X: - if (rdmsrl_safe(MSR_ERROR_CONTROL, &error_control)) + if (rdmsrq_safe(MSR_ERROR_CONTROL, &error_control)) return; error_control |= 2; wrmsrl_safe(MSR_ERROR_CONTROL, error_control); diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 3cf4ef659dbd..949864259ee6 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -411,7 +411,7 @@ void setup_default_sgx_lepubkeyhash(void) * MSRs exist but are read-only (locked and not writable). */ if (!enable_sgx || boot_cpu_has(X86_FEATURE_SGX_LC) || - rdmsrl_safe(MSR_IA32_SGXLEPUBKEYHASH0, &sgx_pubkey_hash[0])) { + rdmsrq_safe(MSR_IA32_SGXLEPUBKEYHASH0, &sgx_pubkey_hash[0])) { sgx_pubkey_hash[0] = 0xa6053e051270b7acULL; sgx_pubkey_hash[1] = 0x6cfbe8ba8b3b413dULL; sgx_pubkey_hash[2] = 0xc4916d99f2b3735dULL; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 0e12a3a97439..9b221bdd3c51 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2850,7 +2850,7 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) fault: WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", - rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); + rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); cr4_clear_bits(X86_CR4_VMXE); return -EFAULT; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 36504a170efc..134c4a186e9c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -590,7 +590,7 @@ static int kvm_probe_user_return_msr(u32 msr) int ret; preempt_disable(); - ret = rdmsrl_safe(msr, &val); + ret = rdmsrq_safe(msr, &val); if (ret) goto out; ret = wrmsrl_safe(msr, val); @@ -630,7 +630,7 @@ static void kvm_user_return_msr_cpu_online(void) int i; for (i = 0; i < kvm_nr_uret_msrs; ++i) { - rdmsrl_safe(kvm_uret_msrs_list[i], &value); + rdmsrq_safe(kvm_uret_msrs_list[i], &value); msrs->values[i].host = value; msrs->values[i].curr = value; } @@ -1660,7 +1660,7 @@ static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, *data = MSR_PLATFORM_INFO_CPUID_FAULT; break; case MSR_IA32_UCODE_REV: - rdmsrl_safe(index, data); + rdmsrq_safe(index, data); break; default: return kvm_x86_call(get_feature_msr)(index, data); @@ -9736,7 +9736,7 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) * with an exception. PAT[0] is set to WB on RESET and also by the * kernel, i.e. failure indicates a kernel bug or broken firmware. */ - if (rdmsrl_safe(MSR_IA32_CR_PAT, &host_pat) || + if (rdmsrq_safe(MSR_IA32_CR_PAT, &host_pat) || (host_pat & GENMASK(2, 0)) != 6) { pr_err("host PAT[0] is not WB\n"); return -EIO; @@ -9770,7 +9770,7 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) kvm_caps.supported_xcr0 = kvm_host.xcr0 & KVM_SUPPORTED_XCR0; } - rdmsrl_safe(MSR_EFER, &kvm_host.efer); + rdmsrq_safe(MSR_EFER, &kvm_host.efer); if (boot_cpu_has(X86_FEATURE_XSAVES)) rdmsrq(MSR_IA32_XSS, kvm_host.xss); @@ -13652,7 +13652,7 @@ int kvm_spec_ctrl_test_value(u64 value) local_irq_save(flags); - if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value)) + if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value)) ret = 1; else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value)) ret = 1; diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c index 7b90f547b109..d533881ce9a0 100644 --- a/arch/x86/lib/msr.c +++ b/arch/x86/lib/msr.c @@ -41,7 +41,7 @@ static int msr_read(u32 msr, struct msr *m) int err; u64 val; - err = rdmsrl_safe(msr, &val); + err = rdmsrq_safe(msr, &val); if (!err) m->q = val; diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 78903d330bec..d5a7b3bc2453 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -125,7 +125,7 @@ static void __save_processor_state(struct saved_context *ctxt) ctxt->cr2 = read_cr2(); ctxt->cr3 = __read_cr3(); ctxt->cr4 = __read_cr4(); - ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE, + ctxt->misc_enable_saved = !rdmsrq_safe(MSR_IA32_MISC_ENABLE, &ctxt->misc_enable); msr_save_context(ctxt); } @@ -414,7 +414,7 @@ static int msr_build_context(const u32 *msr_id, const int num) u64 dummy; msr_array[i].info.msr_no = msr_id[j]; - msr_array[i].valid = !rdmsrl_safe(msr_id[j], &dummy); + msr_array[i].valid = !rdmsrq_safe(msr_id[j], &dummy); msr_array[i].info.reg.q = 0; } saved_msrs->num = total_num; diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c index f7fb7205028d..8465822b6672 100644 --- a/drivers/acpi/acpi_extlog.c +++ b/drivers/acpi/acpi_extlog.c @@ -234,7 +234,7 @@ static int __init extlog_init(void) u64 cap; int rc; - if (rdmsrl_safe(MSR_IA32_MCG_CAP, &cap) || + if (rdmsrq_safe(MSR_IA32_MCG_CAP, &cap) || !(cap & MCG_ELOG_P) || !extlog_get_l1addr()) return -ENODEV; diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c index 794962c5c88e..b8d98b1b48ae 100644 --- a/drivers/acpi/acpi_lpit.c +++ b/drivers/acpi/acpi_lpit.c @@ -39,7 +39,7 @@ static int lpit_read_residency_counter_us(u64 *counter, bool io_mem) return 0; } - err = rdmsrl_safe(residency_info_ffh.gaddr.address, counter); + err = rdmsrq_safe(residency_info_ffh.gaddr.address, counter); if (!err) { u64 mask = GENMASK_ULL(residency_info_ffh.gaddr.bit_offset + residency_info_ffh.gaddr. bit_width - 1, diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index e671bc7d1550..4a1f5b1fbd84 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -90,9 +90,9 @@ static int amd_pstate_ut_check_enabled(u32 index) if (get_shared_mem()) return 0; - ret = rdmsrl_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable); + ret = rdmsrq_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable); if (ret) { - pr_err("%s rdmsrl_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret); + pr_err("%s rdmsrq_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret); return ret; } diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c index 59b19b9975e8..13fed4b9e02b 100644 --- a/drivers/cpufreq/amd_freq_sensitivity.c +++ b/drivers/cpufreq/amd_freq_sensitivity.c @@ -129,7 +129,7 @@ static int __init amd_freq_sensitivity_init(void) pci_dev_put(pcidev); } - if (rdmsrl_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val)) + if (rdmsrq_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val)) return -ENODEV; if (!(val >> CLASS_CODE_SHIFT)) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 3883a2b3e599..a57f494f1032 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1877,7 +1877,7 @@ void notify_hwp_interrupt(void) if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) status_mask |= HWP_HIGHEST_PERF_CHANGE_STATUS; - rdmsrl_safe(MSR_HWP_STATUS, &value); + rdmsrq_safe(MSR_HWP_STATUS, &value); if (!(value & status_mask)) return; diff --git a/drivers/gpu/drm/i915/selftests/librapl.c b/drivers/gpu/drm/i915/selftests/librapl.c index eb03b5b28bad..25b8726b9dff 100644 --- a/drivers/gpu/drm/i915/selftests/librapl.c +++ b/drivers/gpu/drm/i915/selftests/librapl.c @@ -22,12 +22,12 @@ u64 librapl_energy_uJ(void) unsigned long long power; u32 units; - if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) + if (rdmsrq_safe(MSR_RAPL_POWER_UNIT, &power)) return 0; units = (power & 0x1f00) >> 8; - if (rdmsrl_safe(MSR_PP1_ENERGY_STATUS, &power)) + if (rdmsrq_safe(MSR_PP1_ENERGY_STATUS, &power)) return 0; return (1000000 * power) >> units; /* convert to uJ */ diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c index 9ed2c4b6734e..8ecebea53651 100644 --- a/drivers/hwmon/fam15h_power.c +++ b/drivers/hwmon/fam15h_power.c @@ -143,8 +143,8 @@ static void do_read_registers_on_cu(void *_data) */ cu = topology_core_id(smp_processor_id()); - rdmsrl_safe(MSR_F15H_CU_PWR_ACCUMULATOR, &data->cu_acc_power[cu]); - rdmsrl_safe(MSR_F15H_PTSC, &data->cpu_sw_pwr_ptsc[cu]); + rdmsrq_safe(MSR_F15H_CU_PWR_ACCUMULATOR, &data->cu_acc_power[cu]); + rdmsrq_safe(MSR_F15H_PTSC, &data->cpu_sw_pwr_ptsc[cu]); data->cu_on[cu] = 1; } @@ -424,7 +424,7 @@ static int fam15h_power_init_data(struct pci_dev *f4, */ data->cpu_pwr_sample_ratio = cpuid_ecx(0x80000007); - if (rdmsrl_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &tmp)) { + if (rdmsrq_safe(MSR_F15H_CU_MAX_PWR_ACCUMULATOR, &tmp)) { pr_err("Failed to read max compute unit power accumulator MSR\n"); return -ENODEV; } diff --git a/drivers/platform/x86/intel/ifs/core.c b/drivers/platform/x86/intel/ifs/core.c index 1ae50702bdb7..c4328a7ae083 100644 --- a/drivers/platform/x86/intel/ifs/core.c +++ b/drivers/platform/x86/intel/ifs/core.c @@ -115,13 +115,13 @@ static int __init ifs_init(void) if (!m) return -ENODEV; - if (rdmsrl_safe(MSR_IA32_CORE_CAPS, &msrval)) + if (rdmsrq_safe(MSR_IA32_CORE_CAPS, &msrval)) return -ENODEV; if (!(msrval & MSR_IA32_CORE_CAPS_INTEGRITY_CAPS)) return -ENODEV; - if (rdmsrl_safe(MSR_INTEGRITY_CAPS, &msrval)) + if (rdmsrq_safe(MSR_INTEGRITY_CAPS, &msrval)) return -ENODEV; ifs_pkg_auth = kmalloc_array(topology_max_packages(), sizeof(bool), GFP_KERNEL); diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c index 7a1d11f2914f..ff7f64df7c09 100644 --- a/drivers/platform/x86/intel/pmc/core.c +++ b/drivers/platform/x86/intel/pmc/core.c @@ -1082,7 +1082,7 @@ static int pmc_core_pkgc_show(struct seq_file *s, void *unused) unsigned int index; for (index = 0; map[index].name ; index++) { - if (rdmsrl_safe(map[index].bit_mask, &pcstate_count)) + if (rdmsrq_safe(map[index].bit_mask, &pcstate_count)) continue; pcstate_count *= 1000; @@ -1587,7 +1587,7 @@ static __maybe_unused int pmc_core_suspend(struct device *dev) /* Save PKGC residency for checking later */ for (i = 0; i < pmcdev->num_of_pkgc; i++) { - if (rdmsrl_safe(msr_map[i].bit_mask, &pmcdev->pkgc_res_cnt[i])) + if (rdmsrq_safe(msr_map[i].bit_mask, &pmcdev->pkgc_res_cnt[i])) return -EIO; } @@ -1603,7 +1603,7 @@ static inline bool pmc_core_is_deepest_pkgc_failed(struct pmc_dev *pmcdev) u32 deepest_pkgc_msr = msr_map[pmcdev->num_of_pkgc - 1].bit_mask; u64 deepest_pkgc_residency; - if (rdmsrl_safe(deepest_pkgc_msr, &deepest_pkgc_residency)) + if (rdmsrq_safe(deepest_pkgc_msr, &deepest_pkgc_residency)) return false; if (deepest_pkgc_residency == pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1]) @@ -1655,7 +1655,7 @@ int pmc_core_resume_common(struct pmc_dev *pmcdev) for (i = 0; i < pmcdev->num_of_pkgc; i++) { u64 pc_cnt; - if (!rdmsrl_safe(msr_map[i].bit_mask, &pc_cnt)) { + if (!rdmsrq_safe(msr_map[i].bit_mask, &pc_cnt)) { dev_info(dev, "Prev %s cnt = 0x%llx, Current %s cnt = 0x%llx\n", msr_map[i].name, pmcdev->pkgc_res_cnt[i], msr_map[i].name, pc_cnt); diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c index 31239a93dd71..1ebf5624139a 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c @@ -406,7 +406,7 @@ static int isst_if_cpu_online(unsigned int cpu) isst_cpu_info[cpu].numa_node = cpu_to_node(cpu); - ret = rdmsrl_safe(MSR_CPU_BUS_NUMBER, &data); + ret = rdmsrq_safe(MSR_CPU_BUS_NUMBER, &data); if (ret) { /* This is not a fatal error on MSR mailbox only I/F */ isst_cpu_info[cpu].bus_info[0] = -1; @@ -420,12 +420,12 @@ static int isst_if_cpu_online(unsigned int cpu) if (isst_hpm_support) { - ret = rdmsrl_safe(MSR_PM_LOGICAL_ID, &data); + ret = rdmsrq_safe(MSR_PM_LOGICAL_ID, &data); if (!ret) goto set_punit_id; } - ret = rdmsrl_safe(MSR_THREAD_ID_INFO, &data); + ret = rdmsrq_safe(MSR_THREAD_ID_INFO, &data); if (ret) { isst_cpu_info[cpu].punit_cpu_id = -1; return ret; @@ -831,8 +831,8 @@ static int __init isst_if_common_init(void) u64 data; /* Can fail only on some Skylake-X generations */ - if (rdmsrl_safe(MSR_OS_MAILBOX_INTERFACE, &data) || - rdmsrl_safe(MSR_OS_MAILBOX_DATA, &data)) + if (rdmsrq_safe(MSR_OS_MAILBOX_INTERFACE, &data) || + rdmsrq_safe(MSR_OS_MAILBOX_DATA, &data)) return -ENODEV; } diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c index 0d4d67771afa..78989f649aea 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c @@ -176,11 +176,11 @@ static int __init isst_if_mbox_init(void) return -ENODEV; /* Check presence of mailbox MSRs */ - ret = rdmsrl_safe(MSR_OS_MAILBOX_INTERFACE, &data); + ret = rdmsrq_safe(MSR_OS_MAILBOX_INTERFACE, &data); if (ret) return ret; - ret = rdmsrl_safe(MSR_OS_MAILBOX_DATA, &data); + ret = rdmsrq_safe(MSR_OS_MAILBOX_DATA, &data); if (ret) return ret; diff --git a/drivers/platform/x86/intel/tpmi_power_domains.c b/drivers/platform/x86/intel/tpmi_power_domains.c index 2f01cd22a6ee..c21b3cb99b7c 100644 --- a/drivers/platform/x86/intel/tpmi_power_domains.c +++ b/drivers/platform/x86/intel/tpmi_power_domains.c @@ -157,7 +157,7 @@ static int tpmi_get_logical_id(unsigned int cpu, struct tpmi_cpu_info *info) u64 data; int ret; - ret = rdmsrl_safe(MSR_PM_LOGICAL_ID, &data); + ret = rdmsrq_safe(MSR_PM_LOGICAL_ID, &data); if (ret) return ret; @@ -203,7 +203,7 @@ static int __init tpmi_init(void) return -ENODEV; /* Check for MSR 0x54 presence */ - ret = rdmsrl_safe(MSR_PM_LOGICAL_ID, &data); + ret = rdmsrq_safe(MSR_PM_LOGICAL_ID, &data); if (ret) return ret; diff --git a/drivers/platform/x86/intel/turbo_max_3.c b/drivers/platform/x86/intel/turbo_max_3.c index 79a0bcdeffb8..c411ec53145e 100644 --- a/drivers/platform/x86/intel/turbo_max_3.c +++ b/drivers/platform/x86/intel/turbo_max_3.c @@ -48,7 +48,7 @@ static int get_oc_core_priority(unsigned int cpu) } for (i = 0; i < OC_MAILBOX_RETRY_COUNT; ++i) { - ret = rdmsrl_safe(MSR_OC_MAILBOX, &value); + ret = rdmsrq_safe(MSR_OC_MAILBOX, &value); if (ret) { pr_debug("cpu %d OC mailbox read failed\n", cpu); break; diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 2b81aabdb0db..9100300ff868 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -116,7 +116,7 @@ static void rapl_msr_update_func(void *info) struct reg_action *ra = info; u64 val; - ra->err = rdmsrl_safe(ra->reg.msr, &val); + ra->err = rdmsrq_safe(ra->reg.msr, &val); if (ra->err) return; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index c868d8b7bd1c..b0249468b844 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -153,7 +153,7 @@ static ssize_t tcc_offset_degree_celsius_store(struct device *dev, u64 val; int err; - err = rdmsrl_safe(MSR_PLATFORM_INFO, &val); + err = rdmsrq_safe(MSR_PLATFORM_INFO, &val); if (err) return err; diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c index 96a24df79686..9a4cec000910 100644 --- a/drivers/thermal/intel/intel_powerclamp.c +++ b/drivers/thermal/intel/intel_powerclamp.c @@ -340,7 +340,7 @@ static bool has_pkg_state_counter(void) /* check if any one of the counter msrs exists */ while (info->msr_index) { - if (!rdmsrl_safe(info->msr_index, &val)) + if (!rdmsrq_safe(info->msr_index, &val)) return true; info++; } @@ -356,7 +356,7 @@ static u64 pkg_state_counter(void) while (info->msr_index) { if (!info->skip) { - if (!rdmsrl_safe(info->msr_index, &val)) + if (!rdmsrq_safe(info->msr_index, &val)) count += val; else info->skip = true; diff --git a/drivers/thermal/intel/intel_tcc_cooling.c b/drivers/thermal/intel/intel_tcc_cooling.c index 9ff0ebdde0ef..0394897e83cf 100644 --- a/drivers/thermal/intel/intel_tcc_cooling.c +++ b/drivers/thermal/intel/intel_tcc_cooling.c @@ -81,14 +81,14 @@ static int __init tcc_cooling_init(void) if (!id) return -ENODEV; - err = rdmsrl_safe(MSR_PLATFORM_INFO, &val); + err = rdmsrq_safe(MSR_PLATFORM_INFO, &val); if (err) return err; if (!(val & TCC_PROGRAMMABLE)) return -ENODEV; - err = rdmsrl_safe(MSR_IA32_TEMPERATURE_TARGET, &val); + err = rdmsrq_safe(MSR_IA32_TEMPERATURE_TARGET, &val); if (err) return err; From 6fa17efe45440f43fa4e059d7a487179bbba053e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:57 +0200 Subject: [PATCH 097/331] x86/msr: Rename 'wrmsrl_safe()' to 'wrmsrq_safe()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/events/core.c | 2 +- arch/x86/events/intel/core.c | 8 ++++---- arch/x86/events/intel/knc.c | 4 ++-- arch/x86/events/intel/lbr.c | 2 +- arch/x86/events/intel/p4.c | 16 ++++++++-------- arch/x86/events/intel/p6.c | 4 ++-- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/kernel/cpu/amd.c | 6 +++--- arch/x86/kernel/cpu/bus_lock.c | 2 +- arch/x86/kernel/cpu/common.c | 14 +++++++------- arch/x86/kernel/cpu/mce/inject.c | 4 ++-- arch/x86/kernel/cpu/mce/intel.c | 2 +- arch/x86/kernel/cpu/resctrl/core.c | 2 +- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/x86.c | 6 +++--- arch/x86/lib/msr.c | 2 +- drivers/cpufreq/intel_pstate.c | 2 +- .../x86/intel/speed_select_if/isst_if_common.c | 2 +- drivers/platform/x86/intel/turbo_max_3.c | 2 +- drivers/powercap/intel_rapl_msr.c | 2 +- drivers/thermal/intel/therm_throt.c | 2 +- 21 files changed, 45 insertions(+), 45 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 5a2eda65db0a..85b55c1dc162 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -317,7 +317,7 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, if (rdmsrq_safe(reg, &val)) goto msr_fail; val ^= 0xffffUL; - ret = wrmsrl_safe(reg, val); + ret = wrmsrq_safe(reg, val); ret |= rdmsrq_safe(reg, &val_new); if (ret || val != val_new) goto msr_fail; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 1e8c89f5f6ed..394fa83b537b 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2976,13 +2976,13 @@ static void intel_pmu_reset(void) pr_info("clearing PMU state on CPU#%d\n", smp_processor_id()); for_each_set_bit(idx, cntr_mask, INTEL_PMC_MAX_GENERIC) { - wrmsrl_safe(x86_pmu_config_addr(idx), 0ull); - wrmsrl_safe(x86_pmu_event_addr(idx), 0ull); + wrmsrq_safe(x86_pmu_config_addr(idx), 0ull); + wrmsrq_safe(x86_pmu_event_addr(idx), 0ull); } for_each_set_bit(idx, fixed_cntr_mask, INTEL_PMC_MAX_FIXED) { if (fixed_counter_disabled(idx, cpuc->pmu)) continue; - wrmsrl_safe(x86_pmu_fixed_ctr_addr(idx), 0ull); + wrmsrq_safe(x86_pmu_fixed_ctr_addr(idx), 0ull); } if (ds) @@ -5621,7 +5621,7 @@ static bool check_msr(unsigned long msr, u64 mask) if (is_lbr_from(msr)) val_tmp = lbr_from_signext_quirk_wr(val_tmp); - if (wrmsrl_safe(msr, val_tmp) || + if (wrmsrq_safe(msr, val_tmp) || rdmsrq_safe(msr, &val_new)) return false; diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index afb7801bdece..425f6e6eed89 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -182,7 +182,7 @@ knc_pmu_disable_event(struct perf_event *event) val = hwc->config; val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; - (void)wrmsrl_safe(hwc->config_base + hwc->idx, val); + (void)wrmsrq_safe(hwc->config_base + hwc->idx, val); } static void knc_pmu_enable_event(struct perf_event *event) @@ -193,7 +193,7 @@ static void knc_pmu_enable_event(struct perf_event *event) val = hwc->config; val |= ARCH_PERFMON_EVENTSEL_ENABLE; - (void)wrmsrl_safe(hwc->config_base + hwc->idx, val); + (void)wrmsrq_safe(hwc->config_base + hwc->idx, val); } static inline u64 knc_pmu_get_status(void) diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index b55c59ab53d2..48894f7a099f 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1602,7 +1602,7 @@ void __init intel_pmu_arch_lbr_init(void) goto clear_arch_lbr; /* Apply the max depth of Arch LBR */ - if (wrmsrl_safe(MSR_ARCH_LBR_DEPTH, lbr_nr)) + if (wrmsrq_safe(MSR_ARCH_LBR_DEPTH, lbr_nr)) goto clear_arch_lbr; x86_pmu.lbr_depth_mask = eax.split.lbr_depth_mask; diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index 36d4cd23182c..24d811a9608a 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -897,8 +897,8 @@ static void p4_pmu_disable_pebs(void) * So at moment let leave metrics turned on forever -- it's * ok for now but need to be revisited! * - * (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, 0); - * (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, 0); + * (void)wrmsrq_safe(MSR_IA32_PEBS_ENABLE, 0); + * (void)wrmsrq_safe(MSR_P4_PEBS_MATRIX_VERT, 0); */ } @@ -911,7 +911,7 @@ static inline void p4_pmu_disable_event(struct perf_event *event) * state we need to clear P4_CCCR_OVF, otherwise interrupt get * asserted again and again */ - (void)wrmsrl_safe(hwc->config_base, + (void)wrmsrq_safe(hwc->config_base, p4_config_unpack_cccr(hwc->config) & ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED); } @@ -944,8 +944,8 @@ static void p4_pmu_enable_pebs(u64 config) bind = &p4_pebs_bind_map[idx]; - (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, (u64)bind->metric_pebs); - (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, (u64)bind->metric_vert); + (void)wrmsrq_safe(MSR_IA32_PEBS_ENABLE, (u64)bind->metric_pebs); + (void)wrmsrq_safe(MSR_P4_PEBS_MATRIX_VERT, (u64)bind->metric_vert); } static void __p4_pmu_enable_event(struct perf_event *event) @@ -979,8 +979,8 @@ static void __p4_pmu_enable_event(struct perf_event *event) */ p4_pmu_enable_pebs(hwc->config); - (void)wrmsrl_safe(escr_addr, escr_conf); - (void)wrmsrl_safe(hwc->config_base, + (void)wrmsrq_safe(escr_addr, escr_conf); + (void)wrmsrq_safe(hwc->config_base, (cccr & ~P4_CCCR_RESERVED) | P4_CCCR_ENABLE); } @@ -1398,7 +1398,7 @@ __init int p4_pmu_init(void) */ for_each_set_bit(i, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { reg = x86_pmu_config_addr(i); - wrmsrl_safe(reg, 0ULL); + wrmsrq_safe(reg, 0ULL); } return 0; diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c index 6a1fdbc6d20f..35917a776bec 100644 --- a/arch/x86/events/intel/p6.c +++ b/arch/x86/events/intel/p6.c @@ -163,7 +163,7 @@ p6_pmu_disable_event(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; u64 val = P6_NOP_EVENT; - (void)wrmsrl_safe(hwc->config_base, val); + (void)wrmsrq_safe(hwc->config_base, val); } static void p6_pmu_enable_event(struct perf_event *event) @@ -180,7 +180,7 @@ static void p6_pmu_enable_event(struct perf_event *event) * to actually enable the events. */ - (void)wrmsrl_safe(hwc->config_base, val); + (void)wrmsrq_safe(hwc->config_base, val); } PMU_FORMAT_ATTR(event, "config:0-7" ); diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 58554b5e7911..231757554236 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -316,7 +316,7 @@ static __always_inline void wrmsrns(u32 msr, u64 val) /* * 64-bit version of wrmsr_safe(): */ -static inline int wrmsrl_safe(u32 msr, u64 val) +static inline int wrmsrq_safe(u32 msr, u64 val) { return wrmsr_safe(msr, (u32)val, (u32)(val >> 32)); } @@ -385,7 +385,7 @@ static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) } static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { - return wrmsrl_safe(msr_no, q); + return wrmsrq_safe(msr_no, q); } static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) { diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 3c49a92931e8..8937923c9f81 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -612,7 +612,7 @@ static void early_init_amd(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) { if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB)) setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); - else if (c->x86 >= 0x19 && !wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) { + else if (c->x86 >= 0x19 && !wrmsrq_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) { setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE); setup_force_cpu_cap(X86_FEATURE_SBPB); } @@ -790,7 +790,7 @@ static void init_amd_bd(struct cpuinfo_x86 *c) if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) { if (!rdmsrq_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) { value |= 0x1E; - wrmsrl_safe(MSR_F15H_IC_CFG, value); + wrmsrq_safe(MSR_F15H_IC_CFG, value); } } @@ -840,7 +840,7 @@ void init_spectral_chicken(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) { if (!rdmsrq_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) { value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT; - wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value); + wrmsrq_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value); } } #endif diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index 18bd8a876bb8..a18d0f2ea832 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -101,7 +101,7 @@ static bool split_lock_verify_msr(bool on) ctrl |= MSR_TEST_CTRL_SPLIT_LOCK_DETECT; else ctrl &= ~MSR_TEST_CTRL_SPLIT_LOCK_DETECT; - if (wrmsrl_safe(MSR_TEST_CTRL, ctrl)) + if (wrmsrq_safe(MSR_TEST_CTRL, ctrl)) return false; rdmsrq(MSR_TEST_CTRL, tmp); return ctrl == tmp; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index dfccea1f120b..bb986ba8b90c 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -158,7 +158,7 @@ static void ppin_init(struct cpuinfo_x86 *c) /* If PPIN is disabled, try to enable */ if (!(val & 2UL)) { - wrmsrl_safe(info->msr_ppin_ctl, val | 2UL); + wrmsrq_safe(info->msr_ppin_ctl, val | 2UL); rdmsrq_safe(info->msr_ppin_ctl, &val); } @@ -2114,15 +2114,15 @@ static inline void idt_syscall_init(void) * This does not cause SYSENTER to jump to the wrong location, because * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit). */ - wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS); - wrmsrl_safe(MSR_IA32_SYSENTER_ESP, + wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS); + wrmsrq_safe(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1)); - wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat); + wrmsrq_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat); } else { wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore); - wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG); - wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL); - wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL); + wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG); + wrmsrq_safe(MSR_IA32_SYSENTER_ESP, 0ULL); + wrmsrq_safe(MSR_IA32_SYSENTER_EIP, 0ULL); } /* diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 75ff605c8acc..5226f8f7fc98 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -747,9 +747,9 @@ static void check_hw_inj_possible(void) toggle_hw_mce_inject(cpu, true); - wrmsrl_safe(mca_msr_reg(bank, MCA_STATUS), status); + wrmsrq_safe(mca_msr_reg(bank, MCA_STATUS), status); rdmsrq_safe(mca_msr_reg(bank, MCA_STATUS), &status); - wrmsrl_safe(mca_msr_reg(bank, MCA_STATUS), 0); + wrmsrq_safe(mca_msr_reg(bank, MCA_STATUS), 0); if (!status) { hw_injection_possible = false; diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index 9b9ef7d421dc..efcf21e9552e 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -463,7 +463,7 @@ static void intel_imc_init(struct cpuinfo_x86 *c) if (rdmsrq_safe(MSR_ERROR_CONTROL, &error_control)) return; error_control |= 2; - wrmsrl_safe(MSR_ERROR_CONTROL, error_control); + wrmsrq_safe(MSR_ERROR_CONTROL, error_control); break; } } diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 6d408f78873c..280d6900726b 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -145,7 +145,7 @@ static inline void cache_alloc_hsw_probe(void) struct rdt_resource *r = &hw_res->r_resctrl; u64 max_cbm = BIT_ULL_MASK(20) - 1, l3_cbm_0; - if (wrmsrl_safe(MSR_IA32_L3_CBM_BASE, max_cbm)) + if (wrmsrq_safe(MSR_IA32_L3_CBM_BASE, max_cbm)) return; rdmsrq(MSR_IA32_L3_CBM_BASE, l3_cbm_0); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0bc708ee2788..a4aabd666665 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3119,7 +3119,7 @@ static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va) * back to WBINVD if this faults so as not to make any problems worse * by leaving stale encrypted data in the cache. */ - if (WARN_ON_ONCE(wrmsrl_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid))) + if (WARN_ON_ONCE(wrmsrq_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid))) goto do_wbinvd; return; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 134c4a186e9c..b922428236aa 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -593,7 +593,7 @@ static int kvm_probe_user_return_msr(u32 msr) ret = rdmsrq_safe(msr, &val); if (ret) goto out; - ret = wrmsrl_safe(msr, val); + ret = wrmsrq_safe(msr, val); out: preempt_enable(); return ret; @@ -644,7 +644,7 @@ int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) value = (value & mask) | (msrs->values[slot].host & ~mask); if (value == msrs->values[slot].curr) return 0; - err = wrmsrl_safe(kvm_uret_msrs_list[slot], value); + err = wrmsrq_safe(kvm_uret_msrs_list[slot], value); if (err) return 1; @@ -13654,7 +13654,7 @@ int kvm_spec_ctrl_test_value(u64 value) if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value)) ret = 1; - else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value)) + else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value)) ret = 1; else wrmsrq(MSR_IA32_SPEC_CTRL, saved_value); diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c index d533881ce9a0..4ef7c6dcbea6 100644 --- a/arch/x86/lib/msr.c +++ b/arch/x86/lib/msr.c @@ -58,7 +58,7 @@ static int msr_read(u32 msr, struct msr *m) */ static int msr_write(u32 msr, struct msr *m) { - return wrmsrl_safe(msr, m->q); + return wrmsrq_safe(msr, m->q); } static inline int __flip_bit(u32 msr, u8 bit, bool set) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index a57f494f1032..83a43c64c404 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1894,7 +1894,7 @@ void notify_hwp_interrupt(void) return; ack_intr: - wrmsrl_safe(MSR_HWP_STATUS, 0); + wrmsrq_safe(MSR_HWP_STATUS, 0); raw_spin_unlock_irqrestore(&hwp_notify_lock, flags); } diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c index 1ebf5624139a..4368d598beac 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c @@ -211,7 +211,7 @@ static void isst_restore_msr_local(int cpu) hash_for_each_possible(isst_hash, sst_cmd, hnode, punit_msr_white_list[i]) { if (!sst_cmd->mbox_cmd_type && sst_cmd->cpu == cpu) - wrmsrl_safe(sst_cmd->cmd, sst_cmd->data); + wrmsrq_safe(sst_cmd->cmd, sst_cmd->data); } } mutex_unlock(&isst_hash_lock); diff --git a/drivers/platform/x86/intel/turbo_max_3.c b/drivers/platform/x86/intel/turbo_max_3.c index c411ec53145e..7e538bbd5b50 100644 --- a/drivers/platform/x86/intel/turbo_max_3.c +++ b/drivers/platform/x86/intel/turbo_max_3.c @@ -41,7 +41,7 @@ static int get_oc_core_priority(unsigned int cpu) value = cmd << MSR_OC_MAILBOX_CMD_OFFSET; /* Set the busy bit to indicate OS is trying to issue command */ value |= BIT_ULL(MSR_OC_MAILBOX_BUSY_BIT); - ret = wrmsrl_safe(MSR_OC_MAILBOX, value); + ret = wrmsrq_safe(MSR_OC_MAILBOX, value); if (ret) { pr_debug("cpu %d OC mailbox write failed\n", cpu); return ret; diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 9100300ff868..c7f15b01fad8 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -123,7 +123,7 @@ static void rapl_msr_update_func(void *info) val &= ~ra->mask; val |= ra->value; - ra->err = wrmsrl_safe(ra->reg.msr, val); + ra->err = wrmsrq_safe(ra->reg.msr, val); } static int rapl_msr_write_raw(int cpu, struct reg_action *ra) diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c index c4f3f13cd9e8..debc94e2dc16 100644 --- a/drivers/thermal/intel/therm_throt.c +++ b/drivers/thermal/intel/therm_throt.c @@ -643,7 +643,7 @@ static void notify_thresholds(__u64 msr_val) void __weak notify_hwp_interrupt(void) { - wrmsrl_safe(MSR_HWP_STATUS, 0); + wrmsrq_safe(MSR_HWP_STATUS, 0); } /* Thermal transition interrupt handler */ From 5e404cb7ac4c097e95896a4a3fba5f5aabf7f679 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:58 +0200 Subject: [PATCH 098/331] x86/msr: Rename 'rdmsrl_safe_on_cpu()' to 'rdmsrq_safe_on_cpu()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/events/intel/pt.c | 2 +- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/kernel/acpi/cppc.c | 6 +++--- arch/x86/lib/msr-smp.c | 4 ++-- drivers/cpufreq/amd-pstate-ut.c | 2 +- drivers/cpufreq/amd-pstate.c | 2 +- drivers/cpufreq/intel_pstate.c | 6 +++--- drivers/platform/x86/intel/speed_select_if/isst_if_common.c | 2 +- drivers/powercap/intel_rapl_msr.c | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index a645b0993d83..d579f1092357 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -1839,7 +1839,7 @@ static __init int pt_init(void) for_each_online_cpu(cpu) { u64 ctl; - ret = rdmsrl_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl); + ret = rdmsrq_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl); if (!ret && (ctl & RTIT_CTL_TRACEEN)) prior_warn++; } diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 231757554236..c86814c35dec 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -335,7 +335,7 @@ void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu * void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); -int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); +int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); @@ -379,7 +379,7 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) { return wrmsr_safe(msr_no, l, h); } -static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) +static inline int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { return rdmsrq_safe(msr_no, q); } diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 77bfb846490c..6a10d73cb88b 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -49,7 +49,7 @@ int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) { int err; - err = rdmsrl_safe_on_cpu(cpunum, reg->address, val); + err = rdmsrq_safe_on_cpu(cpunum, reg->address, val); if (!err) { u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, reg->bit_offset); @@ -65,7 +65,7 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) u64 rd_val; int err; - err = rdmsrl_safe_on_cpu(cpunum, reg->address, &rd_val); + err = rdmsrq_safe_on_cpu(cpunum, reg->address, &rd_val); if (!err) { u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, reg->bit_offset); @@ -147,7 +147,7 @@ int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) int ret; if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &val); + ret = rdmsrq_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &val); if (ret) goto out; diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index acd463d887e1..0360ce20aae8 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -220,7 +220,7 @@ int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) } EXPORT_SYMBOL(wrmsrl_safe_on_cpu); -int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) +int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { u32 low, high; int err; @@ -230,7 +230,7 @@ int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) return err; } -EXPORT_SYMBOL(rdmsrl_safe_on_cpu); +EXPORT_SYMBOL(rdmsrq_safe_on_cpu); /* * These variants are significantly slower, but allows control over diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 4a1f5b1fbd84..707fa81c749f 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -137,7 +137,7 @@ static int amd_pstate_ut_check_perf(u32 index) lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; lowest_perf = cppc_perf.lowest_perf; } else { - ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + ret = rdmsrq_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); if (ret) { pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret); return ret; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b7caaa83a649..16a3fe2207ba 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -391,7 +391,7 @@ static int msr_init_perf(struct amd_cpudata *cpudata) union perf_cached perf = READ_ONCE(cpudata->perf); u64 cap1, numerator; - int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, + int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); if (ret) return ret; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 83a43c64c404..6f6c14e31795 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2106,13 +2106,13 @@ static int core_get_tdp_ratio(int cpu, u64 plat_info) int err; /* Get the TDP level (0, 1, 2) to get ratios */ - err = rdmsrl_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); + err = rdmsrq_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); if (err) return err; /* TDP MSR are continuous starting at 0x648 */ tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03); - err = rdmsrl_safe_on_cpu(cpu, tdp_msr, &tdp_ratio); + err = rdmsrq_safe_on_cpu(cpu, tdp_msr, &tdp_ratio); if (err) return err; @@ -2149,7 +2149,7 @@ static int core_get_max_pstate(int cpu) return tdp_ratio; } - err = rdmsrl_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar); + err = rdmsrq_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar); if (!err) { int tar_levels; diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c index 4368d598beac..46b96f1aa522 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c @@ -535,7 +535,7 @@ static long isst_if_msr_cmd_req(u8 *cmd_ptr, int *write_only, int resume) } else { u64 data; - ret = rdmsrl_safe_on_cpu(msr_cmd->logical_cpu, + ret = rdmsrq_safe_on_cpu(msr_cmd->logical_cpu, msr_cmd->msr, &data); if (!ret) { msr_cmd->data = data; diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index c7f15b01fad8..6d5853db17ad 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -103,7 +103,7 @@ static int rapl_cpu_down_prep(unsigned int cpu) static int rapl_msr_read_raw(int cpu, struct reg_action *ra) { - if (rdmsrl_safe_on_cpu(cpu, ra->reg.msr, &ra->value)) { + if (rdmsrq_safe_on_cpu(cpu, ra->reg.msr, &ra->value)) { pr_debug("failed to read msr 0x%x on cpu %d\n", ra->reg.msr, cpu); return -EIO; } From 27a23a544a55b55259ee6b438497dd4384c387c5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:28:59 +0200 Subject: [PATCH 099/331] x86/msr: Rename 'wrmsrl_safe_on_cpu()' to 'wrmsrq_safe_on_cpu()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/kernel/acpi/cppc.c | 2 +- arch/x86/lib/msr-smp.c | 4 ++-- drivers/cpufreq/amd-pstate.c | 2 +- drivers/platform/x86/intel/speed_select_if/isst_if_common.c | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index c86814c35dec..52983279c953 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -336,7 +336,7 @@ void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu * int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); -int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); +int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); #else /* CONFIG_SMP */ @@ -383,7 +383,7 @@ static inline int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { return rdmsrq_safe(msr_no, q); } -static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) +static inline int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { return wrmsrq_safe(msr_no, q); } diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 6a10d73cb88b..78b4e4b40985 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -74,7 +74,7 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) val &= mask; rd_val &= ~mask; rd_val |= val; - err = wrmsrl_safe_on_cpu(cpunum, reg->address, rd_val); + err = wrmsrq_safe_on_cpu(cpunum, reg->address, rd_val); } return err; } diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index 0360ce20aae8..434fdc21a86e 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -204,7 +204,7 @@ int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) } EXPORT_SYMBOL(wrmsr_safe_on_cpu); -int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) +int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { int err; struct msr_info rv; @@ -218,7 +218,7 @@ int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) return err ? err : rv.err; } -EXPORT_SYMBOL(wrmsrl_safe_on_cpu); +EXPORT_SYMBOL(wrmsrq_safe_on_cpu); int rdmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 16a3fe2207ba..0615c73306cf 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -371,7 +371,7 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) static inline int msr_cppc_enable(struct cpufreq_policy *policy) { - return wrmsrl_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); + return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); } static int shmem_cppc_enable(struct cpufreq_policy *policy) diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c index 46b96f1aa522..44dcd165b4c0 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c @@ -191,7 +191,7 @@ void isst_resume_common(void) if (cb->registered) isst_mbox_resume_command(cb, sst_cmd); } else { - wrmsrl_safe_on_cpu(sst_cmd->cpu, sst_cmd->cmd, + wrmsrq_safe_on_cpu(sst_cmd->cpu, sst_cmd->cmd, sst_cmd->data); } } @@ -524,7 +524,7 @@ static long isst_if_msr_cmd_req(u8 *cmd_ptr, int *write_only, int resume) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - ret = wrmsrl_safe_on_cpu(msr_cmd->logical_cpu, + ret = wrmsrq_safe_on_cpu(msr_cmd->logical_cpu, msr_cmd->msr, msr_cmd->data); *write_only = 1; From d7484babd2c4dcfa1ca02e7e303fab3fab529d75 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:00 +0200 Subject: [PATCH 100/331] x86/msr: Rename 'rdmsrl_on_cpu()' to 'rdmsrq_on_cpu()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/events/intel/uncore_snbep.c | 2 +- arch/x86/include/asm/msr.h | 4 ++-- arch/x86/kernel/cpu/intel_epb.c | 4 ++-- arch/x86/kernel/cpu/mce/inject.c | 4 ++-- arch/x86/lib/msr-smp.c | 4 ++-- drivers/cpufreq/acpi-cpufreq.c | 4 ++-- drivers/cpufreq/amd-pstate.c | 6 ++--- drivers/cpufreq/intel_pstate.c | 24 +++++++++---------- .../intel/uncore-frequency/uncore-frequency.c | 6 ++--- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 756dd11742ad..dd53dd87cdec 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -3765,7 +3765,7 @@ static int skx_msr_cpu_bus_read(int cpu, u64 *topology) { u64 msr_value; - if (rdmsrl_on_cpu(cpu, SKX_MSR_CPU_BUS_NUMBER, &msr_value) || + if (rdmsrq_on_cpu(cpu, SKX_MSR_CPU_BUS_NUMBER, &msr_value) || !(msr_value & SKX_MSR_CPU_BUS_VALID_BIT)) return -ENXIO; diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 52983279c953..850793b381c6 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -329,7 +329,7 @@ int msr_clear_bit(u32 msr, u8 bit); #ifdef CONFIG_SMP int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); -int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); +int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); @@ -350,7 +350,7 @@ static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) wrmsr(msr_no, l, h); return 0; } -static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) +static inline int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { rdmsrq(msr_no, *q); return 0; diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index 01d81b78d09e..54236defb666 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -135,7 +135,7 @@ static ssize_t energy_perf_bias_show(struct device *dev, u64 epb; int ret; - ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + ret = rdmsrq_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); if (ret < 0) return ret; @@ -157,7 +157,7 @@ static ssize_t energy_perf_bias_store(struct device *dev, else if (kstrtou64(buf, 0, &val) || val > MAX_EPB) return -EINVAL; - ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + ret = rdmsrq_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); if (ret < 0) return ret; diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 5226f8f7fc98..338aeee95bd2 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -589,7 +589,7 @@ static int inj_bank_set(void *data, u64 val) u64 cap; /* Get bank count on target CPU so we can handle non-uniform values. */ - rdmsrl_on_cpu(m->extcpu, MSR_IA32_MCG_CAP, &cap); + rdmsrq_on_cpu(m->extcpu, MSR_IA32_MCG_CAP, &cap); n_banks = cap & MCG_BANKCNT_MASK; if (val >= n_banks) { @@ -613,7 +613,7 @@ static int inj_bank_set(void *data, u64 val) if (cpu_feature_enabled(X86_FEATURE_SMCA)) { u64 ipid; - if (rdmsrl_on_cpu(m->extcpu, MSR_AMD64_SMCA_MCx_IPID(val), &ipid)) { + if (rdmsrq_on_cpu(m->extcpu, MSR_AMD64_SMCA_MCx_IPID(val), &ipid)) { pr_err("Error reading IPID on CPU%d\n", m->extcpu); return -EINVAL; } diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index 434fdc21a86e..b6081fc2217f 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -47,7 +47,7 @@ int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) } EXPORT_SYMBOL(rdmsr_on_cpu); -int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) +int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) { int err; struct msr_info rv; @@ -60,7 +60,7 @@ int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) return err; } -EXPORT_SYMBOL(rdmsrl_on_cpu); +EXPORT_SYMBOL(rdmsrq_on_cpu); int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) { diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index d19867cf816a..8bc08f3b0d5d 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -79,11 +79,11 @@ static bool boost_state(unsigned int cpu) case X86_VENDOR_INTEL: case X86_VENDOR_CENTAUR: case X86_VENDOR_ZHAOXIN: - rdmsrl_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &msr); + rdmsrq_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &msr); return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); case X86_VENDOR_HYGON: case X86_VENDOR_AMD: - rdmsrl_on_cpu(cpu, MSR_K7_HWCR, &msr); + rdmsrq_on_cpu(cpu, MSR_K7_HWCR, &msr); return !(msr & MSR_K7_HWCR_CPB_DIS); } return false; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 0615c73306cf..e987486cbb34 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -197,7 +197,7 @@ static u8 msr_get_epp(struct amd_cpudata *cpudata) u64 value; int ret; - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); if (ret < 0) { pr_debug("Could not retrieve energy perf value (%d)\n", ret); return ret; @@ -769,7 +769,7 @@ static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) goto exit_err; } - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); if (ret) { pr_err_once("failed to read initial CPU boost state!\n"); ret = -EIO; @@ -1491,7 +1491,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) } if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); if (ret) return ret; WRITE_ONCE(cpudata->cppc_req_cached, value); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 6f6c14e31795..8ce9d54ea32e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -620,7 +620,7 @@ static s16 intel_pstate_get_epb(struct cpudata *cpu_data) if (!boot_cpu_has(X86_FEATURE_EPB)) return -ENXIO; - ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + ret = rdmsrq_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); if (ret) return (s16)ret; @@ -637,7 +637,7 @@ static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data) * MSR_HWP_REQUEST, so need to read and get EPP. */ if (!hwp_req_data) { - epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, + epp = rdmsrq_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &hwp_req_data); if (epp) return epp; @@ -659,7 +659,7 @@ static int intel_pstate_set_epb(int cpu, s16 pref) if (!boot_cpu_has(X86_FEATURE_EPB)) return -ENXIO; - ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + ret = rdmsrq_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); if (ret) return ret; @@ -916,7 +916,7 @@ static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf) if (ratio <= 0) { u64 cap; - rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap); + rdmsrq_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap); ratio = HWP_GUARANTEED_PERF(cap); } @@ -1088,7 +1088,7 @@ static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) { u64 cap; - rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap); + rdmsrq_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap); WRITE_ONCE(cpu->hwp_cap_cached, cap); cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap); cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap); @@ -1162,7 +1162,7 @@ static void intel_pstate_hwp_set(unsigned int cpu) if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) min = max; - rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); + rdmsrq_on_cpu(cpu, MSR_HWP_REQUEST, &value); value &= ~HWP_MIN_PERF(~0L); value |= HWP_MIN_PERF(min); @@ -2084,7 +2084,7 @@ static int core_get_min_pstate(int cpu) { u64 value; - rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 40) & 0xFF; } @@ -2092,7 +2092,7 @@ static int core_get_max_pstate_physical(int cpu) { u64 value; - rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 8) & 0xFF; } @@ -2137,7 +2137,7 @@ static int core_get_max_pstate(int cpu) int tdp_ratio; int err; - rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info); + rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info); max_pstate = (plat_info >> 8) & 0xFF; tdp_ratio = core_get_tdp_ratio(cpu, plat_info); @@ -2169,7 +2169,7 @@ static int core_get_turbo_pstate(int cpu) u64 value; int nont, ret; - rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); nont = core_get_max_pstate(cpu); ret = (value) & 255; if (ret <= nont) @@ -2198,7 +2198,7 @@ static int knl_get_turbo_pstate(int cpu) u64 value; int nont, ret; - rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); nont = core_get_max_pstate(cpu); ret = (((value) >> 8) & 0xFF); if (ret <= nont) @@ -3256,7 +3256,7 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) intel_pstate_get_hwp_cap(cpu); - rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value); + rdmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value); WRITE_ONCE(cpu->hwp_req_cached, value); cpu->epp_cached = intel_pstate_get_epp(cpu, value); diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c index 40bbf8e45fa4..5295cd1483f4 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c @@ -51,7 +51,7 @@ static int uncore_read_control_freq(struct uncore_data *data, unsigned int *valu if (data->control_cpu < 0) return -ENXIO; - ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); + ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); if (ret) return ret; @@ -76,7 +76,7 @@ static int uncore_write_control_freq(struct uncore_data *data, unsigned int inpu if (data->control_cpu < 0) return -ENXIO; - ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); + ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); if (ret) return ret; @@ -105,7 +105,7 @@ static int uncore_read_freq(struct uncore_data *data, unsigned int *freq) if (data->control_cpu < 0) return -ENXIO; - ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio); + ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio); if (ret) return ret; From c895ecdab2e4ded78a362721c5a63053060030c9 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:01 +0200 Subject: [PATCH 101/331] x86/msr: Rename 'wrmsrl_on_cpu()' to 'wrmsrq_on_cpu()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/include/asm/msr.h | 4 +-- arch/x86/kernel/cpu/intel_epb.c | 2 +- arch/x86/lib/msr-smp.c | 4 +-- drivers/cpufreq/amd-pstate.c | 6 ++-- drivers/cpufreq/intel_pstate.c | 34 +++++++++---------- .../intel/uncore-frequency/uncore-frequency.c | 4 +-- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 850793b381c6..4335f914d6f8 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -330,7 +330,7 @@ int msr_clear_bit(u32 msr, u8 bit); int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); -int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); +int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q); void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs); int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); @@ -355,7 +355,7 @@ static inline int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) rdmsrq(msr_no, *q); return 0; } -static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) +static inline int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { wrmsrq(msr_no, q); return 0; diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index 54236defb666..bc7671f920a7 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -161,7 +161,7 @@ static ssize_t energy_perf_bias_store(struct device *dev, if (ret < 0) return ret; - ret = wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, + ret = wrmsrq_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, (epb & ~EPB_MASK) | val); if (ret < 0) return ret; diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c index b6081fc2217f..b8f63419e6ae 100644 --- a/arch/x86/lib/msr-smp.c +++ b/arch/x86/lib/msr-smp.c @@ -78,7 +78,7 @@ int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) } EXPORT_SYMBOL(wrmsr_on_cpu); -int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) +int wrmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 q) { int err; struct msr_info rv; @@ -92,7 +92,7 @@ int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) return err; } -EXPORT_SYMBOL(wrmsrl_on_cpu); +EXPORT_SYMBOL(wrmsrq_on_cpu); static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs, diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e987486cbb34..fd1ae77142ad 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -261,7 +261,7 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, wrmsrq(MSR_AMD_CPPC_REQ, value); return 0; } else { - int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); if (ret) return ret; @@ -309,7 +309,7 @@ static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) if (value == prev) return 0; - ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); if (ret) { pr_err("failed to set energy perf value (%d)\n", ret); return ret; @@ -788,7 +788,7 @@ exit_err: static void amd_perf_ctl_reset(unsigned int cpu) { - wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); + wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); } /* diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 8ce9d54ea32e..8e23f3175db7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -664,7 +664,7 @@ static int intel_pstate_set_epb(int cpu, s16 pref) return ret; epb = (epb & ~0x0f) | pref; - wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb); + wrmsrq_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb); return 0; } @@ -762,7 +762,7 @@ static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp) * function, so it cannot run in parallel with the update below. */ WRITE_ONCE(cpu->hwp_req_cached, value); - ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + ret = wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); if (!ret) cpu->epp_cached = epp; @@ -1209,7 +1209,7 @@ static void intel_pstate_hwp_set(unsigned int cpu) } skip_epp: WRITE_ONCE(cpu_data->hwp_req_cached, value); - wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); + wrmsrq_on_cpu(cpu, MSR_HWP_REQUEST, value); } static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata); @@ -1256,7 +1256,7 @@ static void intel_pstate_hwp_offline(struct cpudata *cpu) if (boot_cpu_has(X86_FEATURE_HWP_EPP)) value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE); - wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); mutex_lock(&hybrid_capacity_lock); @@ -1302,7 +1302,7 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata); static void intel_pstate_hwp_reenable(struct cpudata *cpu) { intel_pstate_hwp_enable(cpu); - wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached)); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached)); } static int intel_pstate_suspend(struct cpufreq_policy *policy) @@ -1855,7 +1855,7 @@ static void intel_pstate_notify_work(struct work_struct *work) hybrid_update_capacity(cpudata); } - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); } static DEFINE_RAW_SPINLOCK(hwp_notify_lock); @@ -1905,8 +1905,8 @@ static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata) if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY)) return; - /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */ - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */ + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); raw_spin_lock_irq(&hwp_notify_lock); cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask); @@ -1933,9 +1933,9 @@ static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata) if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) interrupt_mask |= HWP_HIGHEST_PERF_CHANGE_REQ; - /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */ - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask); - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); + /* wrmsrq_on_cpu has to be outside spinlock as this can result in IPC */ + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask); + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); } } @@ -1974,9 +1974,9 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata) { /* First disable HWP notification interrupt till we activate again */ if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); - wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); + wrmsrq_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); intel_pstate_enable_hwp_interrupt(cpudata); @@ -2244,7 +2244,7 @@ static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) * the CPU being updated, so force the register update to run on the * right CPU. */ - wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, + wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } @@ -3102,7 +3102,7 @@ static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max, if (fast_switch) wrmsrq(MSR_HWP_REQUEST, value); else - wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); } static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu, @@ -3112,7 +3112,7 @@ static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu, wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, target_pstate)); else - wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, + wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, target_pstate)); } @@ -3323,7 +3323,7 @@ static int intel_cpufreq_suspend(struct cpufreq_policy *policy) * written by it may not be suitable. */ value &= ~HWP_DESIRED_PERF(~0L); - wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); WRITE_ONCE(cpu->hwp_req_cached, value); } diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c index 5295cd1483f4..6f873765d2d1 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c @@ -88,7 +88,7 @@ static int uncore_write_control_freq(struct uncore_data *data, unsigned int inpu cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input); } - ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap); + ret = wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap); if (ret) return ret; @@ -207,7 +207,7 @@ static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode, if (!data || !data->valid || !data->stored_uncore_data) return 0; - wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, + wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, data->stored_uncore_data); } break; From ebe29309c4d2821d5fdccd5393eba9c77540e260 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:02 +0200 Subject: [PATCH 102/331] x86/msr: Rename 'mce_rdmsrl()' to 'mce_rdmsrq()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/mce/core.c | 32 +++++++++++++++--------------- arch/x86/kernel/cpu/mce/internal.h | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index c2740248b852..765b799c7090 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -388,7 +388,7 @@ void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) } /* MSR access wrappers used for error injection */ -noinstr u64 mce_rdmsrl(u32 msr) +noinstr u64 mce_rdmsrq(u32 msr) { DECLARE_ARGS(val, low, high); @@ -444,7 +444,7 @@ static noinstr void mce_wrmsrl(u32 msr, u64 v) low = (u32)v; high = (u32)(v >> 32); - /* See comment in mce_rdmsrl() */ + /* See comment in mce_rdmsrq() */ asm volatile("1: wrmsr\n" "2:\n" _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE) @@ -468,7 +468,7 @@ static noinstr void mce_gather_info(struct mce_hw_err *err, struct pt_regs *regs instrumentation_end(); m = &err->m; - m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); + m->mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS); if (regs) { /* * Get the address of the instruction at the time of @@ -488,7 +488,7 @@ static noinstr void mce_gather_info(struct mce_hw_err *err, struct pt_regs *regs } /* Use accurate RIP reporting if available. */ if (mca_cfg.rip_msr) - m->ip = mce_rdmsrl(mca_cfg.rip_msr); + m->ip = mce_rdmsrq(mca_cfg.rip_msr); } } @@ -684,10 +684,10 @@ static noinstr void mce_read_aux(struct mce_hw_err *err, int i) struct mce *m = &err->m; if (m->status & MCI_STATUS_MISCV) - m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC)); + m->misc = mce_rdmsrq(mca_msr_reg(i, MCA_MISC)); if (m->status & MCI_STATUS_ADDRV) { - m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR)); + m->addr = mce_rdmsrq(mca_msr_reg(i, MCA_ADDR)); /* * Mask the reported address by the reported granularity. @@ -702,12 +702,12 @@ static noinstr void mce_read_aux(struct mce_hw_err *err, int i) } if (mce_flags.smca) { - m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i)); + m->ipid = mce_rdmsrq(MSR_AMD64_SMCA_MCx_IPID(i)); if (m->status & MCI_STATUS_SYNDV) { - m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i)); - err->vendor.amd.synd1 = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND1(i)); - err->vendor.amd.synd2 = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND2(i)); + m->synd = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND(i)); + err->vendor.amd.synd1 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND1(i)); + err->vendor.amd.synd2 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND2(i)); } } } @@ -753,7 +753,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b) m->bank = i; barrier(); - m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); + m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); /* * Update storm tracking here, before checking for the @@ -887,8 +887,8 @@ quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) */ static noinstr bool quirk_skylake_repmov(void) { - u64 mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); - u64 misc_enable = mce_rdmsrl(MSR_IA32_MISC_ENABLE); + u64 mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS); + u64 misc_enable = mce_rdmsrq(MSR_IA32_MISC_ENABLE); u64 mc1_status; /* @@ -899,7 +899,7 @@ static noinstr bool quirk_skylake_repmov(void) !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) return false; - mc1_status = mce_rdmsrl(MSR_IA32_MCx_STATUS(1)); + mc1_status = mce_rdmsrq(MSR_IA32_MCx_STATUS(1)); /* Check for a software-recoverable data fetch error. */ if ((mc1_status & @@ -955,7 +955,7 @@ static __always_inline int mce_no_way_out(struct mce_hw_err *err, char **msg, un int i; for (i = 0; i < this_cpu_read(mce_num_banks); i++) { - m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); + m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); if (!(m->status & MCI_STATUS_VAL)) continue; @@ -1335,7 +1335,7 @@ __mc_scan_banks(struct mce_hw_err *err, struct pt_regs *regs, m->addr = 0; m->bank = i; - m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); + m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); if (!(m->status & MCI_STATUS_VAL)) continue; diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index 95a504ece43e..b5ba598e54cb 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -312,7 +312,7 @@ static __always_inline void pentium_machine_check(struct pt_regs *regs) {} static __always_inline void winchip_machine_check(struct pt_regs *regs) {} #endif -noinstr u64 mce_rdmsrl(u32 msr); +noinstr u64 mce_rdmsrq(u32 msr); static __always_inline u32 mca_msr_reg(int bank, enum mca_msr reg) { From 8e44e83f57c3289a41507eb79a315400629978ae Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:03 +0200 Subject: [PATCH 103/331] x86/msr: Rename 'mce_wrmsrl()' to 'mce_wrmsrq()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/mce/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 765b799c7090..255927f0284e 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -423,7 +423,7 @@ noinstr u64 mce_rdmsrq(u32 msr) return EAX_EDX_VAL(val, low, high); } -static noinstr void mce_wrmsrl(u32 msr, u64 v) +static noinstr void mce_wrmsrq(u32 msr, u64 v) { u32 low, high; @@ -829,7 +829,7 @@ clear_it: /* * Clear state for this bank. */ - mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); + mce_wrmsrq(mca_msr_reg(i, MCA_STATUS), 0); } /* @@ -910,8 +910,8 @@ static noinstr bool quirk_skylake_repmov(void) MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_AR | MCI_STATUS_S)) { misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING; - mce_wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable); - mce_wrmsrl(MSR_IA32_MCx_STATUS(1), 0); + mce_wrmsrq(MSR_IA32_MISC_ENABLE, misc_enable); + mce_wrmsrq(MSR_IA32_MCx_STATUS(1), 0); instrumentation_begin(); pr_err_once("Erratum detected, disable fast string copy instructions.\n"); @@ -1274,7 +1274,7 @@ static __always_inline void mce_clear_state(unsigned long *toclear) for (i = 0; i < this_cpu_read(mce_num_banks); i++) { if (arch_test_bit(i, toclear)) - mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); + mce_wrmsrq(mca_msr_reg(i, MCA_STATUS), 0); } } @@ -1693,7 +1693,7 @@ out: instrumentation_end(); clear: - mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); + mce_wrmsrq(MSR_IA32_MCG_STATUS, 0); } EXPORT_SYMBOL_GPL(do_machine_check); From e2b8af0c693993e80415997f4842a372359b463e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:04 +0200 Subject: [PATCH 104/331] x86/msr: Rename 'rdmsrl_amd_safe()' to 'rdmsrq_amd_safe()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/amd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 8937923c9f81..0aefa41f1209 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -31,7 +31,7 @@ u16 invlpgb_count_max __ro_after_init; -static inline int rdmsrl_amd_safe(unsigned msr, u64 *p) +static inline int rdmsrq_amd_safe(unsigned msr, u64 *p) { u32 gprs[8] = { 0 }; int err; @@ -636,7 +636,7 @@ static void init_amd_k8(struct cpuinfo_x86 *c) */ if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM) && !cpu_has(c, X86_FEATURE_HYPERVISOR)) { clear_cpu_cap(c, X86_FEATURE_LAHF_LM); - if (!rdmsrl_amd_safe(0xc001100d, &value)) { + if (!rdmsrq_amd_safe(0xc001100d, &value)) { value &= ~BIT_64(32); wrmsrl_amd_safe(0xc001100d, value); } From 604d15d15ebd6148a084ea53d0fa493a74e51b11 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:05 +0200 Subject: [PATCH 105/331] x86/msr: Rename 'wrmsrl_amd_safe()' to 'wrmsrq_amd_safe()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/amd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 0aefa41f1209..0bbe79862aa6 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -49,7 +49,7 @@ static inline int rdmsrq_amd_safe(unsigned msr, u64 *p) return err; } -static inline int wrmsrl_amd_safe(unsigned msr, u64 val) +static inline int wrmsrq_amd_safe(unsigned msr, u64 val) { u32 gprs[8] = { 0 }; @@ -638,7 +638,7 @@ static void init_amd_k8(struct cpuinfo_x86 *c) clear_cpu_cap(c, X86_FEATURE_LAHF_LM); if (!rdmsrq_amd_safe(0xc001100d, &value)) { value &= ~BIT_64(32); - wrmsrl_amd_safe(0xc001100d, value); + wrmsrq_amd_safe(0xc001100d, value); } } From 7cbc2ba7c107a1a537524ae505e192f4f88cc209 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:06 +0200 Subject: [PATCH 106/331] x86/msr: Rename 'native_wrmsrl()' to 'native_wrmsrq()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/hyperv/ivm.c | 2 +- arch/x86/include/asm/microcode.h | 2 +- arch/x86/include/asm/msr.h | 2 +- arch/x86/include/asm/spec-ctrl.h | 2 +- arch/x86/kernel/cpu/microcode/amd.c | 2 +- arch/x86/kernel/cpu/microcode/intel.c | 2 +- arch/x86/kvm/vmx/vmx.c | 8 ++++---- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 77bf05f06b9e..1b8a2415183b 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -115,7 +115,7 @@ static inline u64 rd_ghcb_msr(void) static inline void wr_ghcb_msr(u64 val) { - native_wrmsrl(MSR_AMD64_SEV_ES_GHCB, val); + native_wrmsrq(MSR_AMD64_SEV_ES_GHCB, val); } static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code, diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 695e569159c1..263ea3dd0001 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -61,7 +61,7 @@ static inline u32 intel_get_microcode_revision(void) { u32 rev, dummy; - native_wrmsrl(MSR_IA32_UCODE_REV, 0); + native_wrmsrq(MSR_IA32_UCODE_REV, 0); /* As documented in the SDM: Do a CPUID 1 here */ native_cpuid_eax(1); diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 4335f914d6f8..20deb58308e5 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -109,7 +109,7 @@ do { \ #define native_wrmsr(msr, low, high) \ __wrmsr(msr, low, high) -#define native_wrmsrl(msr, val) \ +#define native_wrmsrq(msr, val) \ __wrmsr((msr), (u32)((u64)(val)), \ (u32)((u64)(val) >> 32)) diff --git a/arch/x86/include/asm/spec-ctrl.h b/arch/x86/include/asm/spec-ctrl.h index 658b690b2ccb..00b7e0398210 100644 --- a/arch/x86/include/asm/spec-ctrl.h +++ b/arch/x86/include/asm/spec-ctrl.h @@ -84,7 +84,7 @@ static inline u64 ssbd_tif_to_amd_ls_cfg(u64 tifn) static __always_inline void __update_spec_ctrl(u64 val) { __this_cpu_write(x86_spec_ctrl_current, val); - native_wrmsrl(MSR_IA32_SPEC_CTRL, val); + native_wrmsrq(MSR_IA32_SPEC_CTRL, val); } #ifdef CONFIG_SMP diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index b61028cf5c8a..041dae089a36 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -602,7 +602,7 @@ static bool __apply_microcode_amd(struct microcode_amd *mc, u32 *cur_rev, if (!verify_sha256_digest(mc->hdr.patch_id, *cur_rev, (const u8 *)p_addr, psize)) return false; - native_wrmsrl(MSR_AMD64_PATCH_LOADER, p_addr); + native_wrmsrq(MSR_AMD64_PATCH_LOADER, p_addr); if (x86_family(bsp_cpuid_1_eax) == 0x17) { unsigned long p_addr_end = p_addr + psize - 1; diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 819199bc0119..86e1047f738f 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -320,7 +320,7 @@ static enum ucode_state __apply_microcode(struct ucode_cpu_info *uci, } /* write microcode via MSR 0x79 */ - native_wrmsrl(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits); + native_wrmsrq(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits); rev = intel_get_microcode_revision(); if (rev != mc->hdr.rev) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9b221bdd3c51..cd0d6c1fcf9c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -382,7 +382,7 @@ static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) msr = __rdmsr(MSR_IA32_MCU_OPT_CTRL); msr |= FB_CLEAR_DIS; - native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr); + native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, msr); /* Cache the MSR value to avoid reading it later */ vmx->msr_ia32_mcu_opt_ctrl = msr; } @@ -393,7 +393,7 @@ static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) return; vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; - native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); + native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); } static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) @@ -6745,7 +6745,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) vcpu->stat.l1d_flush++; if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { - native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); + native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); return; } @@ -7318,7 +7318,7 @@ void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, */ if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) || vmx->spec_ctrl != hostval) - native_wrmsrl(MSR_IA32_SPEC_CTRL, hostval); + native_wrmsrq(MSR_IA32_SPEC_CTRL, hostval); barrier_nospec(); } From eef476f15c8350078efb48bd9b9f3ff50ae1bbbb Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 22:29:07 +0200 Subject: [PATCH 107/331] x86/msr: Rename 'wrmsrl_cstar()' to 'wrmsrq_cstar()' Suggested-by: "H. Peter Anvin" Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds --- arch/x86/kernel/cpu/common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index bb986ba8b90c..079ded4eeb86 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2091,7 +2091,7 @@ DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_ DEFINE_PER_CPU_CACHE_HOT(u64, __x86_call_depth); EXPORT_PER_CPU_SYMBOL(__x86_call_depth); -static void wrmsrl_cstar(unsigned long val) +static void wrmsrq_cstar(unsigned long val) { /* * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR @@ -2107,7 +2107,7 @@ static inline void idt_syscall_init(void) wrmsrq(MSR_LSTAR, (unsigned long)entry_SYSCALL_64); if (ia32_enabled()) { - wrmsrl_cstar((unsigned long)entry_SYSCALL_compat); + wrmsrq_cstar((unsigned long)entry_SYSCALL_compat); /* * This only works on Intel CPUs. * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP. @@ -2119,7 +2119,7 @@ static inline void idt_syscall_init(void) (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1)); wrmsrq_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat); } else { - wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore); + wrmsrq_cstar((unsigned long)entry_SYSCALL32_ignore); wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG); wrmsrq_safe(MSR_IA32_SYSENTER_ESP, 0ULL); wrmsrq_safe(MSR_IA32_SYSENTER_EIP, 0ULL); From d358b45120cc8da9f10d8c1e8ec3559f72525147 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Mar 2025 14:16:01 +0000 Subject: [PATCH 108/331] x86/kexec: Add 8250 serial port output If a serial port was configured for early_printk, use it for debug output from the relocate_kernel exception handler too. Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Andy Shevchenko Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20250326142404.256980-2-dwmw2@infradead.org --- arch/x86/include/asm/kexec.h | 1 + arch/x86/kernel/early_printk.c | 6 +++++ arch/x86/kernel/relocate_kernel_64.S | 39 +++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index fb4537c58b54..9601094ac0ba 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -64,6 +64,7 @@ extern unsigned long kexec_pa_table_page; extern unsigned long kexec_pa_swap_page; extern gate_desc kexec_debug_idt[]; extern unsigned char kexec_debug_exc_vectors[]; +extern uint16_t kexec_debug_8250_port; #endif /* diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c index 611f27e3890c..acb7d05ab6af 100644 --- a/arch/x86/kernel/early_printk.c +++ b/arch/x86/kernel/early_printk.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -144,6 +145,11 @@ static __init void early_serial_hw_init(unsigned divisor) static_call(serial_out)(early_serial_base, DLL, divisor & 0xff); static_call(serial_out)(early_serial_base, DLH, (divisor >> 8) & 0xff); static_call(serial_out)(early_serial_base, LCR, c & ~DLAB); + +#if defined(CONFIG_KEXEC_CORE) && defined(CONFIG_X86_64) + if (static_call_query(serial_in) == io_serial_in) + kexec_debug_8250_port = early_serial_base; +#endif } #define DEFAULT_BAUD 9600 diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 29cb399a34d4..21011cd19c48 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -39,6 +39,7 @@ SYM_DATA(kexec_va_control_page, .quad 0) SYM_DATA(kexec_pa_table_page, .quad 0) SYM_DATA(kexec_pa_swap_page, .quad 0) SYM_DATA_LOCAL(pa_backup_pages_map, .quad 0) +SYM_DATA(kexec_debug_8250_port, .word 0) .balign 16 SYM_DATA_START_LOCAL(kexec_debug_gdt) @@ -380,24 +381,50 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) SYM_CODE_END(swap_pages) /* - * Generic 'print character' routine (as yet unimplemented) + * Generic 'print character' routine * - %al: Character to be printed (may clobber %rax) * - %rdx: MMIO address or port. */ -SYM_CODE_START_LOCAL_NOALIGN(pr_char) +#define XMTRDY 0x20 + +#define TXR 0 /* Transmit register (WRITE) */ +#define LSR 5 /* Line Status */ + +SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250) UNWIND_HINT_FUNC ANNOTATE_NOENDBR + addw $LSR, %dx + xchg %al, %ah +.Lxmtrdy_loop: + inb %dx, %al + testb $XMTRDY, %al + jnz .Lready + rep nop + jmp .Lxmtrdy_loop + +.Lready: + subw $LSR, %dx + xchg %al, %ah + outb %al, %dx +pr_char_null: + ANNOTATE_NOENDBR + ANNOTATE_UNRET_SAFE ret -SYM_CODE_END(pr_char) +SYM_CODE_END(pr_char_8250) /* * Load pr_char function pointer into %rsi and load %rdx with whatever * that function wants to see there (typically port/MMIO address). */ -.macro pr_setup - /* No output; pr_char just returns */ - leaq pr_char(%rip), %rsi +.macro pr_setup + leaq pr_char_8250(%rip), %rsi + movw kexec_debug_8250_port(%rip), %dx + testw %dx, %dx + jnz 1f + + leaq pr_char_null(%rip), %rsi +1: .endm /* Print the nybble in %bl, clobber %rax */ From 7516e7216bdfb9e2fab0a0ca3bd23cb2e61e46ed Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Mar 2025 14:16:02 +0000 Subject: [PATCH 109/331] x86/kexec: Add 8250 MMIO serial port output This supports the same 32-bit MMIO-mapped 8250 as the early_printk code. It's not clear why the early_printk code supports this form and only this form; the actual runtime 8250_pci doesn't seem to support it. But having hacked up QEMU to expose such a device, early_printk does work with it, and now so does the kexec debug code. Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Andy Shevchenko Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20250326142404.256980-3-dwmw2@infradead.org --- arch/x86/include/asm/kexec.h | 1 + arch/x86/kernel/early_printk.c | 3 +++ arch/x86/kernel/machine_kexec_64.c | 17 +++++++++++++++++ arch/x86/kernel/relocate_kernel_64.S | 22 ++++++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 9601094ac0ba..f2ad77929d6e 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -65,6 +65,7 @@ extern unsigned long kexec_pa_swap_page; extern gate_desc kexec_debug_idt[]; extern unsigned char kexec_debug_exc_vectors[]; extern uint16_t kexec_debug_8250_port; +extern unsigned long kexec_debug_8250_mmio32; #endif /* diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c index acb7d05ab6af..9e8707d54f9c 100644 --- a/arch/x86/kernel/early_printk.c +++ b/arch/x86/kernel/early_printk.c @@ -333,6 +333,9 @@ static __init void early_pci_serial_init(char *s) /* WARNING! assuming the address is always in the first 4G */ early_serial_base = (unsigned long)early_ioremap(bar0 & PCI_BASE_ADDRESS_MEM_MASK, 0x10); +#if defined(CONFIG_KEXEC_CORE) && defined(CONFIG_X86_64) + kexec_debug_8250_mmio32 = bar0 & PCI_BASE_ADDRESS_MEM_MASK; +#endif write_pci_config(bus, slot, func, PCI_COMMAND, cmdreg|PCI_COMMAND_MEMORY); } diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index cc73f9708464..ecb0da54abd1 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -76,6 +76,19 @@ map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; } #endif +static int map_mmio_serial(struct x86_mapping_info *info, pgd_t *level4p) +{ + unsigned long mstart, mend; + + if (!kexec_debug_8250_mmio32) + return 0; + + mstart = kexec_debug_8250_mmio32 & PAGE_MASK; + mend = (kexec_debug_8250_mmio32 + PAGE_SIZE + 23) & PAGE_MASK; + pr_info("Map PCI serial at %lx - %lx\n", mstart, mend); + return kernel_ident_mapping_init(info, level4p, mstart, mend); +} + #ifdef CONFIG_KEXEC_FILE const struct kexec_file_ops * const kexec_file_loaders[] = { &kexec_bzImage64_ops, @@ -285,6 +298,10 @@ static int init_pgtable(struct kimage *image, unsigned long control_page) if (result) return result; + result = map_mmio_serial(&info, image->arch.pgd); + if (result) + return result; + /* * This must be last because the intermediate page table pages it * allocates will not be control pages and may overlap the image. diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 21011cd19c48..8808cfca6322 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -39,6 +39,7 @@ SYM_DATA(kexec_va_control_page, .quad 0) SYM_DATA(kexec_pa_table_page, .quad 0) SYM_DATA(kexec_pa_swap_page, .quad 0) SYM_DATA_LOCAL(pa_backup_pages_map, .quad 0) +SYM_DATA(kexec_debug_8250_mmio32, .quad 0) SYM_DATA(kexec_debug_8250_port, .word 0) .balign 16 @@ -413,6 +414,22 @@ pr_char_null: ret SYM_CODE_END(pr_char_8250) +SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250_mmio32) + UNWIND_HINT_FUNC + ANNOTATE_NOENDBR +.Lxmtrdy_loop_mmio: + movb (LSR*4)(%rdx), %ah + testb $XMTRDY, %ah + jnz .Lready_mmio + rep nop + jmp .Lxmtrdy_loop_mmio + +.Lready_mmio: + movb %al, (%rdx) + ANNOTATE_UNRET_SAFE + ret +SYM_CODE_END(pr_char_8250_mmio32) + /* * Load pr_char function pointer into %rsi and load %rdx with whatever * that function wants to see there (typically port/MMIO address). @@ -423,6 +440,11 @@ SYM_CODE_END(pr_char_8250) testw %dx, %dx jnz 1f + leaq pr_char_8250_mmio32(%rip), %rsi + movq kexec_debug_8250_mmio32(%rip), %rdx + testq %rdx, %rdx + jnz 1f + leaq pr_char_null(%rip), %rsi 1: .endm From de085ddd493bccb77a3ec1b99ae7466133540f4d Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Mar 2025 14:16:03 +0000 Subject: [PATCH 110/331] x86/kexec: Invalidate GDT/IDT from relocate_kernel() instead of earlier Reduce the window during which exceptions are unhandled, by leaving the GDT/IDT in place all the way into the relocate_kernel() function, until the moment that %cr3 gets replaced. Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20250326142404.256980-4-dwmw2@infradead.org --- arch/x86/kernel/machine_kexec_64.c | 10 ++-------- arch/x86/kernel/relocate_kernel_64.S | 9 +++++++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index ecb0da54abd1..949c9e4bfad2 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -434,16 +434,10 @@ void __nocfi machine_kexec(struct kimage *image) * with from a table in memory. At no other time is the * descriptor table in memory accessed. * - * I take advantage of this here by force loading the - * segments, before I zap the gdt with an invalid value. + * Take advantage of this here by force loading the segments, + * before the GDT is zapped with an invalid value. */ load_segments(); - /* - * The gdt & idt are now invalid. - * If you want to load them you must set up your own idt & gdt. - */ - native_idt_invalidate(); - native_gdt_invalidate(); /* now call it */ image->start = relocate_kernel_ptr((unsigned long)image->head, diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 8808cfca6322..3062cb3efc44 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -79,8 +79,13 @@ SYM_CODE_START_NOALIGN(relocate_kernel) pushq %r15 pushf - /* zero out flags, and disable interrupts */ - pushq $0 + /* Invalidate GDT/IDT, zero out flags */ + pushq $0 + pushq $0 + + lidt (%rsp) + lgdt (%rsp) + addq $8, %rsp popfq /* Switch to the identity mapped page tables */ From 7615b94b6371dab5d7d8c1ca4f95ed3f866e1c51 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Mar 2025 14:16:04 +0000 Subject: [PATCH 111/331] selftests/kexec: Add x86_64 selftest for kexec-jump and exception handling Add a self test which exercises both the kexec-jump facility, and the kexec exception handling. Invoke a trivial payload which just does an int3 and returns, flip-flopping its entry point for the next invocation between two implementations of the same thing. Signed-off-by: David Woodhouse Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Kees Cook Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20250326142404.256980-5-dwmw2@infradead.org --- tools/testing/selftests/kexec/Makefile | 7 ++ .../testing/selftests/kexec/test_kexec_jump.c | 72 +++++++++++++++++++ .../selftests/kexec/test_kexec_jump.sh | 42 +++++++++++ 3 files changed, 121 insertions(+) create mode 100644 tools/testing/selftests/kexec/test_kexec_jump.c create mode 100755 tools/testing/selftests/kexec/test_kexec_jump.sh diff --git a/tools/testing/selftests/kexec/Makefile b/tools/testing/selftests/kexec/Makefile index 67fe7a46cb62..e3000ccb9a5d 100644 --- a/tools/testing/selftests/kexec/Makefile +++ b/tools/testing/selftests/kexec/Makefile @@ -8,6 +8,13 @@ ifeq ($(ARCH_PROCESSED),$(filter $(ARCH_PROCESSED),x86 ppc64le)) TEST_PROGS := test_kexec_load.sh test_kexec_file_load.sh TEST_FILES := kexec_common_lib.sh +include ../../../scripts/Makefile.arch + +ifeq ($(IS_64_BIT)$(ARCH_PROCESSED),1x86) +TEST_PROGS += test_kexec_jump.sh +test_kexec_jump.sh: $(OUTPUT)/test_kexec_jump +endif + include ../lib.mk endif diff --git a/tools/testing/selftests/kexec/test_kexec_jump.c b/tools/testing/selftests/kexec/test_kexec_jump.c new file mode 100644 index 000000000000..fbce287866f5 --- /dev/null +++ b/tools/testing/selftests/kexec/test_kexec_jump.c @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +asm( + " .code64\n" + " .data\n" + "purgatory_start:\n" + + // Trigger kexec debug exception handling + " int3\n" + + // Set load address for next time + " leaq purgatory_start_b(%rip), %r11\n" + " movq %r11, 8(%rsp)\n" + + // Back to Linux + " ret\n" + + // Same again + "purgatory_start_b:\n" + + // Trigger kexec debug exception handling + " int3\n" + + // Set load address for next time + " leaq purgatory_start(%rip), %r11\n" + " movq %r11, 8(%rsp)\n" + + // Back to Linux + " ret\n" + + "purgatory_end:\n" + ".previous" +); +extern char purgatory_start[], purgatory_end[]; + +int main (void) +{ + struct kexec_segment segment = {}; + int ret; + + segment.buf = purgatory_start; + segment.bufsz = purgatory_end - purgatory_start; + segment.mem = (void *)0x400000; + segment.memsz = 0x1000; + ret = syscall(__NR_kexec_load, 0x400000, 1, &segment, KEXEC_PRESERVE_CONTEXT); + if (ret) { + perror("kexec_load"); + exit(1); + } + + ret = syscall(__NR_reboot, LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, LINUX_REBOOT_CMD_KEXEC); + if (ret) { + perror("kexec reboot"); + exit(1); + } + + ret = syscall(__NR_reboot, LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, LINUX_REBOOT_CMD_KEXEC); + if (ret) { + perror("kexec reboot"); + exit(1); + } + printf("Success\n"); + return 0; +} + diff --git a/tools/testing/selftests/kexec/test_kexec_jump.sh b/tools/testing/selftests/kexec/test_kexec_jump.sh new file mode 100755 index 000000000000..6ae977054ba2 --- /dev/null +++ b/tools/testing/selftests/kexec/test_kexec_jump.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Prevent loading a kernel image via the kexec_load syscall when +# signatures are required. (Dependent on CONFIG_IMA_ARCH_POLICY.) + +TEST="$0" +. ./kexec_common_lib.sh + +# kexec requires root privileges +require_root_privileges + +# get the kernel config +get_kconfig + +kconfig_enabled "CONFIG_KEXEC_JUMP=y" "kexec_jump is enabled" +if [ $? -eq 0 ]; then + log_skip "kexec_jump is not enabled" +fi + +kconfig_enabled "CONFIG_IMA_APPRAISE=y" "IMA enabled" +ima_appraise=$? + +kconfig_enabled "CONFIG_IMA_ARCH_POLICY=y" \ + "IMA architecture specific policy enabled" +arch_policy=$? + +get_secureboot_mode +secureboot=$? + +if [ $secureboot -eq 1 ] && [ $arch_policy -eq 1 ]; then + log_skip "Secure boot and CONFIG_IMA_ARCH_POLICY are enabled" +fi + +./test_kexec_jump +if [ $? -eq 0 ]; then + log_pass "kexec_jump succeeded" +else + # The more likely failure mode if anything went wrong is that the + # kernel just crashes. But if we get back here, sure, whine anyway. + log_fail "kexec_jump failed" +fi From 4334336e769bea1351ab82b22b06118c81bd217f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 11 Apr 2025 07:40:13 +0200 Subject: [PATCH 112/331] x86/alternatives: Improve code-patching scalability by removing false sharing in poke_int3_handler() eBPF programs can be run 50,000,000 times per second on busy servers. Whenever /proc/sys/kernel/bpf_stats_enabled is turned off, hundreds of calls sites are patched from text_poke_bp_batch() and we see a huge loss of performance due to false sharing on bp_desc.refs lasting up to three seconds. 51.30% server_bin [kernel.kallsyms] [k] poke_int3_handler | |--46.45%--poke_int3_handler | exc_int3 | asm_exc_int3 | | | |--24.26%--cls_bpf_classify | | tcf_classify | | __dev_queue_xmit | | ip6_finish_output2 | | ip6_output | | ip6_xmit | | inet6_csk_xmit | | __tcp_transmit_skb Fix this by replacing bp_desc.refs with a per-cpu bp_refs. Before the patch, on a host with 240 cores (480 threads): $ sysctl -wq kernel.bpf_stats_enabled=0 text_poke_bp_batch(nr_entries=164) : Took 2655300 usec $ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 3009063719 run_cnt 82757845 : average cost is 36 nsec per call After this patch: $ sysctl -wq kernel.bpf_stats_enabled=0 text_poke_bp_batch(nr_entries=164) : Took 702 usec $ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 1928223019 run_cnt 67682728 : average cost is 28 nsec per call Ie. text-patching performance improved 3700x: from 2.65 seconds to 0.0007 seconds. Since the atomic_cond_read_acquire(refs, !VAL) spin-loop was not triggered even once in my tests, add an unlikely() annotation, because this appears to be the common case. [ mingo: Improved the changelog some more. ] Signed-off-by: Eric Dumazet Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Brian Gerst Cc: Kees Cook Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250411054105.2341982-2-mingo@kernel.org --- arch/x86/kernel/alternative.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index bf82c6f7d690..85089c79a828 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2474,28 +2474,29 @@ struct text_poke_loc { struct bp_patching_desc { struct text_poke_loc *vec; int nr_entries; - atomic_t refs; }; +static DEFINE_PER_CPU(atomic_t, bp_refs); + static struct bp_patching_desc bp_desc; static __always_inline struct bp_patching_desc *try_get_desc(void) { - struct bp_patching_desc *desc = &bp_desc; + atomic_t *refs = this_cpu_ptr(&bp_refs); - if (!raw_atomic_inc_not_zero(&desc->refs)) + if (!raw_atomic_inc_not_zero(refs)) return NULL; - return desc; + return &bp_desc; } static __always_inline void put_desc(void) { - struct bp_patching_desc *desc = &bp_desc; + atomic_t *refs = this_cpu_ptr(&bp_refs); smp_mb__before_atomic(); - raw_atomic_dec(&desc->refs); + raw_atomic_dec(refs); } static __always_inline void *text_poke_addr(struct text_poke_loc *tp) @@ -2528,9 +2529,9 @@ noinstr int poke_int3_handler(struct pt_regs *regs) * Having observed our INT3 instruction, we now must observe * bp_desc with non-zero refcount: * - * bp_desc.refs = 1 INT3 - * WMB RMB - * write INT3 if (bp_desc.refs != 0) + * bp_refs = 1 INT3 + * WMB RMB + * write INT3 if (bp_refs != 0) */ smp_rmb(); @@ -2636,7 +2637,8 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * Corresponds to the implicit memory barrier in try_get_desc() to * ensure reading a non-zero refcount provides up to date bp_desc data. */ - atomic_set_release(&bp_desc.refs, 1); + for_each_possible_cpu(i) + atomic_set_release(per_cpu_ptr(&bp_refs, i), 1); /* * Function tracing can enable thousands of places that need to be @@ -2750,8 +2752,12 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries /* * Remove and wait for refs to be zero. */ - if (!atomic_dec_and_test(&bp_desc.refs)) - atomic_cond_read_acquire(&bp_desc.refs, !VAL); + for_each_possible_cpu(i) { + atomic_t *refs = per_cpu_ptr(&bp_refs, i); + + if (unlikely(!atomic_dec_and_test(refs))) + atomic_cond_read_acquire(refs, !VAL); + } } static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, From d60e4b2410e1b9f7c5ca347c78c6b07175c2e873 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 11 Apr 2025 07:40:14 +0200 Subject: [PATCH 113/331] x86/alternatives: Document the text_poke_bp_batch() synchronization rules a bit more Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Eric Dumazet Cc: Brian Gerst Cc: Josh Poimboeuf Link: https://lore.kernel.org/r/20250411054105.2341982-3-mingo@kernel.org --- arch/x86/kernel/alternative.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 85089c79a828..5f448142aa99 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2751,6 +2751,13 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries /* * Remove and wait for refs to be zero. + * + * Notably, if after step-3 above the INT3 got removed, then the + * text_poke_sync() will have serialized against any running INT3 + * handlers and the below spin-wait will not happen. + * + * IOW. unless the replacement instruction is INT3, this case goes + * unused. */ for_each_possible_cpu(i) { atomic_t *refs = per_cpu_ptr(&bp_refs, i); From 84e5ba949b0a3fcf2fd0a1b6c9ce14d8436dbbb8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:15 +0200 Subject: [PATCH 114/331] x86/alternatives: Rename 'struct bp_patching_desc' to 'struct text_poke_int3_vec' Follow the INT3 text-poking nomenclature, and also adopt the 'vector' name for the entire object, instead of the rather opaque 'descriptor' naming. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-4-mingo@kernel.org --- arch/x86/kernel/alternative.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 5f448142aa99..8edf7d3fd184 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2471,17 +2471,17 @@ struct text_poke_loc { u8 old; }; -struct bp_patching_desc { +struct text_poke_int3_vec { struct text_poke_loc *vec; int nr_entries; }; static DEFINE_PER_CPU(atomic_t, bp_refs); -static struct bp_patching_desc bp_desc; +static struct text_poke_int3_vec bp_desc; static __always_inline -struct bp_patching_desc *try_get_desc(void) +struct text_poke_int3_vec *try_get_desc(void) { atomic_t *refs = this_cpu_ptr(&bp_refs); @@ -2517,7 +2517,7 @@ static __always_inline int patch_cmp(const void *key, const void *elt) noinstr int poke_int3_handler(struct pt_regs *regs) { - struct bp_patching_desc *desc; + struct text_poke_int3_vec *desc; struct text_poke_loc *tp; int ret = 0; void *ip; From 28fb79092d9f7db3397e886d637d3006551693b3 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:16 +0200 Subject: [PATCH 115/331] x86/alternatives: Rename 'bp_refs' to 'text_poke_array_refs' Make it clear that these reference counts lock access to text_poke_array. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-5-mingo@kernel.org --- arch/x86/kernel/alternative.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 8edf7d3fd184..9bd71c017cfd 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2476,14 +2476,14 @@ struct text_poke_int3_vec { int nr_entries; }; -static DEFINE_PER_CPU(atomic_t, bp_refs); +static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); static struct text_poke_int3_vec bp_desc; static __always_inline struct text_poke_int3_vec *try_get_desc(void) { - atomic_t *refs = this_cpu_ptr(&bp_refs); + atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); if (!raw_atomic_inc_not_zero(refs)) return NULL; @@ -2493,7 +2493,7 @@ struct text_poke_int3_vec *try_get_desc(void) static __always_inline void put_desc(void) { - atomic_t *refs = this_cpu_ptr(&bp_refs); + atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); smp_mb__before_atomic(); raw_atomic_dec(refs); @@ -2529,9 +2529,9 @@ noinstr int poke_int3_handler(struct pt_regs *regs) * Having observed our INT3 instruction, we now must observe * bp_desc with non-zero refcount: * - * bp_refs = 1 INT3 + * text_poke_array_refs = 1 INT3 * WMB RMB - * write INT3 if (bp_refs != 0) + * write INT3 if (text_poke_array_refs != 0) */ smp_rmb(); @@ -2638,7 +2638,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * ensure reading a non-zero refcount provides up to date bp_desc data. */ for_each_possible_cpu(i) - atomic_set_release(per_cpu_ptr(&bp_refs, i), 1); + atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); /* * Function tracing can enable thousands of places that need to be @@ -2760,7 +2760,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries * unused. */ for_each_possible_cpu(i) { - atomic_t *refs = per_cpu_ptr(&bp_refs, i); + atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i); if (unlikely(!atomic_dec_and_test(refs))) atomic_cond_read_acquire(refs, !VAL); From bee4fcfbc128c3ad604539f88307dc2c0fc6f843 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:17 +0200 Subject: [PATCH 116/331] x86/alternatives: Rename 'text_poke_bp_batch()' to 'smp_text_poke_batch_process()' Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-6-mingo@kernel.org --- arch/x86/kernel/alternative.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 9bd71c017cfd..78024e5def29 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2467,7 +2467,7 @@ struct text_poke_loc { u8 len; u8 opcode; const u8 text[POKE_MAX_OPCODE_SIZE]; - /* see text_poke_bp_batch() */ + /* see smp_text_poke_batch_process() */ u8 old; }; @@ -2540,7 +2540,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) return 0; /* - * Discount the INT3. See text_poke_bp_batch(). + * Discount the INT3. See smp_text_poke_batch_process(). */ ip = (void *) regs->ip - INT3_INSN_SIZE; @@ -2602,7 +2602,7 @@ static struct text_poke_loc tp_vec[TP_VEC_MAX]; static int tp_vec_nr; /** - * text_poke_bp_batch() -- update instructions on live kernel on SMP + * smp_text_poke_batch_process() -- update instructions on live kernel on SMP * @tp: vector of instructions to patch * @nr_entries: number of entries in the vector * @@ -2622,7 +2622,7 @@ static int tp_vec_nr; * replacing opcode * - sync cores */ -static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries) +static void smp_text_poke_batch_process(struct text_poke_loc *tp, unsigned int nr_entries) { unsigned char int3 = INT3_INSN_OPCODE; unsigned int i; @@ -2866,7 +2866,7 @@ static bool tp_order_fail(void *addr) static void text_poke_flush(void *addr) { if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) { - text_poke_bp_batch(tp_vec, tp_vec_nr); + smp_text_poke_batch_process(tp_vec, tp_vec_nr); tp_vec_nr = 0; } } @@ -2902,5 +2902,5 @@ void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void * struct text_poke_loc tp; text_poke_loc_init(&tp, addr, opcode, len, emulate); - text_poke_bp_batch(&tp, 1); + smp_text_poke_batch_process(&tp, 1); } From 9586ae48e785b48b8dd25136c34c8d0e3e1d2cc8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:18 +0200 Subject: [PATCH 117/331] x86/alternatives: Rename 'text_poke_bp()' to 'smp_text_poke_single()' Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-7-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 4 ++-- arch/x86/kernel/ftrace.c | 8 ++++---- arch/x86/kernel/jump_label.c | 2 +- arch/x86/kernel/kprobes/opt.c | 2 +- arch/x86/kernel/static_call.c | 2 +- arch/x86/net/bpf_jit_comp.c | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index ab9e143ec9fe..5189188b5e49 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -39,7 +39,7 @@ extern void *text_poke_copy(void *addr, const void *opcode, size_t len); extern void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, bool core_ok); extern void *text_poke_set(void *addr, int c, size_t len); extern int poke_int3_handler(struct pt_regs *regs); -extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate); +extern void smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate); extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate); extern void text_poke_finish(void); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 78024e5def29..222021af7906 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2887,7 +2887,7 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi } /** - * text_poke_bp() -- update instructions on live kernel on SMP + * smp_text_poke_single() -- update instructions on live kernel on SMP * @addr: address to patch * @opcode: opcode of new instruction * @len: length to copy @@ -2897,7 +2897,7 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi * dynamically allocated memory. This function should be used when it is * not possible to allocate memory. */ -void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate) +void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { struct text_poke_loc tp; diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index cace6e8d7cc7..7175a0404def 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -186,11 +186,11 @@ int ftrace_update_ftrace_func(ftrace_func_t func) ip = (unsigned long)(&ftrace_call); new = ftrace_call_replace(ip, (unsigned long)func); - text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_single((void *)ip, new, MCOUNT_INSN_SIZE, NULL); ip = (unsigned long)(&ftrace_regs_call); new = ftrace_call_replace(ip, (unsigned long)func); - text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_single((void *)ip, new, MCOUNT_INSN_SIZE, NULL); return 0; } @@ -492,7 +492,7 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops) mutex_lock(&text_mutex); /* Do a safe modify in case the trampoline is executing */ new = ftrace_call_replace(ip, (unsigned long)func); - text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_single((void *)ip, new, MCOUNT_INSN_SIZE, NULL); mutex_unlock(&text_mutex); } @@ -586,7 +586,7 @@ static int ftrace_mod_jmp(unsigned long ip, void *func) const char *new; new = ftrace_jmp_replace(ip, (unsigned long)func); - text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_single((void *)ip, new, MCOUNT_INSN_SIZE, NULL); return 0; } diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index f5b8ef02d172..166e12037199 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -102,7 +102,7 @@ __jump_label_transform(struct jump_entry *entry, return; } - text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL); + smp_text_poke_single((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL); } static void __ref jump_label_transform(struct jump_entry *entry, diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 36d6809c6c9e..9307a40f4983 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -488,7 +488,7 @@ void arch_optimize_kprobes(struct list_head *oplist) insn_buff[0] = JMP32_INSN_OPCODE; *(s32 *)(&insn_buff[1]) = rel; - text_poke_bp(op->kp.addr, insn_buff, JMP32_INSN_SIZE, NULL); + smp_text_poke_single(op->kp.addr, insn_buff, JMP32_INSN_SIZE, NULL); list_del_init(&op->list); } diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index a59c72e77645..8164a7323c17 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -108,7 +108,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, if (system_state == SYSTEM_BOOTING || modinit) return text_poke_early(insn, code, size); - text_poke_bp(insn, code, size, emulate); + smp_text_poke_single(insn, code, size, emulate); } static void __static_call_validate(u8 *insn, bool tail, bool tramp) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 9e5fe2ba858f..e2b9991c3326 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -629,7 +629,7 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, goto out; ret = 1; if (memcmp(ip, new_insn, X86_PATCH_SIZE)) { - text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL); + smp_text_poke_single(ip, new_insn, X86_PATCH_SIZE, NULL); ret = 0; } out: From 5236b6a0fe921f5de53b8eeea2d8fdd6d643dd7f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:19 +0200 Subject: [PATCH 118/331] x86/alternatives: Rename 'poke_int3_handler()' to 'smp_text_poke_int3_handler()' All related functions in this subsystem already have a text_poke_int3_ prefix - add it to the trap handler as well. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-8-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/traps.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 5189188b5e49..93a6b7bc78bd 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -38,7 +38,7 @@ extern void *text_poke_copy(void *addr, const void *opcode, size_t len); #define text_poke_copy text_poke_copy extern void *text_poke_copy_locked(void *addr, const void *opcode, size_t len, bool core_ok); extern void *text_poke_set(void *addr, int c, size_t len); -extern int poke_int3_handler(struct pt_regs *regs); +extern int smp_text_poke_int3_handler(struct pt_regs *regs); extern void smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate); extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 222021af7906..d2cd0d815130 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2515,7 +2515,7 @@ static __always_inline int patch_cmp(const void *key, const void *elt) return 0; } -noinstr int poke_int3_handler(struct pt_regs *regs) +noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { struct text_poke_int3_vec *desc; struct text_poke_loc *tp; diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 9f88b8a78e50..d67407c623f3 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -882,16 +882,16 @@ static void do_int3_user(struct pt_regs *regs) DEFINE_IDTENTRY_RAW(exc_int3) { /* - * poke_int3_handler() is completely self contained code; it does (and + * smp_text_poke_int3_handler() is completely self contained code; it does (and * must) *NOT* call out to anything, lest it hits upon yet another * INT3. */ - if (poke_int3_handler(regs)) + if (smp_text_poke_int3_handler(regs)) return; /* * irqentry_enter_from_user_mode() uses static_branch_{,un}likely() - * and therefore can trigger INT3, hence poke_int3_handler() must + * and therefore can trigger INT3, hence smp_text_poke_int3_handler() must * be done before. If the entry came from kernel mode, then use * nmi_enter() because the INT3 could have been hit in any context * including NMI. From a5c832e0476e461af46a0aa9bda43a573adbe63f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:20 +0200 Subject: [PATCH 119/331] x86/alternatives: Rename 'poking_mm' to 'text_poke_mm' Put it into the text_poke_* namespace of . Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-9-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 18 +++++++++--------- arch/x86/mm/init.c | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 93a6b7bc78bd..7a95c0820b3e 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -128,7 +128,7 @@ void *text_gen_insn(u8 opcode, const void *addr, const void *dest) } extern int after_bootmem; -extern __ro_after_init struct mm_struct *poking_mm; +extern __ro_after_init struct mm_struct *text_poke_mm; extern __ro_after_init unsigned long poking_addr; #ifndef CONFIG_UML_X86 diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index d2cd0d815130..8ce0d469e32f 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2191,7 +2191,7 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) return temp_state; } -__ro_after_init struct mm_struct *poking_mm; +__ro_after_init struct mm_struct *text_poke_mm; __ro_after_init unsigned long poking_addr; static inline void unuse_temporary_mm(temp_mm_state_t prev_state) @@ -2201,7 +2201,7 @@ static inline void unuse_temporary_mm(temp_mm_state_t prev_state) switch_mm_irqs_off(NULL, prev_state.mm, current); /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ - cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(poking_mm)); + cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(text_poke_mm)); /* * Restore the breakpoints if they were disabled before the temporary mm @@ -2266,7 +2266,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l /* * The lock is not really needed, but this allows to avoid open-coding. */ - ptep = get_locked_pte(poking_mm, poking_addr, &ptl); + ptep = get_locked_pte(text_poke_mm, poking_addr, &ptl); /* * This must not fail; preallocated in poking_init(). @@ -2276,18 +2276,18 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l local_irq_save(flags); pte = mk_pte(pages[0], pgprot); - set_pte_at(poking_mm, poking_addr, ptep, pte); + set_pte_at(text_poke_mm, poking_addr, ptep, pte); if (cross_page_boundary) { pte = mk_pte(pages[1], pgprot); - set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); + set_pte_at(text_poke_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); } /* * Loading the temporary mm behaves as a compiler barrier, which * guarantees that the PTE will be set at the time memcpy() is done. */ - prev = use_temporary_mm(poking_mm); + prev = use_temporary_mm(text_poke_mm); kasan_disable_current(); func((u8 *)poking_addr + offset_in_page(addr), src, len); @@ -2299,9 +2299,9 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l */ barrier(); - pte_clear(poking_mm, poking_addr, ptep); + pte_clear(text_poke_mm, poking_addr, ptep); if (cross_page_boundary) - pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1); + pte_clear(text_poke_mm, poking_addr + PAGE_SIZE, ptep + 1); /* * Loading the previous page-table hierarchy requires a serializing @@ -2314,7 +2314,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * Flushing the TLB might involve IPIs, which would require enabled * IRQs, but not if the mm is not used, as it is in this point. */ - flush_tlb_mm_range(poking_mm, poking_addr, poking_addr + + flush_tlb_mm_range(text_poke_mm, poking_addr, poking_addr + (cross_page_boundary ? 2 : 1) * PAGE_SIZE, PAGE_SHIFT, false); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index bfa444a7dbb0..84b52a1ebd48 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -824,11 +824,11 @@ void __init poking_init(void) spinlock_t *ptl; pte_t *ptep; - poking_mm = mm_alloc(); - BUG_ON(!poking_mm); + text_poke_mm = mm_alloc(); + BUG_ON(!text_poke_mm); /* Xen PV guests need the PGD to be pinned. */ - paravirt_enter_mmap(poking_mm); + paravirt_enter_mmap(text_poke_mm); /* * Randomize the poking address, but make sure that the following page @@ -848,7 +848,7 @@ void __init poking_init(void) * needed for poking now. Later, poking may be performed in an atomic * section, which might cause allocation to fail. */ - ptep = get_locked_pte(poking_mm, poking_addr, &ptl); + ptep = get_locked_pte(text_poke_mm, poking_addr, &ptl); BUG_ON(!ptep); pte_unmap_unlock(ptep, ptl); } From da364fc547897ed98fbf2192d86b5242439d7762 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:21 +0200 Subject: [PATCH 120/331] x86/alternatives: Rename 'poking_addr' to 'text_poke_mm_addr' Put it into the text_poke_* namespace of . Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-10-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 16 ++++++++-------- arch/x86/mm/init.c | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 7a95c0820b3e..c8eac8cf4737 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -129,7 +129,7 @@ void *text_gen_insn(u8 opcode, const void *addr, const void *dest) extern int after_bootmem; extern __ro_after_init struct mm_struct *text_poke_mm; -extern __ro_after_init unsigned long poking_addr; +extern __ro_after_init unsigned long text_poke_mm_addr; #ifndef CONFIG_UML_X86 static __always_inline diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 8ce0d469e32f..62d74442cbd1 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2192,7 +2192,7 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) } __ro_after_init struct mm_struct *text_poke_mm; -__ro_after_init unsigned long poking_addr; +__ro_after_init unsigned long text_poke_mm_addr; static inline void unuse_temporary_mm(temp_mm_state_t prev_state) { @@ -2266,7 +2266,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l /* * The lock is not really needed, but this allows to avoid open-coding. */ - ptep = get_locked_pte(text_poke_mm, poking_addr, &ptl); + ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl); /* * This must not fail; preallocated in poking_init(). @@ -2276,11 +2276,11 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l local_irq_save(flags); pte = mk_pte(pages[0], pgprot); - set_pte_at(text_poke_mm, poking_addr, ptep, pte); + set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte); if (cross_page_boundary) { pte = mk_pte(pages[1], pgprot); - set_pte_at(text_poke_mm, poking_addr + PAGE_SIZE, ptep + 1, pte); + set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte); } /* @@ -2290,7 +2290,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l prev = use_temporary_mm(text_poke_mm); kasan_disable_current(); - func((u8 *)poking_addr + offset_in_page(addr), src, len); + func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len); kasan_enable_current(); /* @@ -2299,9 +2299,9 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l */ barrier(); - pte_clear(text_poke_mm, poking_addr, ptep); + pte_clear(text_poke_mm, text_poke_mm_addr, ptep); if (cross_page_boundary) - pte_clear(text_poke_mm, poking_addr + PAGE_SIZE, ptep + 1); + pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1); /* * Loading the previous page-table hierarchy requires a serializing @@ -2314,7 +2314,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * Flushing the TLB might involve IPIs, which would require enabled * IRQs, but not if the mm is not used, as it is in this point. */ - flush_tlb_mm_range(text_poke_mm, poking_addr, poking_addr + + flush_tlb_mm_range(text_poke_mm, text_poke_mm_addr, text_poke_mm_addr + (cross_page_boundary ? 2 : 1) * PAGE_SIZE, PAGE_SHIFT, false); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 84b52a1ebd48..f8c74d19bebb 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -835,20 +835,20 @@ void __init poking_init(void) * will be mapped at the same PMD. We need 2 pages, so find space for 3, * and adjust the address if the PMD ends after the first one. */ - poking_addr = TASK_UNMAPPED_BASE; + text_poke_mm_addr = TASK_UNMAPPED_BASE; if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) - poking_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) % + text_poke_mm_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) % (TASK_SIZE - TASK_UNMAPPED_BASE - 3 * PAGE_SIZE); - if (((poking_addr + PAGE_SIZE) & ~PMD_MASK) == 0) - poking_addr += PAGE_SIZE; + if (((text_poke_mm_addr + PAGE_SIZE) & ~PMD_MASK) == 0) + text_poke_mm_addr += PAGE_SIZE; /* * We need to trigger the allocation of the page-tables that will be * needed for poking now. Later, poking may be performed in an atomic * section, which might cause allocation to fail. */ - ptep = get_locked_pte(text_poke_mm, poking_addr, &ptl); + ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl); BUG_ON(!ptep); pte_unmap_unlock(ptep, ptl); } From e84c31b9c9ac75ceaa2597bce021c529e76edd26 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:22 +0200 Subject: [PATCH 121/331] x86/alternatives: Rename 'bp_desc' to 'int3_desc' Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-11-mingo@kernel.org --- arch/x86/kernel/alternative.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62d74442cbd1..c6d0ca3d683b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2478,7 +2478,7 @@ struct text_poke_int3_vec { static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); -static struct text_poke_int3_vec bp_desc; +static struct text_poke_int3_vec int3_desc; static __always_inline struct text_poke_int3_vec *try_get_desc(void) @@ -2488,7 +2488,7 @@ struct text_poke_int3_vec *try_get_desc(void) if (!raw_atomic_inc_not_zero(refs)) return NULL; - return &bp_desc; + return &int3_desc; } static __always_inline void put_desc(void) @@ -2527,7 +2527,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) /* * Having observed our INT3 instruction, we now must observe - * bp_desc with non-zero refcount: + * int3_desc with non-zero refcount: * * text_poke_array_refs = 1 INT3 * WMB RMB @@ -2630,12 +2630,12 @@ static void smp_text_poke_batch_process(struct text_poke_loc *tp, unsigned int n lockdep_assert_held(&text_mutex); - bp_desc.vec = tp; - bp_desc.nr_entries = nr_entries; + int3_desc.vec = tp; + int3_desc.nr_entries = nr_entries; /* * Corresponds to the implicit memory barrier in try_get_desc() to - * ensure reading a non-zero refcount provides up to date bp_desc data. + * ensure reading a non-zero refcount provides up to date int3_desc data. */ for_each_possible_cpu(i) atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); From 762255b743b81c5307c0e93b2eeee4e8b7424152 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:23 +0200 Subject: [PATCH 122/331] x86/alternatives: Remove duplicate 'text_poke_early()' prototype It's declared in already. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-12-mingo@kernel.org --- arch/x86/kernel/alternative.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c6d0ca3d683b..b8794f756ea4 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -176,7 +176,6 @@ extern s32 __return_sites[], __return_sites_end[]; extern s32 __cfi_sites[], __cfi_sites_end[]; extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[]; extern s32 __smp_locks[], __smp_locks_end[]; -void text_poke_early(void *addr, const void *opcode, size_t len); /* * Matches NOP and NOPL, not any of the other possible NOPs. From 5224f09a7b57fcf2024245d89dcb26b0756fb1c8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:24 +0200 Subject: [PATCH 123/331] x86/alternatives: Update comments in int3_emulate_push() The idtentry macro in entry_64.S hasn't had a create_gap option for 5 years - update the comment. (Also clean up the entire comment block while at it.) Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-13-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index c8eac8cf4737..7e3527385708 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -142,13 +142,14 @@ static __always_inline void int3_emulate_push(struct pt_regs *regs, unsigned long val) { /* - * The int3 handler in entry_64.S adds a gap between the + * The INT3 handler in entry_64.S adds a gap between the * stack where the break point happened, and the saving of * pt_regs. We can extend the original stack because of - * this gap. See the idtentry macro's create_gap option. + * this gap. See the idtentry macro's X86_TRAP_BP logic. * - * Similarly entry_32.S will have a gap on the stack for (any) hardware - * exception and pt_regs; see FIXUP_FRAME. + * Similarly, entry_32.S will have a gap on the stack for + * (any) hardware exception and pt_regs; see the + * FIXUP_FRAME macro. */ regs->sp -= sizeof(unsigned long); *(unsigned long *)regs->sp = val; From f5afa2e8efda592ecc69cea7528ff660ac1d8096 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:25 +0200 Subject: [PATCH 124/331] x86/alternatives: Remove the confusing, inaccurate & unnecessary 'temp_mm_state_t' abstraction So the temp_mm_state_t abstraction used by use_temporary_mm() and unuse_temporary_mm() is super confusing: - The whole machinery is about temporarily switching to the text_poke_mm utility MM that got allocated during bootup for text-patching purposes alone: temp_mm_state_t prev; /* * Loading the temporary mm behaves as a compiler barrier, which * guarantees that the PTE will be set at the time memcpy() is done. */ prev = use_temporary_mm(text_poke_mm); - Yet the value that gets saved in the temp_mm_state_t variable is not the temporary MM ... but the previous MM... - Ie. we temporarily put the non-temporary MM into a variable that has the temp_mm_state_t type. This makes no sense whatsoever. - The confusion continues in unuse_temporary_mm(): static inline void unuse_temporary_mm(temp_mm_state_t prev_state) Here we unuse an MM that is ... not the temporary MM, but the previous MM. :-/ Fix up all this confusion by removing the unnecessary layer of abstraction and using a bog-standard 'struct mm_struct *prev_mm' variable to save the MM to. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-14-mingo@kernel.org --- arch/x86/kernel/alternative.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index b8794f756ea4..0ee43aa70adf 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2139,10 +2139,6 @@ void __init_or_module text_poke_early(void *addr, const void *opcode, } } -typedef struct { - struct mm_struct *mm; -} temp_mm_state_t; - /* * Using a temporary mm allows to set temporary mappings that are not accessible * by other CPUs. Such mappings are needed to perform sensitive memory writes @@ -2156,9 +2152,9 @@ typedef struct { * loaded, thereby preventing interrupt handler bugs from overriding * the kernel memory protection. */ -static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) +static inline struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) { - temp_mm_state_t temp_state; + struct mm_struct *prev_mm; lockdep_assert_irqs_disabled(); @@ -2170,8 +2166,8 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) if (this_cpu_read(cpu_tlbstate_shared.is_lazy)) leave_mm(); - temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm); - switch_mm_irqs_off(NULL, mm, current); + prev_mm = this_cpu_read(cpu_tlbstate.loaded_mm); + switch_mm_irqs_off(NULL, temp_mm, current); /* * If breakpoints are enabled, disable them while the temporary mm is @@ -2187,17 +2183,17 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) if (hw_breakpoint_active()) hw_breakpoint_disable(); - return temp_state; + return prev_mm; } __ro_after_init struct mm_struct *text_poke_mm; __ro_after_init unsigned long text_poke_mm_addr; -static inline void unuse_temporary_mm(temp_mm_state_t prev_state) +static inline void unuse_temporary_mm(struct mm_struct *prev_mm) { lockdep_assert_irqs_disabled(); - switch_mm_irqs_off(NULL, prev_state.mm, current); + switch_mm_irqs_off(NULL, prev_mm, current); /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(text_poke_mm)); @@ -2228,7 +2224,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; struct page *pages[2] = {NULL}; - temp_mm_state_t prev; + struct mm_struct *prev_mm; unsigned long flags; pte_t pte, *ptep; spinlock_t *ptl; @@ -2286,7 +2282,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * Loading the temporary mm behaves as a compiler barrier, which * guarantees that the PTE will be set at the time memcpy() is done. */ - prev = use_temporary_mm(text_poke_mm); + prev_mm = use_temporary_mm(text_poke_mm); kasan_disable_current(); func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len); @@ -2307,7 +2303,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * instruction that already allows the core to see the updated version. * Xen-PV is assumed to serialize execution in a similar manner. */ - unuse_temporary_mm(prev); + unuse_temporary_mm(prev_mm); /* * Flushing the TLB might involve IPIs, which would require enabled From aedb60c2c66c82be7cceb89e2b1a0d491bb7ca2f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:26 +0200 Subject: [PATCH 125/331] x86/alternatives: Rename 'text_poke_flush()' to 'smp_text_poke_batch_flush()' This name is actually actively confusing, because the simple text_poke*() APIs use MM-switching based code patching, while text_poke_flush() is part of the INT3 based text_poke_int3_*() machinery that is an additional layer of functionality on top of regular text_poke*() functionality. Rename it to smp_text_poke_batch_flush() to make it clear which layer it belongs to. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-15-mingo@kernel.org --- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 0ee43aa70adf..6c8bf2fd7a1c 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2858,7 +2858,7 @@ static bool tp_order_fail(void *addr) return false; } -static void text_poke_flush(void *addr) +static void smp_text_poke_batch_flush(void *addr) { if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) { smp_text_poke_batch_process(tp_vec, tp_vec_nr); @@ -2868,14 +2868,14 @@ static void text_poke_flush(void *addr) void text_poke_finish(void) { - text_poke_flush(NULL); + smp_text_poke_batch_flush(NULL); } void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate) { struct text_poke_loc *tp; - text_poke_flush(addr); + smp_text_poke_batch_flush(addr); tp = &tp_vec[tp_vec_nr++]; text_poke_loc_init(tp, addr, opcode, len, emulate); From e8d7b8c2bbcd5e50c93902af4ba53029fc0497fc Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:27 +0200 Subject: [PATCH 126/331] x86/alternatives: Rename 'text_poke_finish()' to 'smp_text_poke_batch_finish()' This name is actively confusing as well, because the simple text_poke*() APIs use MM-switching based code patching, while text_poke_finish() is part of the INT3 based text_poke_int3_*() machinery that is an additional layer of functionality on top of regular text_poke*() functionality. Rename it to smp_text_poke_batch_finish() to make it clear which layer it belongs to. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-16-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/ftrace.c | 4 ++-- arch/x86/kernel/jump_label.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 7e3527385708..f27d29042f8c 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -42,7 +42,7 @@ extern int smp_text_poke_int3_handler(struct pt_regs *regs); extern void smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate); extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate); -extern void text_poke_finish(void); +extern void smp_text_poke_batch_finish(void); #define INT3_INSN_SIZE 1 #define INT3_INSN_OPCODE 0xCC diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 6c8bf2fd7a1c..0589c051fe83 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2866,7 +2866,7 @@ static void smp_text_poke_batch_flush(void *addr) } } -void text_poke_finish(void) +void smp_text_poke_batch_finish(void) { smp_text_poke_batch_flush(NULL); } diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 7175a0404def..c35a928364b9 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -58,7 +58,7 @@ void ftrace_arch_code_modify_post_process(void) * module load, and we need to finish the text_poke_queue() * that they do, here. */ - text_poke_finish(); + smp_text_poke_batch_finish(); ftrace_poke_late = 0; mutex_unlock(&text_mutex); } @@ -250,7 +250,7 @@ void ftrace_replace_code(int enable) text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL); ftrace_update_record(rec, enable); } - text_poke_finish(); + smp_text_poke_batch_finish(); } void arch_ftrace_update_code(int command) diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 166e12037199..28be6eb6cb3d 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -143,6 +143,6 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, void arch_jump_label_transform_apply(void) { mutex_lock(&text_mutex); - text_poke_finish(); + smp_text_poke_batch_finish(); mutex_unlock(&text_mutex); } From 732c7c33a0c17f68393497766445cbd2878ee95e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:28 +0200 Subject: [PATCH 127/331] x86/alternatives: Rename 'text_poke_queue()' to 'smp_text_poke_batch_add()' This name is actively confusing as well, because the simple text_poke*() APIs use MM-switching based code patching, while text_poke_queue() is part of the INT3 based text_poke_int3_*() machinery that is an additional layer of functionality on top of regular text_poke*() functionality. Rename it to smp_text_poke_batch_add() to make it clear which layer it belongs to. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-17-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/ftrace.c | 6 +++--- arch/x86/kernel/jump_label.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index f27d29042f8c..f3c9b70afb0c 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -41,7 +41,7 @@ extern void *text_poke_set(void *addr, int c, size_t len); extern int smp_text_poke_int3_handler(struct pt_regs *regs); extern void smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate); -extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate); +extern void smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate); extern void smp_text_poke_batch_finish(void); #define INT3_INSN_SIZE 1 diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 0589c051fe83..6865296920d5 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2871,7 +2871,7 @@ void smp_text_poke_batch_finish(void) smp_text_poke_batch_flush(NULL); } -void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate) +void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { struct text_poke_loc *tp; diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index c35a928364b9..0853ba3fd04a 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -55,7 +55,7 @@ void ftrace_arch_code_modify_post_process(void) { /* * ftrace_make_{call,nop}() may be called during - * module load, and we need to finish the text_poke_queue() + * module load, and we need to finish the smp_text_poke_batch_add() * that they do, here. */ smp_text_poke_batch_finish(); @@ -119,7 +119,7 @@ ftrace_modify_code_direct(unsigned long ip, const char *old_code, /* replace the text with the new text */ if (ftrace_poke_late) - text_poke_queue((void *)ip, new_code, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_batch_add((void *)ip, new_code, MCOUNT_INSN_SIZE, NULL); else text_poke_early((void *)ip, new_code, MCOUNT_INSN_SIZE); return 0; @@ -247,7 +247,7 @@ void ftrace_replace_code(int enable) break; } - text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL); + smp_text_poke_batch_add((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL); ftrace_update_record(rec, enable); } smp_text_poke_batch_finish(); diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 28be6eb6cb3d..a7949a54a0ff 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -135,7 +135,7 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, mutex_lock(&text_mutex); jlp = __jump_label_patch(entry, type); - text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL); + smp_text_poke_batch_add((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL); mutex_unlock(&text_mutex); return true; } From fb802d639340d041e32d48057c7f15175a57c2de Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:29 +0200 Subject: [PATCH 128/331] x86/alternatives: Rename 'text_poke_loc_init()' to 'text_poke_int3_loc_init()' This name is actively confusing as well, because the simple text_poke*() APIs use MM-switching based code patching, while text_poke_loc_init() is part of the INT3 based text_poke_int3_*() machinery that is an additional layer of functionality on top of regular text_poke*() functionality. Rename it to text_poke_int3_loc_init() to make it clear which layer it belongs to. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-18-mingo@kernel.org --- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 6865296920d5..ebfd364f947a 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2762,7 +2762,7 @@ static void smp_text_poke_batch_process(struct text_poke_loc *tp, unsigned int n } } -static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, +static void text_poke_int3_loc_init(struct text_poke_loc *tp, void *addr, const void *opcode, size_t len, const void *emulate) { struct insn insn; @@ -2878,7 +2878,7 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c smp_text_poke_batch_flush(addr); tp = &tp_vec[tp_vec_nr++]; - text_poke_loc_init(tp, addr, opcode, len, emulate); + text_poke_int3_loc_init(tp, addr, opcode, len, emulate); } /** @@ -2896,6 +2896,6 @@ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, cons { struct text_poke_loc tp; - text_poke_loc_init(&tp, addr, opcode, len, emulate); + text_poke_int3_loc_init(&tp, addr, opcode, len, emulate); smp_text_poke_batch_process(&tp, 1); } From a81d43c46e6e89fef1961147c5f3faca31f6b84e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:30 +0200 Subject: [PATCH 129/331] x86/alternatives: Rename 'struct text_poke_loc' to 'struct smp_text_poke_loc' Make it clear that this structure is part of the INT3 based SMP patching facility, not the regular text_poke*() MM-switch based facility. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-19-mingo@kernel.org --- arch/x86/kernel/alternative.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index ebfd364f947a..9abb8f0b8416 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2455,7 +2455,7 @@ void text_poke_sync(void) * this thing. When len == 6 everything is prefixed with 0x0f and we map * opcode to Jcc.d8, using len to distinguish. */ -struct text_poke_loc { +struct smp_text_poke_loc { /* addr := _stext + rel_addr */ s32 rel_addr; s32 disp; @@ -2467,7 +2467,7 @@ struct text_poke_loc { }; struct text_poke_int3_vec { - struct text_poke_loc *vec; + struct smp_text_poke_loc *vec; int nr_entries; }; @@ -2494,14 +2494,14 @@ static __always_inline void put_desc(void) raw_atomic_dec(refs); } -static __always_inline void *text_poke_addr(struct text_poke_loc *tp) +static __always_inline void *text_poke_addr(struct smp_text_poke_loc *tp) { return _stext + tp->rel_addr; } static __always_inline int patch_cmp(const void *key, const void *elt) { - struct text_poke_loc *tp = (struct text_poke_loc *) elt; + struct smp_text_poke_loc *tp = (struct smp_text_poke_loc *) elt; if (key < text_poke_addr(tp)) return -1; @@ -2513,7 +2513,7 @@ static __always_inline int patch_cmp(const void *key, const void *elt) noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { struct text_poke_int3_vec *desc; - struct text_poke_loc *tp; + struct smp_text_poke_loc *tp; int ret = 0; void *ip; @@ -2544,7 +2544,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) */ if (unlikely(desc->nr_entries > 1)) { tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, - sizeof(struct text_poke_loc), + sizeof(struct smp_text_poke_loc), patch_cmp); if (!tp) goto out_put; @@ -2592,8 +2592,8 @@ out_put: return ret; } -#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) -static struct text_poke_loc tp_vec[TP_VEC_MAX]; +#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) +static struct smp_text_poke_loc tp_vec[TP_VEC_MAX]; static int tp_vec_nr; /** @@ -2617,7 +2617,7 @@ static int tp_vec_nr; * replacing opcode * - sync cores */ -static void smp_text_poke_batch_process(struct text_poke_loc *tp, unsigned int nr_entries) +static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned int nr_entries) { unsigned char int3 = INT3_INSN_OPCODE; unsigned int i; @@ -2762,7 +2762,7 @@ static void smp_text_poke_batch_process(struct text_poke_loc *tp, unsigned int n } } -static void text_poke_int3_loc_init(struct text_poke_loc *tp, void *addr, +static void text_poke_int3_loc_init(struct smp_text_poke_loc *tp, void *addr, const void *opcode, size_t len, const void *emulate) { struct insn insn; @@ -2843,7 +2843,7 @@ static void text_poke_int3_loc_init(struct text_poke_loc *tp, void *addr, */ static bool tp_order_fail(void *addr) { - struct text_poke_loc *tp; + struct smp_text_poke_loc *tp; if (!tp_vec_nr) return false; @@ -2873,7 +2873,7 @@ void smp_text_poke_batch_finish(void) void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { - struct text_poke_loc *tp; + struct smp_text_poke_loc *tp; smp_text_poke_batch_flush(addr); @@ -2894,7 +2894,7 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c */ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { - struct text_poke_loc tp; + struct smp_text_poke_loc tp; text_poke_int3_loc_init(&tp, addr, opcode, len, emulate); smp_text_poke_batch_process(&tp, 1); From 3bd7546ff24ecf9dbd74adf92b843eebd2862d1c Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:32 +0200 Subject: [PATCH 130/331] x86/alternatives: Rename 'int3_desc' to 'int3_vec' Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-21-mingo@kernel.org --- arch/x86/kernel/alternative.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 9abb8f0b8416..b97abfb38c3b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2473,7 +2473,7 @@ struct text_poke_int3_vec { static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); -static struct text_poke_int3_vec int3_desc; +static struct text_poke_int3_vec int3_vec; static __always_inline struct text_poke_int3_vec *try_get_desc(void) @@ -2483,7 +2483,7 @@ struct text_poke_int3_vec *try_get_desc(void) if (!raw_atomic_inc_not_zero(refs)) return NULL; - return &int3_desc; + return &int3_vec; } static __always_inline void put_desc(void) @@ -2522,7 +2522,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) /* * Having observed our INT3 instruction, we now must observe - * int3_desc with non-zero refcount: + * int3_vec with non-zero refcount: * * text_poke_array_refs = 1 INT3 * WMB RMB @@ -2625,12 +2625,12 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i lockdep_assert_held(&text_mutex); - int3_desc.vec = tp; - int3_desc.nr_entries = nr_entries; + int3_vec.vec = tp; + int3_vec.nr_entries = nr_entries; /* * Corresponds to the implicit memory barrier in try_get_desc() to - * ensure reading a non-zero refcount provides up to date int3_desc data. + * ensure reading a non-zero refcount provides up to date int3_vec data. */ for_each_possible_cpu(i) atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); From 87836af1eafc6616bde680be556f49ba3325f798 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:33 +0200 Subject: [PATCH 131/331] x86/alternatives: Add text_mutex) assert to smp_text_poke_batch_flush() It's possible to escape the text_mutex-held assert in smp_text_poke_batch_process() if the caller uses a properly batched and sorted series of patch requests, so add an explicit lockdep_assert_held() to make sure it's held by all callers. All text_poke_int3_*() APIs will call either smp_text_poke_batch_process() or smp_text_poke_batch_flush() internally. The text_mutex must be held, because tp_vec and tp_vec_nr et al are all globals, and the INT3 patching machinery itself relies on external serialization. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-22-mingo@kernel.org --- arch/x86/kernel/alternative.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index b97abfb38c3b..c53eb3b0454e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2860,6 +2860,8 @@ static bool tp_order_fail(void *addr) static void smp_text_poke_batch_flush(void *addr) { + lockdep_assert_held(&text_mutex); + if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) { smp_text_poke_batch_process(tp_vec, tp_vec_nr); tp_vec_nr = 0; From 2d0cf10a1eb60deded109c2357326a5ca44e3845 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:34 +0200 Subject: [PATCH 132/331] x86/alternatives: Use non-inverted logic instead of 'tp_order_fail()' tp_order_fail() uses inverted logic: it returns true in case something is false, which is only a plus at the IOCCC. Instead rename it to regular parity as 'text_poke_addr_ordered()', and adjust the code accordingly. Also add a comment explaining how the address ordering should be understood. No change in functionality intended. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-23-mingo@kernel.org --- arch/x86/kernel/alternative.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c53eb3b0454e..c5801b4cbb66 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2841,28 +2841,34 @@ static void text_poke_int3_loc_init(struct smp_text_poke_loc *tp, void *addr, * We hard rely on the tp_vec being ordered; ensure this is so by flushing * early if needed. */ -static bool tp_order_fail(void *addr) +static bool text_poke_addr_ordered(void *addr) { struct smp_text_poke_loc *tp; if (!tp_vec_nr) - return false; + return true; if (!addr) /* force */ - return true; + return false; - tp = &tp_vec[tp_vec_nr - 1]; + /* + * If the last current entry's address is higher than the + * new entry's address we'd like to add, then ordering + * is violated and we must first flush all pending patching + * requests: + */ + tp = &tp_vec[tp_vec_nr-1]; if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr) - return true; + return false; - return false; + return true; } static void smp_text_poke_batch_flush(void *addr) { lockdep_assert_held(&text_mutex); - if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) { + if (tp_vec_nr == TP_VEC_MAX || !text_poke_addr_ordered(addr)) { smp_text_poke_batch_process(tp_vec, tp_vec_nr); tp_vec_nr = 0; } From eaa24c9177c8c765ec9b9ccab392ac07ae8acda0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:35 +0200 Subject: [PATCH 133/331] x86/alternatives: Remove the 'addr == NULL means forced-flush' hack from smp_text_poke_batch_finish()/smp_text_poke_batch_flush()/text_poke_addr_ordered() There's this weird hack used by smp_text_poke_batch_finish() to indicate a 'forced flush': smp_text_poke_batch_flush(NULL); Just open-code the vector-flush in a straightforward fashion: smp_text_poke_batch_process(tp_vec, tp_vec_nr); tp_vec_nr = 0; And get rid of !addr hack from text_poke_addr_ordered(). Leave a WARN_ON_ONCE(), just in case some external code learned to rely on this behavior. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-24-mingo@kernel.org --- arch/x86/kernel/alternative.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c5801b4cbb66..16a41e29db17 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2845,12 +2845,11 @@ static bool text_poke_addr_ordered(void *addr) { struct smp_text_poke_loc *tp; + WARN_ON_ONCE(!addr); + if (!tp_vec_nr) return true; - if (!addr) /* force */ - return false; - /* * If the last current entry's address is higher than the * new entry's address we'd like to add, then ordering @@ -2864,6 +2863,14 @@ static bool text_poke_addr_ordered(void *addr) return true; } +void smp_text_poke_batch_finish(void) +{ + if (tp_vec_nr) { + smp_text_poke_batch_process(tp_vec, tp_vec_nr); + tp_vec_nr = 0; + } +} + static void smp_text_poke_batch_flush(void *addr) { lockdep_assert_held(&text_mutex); @@ -2874,11 +2881,6 @@ static void smp_text_poke_batch_flush(void *addr) } } -void smp_text_poke_batch_finish(void) -{ - smp_text_poke_batch_flush(NULL); -} - void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { struct smp_text_poke_loc *tp; From c8976ade0c1b4c0629b8a080d712ff402e8343b3 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:36 +0200 Subject: [PATCH 134/331] x86/alternatives: Simplify smp_text_poke_single() by using tp_vec and existing APIs Instead of constructing a vector on-stack, just use the already available batch-patching vector - which should always be empty at this point. This will allow subsequent simplifications. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-25-mingo@kernel.org --- arch/x86/kernel/alternative.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 16a41e29db17..b4cb8676c2f0 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2904,8 +2904,13 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c */ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { - struct smp_text_poke_loc tp; + struct smp_text_poke_loc *tp; - text_poke_int3_loc_init(&tp, addr, opcode, len, emulate); - smp_text_poke_batch_process(&tp, 1); + /* Batch-patching should not be mixed with single-patching: */ + WARN_ON_ONCE(tp_vec_nr != 0); + + tp = &tp_vec[tp_vec_nr++]; + text_poke_int3_loc_init(tp, addr, opcode, len, emulate); + + smp_text_poke_batch_finish(); } From 476ad071c678d70623cdb14fadd21818f64cc45b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:37 +0200 Subject: [PATCH 135/331] x86/alternatives: Assert that smp_text_poke_int3_handler() can only ever handle 'tp_vec[]' based requests Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-26-mingo@kernel.org --- arch/x86/kernel/alternative.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index b4cb8676c2f0..329f6ee6b3bc 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2510,6 +2510,10 @@ static __always_inline int patch_cmp(const void *key, const void *elt) return 0; } +#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) +static struct smp_text_poke_loc tp_vec[TP_VEC_MAX]; +static int tp_vec_nr; + noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { struct text_poke_int3_vec *desc; @@ -2534,6 +2538,8 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) if (!desc) return 0; + WARN_ON_ONCE(desc->vec != tp_vec); + /* * Discount the INT3. See smp_text_poke_batch_process(). */ @@ -2592,10 +2598,6 @@ out_put: return ret; } -#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) -static struct smp_text_poke_loc tp_vec[TP_VEC_MAX]; -static int tp_vec_nr; - /** * smp_text_poke_batch_process() -- update instructions on live kernel on SMP * @tp: vector of instructions to patch From 37725b64a9912292841cea7e9aebfd0f084ed8c0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:38 +0200 Subject: [PATCH 136/331] x86/alternatives: Assert input parameters in smp_text_poke_batch_process() At this point the 'tp' input parameter must always be the global 'tp_vec' array, and 'nr_entries' must always be equal to 'tp_vec_nr'. Assert these conditions - which will allow the removal of a layer of indirection between these values. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-27-mingo@kernel.org --- arch/x86/kernel/alternative.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 329f6ee6b3bc..4fa26a4351a6 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2627,6 +2627,9 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i lockdep_assert_held(&text_mutex); + WARN_ON_ONCE(tp != tp_vec); + WARN_ON_ONCE(nr_entries != tp_vec_nr); + int3_vec.vec = tp; int3_vec.nr_entries = nr_entries; From 6e7dc03aeeb52fb0147c03377e4f44fea780ef53 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:39 +0200 Subject: [PATCH 137/331] x86/alternatives: Introduce 'struct smp_text_poke_array' and move tp_vec and tp_vec_nr to it struct text_poke_array is an equivalent structure to these global variables: static struct smp_text_poke_loc tp_vec[TP_VEC_MAX]; static int tp_vec_nr; Note that we intentionally mirror much of the naming of 'struct text_poke_int3_vec', which will further highlight the unecessary layering going on in this code, and will ease its removal. No change in functionality. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-28-mingo@kernel.org --- arch/x86/kernel/alternative.c | 43 +++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4fa26a4351a6..0af220394480 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2467,14 +2467,21 @@ struct smp_text_poke_loc { }; struct text_poke_int3_vec { - struct smp_text_poke_loc *vec; int nr_entries; + struct smp_text_poke_loc *vec; }; static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); static struct text_poke_int3_vec int3_vec; +#define TP_ARRAY_NR_ENTRIES_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) + +static struct smp_text_poke_array { + int nr_entries; + struct smp_text_poke_loc vec[TP_ARRAY_NR_ENTRIES_MAX]; +} text_poke_array; + static __always_inline struct text_poke_int3_vec *try_get_desc(void) { @@ -2510,10 +2517,6 @@ static __always_inline int patch_cmp(const void *key, const void *elt) return 0; } -#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) -static struct smp_text_poke_loc tp_vec[TP_VEC_MAX]; -static int tp_vec_nr; - noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { struct text_poke_int3_vec *desc; @@ -2538,7 +2541,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) if (!desc) return 0; - WARN_ON_ONCE(desc->vec != tp_vec); + WARN_ON_ONCE(desc->vec != text_poke_array.vec); /* * Discount the INT3. See smp_text_poke_batch_process(). @@ -2627,8 +2630,8 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i lockdep_assert_held(&text_mutex); - WARN_ON_ONCE(tp != tp_vec); - WARN_ON_ONCE(nr_entries != tp_vec_nr); + WARN_ON_ONCE(tp != text_poke_array.vec); + WARN_ON_ONCE(nr_entries != text_poke_array.nr_entries); int3_vec.vec = tp; int3_vec.nr_entries = nr_entries; @@ -2843,7 +2846,7 @@ static void text_poke_int3_loc_init(struct smp_text_poke_loc *tp, void *addr, } /* - * We hard rely on the tp_vec being ordered; ensure this is so by flushing + * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing * early if needed. */ static bool text_poke_addr_ordered(void *addr) @@ -2852,7 +2855,7 @@ static bool text_poke_addr_ordered(void *addr) WARN_ON_ONCE(!addr); - if (!tp_vec_nr) + if (!text_poke_array.nr_entries) return true; /* @@ -2861,7 +2864,7 @@ static bool text_poke_addr_ordered(void *addr) * is violated and we must first flush all pending patching * requests: */ - tp = &tp_vec[tp_vec_nr-1]; + tp = &text_poke_array.vec[text_poke_array.nr_entries-1]; if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr) return false; @@ -2870,9 +2873,9 @@ static bool text_poke_addr_ordered(void *addr) void smp_text_poke_batch_finish(void) { - if (tp_vec_nr) { - smp_text_poke_batch_process(tp_vec, tp_vec_nr); - tp_vec_nr = 0; + if (text_poke_array.nr_entries) { + smp_text_poke_batch_process(text_poke_array.vec, text_poke_array.nr_entries); + text_poke_array.nr_entries = 0; } } @@ -2880,9 +2883,9 @@ static void smp_text_poke_batch_flush(void *addr) { lockdep_assert_held(&text_mutex); - if (tp_vec_nr == TP_VEC_MAX || !text_poke_addr_ordered(addr)) { - smp_text_poke_batch_process(tp_vec, tp_vec_nr); - tp_vec_nr = 0; + if (text_poke_array.nr_entries == TP_ARRAY_NR_ENTRIES_MAX || !text_poke_addr_ordered(addr)) { + smp_text_poke_batch_process(text_poke_array.vec, text_poke_array.nr_entries); + text_poke_array.nr_entries = 0; } } @@ -2892,7 +2895,7 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c smp_text_poke_batch_flush(addr); - tp = &tp_vec[tp_vec_nr++]; + tp = &text_poke_array.vec[text_poke_array.nr_entries++]; text_poke_int3_loc_init(tp, addr, opcode, len, emulate); } @@ -2912,9 +2915,9 @@ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, cons struct smp_text_poke_loc *tp; /* Batch-patching should not be mixed with single-patching: */ - WARN_ON_ONCE(tp_vec_nr != 0); + WARN_ON_ONCE(text_poke_array.nr_entries != 0); - tp = &tp_vec[tp_vec_nr++]; + tp = &text_poke_array.vec[text_poke_array.nr_entries++]; text_poke_int3_loc_init(tp, addr, opcode, len, emulate); smp_text_poke_batch_finish(); From 0494b16b9caed22ff78adb84cedb7460532eb3f0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:40 +0200 Subject: [PATCH 138/331] x86/alternatives: Remove the tp_vec indirection At this point we are always working out of an uptodate text_poke_array, there's no need for smp_text_poke_int3_handler() to read via the int3_vec indirection - remove it. This simplifies the code: 1 file changed, 5 insertions(+), 15 deletions(-) Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-29-mingo@kernel.org --- arch/x86/kernel/alternative.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 0af220394480..9937345e4c5b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2466,15 +2466,6 @@ struct smp_text_poke_loc { u8 old; }; -struct text_poke_int3_vec { - int nr_entries; - struct smp_text_poke_loc *vec; -}; - -static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); - -static struct text_poke_int3_vec int3_vec; - #define TP_ARRAY_NR_ENTRIES_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) static struct smp_text_poke_array { @@ -2482,15 +2473,17 @@ static struct smp_text_poke_array { struct smp_text_poke_loc vec[TP_ARRAY_NR_ENTRIES_MAX]; } text_poke_array; +static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); + static __always_inline -struct text_poke_int3_vec *try_get_desc(void) +struct smp_text_poke_array *try_get_desc(void) { atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); if (!raw_atomic_inc_not_zero(refs)) return NULL; - return &int3_vec; + return &text_poke_array; } static __always_inline void put_desc(void) @@ -2519,7 +2512,7 @@ static __always_inline int patch_cmp(const void *key, const void *elt) noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { - struct text_poke_int3_vec *desc; + struct smp_text_poke_array *desc; struct smp_text_poke_loc *tp; int ret = 0; void *ip; @@ -2529,7 +2522,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) /* * Having observed our INT3 instruction, we now must observe - * int3_vec with non-zero refcount: + * text_poke_array with non-zero refcount: * * text_poke_array_refs = 1 INT3 * WMB RMB @@ -2633,12 +2626,9 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i WARN_ON_ONCE(tp != text_poke_array.vec); WARN_ON_ONCE(nr_entries != text_poke_array.nr_entries); - int3_vec.vec = tp; - int3_vec.nr_entries = nr_entries; - /* * Corresponds to the implicit memory barrier in try_get_desc() to - * ensure reading a non-zero refcount provides up to date int3_vec data. + * ensure reading a non-zero refcount provides up to date text_poke_array data. */ for_each_possible_cpu(i) atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), 1); From 46f3d9d329dec169c54a3b5b48deb1ba258689fc Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:41 +0200 Subject: [PATCH 139/331] x86/alternatives: Rename 'try_get_desc()' to 'try_get_text_poke_array()' This better reflects what the underlying code is doing, there's no 'descriptor' indirection anymore. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-30-mingo@kernel.org --- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 9937345e4c5b..02f123cff898 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2476,7 +2476,7 @@ static struct smp_text_poke_array { static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); static __always_inline -struct smp_text_poke_array *try_get_desc(void) +struct smp_text_poke_array *try_get_text_poke_array(void) { atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); @@ -2530,7 +2530,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) */ smp_rmb(); - desc = try_get_desc(); + desc = try_get_text_poke_array(); if (!desc) return 0; @@ -2627,7 +2627,7 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i WARN_ON_ONCE(nr_entries != text_poke_array.nr_entries); /* - * Corresponds to the implicit memory barrier in try_get_desc() to + * Corresponds to the implicit memory barrier in try_get_text_poke_array() to * ensure reading a non-zero refcount provides up to date text_poke_array data. */ for_each_possible_cpu(i) From 3916eec5160dd42c8409c2032149470a474cb5f2 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:42 +0200 Subject: [PATCH 140/331] x86/alternatives: Rename 'put_desc()' to 'put_text_poke_array()' Just like with try_get_text_poke_array(), this name better reflects what the underlying code is doing, there's no 'descriptor' indirection anymore. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-31-mingo@kernel.org --- arch/x86/kernel/alternative.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 02f123cff898..f909f4e2f908 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2486,7 +2486,7 @@ struct smp_text_poke_array *try_get_text_poke_array(void) return &text_poke_array; } -static __always_inline void put_desc(void) +static __always_inline void put_text_poke_array(void) { atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); @@ -2590,7 +2590,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) ret = 1; out_put: - put_desc(); + put_text_poke_array(); return ret; } From b6a25841c171c42b02d316a6bf784fb32e39c786 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:43 +0200 Subject: [PATCH 141/331] x86/alternatives: Simplify try_get_text_poke_array() There's no need to return a pointer on success - it's always the same pointer. Return a bool instead. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-32-mingo@kernel.org --- arch/x86/kernel/alternative.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index f909f4e2f908..edc18be4fac4 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2475,15 +2475,14 @@ static struct smp_text_poke_array { static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); -static __always_inline -struct smp_text_poke_array *try_get_text_poke_array(void) +static __always_inline bool try_get_text_poke_array(void) { atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); if (!raw_atomic_inc_not_zero(refs)) - return NULL; + return false; - return &text_poke_array; + return true; } static __always_inline void put_text_poke_array(void) @@ -2530,9 +2529,9 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) */ smp_rmb(); - desc = try_get_text_poke_array(); - if (!desc) + if (!try_get_text_poke_array()) return 0; + desc = &text_poke_array; WARN_ON_ONCE(desc->vec != text_poke_array.vec); From 8e35752f0c334ee3eb2bf075ac0bdb243ad25fac Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:44 +0200 Subject: [PATCH 142/331] x86/alternatives: Simplify smp_text_poke_int3_handler() Remove the 'desc' local variable indirection and use text_poke_array directly. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-33-mingo@kernel.org --- arch/x86/kernel/alternative.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index edc18be4fac4..97cb954467ff 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2511,7 +2511,6 @@ static __always_inline int patch_cmp(const void *key, const void *elt) noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { - struct smp_text_poke_array *desc; struct smp_text_poke_loc *tp; int ret = 0; void *ip; @@ -2531,9 +2530,6 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) if (!try_get_text_poke_array()) return 0; - desc = &text_poke_array; - - WARN_ON_ONCE(desc->vec != text_poke_array.vec); /* * Discount the INT3. See smp_text_poke_batch_process(). @@ -2543,14 +2539,14 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) /* * Skip the binary search if there is a single member in the vector. */ - if (unlikely(desc->nr_entries > 1)) { - tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, + if (unlikely(text_poke_array.nr_entries > 1)) { + tp = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries, sizeof(struct smp_text_poke_loc), patch_cmp); if (!tp) goto out_put; } else { - tp = desc->vec; + tp = text_poke_array.vec; if (text_poke_addr(tp) != ip) goto out_put; } From 74e8e2bf950e8deb0965583e2130f0fb5a705085 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:45 +0200 Subject: [PATCH 143/331] x86/alternatives: Simplify smp_text_poke_batch_process() This function is now using the text_poke_array state exclusively, make that explicit by removing the redundant input parameters. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-34-mingo@kernel.org --- arch/x86/kernel/alternative.c | 43 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 97cb954467ff..08ac3c7ad6f8 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2591,8 +2591,8 @@ out_put: /** * smp_text_poke_batch_process() -- update instructions on live kernel on SMP - * @tp: vector of instructions to patch - * @nr_entries: number of entries in the vector + * @text_poke_array.vec: vector of instructions to patch + * @text_poke_array.nr_entries: number of entries in the vector * * Modify multi-byte instruction by using int3 breakpoint on SMP. * We completely avoid stop_machine() here, and achieve the @@ -2610,7 +2610,7 @@ out_put: * replacing opcode * - sync cores */ -static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned int nr_entries) +static void smp_text_poke_batch_process(void) { unsigned char int3 = INT3_INSN_OPCODE; unsigned int i; @@ -2618,9 +2618,6 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i lockdep_assert_held(&text_mutex); - WARN_ON_ONCE(tp != text_poke_array.vec); - WARN_ON_ONCE(nr_entries != text_poke_array.nr_entries); - /* * Corresponds to the implicit memory barrier in try_get_text_poke_array() to * ensure reading a non-zero refcount provides up to date text_poke_array data. @@ -2640,16 +2637,16 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i /* * Corresponding read barrier in int3 notifier for making sure the - * nr_entries and handler are correctly ordered wrt. patching. + * text_poke_array.nr_entries and handler are correctly ordered wrt. patching. */ smp_wmb(); /* * First step: add a int3 trap to the address that will be patched. */ - for (i = 0; i < nr_entries; i++) { - tp[i].old = *(u8 *)text_poke_addr(&tp[i]); - text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE); + for (i = 0; i < text_poke_array.nr_entries; i++) { + text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]); + text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE); } text_poke_sync(); @@ -2657,15 +2654,15 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i /* * Second step: update all but the first byte of the patched range. */ - for (do_sync = 0, i = 0; i < nr_entries; i++) { - u8 old[POKE_MAX_OPCODE_SIZE+1] = { tp[i].old, }; + for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { + u8 old[POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, }; u8 _new[POKE_MAX_OPCODE_SIZE+1]; - const u8 *new = tp[i].text; - int len = tp[i].len; + const u8 *new = text_poke_array.vec[i].text; + int len = text_poke_array.vec[i].len; if (len - INT3_INSN_SIZE > 0) { memcpy(old + INT3_INSN_SIZE, - text_poke_addr(&tp[i]) + INT3_INSN_SIZE, + text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, len - INT3_INSN_SIZE); if (len == 6) { @@ -2674,7 +2671,7 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i new = _new; } - text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE, + text_poke(text_poke_addr(&text_poke_array.vec[i]) + INT3_INSN_SIZE, new + INT3_INSN_SIZE, len - INT3_INSN_SIZE); @@ -2705,7 +2702,7 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i * The old instruction is recorded so that the event can be * processed forwards or backwards. */ - perf_event_text_poke(text_poke_addr(&tp[i]), old, len, new, len); + perf_event_text_poke(text_poke_addr(&text_poke_array.vec[i]), old, len, new, len); } if (do_sync) { @@ -2721,16 +2718,16 @@ static void smp_text_poke_batch_process(struct smp_text_poke_loc *tp, unsigned i * Third step: replace the first byte (int3) by the first byte of * replacing opcode. */ - for (do_sync = 0, i = 0; i < nr_entries; i++) { - u8 byte = tp[i].text[0]; + for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { + u8 byte = text_poke_array.vec[i].text[0]; - if (tp[i].len == 6) + if (text_poke_array.vec[i].len == 6) byte = 0x0f; if (byte == INT3_INSN_OPCODE) continue; - text_poke(text_poke_addr(&tp[i]), &byte, INT3_INSN_SIZE); + text_poke(text_poke_addr(&text_poke_array.vec[i]), &byte, INT3_INSN_SIZE); do_sync++; } @@ -2859,7 +2856,7 @@ static bool text_poke_addr_ordered(void *addr) void smp_text_poke_batch_finish(void) { if (text_poke_array.nr_entries) { - smp_text_poke_batch_process(text_poke_array.vec, text_poke_array.nr_entries); + smp_text_poke_batch_process(); text_poke_array.nr_entries = 0; } } @@ -2869,7 +2866,7 @@ static void smp_text_poke_batch_flush(void *addr) lockdep_assert_held(&text_mutex); if (text_poke_array.nr_entries == TP_ARRAY_NR_ENTRIES_MAX || !text_poke_addr_ordered(addr)) { - smp_text_poke_batch_process(text_poke_array.vec, text_poke_array.nr_entries); + smp_text_poke_batch_process(); text_poke_array.nr_entries = 0; } } From 0e351aec2b0052d33d6e44ded622223043d4dcd5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:47 +0200 Subject: [PATCH 144/331] x86/alternatives: Move the text_poke_array manipulation into text_poke_int3_loc_init() and rename it to __smp_text_poke_batch_add() This simplifies the code and code generation a bit: text data bss dec hex filename 14802 1029 4112 19943 4de7 alternative.o.before 14784 1029 4112 19925 4dd5 alternative.o.after Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-36-mingo@kernel.org --- arch/x86/kernel/alternative.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 08ac3c7ad6f8..eb0da270043b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2752,12 +2752,14 @@ static void smp_text_poke_batch_process(void) } } -static void text_poke_int3_loc_init(struct smp_text_poke_loc *tp, void *addr, - const void *opcode, size_t len, const void *emulate) +static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { + struct smp_text_poke_loc *tp; struct insn insn; int ret, i = 0; + tp = &text_poke_array.vec[text_poke_array.nr_entries++]; + if (len == 6) i = 1; memcpy((void *)tp->text, opcode+i, len-i); @@ -2873,12 +2875,8 @@ static void smp_text_poke_batch_flush(void *addr) void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { - struct smp_text_poke_loc *tp; - smp_text_poke_batch_flush(addr); - - tp = &text_poke_array.vec[text_poke_array.nr_entries++]; - text_poke_int3_loc_init(tp, addr, opcode, len, emulate); + __smp_text_poke_batch_add(addr, opcode, len, emulate); } /** @@ -2894,13 +2892,9 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c */ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { - struct smp_text_poke_loc *tp; - /* Batch-patching should not be mixed with single-patching: */ WARN_ON_ONCE(text_poke_array.nr_entries != 0); - tp = &text_poke_array.vec[text_poke_array.nr_entries++]; - text_poke_int3_loc_init(tp, addr, opcode, len, emulate); - + __smp_text_poke_batch_add(addr, opcode, len, emulate); smp_text_poke_batch_finish(); } From 8a6a1b4e0ef15dab908a365588e06f23f9c0bad5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:48 +0200 Subject: [PATCH 145/331] x86/alternatives: Remove the mixed-patching restriction on smp_text_poke_single() At this point smp_text_poke_single(addr, opcode, len, emulate) is equivalent to: smp_text_poke_batch_add(addr, opcode, len, emulate); smp_text_poke_batch_finish(); So remove the restriction on mixing single-instruction patching with multi-instruction patching. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-37-mingo@kernel.org --- arch/x86/kernel/alternative.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index eb0da270043b..f0bb2158f27b 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2892,9 +2892,6 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c */ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { - /* Batch-patching should not be mixed with single-patching: */ - WARN_ON_ONCE(text_poke_array.nr_entries != 0); - __smp_text_poke_batch_add(addr, opcode, len, emulate); smp_text_poke_batch_finish(); } From 9647ce465265720509f80f21f0b36c00bb0c0d18 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:49 +0200 Subject: [PATCH 146/331] x86/alternatives: Document 'smp_text_poke_single()' Extend the documentation to better describe its purpose. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-38-mingo@kernel.org --- arch/x86/kernel/alternative.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index f0bb2158f27b..a9726cc96972 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2880,7 +2880,7 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c } /** - * smp_text_poke_single() -- update instructions on live kernel on SMP + * smp_text_poke_single() -- update instruction on live kernel on SMP immediately * @addr: address to patch * @opcode: opcode of new instruction * @len: length to copy @@ -2888,7 +2888,8 @@ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, c * * Update a single instruction with the vector in the stack, avoiding * dynamically allocated memory. This function should be used when it is - * not possible to allocate memory. + * not possible to allocate memory for a vector. The single instruction + * is patched in immediately. */ void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate) { From cca3473956be6ca5c7ad5d2ced5516eb509c1936 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:50 +0200 Subject: [PATCH 147/331] x86/alternatives: Add documentation for smp_text_poke_batch_add() Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-39-mingo@kernel.org --- arch/x86/kernel/alternative.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index a9726cc96972..b47ad0853589 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2873,6 +2873,19 @@ static void smp_text_poke_batch_flush(void *addr) } } +/** + * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched + * @addr: address to patch + * @opcode: opcode of new instruction + * @len: length to copy + * @emulate: instruction to be emulated + * + * Add a new instruction to the current queue of to-be-patched instructions + * the kernel maintains. The patching request will not be executed immediately, + * but becomes part of an array of patching requests, optimized for batched + * execution. All pending patching requests will be executed on the next + * smp_text_poke_batch_finish() call. + */ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { smp_text_poke_batch_flush(addr); From 7fbadb50d95a8bbc0de720e0857c77d4f13ddcaf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:51 +0200 Subject: [PATCH 148/331] x86/alternatives: Move text_poke_array completion from smp_text_poke_batch_finish() and smp_text_poke_batch_flush() to smp_text_poke_batch_process() Simplifies the code and improves code generation a bit: text data bss dec hex filename 14769 1017 4112 19898 4dba alternative.o.before 14742 1017 4112 19871 4d9f alternative.o.after Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-40-mingo@kernel.org --- arch/x86/kernel/alternative.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index b47ad0853589..556a82f576cd 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2750,6 +2750,9 @@ static void smp_text_poke_batch_process(void) if (unlikely(!atomic_dec_and_test(refs))) atomic_cond_read_acquire(refs, !VAL); } + + /* They are all completed: */ + text_poke_array.nr_entries = 0; } static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) @@ -2857,20 +2860,16 @@ static bool text_poke_addr_ordered(void *addr) void smp_text_poke_batch_finish(void) { - if (text_poke_array.nr_entries) { + if (text_poke_array.nr_entries) smp_text_poke_batch_process(); - text_poke_array.nr_entries = 0; - } } static void smp_text_poke_batch_flush(void *addr) { lockdep_assert_held(&text_mutex); - if (text_poke_array.nr_entries == TP_ARRAY_NR_ENTRIES_MAX || !text_poke_addr_ordered(addr)) { + if (text_poke_array.nr_entries == TP_ARRAY_NR_ENTRIES_MAX || !text_poke_addr_ordered(addr)) smp_text_poke_batch_process(); - text_poke_array.nr_entries = 0; - } } /** From 6e4955a9d73ebdc8496e6bff7f6d2bf83c01959f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:52 +0200 Subject: [PATCH 149/331] x86/alternatives: Rename 'text_poke_sync()' to 'smp_text_poke_sync_each_cpu()' Unlike sync_core(), text_poke_sync() is a very heavy operation, as it sends an IPI to every online CPU in the system and waits for completion. Reflect this in the name. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-41-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 12 ++++++------ arch/x86/kernel/kprobes/core.c | 4 ++-- arch/x86/kernel/kprobes/opt.c | 4 ++-- arch/x86/kernel/module.c | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index f3c9b70afb0c..d9dbbe9d9667 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -32,7 +32,7 @@ extern void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u * an inconsistent instruction while you patch. */ extern void *text_poke(void *addr, const void *opcode, size_t len); -extern void text_poke_sync(void); +extern void smp_text_poke_sync_each_cpu(void); extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len); extern void *text_poke_copy(void *addr, const void *opcode, size_t len); #define text_poke_copy text_poke_copy diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 556a82f576cd..e4c51d81a72f 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2445,7 +2445,7 @@ static void do_sync_core(void *info) sync_core(); } -void text_poke_sync(void) +void smp_text_poke_sync_each_cpu(void) { on_each_cpu(do_sync_core, NULL, 1); } @@ -2469,8 +2469,8 @@ struct smp_text_poke_loc { #define TP_ARRAY_NR_ENTRIES_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) static struct smp_text_poke_array { - int nr_entries; struct smp_text_poke_loc vec[TP_ARRAY_NR_ENTRIES_MAX]; + int nr_entries; } text_poke_array; static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); @@ -2649,7 +2649,7 @@ static void smp_text_poke_batch_process(void) text_poke(text_poke_addr(&text_poke_array.vec[i]), &int3, INT3_INSN_SIZE); } - text_poke_sync(); + smp_text_poke_sync_each_cpu(); /* * Second step: update all but the first byte of the patched range. @@ -2711,7 +2711,7 @@ static void smp_text_poke_batch_process(void) * not necessary and we'd be safe even without it. But * better safe than sorry (plus there's not only Intel). */ - text_poke_sync(); + smp_text_poke_sync_each_cpu(); } /* @@ -2732,13 +2732,13 @@ static void smp_text_poke_batch_process(void) } if (do_sync) - text_poke_sync(); + smp_text_poke_sync_each_cpu(); /* * Remove and wait for refs to be zero. * * Notably, if after step-3 above the INT3 got removed, then the - * text_poke_sync() will have serialized against any running INT3 + * smp_text_poke_sync_each_cpu() will have serialized against any running INT3 * handlers and the below spin-wait will not happen. * * IOW. unless the replacement instruction is INT3, this case goes diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 09608fd93687..47cb8eb138ba 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -808,7 +808,7 @@ void arch_arm_kprobe(struct kprobe *p) u8 int3 = INT3_INSN_OPCODE; text_poke(p->addr, &int3, 1); - text_poke_sync(); + smp_text_poke_sync_each_cpu(); perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1); } @@ -818,7 +818,7 @@ void arch_disarm_kprobe(struct kprobe *p) perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1); text_poke(p->addr, &p->opcode, 1); - text_poke_sync(); + smp_text_poke_sync_each_cpu(); } void arch_remove_kprobe(struct kprobe *p) diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 9307a40f4983..0aabd4c4e2c4 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -513,11 +513,11 @@ void arch_unoptimize_kprobe(struct optimized_kprobe *op) JMP32_INSN_SIZE - INT3_INSN_SIZE); text_poke(addr, new, INT3_INSN_SIZE); - text_poke_sync(); + smp_text_poke_sync_each_cpu(); text_poke(addr + INT3_INSN_SIZE, new + INT3_INSN_SIZE, JMP32_INSN_SIZE - INT3_INSN_SIZE); - text_poke_sync(); + smp_text_poke_sync_each_cpu(); perf_event_text_poke(op->kp.addr, old, JMP32_INSN_SIZE, new, JMP32_INSN_SIZE); } diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index a7998f351701..231d6326d1fd 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -206,7 +206,7 @@ static int write_relocate_add(Elf64_Shdr *sechdrs, write, apply); if (!early) { - text_poke_sync(); + smp_text_poke_sync_each_cpu(); mutex_unlock(&text_mutex); } From 0e67e587e2e07be8d6775a1444e679c6afbc87f4 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:53 +0200 Subject: [PATCH 150/331] x86/alternatives: Simplify text_poke_addr_ordered() - Use direct 'void *' pointer comparison, there's no need to force the type to 'unsigned long'. - Remove the 'tp' local variable indirection Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-42-mingo@kernel.org --- arch/x86/kernel/alternative.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index e4c51d81a72f..a747b0885f9a 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2838,8 +2838,6 @@ static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len */ static bool text_poke_addr_ordered(void *addr) { - struct smp_text_poke_loc *tp; - WARN_ON_ONCE(!addr); if (!text_poke_array.nr_entries) @@ -2851,8 +2849,7 @@ static bool text_poke_addr_ordered(void *addr) * is violated and we must first flush all pending patching * requests: */ - tp = &text_poke_array.vec[text_poke_array.nr_entries-1]; - if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr) + if (text_poke_addr(text_poke_array.vec + text_poke_array.nr_entries-1) > addr) return false; return true; From 6af9540379628a769c63b0a101ff371d7719ec04 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:54 +0200 Subject: [PATCH 151/331] x86/alternatives: Constify text_poke_addr() This will also allow the simplification of patch_cmp(). Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-43-mingo@kernel.org --- arch/x86/kernel/alternative.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index a747b0885f9a..14ca17dc36e8 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2493,7 +2493,7 @@ static __always_inline void put_text_poke_array(void) raw_atomic_dec(refs); } -static __always_inline void *text_poke_addr(struct smp_text_poke_loc *tp) +static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tp) { return _stext + tp->rel_addr; } From 3e6f47573ec3ed1d9dc50243fbcf50c87f740853 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:55 +0200 Subject: [PATCH 152/331] x86/alternatives: Simplify and clean up patch_cmp() - No need to cast over to 'struct smp_text_poke_loc *', void * is just fine for a binary search, - Use the canonical (a, b) input parameter nomenclature of cmp_func_t functions and rename the input parameters from (tp, elt) to (tpl_a, tpl_b). Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-44-mingo@kernel.org --- arch/x86/kernel/alternative.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 14ca17dc36e8..f278655f0950 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2498,13 +2498,11 @@ static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tp) return _stext + tp->rel_addr; } -static __always_inline int patch_cmp(const void *key, const void *elt) +static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b) { - struct smp_text_poke_loc *tp = (struct smp_text_poke_loc *) elt; - - if (key < text_poke_addr(tp)) + if (tpl_a < text_poke_addr(tpl_b)) return -1; - if (key > text_poke_addr(tp)) + if (tpl_a > text_poke_addr(tpl_b)) return 1; return 0; } From 22b9662313034e42c780a9d6ebcc1811d47d359b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:56 +0200 Subject: [PATCH 153/331] x86/alternatives: Standardize on 'tpl' local variable names for 'struct smp_text_poke_loc *' There's no toilet paper in this code. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-45-mingo@kernel.org --- arch/x86/kernel/alternative.c | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index f278655f0950..c5abcf95daea 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2493,9 +2493,9 @@ static __always_inline void put_text_poke_array(void) raw_atomic_dec(refs); } -static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tp) +static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl) { - return _stext + tp->rel_addr; + return _stext + tpl->rel_addr; } static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b) @@ -2509,7 +2509,7 @@ static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b) noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) { - struct smp_text_poke_loc *tp; + struct smp_text_poke_loc *tpl; int ret = 0; void *ip; @@ -2538,20 +2538,20 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) * Skip the binary search if there is a single member in the vector. */ if (unlikely(text_poke_array.nr_entries > 1)) { - tp = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries, + tpl = __inline_bsearch(ip, text_poke_array.vec, text_poke_array.nr_entries, sizeof(struct smp_text_poke_loc), patch_cmp); - if (!tp) + if (!tpl) goto out_put; } else { - tp = text_poke_array.vec; - if (text_poke_addr(tp) != ip) + tpl = text_poke_array.vec; + if (text_poke_addr(tpl) != ip) goto out_put; } - ip += tp->len; + ip += tpl->len; - switch (tp->opcode) { + switch (tpl->opcode) { case INT3_INSN_OPCODE: /* * Someone poked an explicit INT3, they'll want to handle it, @@ -2564,16 +2564,16 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) break; case CALL_INSN_OPCODE: - int3_emulate_call(regs, (long)ip + tp->disp); + int3_emulate_call(regs, (long)ip + tpl->disp); break; case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: - int3_emulate_jmp(regs, (long)ip + tp->disp); + int3_emulate_jmp(regs, (long)ip + tpl->disp); break; case 0x70 ... 0x7f: /* Jcc */ - int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); + int3_emulate_jcc(regs, tpl->opcode & 0xf, (long)ip, tpl->disp); break; default: @@ -2755,33 +2755,33 @@ static void smp_text_poke_batch_process(void) static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { - struct smp_text_poke_loc *tp; + struct smp_text_poke_loc *tpl; struct insn insn; int ret, i = 0; - tp = &text_poke_array.vec[text_poke_array.nr_entries++]; + tpl = &text_poke_array.vec[text_poke_array.nr_entries++]; if (len == 6) i = 1; - memcpy((void *)tp->text, opcode+i, len-i); + memcpy((void *)tpl->text, opcode+i, len-i); if (!emulate) emulate = opcode; ret = insn_decode_kernel(&insn, emulate); BUG_ON(ret < 0); - tp->rel_addr = addr - (void *)_stext; - tp->len = len; - tp->opcode = insn.opcode.bytes[0]; + tpl->rel_addr = addr - (void *)_stext; + tpl->len = len; + tpl->opcode = insn.opcode.bytes[0]; if (is_jcc32(&insn)) { /* * Map Jcc.d32 onto Jcc.d8 and use len to distinguish. */ - tp->opcode = insn.opcode.bytes[1] - 0x10; + tpl->opcode = insn.opcode.bytes[1] - 0x10; } - switch (tp->opcode) { + switch (tpl->opcode) { case RET_INSN_OPCODE: case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: @@ -2790,14 +2790,14 @@ static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len * next instruction can be padded with INT3. */ for (i = insn.length; i < len; i++) - BUG_ON(tp->text[i] != INT3_INSN_OPCODE); + BUG_ON(tpl->text[i] != INT3_INSN_OPCODE); break; default: BUG_ON(len != insn.length); } - switch (tp->opcode) { + switch (tpl->opcode) { case INT3_INSN_OPCODE: case RET_INSN_OPCODE: break; @@ -2806,21 +2806,21 @@ static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: case 0x70 ... 0x7f: /* Jcc */ - tp->disp = insn.immediate.value; + tpl->disp = insn.immediate.value; break; default: /* assume NOP */ switch (len) { case 2: /* NOP2 -- emulate as JMP8+0 */ BUG_ON(memcmp(emulate, x86_nops[len], len)); - tp->opcode = JMP8_INSN_OPCODE; - tp->disp = 0; + tpl->opcode = JMP8_INSN_OPCODE; + tpl->disp = 0; break; case 5: /* NOP5 -- emulate as JMP32+0 */ BUG_ON(memcmp(emulate, x86_nops[len], len)); - tp->opcode = JMP32_INSN_OPCODE; - tp->disp = 0; + tpl->opcode = JMP32_INSN_OPCODE; + tpl->disp = 0; break; default: /* unknown instruction */ From 8036fbe5a5d618be3694c4719afb14fd14cc972d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:57 +0200 Subject: [PATCH 154/331] x86/alternatives: Rename 'TP_ARRAY_NR_ENTRIES_MAX' to 'TEXT_POKE_ARRAY_MAX' Standardize on TEXT_POKE_ namespace for CPP constants too. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-46-mingo@kernel.org --- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c5abcf95daea..4b460dea4f4e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2466,10 +2466,10 @@ struct smp_text_poke_loc { u8 old; }; -#define TP_ARRAY_NR_ENTRIES_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) +#define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc)) static struct smp_text_poke_array { - struct smp_text_poke_loc vec[TP_ARRAY_NR_ENTRIES_MAX]; + struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX]; int nr_entries; } text_poke_array; @@ -2863,7 +2863,7 @@ static void smp_text_poke_batch_flush(void *addr) { lockdep_assert_held(&text_mutex); - if (text_poke_array.nr_entries == TP_ARRAY_NR_ENTRIES_MAX || !text_poke_addr_ordered(addr)) + if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) smp_text_poke_batch_process(); } From 3c8454dfc9143c992375a166a620ea3d62c3e434 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:58 +0200 Subject: [PATCH 155/331] x86/alternatives: Rename 'POKE_MAX_OPCODE_SIZE' to 'TEXT_POKE_MAX_OPCODE_SIZE' Join the TEXT_POKE_ namespace. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-47-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 4 ++-- arch/x86/kernel/alternative.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index d9dbbe9d9667..a45ac8a223f3 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -11,7 +11,7 @@ * JUMP_LABEL_NOP_SIZE/RELATIVEJUMP_SIZE, which are 5. * Raise it if needed. */ -#define POKE_MAX_OPCODE_SIZE 5 +#define TEXT_POKE_MAX_OPCODE_SIZE 5 extern void text_poke_early(void *addr, const void *opcode, size_t len); @@ -82,7 +82,7 @@ static __always_inline int text_opcode_size(u8 opcode) } union text_poke_insn { - u8 text[POKE_MAX_OPCODE_SIZE]; + u8 text[TEXT_POKE_MAX_OPCODE_SIZE]; struct { u8 opcode; s32 disp; diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4b460dea4f4e..b8e0b1b2383f 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2461,7 +2461,7 @@ struct smp_text_poke_loc { s32 disp; u8 len; u8 opcode; - const u8 text[POKE_MAX_OPCODE_SIZE]; + const u8 text[TEXT_POKE_MAX_OPCODE_SIZE]; /* see smp_text_poke_batch_process() */ u8 old; }; @@ -2653,8 +2653,8 @@ static void smp_text_poke_batch_process(void) * Second step: update all but the first byte of the patched range. */ for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { - u8 old[POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, }; - u8 _new[POKE_MAX_OPCODE_SIZE+1]; + u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, }; + u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1]; const u8 *new = text_poke_array.vec[i].text; int len = text_poke_array.vec[i].len; From db5c68c88c07337b7e5801abfb926f9191fa7945 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:40:59 +0200 Subject: [PATCH 156/331] x86/alternatives: Simplify the #include section We accumulated lots of unnecessary header inclusions over the years, trim them. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-48-mingo@kernel.org --- arch/x86/kernel/alternative.c | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index b8e0b1b2383f..eb3be5d3efef 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1,36 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only #define pr_fmt(fmt) "SMP alternatives: " fmt -#include -#include +#include #include -#include -#include -#include -#include -#include #include #include -#include -#include -#include -#include -#include -#include -#include + #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include +#include int __read_mostly alternatives_patched; From b1bb39185df6f5ddafcb912304b73e70c6b70c5f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:41:00 +0200 Subject: [PATCH 157/331] x86/alternatives: Move declarations of vmlinux.lds.S defined section symbols to Move it from the middle of a .c file next to the similar declarations of __alt_instructions[] et al. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-49-mingo@kernel.org --- arch/x86/include/asm/alternative.h | 6 ++++++ arch/x86/kernel/alternative.c | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 4a37a8bd87fd..ef84739a77f5 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -82,6 +82,12 @@ struct alt_instr { extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; +extern s32 __retpoline_sites[], __retpoline_sites_end[]; +extern s32 __return_sites[], __return_sites_end[]; +extern s32 __cfi_sites[], __cfi_sites_end[]; +extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[]; +extern s32 __smp_locks[], __smp_locks_end[]; + /* * Debug flag that can be tested to see whether alternative * instructions were patched in already: diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index eb3be5d3efef..cd828c236fd2 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -149,12 +149,6 @@ static void add_nop(u8 *buf, unsigned int len) *buf = INT3_INSN_OPCODE; } -extern s32 __retpoline_sites[], __retpoline_sites_end[]; -extern s32 __return_sites[], __return_sites_end[]; -extern s32 __cfi_sites[], __cfi_sites_end[]; -extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[]; -extern s32 __smp_locks[], __smp_locks_end[]; - /* * Matches NOP and NOPL, not any of the other possible NOPs. */ From 2c373ca0640f25361aee1bcf382fc8081830938e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:41:01 +0200 Subject: [PATCH 158/331] x86/alternatives: Remove 'smp_text_poke_batch_flush()' It only has a single user left, merge it into smp_text_poke_batch_add() and remove the helper function. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-50-mingo@kernel.org --- arch/x86/kernel/alternative.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index cd828c236fd2..c0be0663425e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2831,14 +2831,6 @@ void smp_text_poke_batch_finish(void) smp_text_poke_batch_process(); } -static void smp_text_poke_batch_flush(void *addr) -{ - lockdep_assert_held(&text_mutex); - - if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) - smp_text_poke_batch_process(); -} - /** * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched * @addr: address to patch @@ -2854,7 +2846,8 @@ static void smp_text_poke_batch_flush(void *addr) */ void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { - smp_text_poke_batch_flush(addr); + if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) + smp_text_poke_batch_process(); __smp_text_poke_batch_add(addr, opcode, len, emulate); } From dac0d7542782bae98a4d8cedde3028a07f1915d2 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:41:02 +0200 Subject: [PATCH 159/331] x86/alternatives: Update the comments in smp_text_poke_batch_process() - Capitalize 'INT3' consistently, - make it clear that 'sync cores' means an SMP sync to all CPUs, - fix typos and spelling. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-51-mingo@kernel.org --- arch/x86/kernel/alternative.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index c0be0663425e..9ee6f879bce5 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2561,24 +2561,26 @@ out_put: /** * smp_text_poke_batch_process() -- update instructions on live kernel on SMP - * @text_poke_array.vec: vector of instructions to patch - * @text_poke_array.nr_entries: number of entries in the vector * - * Modify multi-byte instruction by using int3 breakpoint on SMP. - * We completely avoid stop_machine() here, and achieve the - * synchronization using int3 breakpoint. + * Input state: + * text_poke_array.vec: vector of instructions to patch + * text_poke_array.nr_entries: number of entries in the vector + * + * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * We completely avoid using stop_machine() here, and achieve the + * synchronization using INT3 breakpoints and SMP cross-calls. * * The way it is done: * - For each entry in the vector: - * - add a int3 trap to the address that will be patched - * - sync cores + * - add an INT3 trap to the address that will be patched + * - SMP sync all CPUs * - For each entry in the vector: * - update all but the first byte of the patched range - * - sync cores + * - SMP sync all CPUs * - For each entry in the vector: - * - replace the first byte (int3) by the first byte of + * - replace the first byte (INT3) by the first byte of the * replacing opcode - * - sync cores + * - SMP sync all CPUs */ static void smp_text_poke_batch_process(void) { @@ -2606,13 +2608,13 @@ static void smp_text_poke_batch_process(void) cond_resched(); /* - * Corresponding read barrier in int3 notifier for making sure the + * Corresponding read barrier in INT3 notifier for making sure the * text_poke_array.nr_entries and handler are correctly ordered wrt. patching. */ smp_wmb(); /* - * First step: add a int3 trap to the address that will be patched. + * First step: add a INT3 trap to the address that will be patched. */ for (i = 0; i < text_poke_array.nr_entries; i++) { text_poke_array.vec[i].old = *(u8 *)text_poke_addr(&text_poke_array.vec[i]); @@ -2685,7 +2687,7 @@ static void smp_text_poke_batch_process(void) } /* - * Third step: replace the first byte (int3) by the first byte of + * Third step: replace the first byte (INT3) by the first byte of the * replacing opcode. */ for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) { From 023f42dd59203be8ad2fc0574af32d3b4ad041ec Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:41:03 +0200 Subject: [PATCH 160/331] x86/alternatives: Rename 'apply_relocation()' to 'text_poke_apply_relocation()' Join the text_poke_*() API namespace. Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-52-mingo@kernel.org --- arch/x86/include/asm/text-patching.h | 2 +- arch/x86/kernel/alternative.c | 6 +++--- arch/x86/kernel/callthunks.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index a45ac8a223f3..5337f1be18f6 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -15,7 +15,7 @@ extern void text_poke_early(void *addr, const void *opcode, size_t len); -extern void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len); +extern void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len); /* * Clear and restore the kernel write-protection flag on the local CPU. diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 9ee6f879bce5..231b2acbe360 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -340,7 +340,7 @@ static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, } } -void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) +void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) { __apply_relocation(buf, instr, instrlen, repl, repl_len); optimize_nops(instr, buf, instrlen); @@ -496,7 +496,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, for (; insn_buff_sz < a->instrlen; insn_buff_sz++) insn_buff[insn_buff_sz] = 0x90; - apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen); + text_poke_apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen); DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr); DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement); @@ -1981,7 +1981,7 @@ __visible noinline void __init __alt_reloc_selftest(void *arg) static noinline void __init alt_reloc_selftest(void) { /* - * Tests apply_relocation(). + * Tests text_poke_apply_relocation(). * * This has a relative immediate (CALL) in a place other than the first * instruction and additionally on x86_64 we get a RIP-relative LEA: diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c index d86d7d6e750c..a951333c5995 100644 --- a/arch/x86/kernel/callthunks.c +++ b/arch/x86/kernel/callthunks.c @@ -185,7 +185,7 @@ static void *patch_dest(void *dest, bool direct) u8 *pad = dest - tsize; memcpy(insn_buff, skl_call_thunk_template, tsize); - apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize); + text_poke_apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize); /* Already patched? */ if (!bcmp(pad, insn_buff, tsize)) @@ -294,7 +294,7 @@ static bool is_callthunk(void *addr) pad = (void *)(dest - tmpl_size); memcpy(insn_buff, skl_call_thunk_template, tmpl_size); - apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size); + text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size); return !bcmp(pad, insn_buff, tmpl_size); } @@ -312,7 +312,7 @@ int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip) return 0; memcpy(insn_buff, skl_call_thunk_template, tmpl_size); - apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size); + text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size); memcpy(*pprog, insn_buff, tmpl_size); *pprog += tmpl_size; From 4f9534719e524affb1aa8e0ff0c8b30c1c65e574 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 11 Apr 2025 07:41:04 +0200 Subject: [PATCH 161/331] x86/alternatives: Add comment about noinstr expectations Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-53-mingo@kernel.org --- arch/x86/kernel/alternative.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 231b2acbe360..604dd608d281 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2447,6 +2447,11 @@ static struct smp_text_poke_array { static DEFINE_PER_CPU(atomic_t, text_poke_array_refs); +/* + * These four __always_inline annotations imply noinstr, necessary + * due to smp_text_poke_int3_handler() being noinstr: + */ + static __always_inline bool try_get_text_poke_array(void) { atomic_t *refs = this_cpu_ptr(&text_poke_array_refs); From 23a76739d6afe6f98ccdb2517d7985b4335c7a3a Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 11 Apr 2025 07:41:05 +0200 Subject: [PATCH 162/331] x86/alternatives: Make smp_text_poke_batch_process() subsume smp_text_poke_batch_finish() Simplify the alternatives interface some more by moving the poke_batch_finish check into poke_batch_process and renaming the latter. The net effect is one less function name to consider when reading the code. Signed-off-by: Nikolay Borisov Signed-off-by: Ingo Molnar Cc: Juergen Gross Cc: "H . Peter Anvin" Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250411054105.2341982-54-mingo@kernel.org --- arch/x86/kernel/alternative.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 604dd608d281..f785d2335812 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2434,7 +2434,7 @@ struct smp_text_poke_loc { u8 len; u8 opcode; const u8 text[TEXT_POKE_MAX_OPCODE_SIZE]; - /* see smp_text_poke_batch_process() */ + /* see smp_text_poke_batch_finish() */ u8 old; }; @@ -2507,7 +2507,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs) return 0; /* - * Discount the INT3. See smp_text_poke_batch_process(). + * Discount the INT3. See smp_text_poke_batch_finish(). */ ip = (void *) regs->ip - INT3_INSN_SIZE; @@ -2565,7 +2565,7 @@ out_put: } /** - * smp_text_poke_batch_process() -- update instructions on live kernel on SMP + * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP * * Input state: * text_poke_array.vec: vector of instructions to patch @@ -2587,12 +2587,15 @@ out_put: * replacing opcode * - SMP sync all CPUs */ -static void smp_text_poke_batch_process(void) +void smp_text_poke_batch_finish(void) { unsigned char int3 = INT3_INSN_OPCODE; unsigned int i; int do_sync; + if (!text_poke_array.nr_entries) + return; + lockdep_assert_held(&text_mutex); /* @@ -2832,12 +2835,6 @@ static bool text_poke_addr_ordered(void *addr) return true; } -void smp_text_poke_batch_finish(void) -{ - if (text_poke_array.nr_entries) - smp_text_poke_batch_process(); -} - /** * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched * @addr: address to patch @@ -2854,7 +2851,7 @@ void smp_text_poke_batch_finish(void) void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate) { if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr)) - smp_text_poke_batch_process(); + smp_text_poke_batch_finish(); __smp_text_poke_batch_add(addr, opcode, len, emulate); } From 718f9038acc53631cc887676b9c433e6fb9f15dc Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Fri, 11 Apr 2025 09:04:00 +0200 Subject: [PATCH 163/331] x86/cpuid: Remove obsolete CPUID(0x2) iteration macro The CPUID(0x2) cache descriptors iterator at : for_each_leaf_0x2_desc() has no more call sites. Remove it. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20250411070401.1358760-2-darwi@linutronix.de --- arch/x86/include/asm/cpuid/leaf_0x2_api.h | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/arch/x86/include/asm/cpuid/leaf_0x2_api.h b/arch/x86/include/asm/cpuid/leaf_0x2_api.h index 46ecb15e92d9..09fa3070b271 100644 --- a/arch/x86/include/asm/cpuid/leaf_0x2_api.h +++ b/arch/x86/include/asm/cpuid/leaf_0x2_api.h @@ -40,29 +40,6 @@ static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) } } -/** - * for_each_leaf_0x2_desc() - Iterator for CPUID leaf 0x2 descriptors - * @regs: Leaf 0x2 output, as returned by cpuid_get_leaf_0x2_regs() - * @desc: Pointer to the returned descriptor for each iteration - * - * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers - * @regs. Provide each descriptor through @desc. - * - * Note that the first byte is skipped as it is not a descriptor. - * - * Sample usage:: - * - * union leaf_0x2_regs regs; - * u8 *desc; - * - * cpuid_get_leaf_0x2_regs(®s); - * for_each_leaf_0x2_desc(regs, desc) { - * // Handle *desc value - * } - */ -#define for_each_leaf_0x2_desc(regs, desc) \ - for (desc = &(regs).desc[1]; desc < &(regs).desc[16]; desc++) - /** * for_each_leaf_0x2_entry() - Iterator for parsed leaf 0x2 descriptors * @regs: Leaf 0x2 register output, returned by cpuid_get_leaf_0x2_regs() From 62e565273993ee47c82ca2975e65ce4bec3c3697 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Fri, 11 Apr 2025 09:04:01 +0200 Subject: [PATCH 164/331] x86/cacheinfo: Standardize header files and CPUID references Reference header files using their canonical form . Standardize on CPUID(0xN), instead of CPUID(N), for all standard leaves. This removes ambiguity and aligns them with their extended counterparts like CPUID(0x8000001d). References: 0dd09e215a39 ("x86/cacheinfo: Apply maintainer-tip coding style fixes") Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: "H. Peter Anvin" Cc: John Ogness Link: https://lore.kernel.org/r/20250411070401.1358760-3-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 52727f8c0006..cc7ae2bdcf4a 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -3,9 +3,9 @@ * x86 CPU caches detection and configuration * * Previous changes - * - Venkatesh Pallipadi: Cache identification through CPUID(4) + * - Venkatesh Pallipadi: Cache identification through CPUID(0x4) * - Ashok Raj : Work with CPU hotplug infrastructure - * - Andi Kleen / Andreas Herrmann: CPUID(4) emulation on AMD + * - Andi Kleen / Andreas Herrmann: CPUID(0x4) emulation on AMD */ #include @@ -78,7 +78,7 @@ struct _cpuid4_info { unsigned long size; }; -/* Map CPUID(4) EAX.cache_type to linux/cacheinfo.h types */ +/* Map CPUID(0x4) EAX.cache_type to types */ static const enum cache_type cache_type_map[] = { [CTYPE_NULL] = CACHE_TYPE_NOCACHE, [CTYPE_DATA] = CACHE_TYPE_DATA, @@ -87,7 +87,7 @@ static const enum cache_type cache_type_map[] = { }; /* - * Fallback AMD CPUID(4) emulation + * Fallback AMD CPUID(0x4) emulation * AMD CPUs with TOPOEXT can just use CPUID(0x8000001d) * * @AMD_L2_L3_INVALID_ASSOC: cache info for the respective L2/L3 cache should @@ -361,7 +361,7 @@ static void intel_cacheinfo_done(struct cpuinfo_x86 *c, unsigned int l3, { /* * If llc_id is still unset, then cpuid_level < 4, which implies - * that the only possibility left is SMT. Since CPUID(2) doesn't + * that the only possibility left is SMT. Since CPUID(0x2) doesn't * specify any shared caches and SMT shares all caches, we can * unconditionally set LLC ID to the package ID so that all * threads share it. @@ -376,7 +376,7 @@ static void intel_cacheinfo_done(struct cpuinfo_x86 *c, unsigned int l3, } /* - * Legacy Intel CPUID(2) path if CPUID(4) is not available. + * Legacy Intel CPUID(0x2) path if CPUID(0x4) is not available. */ static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) { @@ -466,7 +466,7 @@ static bool intel_cacheinfo_0x4(struct cpuinfo_x86 *c) void init_intel_cacheinfo(struct cpuinfo_x86 *c) { - /* Don't use CPUID(2) if CPUID(4) is supported. */ + /* Don't use CPUID(0x2) if CPUID(0x4) is supported. */ if (intel_cacheinfo_0x4(c)) return; @@ -474,7 +474,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) } /* - * linux/cacheinfo.h shared_cpu_map setup, AMD/Hygon + * shared_cpu_map setup, AMD/Hygon */ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, const struct _cpuid4_info *id4) @@ -533,7 +533,7 @@ static int __cache_amd_cpumap_setup(unsigned int cpu, int index, } /* - * linux/cacheinfo.h shared_cpu_map setup, Intel + fallback AMD/Hygon + * shared_cpu_map setup, Intel + fallback AMD/Hygon */ static void __cache_cpumap_setup(unsigned int cpu, int index, const struct _cpuid4_info *id4) @@ -599,7 +599,7 @@ int init_cache_level(unsigned int cpu) } /* - * The max shared threads number comes from CPUID(4) EAX[25-14] with input + * The max shared threads number comes from CPUID(0x4) EAX[25-14] with input * ECX as cache index. Then right shift apicid by the number's order to get * cache id for this cache node. */ From 0812e096cff0fd58d88a21a413fba56c0e6c3caa Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 2 Apr 2025 11:45:34 +0200 Subject: [PATCH 165/331] x86/mm: Add 'mm' argument to unuse_temporary_mm() In commit 209954cbc7d0 ("x86/mm/tlb: Update mm_cpumask lazily") unuse_temporary_mm() grew the assumption that it gets used on poking_mm exclusively. While this is currently true, lets not hard code this assumption. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-2-mingo@kernel.org --- arch/x86/kernel/alternative.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index f785d2335812..95053e8a1378 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2161,14 +2161,14 @@ static inline struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) __ro_after_init struct mm_struct *text_poke_mm; __ro_after_init unsigned long text_poke_mm_addr; -static inline void unuse_temporary_mm(struct mm_struct *prev_mm) +static inline void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm) { lockdep_assert_irqs_disabled(); switch_mm_irqs_off(NULL, prev_mm, current); /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ - cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(text_poke_mm)); + cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm)); /* * Restore the breakpoints if they were disabled before the temporary mm @@ -2275,7 +2275,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * instruction that already allows the core to see the updated version. * Xen-PV is assumed to serialize execution in a similar manner. */ - unuse_temporary_mm(prev_mm); + unuse_temporary_mm(text_poke_mm, prev_mm); /* * Flushing the TLB might involve IPIs, which would require enabled From 81e3cbdef230fd9adfa8569044b07290afd66708 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 2 Apr 2025 11:45:35 +0200 Subject: [PATCH 166/331] x86/events, x86/insn-eval: Remove incorrect current->active_mm references When decoding an instruction or handling a perf event that references an LDT segment, if we don't have a valid user context, trying to access the LDT by any means other than SLDT is racy. Certainly, using current->active_mm is wrong, as active_mm can point to a real user mm when CR3 and LDTR no longer reference that mm. Clean up the code. If nmi_uaccess_okay() says we don't have a valid context, just fail. Otherwise use current->mm. Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-3-mingo@kernel.org --- arch/x86/events/core.c | 9 ++++++++- arch/x86/lib/insn-eval.c | 13 ++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 6866cc5acb0b..95118b52b606 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2803,8 +2803,15 @@ static unsigned long get_segment_base(unsigned int segment) #ifdef CONFIG_MODIFY_LDT_SYSCALL struct ldt_struct *ldt; + /* + * If we're not in a valid context with a real (not just lazy) + * user mm, then don't even try. + */ + if (!nmi_uaccess_okay()) + return 0; + /* IRQs are off, so this synchronizes with smp_store_release */ - ldt = READ_ONCE(current->active_mm->context.ldt); + ldt = smp_load_acquire(¤t->mm->context.ldt); if (!ldt || idx >= ldt->nr_entries) return 0; diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 98631c0e7a11..f786401ac15d 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -631,14 +631,21 @@ static bool get_desc(struct desc_struct *out, unsigned short sel) /* Bits [15:3] contain the index of the desired entry. */ sel >>= 3; - mutex_lock(¤t->active_mm->context.lock); - ldt = current->active_mm->context.ldt; + /* + * If we're not in a valid context with a real (not just lazy) + * user mm, then don't even try. + */ + if (!nmi_uaccess_okay()) + return false; + + mutex_lock(¤t->mm->context.lock); + ldt = current->mm->context.ldt; if (ldt && sel < ldt->nr_entries) { *out = ldt->entries[sel]; success = true; } - mutex_unlock(¤t->active_mm->context.lock); + mutex_unlock(¤t->mm->context.lock); return success; } From d376972c9825ac4e8ad74872ee0730a5b4292e44 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 2 Apr 2025 11:45:36 +0200 Subject: [PATCH 167/331] x86/mm: Make use_/unuse_temporary_mm() non-static This prepares them for use outside of the alternative machinery. The code is unchanged. Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-4-mingo@kernel.org --- arch/x86/include/asm/mmu_context.h | 3 ++ arch/x86/kernel/alternative.c | 64 ------------------------------ arch/x86/mm/tlb.c | 64 ++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 64 deletions(-) diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index 2398058b6e83..b103e1709a67 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -272,4 +272,7 @@ unsigned long __get_current_cr3_fast(void); #include +extern struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm); +extern void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm); + #endif /* _ASM_X86_MMU_CONTEXT_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 95053e8a1378..bdbdfa0e4d06 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2111,73 +2111,9 @@ void __init_or_module text_poke_early(void *addr, const void *opcode, } } -/* - * Using a temporary mm allows to set temporary mappings that are not accessible - * by other CPUs. Such mappings are needed to perform sensitive memory writes - * that override the kernel memory protections (e.g., W^X), without exposing the - * temporary page-table mappings that are required for these write operations to - * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the - * mapping is torn down. - * - * Context: The temporary mm needs to be used exclusively by a single core. To - * harden security IRQs must be disabled while the temporary mm is - * loaded, thereby preventing interrupt handler bugs from overriding - * the kernel memory protection. - */ -static inline struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) -{ - struct mm_struct *prev_mm; - - lockdep_assert_irqs_disabled(); - - /* - * Make sure not to be in TLB lazy mode, as otherwise we'll end up - * with a stale address space WITHOUT being in lazy mode after - * restoring the previous mm. - */ - if (this_cpu_read(cpu_tlbstate_shared.is_lazy)) - leave_mm(); - - prev_mm = this_cpu_read(cpu_tlbstate.loaded_mm); - switch_mm_irqs_off(NULL, temp_mm, current); - - /* - * If breakpoints are enabled, disable them while the temporary mm is - * used. Userspace might set up watchpoints on addresses that are used - * in the temporary mm, which would lead to wrong signals being sent or - * crashes. - * - * Note that breakpoints are not disabled selectively, which also causes - * kernel breakpoints (e.g., perf's) to be disabled. This might be - * undesirable, but still seems reasonable as the code that runs in the - * temporary mm should be short. - */ - if (hw_breakpoint_active()) - hw_breakpoint_disable(); - - return prev_mm; -} - __ro_after_init struct mm_struct *text_poke_mm; __ro_after_init unsigned long text_poke_mm_addr; -static inline void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm) -{ - lockdep_assert_irqs_disabled(); - - switch_mm_irqs_off(NULL, prev_mm, current); - - /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ - cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm)); - - /* - * Restore the breakpoints if they were disabled before the temporary mm - * was loaded. - */ - if (hw_breakpoint_active()) - hw_breakpoint_restore(); -} - static void text_poke_memcpy(void *dst, const void *src, size_t len) { memcpy(dst, src, len); diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index e459d97ef397..f3da20bfcf0e 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -971,6 +971,70 @@ void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) this_cpu_write(cpu_tlbstate_shared.is_lazy, true); } +/* + * Using a temporary mm allows to set temporary mappings that are not accessible + * by other CPUs. Such mappings are needed to perform sensitive memory writes + * that override the kernel memory protections (e.g., W^X), without exposing the + * temporary page-table mappings that are required for these write operations to + * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the + * mapping is torn down. + * + * Context: The temporary mm needs to be used exclusively by a single core. To + * harden security IRQs must be disabled while the temporary mm is + * loaded, thereby preventing interrupt handler bugs from overriding + * the kernel memory protection. + */ +struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) +{ + struct mm_struct *prev_mm; + + lockdep_assert_irqs_disabled(); + + /* + * Make sure not to be in TLB lazy mode, as otherwise we'll end up + * with a stale address space WITHOUT being in lazy mode after + * restoring the previous mm. + */ + if (this_cpu_read(cpu_tlbstate_shared.is_lazy)) + leave_mm(); + + prev_mm = this_cpu_read(cpu_tlbstate.loaded_mm); + switch_mm_irqs_off(NULL, temp_mm, current); + + /* + * If breakpoints are enabled, disable them while the temporary mm is + * used. Userspace might set up watchpoints on addresses that are used + * in the temporary mm, which would lead to wrong signals being sent or + * crashes. + * + * Note that breakpoints are not disabled selectively, which also causes + * kernel breakpoints (e.g., perf's) to be disabled. This might be + * undesirable, but still seems reasonable as the code that runs in the + * temporary mm should be short. + */ + if (hw_breakpoint_active()) + hw_breakpoint_disable(); + + return prev_mm; +} + +void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm) +{ + lockdep_assert_irqs_disabled(); + + switch_mm_irqs_off(NULL, prev_mm, current); + + /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ + cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm)); + + /* + * Restore the breakpoints if they were disabled before the temporary mm + * was loaded. + */ + if (hw_breakpoint_active()) + hw_breakpoint_restore(); +} + /* * Call this when reinitializing a CPU. It fixes the following potential * problems: From 4873f494bbe4670f353a9b76ce44e6028c811cbb Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 2 Apr 2025 11:45:37 +0200 Subject: [PATCH 168/331] x86/mm: Remove 'mm' argument from unuse_temporary_mm() again Now that unuse_temporary_mm() lives in tlb.c it can access cpu_tlbstate.loaded_mm. [ mingo: Merged it on top of x86/alternatives ] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-5-mingo@kernel.org --- arch/x86/include/asm/mmu_context.h | 2 +- arch/x86/kernel/alternative.c | 2 +- arch/x86/mm/tlb.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index b103e1709a67..988c11792634 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -273,6 +273,6 @@ unsigned long __get_current_cr3_fast(void); #include extern struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm); -extern void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm); +extern void unuse_temporary_mm(struct mm_struct *prev_mm); #endif /* _ASM_X86_MMU_CONTEXT_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index bdbdfa0e4d06..ddbc303e41e3 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2211,7 +2211,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l * instruction that already allows the core to see the updated version. * Xen-PV is assumed to serialize execution in a similar manner. */ - unuse_temporary_mm(text_poke_mm, prev_mm); + unuse_temporary_mm(prev_mm); /* * Flushing the TLB might involve IPIs, which would require enabled diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index f3da20bfcf0e..38fdcf875d5f 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1018,14 +1018,14 @@ struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) return prev_mm; } -void unuse_temporary_mm(struct mm_struct *mm, struct mm_struct *prev_mm) +void unuse_temporary_mm(struct mm_struct *prev_mm) { lockdep_assert_irqs_disabled(); - switch_mm_irqs_off(NULL, prev_mm, current); - /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ - cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), mm_cpumask(this_cpu_read(cpu_tlbstate.loaded_mm))); + + switch_mm_irqs_off(NULL, prev_mm, current); /* * Restore the breakpoints if they were disabled before the temporary mm From 58f8ffa917669a0c8c027e24d5349f0b488f8181 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 2 Apr 2025 11:45:38 +0200 Subject: [PATCH 169/331] x86/mm: Allow temporary MMs when IRQs are on EFI runtime services should use temporary MMs, but EFI runtime services want IRQs on. Preemption must still be disabled in a temporary MM context. At some point, the entirely temporary MM mechanism should be moved out of arch code. Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20250402094540.3586683-6-mingo@kernel.org --- arch/x86/mm/tlb.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 38fdcf875d5f..c9b87e5f569a 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -977,18 +977,23 @@ void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) * that override the kernel memory protections (e.g., W^X), without exposing the * temporary page-table mappings that are required for these write operations to * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the - * mapping is torn down. + * mapping is torn down. Temporary mms can also be used for EFI runtime service + * calls or similar functionality. * - * Context: The temporary mm needs to be used exclusively by a single core. To - * harden security IRQs must be disabled while the temporary mm is - * loaded, thereby preventing interrupt handler bugs from overriding - * the kernel memory protection. + * It is illegal to schedule while using a temporary mm -- the context switch + * code is unaware of the temporary mm and does not know how to context switch. + * Use a real (non-temporary) mm in a kernel thread if you need to sleep. + * + * Note: For sensitive memory writes, the temporary mm needs to be used + * exclusively by a single core, and IRQs should be disabled while the + * temporary mm is loaded, thereby preventing interrupt handler bugs from + * overriding the kernel memory protection. */ struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) { struct mm_struct *prev_mm; - lockdep_assert_irqs_disabled(); + lockdep_assert_preemption_disabled(); /* * Make sure not to be in TLB lazy mode, as otherwise we'll end up @@ -1020,7 +1025,7 @@ struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) void unuse_temporary_mm(struct mm_struct *prev_mm) { - lockdep_assert_irqs_disabled(); + lockdep_assert_preemption_disabled(); /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ cpumask_clear_cpu(smp_processor_id(), mm_cpumask(this_cpu_read(cpu_tlbstate.loaded_mm))); From e7021e2fe0b4335523d3f6e2221000bdfc633b62 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 2 Apr 2025 11:45:39 +0200 Subject: [PATCH 170/331] x86/efi: Make efi_enter/leave_mm() use the use_/unuse_temporary_mm() machinery This should be considerably more robust. It's also necessary for optimized for_each_possible_lazymm_cpu() on x86 -- without this patch, EFI calls in lazy context would remove the lazy mm from mm_cpumask(). [ mingo: Merged it on top of x86/alternatives ] Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-7-mingo@kernel.org --- arch/x86/platform/efi/efi_64.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index ac57259a432b..a5d3496d32a5 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -434,15 +434,12 @@ void __init efi_dump_pagetable(void) */ static void efi_enter_mm(void) { - efi_prev_mm = current->active_mm; - current->active_mm = &efi_mm; - switch_mm(efi_prev_mm, &efi_mm, NULL); + efi_prev_mm = use_temporary_mm(&efi_mm); } static void efi_leave_mm(void) { - current->active_mm = efi_prev_mm; - switch_mm(&efi_mm, efi_prev_mm, NULL); + unuse_temporary_mm(efi_prev_mm); } void arch_efi_call_virt_setup(void) From af8967158f9ad759a93e8e7a933c10e7cbb01ba2 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 2 Apr 2025 11:45:40 +0200 Subject: [PATCH 171/331] x86/mm: Opt-in to IRQs-off activate_mm() We gain nothing by having the core code enable IRQs right before calling activate_mm() only for us to turn them right back off again in switch_mm(). This will save a few cycles, so execve() should be blazingly fast with this patch applied! Signed-off-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: Andrew Morton Link: https://lore.kernel.org/r/20250402094540.3586683-8-mingo@kernel.org --- arch/x86/Kconfig | 1 + arch/x86/include/asm/mmu_context.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 4b9f378e05f6..aeac63b11fc2 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -153,6 +153,7 @@ config X86 select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH + select ARCH_WANT_IRQS_OFF_ACTIVATE_MM select BUILDTIME_TABLE_SORT select CLKEVT_I8253 select CLOCKSOURCE_WATCHDOG diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index 988c11792634..c511f8584ae4 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -190,7 +190,7 @@ extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, #define activate_mm(prev, next) \ do { \ paravirt_enter_mmap(next); \ - switch_mm((prev), (next), NULL); \ + switch_mm_irqs_off((prev), (next), NULL); \ } while (0); #ifdef CONFIG_X86_32 From bcceba3c72c0cf06dfbae77f5aec70fb6187e8df Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:19 +0200 Subject: [PATCH 172/331] x86/asm: Make rip_rel_ptr() usable from fPIC code RIP_REL_REF() is used in non-PIC C code that is called very early, before the kernel virtual mapping is up, which is the mapping that the linker expects. It is currently used in two different ways: - to refer to the value of a global variable, including as an lvalue in assignments; - to take the address of a global variable via the mapping that the code currently executes at. The former case is only needed in non-PIC code, as PIC code will never use absolute symbol references when the address of the symbol is not being used. But taking the address of a variable in PIC code may still require extra care, as a stack allocated struct assignment may be emitted as a memcpy() from a statically allocated copy in .rodata. For instance, this void startup_64_setup_gdt_idt(void) { struct desc_ptr startup_gdt_descr = { .address = (__force unsigned long)gdt_page.gdt, .size = GDT_SIZE - 1, }; may result in an absolute symbol reference in PIC code, even though the struct is allocated on the stack and populated at runtime. To address this case, make rip_rel_ptr() accessible in PIC code, and update any existing uses where the address of a global variable is taken using RIP_REL_REF. Once all code of this nature has been moved into arch/x86/boot/startup and built with -fPIC, RIP_REL_REF() can be retired, and only rip_rel_ptr() will remain. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-14-ardb+git@google.com --- arch/x86/coco/sev/core.c | 2 +- arch/x86/coco/sev/shared.c | 4 ++-- arch/x86/include/asm/asm.h | 2 +- arch/x86/kernel/head64.c | 24 ++++++++++++------------ arch/x86/mm/mem_encrypt_identity.c | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index b0c1a7a57497..832f7a7b10b2 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -2400,7 +2400,7 @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info) * kernel was loaded (physbase), so the get the CA address using * RIP-relative addressing. */ - pa = (u64)&RIP_REL_REF(boot_svsm_ca_page); + pa = (u64)rip_rel_ptr(&boot_svsm_ca_page); /* * Switch over to the boot SVSM CA while the current CA is still diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c index 2e4122f8aa6b..04982d356803 100644 --- a/arch/x86/coco/sev/shared.c +++ b/arch/x86/coco/sev/shared.c @@ -475,7 +475,7 @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid */ static const struct snp_cpuid_table *snp_cpuid_get_table(void) { - return &RIP_REL_REF(cpuid_table_copy); + return rip_rel_ptr(&cpuid_table_copy); } /* @@ -1681,7 +1681,7 @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) * routine is running identity mapped when called, both by the decompressor * code and the early kernel code. */ - if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1)) + if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1)) return false; /* diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index cc2881576c2c..a9f07799e337 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -114,13 +114,13 @@ #endif #ifndef __ASSEMBLER__ -#ifndef __pic__ static __always_inline __pure void *rip_rel_ptr(void *p) { asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p)); return p; } +#ifndef __pic__ #define RIP_REL_REF(var) (*(typeof(&(var)))rip_rel_ptr(&(var))) #else #define RIP_REL_REF(var) (var) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index fa9b6339975f..954d093f187b 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -106,8 +106,8 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, * attribute. */ if (sme_get_me_mask()) { - paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted); - paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted); + paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted); + paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted); for (; paddr < paddr_end; paddr += PMD_SIZE) { /* @@ -144,8 +144,8 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, unsigned long __head __startup_64(unsigned long p2v_offset, struct boot_params *bp) { - pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts); - unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text); + pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts); + unsigned long physaddr = (unsigned long)rip_rel_ptr(_text); unsigned long va_text, va_end; unsigned long pgtable_flags; unsigned long load_delta; @@ -174,18 +174,18 @@ unsigned long __head __startup_64(unsigned long p2v_offset, for (;;); va_text = physaddr - p2v_offset; - va_end = (unsigned long)&RIP_REL_REF(_end) - p2v_offset; + va_end = (unsigned long)rip_rel_ptr(_end) - p2v_offset; /* Include the SME encryption mask in the fixup value */ load_delta += sme_get_me_mask(); /* Fixup the physical addresses in the page table */ - pgd = &RIP_REL_REF(early_top_pgt)->pgd; + pgd = rip_rel_ptr(early_top_pgt); pgd[pgd_index(__START_KERNEL_map)] += load_delta; if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) { - p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt); + p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); p4d[MAX_PTRS_PER_P4D - 1] += load_delta; pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; @@ -258,7 +258,7 @@ unsigned long __head __startup_64(unsigned long p2v_offset, * error, causing the BIOS to halt the system. */ - pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd; + pmd = rip_rel_ptr(level2_kernel_pgt); /* invalidate pages before the kernel image */ for (i = 0; i < pmd_index(va_text); i++) @@ -531,7 +531,7 @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; static void __head startup_64_load_idt(void *vc_handler) { struct desc_ptr desc = { - .address = (unsigned long)&RIP_REL_REF(bringup_idt_table), + .address = (unsigned long)rip_rel_ptr(bringup_idt_table), .size = sizeof(bringup_idt_table) - 1, }; struct idt_data data; @@ -565,11 +565,11 @@ void early_setup_idt(void) */ void __head startup_64_setup_gdt_idt(void) { - struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt; + struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page); void *handler = NULL; struct desc_ptr startup_gdt_descr = { - .address = (unsigned long)&RIP_REL_REF(*gdt), + .address = (unsigned long)gp->gdt, .size = GDT_SIZE - 1, }; @@ -582,7 +582,7 @@ void __head startup_64_setup_gdt_idt(void) "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) - handler = &RIP_REL_REF(vc_no_ghcb); + handler = rip_rel_ptr(vc_no_ghcb); startup_64_load_idt(handler); } diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 5eecdd92da10..e7fb3779b35f 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -318,8 +318,8 @@ void __head sme_encrypt_kernel(struct boot_params *bp) * memory from being cached. */ - kernel_start = (unsigned long)RIP_REL_REF(_text); - kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE); + kernel_start = (unsigned long)rip_rel_ptr(_text); + kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE); kernel_len = kernel_end - kernel_start; initrd_start = 0; @@ -345,7 +345,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp) * pagetable structures for the encryption of the kernel * pagetable structures for workarea (in case not currently mapped) */ - execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea); + execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea); execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE; execute_len = execute_end - execute_start; From 4cecebf200efea0fc865b5656f6d12ead2eb5573 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:20 +0200 Subject: [PATCH 173/331] x86/boot: Move the early GDT/IDT setup code into startup/ Move the early GDT/IDT setup code that runs long before the kernel virtual mapping is up into arch/x86/boot/startup/, and build it in a way that ensures that the code tolerates being called from the 1:1 mapping of memory. The code itself is left unchanged by this patch. Also tweak the sed symbol matching pattern in the decompressor to match on lower case 't' or 'b', as these will be emitted by Clang for symbols with hidden linkage. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-15-ardb+git@google.com --- arch/x86/boot/compressed/Makefile | 2 +- arch/x86/boot/startup/Makefile | 15 ++++++ arch/x86/boot/startup/gdt_idt.c | 84 +++++++++++++++++++++++++++++++ arch/x86/kernel/head64.c | 74 --------------------------- 4 files changed, 100 insertions(+), 75 deletions(-) create mode 100644 arch/x86/boot/startup/gdt_idt.c diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 37b85ce9b2a3..0fcad7b7e007 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -73,7 +73,7 @@ LDFLAGS_vmlinux += -T hostprogs := mkpiggy HOST_EXTRACFLAGS += -I$(srctree)/tools/include -sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' +sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' quiet_cmd_voffset = VOFFSET $@ cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@ diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 8919a1cbcb5a..1beb5de30735 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -1,6 +1,21 @@ # SPDX-License-Identifier: GPL-2.0 KBUILD_AFLAGS += -D__DISABLE_EXPORTS +KBUILD_CFLAGS += -D__DISABLE_EXPORTS -mcmodel=small -fPIC \ + -Os -DDISABLE_BRANCH_PROFILING \ + $(DISABLE_STACKLEAK_PLUGIN) \ + -fno-stack-protector -D__NO_FORTIFY \ + -include $(srctree)/include/linux/hidden.h + +# disable ftrace hooks +KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) +KASAN_SANITIZE := n +KCSAN_SANITIZE := n +KMSAN_SANITIZE := n +UBSAN_SANITIZE := n +KCOV_INSTRUMENT := n + +obj-$(CONFIG_X86_64) += gdt_idt.o lib-$(CONFIG_X86_64) += la57toggle.o lib-$(CONFIG_EFI_MIXED) += efi-mixed.o diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c new file mode 100644 index 000000000000..7e34d0b426b1 --- /dev/null +++ b/arch/x86/boot/startup/gdt_idt.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include +#include + +/* + * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is + * used until the idt_table takes over. On the boot CPU this happens in + * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases + * this happens in the functions called from head_64.S. + * + * The idt_table can't be used that early because all the code modifying it is + * in idt.c and can be instrumented by tracing or KASAN, which both don't work + * during early CPU bringup. Also the idt_table has the runtime vectors + * configured which require certain CPU state to be setup already (like TSS), + * which also hasn't happened yet in early CPU bringup. + */ +static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; + +/* This may run while still in the direct mapping */ +static void __head startup_64_load_idt(void *vc_handler) +{ + struct desc_ptr desc = { + .address = (unsigned long)rip_rel_ptr(bringup_idt_table), + .size = sizeof(bringup_idt_table) - 1, + }; + struct idt_data data; + gate_desc idt_desc; + + /* @vc_handler is set only for a VMM Communication Exception */ + if (vc_handler) { + init_idt_data(&data, X86_TRAP_VC, vc_handler); + idt_init_desc(&idt_desc, &data); + native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc); + } + + native_load_idt(&desc); +} + +/* This is used when running on kernel addresses */ +void early_setup_idt(void) +{ + void *handler = NULL; + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + setup_ghcb(); + handler = vc_boot_ghcb; + } + + startup_64_load_idt(handler); +} + +/* + * Setup boot CPU state needed before kernel switches to virtual addresses. + */ +void __head startup_64_setup_gdt_idt(void) +{ + struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page); + void *handler = NULL; + + struct desc_ptr startup_gdt_descr = { + .address = (unsigned long)gp->gdt, + .size = GDT_SIZE - 1, + }; + + /* Load GDT */ + native_load_gdt(&startup_gdt_descr); + + /* New GDT is live - reload data segment registers */ + asm volatile("movl %%eax, %%ds\n" + "movl %%eax, %%ss\n" + "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + handler = rip_rel_ptr(vc_no_ghcb); + + startup_64_load_idt(handler); +} diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 954d093f187b..9b2ffec4bbad 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -512,77 +512,3 @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data) start_kernel(); } - -/* - * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is - * used until the idt_table takes over. On the boot CPU this happens in - * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases - * this happens in the functions called from head_64.S. - * - * The idt_table can't be used that early because all the code modifying it is - * in idt.c and can be instrumented by tracing or KASAN, which both don't work - * during early CPU bringup. Also the idt_table has the runtime vectors - * configured which require certain CPU state to be setup already (like TSS), - * which also hasn't happened yet in early CPU bringup. - */ -static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; - -/* This may run while still in the direct mapping */ -static void __head startup_64_load_idt(void *vc_handler) -{ - struct desc_ptr desc = { - .address = (unsigned long)rip_rel_ptr(bringup_idt_table), - .size = sizeof(bringup_idt_table) - 1, - }; - struct idt_data data; - gate_desc idt_desc; - - /* @vc_handler is set only for a VMM Communication Exception */ - if (vc_handler) { - init_idt_data(&data, X86_TRAP_VC, vc_handler); - idt_init_desc(&idt_desc, &data); - native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc); - } - - native_load_idt(&desc); -} - -/* This is used when running on kernel addresses */ -void early_setup_idt(void) -{ - void *handler = NULL; - - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - setup_ghcb(); - handler = vc_boot_ghcb; - } - - startup_64_load_idt(handler); -} - -/* - * Setup boot CPU state needed before kernel switches to virtual addresses. - */ -void __head startup_64_setup_gdt_idt(void) -{ - struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page); - void *handler = NULL; - - struct desc_ptr startup_gdt_descr = { - .address = (unsigned long)gp->gdt, - .size = GDT_SIZE - 1, - }; - - /* Load GDT */ - native_load_gdt(&startup_gdt_descr); - - /* New GDT is live - reload data segment registers */ - asm volatile("movl %%eax, %%ds\n" - "movl %%eax, %%ss\n" - "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); - - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) - handler = rip_rel_ptr(vc_no_ghcb); - - startup_64_load_idt(handler); -} From dbe0ad775cbbd614583409d96abbc8ea7edb5eb4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:21 +0200 Subject: [PATCH 174/331] x86/boot: Move early kernel mapping code into startup/ The startup code that constructs the kernel virtual mapping runs from the 1:1 mapping of memory itself, and therefore, cannot use absolute symbol references. Before making changes in subsequent patches, move this code into a separate source file under arch/x86/boot/startup/ where all such code will be kept from now on. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-16-ardb+git@google.com --- arch/x86/boot/startup/Makefile | 2 +- arch/x86/boot/startup/map_kernel.c | 224 +++++++++++++++++++++++++++++ arch/x86/kernel/head64.c | 211 +-------------------------- 3 files changed, 226 insertions(+), 211 deletions(-) create mode 100644 arch/x86/boot/startup/map_kernel.c diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 1beb5de30735..10319aee666b 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -15,7 +15,7 @@ KMSAN_SANITIZE := n UBSAN_SANITIZE := n KCOV_INSTRUMENT := n -obj-$(CONFIG_X86_64) += gdt_idt.o +obj-$(CONFIG_X86_64) += gdt_idt.o map_kernel.o lib-$(CONFIG_X86_64) += la57toggle.o lib-$(CONFIG_EFI_MIXED) += efi-mixed.o diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c new file mode 100644 index 000000000000..5f1b7e0ba26e --- /dev/null +++ b/arch/x86/boot/startup/map_kernel.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; +extern unsigned int next_early_pgt; + +static inline bool check_la57_support(void) +{ + if (!IS_ENABLED(CONFIG_X86_5LEVEL)) + return false; + + /* + * 5-level paging is detected and enabled at kernel decompression + * stage. Only check if it has been enabled there. + */ + if (!(native_read_cr4() & X86_CR4_LA57)) + return false; + + RIP_REL_REF(__pgtable_l5_enabled) = 1; + RIP_REL_REF(pgdir_shift) = 48; + RIP_REL_REF(ptrs_per_p4d) = 512; + RIP_REL_REF(page_offset_base) = __PAGE_OFFSET_BASE_L5; + RIP_REL_REF(vmalloc_base) = __VMALLOC_BASE_L5; + RIP_REL_REF(vmemmap_base) = __VMEMMAP_BASE_L5; + + return true; +} + +static unsigned long __head sme_postprocess_startup(struct boot_params *bp, + pmdval_t *pmd, + unsigned long p2v_offset) +{ + unsigned long paddr, paddr_end; + int i; + + /* Encrypt the kernel and related (if SME is active) */ + sme_encrypt_kernel(bp); + + /* + * Clear the memory encryption mask from the .bss..decrypted section. + * The bss section will be memset to zero later in the initialization so + * there is no need to zero it after changing the memory encryption + * attribute. + */ + if (sme_get_me_mask()) { + paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted); + paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted); + + for (; paddr < paddr_end; paddr += PMD_SIZE) { + /* + * On SNP, transition the page to shared in the RMP table so that + * it is consistent with the page table attribute change. + * + * __start_bss_decrypted has a virtual address in the high range + * mapping (kernel .text). PVALIDATE, by way of + * early_snp_set_memory_shared(), requires a valid virtual + * address but the kernel is currently running off of the identity + * mapping so use the PA to get a *currently* valid virtual address. + */ + early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD); + + i = pmd_index(paddr - p2v_offset); + pmd[i] -= sme_get_me_mask(); + } + } + + /* + * Return the SME encryption mask (if SME is active) to be used as a + * modifier for the initial pgdir entry programmed into CR3. + */ + return sme_get_me_mask(); +} + +/* Code in __startup_64() can be relocated during execution, but the compiler + * doesn't have to generate PC-relative relocations when accessing globals from + * that function. Clang actually does not generate them, which leads to + * boot-time crashes. To work around this problem, every global pointer must + * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined + * by subtracting p2v_offset from the RIP-relative address. + */ +unsigned long __head __startup_64(unsigned long p2v_offset, + struct boot_params *bp) +{ + pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts); + unsigned long physaddr = (unsigned long)rip_rel_ptr(_text); + unsigned long va_text, va_end; + unsigned long pgtable_flags; + unsigned long load_delta; + pgdval_t *pgd; + p4dval_t *p4d; + pudval_t *pud; + pmdval_t *pmd, pmd_entry; + bool la57; + int i; + + la57 = check_la57_support(); + + /* Is the address too large? */ + if (physaddr >> MAX_PHYSMEM_BITS) + for (;;); + + /* + * Compute the delta between the address I am compiled to run at + * and the address I am actually running at. + */ + load_delta = __START_KERNEL_map + p2v_offset; + RIP_REL_REF(phys_base) = load_delta; + + /* Is the address not 2M aligned? */ + if (load_delta & ~PMD_MASK) + for (;;); + + va_text = physaddr - p2v_offset; + va_end = (unsigned long)rip_rel_ptr(_end) - p2v_offset; + + /* Include the SME encryption mask in the fixup value */ + load_delta += sme_get_me_mask(); + + /* Fixup the physical addresses in the page table */ + + pgd = rip_rel_ptr(early_top_pgt); + pgd[pgd_index(__START_KERNEL_map)] += load_delta; + + if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) { + p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); + p4d[MAX_PTRS_PER_P4D - 1] += load_delta; + + pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; + } + + RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta; + RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta; + + for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) + RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta; + + /* + * Set up the identity mapping for the switchover. These + * entries should *NOT* have the global bit set! This also + * creates a bunch of nonsense entries but that is fine -- + * it avoids problems around wraparound. + */ + + pud = &early_pgts[0]->pmd; + pmd = &early_pgts[1]->pmd; + RIP_REL_REF(next_early_pgt) = 2; + + pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask(); + + if (la57) { + p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd; + + i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; + pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; + pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; + + i = physaddr >> P4D_SHIFT; + p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; + p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; + } else { + i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; + pgd[i + 0] = (pgdval_t)pud + pgtable_flags; + pgd[i + 1] = (pgdval_t)pud + pgtable_flags; + } + + i = physaddr >> PUD_SHIFT; + pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; + pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; + + pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; + /* Filter out unsupported __PAGE_KERNEL_* bits: */ + pmd_entry &= RIP_REL_REF(__supported_pte_mask); + pmd_entry += sme_get_me_mask(); + pmd_entry += physaddr; + + for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) { + int idx = i + (physaddr >> PMD_SHIFT); + + pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE; + } + + /* + * Fixup the kernel text+data virtual addresses. Note that + * we might write invalid pmds, when the kernel is relocated + * cleanup_highmap() fixes this up along with the mappings + * beyond _end. + * + * Only the region occupied by the kernel image has so far + * been checked against the table of usable memory regions + * provided by the firmware, so invalidate pages outside that + * region. A page table entry that maps to a reserved area of + * memory would allow processor speculation into that area, + * and on some hardware (particularly the UV platform) even + * speculative access to some reserved areas is caught as an + * error, causing the BIOS to halt the system. + */ + + pmd = rip_rel_ptr(level2_kernel_pgt); + + /* invalidate pages before the kernel image */ + for (i = 0; i < pmd_index(va_text); i++) + pmd[i] &= ~_PAGE_PRESENT; + + /* fixup pages that are part of the kernel image */ + for (; i <= pmd_index(va_end); i++) + if (pmd[i] & _PAGE_PRESENT) + pmd[i] += load_delta; + + /* invalidate pages after the kernel image */ + for (; i < PTRS_PER_PMD; i++) + pmd[i] &= ~_PAGE_PRESENT; + + return sme_postprocess_startup(bp, pmd, p2v_offset); +} diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 9b2ffec4bbad..6b68a206fa7f 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -47,7 +47,7 @@ * Manage page tables very early on. */ extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; -static unsigned int __initdata next_early_pgt; +unsigned int __initdata next_early_pgt; pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX); #ifdef CONFIG_X86_5LEVEL @@ -67,215 +67,6 @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); #endif -static inline bool check_la57_support(void) -{ - if (!IS_ENABLED(CONFIG_X86_5LEVEL)) - return false; - - /* - * 5-level paging is detected and enabled at kernel decompression - * stage. Only check if it has been enabled there. - */ - if (!(native_read_cr4() & X86_CR4_LA57)) - return false; - - RIP_REL_REF(__pgtable_l5_enabled) = 1; - RIP_REL_REF(pgdir_shift) = 48; - RIP_REL_REF(ptrs_per_p4d) = 512; - RIP_REL_REF(page_offset_base) = __PAGE_OFFSET_BASE_L5; - RIP_REL_REF(vmalloc_base) = __VMALLOC_BASE_L5; - RIP_REL_REF(vmemmap_base) = __VMEMMAP_BASE_L5; - - return true; -} - -static unsigned long __head sme_postprocess_startup(struct boot_params *bp, - pmdval_t *pmd, - unsigned long p2v_offset) -{ - unsigned long paddr, paddr_end; - int i; - - /* Encrypt the kernel and related (if SME is active) */ - sme_encrypt_kernel(bp); - - /* - * Clear the memory encryption mask from the .bss..decrypted section. - * The bss section will be memset to zero later in the initialization so - * there is no need to zero it after changing the memory encryption - * attribute. - */ - if (sme_get_me_mask()) { - paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted); - paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted); - - for (; paddr < paddr_end; paddr += PMD_SIZE) { - /* - * On SNP, transition the page to shared in the RMP table so that - * it is consistent with the page table attribute change. - * - * __start_bss_decrypted has a virtual address in the high range - * mapping (kernel .text). PVALIDATE, by way of - * early_snp_set_memory_shared(), requires a valid virtual - * address but the kernel is currently running off of the identity - * mapping so use the PA to get a *currently* valid virtual address. - */ - early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD); - - i = pmd_index(paddr - p2v_offset); - pmd[i] -= sme_get_me_mask(); - } - } - - /* - * Return the SME encryption mask (if SME is active) to be used as a - * modifier for the initial pgdir entry programmed into CR3. - */ - return sme_get_me_mask(); -} - -/* Code in __startup_64() can be relocated during execution, but the compiler - * doesn't have to generate PC-relative relocations when accessing globals from - * that function. Clang actually does not generate them, which leads to - * boot-time crashes. To work around this problem, every global pointer must - * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined - * by subtracting p2v_offset from the RIP-relative address. - */ -unsigned long __head __startup_64(unsigned long p2v_offset, - struct boot_params *bp) -{ - pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts); - unsigned long physaddr = (unsigned long)rip_rel_ptr(_text); - unsigned long va_text, va_end; - unsigned long pgtable_flags; - unsigned long load_delta; - pgdval_t *pgd; - p4dval_t *p4d; - pudval_t *pud; - pmdval_t *pmd, pmd_entry; - bool la57; - int i; - - la57 = check_la57_support(); - - /* Is the address too large? */ - if (physaddr >> MAX_PHYSMEM_BITS) - for (;;); - - /* - * Compute the delta between the address I am compiled to run at - * and the address I am actually running at. - */ - load_delta = __START_KERNEL_map + p2v_offset; - RIP_REL_REF(phys_base) = load_delta; - - /* Is the address not 2M aligned? */ - if (load_delta & ~PMD_MASK) - for (;;); - - va_text = physaddr - p2v_offset; - va_end = (unsigned long)rip_rel_ptr(_end) - p2v_offset; - - /* Include the SME encryption mask in the fixup value */ - load_delta += sme_get_me_mask(); - - /* Fixup the physical addresses in the page table */ - - pgd = rip_rel_ptr(early_top_pgt); - pgd[pgd_index(__START_KERNEL_map)] += load_delta; - - if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) { - p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); - p4d[MAX_PTRS_PER_P4D - 1] += load_delta; - - pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; - } - - RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta; - RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta; - - for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) - RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta; - - /* - * Set up the identity mapping for the switchover. These - * entries should *NOT* have the global bit set! This also - * creates a bunch of nonsense entries but that is fine -- - * it avoids problems around wraparound. - */ - - pud = &early_pgts[0]->pmd; - pmd = &early_pgts[1]->pmd; - RIP_REL_REF(next_early_pgt) = 2; - - pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask(); - - if (la57) { - p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd; - - i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; - pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; - pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; - - i = physaddr >> P4D_SHIFT; - p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; - p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; - } else { - i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; - pgd[i + 0] = (pgdval_t)pud + pgtable_flags; - pgd[i + 1] = (pgdval_t)pud + pgtable_flags; - } - - i = physaddr >> PUD_SHIFT; - pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; - pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; - - pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; - /* Filter out unsupported __PAGE_KERNEL_* bits: */ - pmd_entry &= RIP_REL_REF(__supported_pte_mask); - pmd_entry += sme_get_me_mask(); - pmd_entry += physaddr; - - for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) { - int idx = i + (physaddr >> PMD_SHIFT); - - pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE; - } - - /* - * Fixup the kernel text+data virtual addresses. Note that - * we might write invalid pmds, when the kernel is relocated - * cleanup_highmap() fixes this up along with the mappings - * beyond _end. - * - * Only the region occupied by the kernel image has so far - * been checked against the table of usable memory regions - * provided by the firmware, so invalidate pages outside that - * region. A page table entry that maps to a reserved area of - * memory would allow processor speculation into that area, - * and on some hardware (particularly the UV platform) even - * speculative access to some reserved areas is caught as an - * error, causing the BIOS to halt the system. - */ - - pmd = rip_rel_ptr(level2_kernel_pgt); - - /* invalidate pages before the kernel image */ - for (i = 0; i < pmd_index(va_text); i++) - pmd[i] &= ~_PAGE_PRESENT; - - /* fixup pages that are part of the kernel image */ - for (; i <= pmd_index(va_end); i++) - if (pmd[i] & _PAGE_PRESENT) - pmd[i] += load_delta; - - /* invalidate pages after the kernel image */ - for (; i < PTRS_PER_PMD; i++) - pmd[i] &= ~_PAGE_PRESENT; - - return sme_postprocess_startup(bp, pmd, p2v_offset); -} - /* Wipe all early page tables except for the kernel symbol map */ static void __init reset_early_page_tables(void) { From dafb26f4271b9cc9cad07d9abf3c71c492e14f4c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:22 +0200 Subject: [PATCH 175/331] x86/boot: Drop RIP_REL_REF() uses from early mapping code Now that __startup_64() is built using -fPIC, RIP_REL_REF() has become a NOP and can be removed. Only some occurrences of rip_rel_ptr() will remain, to explicitly take the address of certain global structures in the 1:1 mapping of memory. While at it, update the code comment to describe why this is needed. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-17-ardb+git@google.com --- arch/x86/boot/startup/map_kernel.c | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c index 5f1b7e0ba26e..0eac3f17dbd3 100644 --- a/arch/x86/boot/startup/map_kernel.c +++ b/arch/x86/boot/startup/map_kernel.c @@ -26,12 +26,12 @@ static inline bool check_la57_support(void) if (!(native_read_cr4() & X86_CR4_LA57)) return false; - RIP_REL_REF(__pgtable_l5_enabled) = 1; - RIP_REL_REF(pgdir_shift) = 48; - RIP_REL_REF(ptrs_per_p4d) = 512; - RIP_REL_REF(page_offset_base) = __PAGE_OFFSET_BASE_L5; - RIP_REL_REF(vmalloc_base) = __VMALLOC_BASE_L5; - RIP_REL_REF(vmemmap_base) = __VMEMMAP_BASE_L5; + __pgtable_l5_enabled = 1; + pgdir_shift = 48; + ptrs_per_p4d = 512; + page_offset_base = __PAGE_OFFSET_BASE_L5; + vmalloc_base = __VMALLOC_BASE_L5; + vmemmap_base = __VMEMMAP_BASE_L5; return true; } @@ -81,12 +81,14 @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp, return sme_get_me_mask(); } -/* Code in __startup_64() can be relocated during execution, but the compiler - * doesn't have to generate PC-relative relocations when accessing globals from - * that function. Clang actually does not generate them, which leads to - * boot-time crashes. To work around this problem, every global pointer must - * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined - * by subtracting p2v_offset from the RIP-relative address. +/* + * This code is compiled using PIC codegen because it will execute from the + * early 1:1 mapping of memory, which deviates from the mapping expected by the + * linker. Due to this deviation, taking the address of a global variable will + * produce an ambiguous result when using the plain & operator. Instead, + * rip_rel_ptr() must be used, which will return the RIP-relative address in + * the 1:1 mapping of memory. Kernel virtual addresses can be determined by + * subtracting p2v_offset from the RIP-relative address. */ unsigned long __head __startup_64(unsigned long p2v_offset, struct boot_params *bp) @@ -113,8 +115,7 @@ unsigned long __head __startup_64(unsigned long p2v_offset, * Compute the delta between the address I am compiled to run at * and the address I am actually running at. */ - load_delta = __START_KERNEL_map + p2v_offset; - RIP_REL_REF(phys_base) = load_delta; + phys_base = load_delta = __START_KERNEL_map + p2v_offset; /* Is the address not 2M aligned? */ if (load_delta & ~PMD_MASK) @@ -138,11 +139,11 @@ unsigned long __head __startup_64(unsigned long p2v_offset, pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; } - RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta; - RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta; + level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta; + level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta; for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) - RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta; + level2_fixmap_pgt[i].pmd += load_delta; /* * Set up the identity mapping for the switchover. These @@ -153,12 +154,12 @@ unsigned long __head __startup_64(unsigned long p2v_offset, pud = &early_pgts[0]->pmd; pmd = &early_pgts[1]->pmd; - RIP_REL_REF(next_early_pgt) = 2; + next_early_pgt = 2; pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask(); if (la57) { - p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd; + p4d = &early_pgts[next_early_pgt++]->pmd; i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; @@ -179,7 +180,7 @@ unsigned long __head __startup_64(unsigned long p2v_offset, pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; /* Filter out unsupported __PAGE_KERNEL_* bits: */ - pmd_entry &= RIP_REL_REF(__supported_pte_mask); + pmd_entry &= __supported_pte_mask; pmd_entry += sme_get_me_mask(); pmd_entry += physaddr; From 7ae089ee75f3c77b94ff23204b41ea06da9f0193 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:23 +0200 Subject: [PATCH 176/331] x86/boot: Move early SME init code into startup/ Move the SME initialization code, which runs from the 1:1 mapping of memory as it operates on the kernel virtual mapping, into the new sub-directory arch/x86/boot/startup/ where all startup code will reside that needs to tolerate executing from the 1:1 mapping. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-18-ardb+git@google.com --- arch/x86/boot/startup/Makefile | 1 + arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 2 -- arch/x86/mm/Makefile | 6 ------ 3 files changed, 1 insertion(+), 8 deletions(-) rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (99%) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 10319aee666b..ccdfc42a4d59 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -16,6 +16,7 @@ UBSAN_SANITIZE := n KCOV_INSTRUMENT := n obj-$(CONFIG_X86_64) += gdt_idt.o map_kernel.o +obj-$(CONFIG_AMD_MEM_ENCRYPT) += sme.o lib-$(CONFIG_X86_64) += la57toggle.o lib-$(CONFIG_EFI_MIXED) += efi-mixed.o diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/boot/startup/sme.c similarity index 99% rename from arch/x86/mm/mem_encrypt_identity.c rename to arch/x86/boot/startup/sme.c index e7fb3779b35f..23d10cda5b58 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/boot/startup/sme.c @@ -45,8 +45,6 @@ #include #include -#include "mm_internal.h" - #define PGD_FLAGS _KERNPG_TABLE_NOENC #define P4D_FLAGS _KERNPG_TABLE_NOENC #define PUD_FLAGS _KERNPG_TABLE_NOENC diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 32035d5be5a0..3faa60f13a61 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -3,12 +3,10 @@ KCOV_INSTRUMENT_tlb.o := n KCOV_INSTRUMENT_mem_encrypt.o := n KCOV_INSTRUMENT_mem_encrypt_amd.o := n -KCOV_INSTRUMENT_mem_encrypt_identity.o := n KCOV_INSTRUMENT_pgprot.o := n KASAN_SANITIZE_mem_encrypt.o := n KASAN_SANITIZE_mem_encrypt_amd.o := n -KASAN_SANITIZE_mem_encrypt_identity.o := n KASAN_SANITIZE_pgprot.o := n # Disable KCSAN entirely, because otherwise we get warnings that some functions @@ -16,12 +14,10 @@ KASAN_SANITIZE_pgprot.o := n KCSAN_SANITIZE := n # Avoid recursion by not calling KMSAN hooks for CEA code. KMSAN_SANITIZE_cpu_entry_area.o := n -KMSAN_SANITIZE_mem_encrypt_identity.o := n ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_mem_encrypt.o = -pg CFLAGS_REMOVE_mem_encrypt_amd.o = -pg -CFLAGS_REMOVE_mem_encrypt_identity.o = -pg CFLAGS_REMOVE_pgprot.o = -pg endif @@ -32,7 +28,6 @@ obj-y += pat/ # Make sure __phys_addr has no stackprotector CFLAGS_physaddr.o := -fno-stack-protector -CFLAGS_mem_encrypt_identity.o := -fno-stack-protector CFLAGS_fault.o := -I $(src)/../include/asm/trace @@ -63,5 +58,4 @@ obj-$(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION) += pti.o obj-$(CONFIG_X86_MEM_ENCRYPT) += mem_encrypt.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_amd.o -obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_identity.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o From bee174b27e54462ef18b38f8377d27ac0ad14350 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:24 +0200 Subject: [PATCH 177/331] x86/boot: Drop RIP_REL_REF() uses from SME startup code RIP_REL_REF() has no effect on code residing in arch/x86/boot/startup, as it is built with -fPIC. So remove any occurrences from the SME startup code. Note the SME is the only caller of cc_set_mask() that requires this, so drop it from there as well. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-19-ardb+git@google.com --- arch/x86/boot/startup/sme.c | 11 +++++------ arch/x86/include/asm/coco.h | 2 +- arch/x86/include/asm/mem_encrypt.h | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c index 23d10cda5b58..5738b31c8e60 100644 --- a/arch/x86/boot/startup/sme.c +++ b/arch/x86/boot/startup/sme.c @@ -297,8 +297,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp) * instrumentation or checking boot_cpu_data in the cc_platform_has() * function. */ - if (!sme_get_me_mask() || - RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED) + if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED) return; /* @@ -524,7 +523,7 @@ void __head sme_enable(struct boot_params *bp) me_mask = 1UL << (ebx & 0x3f); /* Check the SEV MSR whether SEV or SME is enabled */ - RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV); + sev_status = msr = __rdmsr(MSR_AMD64_SEV); feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT; /* @@ -560,8 +559,8 @@ void __head sme_enable(struct boot_params *bp) return; } - RIP_REL_REF(sme_me_mask) = me_mask; - RIP_REL_REF(physical_mask) &= ~me_mask; - RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD; + sme_me_mask = me_mask; + physical_mask &= ~me_mask; + cc_vendor = CC_VENDOR_AMD; cc_set_mask(me_mask); } diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h index e7225452963f..e1dbf8df1b69 100644 --- a/arch/x86/include/asm/coco.h +++ b/arch/x86/include/asm/coco.h @@ -22,7 +22,7 @@ static inline u64 cc_get_mask(void) static inline void cc_set_mask(u64 mask) { - RIP_REL_REF(cc_mask) = mask; + cc_mask = mask; } u64 cc_mkenc(u64 val); diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 1530ee301dfe..ea6494628cb0 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -61,7 +61,7 @@ void __init sev_es_init_vc_handling(void); static inline u64 sme_get_me_mask(void) { - return RIP_REL_REF(sme_me_mask); + return sme_me_mask; } #define __bss_decrypted __section(".bss..decrypted") From 221df25fdf827b1fe5b904c6a396af06461a32f6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Apr 2025 15:41:25 +0200 Subject: [PATCH 178/331] x86/sev: Prepare for splitting off early SEV code Prepare for splitting off parts of the SEV core.c source file into a file that carries code that must tolerate being called from the early 1:1 mapping. This will allow special build-time handling of thise code, to ensure that it gets generated in a way that is compatible with the early execution context. So create a de-facto internal SEV API and put the definitions into sev-internal.h. No attempt is made to allow this header file to be included in arbitrary other sources - this is explicitly not the intent. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Linus Torvalds Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250410134117.3713574-20-ardb+git@google.com --- arch/x86/boot/compressed/sev.c | 15 ++-- arch/x86/coco/sev/core.c | 108 +++--------------------- arch/x86/coco/sev/shared.c | 64 +++------------ arch/x86/include/asm/sev-internal.h | 122 ++++++++++++++++++++++++++++ arch/x86/include/asm/sev.h | 37 +++++++++ 5 files changed, 194 insertions(+), 152 deletions(-) create mode 100644 arch/x86/include/asm/sev-internal.h diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index bb55934c1cee..6eadd790f4e5 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -126,20 +126,25 @@ static bool fault_in_kernel_space(unsigned long address) #include "../../lib/inat.c" #include "../../lib/insn.c" -/* Include code for early handlers */ -#include "../../coco/sev/shared.c" +extern struct svsm_ca *boot_svsm_caa; +extern u64 boot_svsm_caa_pa; -static struct svsm_ca *svsm_get_caa(void) +struct svsm_ca *svsm_get_caa(void) { return boot_svsm_caa; } -static u64 svsm_get_caa_pa(void) +u64 svsm_get_caa_pa(void) { return boot_svsm_caa_pa; } -static int svsm_perform_call_protocol(struct svsm_call *call) +int svsm_perform_call_protocol(struct svsm_call *call); + +/* Include code for early handlers */ +#include "../../coco/sev/shared.c" + +int svsm_perform_call_protocol(struct svsm_call *call) { struct ghcb *ghcb; int ret; diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 832f7a7b10b2..aeb7731862c0 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -44,8 +45,6 @@ #include #include -#define DR7_RESET_VALUE 0x400 - /* AP INIT values as documented in the APM2 section "Processor Initialization State" */ #define AP_INIT_CS_LIMIT 0xffff #define AP_INIT_DS_LIMIT 0xffff @@ -82,16 +81,16 @@ static const char * const sev_status_feat_names[] = { }; /* For early boot hypervisor communication in SEV-ES enabled guests */ -static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); +struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); /* * Needs to be in the .data section because we need it NULL before bss is * cleared */ -static struct ghcb *boot_ghcb __section(".data"); +struct ghcb *boot_ghcb __section(".data"); /* Bitmap of SEV features supported by the hypervisor */ -static u64 sev_hv_features __ro_after_init; +u64 sev_hv_features __ro_after_init; /* Secrets page physical address from the CC blob */ static u64 secrets_pa __ro_after_init; @@ -105,54 +104,14 @@ static u64 snp_tsc_scale __ro_after_init; static u64 snp_tsc_offset __ro_after_init; static u64 snp_tsc_freq_khz __ro_after_init; -/* #VC handler runtime per-CPU data */ -struct sev_es_runtime_data { - struct ghcb ghcb_page; - - /* - * Reserve one page per CPU as backup storage for the unencrypted GHCB. - * It is needed when an NMI happens while the #VC handler uses the real - * GHCB, and the NMI handler itself is causing another #VC exception. In - * that case the GHCB content of the first handler needs to be backed up - * and restored. - */ - struct ghcb backup_ghcb; - - /* - * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. - * There is no need for it to be atomic, because nothing is written to - * the GHCB between the read and the write of ghcb_active. So it is safe - * to use it when a nested #VC exception happens before the write. - * - * This is necessary for example in the #VC->NMI->#VC case when the NMI - * happens while the first #VC handler uses the GHCB. When the NMI code - * raises a second #VC handler it might overwrite the contents of the - * GHCB written by the first handler. To avoid this the content of the - * GHCB is saved and restored when the GHCB is detected to be in use - * already. - */ - bool ghcb_active; - bool backup_ghcb_active; - - /* - * Cached DR7 value - write it on DR7 writes and return it on reads. - * That value will never make it to the real hardware DR7 as debugging - * is currently unsupported in SEV-ES guests. - */ - unsigned long dr7; -}; - -struct ghcb_state { - struct ghcb *ghcb; -}; /* For early boot SVSM communication */ -static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); +struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); -static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); -static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); -static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); -static DEFINE_PER_CPU(u64, svsm_caa_pa); +DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); +DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); +DEFINE_PER_CPU(u64, svsm_caa_pa); static __always_inline bool on_vc_stack(struct pt_regs *regs) { @@ -231,7 +190,7 @@ void noinstr __sev_es_ist_exit(void) * * Callers must disable local interrupts around it. */ -static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) +noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) { struct sev_es_runtime_data *data; struct ghcb *ghcb; @@ -274,21 +233,6 @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) return ghcb; } -static inline u64 sev_es_rd_ghcb_msr(void) -{ - return __rdmsr(MSR_AMD64_SEV_ES_GHCB); -} - -static __always_inline void sev_es_wr_ghcb_msr(u64 val) -{ - u32 low, high; - - low = (u32)(val); - high = (u32)(val >> 32); - - native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); -} - static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, unsigned char *buffer) { @@ -601,33 +545,7 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) /* Include code shared with pre-decompression boot stage */ #include "shared.c" -static inline struct svsm_ca *svsm_get_caa(void) -{ - /* - * Use rIP-relative references when called early in the boot. If - * ->use_cas is set, then it is late in the boot and no need - * to worry about rIP-relative references. - */ - if (RIP_REL_REF(sev_cfg).use_cas) - return this_cpu_read(svsm_caa); - else - return RIP_REL_REF(boot_svsm_caa); -} - -static u64 svsm_get_caa_pa(void) -{ - /* - * Use rIP-relative references when called early in the boot. If - * ->use_cas is set, then it is late in the boot and no need - * to worry about rIP-relative references. - */ - if (RIP_REL_REF(sev_cfg).use_cas) - return this_cpu_read(svsm_caa_pa); - else - return RIP_REL_REF(boot_svsm_caa_pa); -} - -static noinstr void __sev_put_ghcb(struct ghcb_state *state) +noinstr void __sev_put_ghcb(struct ghcb_state *state) { struct sev_es_runtime_data *data; struct ghcb *ghcb; @@ -652,7 +570,7 @@ static noinstr void __sev_put_ghcb(struct ghcb_state *state) } } -static int svsm_perform_call_protocol(struct svsm_call *call) +int svsm_perform_call_protocol(struct svsm_call *call) { struct ghcb_state state; unsigned long flags; @@ -761,7 +679,7 @@ static u64 __init get_jump_table_addr(void) return ret; } -static void __head +void __head early_set_pages_state(unsigned long vaddr, unsigned long paddr, unsigned long npages, enum psc_op op) { diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c index 04982d356803..a7c94020e384 100644 --- a/arch/x86/coco/sev/shared.c +++ b/arch/x86/coco/sev/shared.c @@ -38,12 +38,8 @@ */ u8 snp_vmpl __ro_after_init; EXPORT_SYMBOL_GPL(snp_vmpl); -static struct svsm_ca *boot_svsm_caa __ro_after_init; -static u64 boot_svsm_caa_pa __ro_after_init; - -static struct svsm_ca *svsm_get_caa(void); -static u64 svsm_get_caa_pa(void); -static int svsm_perform_call_protocol(struct svsm_call *call); +struct svsm_ca *boot_svsm_caa __ro_after_init; +u64 boot_svsm_caa_pa __ro_after_init; /* I/O parameters for CPUID-related helpers */ struct cpuid_leaf { @@ -55,36 +51,6 @@ struct cpuid_leaf { u32 edx; }; -/* - * Individual entries of the SNP CPUID table, as defined by the SNP - * Firmware ABI, Revision 0.9, Section 7.1, Table 14. - */ -struct snp_cpuid_fn { - u32 eax_in; - u32 ecx_in; - u64 xcr0_in; - u64 xss_in; - u32 eax; - u32 ebx; - u32 ecx; - u32 edx; - u64 __reserved; -} __packed; - -/* - * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, - * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit - * of 64 entries per CPUID table. - */ -#define SNP_CPUID_COUNT_MAX 64 - -struct snp_cpuid_table { - u32 count; - u32 __reserved1; - u64 __reserved2; - struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; -} __packed; - /* * Since feature negotiation related variables are set early in the boot * process they must reside in the .data section so as not to be zeroed @@ -107,7 +73,7 @@ static u32 cpuid_std_range_max __ro_after_init; static u32 cpuid_hyp_range_max __ro_after_init; static u32 cpuid_ext_range_max __ro_after_init; -static bool __init sev_es_check_cpu_features(void) +bool __init sev_es_check_cpu_features(void) { if (!has_cpuflag(X86_FEATURE_RDRAND)) { error("RDRAND instruction not supported - no trusted source of randomness available\n"); @@ -117,7 +83,7 @@ static bool __init sev_es_check_cpu_features(void) return true; } -static void __head __noreturn +void __head __noreturn sev_es_terminate(unsigned int set, unsigned int reason) { u64 val = GHCB_MSR_TERM_REQ; @@ -136,7 +102,7 @@ sev_es_terminate(unsigned int set, unsigned int reason) /* * The hypervisor features are available from GHCB version 2 onward. */ -static u64 get_hv_features(void) +u64 get_hv_features(void) { u64 val; @@ -153,7 +119,7 @@ static u64 get_hv_features(void) return GHCB_MSR_HV_FT_RESP_VAL(val); } -static void snp_register_ghcb_early(unsigned long paddr) +void snp_register_ghcb_early(unsigned long paddr) { unsigned long pfn = paddr >> PAGE_SHIFT; u64 val; @@ -169,7 +135,7 @@ static void snp_register_ghcb_early(unsigned long paddr) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); } -static bool sev_es_negotiate_protocol(void) +bool sev_es_negotiate_protocol(void) { u64 val; @@ -190,12 +156,6 @@ static bool sev_es_negotiate_protocol(void) return true; } -static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) -{ - ghcb->save.sw_exit_code = 0; - __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); -} - static bool vc_decoding_needed(unsigned long exit_code) { /* Exceptions don't require to decode the instruction */ @@ -371,10 +331,10 @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) return svsm_process_result_codes(call); } -static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, - struct es_em_ctxt *ctxt, - u64 exit_code, u64 exit_info_1, - u64 exit_info_2) +enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2) { /* Fill in protocol and format specifiers */ ghcb->protocol_version = ghcb_version; @@ -473,7 +433,7 @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid * while running with the initial identity mapping as well as the * switch-over to kernel virtual addresses later. */ -static const struct snp_cpuid_table *snp_cpuid_get_table(void) +const struct snp_cpuid_table *snp_cpuid_get_table(void) { return rip_rel_ptr(&cpuid_table_copy); } diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h new file mode 100644 index 000000000000..73cb774c3639 --- /dev/null +++ b/arch/x86/include/asm/sev-internal.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#define DR7_RESET_VALUE 0x400 + +extern struct ghcb boot_ghcb_page; +extern struct ghcb *boot_ghcb; +extern u64 sev_hv_features; + +/* #VC handler runtime per-CPU data */ +struct sev_es_runtime_data { + struct ghcb ghcb_page; + + /* + * Reserve one page per CPU as backup storage for the unencrypted GHCB. + * It is needed when an NMI happens while the #VC handler uses the real + * GHCB, and the NMI handler itself is causing another #VC exception. In + * that case the GHCB content of the first handler needs to be backed up + * and restored. + */ + struct ghcb backup_ghcb; + + /* + * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. + * There is no need for it to be atomic, because nothing is written to + * the GHCB between the read and the write of ghcb_active. So it is safe + * to use it when a nested #VC exception happens before the write. + * + * This is necessary for example in the #VC->NMI->#VC case when the NMI + * happens while the first #VC handler uses the GHCB. When the NMI code + * raises a second #VC handler it might overwrite the contents of the + * GHCB written by the first handler. To avoid this the content of the + * GHCB is saved and restored when the GHCB is detected to be in use + * already. + */ + bool ghcb_active; + bool backup_ghcb_active; + + /* + * Cached DR7 value - write it on DR7 writes and return it on reads. + * That value will never make it to the real hardware DR7 as debugging + * is currently unsupported in SEV-ES guests. + */ + unsigned long dr7; +}; + +struct ghcb_state { + struct ghcb *ghcb; +}; + +extern struct svsm_ca boot_svsm_ca_page; + +struct ghcb *__sev_get_ghcb(struct ghcb_state *state); +void __sev_put_ghcb(struct ghcb_state *state); + +DECLARE_PER_CPU(struct sev_es_runtime_data*, runtime_data); +DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa); + +void early_set_pages_state(unsigned long vaddr, unsigned long paddr, + unsigned long npages, enum psc_op op); + +void __noreturn sev_es_terminate(unsigned int set, unsigned int reason); + +DECLARE_PER_CPU(struct svsm_ca *, svsm_caa); +DECLARE_PER_CPU(u64, svsm_caa_pa); + +extern struct svsm_ca *boot_svsm_caa; +extern u64 boot_svsm_caa_pa; + +static __always_inline struct svsm_ca *svsm_get_caa(void) +{ + /* + * Use rIP-relative references when called early in the boot. If + * ->use_cas is set, then it is late in the boot and no need + * to worry about rIP-relative references. + */ + if (RIP_REL_REF(sev_cfg).use_cas) + return this_cpu_read(svsm_caa); + else + return RIP_REL_REF(boot_svsm_caa); +} + +static __always_inline u64 svsm_get_caa_pa(void) +{ + /* + * Use rIP-relative references when called early in the boot. If + * ->use_cas is set, then it is late in the boot and no need + * to worry about rIP-relative references. + */ + if (RIP_REL_REF(sev_cfg).use_cas) + return this_cpu_read(svsm_caa_pa); + else + return RIP_REL_REF(boot_svsm_caa_pa); +} + +int svsm_perform_call_protocol(struct svsm_call *call); + +static inline u64 sev_es_rd_ghcb_msr(void) +{ + return __rdmsr(MSR_AMD64_SEV_ES_GHCB); +} + +static __always_inline void sev_es_wr_ghcb_msr(u64 val) +{ + u32 low, high; + + low = (u32)(val); + high = (u32)(val >> 32); + + native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); +} + +enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2); + +void snp_register_ghcb_early(unsigned long paddr); +bool sev_es_negotiate_protocol(void); +bool sev_es_check_cpu_features(void); +u64 get_hv_features(void); + +const struct snp_cpuid_table *snp_cpuid_get_table(void); diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index ba7999f66abe..a8661dfc9a9a 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -15,6 +15,7 @@ #include #include #include +#include #define GHCB_PROTOCOL_MIN 1ULL #define GHCB_PROTOCOL_MAX 2ULL @@ -83,6 +84,36 @@ extern void vc_no_ghcb(void); extern void vc_boot_ghcb(void); extern bool handle_vc_boot_ghcb(struct pt_regs *regs); +/* + * Individual entries of the SNP CPUID table, as defined by the SNP + * Firmware ABI, Revision 0.9, Section 7.1, Table 14. + */ +struct snp_cpuid_fn { + u32 eax_in; + u32 ecx_in; + u64 xcr0_in; + u64 xss_in; + u32 eax; + u32 ebx; + u32 ecx; + u32 edx; + u64 __reserved; +} __packed; + +/* + * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, + * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit + * of 64 entries per CPUID table. + */ +#define SNP_CPUID_COUNT_MAX 64 + +struct snp_cpuid_table { + u32 count; + u32 __reserved1; + u64 __reserved2; + struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; +} __packed; + /* PVALIDATE return codes */ #define PVALIDATE_FAIL_SIZEMISMATCH 6 @@ -484,6 +515,12 @@ int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req void __init snp_secure_tsc_prepare(void); void __init snp_secure_tsc_init(void); +static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) +{ + ghcb->save.sw_exit_code = 0; + __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); +} + #else /* !CONFIG_AMD_MEM_ENCRYPT */ #define snp_vmpl 0 From d51faee4bd63b3b02fa6f56210ec0d84c5ccb680 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 11 Apr 2025 11:31:17 +0200 Subject: [PATCH 179/331] x86/percpu: Refer __percpu_prefix to __force_percpu_prefix Refer __percpu_prefix to __force_percpu_prefix to avoid duplicate definition. While there, slightly reorder definitions to a more logical sequence, remove unneeded double quotes and move misplaced comment to the right place. No functional changes intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Link: https://lore.kernel.org/r/20250411093130.81389-1-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 5fe314a2e73e..b0d03b6c279b 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -29,6 +29,8 @@ #ifdef CONFIG_SMP +#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":" + #ifdef CONFIG_CC_HAS_NAMED_AS #ifdef __CHECKER__ @@ -36,23 +38,23 @@ # define __seg_fs __attribute__((address_space(__seg_fs))) #endif +#define __percpu_prefix #define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) -#define __percpu_prefix "" #else /* !CONFIG_CC_HAS_NAMED_AS: */ +#define __percpu_prefix __force_percpu_prefix #define __percpu_seg_override -#define __percpu_prefix "%%"__stringify(__percpu_seg)":" #endif /* CONFIG_CC_HAS_NAMED_AS */ -#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":" -#define __my_cpu_offset this_cpu_read(this_cpu_off) - /* * Compared to the generic __my_cpu_offset version, the following * saves one instruction and avoids clobbering a temp register. - * + */ +#define __my_cpu_offset this_cpu_read(this_cpu_off) + +/* * arch_raw_cpu_ptr should not be used in 32-bit VDSO for a 64-bit * kernel, because games are played with CONFIG_X86_64 there and * sizeof(this_cpu_off) becames 4. @@ -77,9 +79,9 @@ #else /* !CONFIG_SMP: */ +#define __force_percpu_prefix +#define __percpu_prefix #define __percpu_seg_override -#define __percpu_prefix "" -#define __force_percpu_prefix "" #define PER_CPU_VAR(var) (var)__percpu_rel @@ -97,8 +99,8 @@ # define __my_cpu_var(var) (*__my_cpu_ptr(&(var))) #endif -#define __percpu_arg(x) __percpu_prefix "%" #x #define __force_percpu_arg(x) __force_percpu_prefix "%" #x +#define __percpu_arg(x) __percpu_prefix "%" #x /* * For arch-specific code, we can use direct single-insn ops (they From 7b3169dfa4ba1b1898d4bdfad6ddabbc9e8a57ed Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Thu, 10 Apr 2025 13:58:35 -0700 Subject: [PATCH 180/331] objtool, x86/hweight: Remove ANNOTATE_IGNORE_ALTERNATIVE Since objtool's inception, frame pointer warnings have been manually silenced for __arch_hweight*() to allow those functions' inline asm to avoid using ASM_CALL_CONSTRAINT. The potentially dubious reasoning for that decision over nine years ago was that since !X86_FEATURE_POPCNT is exceedingly rare, it's not worth hurting the code layout for a function call that will never happen on the vast majority of systems. However, those functions actually started using ASM_CALL_CONSTRAINT with the following commit: 194a613088a8 ("x86/hweight: Use ASM_CALL_CONSTRAINT in inline asm()") And rightfully so, as it makes the code correct. ASM_CALL_CONSTRAINT will soon have no effect for non-FP configs anyway. With ASM_CALL_CONSTRAINT in place, ANNOTATE_IGNORE_ALTERNATIVE no longer has a purpose for the hweight functions. Remove it. Signed-off-by: Josh Poimboeuf Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Link: https://lore.kernel.org/r/e7070dba3278c90f1a836b16157dcd34ccd21e21.1744318586.git.jpoimboe@kernel.org --- arch/x86/include/asm/arch_hweight.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h index cbc6157f0b4b..b5982b94bdba 100644 --- a/arch/x86/include/asm/arch_hweight.h +++ b/arch/x86/include/asm/arch_hweight.h @@ -16,8 +16,7 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { unsigned int res; - asm_inline (ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE - "call __sw_hweight32", + asm_inline (ALTERNATIVE("call __sw_hweight32", "popcntl %[val], %[cnt]", X86_FEATURE_POPCNT) : [cnt] "=" REG_OUT (res), ASM_CALL_CONSTRAINT : [val] REG_IN (w)); @@ -46,8 +45,7 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) { unsigned long res; - asm_inline (ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE - "call __sw_hweight64", + asm_inline (ALTERNATIVE("call __sw_hweight64", "popcntq %[val], %[cnt]", X86_FEATURE_POPCNT) : [cnt] "=" REG_OUT (res), ASM_CALL_CONSTRAINT : [val] REG_IN (w)); From a5447e92e169dafaf02fd653500105c7186d7128 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 20:43:16 +0200 Subject: [PATCH 181/331] x86/msr: Add compatibility wrappers for rdmsrl()/wrmsrl() To reduce the impact of the API renames in -next, add compatibility wrappers for the two most popular MSR access APIs: rdmsrl() and wrmsrl(). Signed-off-by: Ingo Molnar Cc: "H. Peter Anvin" Cc: Peter Zijlstra (Intel) Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Cc: Linus Torvalds Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/msr.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 20deb58308e5..2ccc78ebc3d7 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -396,5 +396,10 @@ static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) return wrmsr_safe_regs(regs); } #endif /* CONFIG_SMP */ + +/* Compatibility wrappers: */ +#define rdmsrl(msr, val) rdmsrq(msr, val) +#define wrmsrl(msr, val) wrmsrq(msr, val) + #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_MSR_H */ From 4850074ff06fce894a9946c5d3ab8ea13fa33e43 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 7 Apr 2025 09:21:04 +0200 Subject: [PATCH 182/331] x86/uaccess: Use asm_inline() instead of asm() in __untagged_addr() Use asm_inline() to instruct the compiler that the size of asm() is the minimum size of one instruction, ignoring how many instructions the compiler thinks it is. ALTERNATIVE macro that expands to several pseudo directives causes instruction length estimate to count more than 20 instructions. bloat-o-meter reports minimal code size increase (x86_64 defconfig with CONFIG_ADDRESS_MASKING, gcc-14.2.1): add/remove: 2/2 grow/shrink: 5/1 up/down: 2365/-1995 (370) Function old new delta ----------------------------------------------------- do_get_mempolicy - 1449 +1449 copy_nodes_to_user - 226 +226 __x64_sys_get_mempolicy 35 213 +178 syscall_user_dispatch_set_config 157 332 +175 __ia32_sys_get_mempolicy 31 206 +175 set_syscall_user_dispatch 29 181 +152 __do_sys_mremap 2073 2083 +10 sp_insert 133 117 -16 task_set_syscall_user_dispatch 172 - -172 kernel_get_mempolicy 1807 - -1807 Total: Before=21423151, After=21423521, chg +0.00% The code size increase is due to the compiler inlining more functions that inline untagged_addr(), e.g: task_set_syscall_user_dispatch() is now fully inlined in set_syscall_user_dispatch(): 000000000010b7e0 : 10b7e0: f3 0f 1e fa endbr64 10b7e4: 49 89 c8 mov %rcx,%r8 10b7e7: 48 89 d1 mov %rdx,%rcx 10b7ea: 48 89 f2 mov %rsi,%rdx 10b7ed: 48 89 fe mov %rdi,%rsi 10b7f0: 65 48 8b 3d 00 00 00 mov %gs:0x0(%rip),%rdi 10b7f7: 00 10b7f8: e9 03 fe ff ff jmp 10b600 that after inlining becomes: 000000000010b730 : 10b730: f3 0f 1e fa endbr64 10b734: 65 48 8b 05 00 00 00 mov %gs:0x0(%rip),%rax 10b73b: 00 10b73c: 48 85 ff test %rdi,%rdi 10b73f: 74 54 je 10b795 10b741: 48 83 ff 01 cmp $0x1,%rdi 10b745: 74 06 je 10b74d 10b747: b8 ea ff ff ff mov $0xffffffea,%eax 10b74c: c3 ret 10b74d: 48 85 f6 test %rsi,%rsi 10b750: 75 7b jne 10b7cd 10b752: 48 85 c9 test %rcx,%rcx 10b755: 74 1a je 10b771 10b757: 48 89 cf mov %rcx,%rdi 10b75a: 49 b8 ef cd ab 89 67 movabs $0x123456789abcdef,%r8 10b761: 45 23 01 10b764: 90 nop 10b765: 90 nop 10b766: 90 nop 10b767: 90 nop 10b768: 90 nop 10b769: 90 nop 10b76a: 90 nop 10b76b: 90 nop 10b76c: 49 39 f8 cmp %rdi,%r8 10b76f: 72 6e jb 10b7df 10b771: 48 89 88 48 08 00 00 mov %rcx,0x848(%rax) 10b778: 48 89 b0 50 08 00 00 mov %rsi,0x850(%rax) 10b77f: 48 89 90 58 08 00 00 mov %rdx,0x858(%rax) 10b786: c6 80 60 08 00 00 00 movb $0x0,0x860(%rax) 10b78d: f0 80 48 08 20 lock orb $0x20,0x8(%rax) 10b792: 31 c0 xor %eax,%eax 10b794: c3 ret 10b795: 48 09 d1 or %rdx,%rcx 10b798: 48 09 f1 or %rsi,%rcx 10b79b: 75 aa jne 10b747 10b79d: 48 c7 80 48 08 00 00 movq $0x0,0x848(%rax) 10b7a4: 00 00 00 00 10b7a8: 48 c7 80 50 08 00 00 movq $0x0,0x850(%rax) 10b7af: 00 00 00 00 10b7b3: 48 c7 80 58 08 00 00 movq $0x0,0x858(%rax) 10b7ba: 00 00 00 00 10b7be: c6 80 60 08 00 00 00 movb $0x0,0x860(%rax) 10b7c5: f0 80 60 08 df lock andb $0xdf,0x8(%rax) 10b7ca: 31 c0 xor %eax,%eax 10b7cc: c3 ret 10b7cd: 48 8d 3c 16 lea (%rsi,%rdx,1),%rdi 10b7d1: 48 39 fe cmp %rdi,%rsi 10b7d4: 0f 82 78 ff ff ff jb 10b752 10b7da: e9 68 ff ff ff jmp 10b747 10b7df: b8 f2 ff ff ff mov $0xfffffff2,%eax 10b7e4: c3 ret Please note a series of NOPs that get replaced with an alternative: 11f0: 65 48 23 05 00 00 00 and %gs:0x0(%rip),%rax 11f7: 00 Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250407072129.33440-1-ubizjak@gmail.com --- arch/x86/include/asm/uaccess_64.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 4c13883371aa..c8a5ae35c871 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -26,8 +26,8 @@ extern unsigned long USER_PTR_MAX; */ static inline unsigned long __untagged_addr(unsigned long addr) { - asm (ALTERNATIVE("", - "and " __percpu_arg([mask]) ", %[addr]", X86_FEATURE_LAM) + asm_inline (ALTERNATIVE("", "and " __percpu_arg([mask]) ", %[addr]", + X86_FEATURE_LAM) : [addr] "+r" (addr) : [mask] "m" (__my_cpu_var(tlbstate_untag_mask))); From 031b33ef1a6a1129a1a02a16b89608ded2eff9be Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Thu, 20 Mar 2025 16:42:52 -0700 Subject: [PATCH 183/331] x86/fpu/xstate: Remove xstate offset check Traditionally, new xstate components have been assigned sequentially, aligning feature numbers with their offsets in the XSAVE buffer. However, this ordering is not architecturally mandated in the non-compacted format, where a component's offset may not correspond to its feature number. The kernel caches CPUID-reported xstate component details, including size and offset in the non-compacted format. As part of this process, a sanity check is also conducted to ensure alignment between feature numbers and offsets. This check was likely intended as a general guideline rather than a strict requirement. Upcoming changes will support out-of-order offsets. Remove the check as becoming obsolete. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250320234301.8342-2-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 6a41d1610d8b..542c6981180d 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -216,9 +216,6 @@ static bool xfeature_enabled(enum xfeature xfeature) static void __init setup_xstate_cache(void) { u32 eax, ebx, ecx, edx, i; - /* start at the beginning of the "extended state" */ - unsigned int last_good_offset = offsetof(struct xregs_state, - extended_state_area); /* * The FP xstates and SSE xstates are legacy states. They are always * in the fixed offsets in the xsave area in either compacted form @@ -246,16 +243,6 @@ static void __init setup_xstate_cache(void) continue; xstate_offsets[i] = ebx; - - /* - * In our xstate size checks, we assume that the highest-numbered - * xstate feature has the highest offset in the buffer. Ensure - * it does. - */ - WARN_ONCE(last_good_offset > xstate_offsets[i], - "x86/fpu: misordered xstate at %d\n", last_good_offset); - - last_good_offset = xstate_offsets[i]; } } From 15d51a2f6f3f7057311d6e37d5190d58597a94a9 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Thu, 20 Mar 2025 16:42:53 -0700 Subject: [PATCH 184/331] x86/fpu/xstate: Introduce xfeature order table and accessor macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel has largely assumed that higher xstate component numbers correspond to later offsets in the buffer. However, this assumption no longer holds for the non-compacted format, where a newer state component may have a lower offset. When iterating over xstate components in offset order, using the feature number as an index may be misleading. At the same time, the CPU exposes each component’s size and offset based on its feature number, making it a key for state information. To provide flexibility in handling xstate ordering, introduce a mapping table: feature order -> feature number. The table is dynamically populated based on the CPU-exposed features and is sorted in offset order at boot time. Additionally, add an accessor macro to facilitate sequential traversal of xstate components based on their actual buffer positions, given a feature bitmask. This accessor macro will be particularly useful for computing custom non-compacted format sizes and iterating over xstate offsets in non-compacted buffers. Suggested-by: Dave Hansen Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250320234301.8342-3-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 542c6981180d..1e22103a8e17 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,31 @@ static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init; +/* + * Ordering of xstate components in uncompacted format: The xfeature + * number does not necessarily indicate its position in the XSAVE buffer. + * This array defines the traversal order of xstate features. + */ +static unsigned int xfeature_uncompact_order[XFEATURE_MAX] __ro_after_init = + { [ 0 ... XFEATURE_MAX - 1] = -1}; + +static inline unsigned int next_xfeature_order(unsigned int i, u64 mask) +{ + for (; xfeature_uncompact_order[i] != -1; i++) { + if (mask & BIT_ULL(xfeature_uncompact_order[i])) + break; + } + + return i; +} + +/* Iterate xstate features in uncompacted order: */ +#define for_each_extended_xfeature_in_order(i, mask) \ + for (i = 0; \ + i = next_xfeature_order(i, mask), \ + xfeature_uncompact_order[i] != -1; \ + i++) + #define XSTATE_FLAG_SUPERVISOR BIT(0) #define XSTATE_FLAG_ALIGNED64 BIT(1) @@ -209,13 +235,20 @@ static bool xfeature_enabled(enum xfeature xfeature) return fpu_kernel_cfg.max_features & BIT_ULL(xfeature); } +static int compare_xstate_offsets(const void *xfeature1, const void *xfeature2) +{ + return xstate_offsets[*(unsigned int *)xfeature1] - + xstate_offsets[*(unsigned int *)xfeature2]; +} + /* * Record the offsets and sizes of various xstates contained - * in the XSAVE state memory layout. + * in the XSAVE state memory layout. Also, create an ordered + * list of xfeatures for handling out-of-order offsets. */ static void __init setup_xstate_cache(void) { - u32 eax, ebx, ecx, edx, i; + u32 eax, ebx, ecx, edx, xfeature, i = 0; /* * The FP xstates and SSE xstates are legacy states. They are always * in the fixed offsets in the xsave area in either compacted form @@ -229,21 +262,30 @@ static void __init setup_xstate_cache(void) xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, xmm_space); - for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { - cpuid_count(CPUID_LEAF_XSTATE, i, &eax, &ebx, &ecx, &edx); + for_each_extended_xfeature(xfeature, fpu_kernel_cfg.max_features) { + cpuid_count(CPUID_LEAF_XSTATE, xfeature, &eax, &ebx, &ecx, &edx); - xstate_sizes[i] = eax; - xstate_flags[i] = ecx; + xstate_sizes[xfeature] = eax; + xstate_flags[xfeature] = ecx; /* * If an xfeature is supervisor state, the offset in EBX is * invalid, leave it to -1. */ - if (xfeature_is_supervisor(i)) + if (xfeature_is_supervisor(xfeature)) continue; - xstate_offsets[i] = ebx; + xstate_offsets[xfeature] = ebx; + + /* Populate the list of xfeatures before sorting */ + xfeature_uncompact_order[i++] = xfeature; } + + /* + * Sort xfeatures by their offsets to support out-of-order + * offsets in the uncompacted format. + */ + sort(xfeature_uncompact_order, i, sizeof(unsigned int), compare_xstate_offsets, NULL); } /* From a758ae2885eacaa68e4de9a01539a242a6bbb403 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Thu, 20 Mar 2025 16:42:54 -0700 Subject: [PATCH 185/331] x86/fpu/xstate: Adjust XSAVE buffer size calculation The current xstate size calculation assumes that the highest-numbered xstate feature has the highest offset in the buffer, determining the size based on the topmost bit in the feature mask. However, this assumption is not architecturally guaranteed -- higher-numbered features may have lower offsets. With the introduction of the xfeature order table and its helper macro, xstate components can now be traversed in their positional order. Update the non-compacted format handling to iterate through the table to determine the last-positioned feature. Then, set the offset accordingly. Since size calculation primarily occurs during initialization or in non-critical paths, looping to find the last feature is not expected to have a meaningful performance impact. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250320234301.8342-4-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 1e22103a8e17..93f94013b094 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -581,13 +581,20 @@ static bool __init check_xstate_against_struct(int nr) static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) { unsigned int topmost = fls64(xfeatures) - 1; - unsigned int offset = xstate_offsets[topmost]; + unsigned int offset, i; if (topmost <= XFEATURE_SSE) return sizeof(struct xregs_state); - if (compacted) + if (compacted) { offset = xfeature_get_offset(xfeatures, topmost); + } else { + /* Walk through the xfeature order to pick the last */ + for_each_extended_xfeature_in_order(i, xfeatures) + topmost = xfeature_uncompact_order[i]; + offset = xstate_offsets[topmost]; + } + return offset + xstate_sizes[topmost]; } From cbe8e4dab16c56ac84765dcd53e418160c8bc0db Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Thu, 20 Mar 2025 16:42:55 -0700 Subject: [PATCH 186/331] x86/fpu/xstate: Adjust xstate copying logic for user ABI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit == Background == As feature positions in the userspace XSAVE buffer do not always align with their feature numbers, the XSAVE format conversion needs to be reconsidered to align with the revised xstate size calculation logic. * For signal handling, XSAVE and XRSTOR are used directly to save and restore extended registers. * For ptrace, KVM, and signal returns (for 32-bit frame), the kernel copies data between its internal buffer and the userspace XSAVE buffer. If memcpy() were used for these cases, existing offset helpers — such as __raw_xsave_addr() or xstate_offsets[] — would be sufficient to handle the format conversion. == Problem == When copying data from the compacted in-kernel buffer to the non-compacted userspace buffer, the function follows the user_regset_get2_fn() prototype. This means it utilizes struct membuf helpers for the destination buffer. As defined in regset.h, these helpers update the memory pointer during the copy process, enforcing sequential writes within the loop. Since xstate components are processed sequentially, any component whose buffer position does not align with its feature number has an issue. == Solution == Replace for_each_extended_xfeature() with the newly introduced for_each_extended_xfeature_in_order(). This macro ensures xstate components are handled in the correct order based on their actual positions in the destination buffer, rather than their feature numbers. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250320234301.8342-5-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 93f94013b094..46c45e2f2a5a 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1107,10 +1107,9 @@ void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); struct xregs_state *xinit = &init_fpstate.regs.xsave; struct xregs_state *xsave = &fpstate->regs.xsave; + unsigned int zerofrom, i, xfeature; struct xstate_header header; - unsigned int zerofrom; u64 mask; - int i; memset(&header, 0, sizeof(header)); header.xfeatures = xsave->header.xfeatures; @@ -1179,15 +1178,16 @@ void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, */ mask = header.xfeatures; - for_each_extended_xfeature(i, mask) { + for_each_extended_xfeature_in_order(i, mask) { + xfeature = xfeature_uncompact_order[i]; /* * If there was a feature or alignment gap, zero the space * in the destination buffer. */ - if (zerofrom < xstate_offsets[i]) - membuf_zero(&to, xstate_offsets[i] - zerofrom); + if (zerofrom < xstate_offsets[xfeature]) + membuf_zero(&to, xstate_offsets[xfeature] - zerofrom); - if (i == XFEATURE_PKRU) { + if (xfeature == XFEATURE_PKRU) { struct pkru_state pkru = {0}; /* * PKRU is not necessarily up to date in the @@ -1197,14 +1197,14 @@ void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, membuf_write(&to, &pkru, sizeof(pkru)); } else { membuf_write(&to, - __raw_xsave_addr(xsave, i), - xstate_sizes[i]); + __raw_xsave_addr(xsave, xfeature), + xstate_sizes[xfeature]); } /* * Keep track of the last copied state in the non-compacted * target buffer for gap zeroing. */ - zerofrom = xstate_offsets[i] + xstate_sizes[i]; + zerofrom = xstate_offsets[xfeature] + xstate_sizes[xfeature]; } out: From 77fbccede633a5565cae084348b5459f6849086d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:20 +0200 Subject: [PATCH 187/331] x86/fpu: Introduce the x86_task_fpu() helper method The per-task FPU context/save area is allocated right next to task_struct, currently in a variable-size array via task_struct::thread.fpu[], but we plan to fully hide it from the C type scope. Introduce the x86_task_fpu() accessor that gets to the FPU context pointer explicitly from the task pointer. Right now this is a simple (task)->thread.fpu wrapper. Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250409211127.3544993-2-mingo@kernel.org --- arch/x86/include/asm/processor.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 5d2f7e5aff26..2f631e0adea3 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -523,6 +523,8 @@ struct thread_struct { */ }; +#define x86_task_fpu(task) (&(task)->thread.fpu) + extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size); static inline void arch_thread_struct_whitelist(unsigned long *offset, From e3bfa3859936da3edd1e16d0b74fdaaa19bb5087 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:21 +0200 Subject: [PATCH 188/331] x86/fpu: Convert task_struct::thread.fpu accesses to use x86_task_fpu() This will make the removal of the task_struct::thread.fpu array easier. No change in functionality - code generated before and after this commit is identical on x86-defconfig: kepler:~/tip> diff -up vmlinux.before.asm vmlinux.after.asm kepler:~/tip> Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250409211127.3544993-3-mingo@kernel.org --- arch/x86/include/asm/fpu/sched.h | 2 +- arch/x86/kernel/fpu/context.h | 4 ++-- arch/x86/kernel/fpu/core.c | 30 +++++++++++++++--------------- arch/x86/kernel/fpu/init.c | 8 ++++---- arch/x86/kernel/fpu/regset.c | 22 +++++++++++----------- arch/x86/kernel/fpu/signal.c | 18 +++++++++--------- arch/x86/kernel/fpu/xstate.c | 22 +++++++++++----------- arch/x86/kernel/fpu/xstate.h | 6 +++--- arch/x86/kernel/process.c | 6 +++--- arch/x86/kernel/signal.c | 6 +++--- arch/x86/kernel/traps.c | 2 +- arch/x86/math-emu/fpu_aux.c | 2 +- arch/x86/math-emu/fpu_entry.c | 4 ++-- arch/x86/math-emu/fpu_system.h | 2 +- arch/x86/mm/extable.c | 2 +- 15 files changed, 68 insertions(+), 68 deletions(-) diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index c485f1944c5f..1feaa68b7567 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -41,7 +41,7 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu) { if (cpu_feature_enabled(X86_FEATURE_FPU) && !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) { - struct fpu *old_fpu = &old->thread.fpu; + struct fpu *old_fpu = x86_task_fpu(old); save_fpregs_to_fpstate(old_fpu); /* diff --git a/arch/x86/kernel/fpu/context.h b/arch/x86/kernel/fpu/context.h index f6d856bd50bc..10d0a720659c 100644 --- a/arch/x86/kernel/fpu/context.h +++ b/arch/x86/kernel/fpu/context.h @@ -53,7 +53,7 @@ static inline void fpregs_activate(struct fpu *fpu) /* Internal helper for switch_fpu_return() and signal frame setup */ static inline void fpregs_restore_userregs(void) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); int cpu = smp_processor_id(); if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER))) @@ -67,7 +67,7 @@ static inline void fpregs_restore_userregs(void) * If PKRU is enabled, then the PKRU value is already * correct because it was either set in switch_to() or in * flush_thread(). So it is excluded because it might be - * not up to date in current->thread.fpu.xsave state. + * not up to date in current->thread.fpu->xsave state. * * XFD state is handled in restore_fpregs_from_fpstate(). */ diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 91d6341f281f..dc6d7f93c446 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -211,7 +211,7 @@ static void fpu_init_guest_permissions(struct fpu_guest *gfpu) return; spin_lock_irq(¤t->sighand->siglock); - fpuperm = ¤t->group_leader->thread.fpu.guest_perm; + fpuperm = &x86_task_fpu(current->group_leader)->guest_perm; perm = fpuperm->__state_perm; /* First fpstate allocation locks down permissions. */ @@ -323,7 +323,7 @@ EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); */ void fpu_sync_guest_vmexit_xfd_state(void) { - struct fpstate *fps = current->thread.fpu.fpstate; + struct fpstate *fps = x86_task_fpu(current)->fpstate; lockdep_assert_irqs_disabled(); if (fpu_state_size_dynamic()) { @@ -337,7 +337,7 @@ EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) { struct fpstate *guest_fps = guest_fpu->fpstate; - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); struct fpstate *cur_fps = fpu->fpstate; fpregs_lock(); @@ -438,7 +438,7 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && !test_thread_flag(TIF_NEED_FPU_LOAD)) { set_thread_flag(TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(¤t->thread.fpu); + save_fpregs_to_fpstate(x86_task_fpu(current)); } __cpu_invalidate_fpregs_state(); @@ -467,7 +467,7 @@ EXPORT_SYMBOL_GPL(kernel_fpu_end); */ void fpu_sync_fpstate(struct fpu *fpu) { - WARN_ON_FPU(fpu != ¤t->thread.fpu); + WARN_ON_FPU(fpu != x86_task_fpu(current)); fpregs_lock(); trace_x86_fpu_before_save(fpu); @@ -552,7 +552,7 @@ void fpstate_reset(struct fpu *fpu) static inline void fpu_inherit_perms(struct fpu *dst_fpu) { if (fpu_state_size_dynamic()) { - struct fpu *src_fpu = ¤t->group_leader->thread.fpu; + struct fpu *src_fpu = x86_task_fpu(current->group_leader); spin_lock_irq(¤t->sighand->siglock); /* Fork also inherits the permissions of the parent */ @@ -572,7 +572,7 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) if (!ssp) return 0; - xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave, + xstate = get_xsave_addr(&x86_task_fpu(dst)->fpstate->regs.xsave, XFEATURE_CET_USER); /* @@ -593,8 +593,8 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, unsigned long ssp) { - struct fpu *src_fpu = ¤t->thread.fpu; - struct fpu *dst_fpu = &dst->thread.fpu; + struct fpu *src_fpu = x86_task_fpu(current); + struct fpu *dst_fpu = x86_task_fpu(dst); /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; @@ -686,7 +686,7 @@ void fpu__drop(struct fpu *fpu) { preempt_disable(); - if (fpu == ¤t->thread.fpu) { + if (fpu == x86_task_fpu(current)) { /* Ignore delayed exceptions from user space */ asm volatile("1: fwait\n" "2:\n" @@ -720,7 +720,7 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask) */ static void fpu_reset_fpregs(void) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); fpregs_lock(); __fpu_invalidate_fpregs_state(fpu); @@ -749,7 +749,7 @@ static void fpu_reset_fpregs(void) */ void fpu__clear_user_states(struct fpu *fpu) { - WARN_ON_FPU(fpu != ¤t->thread.fpu); + WARN_ON_FPU(fpu != x86_task_fpu(current)); fpregs_lock(); if (!cpu_feature_enabled(X86_FEATURE_FPU)) { @@ -782,7 +782,7 @@ void fpu__clear_user_states(struct fpu *fpu) void fpu_flush_thread(void) { - fpstate_reset(¤t->thread.fpu); + fpstate_reset(x86_task_fpu(current)); fpu_reset_fpregs(); } /* @@ -823,7 +823,7 @@ void fpregs_lock_and_load(void) */ void fpregs_assert_state_consistent(void) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); if (test_thread_flag(TIF_NEED_FPU_LOAD)) return; @@ -835,7 +835,7 @@ EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); void fpregs_mark_activate(void) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); fpregs_activate(fpu); fpu->last_cpu = smp_processor_id(); diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 998a08f17e33..ad5cb2943d37 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -38,7 +38,7 @@ static void fpu__init_cpu_generic(void) /* Flush out any pending x87 state: */ #ifdef CONFIG_MATH_EMULATION if (!boot_cpu_has(X86_FEATURE_FPU)) - fpstate_init_soft(¤t->thread.fpu.fpstate->regs.soft); + fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft); else #endif asm volatile ("fninit"); @@ -154,7 +154,7 @@ static void __init fpu__init_task_struct_size(void) * Subtract off the static size of the register state. * It potentially has a bunch of padding. */ - task_size -= sizeof(current->thread.fpu.__fpstate.regs); + task_size -= sizeof(union fpregs_state); /* * Add back the dynamically-calculated register state @@ -204,7 +204,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) fpu_kernel_cfg.default_size = size; fpu_user_cfg.max_size = size; fpu_user_cfg.default_size = size; - fpstate_reset(¤t->thread.fpu); + fpstate_reset(x86_task_fpu(current)); } /* @@ -213,7 +213,7 @@ static void __init fpu__init_system_xstate_size_legacy(void) */ void __init fpu__init_system(void) { - fpstate_reset(¤t->thread.fpu); + fpstate_reset(x86_task_fpu(current)); fpu__init_system_early_generic(); /* diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c index 887b0b8e21e3..0986c2200adc 100644 --- a/arch/x86/kernel/fpu/regset.c +++ b/arch/x86/kernel/fpu/regset.c @@ -45,7 +45,7 @@ int regset_xregset_fpregs_active(struct task_struct *target, const struct user_r */ static void sync_fpstate(struct fpu *fpu) { - if (fpu == ¤t->thread.fpu) + if (fpu == x86_task_fpu(current)) fpu_sync_fpstate(fpu); } @@ -63,7 +63,7 @@ static void fpu_force_restore(struct fpu *fpu) * Only stopped child tasks can be used to modify the FPU * state in the fpstate buffer: */ - WARN_ON_FPU(fpu == ¤t->thread.fpu); + WARN_ON_FPU(fpu == x86_task_fpu(current)); __fpu_invalidate_fpregs_state(fpu); } @@ -71,7 +71,7 @@ static void fpu_force_restore(struct fpu *fpu) int xfpregs_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); if (!cpu_feature_enabled(X86_FEATURE_FXSR)) return -ENODEV; @@ -91,7 +91,7 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct fxregs_state newstate; int ret; @@ -133,7 +133,7 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset, if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) return -ENODEV; - sync_fpstate(&target->thread.fpu); + sync_fpstate(x86_task_fpu(target)); copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_XSAVE); return 0; @@ -143,7 +143,7 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct xregs_state *tmpbuf = NULL; int ret; @@ -187,7 +187,7 @@ int ssp_active(struct task_struct *target, const struct user_regset *regset) int ssp_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct cet_user_state *cetregs; if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || @@ -214,7 +214,7 @@ int ssp_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct xregs_state *xsave = &fpu->fpstate->regs.xsave; struct cet_user_state *cetregs; unsigned long user_ssp; @@ -368,7 +368,7 @@ static void __convert_from_fxsr(struct user_i387_ia32_struct *env, void convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) { - __convert_from_fxsr(env, tsk, &tsk->thread.fpu.fpstate->regs.fxsave); + __convert_from_fxsr(env, tsk, &x86_task_fpu(tsk)->fpstate->regs.fxsave); } void convert_to_fxsr(struct fxregs_state *fxsave, @@ -401,7 +401,7 @@ void convert_to_fxsr(struct fxregs_state *fxsave, int fpregs_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct user_i387_ia32_struct env; struct fxregs_state fxsave, *fx; @@ -433,7 +433,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct fpu *fpu = &target->thread.fpu; + struct fpu *fpu = x86_task_fpu(target); struct user_i387_ia32_struct env; int ret; diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 6c69cb28b298..b8b4fa9c2d04 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -43,13 +43,13 @@ static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, * fpstate layout with out copying the extended state information * in the memory layout. */ - if (__get_user(magic2, (__u32 __user *)(fpstate + current->thread.fpu.fpstate->user_size))) + if (__get_user(magic2, (__u32 __user *)(fpstate + x86_task_fpu(current)->fpstate->user_size))) return false; if (likely(magic2 == FP_XSTATE_MAGIC2)) return true; setfx: - trace_x86_fpu_xstate_check_failed(¤t->thread.fpu); + trace_x86_fpu_xstate_check_failed(x86_task_fpu(current)); /* Set the parameters for fx only state */ fx_sw->magic1 = 0; @@ -64,13 +64,13 @@ setfx: static inline bool save_fsave_header(struct task_struct *tsk, void __user *buf) { if (use_fxsr()) { - struct xregs_state *xsave = &tsk->thread.fpu.fpstate->regs.xsave; + struct xregs_state *xsave = &x86_task_fpu(tsk)->fpstate->regs.xsave; struct user_i387_ia32_struct env; struct _fpstate_32 __user *fp = buf; fpregs_lock(); if (!test_thread_flag(TIF_NEED_FPU_LOAD)) - fxsave(&tsk->thread.fpu.fpstate->regs.fxsave); + fxsave(&x86_task_fpu(tsk)->fpstate->regs.fxsave); fpregs_unlock(); convert_from_fxsr(&env, tsk); @@ -184,7 +184,7 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf, u32 pk bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u32 pkru) { struct task_struct *tsk = current; - struct fpstate *fpstate = tsk->thread.fpu.fpstate; + struct fpstate *fpstate = x86_task_fpu(tsk)->fpstate; bool ia32_fxstate = (buf != buf_fx); int ret; @@ -272,7 +272,7 @@ static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures, */ static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); int ret; /* Restore enabled features only. */ @@ -332,7 +332,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx, bool ia32_fxstate) { struct task_struct *tsk = current; - struct fpu *fpu = &tsk->thread.fpu; + struct fpu *fpu = x86_task_fpu(tsk); struct user_i387_ia32_struct env; bool success, fx_only = false; union fpregs_state *fpregs; @@ -452,7 +452,7 @@ static inline unsigned int xstate_sigframe_size(struct fpstate *fpstate) */ bool fpu__restore_sig(void __user *buf, int ia32_frame) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); void __user *buf_fx = buf; bool ia32_fxstate = false; bool success = false; @@ -499,7 +499,7 @@ unsigned long fpu__alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx, unsigned long *size) { - unsigned long frame_size = xstate_sigframe_size(current->thread.fpu.fpstate); + unsigned long frame_size = xstate_sigframe_size(x86_task_fpu(current)->fpstate); *buf_fx = sp = round_down(sp - frame_size, 64); if (ia32_frame && use_fxsr()) { diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 46c45e2f2a5a..253da5aec915 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -763,7 +763,7 @@ static void __init fpu__init_disable_system_xstate(unsigned int legacy_size) */ init_fpstate.xfd = 0; - fpstate_reset(¤t->thread.fpu); + fpstate_reset(x86_task_fpu(current)); } /* @@ -871,7 +871,7 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) goto out_disable; /* Reset the state for the current task */ - fpstate_reset(¤t->thread.fpu); + fpstate_reset(x86_task_fpu(current)); /* * Update info used for ptrace frames; use standard-format size and no @@ -945,7 +945,7 @@ void fpu__resume_cpu(void) } if (fpu_state_size_dynamic()) - wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd); + wrmsrl(MSR_IA32_XFD, x86_task_fpu(current)->fpstate->xfd); } /* @@ -1227,8 +1227,8 @@ out: void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, enum xstate_copy_mode copy_mode) { - __copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate, - tsk->thread.fpu.fpstate->user_xfeatures, + __copy_xstate_to_uabi_buf(to, x86_task_fpu(tsk)->fpstate, + x86_task_fpu(tsk)->fpstate->user_xfeatures, tsk->thread.pkru, copy_mode); } @@ -1368,7 +1368,7 @@ int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf) { - return copy_uabi_to_xstate(tsk->thread.fpu.fpstate, NULL, ubuf, &tsk->thread.pkru); + return copy_uabi_to_xstate(x86_task_fpu(tsk)->fpstate, NULL, ubuf, &tsk->thread.pkru); } static bool validate_independent_components(u64 mask) @@ -1462,7 +1462,7 @@ static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor) * The XFD MSR does not match fpstate->xfd. That's invalid when * the passed in fpstate is current's fpstate. */ - if (fpstate->xfd == current->thread.fpu.fpstate->xfd) + if (fpstate->xfd == x86_task_fpu(current)->fpstate->xfd) return false; /* @@ -1539,7 +1539,7 @@ void fpstate_free(struct fpu *fpu) static int fpstate_realloc(u64 xfeatures, unsigned int ksize, unsigned int usize, struct fpu_guest *guest_fpu) { - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); struct fpstate *curfps, *newfps = NULL; unsigned int fpsize; bool in_use; @@ -1632,7 +1632,7 @@ static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) * AVX512. */ bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); - struct fpu *fpu = ¤t->group_leader->thread.fpu; + struct fpu *fpu = x86_task_fpu(current->group_leader); struct fpu_state_perm *perm; unsigned int ksize, usize; u64 mask; @@ -1735,7 +1735,7 @@ int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu) return -EPERM; } - fpu = ¤t->group_leader->thread.fpu; + fpu = x86_task_fpu(current->group_leader); perm = guest_fpu ? &fpu->guest_perm : &fpu->perm; ksize = perm->__state_size; usize = perm->__user_state_size; @@ -1840,7 +1840,7 @@ long fpu_xstate_prctl(int option, unsigned long arg2) */ static void avx512_status(struct seq_file *m, struct task_struct *task) { - unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); + unsigned long timestamp = READ_ONCE(x86_task_fpu(task)->avx512_timestamp); long delta; if (!timestamp) { diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h index 0fd34f53f025..9a3a8ccf13bf 100644 --- a/arch/x86/kernel/fpu/xstate.h +++ b/arch/x86/kernel/fpu/xstate.h @@ -22,7 +22,7 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) static inline u64 xstate_get_group_perm(bool guest) { - struct fpu *fpu = ¤t->group_leader->thread.fpu; + struct fpu *fpu = x86_task_fpu(current->group_leader); struct fpu_state_perm *perm; /* Pairs with WRITE_ONCE() in xstate_request_perm() */ @@ -288,7 +288,7 @@ static inline int xsave_to_user_sigframe(struct xregs_state __user *buf, u32 pkr * internally, e.g. PKRU. That's user space ABI and also required * to allow the signal handler to modify PKRU. */ - struct fpstate *fpstate = current->thread.fpu.fpstate; + struct fpstate *fpstate = x86_task_fpu(current)->fpstate; u64 mask = fpstate->user_xfeatures; u32 lmask; u32 hmask; @@ -322,7 +322,7 @@ static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 u32 hmask = mask >> 32; int err; - xfd_validate_state(current->thread.fpu.fpstate, mask, true); + xfd_validate_state(x86_task_fpu(current)->fpstate, mask, true); stac(); XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 962c3ce39323..47694e391506 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -103,7 +103,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) dst->thread.vm86 = NULL; #endif /* Drop the copied pointer to current's fpstate */ - dst->thread.fpu.fpstate = NULL; + x86_task_fpu(dst)->fpstate = NULL; return 0; } @@ -112,7 +112,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) void arch_release_task_struct(struct task_struct *tsk) { if (fpu_state_size_dynamic()) - fpstate_free(&tsk->thread.fpu); + fpstate_free(x86_task_fpu(tsk)); } #endif @@ -122,7 +122,7 @@ void arch_release_task_struct(struct task_struct *tsk) void exit_thread(struct task_struct *tsk) { struct thread_struct *t = &tsk->thread; - struct fpu *fpu = &t->fpu; + struct fpu *fpu = x86_task_fpu(tsk); if (test_thread_flag(TIF_IO_BITMAP)) io_bitmap_exit(tsk); diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 5f441039b572..2404233336ab 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -255,7 +255,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) { bool stepping, failed; - struct fpu *fpu = ¤t->thread.fpu; + struct fpu *fpu = x86_task_fpu(current); if (v8086_mode(regs)) save_v86_state((struct kernel_vm86_regs *) regs, VM86_SIGNAL); @@ -423,14 +423,14 @@ bool sigaltstack_size_valid(size_t ss_size) if (!fpu_state_size_dynamic() && !strict_sigaltstack_size) return true; - fsize += current->group_leader->thread.fpu.perm.__user_state_size; + fsize += x86_task_fpu(current->group_leader)->perm.__user_state_size; if (likely(ss_size > fsize)) return true; if (strict_sigaltstack_size) return ss_size > fsize; - mask = current->group_leader->thread.fpu.perm.__state_perm; + mask = x86_task_fpu(current->group_leader)->perm.__state_perm; if (mask & XFEATURE_MASK_USER_DYNAMIC) return ss_size > fsize; diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 9f88b8a78e50..f48325dfaa01 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -1295,7 +1295,7 @@ DEFINE_IDTENTRY_RAW(exc_debug) static void math_error(struct pt_regs *regs, int trapnr) { struct task_struct *task = current; - struct fpu *fpu = &task->thread.fpu; + struct fpu *fpu = x86_task_fpu(task); int si_code; char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" : "simd exception"; diff --git a/arch/x86/math-emu/fpu_aux.c b/arch/x86/math-emu/fpu_aux.c index d62662bdd460..5f253ae406b6 100644 --- a/arch/x86/math-emu/fpu_aux.c +++ b/arch/x86/math-emu/fpu_aux.c @@ -53,7 +53,7 @@ void fpstate_init_soft(struct swregs_state *soft) void finit(void) { - fpstate_init_soft(¤t->thread.fpu.fpstate->regs.soft); + fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft); } /* diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c index 91c52ead1226..5034df617740 100644 --- a/arch/x86/math-emu/fpu_entry.c +++ b/arch/x86/math-emu/fpu_entry.c @@ -641,7 +641,7 @@ int fpregs_soft_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct swregs_state *s387 = &target->thread.fpu.fpstate->regs.soft; + struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft; void *space = s387->st_space; int ret; int offset, other, i, tags, regnr, tag, newtop; @@ -692,7 +692,7 @@ int fpregs_soft_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct swregs_state *s387 = &target->thread.fpu.fpstate->regs.soft; + struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft; const void *space = s387->st_space; int offset = (S387->ftop & 7) * 10, other = 80 - offset; diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h index eec3e4805c75..5e238e930fe3 100644 --- a/arch/x86/math-emu/fpu_system.h +++ b/arch/x86/math-emu/fpu_system.h @@ -73,7 +73,7 @@ static inline bool seg_writable(struct desc_struct *d) return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_WRITABLE; } -#define I387 (¤t->thread.fpu.fpstate->regs) +#define I387 (&x86_task_fpu(current)->fpstate->regs) #define FPU_info (I387->soft.info) #define FPU_CS (*(unsigned short *) &(FPU_info->regs->cs)) diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 51986e8a9d35..bf8dab18be97 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -111,7 +111,7 @@ static bool ex_handler_sgx(const struct exception_table_entry *fixup, /* * Handler for when we fail to restore a task's FPU state. We should never get - * here because the FPU state of a task using the FPU (task->thread.fpu.state) + * here because the FPU state of a task using the FPU (struct fpu::fpstate) * should always be valid. However, past bugs have allowed userspace to set * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn(). * These caused XRSTOR to fail when switching to the task, leaking the FPU From cb7ca40a3882360ce87191793449d48df0b29184 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:22 +0200 Subject: [PATCH 189/331] x86/fpu: Make task_struct::thread constant size Turn thread.fpu into a pointer. Since most FPU code internals work by passing around the FPU pointer already, the code generation impact is small. This allows us to remove the old kludge of task_struct being variable size: struct task_struct { ... /* * New fields for task_struct should be added above here, so that * they are included in the randomized portion of task_struct. */ randomized_struct_fields_end /* CPU-specific state of this task: */ struct thread_struct thread; /* * WARNING: on x86, 'thread_struct' contains a variable-sized * structure. It *MUST* be at the end of 'task_struct'. * * Do not put anything below here! */ }; ... which creates a number of problems, such as requiring thread_struct to be the last member of the struct - not allowing it to be struct-randomized, etc. But the primary motivation is to allow the decoupling of task_struct from hardware details ( in particular), and to eventually allow the per-task infrastructure: DECLARE_PER_TASK(type, name); ... per_task(current, name) = val; ... which requires task_struct to be a constant size struct. The fpu_thread_struct_whitelist() quirk to hardened usercopy can be removed, now that the FPU structure is not embedded in the task struct anymore, which reduces text footprint a bit. Fixed-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250409211127.3544993-4-mingo@kernel.org --- arch/x86/include/asm/processor.h | 20 +++++++++----------- arch/x86/kernel/fpu/core.c | 23 ++++++++++++----------- arch/x86/kernel/fpu/init.c | 17 ++++++++++------- arch/x86/kernel/process.c | 2 +- include/linux/sched.h | 15 ++++----------- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 2f631e0adea3..5ea7e5d2c4de 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -516,21 +516,19 @@ struct thread_struct { #endif /* Floating point and extended processor state */ - struct fpu fpu; - /* - * WARNING: 'fpu' is dynamically-sized. It *MUST* be at - * the end. - */ + struct fpu *fpu; }; -#define x86_task_fpu(task) (&(task)->thread.fpu) +#define x86_task_fpu(task) ((task)->thread.fpu) -extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size); - -static inline void arch_thread_struct_whitelist(unsigned long *offset, - unsigned long *size) +/* + * X86 doesn't need any embedded-FPU-struct quirks: + */ +static inline void +arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) { - fpu_thread_struct_whitelist(offset, size); + *offset = 0; + *size = 0; } static inline void diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index dc6d7f93c446..853a738fdf2d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -593,8 +593,19 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, unsigned long ssp) { + /* + * We allocate the new FPU structure right after the end of the task struct. + * task allocation size already took this into account. + * + * This is safe because task_struct size is a multiple of cacheline size. + */ struct fpu *src_fpu = x86_task_fpu(current); - struct fpu *dst_fpu = x86_task_fpu(dst); + struct fpu *dst_fpu = (void *)dst + sizeof(*dst); + + BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0); + BUG_ON(!src_fpu); + + dst->thread.fpu = dst_fpu; /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; @@ -663,16 +674,6 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, return 0; } -/* - * Whitelist the FPU register state embedded into task_struct for hardened - * usercopy. - */ -void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) -{ - *offset = offsetof(struct thread_struct, fpu.__fpstate.regs); - *size = fpu_kernel_cfg.default_size; -} - /* * Drops current FPU state: deactivates the fpregs and * the fpstate. NOTE: it still leaves previous contents diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index ad5cb2943d37..848ea79886ba 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -71,8 +71,15 @@ static bool __init fpu__probe_without_cpuid(void) return fsw == 0 && (fcw & 0x103f) == 0x003f; } +static struct fpu x86_init_fpu __attribute__ ((aligned (64))) __read_mostly; + static void __init fpu__init_system_early_generic(void) { + fpstate_reset(&x86_init_fpu); + current->thread.fpu = &x86_init_fpu; + set_thread_flag(TIF_NEED_FPU_LOAD); + x86_init_fpu.last_cpu = -1; + if (!boot_cpu_has(X86_FEATURE_CPUID) && !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { if (fpu__probe_without_cpuid()) @@ -150,6 +157,8 @@ static void __init fpu__init_task_struct_size(void) { int task_size = sizeof(struct task_struct); + task_size += sizeof(struct fpu); + /* * Subtract off the static size of the register state. * It potentially has a bunch of padding. @@ -164,14 +173,9 @@ static void __init fpu__init_task_struct_size(void) /* * We dynamically size 'struct fpu', so we require that - * it be at the end of 'thread_struct' and that - * 'thread_struct' be at the end of 'task_struct'. If - * you hit a compile error here, check the structure to - * see if something got added to the end. + * 'state' be at the end of 'it: */ CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate); - CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu); - CHECK_MEMBER_AT_END_OF(struct task_struct, thread); arch_task_struct_size = task_size; } @@ -213,7 +217,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) */ void __init fpu__init_system(void) { - fpstate_reset(x86_task_fpu(current)); fpu__init_system_early_generic(); /* diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 47694e391506..3ce4cce46f3f 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -103,7 +103,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) dst->thread.vm86 = NULL; #endif /* Drop the copied pointer to current's fpstate */ - x86_task_fpu(dst)->fpstate = NULL; + dst->thread.fpu = NULL; return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index f96ac1982893..4ecc0c6b1cb0 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1646,22 +1646,15 @@ struct task_struct { struct user_event_mm *user_event_mm; #endif + /* CPU-specific state of this task: */ + struct thread_struct thread; + /* * New fields for task_struct should be added above here, so that * they are included in the randomized portion of task_struct. */ randomized_struct_fields_end - - /* CPU-specific state of this task: */ - struct thread_struct thread; - - /* - * WARNING: on x86, 'thread_struct' contains a variable-sized - * structure. It *MUST* be at the end of 'task_struct'. - * - * Do not put anything below here! - */ -}; +} __attribute__ ((aligned (64))); #define TASK_REPORT_IDLE (TASK_REPORT + 1) #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1) From 55bc30f2e34dcc17a370d1f6c1c992be107c4502 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:23 +0200 Subject: [PATCH 190/331] x86/fpu: Remove the thread::fpu pointer As suggested by Oleg, remove the thread::fpu pointer, as we can calculate it via x86_task_fpu() at compile-time. This improves code generation a bit: kepler:~/tip> size vmlinux.before vmlinux.after text data bss dec hex filename 26475405 10435342 1740804 38651551 24dc69f vmlinux.before 26475339 10959630 1216516 38651485 24dc65d vmlinux.after Suggested-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250409211127.3544993-5-mingo@kernel.org --- arch/x86/include/asm/processor.h | 5 +---- arch/x86/kernel/fpu/core.c | 4 +--- arch/x86/kernel/fpu/init.c | 1 - arch/x86/kernel/process.c | 2 -- arch/x86/kernel/vmlinux.lds.S | 4 ++++ 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 5ea7e5d2c4de..b7f7c9c83409 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -514,12 +514,9 @@ struct thread_struct { struct thread_shstk shstk; #endif - - /* Floating point and extended processor state */ - struct fpu *fpu; }; -#define x86_task_fpu(task) ((task)->thread.fpu) +#define x86_task_fpu(task) ((struct fpu *)((void *)(task) + sizeof(*(task)))) /* * X86 doesn't need any embedded-FPU-struct quirks: diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 853a738fdf2d..974b276ff0da 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -600,13 +600,11 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, * This is safe because task_struct size is a multiple of cacheline size. */ struct fpu *src_fpu = x86_task_fpu(current); - struct fpu *dst_fpu = (void *)dst + sizeof(*dst); + struct fpu *dst_fpu = x86_task_fpu(dst); BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0); BUG_ON(!src_fpu); - dst->thread.fpu = dst_fpu; - /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 848ea79886ba..da41a1d2c40f 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -76,7 +76,6 @@ static struct fpu x86_init_fpu __attribute__ ((aligned (64))) __read_mostly; static void __init fpu__init_system_early_generic(void) { fpstate_reset(&x86_init_fpu); - current->thread.fpu = &x86_init_fpu; set_thread_flag(TIF_NEED_FPU_LOAD); x86_init_fpu.last_cpu = -1; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 3ce4cce46f3f..88868a90459e 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -102,8 +102,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) #ifdef CONFIG_VM86 dst->thread.vm86 = NULL; #endif - /* Drop the copied pointer to current's fpstate */ - dst->thread.fpu = NULL; return 0; } diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index ccdc45e5b759..d9ca2d1754da 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -181,6 +181,10 @@ SECTIONS /* equivalent to task_pt_regs(&init_task) */ __top_init_kernel_stack = __end_init_stack - TOP_OF_KERNEL_STACK_PADDING - PTREGS_SIZE; + __x86_init_fpu_begin = .; + . = __x86_init_fpu_begin + 128*PAGE_SIZE; + __x86_init_fpu_end = .; + #ifdef CONFIG_X86_32 /* 32 bit has nosave before _edata */ NOSAVE_DATA From ec2227e03a46a162f2721917262adc553b212e2d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:24 +0200 Subject: [PATCH 191/331] x86/fpu: Push 'fpu' pointer calculation into the fpu__drop() call This encapsulates the fpu__drop() functionality better, and it will also enable other changes that want to check a task for PF_KTHREAD before calling x86_task_fpu(). Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250409211127.3544993-6-mingo@kernel.org --- arch/x86/include/asm/fpu/sched.h | 2 +- arch/x86/kernel/fpu/core.c | 4 +++- arch/x86/kernel/process.c | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index 1feaa68b7567..5fd12634bcc4 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -10,7 +10,7 @@ #include extern void save_fpregs_to_fpstate(struct fpu *fpu); -extern void fpu__drop(struct fpu *fpu); +extern void fpu__drop(struct task_struct *tsk); extern int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, unsigned long shstk_addr); extern void fpu_flush_thread(void); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 974b276ff0da..e4c20908ee49 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -681,8 +681,10 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, * a state-restore is coming: either an explicit one, * or a reschedule. */ -void fpu__drop(struct fpu *fpu) +void fpu__drop(struct task_struct *tsk) { + struct fpu *fpu = x86_task_fpu(tsk); + preempt_disable(); if (fpu == x86_task_fpu(current)) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 88868a90459e..5fb502c97b08 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -120,7 +120,6 @@ void arch_release_task_struct(struct task_struct *tsk) void exit_thread(struct task_struct *tsk) { struct thread_struct *t = &tsk->thread; - struct fpu *fpu = x86_task_fpu(tsk); if (test_thread_flag(TIF_IO_BITMAP)) io_bitmap_exit(tsk); @@ -128,7 +127,7 @@ void exit_thread(struct task_struct *tsk) free_vm86(t); shstk_free(tsk); - fpu__drop(fpu); + fpu__drop(tsk); } static int set_new_tls(struct task_struct *p, unsigned long tls) From c360bdc593b8a8b6e94166026728764085919cff Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:25 +0200 Subject: [PATCH 192/331] x86/fpu: Make sure x86_task_fpu() doesn't get called for PF_KTHREAD|PF_USER_WORKER tasks during exit fpu__drop() and arch_release_task_struct() calls x86_task_fpu() unconditionally, while the FPU context area will not be present if it's the init task, and should not be in use when it's some other type of kthread. Return early for PF_KTHREAD or PF_USER_WORKER tasks. The debug warning in x86_task_fpu() will catch any kthreads attempting to use the FPU save area. Fixed-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250409211127.3544993-7-mingo@kernel.org --- arch/x86/kernel/fpu/core.c | 8 +++++++- arch/x86/kernel/process.c | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index e4c20908ee49..4a2193892e5d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -683,7 +683,13 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, */ void fpu__drop(struct task_struct *tsk) { - struct fpu *fpu = x86_task_fpu(tsk); + struct fpu *fpu; + + /* PF_KTHREAD tasks do not use the FPU context area: */ + if (tsk->flags & (PF_KTHREAD | PF_USER_WORKER)) + return; + + fpu = x86_task_fpu(tsk); preempt_disable(); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 5fb502c97b08..7a1bfb61d86f 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -109,7 +109,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) #ifdef CONFIG_X86_64 void arch_release_task_struct(struct task_struct *tsk) { - if (fpu_state_size_dynamic()) + if (fpu_state_size_dynamic() && !(tsk->flags & (PF_KTHREAD | PF_USER_WORKER))) fpstate_free(x86_task_fpu(tsk)); } #endif From 22aafe3bcb67472effdea1ccf0df20280192bbaf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:26 +0200 Subject: [PATCH 193/331] x86/fpu: Remove init_task FPU state dependencies, add debugging warning for PF_KTHREAD tasks init_task's FPU state initialization was a bit of a hack: __x86_init_fpu_begin = .; . = __x86_init_fpu_begin + 128*PAGE_SIZE; __x86_init_fpu_end = .; But the init task isn't supposed to be using the FPU context in any case, so remove the hack and add in some debug warnings. As Linus noted in the discussion, the init task (and other PF_KTHREAD tasks) *can* use the FPU via kernel_fpu_begin()/_end(), but they don't need the context area because their FPU use is not preemptible or reentrant, and they don't return to user-space. Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250409211127.3544993-8-mingo@kernel.org --- arch/x86/include/asm/processor.h | 6 +++++- arch/x86/kernel/fpu/core.c | 15 +++++++++++---- arch/x86/kernel/fpu/init.c | 3 +-- arch/x86/kernel/fpu/xstate.c | 3 --- arch/x86/kernel/vmlinux.lds.S | 4 ---- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index b7f7c9c83409..eaa7214d6953 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -516,7 +516,11 @@ struct thread_struct { #endif }; -#define x86_task_fpu(task) ((struct fpu *)((void *)(task) + sizeof(*(task)))) +#ifdef CONFIG_X86_DEBUG_FPU +extern struct fpu *x86_task_fpu(struct task_struct *task); +#else +# define x86_task_fpu(task) ((struct fpu *)((void *)(task) + sizeof(*(task)))) +#endif /* * X86 doesn't need any embedded-FPU-struct quirks: diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 4a2193892e5d..4d1a205b7ce2 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -51,6 +51,16 @@ static DEFINE_PER_CPU(bool, in_kernel_fpu); */ DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); +#ifdef CONFIG_X86_DEBUG_FPU +struct fpu *x86_task_fpu(struct task_struct *task) +{ + if (WARN_ON_ONCE(task->flags & PF_KTHREAD)) + return NULL; + + return (void *)task + sizeof(*task); +} +#endif + /* * Can we use the FPU in kernel mode with the * whole "kernel_fpu_begin/end()" sequence? @@ -599,11 +609,9 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, * * This is safe because task_struct size is a multiple of cacheline size. */ - struct fpu *src_fpu = x86_task_fpu(current); - struct fpu *dst_fpu = x86_task_fpu(dst); + struct fpu *dst_fpu = (void *)dst + sizeof(*dst); BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0); - BUG_ON(!src_fpu); /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; @@ -666,7 +674,6 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, if (update_fpu_shstk(dst, ssp)) return 1; - trace_x86_fpu_copy_src(src_fpu); trace_x86_fpu_copy_dst(dst_fpu); return 0; diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index da41a1d2c40f..16b6611634c3 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -38,7 +38,7 @@ static void fpu__init_cpu_generic(void) /* Flush out any pending x87 state: */ #ifdef CONFIG_MATH_EMULATION if (!boot_cpu_has(X86_FEATURE_FPU)) - fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft); + ; else #endif asm volatile ("fninit"); @@ -207,7 +207,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) fpu_kernel_cfg.default_size = size; fpu_user_cfg.max_size = size; fpu_user_cfg.default_size = size; - fpstate_reset(x86_task_fpu(current)); } /* diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 253da5aec915..4c771b9bd270 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -870,9 +870,6 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) if (err) goto out_disable; - /* Reset the state for the current task */ - fpstate_reset(x86_task_fpu(current)); - /* * Update info used for ptrace frames; use standard-format size and no * supervisor xstates: diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index d9ca2d1754da..ccdc45e5b759 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -181,10 +181,6 @@ SECTIONS /* equivalent to task_pt_regs(&init_task) */ __top_init_kernel_stack = __end_init_stack - TOP_OF_KERNEL_STACK_PADDING - PTREGS_SIZE; - __x86_init_fpu_begin = .; - . = __x86_init_fpu_begin + 128*PAGE_SIZE; - __x86_init_fpu_end = .; - #ifdef CONFIG_X86_32 /* 32 bit has nosave before _edata */ NOSAVE_DATA From 8b2a7a7294b34fa00adbecbd352ef19eb780261b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Apr 2025 23:11:27 +0200 Subject: [PATCH 194/331] x86/fpu: Use 'fpstate' variable names consistently A few uses of 'fps' snuck in, which is rather confusing (to me) as it suggests frames-per-second. ;-) Rename them to the canonical 'fpstate' name. No change in functionality. Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250409211127.3544993-9-mingo@kernel.org --- arch/x86/include/asm/fpu/api.h | 2 +- arch/x86/kernel/fpu/core.c | 14 +++++++------- arch/x86/kernel/fpu/xstate.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index f42de5f05e7e..8e6848f55dcd 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -136,7 +136,7 @@ static inline void fpstate_free(struct fpu *fpu) { } #endif /* fpstate-related functions which are exported to KVM */ -extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature); +extern void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature); extern u64 xstate_get_guest_group_perm(void); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 4d1a205b7ce2..d0a45f6492cb 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -273,16 +273,16 @@ EXPORT_SYMBOL_GPL(fpu_alloc_guest_fpstate); void fpu_free_guest_fpstate(struct fpu_guest *gfpu) { - struct fpstate *fps = gfpu->fpstate; + struct fpstate *fpstate = gfpu->fpstate; - if (!fps) + if (!fpstate) return; - if (WARN_ON_ONCE(!fps->is_valloc || !fps->is_guest || fps->in_use)) + if (WARN_ON_ONCE(!fpstate->is_valloc || !fpstate->is_guest || fpstate->in_use)) return; gfpu->fpstate = NULL; - vfree(fps); + vfree(fpstate); } EXPORT_SYMBOL_GPL(fpu_free_guest_fpstate); @@ -333,12 +333,12 @@ EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); */ void fpu_sync_guest_vmexit_xfd_state(void) { - struct fpstate *fps = x86_task_fpu(current)->fpstate; + struct fpstate *fpstate = x86_task_fpu(current)->fpstate; lockdep_assert_irqs_disabled(); if (fpu_state_size_dynamic()) { - rdmsrl(MSR_IA32_XFD, fps->xfd); - __this_cpu_write(xfd_state, fps->xfd); + rdmsrl(MSR_IA32_XFD, fpstate->xfd); + __this_cpu_write(xfd_state, fpstate->xfd); } } EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 4c771b9bd270..a288597065fd 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1431,9 +1431,9 @@ void xrstors(struct xregs_state *xstate, u64 mask) } #if IS_ENABLED(CONFIG_KVM) -void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature) +void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature) { - void *addr = get_xsave_addr(&fps->regs.xsave, xfeature); + void *addr = get_xsave_addr(&fpstate->regs.xsave, xfeature); if (addr) memset(addr, 0, xstate_sizes[xfeature]); From e3a52b67f54aa36ab21265eeea016460b5fe1c46 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 10 Apr 2025 12:52:16 +0200 Subject: [PATCH 195/331] x86/fpu: Clarify FPU context cacheline alignment Suggested-by: Peter Zijlstra Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Chang S. Bae Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/Z_ejggklB5-IWB5W@gmail.com --- arch/x86/kernel/fpu/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index d0a45f6492cb..3a19877a314e 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -607,7 +607,8 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, * We allocate the new FPU structure right after the end of the task struct. * task allocation size already took this into account. * - * This is safe because task_struct size is a multiple of cacheline size. + * This is safe because task_struct size is a multiple of cacheline size, + * thus x86_task_fpu() will always be cacheline aligned as well. */ struct fpu *dst_fpu = (void *)dst + sizeof(*dst); From 3846389c03a8518884f09056611619bd1461ffc7 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:39 +0200 Subject: [PATCH 196/331] x86/platform/amd: Move the header to Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Link: https://lore.kernel.org/r/20250413084144.3746608-2-mingo@kernel.org --- arch/x86/events/amd/ibs.c | 2 +- arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} | 0 tools/perf/check-headers.sh | 2 +- tools/perf/util/amd-sample-raw.c | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} (100%) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 0252b7ea8bca..172619932fe3 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -26,7 +26,7 @@ static u32 ibs_caps; #include #include -#include +#include /* attr.config2 */ #define IBS_SW_FILTER_MASK 1 diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd/ibs.h similarity index 100% rename from arch/x86/include/asm/amd-ibs.h rename to arch/x86/include/asm/amd/ibs.h diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index a4499e5a6f9c..493a13830419 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -185,7 +185,7 @@ done # diff with extra ignore lines check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))" -I"^#include "' check arch/x86/lib/memset_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' -check arch/x86/include/asm/amd-ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' +check arch/x86/include/asm/amd/ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' check include/linux/unaligned.h '-I "^#include " -I "^#include " -I "^#pragma GCC diagnostic"' check include/uapi/asm-generic/mman.h '-I "^#include <\(uapi/\)*asm-generic/mman-common\(-tools\)*.h>"' diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 9d0ce88e90e4..456ce64ad822 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -9,7 +9,7 @@ #include #include -#include "../../arch/x86/include/asm/amd-ibs.h" +#include "../../arch/x86/include/asm/amd/ibs.h" #include "debug.h" #include "session.h" From 861c6b1185fbb2e3fa158aa3aca75d2f767db2a8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:40 +0200 Subject: [PATCH 197/331] x86/platform/amd: Add standard header guards to Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Link: https://lore.kernel.org/r/20250413084144.3746608-3-mingo@kernel.org --- arch/x86/include/asm/amd/ibs.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/include/asm/amd/ibs.h b/arch/x86/include/asm/amd/ibs.h index 77f3a589a99a..3ee5903982c2 100644 --- a/arch/x86/include/asm/amd/ibs.h +++ b/arch/x86/include/asm/amd/ibs.h @@ -1,4 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_AMD_IBS_H +#define _ASM_X86_AMD_IBS_H + /* * From PPR Vol 1 for AMD Family 19h Model 01h B1 * 55898 Rev 0.35 - Feb 5, 2021 @@ -151,3 +154,5 @@ struct perf_ibs_data { }; u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; }; + +#endif /* _ASM_X86_AMD_IBS_H */ From bcbb65559532891148d990527e9df6b8fc98e98d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 14 Apr 2025 09:32:04 +0200 Subject: [PATCH 198/331] x86/platform/amd: Move the header to Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Link: https://lore.kernel.org/r/20250413084144.3746608-4-mingo@kernel.org --- arch/x86/include/asm/{amd_nb.h => amd/nb.h} | 0 arch/x86/kernel/amd_gart_64.c | 2 +- arch/x86/kernel/amd_nb.c | 2 +- arch/x86/kernel/aperture_64.c | 2 +- arch/x86/kernel/cpu/amd_cache_disable.c | 2 +- arch/x86/kernel/cpu/cacheinfo.c | 2 +- arch/x86/kernel/cpu/mce/inject.c | 2 +- arch/x86/mm/amdtopology.c | 2 +- arch/x86/mm/numa.c | 2 +- arch/x86/pci/amd_bus.c | 2 +- drivers/char/agp/amd64-agp.c | 2 +- drivers/edac/amd64_edac.c | 2 +- drivers/platform/x86/amd/pmc/mp1_stb.c | 2 +- drivers/pnp/quirks.c | 2 +- drivers/ras/amd/atl/internal.h | 2 +- 15 files changed, 14 insertions(+), 14 deletions(-) rename arch/x86/include/asm/{amd_nb.h => amd/nb.h} (100%) diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd/nb.h similarity index 100% rename from arch/x86/include/asm/amd_nb.h rename to arch/x86/include/asm/amd/nb.h diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index c884deca839b..3485d419c2f5 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include static unsigned long iommu_bus_base; /* GART remapping area (physical) */ diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 5a8cc48f80be..ffaad175cee2 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include static u32 *flush_words; diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index 89c0c8a3fc7e..769321185a08 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/kernel/cpu/amd_cache_disable.c b/arch/x86/kernel/cpu/amd_cache_disable.c index d860ad3f8a5a..8843b9557aea 100644 --- a/arch/x86/kernel/cpu/amd_cache_disable.c +++ b/arch/x86/kernel/cpu/amd_cache_disable.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index cc7ae2bdcf4a..f866d94352fb 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 06e3cf7229ce..bb060f8326ef 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/mm/amdtopology.c b/arch/x86/mm/amdtopology.c index 628833afee37..f980b0eb0105 100644 --- a/arch/x86/mm/amdtopology.c +++ b/arch/x86/mm/amdtopology.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include static unsigned char __initdata nodeids[8]; diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 64e5cdb2460a..fed02d1073c7 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include "numa_internal.h" diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index 631512f7ec85..95ae1971a5f1 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/char/agp/amd64-agp.c b/drivers/char/agp/amd64-agp.c index 8e41731d3642..bf490967241a 100644 --- a/drivers/char/agp/amd64-agp.c +++ b/drivers/char/agp/amd64-agp.c @@ -16,7 +16,7 @@ #include #include /* PAGE_SIZE */ #include -#include +#include #include #include "agp.h" diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 90f0eb7cc5b9..417940f455d3 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2,7 +2,7 @@ #include #include #include "amd64_edac.h" -#include +#include #include static struct edac_pci_ctl_info *pci_ctl; diff --git a/drivers/platform/x86/amd/pmc/mp1_stb.c b/drivers/platform/x86/amd/pmc/mp1_stb.c index c005f00988f7..3b9b9f30faa3 100644 --- a/drivers/platform/x86/amd/pmc/mp1_stb.c +++ b/drivers/platform/x86/amd/pmc/mp1_stb.c @@ -11,7 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include #include #include #include diff --git a/drivers/pnp/quirks.c b/drivers/pnp/quirks.c index 6085a1471de2..6e1d4bfd28ac 100644 --- a/drivers/pnp/quirks.c +++ b/drivers/pnp/quirks.c @@ -290,7 +290,7 @@ static void quirk_system_pci_resources(struct pnp_dev *dev) #ifdef CONFIG_AMD_NB -#include +#include static void quirk_amd_mmconfig_area(struct pnp_dev *dev) { diff --git a/drivers/ras/amd/atl/internal.h b/drivers/ras/amd/atl/internal.h index f9be26d25348..c63fee30cc18 100644 --- a/drivers/ras/amd/atl/internal.h +++ b/drivers/ras/amd/atl/internal.h @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include "reg_fields.h" From d96c78684166bc0e7997d12d845882cb5824eed3 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:42 +0200 Subject: [PATCH 199/331] x86/platform/amd: Move the header to Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: Carlos Bilbao Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Cc: Naveen Krishna Chatradhi Link: https://lore.kernel.org/r/20250413084144.3746608-5-mingo@kernel.org --- MAINTAINERS | 2 +- arch/x86/include/asm/{amd_hsmp.h => amd/hsmp.h} | 0 drivers/platform/x86/amd/hsmp/acpi.c | 2 +- drivers/platform/x86/amd/hsmp/hsmp.c | 2 +- drivers/platform/x86/amd/hsmp/plat.c | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename arch/x86/include/asm/{amd_hsmp.h => amd/hsmp.h} (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 61b9cc04c814..d4ea6d78db51 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1097,7 +1097,7 @@ R: Carlos Bilbao L: platform-driver-x86@vger.kernel.org S: Maintained F: Documentation/arch/x86/amd_hsmp.rst -F: arch/x86/include/asm/amd_hsmp.h +F: arch/x86/include/asm/amd/hsmp.h F: arch/x86/include/uapi/asm/amd_hsmp.h F: drivers/platform/x86/amd/hsmp/ diff --git a/arch/x86/include/asm/amd_hsmp.h b/arch/x86/include/asm/amd/hsmp.h similarity index 100% rename from arch/x86/include/asm/amd_hsmp.h rename to arch/x86/include/asm/amd/hsmp.h diff --git a/drivers/platform/x86/amd/hsmp/acpi.c b/drivers/platform/x86/amd/hsmp/acpi.c index c1eccb3c80c5..3c7acb90c3ab 100644 --- a/drivers/platform/x86/amd/hsmp/acpi.c +++ b/drivers/platform/x86/amd/hsmp/acpi.c @@ -9,7 +9,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include #include #include diff --git a/drivers/platform/x86/amd/hsmp/hsmp.c b/drivers/platform/x86/amd/hsmp/hsmp.c index a3ac09a90de4..e262e8a97b45 100644 --- a/drivers/platform/x86/amd/hsmp/hsmp.c +++ b/drivers/platform/x86/amd/hsmp/hsmp.c @@ -7,7 +7,7 @@ * This file provides a device implementation for HSMP interface */ -#include +#include #include #include diff --git a/drivers/platform/x86/amd/hsmp/plat.c b/drivers/platform/x86/amd/hsmp/plat.c index b9782a078dbd..0eb73fcb9806 100644 --- a/drivers/platform/x86/amd/hsmp/plat.c +++ b/drivers/platform/x86/amd/hsmp/plat.c @@ -9,7 +9,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include #include #include From 5bb144e52c666dc0082a90662a5406133415cacc Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:43 +0200 Subject: [PATCH 200/331] x86/platform/amd: Clean up the header guards a bit - There's no need for a newline after the SPDX line - But there's a need for one before the closing header guard. Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: Carlos Bilbao Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Cc: Naveen Krishna Chatradhi Link: https://lore.kernel.org/r/20250413084144.3746608-6-mingo@kernel.org --- arch/x86/include/asm/amd/hsmp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/amd/hsmp.h b/arch/x86/include/asm/amd/hsmp.h index 03c2ce3edaf5..2137f62853ed 100644 --- a/arch/x86/include/asm/amd/hsmp.h +++ b/arch/x86/include/asm/amd/hsmp.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - #ifndef _ASM_X86_AMD_HSMP_H_ #define _ASM_X86_AMD_HSMP_H_ @@ -13,4 +12,5 @@ static inline int hsmp_send_message(struct hsmp_message *msg) return -ENODEV; } #endif + #endif /*_ASM_X86_AMD_HSMP_H_*/ From 0a35c9280a9105e601cfe23b7c15522a195fa412 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:44 +0200 Subject: [PATCH 201/331] x86/platform/amd: Move the header to Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Link: https://lore.kernel.org/r/20250413084144.3746608-7-mingo@kernel.org --- MAINTAINERS | 2 +- arch/x86/include/asm/amd/nb.h | 2 +- arch/x86/include/asm/{amd_node.h => amd/node.h} | 0 arch/x86/kernel/amd_node.c | 2 +- arch/x86/pci/fixup.c | 2 +- drivers/edac/amd64_edac.c | 2 +- drivers/hwmon/k10temp.c | 2 +- drivers/platform/x86/amd/hsmp/acpi.c | 2 +- drivers/platform/x86/amd/hsmp/plat.c | 2 +- drivers/platform/x86/amd/pmc/pmc.c | 2 +- drivers/platform/x86/amd/pmf/core.c | 2 +- drivers/ras/amd/atl/internal.h | 2 +- sound/soc/amd/acp/acp-rembrandt.c | 2 +- sound/soc/amd/acp/acp63.c | 2 +- sound/soc/amd/acp/acp70.c | 2 +- sound/soc/sof/amd/acp.c | 2 +- 16 files changed, 15 insertions(+), 15 deletions(-) rename arch/x86/include/asm/{amd_node.h => amd/node.h} (100%) diff --git a/MAINTAINERS b/MAINTAINERS index d4ea6d78db51..14bbb43a19ba 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1142,7 +1142,7 @@ M: Mario Limonciello M: Yazen Ghannam L: linux-kernel@vger.kernel.org S: Supported -F: arch/x86/include/asm/amd_node.h +F: arch/x86/include/asm/amd/node.h F: arch/x86/kernel/amd_node.c AMD PDS CORE DRIVER diff --git a/arch/x86/include/asm/amd/nb.h b/arch/x86/include/asm/amd/nb.h index adfa0854cf2d..ddb5108cf46c 100644 --- a/arch/x86/include/asm/amd/nb.h +++ b/arch/x86/include/asm/amd/nb.h @@ -4,7 +4,7 @@ #include #include -#include +#include struct amd_nb_bus_dev_range { u8 bus; diff --git a/arch/x86/include/asm/amd_node.h b/arch/x86/include/asm/amd/node.h similarity index 100% rename from arch/x86/include/asm/amd_node.h rename to arch/x86/include/asm/amd/node.h diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c index b670fa85c61b..a40176b62eb5 100644 --- a/arch/x86/kernel/amd_node.c +++ b/arch/x86/kernel/amd_node.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * AMD Nodes are a physical collection of I/O devices within an SoC. There can be one diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index efefeb82ab61..36336299596b 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 417940f455d3..25180165ca8e 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -3,7 +3,7 @@ #include #include "amd64_edac.h" #include -#include +#include static struct edac_pci_ctl_info *pci_ctl; diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c index 3685906cc57c..472bcf6092f6 100644 --- a/drivers/hwmon/k10temp.c +++ b/drivers/hwmon/k10temp.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include MODULE_DESCRIPTION("AMD Family 10h+ CPU core temperature monitor"); diff --git a/drivers/platform/x86/amd/hsmp/acpi.c b/drivers/platform/x86/amd/hsmp/acpi.c index 3c7acb90c3ab..02e22c1f3a1d 100644 --- a/drivers/platform/x86/amd/hsmp/acpi.c +++ b/drivers/platform/x86/amd/hsmp/acpi.c @@ -23,7 +23,7 @@ #include -#include +#include #include "hsmp.h" diff --git a/drivers/platform/x86/amd/hsmp/plat.c b/drivers/platform/x86/amd/hsmp/plat.c index 0eb73fcb9806..c9a1b1ed4224 100644 --- a/drivers/platform/x86/amd/hsmp/plat.c +++ b/drivers/platform/x86/amd/hsmp/plat.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include "hsmp.h" diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c index d789d6cab794..e5c4e6be37f7 100644 --- a/drivers/platform/x86/amd/pmc/pmc.c +++ b/drivers/platform/x86/amd/pmc/pmc.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include "pmc.h" diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c index a2cb2d5544f5..cecadae3f3bf 100644 --- a/drivers/platform/x86/amd/pmf/core.c +++ b/drivers/platform/x86/amd/pmf/core.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "pmf.h" /* PMF-SMU communication registers */ diff --git a/drivers/ras/amd/atl/internal.h b/drivers/ras/amd/atl/internal.h index c63fee30cc18..05bbee20eeaf 100644 --- a/drivers/ras/amd/atl/internal.h +++ b/drivers/ras/amd/atl/internal.h @@ -18,7 +18,7 @@ #include #include -#include +#include #include "reg_fields.h" diff --git a/sound/soc/amd/acp/acp-rembrandt.c b/sound/soc/amd/acp/acp-rembrandt.c index 746b6ed72029..c30a7b59fbff 100644 --- a/sound/soc/amd/acp/acp-rembrandt.c +++ b/sound/soc/amd/acp/acp-rembrandt.c @@ -22,7 +22,7 @@ #include #include -#include +#include #include "amd.h" #include "../mach-config.h" diff --git a/sound/soc/amd/acp/acp63.c b/sound/soc/amd/acp/acp63.c index 52d895e624c7..0ddde14da7d5 100644 --- a/sound/soc/amd/acp/acp63.c +++ b/sound/soc/amd/acp/acp63.c @@ -21,7 +21,7 @@ #include #include -#include +#include #include "amd.h" #include "acp-mach.h" diff --git a/sound/soc/amd/acp/acp70.c b/sound/soc/amd/acp/acp70.c index 6d5f5ade075c..7f4a25b46576 100644 --- a/sound/soc/amd/acp/acp70.c +++ b/sound/soc/amd/acp/acp70.c @@ -23,7 +23,7 @@ #include "amd.h" #include "acp-mach.h" -#include +#include #define DRV_NAME "acp_asoc_acp70" diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c index 7c6d647fa253..7e6f10726ff0 100644 --- a/sound/soc/sof/amd/acp.c +++ b/sound/soc/sof/amd/acp.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include "../ops.h" #include "acp.h" From f99002b9a9cc441a8f362e6fb32cf8a5a990261a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 14 Apr 2025 09:39:33 +0200 Subject: [PATCH 202/331] x86/alternatives, um: Rename UML's text_poke_sync() wrapper to smp_text_poke_sync_each_cpu() Missed this UML wrapper in the rename. Fixes: 6e4955a9d73e ("x86/alternatives: Rename 'text_poke_sync()' to 'smp_text_poke_sync_each_cpu()'") Reported-by: kernel test robot Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/202504141003.kc69fVoj-lkp@intel.com --- arch/um/kernel/um_arch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index d4b3b6742ec8..2f5ee045bc7a 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -477,7 +477,7 @@ void *text_poke_copy(void *addr, const void *opcode, size_t len) return text_poke(addr, opcode, len); } -void text_poke_sync(void) +void smp_text_poke_sync_each_cpu(void) { } From 9fb6938d55348ee3e47deff9646fae59a15ed1c9 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Mon, 14 Apr 2025 11:41:29 +0200 Subject: [PATCH 203/331] x86/cpuid: Align macro linebreaks vertically Align the backspaces vertically again, after recent cleanups. No functional changes. Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Cc: Ahmed S. Darwish Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250414094130.6768-1-bp@kernel.org --- arch/x86/include/asm/cpuid/api.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index bf75c6267311..bf76a1706d02 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -36,9 +36,9 @@ static inline void native_cpuid(u32 *eax, u32 *ebx, } #define NATIVE_CPUID_REG(reg) \ -static inline u32 native_cpuid_##reg(u32 op) \ +static inline u32 native_cpuid_##reg(u32 op) \ { \ - u32 eax = op, ebx, ecx = 0, edx; \ + u32 eax = op, ebx, ecx = 0, edx; \ \ native_cpuid(&eax, &ebx, &ecx, &edx); \ \ From dd86a1d013e0c94fedd060514b9e7be2988ef320 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Mon, 14 Apr 2025 17:09:51 +0200 Subject: [PATCH 204/331] x86/bugs: Remove X86_BUG_MMIO_UNKNOWN Whack this thing because: - the "unknown" handling is done only for this vuln and not for the others - it doesn't do anything besides reporting things differently. It doesn't apply any mitigations - it is simply causing unnecessary complications to the code which don't bring anything besides maintenance overhead to what is already a very nasty spaghetti pile - all the currently unaffected CPUs can also be in "unknown" status so there's no need for special handling here so get rid of it. Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: David Kaplan Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Pawan Gupta Link: https://lore.kernel.org/r/20250414150951.5345-1-bp@kernel.org --- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/kernel/cpu/bugs.c | 12 +----------- arch/x86/kernel/cpu/common.c | 5 ----- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 6c2c152d8a67..e8f8d43a3825 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -519,7 +519,7 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* "itlb_multihit" CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* "srbds" CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* "mmio_stale_data" CPU is affected by Processor MMIO Stale Data vulnerabilities */ -#define X86_BUG_MMIO_UNKNOWN X86_BUG(26) /* "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */ +/* unused, was #define X86_BUG_MMIO_UNKNOWN X86_BUG(26) "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */ #define X86_BUG_RETBLEED X86_BUG(27) /* "retbleed" CPU is affected by RETBleed */ #define X86_BUG_EIBRS_PBRSB X86_BUG(28) /* "eibrs_pbrsb" EIBRS is vulnerable to Post Barrier RSB Predictions */ #define X86_BUG_SMT_RSB X86_BUG(29) /* "smt_rsb" CPU is vulnerable to Cross-Thread Return Address Predictions */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 4386aa6c69e1..a91a1cac6183 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -428,7 +428,6 @@ static const char * const mmio_strings[] = { static void __init mmio_select_mitigation(void) { if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || - boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || cpu_mitigations_off()) { mmio_mitigation = MMIO_MITIGATION_OFF; return; @@ -591,8 +590,6 @@ out: pr_info("TAA: %s\n", taa_strings[taa_mitigation]); if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); - else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) - pr_info("MMIO Stale Data: Unknown: No mitigations\n"); if (boot_cpu_has_bug(X86_BUG_RFDS)) pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]); } @@ -2819,9 +2816,6 @@ static ssize_t tsx_async_abort_show_state(char *buf) static ssize_t mmio_stale_data_show_state(char *buf) { - if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) - return sysfs_emit(buf, "Unknown: No mitigations\n"); - if (mmio_mitigation == MMIO_MITIGATION_OFF) return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); @@ -3006,7 +3000,6 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr return srbds_show_state(buf); case X86_BUG_MMIO_STALE_DATA: - case X86_BUG_MMIO_UNKNOWN: return mmio_stale_data_show_state(buf); case X86_BUG_RETBLEED: @@ -3075,10 +3068,7 @@ ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char * ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) { - if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) - return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); - else - return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); + return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); } ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 12126adbc3a9..4ada55f126ae 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1402,15 +1402,10 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) * Affected CPU list is generally enough to enumerate the vulnerability, * but for virtualization case check for ARCH_CAP MSR bits also, VMM may * not want the guest to enumerate the bug. - * - * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist, - * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits. */ if (!arch_cap_mmio_immune(x86_arch_cap_msr)) { if (cpu_matches(cpu_vuln_blacklist, MMIO)) setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA); - else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO)) - setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN); } if (!cpu_has(c, X86_FEATURE_BTC_NO)) { diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 9e3fa7942e7d..e88500d90309 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -508,7 +508,7 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* "itlb_multihit" CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* "srbds" CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* "mmio_stale_data" CPU is affected by Processor MMIO Stale Data vulnerabilities */ -#define X86_BUG_MMIO_UNKNOWN X86_BUG(26) /* "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */ +/* unused, was #define X86_BUG_MMIO_UNKNOWN X86_BUG(26) * "mmio_unknown" CPU is too old and its MMIO Stale Data status is unknown */ #define X86_BUG_RETBLEED X86_BUG(27) /* "retbleed" CPU is affected by RETBleed */ #define X86_BUG_EIBRS_PBRSB X86_BUG(28) /* "eibrs_pbrsb" EIBRS is vulnerable to Post Barrier RSB Predictions */ #define X86_BUG_SMT_RSB X86_BUG(29) /* "smt_rsb" CPU is vulnerable to Cross-Thread Return Address Predictions */ From 282cc5b6762310a73255c6dd1d79de1609042eeb Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 15 Apr 2025 10:54:08 -0700 Subject: [PATCH 205/331] x86/cpufeatures: Clean up formatting It is a special file with special formatting so remove one whitespace damage and format newer defines like the rest. No functional changes. [ Xin: Do the same to tools/arch/x86/include/asm/cpufeatures.h. ] Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Xin Li (Intel) Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250415175410.2944032-2-xin@zytor.com --- arch/x86/include/asm/cpufeatures.h | 20 ++++++++++---------- tools/arch/x86/include/asm/cpufeatures.h | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index e8f8d43a3825..60b4a4c00491 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -477,10 +477,10 @@ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ -#define X86_FEATURE_AMD_FAST_CPPC (21*32 + 5) /* Fast CPPC */ -#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32 + 6) /* Heterogeneous Core Topology */ -#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */ -#define X86_FEATURE_PREFER_YMM (21*32 + 8) /* Avoid ZMM registers due to downclocking */ +#define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ +#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ +#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ +#define X86_FEATURE_PREFER_YMM (21*32+ 8) /* Avoid ZMM registers due to downclocking */ /* * BUG word(s) @@ -527,10 +527,10 @@ #define X86_BUG_TDX_PW_MCE X86_BUG(31) /* "tdx_pw_mce" CPU may incur #MC if non-TD software does partial write to TDX private memory */ /* BUG word 2 */ -#define X86_BUG_SRSO X86_BUG(1*32 + 0) /* "srso" AMD SRSO bug */ -#define X86_BUG_DIV0 X86_BUG(1*32 + 1) /* "div0" AMD DIV0 speculation bug */ -#define X86_BUG_RFDS X86_BUG(1*32 + 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */ -#define X86_BUG_BHI X86_BUG(1*32 + 3) /* "bhi" CPU is affected by Branch History Injection */ -#define X86_BUG_IBPB_NO_RET X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */ -#define X86_BUG_SPECTRE_V2_USER X86_BUG(1*32 + 5) /* "spectre_v2_user" CPU is affected by Spectre variant 2 attack between user processes */ +#define X86_BUG_SRSO X86_BUG( 1*32+ 0) /* "srso" AMD SRSO bug */ +#define X86_BUG_DIV0 X86_BUG( 1*32+ 1) /* "div0" AMD DIV0 speculation bug */ +#define X86_BUG_RFDS X86_BUG( 1*32+ 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */ +#define X86_BUG_BHI X86_BUG( 1*32+ 3) /* "bhi" CPU is affected by Branch History Injection */ +#define X86_BUG_IBPB_NO_RET X86_BUG( 1*32+ 4) /* "ibpb_no_ret" IBPB omits return target predictions */ +#define X86_BUG_SPECTRE_V2_USER X86_BUG( 1*32+ 5) /* "spectre_v2_user" CPU is affected by Spectre variant 2 attack between user processes */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index e88500d90309..2e219be047d4 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -467,9 +467,10 @@ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ -#define X86_FEATURE_AMD_FAST_CPPC (21*32 + 5) /* Fast CPPC */ -#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32 + 6) /* Heterogeneous Core Topology */ -#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */ +#define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ +#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ +#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ +#define X86_FEATURE_PREFER_YMM (21*32+ 8) /* Avoid ZMM registers due to downclocking */ /* * BUG word(s) @@ -516,9 +517,10 @@ #define X86_BUG_TDX_PW_MCE X86_BUG(31) /* "tdx_pw_mce" CPU may incur #MC if non-TD software does partial write to TDX private memory */ /* BUG word 2 */ -#define X86_BUG_SRSO X86_BUG(1*32 + 0) /* "srso" AMD SRSO bug */ -#define X86_BUG_DIV0 X86_BUG(1*32 + 1) /* "div0" AMD DIV0 speculation bug */ -#define X86_BUG_RFDS X86_BUG(1*32 + 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */ -#define X86_BUG_BHI X86_BUG(1*32 + 3) /* "bhi" CPU is affected by Branch History Injection */ -#define X86_BUG_IBPB_NO_RET X86_BUG(1*32 + 4) /* "ibpb_no_ret" IBPB omits return target predictions */ +#define X86_BUG_SRSO X86_BUG( 1*32+ 0) /* "srso" AMD SRSO bug */ +#define X86_BUG_DIV0 X86_BUG( 1*32+ 1) /* "div0" AMD DIV0 speculation bug */ +#define X86_BUG_RFDS X86_BUG( 1*32+ 2) /* "rfds" CPU is vulnerable to Register File Data Sampling */ +#define X86_BUG_BHI X86_BUG( 1*32+ 3) /* "bhi" CPU is affected by Branch History Injection */ +#define X86_BUG_IBPB_NO_RET X86_BUG( 1*32+ 4) /* "ibpb_no_ret" IBPB omits return target predictions */ +#define X86_BUG_SPECTRE_V2_USER X86_BUG( 1*32+ 5) /* "spectre_v2_user" CPU is affected by Spectre variant 2 attack between user processes */ #endif /* _ASM_X86_CPUFEATURES_H */ From 13327fada7ff0ae858e28b9515cd7d6ccb5fccc7 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Tue, 15 Apr 2025 10:54:09 -0700 Subject: [PATCH 206/331] x86/cpufeatures: Shorten X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT Shorten X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT to X86_FEATURE_CLEAR_BHB_VMEXIT to make the last column aligned consistently in the whole file. There's no need to explain in the name what the mitigation does. No functional changes. Suggested-by: Borislav Petkov (AMD) Signed-off-by: Xin Li (Intel) Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250415175410.2944032-3-xin@zytor.com --- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/include/asm/nospec-branch.h | 2 +- arch/x86/kernel/cpu/bugs.c | 6 +++--- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 60b4a4c00491..bd27a1d70177 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -476,7 +476,7 @@ #define X86_FEATURE_CLEAR_BHB_LOOP (21*32+ 1) /* Clear branch history at syscall entry using SW loop */ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ -#define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ +#define X86_FEATURE_CLEAR_BHB_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ #define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ #define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 8a5cc8e70439..707ee5298d2a 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -327,7 +327,7 @@ .endm .macro CLEAR_BRANCH_HISTORY_VMEXIT - ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT + ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_VMEXIT .endm #else #define CLEAR_BRANCH_HISTORY diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index a91a1cac6183..3228f5dd6bd7 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -1701,13 +1701,13 @@ static void __init bhi_select_mitigation(void) if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) { pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n"); - setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT); + setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); return; } pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n"); setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP); - setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT); + setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_VMEXIT); } static void __init spectre_v2_select_mitigation(void) @@ -2891,7 +2891,7 @@ static const char *spectre_bhi_state(void) !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) && rrsba_disabled) return "; BHI: Retpoline"; - else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT)) + else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT)) return "; BHI: Vulnerable, KVM: SW loop"; return "; BHI: Vulnerable"; diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 2e219be047d4..e10c3f4ab328 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -466,7 +466,7 @@ #define X86_FEATURE_CLEAR_BHB_LOOP (21*32+ 1) /* Clear branch history at syscall entry using SW loop */ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ -#define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ +#define X86_FEATURE_CLEAR_BHB_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ #define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ #define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ From 3aba0b40cacdfba4a604dd09315fa6cdbeb0ed90 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Tue, 15 Apr 2025 10:54:10 -0700 Subject: [PATCH 207/331] x86/cpufeatures: Shorten X86_FEATURE_AMD_HETEROGENEOUS_CORES Shorten X86_FEATURE_AMD_HETEROGENEOUS_CORES to X86_FEATURE_AMD_HTR_CORES to make the last column aligned consistently in the whole file. No functional changes. Suggested-by: Borislav Petkov (AMD) Signed-off-by: Xin Li (Intel) Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250415175410.2944032-4-xin@zytor.com --- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/kernel/acpi/cppc.c | 2 +- arch/x86/kernel/cpu/scattered.c | 2 +- arch/x86/kernel/cpu/topology_amd.c | 2 +- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index bd27a1d70177..bc81b9d1aeca 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -478,7 +478,7 @@ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ #define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ -#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ +#define X86_FEATURE_AMD_HTR_CORES (21*32+ 6) /* Heterogeneous Core Topology */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ #define X86_FEATURE_PREFER_YMM (21*32+ 8) /* Avoid ZMM registers due to downclocking */ diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 77bfb846490c..62ca714aae77 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -272,7 +272,7 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) } /* detect if running on heterogeneous design */ - if (cpu_feature_enabled(X86_FEATURE_AMD_HETEROGENEOUS_CORES)) { + if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { switch (core_type) { case TOPO_CPU_TYPE_UNKNOWN: pr_warn("Undefined core type found for cpu %d\n", cpu); diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 16f3ca30626a..c75c57b32b74 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -53,7 +53,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_PMC_FREEZE, CPUID_EAX, 2, 0x80000022, 0 }, - { X86_FEATURE_AMD_HETEROGENEOUS_CORES, CPUID_EAX, 30, 0x80000026, 0 }, + { X86_FEATURE_AMD_HTR_CORES, CPUID_EAX, 30, 0x80000026, 0 }, { 0, 0, 0, 0, 0 } }; diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index 03b3c9c3a45e..eb799e2405f9 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -182,7 +182,7 @@ static void parse_topology_amd(struct topo_scan *tscan) if (cpu_feature_enabled(X86_FEATURE_TOPOEXT)) has_topoext = cpu_parse_topology_ext(tscan); - if (cpu_feature_enabled(X86_FEATURE_AMD_HETEROGENEOUS_CORES)) + if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) tscan->c->topo.cpu_type = cpuid_ebx(0x80000026); if (!has_topoext && !parse_8000_0008(tscan)) diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index e10c3f4ab328..fdbc92aca812 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -468,7 +468,7 @@ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ #define X86_FEATURE_AMD_FAST_CPPC (21*32+ 5) /* Fast CPPC */ -#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32+ 6) /* Heterogeneous Core Topology */ +#define X86_FEATURE_AMD_HTR_CORES (21*32+ 6) /* Heterogeneous Core Topology */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ #define X86_FEATURE_PREFER_YMM (21*32+ 8) /* Avoid ZMM registers due to downclocking */ From b02dc185ee86836cf1d8a37b81349374e4018ee0 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:51 -0700 Subject: [PATCH 208/331] x86/cpufeatures: Add X86_FEATURE_APX Intel Advanced Performance Extensions (APX) introduce a new set of general-purpose registers, managed as an extended state component via the xstate management facility. Before enabling this new xstate, define a feature flag to clarify the dependency in xsave_cpuid_features[]. APX is enumerated under CPUID level 7 with EDX=1. Since this CPUID leaf is not yet allocated, place the flag in a scattered feature word. While this feature is intended only for userspace, exposing it via /proc/cpuinfo is unnecessary. Instead, the existing arch_prctl(2) mechanism with the ARCH_GET_XCOMP_SUPP option can be used to query the feature availability. Finally, clarify that APX depends on XSAVE. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Reviewed-by: Sohil Mehta Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-2-chang.seok.bae@intel.com --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/cpuid-deps.c | 1 + arch/x86/kernel/cpu/scattered.c | 1 + 3 files changed, 3 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index bc81b9d1aeca..478ab362fda2 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -481,6 +481,7 @@ #define X86_FEATURE_AMD_HTR_CORES (21*32+ 6) /* Heterogeneous Core Topology */ #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32+ 7) /* Workload Classification */ #define X86_FEATURE_PREFER_YMM (21*32+ 8) /* Avoid ZMM registers due to downclocking */ +#define X86_FEATURE_APX (21*32+ 9) /* Advanced Performance Extensions */ /* * BUG word(s) diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index 94c062cddfa4..46efcbd6afa4 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -28,6 +28,7 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_PKU, X86_FEATURE_XSAVE }, { X86_FEATURE_MPX, X86_FEATURE_XSAVE }, { X86_FEATURE_XGETBV1, X86_FEATURE_XSAVE }, + { X86_FEATURE_APX, X86_FEATURE_XSAVE }, { X86_FEATURE_CMOV, X86_FEATURE_FXSR }, { X86_FEATURE_MMX, X86_FEATURE_FXSR }, { X86_FEATURE_MMXEXT, X86_FEATURE_MMX }, diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index c75c57b32b74..dbf6d71bdf18 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -27,6 +27,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_APERFMPERF, CPUID_ECX, 0, 0x00000006, 0 }, { X86_FEATURE_EPB, CPUID_ECX, 3, 0x00000006, 0 }, { X86_FEATURE_INTEL_PPIN, CPUID_EBX, 0, 0x00000007, 1 }, + { X86_FEATURE_APX, CPUID_EDX, 21, 0x00000007, 1 }, { X86_FEATURE_RRSBA_CTRL, CPUID_EDX, 2, 0x00000007, 2 }, { X86_FEATURE_BHI_CTRL, CPUID_EDX, 4, 0x00000007, 2 }, { X86_FEATURE_CQM_LLC, CPUID_EDX, 1, 0x0000000f, 0 }, From bd0b10b795c5c4c587e83c0498251356874c655c Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:52 -0700 Subject: [PATCH 209/331] x86/fpu/apx: Define APX state component Advanced Performance Extensions (APX) is associated with a new state component number 19. To support saving and restoring of the corresponding registers via the XSAVE mechanism, introduce the component definition along with the necessary sanity checks. Define the new component number, state name, and those register data type. Then, extend the size checker to validate the register data type and explicitly list the APX feature flag as a dependency for the new component in xsave_cpuid_features[]. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Reviewed-by: Sohil Mehta Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-3-chang.seok.bae@intel.com --- arch/x86/include/asm/fpu/types.h | 9 +++++++++ arch/x86/kernel/fpu/xstate.c | 3 +++ 2 files changed, 12 insertions(+) diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h index de16862bf230..97310df3ea13 100644 --- a/arch/x86/include/asm/fpu/types.h +++ b/arch/x86/include/asm/fpu/types.h @@ -125,6 +125,7 @@ enum xfeature { XFEATURE_RSRVD_COMP_16, XFEATURE_XTILE_CFG, XFEATURE_XTILE_DATA, + XFEATURE_APX, XFEATURE_MAX, }; @@ -145,6 +146,7 @@ enum xfeature { #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) +#define XFEATURE_MASK_APX (1 << XFEATURE_APX) #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ @@ -303,6 +305,13 @@ struct xtile_data { struct reg_1024_byte tmm; } __packed; +/* + * State component 19: 8B extended general purpose register. + */ +struct apx_state { + u64 egpr[16]; +} __packed; + /* * State component 10 is supervisor state used for context-switching the * PASID state. diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index a288597065fd..dfd07af10037 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -63,6 +63,7 @@ static const char *xfeature_names[] = "unknown xstate feature", "AMX Tile config", "AMX Tile data", + "APX registers", "unknown xstate feature", }; @@ -81,6 +82,7 @@ static unsigned short xsave_cpuid_features[] __initdata = { [XFEATURE_CET_USER] = X86_FEATURE_SHSTK, [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE, [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE, + [XFEATURE_APX] = X86_FEATURE_APX, }; static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = @@ -569,6 +571,7 @@ static bool __init check_xstate_against_struct(int nr) case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state); case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg); case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state); + case XFEATURE_APX: return XCHECK_SZ(sz, nr, struct apx_state); case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true; default: XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr); From ea68e39190cff86f457bd286c70b535e2a99a94d Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:53 -0700 Subject: [PATCH 210/331] x86/fpu/apx: Disallow conflicting MPX presence XSTATE components are architecturally independent. There is no rule requiring their offsets in the non-compacted format to be strictly ascending or mutually non-overlapping. However, in practice, such overlaps have not occurred -- until now. APX is introduced as xstate component 19, following AMX. In the non-compacted XSAVE format, its offset overlaps with the space previously occupied by the now-deprecated MPX feature: 45fc24e89b7c ("x86/mpx: remove MPX from arch/x86") To prevent conflicts, the kernel must ensure the CPU never expose both features at the same time. If so, it indicates unreliable hardware. In such cases, XSAVE should be disabled entirely as a precautionary measure. Add a sanity check to detect this condition and disable XSAVE if an invalid hardware configuration is identified. Note: MPX state components remain enabled on legacy systems solely for KVM guest support. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Reviewed-by: Sohil Mehta Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-4-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index dfd07af10037..14f5c1bb2080 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -814,6 +814,17 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) goto out_disable; } + if (fpu_kernel_cfg.max_features & XFEATURE_MASK_APX && + fpu_kernel_cfg.max_features & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR)) { + /* + * This is a problematic CPU configuration where two + * conflicting state components are both enumerated. + */ + pr_err("x86/fpu: Both APX/MPX present in the CPU's xstate features: 0x%llx, disabling XSAVE.\n", + fpu_kernel_cfg.max_features); + goto out_disable; + } + fpu_kernel_cfg.independent_features = fpu_kernel_cfg.max_features & XFEATURE_MASK_INDEPENDENT; From 50c5b071e2833d2b61e3774cd792620311df157c Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:54 -0700 Subject: [PATCH 211/331] x86/fpu/apx: Enable APX state support With securing APX against conflicting MPX, it is now ready to be enabled. Include APX in the enabled xfeature set. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Reviewed-by: Sohil Mehta Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-5-chang.seok.bae@intel.com --- arch/x86/include/asm/fpu/xstate.h | 3 ++- arch/x86/kernel/fpu/xstate.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 7f39fe7980c5..b308a76afbb7 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -32,7 +32,8 @@ XFEATURE_MASK_PKRU | \ XFEATURE_MASK_BNDREGS | \ XFEATURE_MASK_BNDCSR | \ - XFEATURE_MASK_XTILE) + XFEATURE_MASK_XTILE | \ + XFEATURE_MASK_APX) /* * Features which are restored when returning to user space. diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 14f5c1bb2080..2ac1fc182273 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -371,7 +371,8 @@ static __init void os_xrstor_booting(struct xregs_state *xstate) XFEATURE_MASK_BNDCSR | \ XFEATURE_MASK_PASID | \ XFEATURE_MASK_CET_USER | \ - XFEATURE_MASK_XTILE) + XFEATURE_MASK_XTILE | \ + XFEATURE_MASK_APX) /* * setup the xstate image representing the init state From ab6f87ddd0c6d3fb114cdf897eb9839cbd429439 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:55 -0700 Subject: [PATCH 212/331] selftests/x86/apx: Add APX test The extended general-purpose registers for APX may contain random data, which is currently assumed by the xstate testing framework. This allows the testing of the new userspace feature using the common test code. Invoke the test entry function from apx.c after enumerating the state component and adding it to the support list Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Reviewed-by: Sohil Mehta Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-6-chang.seok.bae@intel.com --- tools/testing/selftests/x86/Makefile | 3 ++- tools/testing/selftests/x86/apx.c | 10 ++++++++++ tools/testing/selftests/x86/xstate.c | 3 ++- tools/testing/selftests/x86/xstate.h | 2 ++ 4 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/x86/apx.c diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile index 28422c32cc8f..f703fcfe9f7c 100644 --- a/tools/testing/selftests/x86/Makefile +++ b/tools/testing/selftests/x86/Makefile @@ -19,7 +19,7 @@ TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ test_FCMOV test_FCOMI test_FISTTP \ vdso_restorer TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \ - corrupt_xstate_header amx lam test_shadow_stack avx + corrupt_xstate_header amx lam test_shadow_stack avx apx # Some selftests require 32bit support enabled also on 64bit systems TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall @@ -136,3 +136,4 @@ $(OUTPUT)/nx_stack_64: CFLAGS += -Wl,-z,noexecstack $(OUTPUT)/avx_64: CFLAGS += -mno-avx -mno-avx512f $(OUTPUT)/amx_64: EXTRA_FILES += xstate.c $(OUTPUT)/avx_64: EXTRA_FILES += xstate.c +$(OUTPUT)/apx_64: EXTRA_FILES += xstate.c diff --git a/tools/testing/selftests/x86/apx.c b/tools/testing/selftests/x86/apx.c new file mode 100644 index 000000000000..d9c8d41b8c5a --- /dev/null +++ b/tools/testing/selftests/x86/apx.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE + +#include "xstate.h" + +int main(void) +{ + test_xstate(XFEATURE_APX); +} diff --git a/tools/testing/selftests/x86/xstate.c b/tools/testing/selftests/x86/xstate.c index 23c1d6c964ea..97fe4bd8bc77 100644 --- a/tools/testing/selftests/x86/xstate.c +++ b/tools/testing/selftests/x86/xstate.c @@ -31,7 +31,8 @@ (1 << XFEATURE_OPMASK) | \ (1 << XFEATURE_ZMM_Hi256) | \ (1 << XFEATURE_Hi16_ZMM) | \ - (1 << XFEATURE_XTILEDATA)) + (1 << XFEATURE_XTILEDATA) | \ + (1 << XFEATURE_APX)) static inline uint64_t xgetbv(uint32_t index) { diff --git a/tools/testing/selftests/x86/xstate.h b/tools/testing/selftests/x86/xstate.h index 42af36ec852f..e91e3092b5d2 100644 --- a/tools/testing/selftests/x86/xstate.h +++ b/tools/testing/selftests/x86/xstate.h @@ -33,6 +33,7 @@ enum xfeature { XFEATURE_RSRVD_COMP_16, XFEATURE_XTILECFG, XFEATURE_XTILEDATA, + XFEATURE_APX, XFEATURE_MAX, }; @@ -59,6 +60,7 @@ static const char *xfeature_names[] = "unknown xstate feature", "AMX Tile config", "AMX Tile data", + "APX registers", "unknown xstate feature", }; From 39cd7fad39ce2ffbeb21939c805ee55f3ec808d4 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:56 -0700 Subject: [PATCH 213/331] x86/fpu: Log XSAVE disablement consistently Not all paths that lead to fpu__init_disable_system_xstate() currently emit a message indicating that XSAVE has been disabled. Move the print statement into the function to ensure the message in all cases. Suggested-by: Dave Hansen Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250416021720.12305-7-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 2ac1fc182273..8b14c9d3a1df 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -751,6 +751,8 @@ static int __init init_xstate_size(void) */ static void __init fpu__init_disable_system_xstate(unsigned int legacy_size) { + pr_info("x86/fpu: XSAVE disabled\n"); + fpu_kernel_cfg.max_features = 0; cr4_clear_bits(X86_CR4_OSXSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVE); @@ -821,7 +823,7 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) * This is a problematic CPU configuration where two * conflicting state components are both enumerated. */ - pr_err("x86/fpu: Both APX/MPX present in the CPU's xstate features: 0x%llx, disabling XSAVE.\n", + pr_err("x86/fpu: Both APX/MPX present in the CPU's xstate features: 0x%llx.\n", fpu_kernel_cfg.max_features); goto out_disable; } @@ -900,7 +902,7 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) init_fpstate.xfeatures = fpu_kernel_cfg.default_features; if (init_fpstate.size > sizeof(init_fpstate.regs)) { - pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d), disabling XSAVE\n", + pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d)\n", sizeof(init_fpstate.regs), init_fpstate.size); goto out_disable; } @@ -912,7 +914,7 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) * xfeatures mask. */ if (xfeatures != fpu_kernel_cfg.max_features) { - pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n", + pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init\n", xfeatures, fpu_kernel_cfg.max_features); goto out_disable; } From 64e54461ab6e8524a8de4e63b7d1a3e4481b5cf3 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:57 -0700 Subject: [PATCH 214/331] x86/fpu: Refactor xfeature bitmask update code for sigframe XSAVE Currently, saving register states in the signal frame, the legacy feature bits are always set in xregs_state->header->xfeatures. This code sequence can be generalized for reuse in similar cases. Refactor the logic to ensure a consistent approach across similar usages. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-8-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/signal.c | 11 +---------- arch/x86/kernel/fpu/xstate.h | 13 +++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index b8b4fa9c2d04..c3ec2512f2bb 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -114,7 +114,6 @@ static inline bool save_xstate_epilog(void __user *buf, int ia32_frame, { struct xregs_state __user *x = buf; struct _fpx_sw_bytes sw_bytes = {}; - u32 xfeatures; int err; /* Setup the bytes not touched by the [f]xsave and reserved for SW. */ @@ -127,12 +126,6 @@ static inline bool save_xstate_epilog(void __user *buf, int ia32_frame, err |= __put_user(FP_XSTATE_MAGIC2, (__u32 __user *)(buf + fpstate->user_size)); - /* - * Read the xfeatures which we copied (directly from the cpu or - * from the state in task struct) to the user buffers. - */ - err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures); - /* * For legacy compatible, we always set FP/SSE bits in the bit * vector while saving the state to the user context. This will @@ -144,9 +137,7 @@ static inline bool save_xstate_epilog(void __user *buf, int ia32_frame, * header as well as change any contents in the memory layout. * xrestore as part of sigreturn will capture all the changes. */ - xfeatures |= XFEATURE_MASK_FPSSE; - - err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures); + err |= set_xfeature_in_sigframe(x, XFEATURE_MASK_FPSSE); return !err; } diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h index 9a3a8ccf13bf..4231e449f0b4 100644 --- a/arch/x86/kernel/fpu/xstate.h +++ b/arch/x86/kernel/fpu/xstate.h @@ -69,6 +69,19 @@ static inline u64 xfeatures_mask_independent(void) return fpu_kernel_cfg.independent_features; } +static inline int set_xfeature_in_sigframe(struct xregs_state __user *xbuf, u64 mask) +{ + u64 xfeatures; + int err; + + /* Read the xfeatures value already saved in the user buffer */ + err = __get_user(xfeatures, &xbuf->header.xfeatures); + xfeatures |= mask; + err |= __put_user(xfeatures, &xbuf->header.xfeatures); + + return err; +} + /* * Update the value of PKRU register that was already pushed onto the signal frame. */ From d1e420772cd1eb0afe5858619c73ce36f3e781a1 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:58 -0700 Subject: [PATCH 215/331] x86/pkeys: Simplify PKRU update in signal frame The signal delivery logic was modified to always set the PKRU bit in xregs_state->header->xfeatures by this commit: ae6012d72fa6 ("x86/pkeys: Ensure updated PKRU value is XRSTOR'd") However, the change derives the bitmask value using XGETBV(1), rather than simply updating the buffer that already holds the value. Thus, this approach induces an unnecessary dependency on XGETBV1 for PKRU handling. Eliminate the dependency by using the established helper function. Subsequently, remove the now-unused 'mask' argument. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Aruna Ramakrishna Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Tony W Wang-oc Cc: Dave Hansen Link: https://lore.kernel.org/r/20250416021720.12305-9-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/xstate.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h index 4231e449f0b4..a0256ef34ecb 100644 --- a/arch/x86/kernel/fpu/xstate.h +++ b/arch/x86/kernel/fpu/xstate.h @@ -85,18 +85,15 @@ static inline int set_xfeature_in_sigframe(struct xregs_state __user *xbuf, u64 /* * Update the value of PKRU register that was already pushed onto the signal frame. */ -static inline int update_pkru_in_sigframe(struct xregs_state __user *buf, u64 mask, u32 pkru) +static inline int update_pkru_in_sigframe(struct xregs_state __user *buf, u32 pkru) { - u64 xstate_bv; int err; if (unlikely(!cpu_feature_enabled(X86_FEATURE_OSPKE))) return 0; /* Mark PKRU as in-use so that it is restored correctly. */ - xstate_bv = (mask & xfeatures_in_use()) | XFEATURE_MASK_PKRU; - - err = __put_user(xstate_bv, &buf->header.xfeatures); + err = set_xfeature_in_sigframe(buf, XFEATURE_MASK_PKRU); if (err) return err; @@ -320,7 +317,7 @@ static inline int xsave_to_user_sigframe(struct xregs_state __user *buf, u32 pkr clac(); if (!err) - err = update_pkru_in_sigframe(buf, mask, pkru); + err = update_pkru_in_sigframe(buf, pkru); return err; } From 70fe4a0266ef156f3a49071da0d9ea6af0f49c44 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:16:59 -0700 Subject: [PATCH 216/331] x86/fpu: Remove export of mxcsr_feature_mask The variable was previously referenced in KVM code but the last usage was removed by: ea4d6938d4c0 ("x86/fpu: Replace KVMs home brewed FPU copy from user") Remove its export symbol. Reviewed-by: Nikolay Borisov Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-10-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/init.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 16b6611634c3..2d9b5e677559 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -100,7 +100,6 @@ static void __init fpu__init_system_early_generic(void) * Boot time FPU feature detection code: */ unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu; -EXPORT_SYMBOL_GPL(mxcsr_feature_mask); static void __init fpu__init_system_mxcsr(void) { From de8304c319bc020ef79d109909ad40e944d82c82 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Tue, 15 Apr 2025 19:17:00 -0700 Subject: [PATCH 217/331] x86/fpu: Rename fpu_reset_fpregs() to fpu_reset_fpstate_regs() The original function name came from an overly compressed form of 'fpstate_regs' by commit: e61d6310a0f8 ("x86/fpu: Reset permission and fpstate on exec()") However, the term 'fpregs' typically refers to physical FPU registers. In contrast, this function copies the init values to fpu->fpstate->regs, not hardware registers. Rename the function to better reflect what it actually does. No functional change. Signed-off-by: Chang S. Bae Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Link: https://lore.kernel.org/r/20250416021720.12305-11-chang.seok.bae@intel.com --- arch/x86/kernel/fpu/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 3a19877a314e..8d674435f173 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -733,7 +733,7 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask) /* * Reset current->fpu memory state to the init values. */ -static void fpu_reset_fpregs(void) +static void fpu_reset_fpstate_regs(void) { struct fpu *fpu = x86_task_fpu(current); @@ -768,7 +768,7 @@ void fpu__clear_user_states(struct fpu *fpu) fpregs_lock(); if (!cpu_feature_enabled(X86_FEATURE_FPU)) { - fpu_reset_fpregs(); + fpu_reset_fpstate_regs(); fpregs_unlock(); return; } @@ -798,7 +798,7 @@ void fpu__clear_user_states(struct fpu *fpu) void fpu_flush_thread(void) { fpstate_reset(x86_task_fpu(current)); - fpu_reset_fpregs(); + fpu_reset_fpstate_regs(); } /* * Load FPU context before returning to userspace. From d9b79111fd9945931b7a2b2a3e7db7625dd953fe Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Wed, 16 Apr 2025 06:47:51 -0700 Subject: [PATCH 218/331] x86/bugs: Rename mmio_stale_data_clear to cpu_buf_vm_clear The static key mmio_stale_data_clear controls the KVM-only mitigation for MMIO Stale Data vulnerability. Rename it to reflect its purpose. No functional change. Signed-off-by: Pawan Gupta Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250416-mmio-rename-v2-1-ad1f5488767c@linux.intel.com --- arch/x86/include/asm/nospec-branch.h | 2 +- arch/x86/kernel/cpu/bugs.c | 16 ++++++++++------ arch/x86/kvm/vmx/vmx.c | 6 +++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 5c43f145454d..81c4a13e4c0d 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -561,7 +561,7 @@ DECLARE_STATIC_KEY_FALSE(mds_idle_clear); DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); -DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear); +DECLARE_STATIC_KEY_FALSE(cpu_buf_vm_clear); extern u16 mds_verw_sel; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 362602b705cc..9131e612de17 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -127,9 +127,13 @@ EXPORT_SYMBOL_GPL(mds_idle_clear); */ DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); -/* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ -DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); -EXPORT_SYMBOL_GPL(mmio_stale_data_clear); +/* + * Controls CPU Fill buffer clear before VMenter. This is a subset of + * X86_FEATURE_CLEAR_CPU_BUF, and should only be enabled when KVM-only + * mitigation is required. + */ +DEFINE_STATIC_KEY_FALSE(cpu_buf_vm_clear); +EXPORT_SYMBOL_GPL(cpu_buf_vm_clear); void __init cpu_select_mitigations(void) { @@ -449,9 +453,9 @@ static void __init mmio_select_mitigation(void) * mitigations, disable KVM-only mitigation in that case. */ if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) - static_branch_disable(&mmio_stale_data_clear); + static_branch_disable(&cpu_buf_vm_clear); else - static_branch_enable(&mmio_stale_data_clear); + static_branch_enable(&cpu_buf_vm_clear); /* * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can @@ -571,7 +575,7 @@ static void __init md_clear_update_mitigation(void) taa_select_mitigation(); } /* - * MMIO_MITIGATION_OFF is not checked here so that mmio_stale_data_clear + * MMIO_MITIGATION_OFF is not checked here so that cpu_buf_vm_clear * gets updated correctly as per X86_FEATURE_CLEAR_CPU_BUF state. */ if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 5c5766467a61..a1754f7ba889 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7358,10 +7358,14 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, * mitigation for MDS is done late in VMentry and is still * executed in spite of L1D Flush. This is because an extra VERW * should not matter much after the big hammer L1D Flush. + * + * cpu_buf_vm_clear is used when system is not vulnerable to MDS/TAA, + * and is affected by MMIO Stale Data. In such cases mitigation in only + * needed against an MMIO capable guest. */ if (static_branch_unlikely(&vmx_l1d_should_flush)) vmx_l1d_flush(vcpu); - else if (static_branch_unlikely(&mmio_stale_data_clear) && + else if (static_branch_unlikely(&cpu_buf_vm_clear) && kvm_arch_has_assigned_device(vcpu->kvm)) mds_clear_cpu_buffers(); From 498cb872a111e25021ca5e2d91af7b7a2e62630f Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 14 Apr 2025 12:26:07 -0700 Subject: [PATCH 219/331] x86/boot/startup: Disable LTO for the startup code When building with CONFIG_LTO_CLANG, there is an error in the x86 boot startup code because it builds with a different code model than the rest of the kernel: ld.lld: error: Function Import: link error: linking module flags 'Code Model': IDs have conflicting values: 'i32 2' from vmlinux.a(head64.o at 1302448), and 'i32 1' from vmlinux.a(map_kernel.o at 1314208) ld.lld: error: Function Import: link error: linking module flags 'Code Model': IDs have conflicting values: 'i32 2' from vmlinux.a(common.o at 1306108), and 'i32 1' from vmlinux.a(gdt_idt.o at 1314148) As this directory is for code that only runs during early system initialization, LTO is not very important, so filter out the LTO flags from KBUILD_CFLAGS for arch/x86/boot/startup to resolve the build error. Fixes: 4cecebf200ef ("x86/boot: Move the early GDT/IDT setup code into startup/") Reported-by: Linux Kernel Functional Testing Signed-off-by: Nathan Chancellor Signed-off-by: Ingo Molnar Acked-by: Ard Biesheuvel Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Cc: David Woodhouse Cc: llvm@lists.linux.dev Link: https://lore.kernel.org/r/20250414-x86-boot-startup-lto-error-v1-1-7c8bed7c131c@kernel.org Closes: https://lore.kernel.org/CA+G9fYvnun+bhYgtt425LWxzOmj+8Jf3ruKeYxQSx-F6U7aisg@mail.gmail.com/ --- arch/x86/boot/startup/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index ccdfc42a4d59..bb7c826e586c 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -7,8 +7,9 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS -mcmodel=small -fPIC \ -fno-stack-protector -D__NO_FORTIFY \ -include $(srctree)/include/linux/hidden.h -# disable ftrace hooks +# disable ftrace hooks and LTO KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) KASAN_SANITIZE := n KCSAN_SANITIZE := n KMSAN_SANITIZE := n From 52ebfe7412ce4b3af54fe962af58efe9b25cd9a9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 17 Apr 2025 14:34:13 +0200 Subject: [PATCH 220/331] x86/mm: Remove the mm_cpumask(prev) warning from switch_mm_irqs_off() The CONFIG_DEBUG_VM=y warning in switch_mm_irqs_off() started triggering in testing: VM_WARN_ON_ONCE(prev != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(prev))); AFAIU what happens is that unuse_temporary_mm() clears the mm_cpumask() for the current CPU, while switch_mm_irqs_off() then checks that the mm_cpumask() bit is set for the current CPU. While this behaviour hasn't really changed since the following commit: 209954cbc7d0 ("x86/mm/tlb: Update mm_cpumask lazily") introduced both, but the warning is wrong, so remove it. [ mingo: Patchified Peter's email. ] Reported-by: syzbot+c2537ce72a879a38113e@syzkaller.appspotmail.com Reported-by: Borislav Petkov Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: Andrew Cooper Cc: Rik van Riel Cc: "H. Peter Anvin" Cc: Linus Torvalds Cc: linux-kernel@vger.kernel.org Link: https://lore.kernel.org/r/20250414135629.GA17910@noisy.programming.kicks-ass.net --- arch/x86/mm/tlb.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index c9b87e5f569a..79c124f6f3f2 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -905,14 +905,6 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING); barrier(); - /* - * Leave this CPU in prev's mm_cpumask. Atomic writes to - * mm_cpumask can be expensive under contention. The CPU - * will be removed lazily at TLB flush time. - */ - VM_WARN_ON_ONCE(prev != &init_mm && !cpumask_test_cpu(cpu, - mm_cpumask(prev))); - /* Start receiving IPIs and then read tlb_gen (and LAM below) */ if (next != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(next))) cpumask_set_cpu(cpu, mm_cpumask(next)); From 780f97e309302fdee05b31c91a4dc81ded4c3702 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:34 -0700 Subject: [PATCH 221/331] x86/mm: Always allocate a whole page for PAE PGDs A hardware PAE PGD is only 32 bytes. A PGD is PAGE_SIZE in the other paging modes. But for reasons*, the kernel _sometimes_ allocates a whole page even though it only ever uses 32 bytes. Make PAE less weird. Just allocate a page like the other paging modes. This was already being done for PTI (and Xen in the past) and nobody screamed that loudly about it so it can't be that bad. * The original reason for PAGE_SIZE allocations for the PAE PGDs was Xen's need to detect page table writes. But 32-bit PTI forced it too for reasons I'm unclear about. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173234.D34F0C3E%40davehans-spike.ostc.intel.com --- arch/x86/mm/pgtable.c | 62 +++---------------------------------------- 1 file changed, 4 insertions(+), 58 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index a05fcddfc811..ea01b5572442 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -318,68 +318,15 @@ static void pgd_prepopulate_user_pmd(struct mm_struct *mm, { } #endif -/* - * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also - * assumes that pgd should be in one page. - * - * But kernel with PAE paging that is not running as a Xen domain - * only needs to allocate 32 bytes for pgd instead of one page. - */ -#ifdef CONFIG_X86_PAE - -#include - -#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) -#define PGD_ALIGN 32 - -static struct kmem_cache *pgd_cache; - -void __init pgtable_cache_init(void) -{ - /* - * When PAE kernel is running as a Xen domain, it does not use - * shared kernel pmd. And this requires a whole page for pgd. - */ - if (!SHARED_KERNEL_PMD) - return; - - /* - * when PAE kernel is not running as a Xen domain, it uses - * shared kernel pmd. Shared kernel pmd does not require a whole - * page for pgd. We are able to just allocate a 32-byte for pgd. - * During boot time, we create a 32-byte slab for pgd table allocation. - */ - pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN, - SLAB_PANIC, NULL); -} static inline pgd_t *_pgd_alloc(struct mm_struct *mm) { /* - * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain. - * We allocate one page for pgd. + * PTI and Xen need a whole page for the PAE PGD + * even though the hardware only needs 32 bytes. + * + * For simplicity, allocate a page for all users. */ - if (!SHARED_KERNEL_PMD) - return __pgd_alloc(mm, PGD_ALLOCATION_ORDER); - - /* - * Now PAE kernel is not running as a Xen domain. We can allocate - * a 32-byte slab for pgd to save memory space. - */ - return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER); -} - -static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd) -{ - if (!SHARED_KERNEL_PMD) - __pgd_free(mm, pgd); - else - kmem_cache_free(pgd_cache, pgd); -} -#else - -static inline pgd_t *_pgd_alloc(struct mm_struct *mm) -{ return __pgd_alloc(mm, PGD_ALLOCATION_ORDER); } @@ -387,7 +334,6 @@ static inline void _pgd_free(struct mm_struct *mm, pgd_t *pgd) { __pgd_free(mm, pgd); } -#endif /* CONFIG_X86_PAE */ pgd_t *pgd_alloc(struct mm_struct *mm) { From b0cc4d19f198cdfd1b58c8f5536670d1dc68cbbd Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:35 -0700 Subject: [PATCH 222/331] x86/mm: Always "broadcast" PMD setting operations Kernel PMDs can either be shared across processes or private to a process. On 64-bit, they are always shared. 32-bit non-PAE hardware does not have PMDs, but the kernel logically squishes them into the PGD and treats them as private. Here are the four cases: 64-bit: Shared 32-bit: non-PAE: Private 32-bit: PAE+ PTI: Private 32-bit: PAE+noPTI: Shared Note that 32-bit is all "Private" except for PAE+noPTI being an oddball. The 32-bit+PAE+noPTI case will be made like the rest of 32-bit shortly. But until that can be done, temporarily treat the 32-bit+PAE+noPTI case as Private. This will do unnecessary walks across pgd_list and unnecessary PTE setting but should be otherwise harmless. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173235.F63F50D1%40davehans-spike.ostc.intel.com --- arch/x86/mm/pat/set_memory.c | 4 ++-- arch/x86/mm/pgtable.c | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index def3d9284254..30ab4aced761 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -889,7 +889,7 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) /* change init_mm */ set_pte_atomic(kpte, pte); #ifdef CONFIG_X86_32 - if (!SHARED_KERNEL_PMD) { + { struct page *page; list_for_each_entry(page, &pgd_list, lru) { @@ -1293,7 +1293,7 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr, /* Queue the page table to be freed after TLB flush */ list_add(&page_ptdesc(pmd_page(old_pmd))->pt_list, pgtables); - if (IS_ENABLED(CONFIG_X86_32) && !SHARED_KERNEL_PMD) { + if (IS_ENABLED(CONFIG_X86_32)) { struct page *page; /* Update all PGD tables to use the same large page */ diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index ea01b5572442..f1c58860b926 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -97,18 +97,13 @@ static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) KERNEL_PGD_PTRS); } - /* list required to sync kernel mapping updates */ - if (!SHARED_KERNEL_PMD) { - pgd_set_mm(pgd, mm); - pgd_list_add(pgd); - } + /* List used to sync kernel mapping updates */ + pgd_set_mm(pgd, mm); + pgd_list_add(pgd); } static void pgd_dtor(pgd_t *pgd) { - if (SHARED_KERNEL_PMD) - return; - spin_lock(&pgd_lock); pgd_list_del(pgd); spin_unlock(&pgd_lock); From eb9c7f00f22d6ea2a94e00eb4f33a79064681564 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:37 -0700 Subject: [PATCH 223/331] x86/mm: Always tell core mm to sync kernel mappings Each mm_struct has its own copy of the page tables. When core mm code makes changes to a copy of the page tables those changes sometimes need to be synchronized with other mms' copies of the page tables. But when this synchronization actually needs to happen is highly architecture and configuration specific. In cases where kernel PMDs are shared across processes (SHARED_KERNEL_PMD) the core mm does not itself need to do that synchronization for kernel PMD changes. The x86 code communicates this by clearing the PGTBL_PMD_MODIFIED bit cleared in those configs to avoid expensive synchronization. The kernel is moving toward never sharing kernel PMDs on 32-bit. Prepare for that and make 32-bit PAE always set PGTBL_PMD_MODIFIED, even if there is no modification to synchronize. This obviously adds some synchronization overhead in cases where the kernel page tables are being changed. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173237.EC790E95%40davehans-spike.ostc.intel.com --- arch/x86/include/asm/pgtable-3level_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h index 9d5b257d44e3..9759fa0eb6a3 100644 --- a/arch/x86/include/asm/pgtable-3level_types.h +++ b/arch/x86/include/asm/pgtable-3level_types.h @@ -29,7 +29,7 @@ typedef union { #define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI)) -#define ARCH_PAGE_TABLE_SYNC_MASK (SHARED_KERNEL_PMD ? 0 : PGTBL_PMD_MODIFIED) +#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED /* * PGDIR_SHIFT determines what a top-level page table entry can map From 45fb940563f80b8138f465f18d71c2d3e4a0724e Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:38 -0700 Subject: [PATCH 224/331] x86/mm: Simplify PAE PGD sharing macros There are a few too many levels of abstraction here. First, just expand the PREALLOCATED_PMDS macro in place to make it clear that it is only conditional on PTI. Second, MAX_PREALLOCATED_PMDS is only used in one spot for an on-stack allocation. It has a *maximum* value of 4. Do not bother with the macro MAX() magic. Just set it to 4. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173238.6E3CDA56%40davehans-spike.ostc.intel.com --- arch/x86/mm/pgtable.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index f1c58860b926..027e1d32925c 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -68,12 +68,6 @@ static inline void pgd_list_del(pgd_t *pgd) list_del(&ptdesc->pt_list); } -#define UNSHARED_PTRS_PER_PGD \ - (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) -#define MAX_UNSHARED_PTRS_PER_PGD \ - MAX_T(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) - - static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) { virt_to_ptdesc(pgd)->pt_mm = mm; @@ -132,8 +126,9 @@ static void pgd_dtor(pgd_t *pgd) * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate * and initialize the kernel pmds here. */ -#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD -#define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD +#define PREALLOCATED_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \ + PTRS_PER_PGD : KERNEL_PGD_BOUNDARY) +#define MAX_PREALLOCATED_PMDS PTRS_PER_PGD /* * We allocate separate PMDs for the kernel part of the user page-table From 82f120010f3b86c2f9c1279452c1ecab7bc117d2 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:40 -0700 Subject: [PATCH 225/331] x86/mm: Fix up comments around PMD preallocation The "paravirt environment" is no longer in the tree. Axe that part of the comment. Also add a blurb to remind readers that "USER_PMDS" refer to the PTI user *copy* of the page tables, not the user *portion*. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173240.5B1AB322%40davehans-spike.ostc.intel.com --- arch/x86/mm/pgtable.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 027e1d32925c..ca07db510d26 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -121,16 +121,17 @@ static void pgd_dtor(pgd_t *pgd) * processor notices the update. Since this is expensive, and * all 4 top-level entries are used almost immediately in a * new process's life, we just pre-populate them here. - * - * Also, if we're in a paravirt environment where the kernel pmd is - * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate - * and initialize the kernel pmds here. */ #define PREALLOCATED_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \ PTRS_PER_PGD : KERNEL_PGD_BOUNDARY) #define MAX_PREALLOCATED_PMDS PTRS_PER_PGD /* + * "USER_PMDS" are the PMDs for the user copy of the page tables when + * PTI is enabled. They do not exist when PTI is disabled. Note that + * this is distinct from the user _portion_ of the kernel page tables + * which always exists. + * * We allocate separate PMDs for the kernel part of the user page-table * when PTI is enabled. We need them to map the per-process LDT into the * user-space page-table. From 454e65b4fb38ddeea62472649ef16b5e8d285015 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:41 -0700 Subject: [PATCH 226/331] x86/mm: Preallocate all PAE page tables Finally, move away from having PAE kernels share any PMDs across processes. This was already the default on PTI kernels which are the common case. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173241.1288CAB4%40davehans-spike.ostc.intel.com --- arch/x86/mm/pgtable.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index ca07db510d26..f4fa8fabf326 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -80,16 +80,11 @@ struct mm_struct *pgd_page_get_mm(struct page *page) static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) { - /* If the pgd points to a shared pagetable level (either the - ptes in non-PAE, or shared PMD in PAE), then just copy the - references from swapper_pg_dir. */ - if (CONFIG_PGTABLE_LEVELS == 2 || - (CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) || - CONFIG_PGTABLE_LEVELS >= 4) { + /* PAE preallocates all its PMDs. No cloning needed. */ + if (!IS_ENABLED(CONFIG_X86_PAE)) clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, swapper_pg_dir + KERNEL_PGD_BOUNDARY, KERNEL_PGD_PTRS); - } /* List used to sync kernel mapping updates */ pgd_set_mm(pgd, mm); @@ -122,8 +117,7 @@ static void pgd_dtor(pgd_t *pgd) * all 4 top-level entries are used almost immediately in a * new process's life, we just pre-populate them here. */ -#define PREALLOCATED_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \ - PTRS_PER_PGD : KERNEL_PGD_BOUNDARY) +#define PREALLOCATED_PMDS PTRS_PER_PGD #define MAX_PREALLOCATED_PMDS PTRS_PER_PGD /* From 99b8f0c54f571616d7bf4a776a2863a321c38cb1 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:42 -0700 Subject: [PATCH 227/331] x86/mm: Remove duplicated PMD preallocation macro MAX_PREALLOCATED_PMDS and PREALLOCATED_PMDS are now identical. Just use PREALLOCATED_PMDS and remove "MAX". Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173242.5ED13A5B%40davehans-spike.ostc.intel.com --- arch/x86/mm/pgtable.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index f4fa8fabf326..c1144e2f24e2 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -118,7 +118,6 @@ static void pgd_dtor(pgd_t *pgd) * new process's life, we just pre-populate them here. */ #define PREALLOCATED_PMDS PTRS_PER_PGD -#define MAX_PREALLOCATED_PMDS PTRS_PER_PGD /* * "USER_PMDS" are the PMDs for the user copy of the page tables when @@ -154,7 +153,6 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) /* No need to prepopulate any pagetable entries in non-PAE modes. */ #define PREALLOCATED_PMDS 0 -#define MAX_PREALLOCATED_PMDS 0 #define PREALLOCATED_USER_PMDS 0 #define MAX_PREALLOCATED_USER_PMDS 0 #endif /* CONFIG_X86_PAE */ @@ -324,7 +322,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *pgd; pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS]; - pmd_t *pmds[MAX_PREALLOCATED_PMDS]; + pmd_t *pmds[PREALLOCATED_PMDS]; pgd = _pgd_alloc(mm); From eaa607deb29e0b6fd24b9adf230fbc765f342521 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 14 Apr 2025 10:32:44 -0700 Subject: [PATCH 228/331] x86/mm: Remove now unused SHARED_KERNEL_PMD All the users of SHARED_KERNEL_PMD are gone. Zap it. Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250414173244.1125BEC3%40davehans-spike.ostc.intel.com --- arch/x86/include/asm/pgtable-2level_types.h | 2 -- arch/x86/include/asm/pgtable-3level_types.h | 2 -- arch/x86/include/asm/pgtable_64_types.h | 2 -- 3 files changed, 6 deletions(-) diff --git a/arch/x86/include/asm/pgtable-2level_types.h b/arch/x86/include/asm/pgtable-2level_types.h index 66425424ce91..54690bd4ddbe 100644 --- a/arch/x86/include/asm/pgtable-2level_types.h +++ b/arch/x86/include/asm/pgtable-2level_types.h @@ -18,8 +18,6 @@ typedef union { } pte_t; #endif /* !__ASSEMBLER__ */ -#define SHARED_KERNEL_PMD 0 - #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED /* diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h index 9759fa0eb6a3..580b09bf6a45 100644 --- a/arch/x86/include/asm/pgtable-3level_types.h +++ b/arch/x86/include/asm/pgtable-3level_types.h @@ -27,8 +27,6 @@ typedef union { } pmd_t; #endif /* !__ASSEMBLER__ */ -#define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI)) - #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED /* diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index 5bb782d856f2..e83721db18c9 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -46,8 +46,6 @@ extern unsigned int ptrs_per_p4d; #endif /* !__ASSEMBLER__ */ -#define SHARED_KERNEL_PMD 0 - #ifdef CONFIG_X86_5LEVEL /* From 0dcc51477b94d87f23aeb400b78fbdfb09363000 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 18 Apr 2025 09:13:50 +0200 Subject: [PATCH 229/331] x86/boot: Remove semicolon from "rep" prefixes Minimum version of binutils required to compile the kernel is 2.25. This version correctly handles the "rep" prefixes, so it is possible to remove the semicolon, which was used to support ancient versions of GNU as. Due to the semicolon, the compiler considers "rep; insn" (or its alternate "rep\n\tinsn" form) as two separate instructions. Removing the semicolon makes asm length calculations more accurate, consequently making scheduling and inlining decisions of the compiler more accurate. Removing the semicolon also enables assembler checks involving "rep" prefixes. Trying to assemble e.g. "rep addl %eax, %ebx" results in: Error: invalid instruction `add' after `rep' Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Martin Mares Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250418071437.4144391-1-ubizjak@gmail.com --- arch/x86/boot/bioscall.S | 4 ++-- arch/x86/boot/boot.h | 4 ++-- arch/x86/boot/compressed/string.c | 8 ++++---- arch/x86/boot/copy.S | 8 ++++---- arch/x86/boot/header.S | 2 +- arch/x86/boot/string.c | 2 +- arch/x86/boot/video.c | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/x86/boot/bioscall.S b/arch/x86/boot/bioscall.S index aa9b96457584..cf4a6155714e 100644 --- a/arch/x86/boot/bioscall.S +++ b/arch/x86/boot/bioscall.S @@ -32,7 +32,7 @@ intcall: movw %dx, %si movw %sp, %di movw $11, %cx - rep; movsl + rep movsl /* Pop full state from the stack */ popal @@ -67,7 +67,7 @@ intcall: jz 4f movw %sp, %si movw $11, %cx - rep; movsl + rep movsl 4: addw $44, %sp /* Restore state and return */ diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h index 38f17a1e1e36..f3771a6373c7 100644 --- a/arch/x86/boot/boot.h +++ b/arch/x86/boot/boot.h @@ -155,14 +155,14 @@ static inline void wrgs32(u32 v, addr_t addr) static inline bool memcmp_fs(const void *s1, addr_t s2, size_t len) { bool diff; - asm volatile("fs; repe; cmpsb" CC_SET(nz) + asm volatile("fs repe cmpsb" CC_SET(nz) : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } static inline bool memcmp_gs(const void *s1, addr_t s2, size_t len) { bool diff; - asm volatile("gs; repe; cmpsb" CC_SET(nz) + asm volatile("gs repe cmpsb" CC_SET(nz) : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } diff --git a/arch/x86/boot/compressed/string.c b/arch/x86/boot/compressed/string.c index 81fc1eaa3229..9af19d9614cb 100644 --- a/arch/x86/boot/compressed/string.c +++ b/arch/x86/boot/compressed/string.c @@ -15,9 +15,9 @@ static void *____memcpy(void *dest, const void *src, size_t n) { int d0, d1, d2; asm volatile( - "rep ; movsl\n\t" + "rep movsl\n\t" "movl %4,%%ecx\n\t" - "rep ; movsb\n\t" + "rep movsb" : "=&c" (d0), "=&D" (d1), "=&S" (d2) : "0" (n >> 2), "g" (n & 3), "1" (dest), "2" (src) : "memory"); @@ -29,9 +29,9 @@ static void *____memcpy(void *dest, const void *src, size_t n) { long d0, d1, d2; asm volatile( - "rep ; movsq\n\t" + "rep movsq\n\t" "movq %4,%%rcx\n\t" - "rep ; movsb\n\t" + "rep movsb" : "=&c" (d0), "=&D" (d1), "=&S" (d2) : "0" (n >> 3), "g" (n & 7), "1" (dest), "2" (src) : "memory"); diff --git a/arch/x86/boot/copy.S b/arch/x86/boot/copy.S index 6afd05e819d2..3973a67cd04e 100644 --- a/arch/x86/boot/copy.S +++ b/arch/x86/boot/copy.S @@ -22,10 +22,10 @@ SYM_FUNC_START_NOALIGN(memcpy) movw %dx, %si pushw %cx shrw $2, %cx - rep; movsl + rep movsl popw %cx andw $3, %cx - rep; movsb + rep movsb popw %di popw %si retl @@ -38,10 +38,10 @@ SYM_FUNC_START_NOALIGN(memset) imull $0x01010101,%eax pushw %cx shrw $2, %cx - rep; stosl + rep stosl popw %cx andw $3, %cx - rep; stosb + rep stosb popw %di retl SYM_FUNC_END(memset) diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index b5c79f43359b..9cb91421b4e4 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -585,7 +585,7 @@ start_of_setup: xorl %eax, %eax subw %di, %cx shrw $2, %cx - rep; stosl + rep stosl # Jump to C code (should not return) calll main diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index 84f7a883ce1e..f35369bb14c5 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -32,7 +32,7 @@ int memcmp(const void *s1, const void *s2, size_t len) { bool diff; - asm("repe; cmpsb" CC_SET(nz) + asm("repe cmpsb" CC_SET(nz) : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } diff --git a/arch/x86/boot/video.c b/arch/x86/boot/video.c index f2e96905b3fe..0641c8c46aee 100644 --- a/arch/x86/boot/video.c +++ b/arch/x86/boot/video.c @@ -292,7 +292,7 @@ static void restore_screen(void) "shrw %%cx ; " "jnc 1f ; " "stosw \n\t" - "1: rep;stosl ; " + "1: rep stosl ; " "popw %%es" : "+D" (dst), "+c" (npad) : "bdS" (video_segment), From 42c782fae38f0559e6355707ee0afde9ac16dcc3 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 18 Apr 2025 09:13:51 +0200 Subject: [PATCH 230/331] x86/asm: Remove semicolon from "rep" prefixes Minimum version of binutils required to compile the kernel is 2.25. This version correctly handles the "rep" prefixes, so it is possible to remove the semicolon, which was used to support ancient versions of GNU as. Due to the semicolon, the compiler considers "rep; insn" (or its alternate "rep\n\tinsn" form) as two separate instructions. Removing the semicolon makes asm length calculations more accurate, consequently making scheduling and inlining decisions of the compiler more accurate. Removing the semicolon also enables assembler checks involving "rep" prefixes. Trying to assemble e.g. "rep addl %eax, %ebx" results in: Error: invalid instruction `add' after `rep' Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Boris Ostrovsky Cc: Brian Gerst Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Pavel Machek Cc: Rafael J. Wysocki Link: https://lore.kernel.org/r/20250418071437.4144391-2-ubizjak@gmail.com --- arch/x86/include/asm/io.h | 6 +++--- arch/x86/include/asm/string_32.h | 15 ++++++--------- arch/x86/kernel/head_32.S | 8 +++----- arch/x86/kernel/relocate_kernel_32.S | 6 +++--- arch/x86/kernel/relocate_kernel_64.S | 6 +++--- arch/x86/lib/iomem.c | 2 +- arch/x86/lib/string_32.c | 17 ++++++----------- arch/x86/lib/strstr_32.c | 6 ++---- arch/x86/lib/usercopy_32.c | 18 +++++++++--------- arch/x86/platform/pvh/head.S | 3 +-- arch/x86/power/hibernate_asm_32.S | 3 +-- arch/x86/power/hibernate_asm_64.S | 3 +-- 12 files changed, 39 insertions(+), 54 deletions(-) diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index e889c3bab5a2..ca309a3227c7 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -217,7 +217,7 @@ void memset_io(volatile void __iomem *, int, size_t); static inline void __iowrite32_copy(void __iomem *to, const void *from, size_t count) { - asm volatile("rep ; movsl" + asm volatile("rep movsl" : "=&c"(count), "=&D"(to), "=&S"(from) : "0"(count), "1"(to), "2"(from) : "memory"); @@ -282,7 +282,7 @@ static inline void outs##bwl(u16 port, const void *addr, unsigned long count) \ count--; \ } \ } else { \ - asm volatile("rep; outs" #bwl \ + asm volatile("rep outs" #bwl \ : "+S"(addr), "+c"(count) \ : "d"(port) : "memory"); \ } \ @@ -298,7 +298,7 @@ static inline void ins##bwl(u16 port, void *addr, unsigned long count) \ count--; \ } \ } else { \ - asm volatile("rep; ins" #bwl \ + asm volatile("rep ins" #bwl \ : "+D"(addr), "+c"(count) \ : "d"(port) : "memory"); \ } \ diff --git a/arch/x86/include/asm/string_32.h b/arch/x86/include/asm/string_32.h index 32c0d981a82a..e9cce169bb4c 100644 --- a/arch/x86/include/asm/string_32.h +++ b/arch/x86/include/asm/string_32.h @@ -33,11 +33,11 @@ extern size_t strlen(const char *s); static __always_inline void *__memcpy(void *to, const void *from, size_t n) { int d0, d1, d2; - asm volatile("rep ; movsl\n\t" + asm volatile("rep movsl\n\t" "movl %4,%%ecx\n\t" "andl $3,%%ecx\n\t" "jz 1f\n\t" - "rep ; movsb\n\t" + "rep movsb\n\t" "1:" : "=&c" (d0), "=&D" (d1), "=&S" (d2) : "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from) @@ -89,7 +89,7 @@ static __always_inline void *__constant_memcpy(void *to, const void *from, if (n >= 5 * 4) { /* large block: use rep prefix */ int ecx; - asm volatile("rep ; movsl" + asm volatile("rep movsl" : "=&c" (ecx), "=&D" (edi), "=&S" (esi) : "0" (n / 4), "1" (edi), "2" (esi) : "memory" @@ -165,8 +165,7 @@ extern void *memchr(const void *cs, int c, size_t count); static inline void *__memset_generic(void *s, char c, size_t count) { int d0, d1; - asm volatile("rep\n\t" - "stosb" + asm volatile("rep stosb" : "=&c" (d0), "=&D" (d1) : "a" (c), "1" (s), "0" (count) : "memory"); @@ -199,8 +198,7 @@ extern void *memset(void *, int, size_t); static inline void *memset16(uint16_t *s, uint16_t v, size_t n) { int d0, d1; - asm volatile("rep\n\t" - "stosw" + asm volatile("rep stosw" : "=&c" (d0), "=&D" (d1) : "a" (v), "1" (s), "0" (n) : "memory"); @@ -211,8 +209,7 @@ static inline void *memset16(uint16_t *s, uint16_t v, size_t n) static inline void *memset32(uint32_t *s, uint32_t v, size_t n) { int d0, d1; - asm volatile("rep\n\t" - "stosl" + asm volatile("rep stosl" : "=&c" (d0), "=&D" (d1) : "a" (v), "1" (s), "0" (n) : "memory"); diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 2e42056d2306..76743dfad6ab 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -86,7 +86,7 @@ SYM_CODE_START(startup_32) movl $pa(__bss_stop),%ecx subl %edi,%ecx shrl $2,%ecx - rep ; stosl + rep stosl /* * Copy bootup parameters out of the way. * Note: %esi still has the pointer to the real-mode data. @@ -98,15 +98,13 @@ SYM_CODE_START(startup_32) movl $pa(boot_params),%edi movl $(PARAM_SIZE/4),%ecx cld - rep - movsl + rep movsl movl pa(boot_params) + NEW_CL_POINTER,%esi andl %esi,%esi jz 1f # No command line movl $pa(boot_command_line),%edi movl $(COMMAND_LINE_SIZE/4),%ecx - rep - movsl + rep movsl 1: #ifdef CONFIG_OLPC diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S index c7c4b1917336..57276f134d12 100644 --- a/arch/x86/kernel/relocate_kernel_32.S +++ b/arch/x86/kernel/relocate_kernel_32.S @@ -263,17 +263,17 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) movl %edx, %edi movl $1024, %ecx - rep ; movsl + rep movsl movl %ebp, %edi movl %eax, %esi movl $1024, %ecx - rep ; movsl + rep movsl movl %eax, %edi movl %edx, %esi movl $1024, %ecx - rep ; movsl + rep movsl lea PAGE_SIZE(%ebp), %esi jmp 0b diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 3062cb3efc44..1d6355227bf6 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -363,20 +363,20 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) /* copy source page to swap page */ movq kexec_pa_swap_page(%rip), %rdi movl $512, %ecx - rep ; movsq + rep movsq /* copy destination page to source page */ movq %rax, %rdi movq %rdx, %rsi movl $512, %ecx - rep ; movsq + rep movsq /* copy swap page to destination page */ movq %rdx, %rdi movq kexec_pa_swap_page(%rip), %rsi .Lnoswap: movl $512, %ecx - rep ; movsq + rep movsq lea PAGE_SIZE(%rax), %rsi jmp .Lloop diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c index 5eecb45d05d5..c20e04764edc 100644 --- a/arch/x86/lib/iomem.c +++ b/arch/x86/lib/iomem.c @@ -10,7 +10,7 @@ static __always_inline void rep_movs(void *to, const void *from, size_t n) { unsigned long d0, d1, d2; - asm volatile("rep ; movsl\n\t" + asm volatile("rep movsl\n\t" "testb $2,%b4\n\t" "je 1f\n\t" "movsw\n" diff --git a/arch/x86/lib/string_32.c b/arch/x86/lib/string_32.c index 53b3f202267c..f87ec24fa579 100644 --- a/arch/x86/lib/string_32.c +++ b/arch/x86/lib/string_32.c @@ -40,8 +40,7 @@ char *strncpy(char *dest, const char *src, size_t count) "stosb\n\t" "testb %%al,%%al\n\t" "jne 1b\n\t" - "rep\n\t" - "stosb\n" + "rep stosb\n" "2:" : "=&S" (d0), "=&D" (d1), "=&c" (d2), "=&a" (d3) : "0" (src), "1" (dest), "2" (count) : "memory"); @@ -54,8 +53,7 @@ EXPORT_SYMBOL(strncpy); char *strcat(char *dest, const char *src) { int d0, d1, d2, d3; - asm volatile("repne\n\t" - "scasb\n\t" + asm volatile("repne scasb\n\t" "decl %1\n" "1:\tlodsb\n\t" "stosb\n\t" @@ -72,8 +70,7 @@ EXPORT_SYMBOL(strcat); char *strncat(char *dest, const char *src, size_t count) { int d0, d1, d2, d3; - asm volatile("repne\n\t" - "scasb\n\t" + asm volatile("repne scasb\n\t" "decl %1\n\t" "movl %8,%3\n" "1:\tdecl %3\n\t" @@ -167,8 +164,7 @@ size_t strlen(const char *s) { int d0; size_t res; - asm volatile("repne\n\t" - "scasb" + asm volatile("repne scasb" : "=c" (res), "=&D" (d0) : "1" (s), "a" (0), "0" (0xffffffffu) : "memory"); @@ -184,8 +180,7 @@ void *memchr(const void *cs, int c, size_t count) void *res; if (!count) return NULL; - asm volatile("repne\n\t" - "scasb\n\t" + asm volatile("repne scasb\n\t" "je 1f\n\t" "movl $1,%0\n" "1:\tdecl %0" @@ -202,7 +197,7 @@ void *memscan(void *addr, int c, size_t size) { if (!size) return addr; - asm volatile("repnz; scasb\n\t" + asm volatile("repnz scasb\n\t" "jnz 1f\n\t" "dec %%edi\n" "1:" diff --git a/arch/x86/lib/strstr_32.c b/arch/x86/lib/strstr_32.c index 38f37df056f7..28267985e85f 100644 --- a/arch/x86/lib/strstr_32.c +++ b/arch/x86/lib/strstr_32.c @@ -8,16 +8,14 @@ int d0, d1; register char *__res; __asm__ __volatile__( "movl %6,%%edi\n\t" - "repne\n\t" - "scasb\n\t" + "repne scasb\n\t" "notl %%ecx\n\t" "decl %%ecx\n\t" /* NOTE! This also sets Z if searchstring='' */ "movl %%ecx,%%edx\n" "1:\tmovl %6,%%edi\n\t" "movl %%esi,%%eax\n\t" "movl %%edx,%%ecx\n\t" - "repe\n\t" - "cmpsb\n\t" + "repe cmpsb\n\t" "je 2f\n\t" /* also works for empty string, see above */ "xchgl %%eax,%%esi\n\t" "incl %%esi\n\t" diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index 422257c350c6..f6f436f1d573 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -38,9 +38,9 @@ do { \ might_fault(); \ __asm__ __volatile__( \ ASM_STAC "\n" \ - "0: rep; stosl\n" \ + "0: rep stosl\n" \ " movl %2,%0\n" \ - "1: rep; stosb\n" \ + "1: rep stosb\n" \ "2: " ASM_CLAC "\n" \ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2) \ _ASM_EXTABLE_UA(1b, 2b) \ @@ -140,9 +140,9 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) " shrl $2, %0\n" " andl $3, %%eax\n" " cld\n" - "99: rep; movsl\n" + "99: rep movsl\n" "36: movl %%eax, %0\n" - "37: rep; movsb\n" + "37: rep movsb\n" "100:\n" _ASM_EXTABLE_UA(1b, 100b) _ASM_EXTABLE_UA(2b, 100b) @@ -242,9 +242,9 @@ static unsigned long __copy_user_intel_nocache(void *to, " shrl $2, %0\n" " andl $3, %%eax\n" " cld\n" - "6: rep; movsl\n" + "6: rep movsl\n" " movl %%eax,%0\n" - "7: rep; movsb\n" + "7: rep movsb\n" "8:\n" _ASM_EXTABLE_UA(0b, 8b) _ASM_EXTABLE_UA(1b, 8b) @@ -293,14 +293,14 @@ do { \ " negl %0\n" \ " andl $7,%0\n" \ " subl %0,%3\n" \ - "4: rep; movsb\n" \ + "4: rep movsb\n" \ " movl %3,%0\n" \ " shrl $2,%0\n" \ " andl $3,%3\n" \ " .align 2,0x90\n" \ - "0: rep; movsl\n" \ + "0: rep movsl\n" \ " movl %3,%0\n" \ - "1: rep; movsb\n" \ + "1: rep movsb\n" \ "2:\n" \ _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3) \ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3) \ diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index cfa18ec7d55f..1d78e5631bb8 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -87,8 +87,7 @@ SYM_CODE_START(pvh_start_xen) mov %ebx, %esi movl rva(pvh_start_info_sz)(%ebp), %ecx shr $2,%ecx - rep - movsl + rep movsl leal rva(early_stack_end)(%ebp), %esp diff --git a/arch/x86/power/hibernate_asm_32.S b/arch/x86/power/hibernate_asm_32.S index 5606a15cf9a1..fb910d9f8471 100644 --- a/arch/x86/power/hibernate_asm_32.S +++ b/arch/x86/power/hibernate_asm_32.S @@ -69,8 +69,7 @@ copy_loop: movl pbe_orig_address(%edx), %edi movl $(PAGE_SIZE >> 2), %ecx - rep - movsl + rep movsl movl pbe_next(%edx), %edx jmp copy_loop diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index 8c534c36adfa..f5bda01904c7 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -138,8 +138,7 @@ SYM_FUNC_START(core_restore_code) movq pbe_address(%rdx), %rsi movq pbe_orig_address(%rdx), %rdi movq $(PAGE_SIZE >> 3), %rcx - rep - movsq + rep movsq /* progress to the next pbe */ movq pbe_next(%rdx), %rdx From d109ff4f0bc32de354247a6e8ede3ffc8ef14cd0 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 18 Apr 2025 10:07:44 +0200 Subject: [PATCH 231/331] x86/asm: Replace "REP; NOP" with PAUSE mnemonic Current minimum required version of binutils is 2.25, which supports PAUSE instruction mnemonic. Replace "REP; NOP" with this proper mnemonic. No functional change intended. Reviewed-by: Nikolay Borisov Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Link: https://lore.kernel.org/r/20250418080805.83679-2-ubizjak@gmail.com --- arch/x86/boot/boot.h | 2 +- arch/x86/include/asm/vdso/processor.h | 4 ++-- arch/x86/kernel/relocate_kernel_64.S | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/boot/boot.h b/arch/x86/boot/boot.h index 38f17a1e1e36..f899717f9e48 100644 --- a/arch/x86/boot/boot.h +++ b/arch/x86/boot/boot.h @@ -34,7 +34,7 @@ extern struct setup_header hdr; extern struct boot_params boot_params; -#define cpu_relax() asm volatile("rep; nop") +#define cpu_relax() asm volatile("pause") static inline void io_delay(void) { diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index c9b2ba7a9ec4..240d761b8ce4 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -7,10 +7,10 @@ #ifndef __ASSEMBLER__ -/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ +/* PAUSE is a good thing to insert into busy-wait loops. */ static __always_inline void rep_nop(void) { - asm volatile("rep; nop" ::: "memory"); + asm volatile("pause" ::: "memory"); } static __always_inline void cpu_relax(void) diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 1d6355227bf6..ea604f4d0b52 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -405,7 +405,7 @@ SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250) inb %dx, %al testb $XMTRDY, %al jnz .Lready - rep nop + pause jmp .Lxmtrdy_loop .Lready: @@ -426,7 +426,7 @@ SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250_mmio32) movb (LSR*4)(%rdx), %ah testb $XMTRDY, %ah jnz .Lready_mmio - rep nop + pause jmp .Lxmtrdy_loop_mmio .Lready_mmio: From 3ce4b1f1f24bbd9f1c349ecb6641dfa038bd0b5a Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 18 Apr 2025 10:07:43 +0200 Subject: [PATCH 232/331] x86/asm: Rename rep_nop() to native_pause() Rename rep_nop() function to what it really does. No functional change intended. Suggested-by: David Laight Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Link: https://lore.kernel.org/r/20250418080805.83679-1-ubizjak@gmail.com --- arch/x86/include/asm/vdso/processor.h | 4 ++-- arch/x86/kernel/apic/io_apic.c | 2 +- arch/x86/lib/delay.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index 240d761b8ce4..7000aeb59aa2 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -8,14 +8,14 @@ #ifndef __ASSEMBLER__ /* PAUSE is a good thing to insert into busy-wait loops. */ -static __always_inline void rep_nop(void) +static __always_inline void native_pause(void) { asm volatile("pause" ::: "memory"); } static __always_inline void cpu_relax(void) { - rep_nop(); + native_pause(); } struct getcpu_cache; diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index eebc360ed1bb..ba5a4ccda37a 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1486,7 +1486,7 @@ static void __init delay_with_tsc(void) * 1 GHz == 40 jiffies */ do { - rep_nop(); + native_pause(); now = rdtsc(); } while ((now - start) < 40000000000ULL / HZ && time_before_eq(jiffies, end)); } diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c index e86eda2c0b04..eb2d2e1cbddd 100644 --- a/arch/x86/lib/delay.c +++ b/arch/x86/lib/delay.c @@ -75,7 +75,7 @@ static void delay_tsc(u64 cycles) /* Allow RT tasks to run */ preempt_enable(); - rep_nop(); + native_pause(); preempt_disable(); /* From aef1d0209ddf127a8069aca5fa3a062be4136b76 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 18 Apr 2025 11:50:34 +0200 Subject: [PATCH 233/331] x86/mm: Fix {,un}use_temporary_mm() IRQ state As the function switch_mm_irqs_off() implies, it ought to be called with IRQs *off*. Commit 58f8ffa91766 ("x86/mm: Allow temporary MMs when IRQs are on") caused this to not be the case for EFI. Ensure IRQs are off where it matters. Fixes: 58f8ffa91766 ("x86/mm: Allow temporary MMs when IRQs are on") Reported-by: Borislav Petkov (AMD) Tested-by: Borislav Petkov (AMD) Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Morton Cc: Andy Lutomirski Cc: Linus Torvalds Cc: Rik van Riel Link: https://lore.kernel.org/r/20250418095034.GR38216@noisy.programming.kicks-ass.net --- arch/x86/mm/tlb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 79c124f6f3f2..39761c7765bd 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -986,6 +986,7 @@ struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) struct mm_struct *prev_mm; lockdep_assert_preemption_disabled(); + guard(irqsave)(); /* * Make sure not to be in TLB lazy mode, as otherwise we'll end up @@ -1018,6 +1019,7 @@ struct mm_struct *use_temporary_mm(struct mm_struct *temp_mm) void unuse_temporary_mm(struct mm_struct *prev_mm) { lockdep_assert_preemption_disabled(); + guard(irqsave)(); /* Clear the cpumask, to indicate no TLB flushing is needed anywhere */ cpumask_clear_cpu(smp_processor_id(), mm_cpumask(this_cpu_read(cpu_tlbstate.loaded_mm))); From 4e2c719782a84702db7fc2dc07ced796f308fec7 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Tue, 22 Apr 2025 08:32:47 +0200 Subject: [PATCH 234/331] x86/cpu: Help users notice when running old Intel microcode Old microcode is bad for users and for kernel developers. For users, it exposes them to known fixed security and/or functional issues. These obviously rarely result in instant dumpster fires in every environment. But it is as important to keep your microcode up to date as it is to keep your kernel up to date. Old microcode also makes kernels harder to debug. A developer looking at an oops need to consider kernel bugs, known CPU issues and unknown CPU issues as possible causes. If they know the microcode is up to date, they can mostly eliminate known CPU issues as the cause. Make it easier to tell if CPU microcode is out of date. Add a list of released microcode. If the loaded microcode is older than the release, tell users in a place that folks can find it: /sys/devices/system/cpu/vulnerabilities/old_microcode Tell kernel kernel developers about it with the existing taint flag: TAINT_CPU_OUT_OF_SPEC == Discussion == When a user reports a potential kernel issue, it is very common to ask them to reproduce the issue on mainline. Running mainline, they will (independently from the distro) acquire a more up-to-date microcode version list. If their microcode is old, they will get a warning about the taint and kernel developers can take that into consideration when debugging. Just like any other entry in "vulnerabilities/", users are free to make their own assessment of their exposure. == Microcode Revision Discussion == The microcode versions in the table were generated from the Intel microcode git repo: 8ac9378a8487 ("microcode-20241112 Release") which as of this writing lags behind the latest microcode-20250211. It can be argued that the versions that the kernel picks to call "old" should be a revision or two old. Which specific version is picked is less important to me than picking *a* version and enforcing it. This repository contains only microcode versions that Intel has deemed to be OS-loadable. It is quite possible that the BIOS has loaded a newer microcode than the latest in this repo. If this happens, the system is considered to have new microcode, not old. Specifically, the sysfs file and taint flag answer the question: Is the CPU running on the latest OS-loadable microcode, or something even later that the BIOS loaded? In other words, Intel never publishes an authoritative list of CPUs and latest microcode revisions. Until it does, this is the best that Linux can do. Also note that the "intel-ucode-defs.h" file is simple, ugly and has lots of magic numbers. That's on purpose and should allow a single file to be shared across lots of stable kernel regardless of if they have the new "VFM" infrastructure or not. It was generated with a dumb script. == FAQ == Q: Does this tell me if my system is secure or insecure? A: No. It only tells you if your microcode was old when the system booted. Q: Should the kernel warn if the microcode list itself is too old? A: No. New kernels will get new microcode lists, both mainline and stable. The only way to have an old list is to be running an old kernel in which case you have bigger problems. Q: Is this for security or functional issues? A: Both. Q: If a given microcode update only has functional problems but no security issues, will it be considered old? A: Yes. All microcode image versions within a microcode release are treated identically. Intel appears to make security updates without disclosing them in the release notes. Thus, all updates are considered to be security-relevant. Q: Who runs old microcode? A: Anybody with an old distro. This happens all the time inside of Intel where there are lots of weird systems in labs that might not be getting regular distro updates and might also be running rather exotic microcode images. Q: If I update my microcode after booting will it stop saying "Vulnerable"? A: No. Just like all the other vulnerabilies, you need to reboot before the kernel will reassess your vulnerability. Signed-off-by: Dave Hansen Signed-off-by: Ingo Molnar Cc: "Ahmed S. Darwish" Cc: Andrew Cooper Cc: Andy Lutomirski Cc: Brian Gerst Cc: John Ogness Cc: Josh Poimboeuf Cc: Juergen Gross Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Link: https://lore.kernel.org/all/20250421195659.CF426C07%40davehans-spike.ostc.intel.com (cherry picked from commit 9127865b15eb0a1bd05ad7efe29489c44394bdc1) --- .../ABI/testing/sysfs-devices-system-cpu | 1 + .../admin-guide/hw-vuln/old_microcode.rst | 21 +++ arch/x86/include/asm/cpufeatures.h | 2 + arch/x86/kernel/cpu/bugs.c | 16 ++ arch/x86/kernel/cpu/common.c | 42 +++++ .../kernel/cpu/microcode/intel-ucode-defs.h | 150 ++++++++++++++++++ drivers/base/cpu.c | 3 + include/linux/cpu.h | 2 + 8 files changed, 237 insertions(+) create mode 100644 Documentation/admin-guide/hw-vuln/old_microcode.rst create mode 100644 arch/x86/kernel/cpu/microcode/intel-ucode-defs.h diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 206079d3bd5b..766a9747b154 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -516,6 +516,7 @@ What: /sys/devices/system/cpu/vulnerabilities /sys/devices/system/cpu/vulnerabilities/mds /sys/devices/system/cpu/vulnerabilities/meltdown /sys/devices/system/cpu/vulnerabilities/mmio_stale_data + /sys/devices/system/cpu/vulnerabilities/old_microcode /sys/devices/system/cpu/vulnerabilities/reg_file_data_sampling /sys/devices/system/cpu/vulnerabilities/retbleed /sys/devices/system/cpu/vulnerabilities/spec_store_bypass diff --git a/Documentation/admin-guide/hw-vuln/old_microcode.rst b/Documentation/admin-guide/hw-vuln/old_microcode.rst new file mode 100644 index 000000000000..6ded8f86b8d0 --- /dev/null +++ b/Documentation/admin-guide/hw-vuln/old_microcode.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +Old Microcode +============= + +The kernel keeps a table of released microcode. Systems that had +microcode older than this at boot will say "Vulnerable". This means +that the system was vulnerable to some known CPU issue. It could be +security or functional, the kernel does not know or care. + +You should update the CPU microcode to mitigate any exposure. This is +usually accomplished by updating the files in +/lib/firmware/intel-ucode/ via normal distribution updates. Intel also +distributes these files in a github repo: + + https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files.git + +Just like all the other hardware vulnerabilities, exposure is +determined at boot. Runtime microcode updates do not change the status +of this vulnerability. diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index bc81b9d1aeca..426c7e8f0158 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -533,4 +533,6 @@ #define X86_BUG_BHI X86_BUG( 1*32+ 3) /* "bhi" CPU is affected by Branch History Injection */ #define X86_BUG_IBPB_NO_RET X86_BUG( 1*32+ 4) /* "ibpb_no_ret" IBPB omits return target predictions */ #define X86_BUG_SPECTRE_V2_USER X86_BUG( 1*32+ 5) /* "spectre_v2_user" CPU is affected by Spectre variant 2 attack between user processes */ +#define X86_BUG_OLD_MICROCODE X86_BUG( 1*32+ 6) /* "old_microcode" CPU has old microcode, it is surely vulnerable to something */ + #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 606c0cb97e86..aa9f86bf6103 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -2794,6 +2794,14 @@ static ssize_t rfds_show_state(char *buf) return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]); } +static ssize_t old_microcode_show_state(char *buf) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return sysfs_emit(buf, "Unknown: running under hypervisor"); + + return sysfs_emit(buf, "Vulnerable\n"); +} + static char *stibp_state(void) { if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && @@ -2975,6 +2983,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr case X86_BUG_RFDS: return rfds_show_state(buf); + case X86_BUG_OLD_MICROCODE: + return old_microcode_show_state(buf); + default: break; } @@ -3051,6 +3062,11 @@ ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attrib { return cpu_show_common(dev, attr, buf, X86_BUG_RFDS); } + +ssize_t cpu_show_old_microcode(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_OLD_MICROCODE); +} #endif void __warn_thunk(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 4ada55f126ae..19893eb4f034 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1318,10 +1318,52 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return cpu_matches(cpu_vuln_blacklist, RFDS); } +static struct x86_cpu_id cpu_latest_microcode[] = { +#include "microcode/intel-ucode-defs.h" + {} +}; + +static bool __init cpu_has_old_microcode(void) +{ + const struct x86_cpu_id *m = x86_match_cpu(cpu_latest_microcode); + + /* Give unknown CPUs a pass: */ + if (!m) { + /* Intel CPUs should be in the list. Warn if not: */ + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) + pr_info("x86/CPU: Model not found in latest microcode list\n"); + return false; + } + + /* + * Hosts usually lie to guests with a super high microcode + * version. Just ignore what hosts tell guests: + */ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return false; + + /* Consider all debug microcode to be old: */ + if (boot_cpu_data.microcode & BIT(31)) + return true; + + /* Give new microcode a pass: */ + if (boot_cpu_data.microcode >= m->driver_data) + return false; + + /* Uh oh, too old: */ + return true; +} + static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) { u64 x86_arch_cap_msr = x86_read_arch_cap_msr(); + if (cpu_has_old_microcode()) { + pr_warn("x86/CPU: Running old microcode\n"); + setup_force_cpu_bug(X86_BUG_OLD_MICROCODE); + add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK); + } + /* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */ if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) && !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO)) diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h new file mode 100644 index 000000000000..cb6e601701ab --- /dev/null +++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h @@ -0,0 +1,150 @@ +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .driver_data = 0x45 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .driver_data = 0x40 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .driver_data = 0x2c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .driver_data = 0x10 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .driver_data = 0x14 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .driver_data = 0x38 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .driver_data = 0x2e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .driver_data = 0x11 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .driver_data = 0x5 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .driver_data = 0x47 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .driver_data = 0x1 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .driver_data = 0x1d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .driver_data = 0x2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .driver_data = 0x18 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .driver_data = 0x39 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .driver_data = 0x59 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .driver_data = 0x5d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .driver_data = 0xd2 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .driver_data = 0x6b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .driver_data = 0x95 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .driver_data = 0xbc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .driver_data = 0xa4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .driver_data = 0x44 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .driver_data = 0x60f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .driver_data = 0x70a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .driver_data = 0xa0b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .driver_data = 0x12 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .driver_data = 0x1d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .driver_data = 0x219 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .driver_data = 0x107 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .driver_data = 0x29 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .driver_data = 0x11 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .driver_data = 0x7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .driver_data = 0x105 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .driver_data = 0x1f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .driver_data = 0x621 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .driver_data = 0x71a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .driver_data = 0xd }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .driver_data = 0x3b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .driver_data = 0x838 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .driver_data = 0x90d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .driver_data = 0x21 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .driver_data = 0x28 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .driver_data = 0x42e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .driver_data = 0x600 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .driver_data = 0x715 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .driver_data = 0x49 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .driver_data = 0x1a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .driver_data = 0x26 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .driver_data = 0x1c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .driver_data = 0x22 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .driver_data = 0x368 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .driver_data = 0x411 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .driver_data = 0x12d }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .driver_data = 0xf0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .driver_data = 0x1000191 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .driver_data = 0x2007006 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .driver_data = 0x3000010 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0040, .driver_data = 0x4003605 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .driver_data = 0x5003707 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .driver_data = 0x7002904 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .driver_data = 0x1c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .driver_data = 0x700001c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .driver_data = 0xf00001a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .driver_data = 0xe000015 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .driver_data = 0x14 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .driver_data = 0x48 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .driver_data = 0x28 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .driver_data = 0xf0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .driver_data = 0x3e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .driver_data = 0x2a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .driver_data = 0xc0002f0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .driver_data = 0xd0003e7 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .driver_data = 0x10002b0 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .driver_data = 0x42 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .driver_data = 0x24 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .driver_data = 0xc6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .driver_data = 0x33 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .driver_data = 0xb8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .driver_data = 0x38 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .driver_data = 0x52 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .driver_data = 0xfc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .driver_data = 0x2c000390 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .driver_data = 0x2b000603 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .driver_data = 0x2c000390 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .driver_data = 0x2c000390 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .driver_data = 0x2c000390 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .driver_data = 0x1a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .driver_data = 0x37 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .driver_data = 0x37 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .driver_data = 0x37 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .driver_data = 0x37 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .driver_data = 0x435 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .driver_data = 0x435 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .driver_data = 0x24000026 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .driver_data = 0xf8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .driver_data = 0xf8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .driver_data = 0xf6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .driver_data = 0xf8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .driver_data = 0x100 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .driver_data = 0xfc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .driver_data = 0xfc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .driver_data = 0xfc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .driver_data = 0xfe }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .driver_data = 0xfc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .driver_data = 0x62 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .driver_data = 0x20 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .driver_data = 0x12b }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .driver_data = 0x4123 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .driver_data = 0x4123 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .driver_data = 0x4123 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .driver_data = 0x1a }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .driver_data = 0x21000283 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .driver_data = 0x21000283 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .driver_data = 0x12 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .driver_data = 0x15 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .driver_data = 0x2e }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .driver_data = 0x21 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .driver_data = 0x2c }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .driver_data = 0x10 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .driver_data = 0x39 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .driver_data = 0x2f }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .driver_data = 0xa }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .driver_data = 0xc }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .driver_data = 0x17 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .driver_data = 0x17 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .driver_data = 0x5 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .driver_data = 0x6 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .driver_data = 0xe }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .driver_data = 0x3 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .driver_data = 0x4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .driver_data = 0xf }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .driver_data = 0x4 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .driver_data = 0x8 }, +{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .driver_data = 0x9 }, diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index a7e511849875..1c4359366cd7 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -600,6 +600,7 @@ CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow); CPU_SHOW_VULN_FALLBACK(gds); CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling); CPU_SHOW_VULN_FALLBACK(ghostwrite); +CPU_SHOW_VULN_FALLBACK(old_microcode); static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); @@ -616,6 +617,7 @@ static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NU static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL); static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL); static DEVICE_ATTR(ghostwrite, 0444, cpu_show_ghostwrite, NULL); +static DEVICE_ATTR(old_microcode, 0444, cpu_show_old_microcode, NULL); static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_meltdown.attr, @@ -633,6 +635,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = { &dev_attr_gather_data_sampling.attr, &dev_attr_reg_file_data_sampling.attr, &dev_attr_ghostwrite.attr, + &dev_attr_old_microcode.attr, NULL }; diff --git a/include/linux/cpu.h b/include/linux/cpu.h index e3049543008b..1f5cfc4cc04f 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -78,6 +78,8 @@ extern ssize_t cpu_show_gds(struct device *dev, extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf); extern ssize_t cpu_show_ghostwrite(struct device *dev, struct device_attribute *attr, char *buf); +extern ssize_t cpu_show_old_microcode(struct device *dev, + struct device_attribute *attr, char *buf); extern __printf(4, 5) struct device *cpu_device_create(struct device *parent, void *drvdata, From 092071e0f63c2d2c54810a427d4d9a0df6aad52b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:12:55 +0200 Subject: [PATCH 235/331] vmlinux.lds: Include .data.rel[.local] into .data section When running in -fPIC mode, the compiler may decide to emit statically initialized data objects into .data.rel or .data.rel.local if they contain absolute references to global or local objects, respectively, which require fixing up at load time. This distinction is irrelevant for the kernel, so fold .data.rel and .data.rel.local into .data. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-9-ardb+git@google.com --- include/asm-generic/vmlinux.lds.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 58a635a6d5bd..66409bc3a4e0 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -108,13 +108,13 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) #define TEXT_MAIN .text #endif #if defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || defined(CONFIG_LTO_CLANG) -#define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data..L* .data..compoundliteral* .data.$__unnamed_* .data.$L* +#define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data.rel.* .data..L* .data..compoundliteral* .data.$__unnamed_* .data.$L* #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* #define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* .rodata..L* #define BSS_MAIN .bss .bss.[0-9a-zA-Z_]* .bss..L* .bss..compoundliteral* #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* #else -#define DATA_MAIN .data +#define DATA_MAIN .data .data.rel .data.rel.local #define SDATA_MAIN .sdata #define RODATA_MAIN .rodata #define BSS_MAIN .bss From b66fcee1574e72663a0c6dd7112a9e22774dbe9f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:12:56 +0200 Subject: [PATCH 236/331] x86/sev: Move noinstr NMI handling code into separate source file GCC may ignore the __no_sanitize_address function attribute when inlining, resulting in KASAN instrumentation in code tagged as 'noinstr'. Move the SEV NMI handling code, which is noinstr, into a separate source file so KASAN can be disabled on the whole file without losing coverage of other SEV core code, once the startup code is split off from it too. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-10-ardb+git@google.com --- arch/x86/coco/sev/Makefile | 5 +- arch/x86/coco/sev/core.c | 89 ----------------------------- arch/x86/coco/sev/sev-nmi.c | 108 ++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 90 deletions(-) create mode 100644 arch/x86/coco/sev/sev-nmi.c diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile index dcb06dc8b5ae..bc4baa400c55 100644 --- a/arch/x86/coco/sev/Makefile +++ b/arch/x86/coco/sev/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += core.o +obj-y += core.o sev-nmi.o # jump tables are emitted using absolute references in non-PIC code # so they cannot be used in the early SEV startup code @@ -20,3 +20,6 @@ KCSAN_SANITIZE := n # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining UBSAN_SANITIZE := n + +# GCC may fail to respect __no_sanitize_address when inlining +KASAN_SANITIZE_sev-nmi.o := n diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index aeb7731862c0..c7a0f3a2f4ac 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -113,77 +113,6 @@ DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); DEFINE_PER_CPU(u64, svsm_caa_pa); -static __always_inline bool on_vc_stack(struct pt_regs *regs) -{ - unsigned long sp = regs->sp; - - /* User-mode RSP is not trusted */ - if (user_mode(regs)) - return false; - - /* SYSCALL gap still has user-mode RSP */ - if (ip_within_syscall_gap(regs)) - return false; - - return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); -} - -/* - * This function handles the case when an NMI is raised in the #VC - * exception handler entry code, before the #VC handler has switched off - * its IST stack. In this case, the IST entry for #VC must be adjusted, - * so that any nested #VC exception will not overwrite the stack - * contents of the interrupted #VC handler. - * - * The IST entry is adjusted unconditionally so that it can be also be - * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a - * nested sev_es_ist_exit() call may adjust back the IST entry too - * early. - * - * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run - * on the NMI IST stack, as they are only called from NMI handling code - * right now. - */ -void noinstr __sev_es_ist_enter(struct pt_regs *regs) -{ - unsigned long old_ist, new_ist; - - /* Read old IST entry */ - new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); - - /* - * If NMI happened while on the #VC IST stack, set the new IST - * value below regs->sp, so that the interrupted stack frame is - * not overwritten by subsequent #VC exceptions. - */ - if (on_vc_stack(regs)) - new_ist = regs->sp; - - /* - * Reserve additional 8 bytes and store old IST value so this - * adjustment can be unrolled in __sev_es_ist_exit(). - */ - new_ist -= sizeof(old_ist); - *(unsigned long *)new_ist = old_ist; - - /* Set new IST entry */ - this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); -} - -void noinstr __sev_es_ist_exit(void) -{ - unsigned long ist; - - /* Read IST entry */ - ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); - - if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) - return; - - /* Read back old IST entry and write it to the TSS */ - this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); -} - /* * Nothing shall interrupt this code path while holding the per-CPU * GHCB. The backup GHCB is only for NMIs interrupting this path. @@ -608,24 +537,6 @@ int svsm_perform_call_protocol(struct svsm_call *call) return ret; } -void noinstr __sev_es_nmi_complete(void) -{ - struct ghcb_state state; - struct ghcb *ghcb; - - ghcb = __sev_get_ghcb(&state); - - vc_ghcb_invalidate(ghcb); - ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); - ghcb_set_sw_exit_info_1(ghcb, 0); - ghcb_set_sw_exit_info_2(ghcb, 0); - - sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); - VMGEXIT(); - - __sev_put_ghcb(&state); -} - static u64 __init get_snp_jump_table_addr(void) { struct snp_secrets_page *secrets; diff --git a/arch/x86/coco/sev/sev-nmi.c b/arch/x86/coco/sev/sev-nmi.c new file mode 100644 index 000000000000..d8dfaddfb367 --- /dev/null +++ b/arch/x86/coco/sev/sev-nmi.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#define pr_fmt(fmt) "SEV: " fmt + +#include +#include + +#include +#include +#include +#include +#include + +static __always_inline bool on_vc_stack(struct pt_regs *regs) +{ + unsigned long sp = regs->sp; + + /* User-mode RSP is not trusted */ + if (user_mode(regs)) + return false; + + /* SYSCALL gap still has user-mode RSP */ + if (ip_within_syscall_gap(regs)) + return false; + + return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); +} + +/* + * This function handles the case when an NMI is raised in the #VC + * exception handler entry code, before the #VC handler has switched off + * its IST stack. In this case, the IST entry for #VC must be adjusted, + * so that any nested #VC exception will not overwrite the stack + * contents of the interrupted #VC handler. + * + * The IST entry is adjusted unconditionally so that it can be also be + * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a + * nested sev_es_ist_exit() call may adjust back the IST entry too + * early. + * + * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run + * on the NMI IST stack, as they are only called from NMI handling code + * right now. + */ +void noinstr __sev_es_ist_enter(struct pt_regs *regs) +{ + unsigned long old_ist, new_ist; + + /* Read old IST entry */ + new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + /* + * If NMI happened while on the #VC IST stack, set the new IST + * value below regs->sp, so that the interrupted stack frame is + * not overwritten by subsequent #VC exceptions. + */ + if (on_vc_stack(regs)) + new_ist = regs->sp; + + /* + * Reserve additional 8 bytes and store old IST value so this + * adjustment can be unrolled in __sev_es_ist_exit(). + */ + new_ist -= sizeof(old_ist); + *(unsigned long *)new_ist = old_ist; + + /* Set new IST entry */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); +} + +void noinstr __sev_es_ist_exit(void) +{ + unsigned long ist; + + /* Read IST entry */ + ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); + + if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) + return; + + /* Read back old IST entry and write it to the TSS */ + this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); +} + +void noinstr __sev_es_nmi_complete(void) +{ + struct ghcb_state state; + struct ghcb *ghcb; + + ghcb = __sev_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); + ghcb_set_sw_exit_info_1(ghcb, 0); + ghcb_set_sw_exit_info_2(ghcb, 0); + + sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); + VMGEXIT(); + + __sev_put_ghcb(&state); +} From 234cf67fc3bd290bb16a377647a9c3f5a7f28a47 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:12:57 +0200 Subject: [PATCH 237/331] x86/sev: Split off startup code from core code Disentangle the SEV core code and the SEV code that is called during early boot. The latter piece will be moved into startup/ in a subsequent patch. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-11-ardb+git@google.com --- arch/x86/boot/compressed/sev.c | 2 + arch/x86/coco/sev/Makefile | 12 +- arch/x86/coco/sev/core.c | 1583 ++++++-------------------------- arch/x86/coco/sev/shared.c | 281 ------ arch/x86/coco/sev/startup.c | 1395 ++++++++++++++++++++++++++++ 5 files changed, 1662 insertions(+), 1611 deletions(-) create mode 100644 arch/x86/coco/sev/startup.c diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 478eca4f7180..714e30c66eae 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -141,6 +141,8 @@ u64 svsm_get_caa_pa(void) int svsm_perform_call_protocol(struct svsm_call *call); +u8 snp_vmpl; + /* Include code for early handlers */ #include "../../coco/sev/shared.c" diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile index bc4baa400c55..57e25f946663 100644 --- a/arch/x86/coco/sev/Makefile +++ b/arch/x86/coco/sev/Makefile @@ -1,18 +1,18 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += core.o sev-nmi.o +obj-y += core.o sev-nmi.o startup.o # jump tables are emitted using absolute references in non-PIC code # so they cannot be used in the early SEV startup code -CFLAGS_core.o += -fno-jump-tables +CFLAGS_startup.o += -fno-jump-tables ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_core.o = -pg +CFLAGS_REMOVE_startup.o = -pg endif -KASAN_SANITIZE_core.o := n -KMSAN_SANITIZE_core.o := n -KCOV_INSTRUMENT_core.o := n +KASAN_SANITIZE_startup.o := n +KMSAN_SANITIZE_startup.o := n +KCOV_INSTRUMENT_startup.o := n # With some compiler versions the generated code results in boot hangs, caused # by several compilation units. To be safe, disable all instrumentation. diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index c7a0f3a2f4ac..617988a5f3d7 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -80,18 +80,6 @@ static const char * const sev_status_feat_names[] = { [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", }; -/* For early boot hypervisor communication in SEV-ES enabled guests */ -struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); - -/* - * Needs to be in the .data section because we need it NULL before bss is - * cleared - */ -struct ghcb *boot_ghcb __section(".data"); - -/* Bitmap of SEV features supported by the hypervisor */ -u64 sev_hv_features __ro_after_init; - /* Secrets page physical address from the CC blob */ static u64 secrets_pa __ro_after_init; @@ -104,438 +92,16 @@ static u64 snp_tsc_scale __ro_after_init; static u64 snp_tsc_offset __ro_after_init; static u64 snp_tsc_freq_khz __ro_after_init; - -/* For early boot SVSM communication */ -struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); - DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); -DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); -DEFINE_PER_CPU(u64, svsm_caa_pa); /* - * Nothing shall interrupt this code path while holding the per-CPU - * GHCB. The backup GHCB is only for NMIs interrupting this path. - * - * Callers must disable local interrupts around it. + * SVSM related information: + * When running under an SVSM, the VMPL that Linux is executing at must be + * non-zero. The VMPL is therefore used to indicate the presence of an SVSM. */ -noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) -{ - struct sev_es_runtime_data *data; - struct ghcb *ghcb; - - WARN_ON(!irqs_disabled()); - - data = this_cpu_read(runtime_data); - ghcb = &data->ghcb_page; - - if (unlikely(data->ghcb_active)) { - /* GHCB is already in use - save its contents */ - - if (unlikely(data->backup_ghcb_active)) { - /* - * Backup-GHCB is also already in use. There is no way - * to continue here so just kill the machine. To make - * panic() work, mark GHCBs inactive so that messages - * can be printed out. - */ - data->ghcb_active = false; - data->backup_ghcb_active = false; - - instrumentation_begin(); - panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); - instrumentation_end(); - } - - /* Mark backup_ghcb active before writing to it */ - data->backup_ghcb_active = true; - - state->ghcb = &data->backup_ghcb; - - /* Backup GHCB content */ - *state->ghcb = *ghcb; - } else { - state->ghcb = NULL; - data->ghcb_active = true; - } - - return ghcb; -} - -static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, - unsigned char *buffer) -{ - return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); -} - -static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) -{ - char buffer[MAX_INSN_SIZE]; - int insn_bytes; - - insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); - if (insn_bytes == 0) { - /* Nothing could be copied */ - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; - } else if (insn_bytes == -EINVAL) { - /* Effective RIP could not be calculated */ - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - ctxt->fi.cr2 = 0; - return ES_EXCEPTION; - } - - if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) - return ES_DECODE_FAILED; - - if (ctxt->insn.immediate.got) - return ES_OK; - else - return ES_DECODE_FAILED; -} - -static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) -{ - char buffer[MAX_INSN_SIZE]; - int res, ret; - - res = vc_fetch_insn_kernel(ctxt, buffer); - if (res) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = X86_PF_INSTR; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; - } - - ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); - if (ret < 0) - return ES_DECODE_FAILED; - else - return ES_OK; -} - -static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) -{ - if (user_mode(ctxt->regs)) - return __vc_decode_user_insn(ctxt); - else - return __vc_decode_kern_insn(ctxt); -} - -static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, - char *dst, char *buf, size_t size) -{ - unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; - - /* - * This function uses __put_user() independent of whether kernel or user - * memory is accessed. This works fine because __put_user() does no - * sanity checks of the pointer being accessed. All that it does is - * to report when the access failed. - * - * Also, this function runs in atomic context, so __put_user() is not - * allowed to sleep. The page-fault handler detects that it is running - * in atomic context and will not try to take mmap_sem and handle the - * fault, so additional pagefault_enable()/disable() calls are not - * needed. - * - * The access can't be done via copy_to_user() here because - * vc_write_mem() must not use string instructions to access unsafe - * memory. The reason is that MOVS is emulated by the #VC handler by - * splitting the move up into a read and a write and taking a nested #VC - * exception on whatever of them is the MMIO access. Using string - * instructions here would cause infinite nesting. - */ - switch (size) { - case 1: { - u8 d1; - u8 __user *target = (u8 __user *)dst; - - memcpy(&d1, buf, 1); - if (__put_user(d1, target)) - goto fault; - break; - } - case 2: { - u16 d2; - u16 __user *target = (u16 __user *)dst; - - memcpy(&d2, buf, 2); - if (__put_user(d2, target)) - goto fault; - break; - } - case 4: { - u32 d4; - u32 __user *target = (u32 __user *)dst; - - memcpy(&d4, buf, 4); - if (__put_user(d4, target)) - goto fault; - break; - } - case 8: { - u64 d8; - u64 __user *target = (u64 __user *)dst; - - memcpy(&d8, buf, 8); - if (__put_user(d8, target)) - goto fault; - break; - } - default: - WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); - return ES_UNSUPPORTED; - } - - return ES_OK; - -fault: - if (user_mode(ctxt->regs)) - error_code |= X86_PF_USER; - - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = error_code; - ctxt->fi.cr2 = (unsigned long)dst; - - return ES_EXCEPTION; -} - -static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, - char *src, char *buf, size_t size) -{ - unsigned long error_code = X86_PF_PROT; - - /* - * This function uses __get_user() independent of whether kernel or user - * memory is accessed. This works fine because __get_user() does no - * sanity checks of the pointer being accessed. All that it does is - * to report when the access failed. - * - * Also, this function runs in atomic context, so __get_user() is not - * allowed to sleep. The page-fault handler detects that it is running - * in atomic context and will not try to take mmap_sem and handle the - * fault, so additional pagefault_enable()/disable() calls are not - * needed. - * - * The access can't be done via copy_from_user() here because - * vc_read_mem() must not use string instructions to access unsafe - * memory. The reason is that MOVS is emulated by the #VC handler by - * splitting the move up into a read and a write and taking a nested #VC - * exception on whatever of them is the MMIO access. Using string - * instructions here would cause infinite nesting. - */ - switch (size) { - case 1: { - u8 d1; - u8 __user *s = (u8 __user *)src; - - if (__get_user(d1, s)) - goto fault; - memcpy(buf, &d1, 1); - break; - } - case 2: { - u16 d2; - u16 __user *s = (u16 __user *)src; - - if (__get_user(d2, s)) - goto fault; - memcpy(buf, &d2, 2); - break; - } - case 4: { - u32 d4; - u32 __user *s = (u32 __user *)src; - - if (__get_user(d4, s)) - goto fault; - memcpy(buf, &d4, 4); - break; - } - case 8: { - u64 d8; - u64 __user *s = (u64 __user *)src; - if (__get_user(d8, s)) - goto fault; - memcpy(buf, &d8, 8); - break; - } - default: - WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); - return ES_UNSUPPORTED; - } - - return ES_OK; - -fault: - if (user_mode(ctxt->regs)) - error_code |= X86_PF_USER; - - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = error_code; - ctxt->fi.cr2 = (unsigned long)src; - - return ES_EXCEPTION; -} - -static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, - unsigned long vaddr, phys_addr_t *paddr) -{ - unsigned long va = (unsigned long)vaddr; - unsigned int level; - phys_addr_t pa; - pgd_t *pgd; - pte_t *pte; - - pgd = __va(read_cr3_pa()); - pgd = &pgd[pgd_index(va)]; - pte = lookup_address_in_pgd(pgd, va, &level); - if (!pte) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.cr2 = vaddr; - ctxt->fi.error_code = 0; - - if (user_mode(ctxt->regs)) - ctxt->fi.error_code |= X86_PF_USER; - - return ES_EXCEPTION; - } - - if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) - /* Emulated MMIO to/from encrypted memory not supported */ - return ES_UNSUPPORTED; - - pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; - pa |= va & ~page_level_mask(level); - - *paddr = pa; - - return ES_OK; -} - -static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) -{ - BUG_ON(size > 4); - - if (user_mode(ctxt->regs)) { - struct thread_struct *t = ¤t->thread; - struct io_bitmap *iobm = t->io_bitmap; - size_t idx; - - if (!iobm) - goto fault; - - for (idx = port; idx < port + size; ++idx) { - if (test_bit(idx, iobm->bitmap)) - goto fault; - } - } - - return ES_OK; - -fault: - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - - return ES_EXCEPTION; -} - -static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) -{ - long error_code = ctxt->fi.error_code; - int trapnr = ctxt->fi.vector; - - ctxt->regs->orig_ax = ctxt->fi.error_code; - - switch (trapnr) { - case X86_TRAP_GP: - exc_general_protection(ctxt->regs, error_code); - break; - case X86_TRAP_UD: - exc_invalid_op(ctxt->regs); - break; - case X86_TRAP_PF: - write_cr2(ctxt->fi.cr2); - exc_page_fault(ctxt->regs, error_code); - break; - case X86_TRAP_AC: - exc_alignment_check(ctxt->regs, error_code); - break; - default: - pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); - BUG(); - } -} - -/* Include code shared with pre-decompression boot stage */ -#include "shared.c" - -noinstr void __sev_put_ghcb(struct ghcb_state *state) -{ - struct sev_es_runtime_data *data; - struct ghcb *ghcb; - - WARN_ON(!irqs_disabled()); - - data = this_cpu_read(runtime_data); - ghcb = &data->ghcb_page; - - if (state->ghcb) { - /* Restore GHCB from Backup */ - *ghcb = *state->ghcb; - data->backup_ghcb_active = false; - state->ghcb = NULL; - } else { - /* - * Invalidate the GHCB so a VMGEXIT instruction issued - * from userspace won't appear to be valid. - */ - vc_ghcb_invalidate(ghcb); - data->ghcb_active = false; - } -} - -int svsm_perform_call_protocol(struct svsm_call *call) -{ - struct ghcb_state state; - unsigned long flags; - struct ghcb *ghcb; - int ret; - - /* - * This can be called very early in the boot, use native functions in - * order to avoid paravirt issues. - */ - flags = native_local_irq_save(); - - /* - * Use rip-relative references when called early in the boot. If - * ghcbs_initialized is set, then it is late in the boot and no need - * to worry about rip-relative references in called functions. - */ - if (RIP_REL_REF(sev_cfg).ghcbs_initialized) - ghcb = __sev_get_ghcb(&state); - else if (RIP_REL_REF(boot_ghcb)) - ghcb = RIP_REL_REF(boot_ghcb); - else - ghcb = NULL; - - do { - ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) - : svsm_perform_msr_protocol(call); - } while (ret == -EAGAIN); - - if (RIP_REL_REF(sev_cfg).ghcbs_initialized) - __sev_put_ghcb(&state); - - native_local_irq_restore(flags); - - return ret; -} +u8 snp_vmpl __ro_after_init; +EXPORT_SYMBOL_GPL(snp_vmpl); static u64 __init get_snp_jump_table_addr(void) { @@ -590,85 +156,280 @@ static u64 __init get_jump_table_addr(void) return ret; } -void __head -early_set_pages_state(unsigned long vaddr, unsigned long paddr, - unsigned long npages, enum psc_op op) +static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size, + int ret, u64 svsm_ret) { - unsigned long paddr_end; - u64 val; + WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n", + pfn, action, page_size, ret, svsm_ret); - vaddr = vaddr & PAGE_MASK; + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); +} - paddr = paddr & PAGE_MASK; - paddr_end = paddr + (npages << PAGE_SHIFT); +static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret) +{ + unsigned int page_size; + bool action; + u64 pfn; - while (paddr < paddr_end) { - /* Page validation must be rescinded before changing to shared */ - if (op == SNP_PAGE_STATE_SHARED) - pvalidate_4k_page(vaddr, paddr, false); + pfn = pc->entry[pc->cur_index].pfn; + action = pc->entry[pc->cur_index].action; + page_size = pc->entry[pc->cur_index].page_size; - /* - * Use the MSR protocol because this function can be called before - * the GHCB is established. - */ - sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); - VMGEXIT(); + __pval_terminate(pfn, action, page_size, ret, svsm_ret); +} - val = sev_es_rd_ghcb_msr(); +static void pval_pages(struct snp_psc_desc *desc) +{ + struct psc_entry *e; + unsigned long vaddr; + unsigned int size; + unsigned int i; + bool validate; + u64 pfn; + int rc; - if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) - goto e_term; + for (i = 0; i <= desc->hdr.end_entry; i++) { + e = &desc->entries[i]; - if (GHCB_MSR_PSC_RESP_VAL(val)) - goto e_term; + pfn = e->gfn; + vaddr = (unsigned long)pfn_to_kaddr(pfn); + size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; + validate = e->operation == SNP_PAGE_STATE_PRIVATE; - /* Page validation must be performed after changing to private */ - if (op == SNP_PAGE_STATE_PRIVATE) - pvalidate_4k_page(vaddr, paddr, true); + rc = pvalidate(vaddr, size, validate); + if (!rc) + continue; - vaddr += PAGE_SIZE; - paddr += PAGE_SIZE; + if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { + unsigned long vaddr_end = vaddr + PMD_SIZE; + + for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) { + rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); + if (rc) + __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0); + } + } else { + __pval_terminate(pfn, validate, size, rc, 0); + } + } +} + +static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action, + struct svsm_pvalidate_call *pc) +{ + struct svsm_pvalidate_entry *pe; + + /* Nothing in the CA yet */ + pc->num_entries = 0; + pc->cur_index = 0; + + pe = &pc->entry[0]; + + while (pfn < pfn_end) { + pe->page_size = RMP_PG_SIZE_4K; + pe->action = action; + pe->ignore_cf = 0; + pe->pfn = pfn; + + pe++; + pfn++; + + pc->num_entries++; + if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) + break; } - return; - -e_term: - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); + return pfn; } -void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, - unsigned long npages) +static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry, + struct svsm_pvalidate_call *pc) { - /* - * This can be invoked in early boot while running identity mapped, so - * use an open coded check for SNP instead of using cc_platform_has(). - * This eliminates worries about jump tables or checking boot_cpu_data - * in the cc_platform_has() function. - */ - if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) - return; + struct svsm_pvalidate_entry *pe; + struct psc_entry *e; - /* - * Ask the hypervisor to mark the memory pages as private in the RMP - * table. - */ - early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE); + /* Nothing in the CA yet */ + pc->num_entries = 0; + pc->cur_index = 0; + + pe = &pc->entry[0]; + e = &desc->entries[desc_entry]; + + while (desc_entry <= desc->hdr.end_entry) { + pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; + pe->action = e->operation == SNP_PAGE_STATE_PRIVATE; + pe->ignore_cf = 0; + pe->pfn = e->gfn; + + pe++; + e++; + + desc_entry++; + pc->num_entries++; + if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) + break; + } + + return desc_entry; } -void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, - unsigned long npages) +static void svsm_pval_pages(struct snp_psc_desc *desc) { - /* - * This can be invoked in early boot while running identity mapped, so - * use an open coded check for SNP instead of using cc_platform_has(). - * This eliminates worries about jump tables or checking boot_cpu_data - * in the cc_platform_has() function. - */ - if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) - return; + struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY]; + unsigned int i, pv_4k_count = 0; + struct svsm_pvalidate_call *pc; + struct svsm_call call = {}; + unsigned long flags; + bool action; + u64 pc_pa; + int ret; - /* Ask hypervisor to mark the memory pages shared in the RMP table. */ - early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); + /* + * This can be called very early in the boot, use native functions in + * order to avoid paravirt issues. + */ + flags = native_local_irq_save(); + + /* + * The SVSM calling area (CA) can support processing 510 entries at a + * time. Loop through the Page State Change descriptor until the CA is + * full or the last entry in the descriptor is reached, at which time + * the SVSM is invoked. This repeats until all entries in the descriptor + * are processed. + */ + call.caa = svsm_get_caa(); + + pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer; + pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); + + /* Protocol 0, Call ID 1 */ + call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE); + call.rcx = pc_pa; + + for (i = 0; i <= desc->hdr.end_entry;) { + i = svsm_build_ca_from_psc_desc(desc, i, pc); + + do { + ret = svsm_perform_call_protocol(&call); + if (!ret) + continue; + + /* + * Check if the entry failed because of an RMP mismatch (a + * PVALIDATE at 2M was requested, but the page is mapped in + * the RMP as 4K). + */ + + if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH && + pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) { + /* Save this entry for post-processing at 4K */ + pv_4k[pv_4k_count++] = pc->entry[pc->cur_index]; + + /* Skip to the next one unless at the end of the list */ + pc->cur_index++; + if (pc->cur_index < pc->num_entries) + ret = -EAGAIN; + else + ret = 0; + } + } while (ret == -EAGAIN); + + if (ret) + svsm_pval_terminate(pc, ret, call.rax_out); + } + + /* Process any entries that failed to be validated at 2M and validate them at 4K */ + for (i = 0; i < pv_4k_count; i++) { + u64 pfn, pfn_end; + + action = pv_4k[i].action; + pfn = pv_4k[i].pfn; + pfn_end = pfn + 512; + + while (pfn < pfn_end) { + pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc); + + ret = svsm_perform_call_protocol(&call); + if (ret) + svsm_pval_terminate(pc, ret, call.rax_out); + } + } + + native_local_irq_restore(flags); +} + +static void pvalidate_pages(struct snp_psc_desc *desc) +{ + if (snp_vmpl) + svsm_pval_pages(desc); + else + pval_pages(desc); +} + +static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) +{ + int cur_entry, end_entry, ret = 0; + struct snp_psc_desc *data; + struct es_em_ctxt ctxt; + + vc_ghcb_invalidate(ghcb); + + /* Copy the input desc into GHCB shared buffer */ + data = (struct snp_psc_desc *)ghcb->shared_buffer; + memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); + + /* + * As per the GHCB specification, the hypervisor can resume the guest + * before processing all the entries. Check whether all the entries + * are processed. If not, then keep retrying. Note, the hypervisor + * will update the data memory directly to indicate the status, so + * reference the data->hdr everywhere. + * + * The strategy here is to wait for the hypervisor to change the page + * state in the RMP table before guest accesses the memory pages. If the + * page state change was not successful, then later memory access will + * result in a crash. + */ + cur_entry = data->hdr.cur_entry; + end_entry = data->hdr.end_entry; + + while (data->hdr.cur_entry <= data->hdr.end_entry) { + ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); + + /* This will advance the shared buffer data points to. */ + ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); + + /* + * Page State Change VMGEXIT can pass error code through + * exit_info_2. + */ + if (WARN(ret || ghcb->save.sw_exit_info_2, + "SNP: PSC failed ret=%d exit_info_2=%llx\n", + ret, ghcb->save.sw_exit_info_2)) { + ret = 1; + goto out; + } + + /* Verify that reserved bit is not set */ + if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { + ret = 1; + goto out; + } + + /* + * Sanity check that entry processing is not going backwards. + * This will happen only if hypervisor is tricking us. + */ + if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, +"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", + end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { + ret = 1; + goto out; + } + } + +out: + return ret; } static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr, @@ -1246,90 +1007,6 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd) return 0; } -/* Writes to the SVSM CAA MSR are ignored */ -static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) -{ - if (write) - return ES_OK; - - regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); - regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); - - return ES_OK; -} - -/* - * TSC related accesses should not exit to the hypervisor when a guest is - * executing with Secure TSC enabled, so special handling is required for - * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ. - */ -static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write) -{ - u64 tsc; - - /* - * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled. - * Terminate the SNP guest when the interception is enabled. - */ - if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ) - return ES_VMM_ERROR; - - /* - * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC - * to return undefined values, so ignore all writes. - * - * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use - * the value returned by rdtsc_ordered(). - */ - if (write) { - WARN_ONCE(1, "TSC MSR writes are verboten!\n"); - return ES_OK; - } - - tsc = rdtsc_ordered(); - regs->ax = lower_32_bits(tsc); - regs->dx = upper_32_bits(tsc); - - return ES_OK; -} - -static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct pt_regs *regs = ctxt->regs; - enum es_result ret; - bool write; - - /* Is it a WRMSR? */ - write = ctxt->insn.opcode.bytes[1] == 0x30; - - switch (regs->cx) { - case MSR_SVSM_CAA: - return __vc_handle_msr_caa(regs, write); - case MSR_IA32_TSC: - case MSR_AMD64_GUEST_TSC_FREQ: - if (sev_status & MSR_AMD64_SNP_SECURE_TSC) - return __vc_handle_secure_tsc_msrs(regs, write); - break; - default: - break; - } - - ghcb_set_rcx(ghcb, regs->cx); - if (write) { - ghcb_set_rax(ghcb, regs->ax); - ghcb_set_rdx(ghcb, regs->dx); - } - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); - - if ((ret == ES_OK) && !write) { - regs->ax = ghcb->save.rax; - regs->dx = ghcb->save.rdx; - } - - return ret; -} - static void snp_register_per_cpu_ghcb(void) { struct sev_es_runtime_data *data; @@ -1542,748 +1219,6 @@ void __init sev_es_init_vc_handling(void) initial_vc_handler = (unsigned long)kernel_exc_vmm_communication; } -static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) -{ - int trapnr = ctxt->fi.vector; - - if (trapnr == X86_TRAP_PF) - native_write_cr2(ctxt->fi.cr2); - - ctxt->regs->orig_ax = ctxt->fi.error_code; - do_early_exception(ctxt->regs, trapnr); -} - -static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) -{ - long *reg_array; - int offset; - - reg_array = (long *)ctxt->regs; - offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); - - if (offset < 0) - return NULL; - - offset /= sizeof(long); - - return reg_array + offset; -} -static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, - unsigned int bytes, bool read) -{ - u64 exit_code, exit_info_1, exit_info_2; - unsigned long ghcb_pa = __pa(ghcb); - enum es_result res; - phys_addr_t paddr; - void __user *ref; - - ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); - if (ref == (void __user *)-1L) - return ES_UNSUPPORTED; - - exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; - - res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); - if (res != ES_OK) { - if (res == ES_EXCEPTION && !read) - ctxt->fi.error_code |= X86_PF_WRITE; - - return res; - } - - exit_info_1 = paddr; - /* Can never be greater than 8 */ - exit_info_2 = bytes; - - ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); - - return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); -} - -/* - * The MOVS instruction has two memory operands, which raises the - * problem that it is not known whether the access to the source or the - * destination caused the #VC exception (and hence whether an MMIO read - * or write operation needs to be emulated). - * - * Instead of playing games with walking page-tables and trying to guess - * whether the source or destination is an MMIO range, split the move - * into two operations, a read and a write with only one memory operand. - * This will cause a nested #VC exception on the MMIO address which can - * then be handled. - * - * This implementation has the benefit that it also supports MOVS where - * source _and_ destination are MMIO regions. - * - * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a - * rare operation. If it turns out to be a performance problem the split - * operations can be moved to memcpy_fromio() and memcpy_toio(). - */ -static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, - unsigned int bytes) -{ - unsigned long ds_base, es_base; - unsigned char *src, *dst; - unsigned char buffer[8]; - enum es_result ret; - bool rep; - int off; - - ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); - es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); - - if (ds_base == -1L || es_base == -1L) { - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; - } - - src = ds_base + (unsigned char *)ctxt->regs->si; - dst = es_base + (unsigned char *)ctxt->regs->di; - - ret = vc_read_mem(ctxt, src, buffer, bytes); - if (ret != ES_OK) - return ret; - - ret = vc_write_mem(ctxt, dst, buffer, bytes); - if (ret != ES_OK) - return ret; - - if (ctxt->regs->flags & X86_EFLAGS_DF) - off = -bytes; - else - off = bytes; - - ctxt->regs->si += off; - ctxt->regs->di += off; - - rep = insn_has_rep_prefix(&ctxt->insn); - if (rep) - ctxt->regs->cx -= 1; - - if (!rep || ctxt->regs->cx == 0) - return ES_OK; - else - return ES_RETRY; -} - -static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct insn *insn = &ctxt->insn; - enum insn_mmio_type mmio; - unsigned int bytes = 0; - enum es_result ret; - u8 sign_byte; - long *reg_data; - - mmio = insn_decode_mmio(insn, &bytes); - if (mmio == INSN_MMIO_DECODE_FAILED) - return ES_DECODE_FAILED; - - if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { - reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); - if (!reg_data) - return ES_DECODE_FAILED; - } - - if (user_mode(ctxt->regs)) - return ES_UNSUPPORTED; - - switch (mmio) { - case INSN_MMIO_WRITE: - memcpy(ghcb->shared_buffer, reg_data, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); - break; - case INSN_MMIO_WRITE_IMM: - memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); - break; - case INSN_MMIO_READ: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Zero-extend for 32-bit operation */ - if (bytes == 4) - *reg_data = 0; - - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_READ_ZERO_EXTEND: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Zero extend based on operand size */ - memset(reg_data, 0, insn->opnd_bytes); - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_READ_SIGN_EXTEND: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - if (bytes == 1) { - u8 *val = (u8 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x80) ? 0xff : 0x00; - } else { - u16 *val = (u16 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x8000) ? 0xff : 0x00; - } - - /* Sign extend based on operand size */ - memset(reg_data, sign_byte, insn->opnd_bytes); - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_MOVS: - ret = vc_handle_mmio_movs(ctxt, bytes); - break; - default: - ret = ES_UNSUPPORTED; - break; - } - - return ret; -} - -static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct sev_es_runtime_data *data = this_cpu_read(runtime_data); - long val, *reg = vc_insn_get_rm(ctxt); - enum es_result ret; - - if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) - return ES_VMM_ERROR; - - if (!reg) - return ES_DECODE_FAILED; - - val = *reg; - - /* Upper 32 bits must be written as zeroes */ - if (val >> 32) { - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; - } - - /* Clear out other reserved bits and set bit 10 */ - val = (val & 0xffff23ffL) | BIT(10); - - /* Early non-zero writes to DR7 are not supported */ - if (!data && (val & ~DR7_RESET_VALUE)) - return ES_UNSUPPORTED; - - /* Using a value of 0 for ExitInfo1 means RAX holds the value */ - ghcb_set_rax(ghcb, val); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); - if (ret != ES_OK) - return ret; - - if (data) - data->dr7 = val; - - return ES_OK; -} - -static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct sev_es_runtime_data *data = this_cpu_read(runtime_data); - long *reg = vc_insn_get_rm(ctxt); - - if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) - return ES_VMM_ERROR; - - if (!reg) - return ES_DECODE_FAILED; - - if (data) - *reg = data->dr7; - else - *reg = DR7_RESET_VALUE; - - return ES_OK; -} - -static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); -} - -static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - enum es_result ret; - - ghcb_set_rcx(ghcb, ctxt->regs->cx); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); - if (ret != ES_OK) - return ret; - - if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) - return ES_VMM_ERROR; - - ctxt->regs->ax = ghcb->save.rax; - ctxt->regs->dx = ghcb->save.rdx; - - return ES_OK; -} - -static enum es_result vc_handle_monitor(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* - * Treat it as a NOP and do not leak a physical address to the - * hypervisor. - */ - return ES_OK; -} - -static enum es_result vc_handle_mwait(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* Treat the same as MONITOR/MONITORX */ - return ES_OK; -} - -static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - enum es_result ret; - - ghcb_set_rax(ghcb, ctxt->regs->ax); - ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); - - if (x86_platform.hyper.sev_es_hcall_prepare) - x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); - if (ret != ES_OK) - return ret; - - if (!ghcb_rax_is_valid(ghcb)) - return ES_VMM_ERROR; - - ctxt->regs->ax = ghcb->save.rax; - - /* - * Call sev_es_hcall_finish() after regs->ax is already set. - * This allows the hypervisor handler to overwrite it again if - * necessary. - */ - if (x86_platform.hyper.sev_es_hcall_finish && - !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) - return ES_VMM_ERROR; - - return ES_OK; -} - -static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* - * Calling ecx_alignment_check() directly does not work, because it - * enables IRQs and the GHCB is active. Forward the exception and call - * it later from vc_forward_exception(). - */ - ctxt->fi.vector = X86_TRAP_AC; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; -} - -static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, - struct ghcb *ghcb, - unsigned long exit_code) -{ - enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); - - if (result != ES_OK) - return result; - - switch (exit_code) { - case SVM_EXIT_READ_DR7: - result = vc_handle_dr7_read(ghcb, ctxt); - break; - case SVM_EXIT_WRITE_DR7: - result = vc_handle_dr7_write(ghcb, ctxt); - break; - case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: - result = vc_handle_trap_ac(ghcb, ctxt); - break; - case SVM_EXIT_RDTSC: - case SVM_EXIT_RDTSCP: - result = vc_handle_rdtsc(ghcb, ctxt, exit_code); - break; - case SVM_EXIT_RDPMC: - result = vc_handle_rdpmc(ghcb, ctxt); - break; - case SVM_EXIT_INVD: - pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); - result = ES_UNSUPPORTED; - break; - case SVM_EXIT_CPUID: - result = vc_handle_cpuid(ghcb, ctxt); - break; - case SVM_EXIT_IOIO: - result = vc_handle_ioio(ghcb, ctxt); - break; - case SVM_EXIT_MSR: - result = vc_handle_msr(ghcb, ctxt); - break; - case SVM_EXIT_VMMCALL: - result = vc_handle_vmmcall(ghcb, ctxt); - break; - case SVM_EXIT_WBINVD: - result = vc_handle_wbinvd(ghcb, ctxt); - break; - case SVM_EXIT_MONITOR: - result = vc_handle_monitor(ghcb, ctxt); - break; - case SVM_EXIT_MWAIT: - result = vc_handle_mwait(ghcb, ctxt); - break; - case SVM_EXIT_NPF: - result = vc_handle_mmio(ghcb, ctxt); - break; - default: - /* - * Unexpected #VC exception - */ - result = ES_UNSUPPORTED; - } - - return result; -} - -static __always_inline bool is_vc2_stack(unsigned long sp) -{ - return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); -} - -static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) -{ - unsigned long sp, prev_sp; - - sp = (unsigned long)regs; - prev_sp = regs->sp; - - /* - * If the code was already executing on the VC2 stack when the #VC - * happened, let it proceed to the normal handling routine. This way the - * code executing on the VC2 stack can cause #VC exceptions to get handled. - */ - return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); -} - -static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) -{ - struct ghcb_state state; - struct es_em_ctxt ctxt; - enum es_result result; - struct ghcb *ghcb; - bool ret = true; - - ghcb = __sev_get_ghcb(&state); - - vc_ghcb_invalidate(ghcb); - result = vc_init_em_ctxt(&ctxt, regs, error_code); - - if (result == ES_OK) - result = vc_handle_exitcode(&ctxt, ghcb, error_code); - - __sev_put_ghcb(&state); - - /* Done - now check the result */ - switch (result) { - case ES_OK: - vc_finish_insn(&ctxt); - break; - case ES_UNSUPPORTED: - pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_VMM_ERROR: - pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_DECODE_FAILED: - pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_EXCEPTION: - vc_forward_exception(&ctxt); - break; - case ES_RETRY: - /* Nothing to do */ - break; - default: - pr_emerg("Unknown result in %s():%d\n", __func__, result); - /* - * Emulating the instruction which caused the #VC exception - * failed - can't continue so print debug information - */ - BUG(); - } - - return ret; -} - -static __always_inline bool vc_is_db(unsigned long error_code) -{ - return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; -} - -/* - * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode - * and will panic when an error happens. - */ -DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) -{ - irqentry_state_t irq_state; - - /* - * With the current implementation it is always possible to switch to a - * safe stack because #VC exceptions only happen at known places, like - * intercepted instructions or accesses to MMIO areas/IO ports. They can - * also happen with code instrumentation when the hypervisor intercepts - * #DB, but the critical paths are forbidden to be instrumented, so #DB - * exceptions currently also only happen in safe places. - * - * But keep this here in case the noinstr annotations are violated due - * to bug elsewhere. - */ - if (unlikely(vc_from_invalid_context(regs))) { - instrumentation_begin(); - panic("Can't handle #VC exception from unsupported context\n"); - instrumentation_end(); - } - - /* - * Handle #DB before calling into !noinstr code to avoid recursive #DB. - */ - if (vc_is_db(error_code)) { - exc_debug(regs); - return; - } - - irq_state = irqentry_nmi_enter(regs); - - instrumentation_begin(); - - if (!vc_raw_handle_exception(regs, error_code)) { - /* Show some debug info */ - show_regs(regs); - - /* Ask hypervisor to sev_es_terminate */ - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); - - /* If that fails and we get here - just panic */ - panic("Returned from Terminate-Request to Hypervisor\n"); - } - - instrumentation_end(); - irqentry_nmi_exit(regs, irq_state); -} - -/* - * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode - * and will kill the current task with SIGBUS when an error happens. - */ -DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) -{ - /* - * Handle #DB before calling into !noinstr code to avoid recursive #DB. - */ - if (vc_is_db(error_code)) { - noist_exc_debug(regs); - return; - } - - irqentry_enter_from_user_mode(regs); - instrumentation_begin(); - - if (!vc_raw_handle_exception(regs, error_code)) { - /* - * Do not kill the machine if user-space triggered the - * exception. Send SIGBUS instead and let user-space deal with - * it. - */ - force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); - } - - instrumentation_end(); - irqentry_exit_to_user_mode(regs); -} - -bool __init handle_vc_boot_ghcb(struct pt_regs *regs) -{ - unsigned long exit_code = regs->orig_ax; - struct es_em_ctxt ctxt; - enum es_result result; - - vc_ghcb_invalidate(boot_ghcb); - - result = vc_init_em_ctxt(&ctxt, regs, exit_code); - if (result == ES_OK) - result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); - - /* Done - now check the result */ - switch (result) { - case ES_OK: - vc_finish_insn(&ctxt); - break; - case ES_UNSUPPORTED: - early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_VMM_ERROR: - early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_DECODE_FAILED: - early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_EXCEPTION: - vc_early_forward_exception(&ctxt); - break; - case ES_RETRY: - /* Nothing to do */ - break; - default: - BUG(); - } - - return true; - -fail: - show_regs(regs); - - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); -} - -/* - * Initial set up of SNP relies on information provided by the - * Confidential Computing blob, which can be passed to the kernel - * in the following ways, depending on how it is booted: - * - * - when booted via the boot/decompress kernel: - * - via boot_params - * - * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH): - * - via a setup_data entry, as defined by the Linux Boot Protocol - * - * Scan for the blob in that order. - */ -static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) -{ - struct cc_blob_sev_info *cc_info; - - /* Boot kernel would have passed the CC blob via boot_params. */ - if (bp->cc_blob_address) { - cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address; - goto found_cc_info; - } - - /* - * If kernel was booted directly, without the use of the - * boot/decompression kernel, the CC blob may have been passed via - * setup_data instead. - */ - cc_info = find_cc_blob_setup_data(bp); - if (!cc_info) - return NULL; - -found_cc_info: - if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) - snp_abort(); - - return cc_info; -} - -static __head void svsm_setup(struct cc_blob_sev_info *cc_info) -{ - struct svsm_call call = {}; - int ret; - u64 pa; - - /* - * Record the SVSM Calling Area address (CAA) if the guest is not - * running at VMPL0. The CA will be used to communicate with the - * SVSM to perform the SVSM services. - */ - if (!svsm_setup_ca(cc_info)) - return; - - /* - * It is very early in the boot and the kernel is running identity - * mapped but without having adjusted the pagetables to where the - * kernel was loaded (physbase), so the get the CA address using - * RIP-relative addressing. - */ - pa = (u64)rip_rel_ptr(&boot_svsm_ca_page); - - /* - * Switch over to the boot SVSM CA while the current CA is still - * addressable. There is no GHCB at this point so use the MSR protocol. - * - * SVSM_CORE_REMAP_CA call: - * RAX = 0 (Protocol=0, CallID=0) - * RCX = New CA GPA - */ - call.caa = svsm_get_caa(); - call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); - call.rcx = pa; - ret = svsm_perform_call_protocol(&call); - if (ret) - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL); - - RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa; - RIP_REL_REF(boot_svsm_caa_pa) = pa; -} - -bool __head snp_init(struct boot_params *bp) -{ - struct cc_blob_sev_info *cc_info; - - if (!bp) - return false; - - cc_info = find_cc_blob(bp); - if (!cc_info) - return false; - - if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE) - secrets_pa = cc_info->secrets_phys; - else - return false; - - setup_cpuid_table(cc_info); - - svsm_setup(cc_info); - - /* - * The CC blob will be used later to access the secrets page. Cache - * it here like the boot kernel does. - */ - bp->cc_blob_address = (u32)(unsigned long)cc_info; - - return true; -} - -void __head __noreturn snp_abort(void) -{ - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); -} - /* * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are * enabled, as the alternative (fallback) logic for DMI probing in the legacy diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c index a7c94020e384..815542295f16 100644 --- a/arch/x86/coco/sev/shared.c +++ b/arch/x86/coco/sev/shared.c @@ -27,17 +27,12 @@ /* * SVSM related information: - * When running under an SVSM, the VMPL that Linux is executing at must be - * non-zero. The VMPL is therefore used to indicate the presence of an SVSM. - * * During boot, the page tables are set up as identity mapped and later * changed to use kernel virtual addresses. Maintain separate virtual and * physical addresses for the CAA to allow SVSM functions to be used during * early boot, both with identity mapped virtual addresses and proper kernel * virtual addresses. */ -u8 snp_vmpl __ro_after_init; -EXPORT_SYMBOL_GPL(snp_vmpl); struct svsm_ca *boot_svsm_caa __ro_after_init; u64 boot_svsm_caa_pa __ro_after_init; @@ -1192,28 +1187,6 @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) } } -static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size, - int ret, u64 svsm_ret) -{ - WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n", - pfn, action, page_size, ret, svsm_ret); - - sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); -} - -static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret) -{ - unsigned int page_size; - bool action; - u64 pfn; - - pfn = pc->entry[pc->cur_index].pfn; - action = pc->entry[pc->cur_index].action; - page_size = pc->entry[pc->cur_index].page_size; - - __pval_terminate(pfn, action, page_size, ret, svsm_ret); -} - static void __head svsm_pval_4k_page(unsigned long paddr, bool validate) { struct svsm_pvalidate_call *pc; @@ -1269,260 +1242,6 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, } } -static void pval_pages(struct snp_psc_desc *desc) -{ - struct psc_entry *e; - unsigned long vaddr; - unsigned int size; - unsigned int i; - bool validate; - u64 pfn; - int rc; - - for (i = 0; i <= desc->hdr.end_entry; i++) { - e = &desc->entries[i]; - - pfn = e->gfn; - vaddr = (unsigned long)pfn_to_kaddr(pfn); - size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; - validate = e->operation == SNP_PAGE_STATE_PRIVATE; - - rc = pvalidate(vaddr, size, validate); - if (!rc) - continue; - - if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { - unsigned long vaddr_end = vaddr + PMD_SIZE; - - for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) { - rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); - if (rc) - __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0); - } - } else { - __pval_terminate(pfn, validate, size, rc, 0); - } - } -} - -static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action, - struct svsm_pvalidate_call *pc) -{ - struct svsm_pvalidate_entry *pe; - - /* Nothing in the CA yet */ - pc->num_entries = 0; - pc->cur_index = 0; - - pe = &pc->entry[0]; - - while (pfn < pfn_end) { - pe->page_size = RMP_PG_SIZE_4K; - pe->action = action; - pe->ignore_cf = 0; - pe->pfn = pfn; - - pe++; - pfn++; - - pc->num_entries++; - if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) - break; - } - - return pfn; -} - -static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry, - struct svsm_pvalidate_call *pc) -{ - struct svsm_pvalidate_entry *pe; - struct psc_entry *e; - - /* Nothing in the CA yet */ - pc->num_entries = 0; - pc->cur_index = 0; - - pe = &pc->entry[0]; - e = &desc->entries[desc_entry]; - - while (desc_entry <= desc->hdr.end_entry) { - pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; - pe->action = e->operation == SNP_PAGE_STATE_PRIVATE; - pe->ignore_cf = 0; - pe->pfn = e->gfn; - - pe++; - e++; - - desc_entry++; - pc->num_entries++; - if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) - break; - } - - return desc_entry; -} - -static void svsm_pval_pages(struct snp_psc_desc *desc) -{ - struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY]; - unsigned int i, pv_4k_count = 0; - struct svsm_pvalidate_call *pc; - struct svsm_call call = {}; - unsigned long flags; - bool action; - u64 pc_pa; - int ret; - - /* - * This can be called very early in the boot, use native functions in - * order to avoid paravirt issues. - */ - flags = native_local_irq_save(); - - /* - * The SVSM calling area (CA) can support processing 510 entries at a - * time. Loop through the Page State Change descriptor until the CA is - * full or the last entry in the descriptor is reached, at which time - * the SVSM is invoked. This repeats until all entries in the descriptor - * are processed. - */ - call.caa = svsm_get_caa(); - - pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer; - pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); - - /* Protocol 0, Call ID 1 */ - call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE); - call.rcx = pc_pa; - - for (i = 0; i <= desc->hdr.end_entry;) { - i = svsm_build_ca_from_psc_desc(desc, i, pc); - - do { - ret = svsm_perform_call_protocol(&call); - if (!ret) - continue; - - /* - * Check if the entry failed because of an RMP mismatch (a - * PVALIDATE at 2M was requested, but the page is mapped in - * the RMP as 4K). - */ - - if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH && - pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) { - /* Save this entry for post-processing at 4K */ - pv_4k[pv_4k_count++] = pc->entry[pc->cur_index]; - - /* Skip to the next one unless at the end of the list */ - pc->cur_index++; - if (pc->cur_index < pc->num_entries) - ret = -EAGAIN; - else - ret = 0; - } - } while (ret == -EAGAIN); - - if (ret) - svsm_pval_terminate(pc, ret, call.rax_out); - } - - /* Process any entries that failed to be validated at 2M and validate them at 4K */ - for (i = 0; i < pv_4k_count; i++) { - u64 pfn, pfn_end; - - action = pv_4k[i].action; - pfn = pv_4k[i].pfn; - pfn_end = pfn + 512; - - while (pfn < pfn_end) { - pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc); - - ret = svsm_perform_call_protocol(&call); - if (ret) - svsm_pval_terminate(pc, ret, call.rax_out); - } - } - - native_local_irq_restore(flags); -} - -static void pvalidate_pages(struct snp_psc_desc *desc) -{ - if (snp_vmpl) - svsm_pval_pages(desc); - else - pval_pages(desc); -} - -static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) -{ - int cur_entry, end_entry, ret = 0; - struct snp_psc_desc *data; - struct es_em_ctxt ctxt; - - vc_ghcb_invalidate(ghcb); - - /* Copy the input desc into GHCB shared buffer */ - data = (struct snp_psc_desc *)ghcb->shared_buffer; - memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); - - /* - * As per the GHCB specification, the hypervisor can resume the guest - * before processing all the entries. Check whether all the entries - * are processed. If not, then keep retrying. Note, the hypervisor - * will update the data memory directly to indicate the status, so - * reference the data->hdr everywhere. - * - * The strategy here is to wait for the hypervisor to change the page - * state in the RMP table before guest accesses the memory pages. If the - * page state change was not successful, then later memory access will - * result in a crash. - */ - cur_entry = data->hdr.cur_entry; - end_entry = data->hdr.end_entry; - - while (data->hdr.cur_entry <= data->hdr.end_entry) { - ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); - - /* This will advance the shared buffer data points to. */ - ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); - - /* - * Page State Change VMGEXIT can pass error code through - * exit_info_2. - */ - if (WARN(ret || ghcb->save.sw_exit_info_2, - "SNP: PSC failed ret=%d exit_info_2=%llx\n", - ret, ghcb->save.sw_exit_info_2)) { - ret = 1; - goto out; - } - - /* Verify that reserved bit is not set */ - if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { - ret = 1; - goto out; - } - - /* - * Sanity check that entry processing is not going backwards. - * This will happen only if hypervisor is tricking us. - */ - if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, -"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", - end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { - ret = 1; - goto out; - } - } - -out: - return ret; -} - static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt, unsigned long exit_code) { diff --git a/arch/x86/coco/sev/startup.c b/arch/x86/coco/sev/startup.c new file mode 100644 index 000000000000..9f5dc70cfb44 --- /dev/null +++ b/arch/x86/coco/sev/startup.c @@ -0,0 +1,1395 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#define pr_fmt(fmt) "SEV: " fmt + +#include /* For show_regs() */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* For early boot hypervisor communication in SEV-ES enabled guests */ +struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); + +/* + * Needs to be in the .data section because we need it NULL before bss is + * cleared + */ +struct ghcb *boot_ghcb __section(".data"); + +/* Bitmap of SEV features supported by the hypervisor */ +u64 sev_hv_features __ro_after_init; + +/* Secrets page physical address from the CC blob */ +static u64 secrets_pa __ro_after_init; + +/* For early boot SVSM communication */ +struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); + +DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); +DEFINE_PER_CPU(u64, svsm_caa_pa); + +/* + * Nothing shall interrupt this code path while holding the per-CPU + * GHCB. The backup GHCB is only for NMIs interrupting this path. + * + * Callers must disable local interrupts around it. + */ +noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + WARN_ON(!irqs_disabled()); + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (unlikely(data->ghcb_active)) { + /* GHCB is already in use - save its contents */ + + if (unlikely(data->backup_ghcb_active)) { + /* + * Backup-GHCB is also already in use. There is no way + * to continue here so just kill the machine. To make + * panic() work, mark GHCBs inactive so that messages + * can be printed out. + */ + data->ghcb_active = false; + data->backup_ghcb_active = false; + + instrumentation_begin(); + panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); + instrumentation_end(); + } + + /* Mark backup_ghcb active before writing to it */ + data->backup_ghcb_active = true; + + state->ghcb = &data->backup_ghcb; + + /* Backup GHCB content */ + *state->ghcb = *ghcb; + } else { + state->ghcb = NULL; + data->ghcb_active = true; + } + + return ghcb; +} + +static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, + unsigned char *buffer) +{ + return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); +} + +static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + int insn_bytes; + + insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); + if (insn_bytes == 0) { + /* Nothing could be copied */ + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } else if (insn_bytes == -EINVAL) { + /* Effective RIP could not be calculated */ + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + ctxt->fi.cr2 = 0; + return ES_EXCEPTION; + } + + if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) + return ES_DECODE_FAILED; + + if (ctxt->insn.immediate.got) + return ES_OK; + else + return ES_DECODE_FAILED; +} + +static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + int res, ret; + + res = vc_fetch_insn_kernel(ctxt, buffer); + if (res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); + if (ret < 0) + return ES_DECODE_FAILED; + else + return ES_OK; +} + +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + if (user_mode(ctxt->regs)) + return __vc_decode_user_insn(ctxt); + else + return __vc_decode_kern_insn(ctxt); +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + char *dst, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; + + /* + * This function uses __put_user() independent of whether kernel or user + * memory is accessed. This works fine because __put_user() does no + * sanity checks of the pointer being accessed. All that it does is + * to report when the access failed. + * + * Also, this function runs in atomic context, so __put_user() is not + * allowed to sleep. The page-fault handler detects that it is running + * in atomic context and will not try to take mmap_sem and handle the + * fault, so additional pagefault_enable()/disable() calls are not + * needed. + * + * The access can't be done via copy_to_user() here because + * vc_write_mem() must not use string instructions to access unsafe + * memory. The reason is that MOVS is emulated by the #VC handler by + * splitting the move up into a read and a write and taking a nested #VC + * exception on whatever of them is the MMIO access. Using string + * instructions here would cause infinite nesting. + */ + switch (size) { + case 1: { + u8 d1; + u8 __user *target = (u8 __user *)dst; + + memcpy(&d1, buf, 1); + if (__put_user(d1, target)) + goto fault; + break; + } + case 2: { + u16 d2; + u16 __user *target = (u16 __user *)dst; + + memcpy(&d2, buf, 2); + if (__put_user(d2, target)) + goto fault; + break; + } + case 4: { + u32 d4; + u32 __user *target = (u32 __user *)dst; + + memcpy(&d4, buf, 4); + if (__put_user(d4, target)) + goto fault; + break; + } + case 8: { + u64 d8; + u64 __user *target = (u64 __user *)dst; + + memcpy(&d8, buf, 8); + if (__put_user(d8, target)) + goto fault; + break; + } + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)dst; + + return ES_EXCEPTION; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + char *src, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT; + + /* + * This function uses __get_user() independent of whether kernel or user + * memory is accessed. This works fine because __get_user() does no + * sanity checks of the pointer being accessed. All that it does is + * to report when the access failed. + * + * Also, this function runs in atomic context, so __get_user() is not + * allowed to sleep. The page-fault handler detects that it is running + * in atomic context and will not try to take mmap_sem and handle the + * fault, so additional pagefault_enable()/disable() calls are not + * needed. + * + * The access can't be done via copy_from_user() here because + * vc_read_mem() must not use string instructions to access unsafe + * memory. The reason is that MOVS is emulated by the #VC handler by + * splitting the move up into a read and a write and taking a nested #VC + * exception on whatever of them is the MMIO access. Using string + * instructions here would cause infinite nesting. + */ + switch (size) { + case 1: { + u8 d1; + u8 __user *s = (u8 __user *)src; + + if (__get_user(d1, s)) + goto fault; + memcpy(buf, &d1, 1); + break; + } + case 2: { + u16 d2; + u16 __user *s = (u16 __user *)src; + + if (__get_user(d2, s)) + goto fault; + memcpy(buf, &d2, 2); + break; + } + case 4: { + u32 d4; + u32 __user *s = (u32 __user *)src; + + if (__get_user(d4, s)) + goto fault; + memcpy(buf, &d4, 4); + break; + } + case 8: { + u64 d8; + u64 __user *s = (u64 __user *)src; + if (__get_user(d8, s)) + goto fault; + memcpy(buf, &d8, 8); + break; + } + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)src; + + return ES_EXCEPTION; +} + +static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned long vaddr, phys_addr_t *paddr) +{ + unsigned long va = (unsigned long)vaddr; + unsigned int level; + phys_addr_t pa; + pgd_t *pgd; + pte_t *pte; + + pgd = __va(read_cr3_pa()); + pgd = &pgd[pgd_index(va)]; + pte = lookup_address_in_pgd(pgd, va, &level); + if (!pte) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.cr2 = vaddr; + ctxt->fi.error_code = 0; + + if (user_mode(ctxt->regs)) + ctxt->fi.error_code |= X86_PF_USER; + + return ES_EXCEPTION; + } + + if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) + /* Emulated MMIO to/from encrypted memory not supported */ + return ES_UNSUPPORTED; + + pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; + pa |= va & ~page_level_mask(level); + + *paddr = pa; + + return ES_OK; +} + +static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) +{ + BUG_ON(size > 4); + + if (user_mode(ctxt->regs)) { + struct thread_struct *t = ¤t->thread; + struct io_bitmap *iobm = t->io_bitmap; + size_t idx; + + if (!iobm) + goto fault; + + for (idx = port; idx < port + size; ++idx) { + if (test_bit(idx, iobm->bitmap)) + goto fault; + } + } + + return ES_OK; + +fault: + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + + return ES_EXCEPTION; +} + +static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) +{ + long error_code = ctxt->fi.error_code; + int trapnr = ctxt->fi.vector; + + ctxt->regs->orig_ax = ctxt->fi.error_code; + + switch (trapnr) { + case X86_TRAP_GP: + exc_general_protection(ctxt->regs, error_code); + break; + case X86_TRAP_UD: + exc_invalid_op(ctxt->regs); + break; + case X86_TRAP_PF: + write_cr2(ctxt->fi.cr2); + exc_page_fault(ctxt->regs, error_code); + break; + case X86_TRAP_AC: + exc_alignment_check(ctxt->regs, error_code); + break; + default: + pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); + BUG(); + } +} + +/* Include code shared with pre-decompression boot stage */ +#include "shared.c" + +noinstr void __sev_put_ghcb(struct ghcb_state *state) +{ + struct sev_es_runtime_data *data; + struct ghcb *ghcb; + + WARN_ON(!irqs_disabled()); + + data = this_cpu_read(runtime_data); + ghcb = &data->ghcb_page; + + if (state->ghcb) { + /* Restore GHCB from Backup */ + *ghcb = *state->ghcb; + data->backup_ghcb_active = false; + state->ghcb = NULL; + } else { + /* + * Invalidate the GHCB so a VMGEXIT instruction issued + * from userspace won't appear to be valid. + */ + vc_ghcb_invalidate(ghcb); + data->ghcb_active = false; + } +} + +int svsm_perform_call_protocol(struct svsm_call *call) +{ + struct ghcb_state state; + unsigned long flags; + struct ghcb *ghcb; + int ret; + + /* + * This can be called very early in the boot, use native functions in + * order to avoid paravirt issues. + */ + flags = native_local_irq_save(); + + /* + * Use rip-relative references when called early in the boot. If + * ghcbs_initialized is set, then it is late in the boot and no need + * to worry about rip-relative references in called functions. + */ + if (RIP_REL_REF(sev_cfg).ghcbs_initialized) + ghcb = __sev_get_ghcb(&state); + else if (RIP_REL_REF(boot_ghcb)) + ghcb = RIP_REL_REF(boot_ghcb); + else + ghcb = NULL; + + do { + ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) + : svsm_perform_msr_protocol(call); + } while (ret == -EAGAIN); + + if (RIP_REL_REF(sev_cfg).ghcbs_initialized) + __sev_put_ghcb(&state); + + native_local_irq_restore(flags); + + return ret; +} + +void __head +early_set_pages_state(unsigned long vaddr, unsigned long paddr, + unsigned long npages, enum psc_op op) +{ + unsigned long paddr_end; + u64 val; + + vaddr = vaddr & PAGE_MASK; + + paddr = paddr & PAGE_MASK; + paddr_end = paddr + (npages << PAGE_SHIFT); + + while (paddr < paddr_end) { + /* Page validation must be rescinded before changing to shared */ + if (op == SNP_PAGE_STATE_SHARED) + pvalidate_4k_page(vaddr, paddr, false); + + /* + * Use the MSR protocol because this function can be called before + * the GHCB is established. + */ + sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); + VMGEXIT(); + + val = sev_es_rd_ghcb_msr(); + + if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) + goto e_term; + + if (GHCB_MSR_PSC_RESP_VAL(val)) + goto e_term; + + /* Page validation must be performed after changing to private */ + if (op == SNP_PAGE_STATE_PRIVATE) + pvalidate_4k_page(vaddr, paddr, true); + + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + return; + +e_term: + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); +} + +void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, + unsigned long npages) +{ + /* + * This can be invoked in early boot while running identity mapped, so + * use an open coded check for SNP instead of using cc_platform_has(). + * This eliminates worries about jump tables or checking boot_cpu_data + * in the cc_platform_has() function. + */ + if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) + return; + + /* + * Ask the hypervisor to mark the memory pages as private in the RMP + * table. + */ + early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE); +} + +void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, + unsigned long npages) +{ + /* + * This can be invoked in early boot while running identity mapped, so + * use an open coded check for SNP instead of using cc_platform_has(). + * This eliminates worries about jump tables or checking boot_cpu_data + * in the cc_platform_has() function. + */ + if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) + return; + + /* Ask hypervisor to mark the memory pages shared in the RMP table. */ + early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); +} + +/* Writes to the SVSM CAA MSR are ignored */ +static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) +{ + if (write) + return ES_OK; + + regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); + regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); + + return ES_OK; +} + +/* + * TSC related accesses should not exit to the hypervisor when a guest is + * executing with Secure TSC enabled, so special handling is required for + * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ. + */ +static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write) +{ + u64 tsc; + + /* + * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled. + * Terminate the SNP guest when the interception is enabled. + */ + if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ) + return ES_VMM_ERROR; + + /* + * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC + * to return undefined values, so ignore all writes. + * + * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use + * the value returned by rdtsc_ordered(). + */ + if (write) { + WARN_ONCE(1, "TSC MSR writes are verboten!\n"); + return ES_OK; + } + + tsc = rdtsc_ordered(); + regs->ax = lower_32_bits(tsc); + regs->dx = upper_32_bits(tsc); + + return ES_OK; +} + +static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + enum es_result ret; + bool write; + + /* Is it a WRMSR? */ + write = ctxt->insn.opcode.bytes[1] == 0x30; + + switch (regs->cx) { + case MSR_SVSM_CAA: + return __vc_handle_msr_caa(regs, write); + case MSR_IA32_TSC: + case MSR_AMD64_GUEST_TSC_FREQ: + if (sev_status & MSR_AMD64_SNP_SECURE_TSC) + return __vc_handle_secure_tsc_msrs(regs, write); + break; + default: + break; + } + + ghcb_set_rcx(ghcb, regs->cx); + if (write) { + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rdx(ghcb, regs->dx); + } + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); + + if ((ret == ES_OK) && !write) { + regs->ax = ghcb->save.rax; + regs->dx = ghcb->save.rdx; + } + + return ret; +} + +static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) +{ + int trapnr = ctxt->fi.vector; + + if (trapnr == X86_TRAP_PF) + native_write_cr2(ctxt->fi.cr2); + + ctxt->regs->orig_ax = ctxt->fi.error_code; + do_early_exception(ctxt->regs, trapnr); +} + +static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} +static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned int bytes, bool read) +{ + u64 exit_code, exit_info_1, exit_info_2; + unsigned long ghcb_pa = __pa(ghcb); + enum es_result res; + phys_addr_t paddr; + void __user *ref; + + ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); + if (ref == (void __user *)-1L) + return ES_UNSUPPORTED; + + exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; + + res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); + if (res != ES_OK) { + if (res == ES_EXCEPTION && !read) + ctxt->fi.error_code |= X86_PF_WRITE; + + return res; + } + + exit_info_1 = paddr; + /* Can never be greater than 8 */ + exit_info_2 = bytes; + + ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); + + return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); +} + +/* + * The MOVS instruction has two memory operands, which raises the + * problem that it is not known whether the access to the source or the + * destination caused the #VC exception (and hence whether an MMIO read + * or write operation needs to be emulated). + * + * Instead of playing games with walking page-tables and trying to guess + * whether the source or destination is an MMIO range, split the move + * into two operations, a read and a write with only one memory operand. + * This will cause a nested #VC exception on the MMIO address which can + * then be handled. + * + * This implementation has the benefit that it also supports MOVS where + * source _and_ destination are MMIO regions. + * + * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a + * rare operation. If it turns out to be a performance problem the split + * operations can be moved to memcpy_fromio() and memcpy_toio(). + */ +static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, + unsigned int bytes) +{ + unsigned long ds_base, es_base; + unsigned char *src, *dst; + unsigned char buffer[8]; + enum es_result ret; + bool rep; + int off; + + ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + if (ds_base == -1L || es_base == -1L) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + src = ds_base + (unsigned char *)ctxt->regs->si; + dst = es_base + (unsigned char *)ctxt->regs->di; + + ret = vc_read_mem(ctxt, src, buffer, bytes); + if (ret != ES_OK) + return ret; + + ret = vc_write_mem(ctxt, dst, buffer, bytes); + if (ret != ES_OK) + return ret; + + if (ctxt->regs->flags & X86_EFLAGS_DF) + off = -bytes; + else + off = bytes; + + ctxt->regs->si += off; + ctxt->regs->di += off; + + rep = insn_has_rep_prefix(&ctxt->insn); + if (rep) + ctxt->regs->cx -= 1; + + if (!rep || ctxt->regs->cx == 0) + return ES_OK; + else + return ES_RETRY; +} + +static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + enum insn_mmio_type mmio; + unsigned int bytes = 0; + enum es_result ret; + u8 sign_byte; + long *reg_data; + + mmio = insn_decode_mmio(insn, &bytes); + if (mmio == INSN_MMIO_DECODE_FAILED) + return ES_DECODE_FAILED; + + if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { + reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); + if (!reg_data) + return ES_DECODE_FAILED; + } + + if (user_mode(ctxt->regs)) + return ES_UNSUPPORTED; + + switch (mmio) { + case INSN_MMIO_WRITE: + memcpy(ghcb->shared_buffer, reg_data, bytes); + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + case INSN_MMIO_WRITE_IMM: + memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + case INSN_MMIO_READ: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero-extend for 32-bit operation */ + if (bytes == 4) + *reg_data = 0; + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_ZERO_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero extend based on operand size */ + memset(reg_data, 0, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_SIGN_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + if (bytes == 1) { + u8 *val = (u8 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + u16 *val = (u16 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + + /* Sign extend based on operand size */ + memset(reg_data, sign_byte, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_MOVS: + ret = vc_handle_mmio_movs(ctxt, bytes); + break; + default: + ret = ES_UNSUPPORTED; + break; + } + + return ret; +} + +static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long val, *reg = vc_insn_get_rm(ctxt); + enum es_result ret; + + if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) + return ES_VMM_ERROR; + + if (!reg) + return ES_DECODE_FAILED; + + val = *reg; + + /* Upper 32 bits must be written as zeroes */ + if (val >> 32) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + /* Clear out other reserved bits and set bit 10 */ + val = (val & 0xffff23ffL) | BIT(10); + + /* Early non-zero writes to DR7 are not supported */ + if (!data && (val & ~DR7_RESET_VALUE)) + return ES_UNSUPPORTED; + + /* Using a value of 0 for ExitInfo1 means RAX holds the value */ + ghcb_set_rax(ghcb, val); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); + if (ret != ES_OK) + return ret; + + if (data) + data->dr7 = val; + + return ES_OK; +} + +static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long *reg = vc_insn_get_rm(ctxt); + + if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) + return ES_VMM_ERROR; + + if (!reg) + return ES_DECODE_FAILED; + + if (data) + *reg = data->dr7; + else + *reg = DR7_RESET_VALUE; + + return ES_OK; +} + +static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); +} + +static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rcx(ghcb, ctxt->regs->cx); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + + return ES_OK; +} + +static enum es_result vc_handle_monitor(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Treat it as a NOP and do not leak a physical address to the + * hypervisor. + */ + return ES_OK; +} + +static enum es_result vc_handle_mwait(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* Treat the same as MONITOR/MONITORX */ + return ES_OK; +} + +static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rax(ghcb, ctxt->regs->ax); + ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + + if (x86_platform.hyper.sev_es_hcall_prepare) + x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); + if (ret != ES_OK) + return ret; + + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + + /* + * Call sev_es_hcall_finish() after regs->ax is already set. + * This allows the hypervisor handler to overwrite it again if + * necessary. + */ + if (x86_platform.hyper.sev_es_hcall_finish && + !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) + return ES_VMM_ERROR; + + return ES_OK; +} + +static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Calling ecx_alignment_check() directly does not work, because it + * enables IRQs and the GHCB is active. Forward the exception and call + * it later from vc_forward_exception(). + */ + ctxt->fi.vector = X86_TRAP_AC; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; +} + +static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, + struct ghcb *ghcb, + unsigned long exit_code) +{ + enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); + + if (result != ES_OK) + return result; + + switch (exit_code) { + case SVM_EXIT_READ_DR7: + result = vc_handle_dr7_read(ghcb, ctxt); + break; + case SVM_EXIT_WRITE_DR7: + result = vc_handle_dr7_write(ghcb, ctxt); + break; + case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: + result = vc_handle_trap_ac(ghcb, ctxt); + break; + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(ghcb, ctxt, exit_code); + break; + case SVM_EXIT_RDPMC: + result = vc_handle_rdpmc(ghcb, ctxt); + break; + case SVM_EXIT_INVD: + pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); + result = ES_UNSUPPORTED; + break; + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(ghcb, ctxt); + break; + case SVM_EXIT_IOIO: + result = vc_handle_ioio(ghcb, ctxt); + break; + case SVM_EXIT_MSR: + result = vc_handle_msr(ghcb, ctxt); + break; + case SVM_EXIT_VMMCALL: + result = vc_handle_vmmcall(ghcb, ctxt); + break; + case SVM_EXIT_WBINVD: + result = vc_handle_wbinvd(ghcb, ctxt); + break; + case SVM_EXIT_MONITOR: + result = vc_handle_monitor(ghcb, ctxt); + break; + case SVM_EXIT_MWAIT: + result = vc_handle_mwait(ghcb, ctxt); + break; + case SVM_EXIT_NPF: + result = vc_handle_mmio(ghcb, ctxt); + break; + default: + /* + * Unexpected #VC exception + */ + result = ES_UNSUPPORTED; + } + + return result; +} + +static __always_inline bool is_vc2_stack(unsigned long sp) +{ + return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); +} + +static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) +{ + unsigned long sp, prev_sp; + + sp = (unsigned long)regs; + prev_sp = regs->sp; + + /* + * If the code was already executing on the VC2 stack when the #VC + * happened, let it proceed to the normal handling routine. This way the + * code executing on the VC2 stack can cause #VC exceptions to get handled. + */ + return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); +} + +static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) +{ + struct ghcb_state state; + struct es_em_ctxt ctxt; + enum es_result result; + struct ghcb *ghcb; + bool ret = true; + + ghcb = __sev_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + result = vc_init_em_ctxt(&ctxt, regs, error_code); + + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, ghcb, error_code); + + __sev_put_ghcb(&state); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_VMM_ERROR: + pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_DECODE_FAILED: + pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_EXCEPTION: + vc_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + pr_emerg("Unknown result in %s():%d\n", __func__, result); + /* + * Emulating the instruction which caused the #VC exception + * failed - can't continue so print debug information + */ + BUG(); + } + + return ret; +} + +static __always_inline bool vc_is_db(unsigned long error_code) +{ + return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; +} + +/* + * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode + * and will panic when an error happens. + */ +DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) +{ + irqentry_state_t irq_state; + + /* + * With the current implementation it is always possible to switch to a + * safe stack because #VC exceptions only happen at known places, like + * intercepted instructions or accesses to MMIO areas/IO ports. They can + * also happen with code instrumentation when the hypervisor intercepts + * #DB, but the critical paths are forbidden to be instrumented, so #DB + * exceptions currently also only happen in safe places. + * + * But keep this here in case the noinstr annotations are violated due + * to bug elsewhere. + */ + if (unlikely(vc_from_invalid_context(regs))) { + instrumentation_begin(); + panic("Can't handle #VC exception from unsupported context\n"); + instrumentation_end(); + } + + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (vc_is_db(error_code)) { + exc_debug(regs); + return; + } + + irq_state = irqentry_nmi_enter(regs); + + instrumentation_begin(); + + if (!vc_raw_handle_exception(regs, error_code)) { + /* Show some debug info */ + show_regs(regs); + + /* Ask hypervisor to sev_es_terminate */ + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); + + /* If that fails and we get here - just panic */ + panic("Returned from Terminate-Request to Hypervisor\n"); + } + + instrumentation_end(); + irqentry_nmi_exit(regs, irq_state); +} + +/* + * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode + * and will kill the current task with SIGBUS when an error happens. + */ +DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) +{ + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (vc_is_db(error_code)) { + noist_exc_debug(regs); + return; + } + + irqentry_enter_from_user_mode(regs); + instrumentation_begin(); + + if (!vc_raw_handle_exception(regs, error_code)) { + /* + * Do not kill the machine if user-space triggered the + * exception. Send SIGBUS instead and let user-space deal with + * it. + */ + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); + } + + instrumentation_end(); + irqentry_exit_to_user_mode(regs); +} + +bool __init handle_vc_boot_ghcb(struct pt_regs *regs) +{ + unsigned long exit_code = regs->orig_ax; + struct es_em_ctxt ctxt; + enum es_result result; + + vc_ghcb_invalidate(boot_ghcb); + + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_early_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + BUG(); + } + + return true; + +fail: + show_regs(regs); + + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); +} + +/* + * Initial set up of SNP relies on information provided by the + * Confidential Computing blob, which can be passed to the kernel + * in the following ways, depending on how it is booted: + * + * - when booted via the boot/decompress kernel: + * - via boot_params + * + * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH): + * - via a setup_data entry, as defined by the Linux Boot Protocol + * + * Scan for the blob in that order. + */ +static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) +{ + struct cc_blob_sev_info *cc_info; + + /* Boot kernel would have passed the CC blob via boot_params. */ + if (bp->cc_blob_address) { + cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address; + goto found_cc_info; + } + + /* + * If kernel was booted directly, without the use of the + * boot/decompression kernel, the CC blob may have been passed via + * setup_data instead. + */ + cc_info = find_cc_blob_setup_data(bp); + if (!cc_info) + return NULL; + +found_cc_info: + if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) + snp_abort(); + + return cc_info; +} + +static __head void svsm_setup(struct cc_blob_sev_info *cc_info) +{ + struct svsm_call call = {}; + int ret; + u64 pa; + + /* + * Record the SVSM Calling Area address (CAA) if the guest is not + * running at VMPL0. The CA will be used to communicate with the + * SVSM to perform the SVSM services. + */ + if (!svsm_setup_ca(cc_info)) + return; + + /* + * It is very early in the boot and the kernel is running identity + * mapped but without having adjusted the pagetables to where the + * kernel was loaded (physbase), so the get the CA address using + * RIP-relative addressing. + */ + pa = (u64)rip_rel_ptr(&boot_svsm_ca_page); + + /* + * Switch over to the boot SVSM CA while the current CA is still + * addressable. There is no GHCB at this point so use the MSR protocol. + * + * SVSM_CORE_REMAP_CA call: + * RAX = 0 (Protocol=0, CallID=0) + * RCX = New CA GPA + */ + call.caa = svsm_get_caa(); + call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); + call.rcx = pa; + ret = svsm_perform_call_protocol(&call); + if (ret) + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL); + + RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa; + RIP_REL_REF(boot_svsm_caa_pa) = pa; +} + +bool __head snp_init(struct boot_params *bp) +{ + struct cc_blob_sev_info *cc_info; + + if (!bp) + return false; + + cc_info = find_cc_blob(bp); + if (!cc_info) + return false; + + if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE) + secrets_pa = cc_info->secrets_phys; + else + return false; + + setup_cpuid_table(cc_info); + + svsm_setup(cc_info); + + /* + * The CC blob will be used later to access the secrets page. Cache + * it here like the boot kernel does. + */ + bp->cc_blob_address = (u32)(unsigned long)cc_info; + + return true; +} + +void __head __noreturn snp_abort(void) +{ + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); +} From a3cbbb4717e120f9e53fa7685909b310f7e99bf5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:12:58 +0200 Subject: [PATCH 238/331] x86/boot: Move SEV startup code into startup/ Move the SEV startup code into arch/x86/boot/startup/, where it will reside along with other code that executes extremely early, and therefore needs to be built in a special manner. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-12-ardb+git@google.com --- arch/x86/boot/compressed/sev.c | 2 +- arch/x86/boot/startup/Makefile | 2 +- .../shared.c => boot/startup/sev-shared.c} | 0 .../startup.c => boot/startup/sev-startup.c} | 2 +- arch/x86/coco/sev/Makefile | 20 ++----------------- 5 files changed, 5 insertions(+), 21 deletions(-) rename arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c} (100%) rename arch/x86/{coco/sev/startup.c => boot/startup/sev-startup.c} (99%) diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 714e30c66eae..478c65149cf0 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -144,7 +144,7 @@ int svsm_perform_call_protocol(struct svsm_call *call); u8 snp_vmpl; /* Include code for early handlers */ -#include "../../coco/sev/shared.c" +#include "../../boot/startup/sev-shared.c" int svsm_perform_call_protocol(struct svsm_call *call) { diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index bb7c826e586c..21d911b61459 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -17,7 +17,7 @@ UBSAN_SANITIZE := n KCOV_INSTRUMENT := n obj-$(CONFIG_X86_64) += gdt_idt.o map_kernel.o -obj-$(CONFIG_AMD_MEM_ENCRYPT) += sme.o +obj-$(CONFIG_AMD_MEM_ENCRYPT) += sme.o sev-startup.o lib-$(CONFIG_X86_64) += la57toggle.o lib-$(CONFIG_EFI_MIXED) += efi-mixed.o diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/boot/startup/sev-shared.c similarity index 100% rename from arch/x86/coco/sev/shared.c rename to arch/x86/boot/startup/sev-shared.c diff --git a/arch/x86/coco/sev/startup.c b/arch/x86/boot/startup/sev-startup.c similarity index 99% rename from arch/x86/coco/sev/startup.c rename to arch/x86/boot/startup/sev-startup.c index 9f5dc70cfb44..10b636009d1c 100644 --- a/arch/x86/coco/sev/startup.c +++ b/arch/x86/boot/startup/sev-startup.c @@ -422,7 +422,7 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) } /* Include code shared with pre-decompression boot stage */ -#include "shared.c" +#include "sev-shared.c" noinstr void __sev_put_ghcb(struct ghcb_state *state) { diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile index 57e25f946663..2919dcfc4b02 100644 --- a/arch/x86/coco/sev/Makefile +++ b/arch/x86/coco/sev/Makefile @@ -1,25 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += core.o sev-nmi.o startup.o - -# jump tables are emitted using absolute references in non-PIC code -# so they cannot be used in the early SEV startup code -CFLAGS_startup.o += -fno-jump-tables - -ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_startup.o = -pg -endif - -KASAN_SANITIZE_startup.o := n -KMSAN_SANITIZE_startup.o := n -KCOV_INSTRUMENT_startup.o := n - -# With some compiler versions the generated code results in boot hangs, caused -# by several compilation units. To be safe, disable all instrumentation. -KCSAN_SANITIZE := n +obj-y += core.o sev-nmi.o # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining -UBSAN_SANITIZE := n +UBSAN_SANITIZE_sev-nmi.o := n # GCC may fail to respect __no_sanitize_address when inlining KASAN_SANITIZE_sev-nmi.o := n From 681e2901330c5c27ce8b58dfdd92a3c339e47caf Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:12:59 +0200 Subject: [PATCH 239/331] x86/boot: Drop RIP_REL_REF() uses from early SEV code Now that the early SEV code is built with -fPIC, RIP_REL_REF() has no effect and can be dropped. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-13-ardb+git@google.com --- arch/x86/boot/startup/sev-shared.c | 26 +++++++++++--------------- arch/x86/boot/startup/sev-startup.c | 21 ++++++++------------- arch/x86/include/asm/sev-internal.h | 18 ++++-------------- 3 files changed, 23 insertions(+), 42 deletions(-) diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c index 815542295f16..173f3d1f777a 100644 --- a/arch/x86/boot/startup/sev-shared.c +++ b/arch/x86/boot/startup/sev-shared.c @@ -299,7 +299,7 @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) * Fill in protocol and format specifiers. This can be called very early * in the boot, so use rip-relative references as needed. */ - ghcb->protocol_version = RIP_REL_REF(ghcb_version); + ghcb->protocol_version = ghcb_version; ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL); @@ -656,9 +656,9 @@ snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0; /* Skip post-processing for out-of-range zero leafs. */ - if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) || - (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) || - (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max)))) + if (!(leaf->fn <= cpuid_std_range_max || + (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) || + (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max))) return 0; } @@ -1179,11 +1179,11 @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; if (fn->eax_in == 0x0) - RIP_REL_REF(cpuid_std_range_max) = fn->eax; + cpuid_std_range_max = fn->eax; else if (fn->eax_in == 0x40000000) - RIP_REL_REF(cpuid_hyp_range_max) = fn->eax; + cpuid_hyp_range_max = fn->eax; else if (fn->eax_in == 0x80000000) - RIP_REL_REF(cpuid_ext_range_max) = fn->eax; + cpuid_ext_range_max = fn->eax; } } @@ -1229,11 +1229,7 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, { int ret; - /* - * This can be called very early during boot, so use rIP-relative - * references as needed. - */ - if (RIP_REL_REF(snp_vmpl)) { + if (snp_vmpl) { svsm_pval_4k_page(paddr, validate); } else { ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); @@ -1377,7 +1373,7 @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) if (!secrets_page->svsm_guest_vmpl) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0); - RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl; + snp_vmpl = secrets_page->svsm_guest_vmpl; caa = secrets_page->svsm_caa; @@ -1392,8 +1388,8 @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) * The CA is identity mapped when this routine is called, both by the * decompressor code and the early kernel code. */ - RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa; - RIP_REL_REF(boot_svsm_caa_pa) = caa; + boot_svsm_caa = (struct svsm_ca *)caa; + boot_svsm_caa_pa = caa; /* Advertise the SVSM presence via CPUID. */ cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table(); diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c index 10b636009d1c..36a75c5096b0 100644 --- a/arch/x86/boot/startup/sev-startup.c +++ b/arch/x86/boot/startup/sev-startup.c @@ -462,15 +462,10 @@ int svsm_perform_call_protocol(struct svsm_call *call) */ flags = native_local_irq_save(); - /* - * Use rip-relative references when called early in the boot. If - * ghcbs_initialized is set, then it is late in the boot and no need - * to worry about rip-relative references in called functions. - */ - if (RIP_REL_REF(sev_cfg).ghcbs_initialized) + if (sev_cfg.ghcbs_initialized) ghcb = __sev_get_ghcb(&state); - else if (RIP_REL_REF(boot_ghcb)) - ghcb = RIP_REL_REF(boot_ghcb); + else if (boot_ghcb) + ghcb = boot_ghcb; else ghcb = NULL; @@ -479,7 +474,7 @@ int svsm_perform_call_protocol(struct svsm_call *call) : svsm_perform_msr_protocol(call); } while (ret == -EAGAIN); - if (RIP_REL_REF(sev_cfg).ghcbs_initialized) + if (sev_cfg.ghcbs_initialized) __sev_put_ghcb(&state); native_local_irq_restore(flags); @@ -542,7 +537,7 @@ void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long padd * This eliminates worries about jump tables or checking boot_cpu_data * in the cc_platform_has() function. */ - if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) + if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED)) return; /* @@ -561,7 +556,7 @@ void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr * This eliminates worries about jump tables or checking boot_cpu_data * in the cc_platform_has() function. */ - if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) + if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED)) return; /* Ask hypervisor to mark the memory pages shared in the RMP table. */ @@ -1356,8 +1351,8 @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info) if (ret) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL); - RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa; - RIP_REL_REF(boot_svsm_caa_pa) = pa; + boot_svsm_caa = (struct svsm_ca *)pa; + boot_svsm_caa_pa = pa; } bool __head snp_init(struct boot_params *bp) diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h index 73cb774c3639..e54847a69107 100644 --- a/arch/x86/include/asm/sev-internal.h +++ b/arch/x86/include/asm/sev-internal.h @@ -68,28 +68,18 @@ extern u64 boot_svsm_caa_pa; static __always_inline struct svsm_ca *svsm_get_caa(void) { - /* - * Use rIP-relative references when called early in the boot. If - * ->use_cas is set, then it is late in the boot and no need - * to worry about rIP-relative references. - */ - if (RIP_REL_REF(sev_cfg).use_cas) + if (sev_cfg.use_cas) return this_cpu_read(svsm_caa); else - return RIP_REL_REF(boot_svsm_caa); + return boot_svsm_caa; } static __always_inline u64 svsm_get_caa_pa(void) { - /* - * Use rIP-relative references when called early in the boot. If - * ->use_cas is set, then it is late in the boot and no need - * to worry about rIP-relative references. - */ - if (RIP_REL_REF(sev_cfg).use_cas) + if (sev_cfg.use_cas) return this_cpu_read(svsm_caa_pa); else - return RIP_REL_REF(boot_svsm_caa_pa); + return boot_svsm_caa_pa; } int svsm_perform_call_protocol(struct svsm_call *call); From ff4c0560ab020d34baf0aa6434f66333d25ae524 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Apr 2025 16:13:00 +0200 Subject: [PATCH 240/331] x86/asm: Retire RIP_REL_REF() Now that all users have been moved into startup/ where PIC codegen is used, RIP_REL_REF() is no longer needed. Remove it. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Juergen Gross Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Link: https://lore.kernel.org/r/20250418141253.2601348-14-ardb+git@google.com --- arch/x86/include/asm/asm.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index a9f07799e337..eef0771512de 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -120,11 +120,6 @@ static __always_inline __pure void *rip_rel_ptr(void *p) return p; } -#ifndef __pic__ -#define RIP_REL_REF(var) (*(typeof(&(var)))rip_rel_ptr(&(var))) -#else -#define RIP_REL_REF(var) (var) -#endif #endif /* From 121c335b36e02d6aefb72501186e060474fdf33c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 22 Apr 2025 23:05:11 +0200 Subject: [PATCH 241/331] x86/boot: Disable jump tables in PIC code objtool already struggles to identify jump tables correctly in non-PIC code, where the idiom is something like jmpq *table(,%idx,8) and the table is a list of absolute addresses of jump targets. When using -fPIC, both the table reference as well as the jump targets are emitted in a RIP-relative manner, resulting in something like leaq table(%rip), %tbl movslq (%tbl,%idx,4), %offset addq %offset, %tbl jmpq *%tbl and the table is a list of offsets of the jump targets relative to the start of the entire table. Considering that this sequence of instructions can be interleaved with other instructions that have nothing to do with the jump table in question, it is extremely difficult to infer the control flow by deriving the jump targets from the indirect jump, the location of the table and the relative offsets it contains. So let's not bother and disable jump tables for code built with -fPIC under arch/x86/boot/startup. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250422210510.600354-2-ardb+git@google.com --- arch/x86/boot/startup/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile index 21d911b61459..b514f7e81332 100644 --- a/arch/x86/boot/startup/Makefile +++ b/arch/x86/boot/startup/Makefile @@ -5,6 +5,7 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS -mcmodel=small -fPIC \ -Os -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) \ -fno-stack-protector -D__NO_FORTIFY \ + -fno-jump-tables \ -include $(srctree)/include/linux/hidden.h # disable ftrace hooks and LTO From 18ea89eae404d119ced26d80ac3e62255ce15409 Mon Sep 17 00:00:00 2001 From: Tom Lendacky Date: Wed, 23 Apr 2025 10:22:31 -0500 Subject: [PATCH 242/331] x86/sev: Share the sev_secrets_pa value again This commits breaks SNP guests: 234cf67fc3bd ("x86/sev: Split off startup code from core code") The SNP guest boots, but no longer has access to the VMPCK keys needed to communicate with the ASP, which is used, for example, to obtain an attestation report. The secrets_pa value is defined as static in both startup.c and core.c. It is set by a function in startup.c and so when used in core.c its value will be 0. Share it again and add the sev_ prefix to put it into the global SEV symbols namespace. [ mingo: Renamed to sev_secrets_pa ] Signed-off-by: Tom Lendacky Signed-off-by: Ingo Molnar Acked-by: Ard Biesheuvel Cc: Dionna Amalie Glaze Cc: Kevin Loughlin Link: https://lore.kernel.org/r/cf878810-81ed-3017-52c6-ce6aa41b5f01@amd.com --- arch/x86/boot/startup/sev-startup.c | 4 ++-- arch/x86/coco/sev/core.c | 7 ++----- arch/x86/include/asm/sev-internal.h | 1 + 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c index 36a75c5096b0..f901ce9680e6 100644 --- a/arch/x86/boot/startup/sev-startup.c +++ b/arch/x86/boot/startup/sev-startup.c @@ -55,7 +55,7 @@ struct ghcb *boot_ghcb __section(".data"); u64 sev_hv_features __ro_after_init; /* Secrets page physical address from the CC blob */ -static u64 secrets_pa __ro_after_init; +u64 sev_secrets_pa __ro_after_init; /* For early boot SVSM communication */ struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); @@ -1367,7 +1367,7 @@ bool __head snp_init(struct boot_params *bp) return false; if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE) - secrets_pa = cc_info->secrets_phys; + sev_secrets_pa = cc_info->secrets_phys; else return false; diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 617988a5f3d7..ac400525de73 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -80,9 +80,6 @@ static const char * const sev_status_feat_names[] = { [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", }; -/* Secrets page physical address from the CC blob */ -static u64 secrets_pa __ro_after_init; - /* * For Secure TSC guests, the BSP fetches TSC_INFO using SNP guest messaging and * initializes snp_tsc_scale and snp_tsc_offset. These values are replicated @@ -109,7 +106,7 @@ static u64 __init get_snp_jump_table_addr(void) void __iomem *mem; u64 addr; - mem = ioremap_encrypted(secrets_pa, PAGE_SIZE); + mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE); if (!mem) { pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n"); return 0; @@ -1599,7 +1596,7 @@ struct snp_msg_desc *snp_msg_alloc(void) if (!mdesc) return ERR_PTR(-ENOMEM); - mem = ioremap_encrypted(secrets_pa, PAGE_SIZE); + mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE); if (!mem) goto e_free_mdesc; diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h index e54847a69107..a78f97208a39 100644 --- a/arch/x86/include/asm/sev-internal.h +++ b/arch/x86/include/asm/sev-internal.h @@ -5,6 +5,7 @@ extern struct ghcb boot_ghcb_page; extern struct ghcb *boot_ghcb; extern u64 sev_hv_features; +extern u64 sev_secrets_pa; /* #VC handler runtime per-CPU data */ struct sev_es_runtime_data { From 7e173eb82ae97175fd1ae8390259227f1a76a41f Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 22 Apr 2025 18:48:27 -0500 Subject: [PATCH 243/331] i2c: piix4: Make CONFIG_I2C_PIIX4 dependent on CONFIG_X86 PIIX4 and compatible controllers are only for X86. As some headers are being moved into x86 specific headers PIIX4 won't compile on non-x86. Suggested-by: Ingo Molnar Signed-off-by: Mario Limonciello Signed-off-by: Ingo Molnar Acked-by: Andi Shyti Cc: Andy Shevchenko Cc: H. Peter Anvin Cc: Sanket Goswami Cc: Shyam Sundar S K Cc: Yazen Ghannam Cc: linux-i2c@vger.kernel.org Link: https://lore.kernel.org/r/20250422234830.2840784-3-superm1@kernel.org --- drivers/i2c/busses/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 83c88c79afe2..bbbd6240fa6e 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -200,7 +200,7 @@ config I2C_ISMT config I2C_PIIX4 tristate "Intel PIIX4 and compatible (ATI/AMD/Serverworks/Broadcom/SMSC)" - depends on PCI && HAS_IOPORT + depends on PCI && HAS_IOPORT && X86 select I2C_SMBUS help If you say yes to this option, support will be included for the Intel From 624b0d5696a89b138408d385899dd35372db324b Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 22 Apr 2025 18:48:28 -0500 Subject: [PATCH 244/331] i2c: piix4, x86/platform: Move the SB800 PIIX4 FCH definitions to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SB800_PIIX4_FCH_PM_ADDR is used to indicate the base address for the FCH PM registers. Multiple drivers may need this base address, so move related defines to a common header location and rename them accordingly. Signed-off-by: Mario Limonciello Signed-off-by: Ingo Molnar Acked-by: Andi Shyti Cc: Andy Shevchenko Cc: H. Peter Anvin Cc: Hans de Goede Cc: Ilpo Järvinen Cc: Jean Delvare Cc: Jonathan Corbet Cc: Sanket Goswami Cc: Shyam Sundar S K Cc: Yazen Ghannam Cc: linux-i2c@vger.kernel.org Link: https://lore.kernel.org/r/20250422234830.2840784-4-superm1@kernel.org --- arch/x86/include/asm/amd/fch.h | 11 +++++++++++ drivers/i2c/busses/i2c-piix4.c | 18 +++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 arch/x86/include/asm/amd/fch.h diff --git a/arch/x86/include/asm/amd/fch.h b/arch/x86/include/asm/amd/fch.h new file mode 100644 index 000000000000..28972bd8f715 --- /dev/null +++ b/arch/x86/include/asm/amd/fch.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_AMD_FCH_H_ +#define _ASM_X86_AMD_FCH_H_ + +#define FCH_PM_BASE 0xFED80300 + +/* Register offsets from PM base: */ +#define FCH_PM_DECODEEN 0x00 +#define FCH_PM_DECODEEN_SMBUS0SEL GENMASK(20, 19) + +#endif /* _ASM_X86_AMD_FCH_H_ */ diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c index dd75916157f0..59ecaa990bce 100644 --- a/drivers/i2c/busses/i2c-piix4.c +++ b/drivers/i2c/busses/i2c-piix4.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "i2c-piix4.h" @@ -80,12 +81,11 @@ #define SB800_PIIX4_PORT_IDX_MASK 0x06 #define SB800_PIIX4_PORT_IDX_SHIFT 1 -/* On kerncz and Hudson2, SmBus0Sel is at bit 20:19 of PMx00 DecodeEn */ -#define SB800_PIIX4_PORT_IDX_KERNCZ 0x02 -#define SB800_PIIX4_PORT_IDX_MASK_KERNCZ 0x18 +/* SmBus0Sel is at bit 20:19 of PMx00 DecodeEn */ +#define SB800_PIIX4_PORT_IDX_KERNCZ (FCH_PM_DECODEEN + 0x02) +#define SB800_PIIX4_PORT_IDX_MASK_KERNCZ (FCH_PM_DECODEEN_SMBUS0SEL >> 16) #define SB800_PIIX4_PORT_IDX_SHIFT_KERNCZ 3 -#define SB800_PIIX4_FCH_PM_ADDR 0xFED80300 #define SB800_PIIX4_FCH_PM_SIZE 8 #define SB800_ASF_ACPI_PATH "\\_SB.ASFC" @@ -162,19 +162,19 @@ int piix4_sb800_region_request(struct device *dev, struct sb800_mmio_cfg *mmio_c if (mmio_cfg->use_mmio) { void __iomem *addr; - if (!request_mem_region_muxed(SB800_PIIX4_FCH_PM_ADDR, + if (!request_mem_region_muxed(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE, "sb800_piix4_smb")) { dev_err(dev, "SMBus base address memory region 0x%x already in use.\n", - SB800_PIIX4_FCH_PM_ADDR); + FCH_PM_BASE); return -EBUSY; } - addr = ioremap(SB800_PIIX4_FCH_PM_ADDR, + addr = ioremap(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); if (!addr) { - release_mem_region(SB800_PIIX4_FCH_PM_ADDR, + release_mem_region(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); dev_err(dev, "SMBus base address mapping failed.\n"); return -ENOMEM; @@ -201,7 +201,7 @@ void piix4_sb800_region_release(struct device *dev, struct sb800_mmio_cfg *mmio_ { if (mmio_cfg->use_mmio) { iounmap(mmio_cfg->addr); - release_mem_region(SB800_PIIX4_FCH_PM_ADDR, + release_mem_region(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); return; } From 7094702a9e6d1256f1444ceff7be97414db51dbb Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 22 Apr 2025 18:48:29 -0500 Subject: [PATCH 245/331] platform/x86/amd/pmc: Use FCH_PM_BASE definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The s2idle MMIO quirk uses a scratch register in the FCH. Adjust the code to clarify that. Signed-off-by: Mario Limonciello Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Hans de Goede Cc: Ilpo Järvinen Cc: Shyam Sundar S K Cc: Yazen Ghannam Cc: platform-driver-x86@vger.kernel.org Link: https://lore.kernel.org/r/20250422234830.2840784-5-superm1@kernel.org --- arch/x86/include/asm/amd/fch.h | 1 + drivers/platform/x86/amd/pmc/pmc-quirks.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/amd/fch.h b/arch/x86/include/asm/amd/fch.h index 28972bd8f715..01ee15bcd25f 100644 --- a/arch/x86/include/asm/amd/fch.h +++ b/arch/x86/include/asm/amd/fch.h @@ -7,5 +7,6 @@ /* Register offsets from PM base: */ #define FCH_PM_DECODEEN 0x00 #define FCH_PM_DECODEEN_SMBUS0SEL GENMASK(20, 19) +#define FCH_PM_SCRATCH 0x80 #endif /* _ASM_X86_AMD_FCH_H_ */ diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index b4f49720c87f..b706b1f4d94b 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "pmc.h" @@ -20,7 +21,7 @@ struct quirk_entry { }; static struct quirk_entry quirk_s2idle_bug = { - .s2idle_bug_mmio = 0xfed80380, + .s2idle_bug_mmio = FCH_PM_BASE + FCH_PM_SCRATCH, }; static struct quirk_entry quirk_spurious_8042 = { From 559c758bc722ca0630d2b7f433f490cb76fe6128 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:06 -0500 Subject: [PATCH 246/331] x86/bugs: Restructure MDS mitigation Restructure MDS mitigation selection to use select/update/apply functions to create consistent vulnerability handling. [ bp: rename and beef up comment over VERW mitigation selected var for maximum clarity. ] Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-2-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 61 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 9131e612de17..f697e6b41afd 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -34,6 +34,25 @@ #include "cpu.h" +/* + * Speculation Vulnerability Handling + * + * Each vulnerability is handled with the following functions: + * _select_mitigation() -- Selects a mitigation to use. This should + * take into account all relevant command line + * options. + * _update_mitigation() -- This is called after all vulnerabilities have + * selected a mitigation, in case the selection + * may want to change based on other choices + * made. This function is optional. + * _apply_mitigation() -- Enable the selected mitigation. + * + * The compile-time mitigation in all cases should be AUTO. An explicit + * command-line option can override AUTO. If no such option is + * provided, _select_mitigation() will override AUTO to the best + * mitigation option. + */ + static void __init spectre_v1_select_mitigation(void); static void __init spectre_v2_select_mitigation(void); static void __init retbleed_select_mitigation(void); @@ -41,6 +60,8 @@ static void __init spectre_v2_user_select_mitigation(void); static void __init ssb_select_mitigation(void); static void __init l1tf_select_mitigation(void); static void __init mds_select_mitigation(void); +static void __init mds_update_mitigation(void); +static void __init mds_apply_mitigation(void); static void __init md_clear_update_mitigation(void); static void __init md_clear_select_mitigation(void); static void __init taa_select_mitigation(void); @@ -172,6 +193,7 @@ void __init cpu_select_mitigations(void) spectre_v2_user_select_mitigation(); ssb_select_mitigation(); l1tf_select_mitigation(); + mds_select_mitigation(); md_clear_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); @@ -182,6 +204,14 @@ void __init cpu_select_mitigations(void) */ srso_select_mitigation(); gds_select_mitigation(); + + /* + * After mitigations are selected, some may need to update their + * choices. + */ + mds_update_mitigation(); + + mds_apply_mitigation(); } /* @@ -284,6 +314,12 @@ enum rfds_mitigations { static enum rfds_mitigations rfds_mitigation __ro_after_init = IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF; +/* + * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing + * through X86_FEATURE_CLEAR_CPU_BUF on kernel and guest entry. + */ +static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init; + static void __init mds_select_mitigation(void) { if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { @@ -294,12 +330,34 @@ static void __init mds_select_mitigation(void) if (mds_mitigation == MDS_MITIGATION_AUTO) mds_mitigation = MDS_MITIGATION_FULL; + if (mds_mitigation == MDS_MITIGATION_OFF) + return; + + verw_clear_cpu_buf_mitigation_selected = true; +} + +static void __init mds_update_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) + return; + + /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */ + if (verw_clear_cpu_buf_mitigation_selected) + mds_mitigation = MDS_MITIGATION_FULL; + if (mds_mitigation == MDS_MITIGATION_FULL) { if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) mds_mitigation = MDS_MITIGATION_VMWERV; + } + pr_info("%s\n", mds_strings[mds_mitigation]); +} + +static void __init mds_apply_mitigation(void) +{ + if (mds_mitigation == MDS_MITIGATION_FULL || + mds_mitigation == MDS_MITIGATION_VMWERV) { setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); - if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && (mds_nosmt || cpu_mitigations_auto_nosmt())) cpu_smt_disable(false); @@ -602,7 +660,6 @@ out: static void __init md_clear_select_mitigation(void) { - mds_select_mitigation(); taa_select_mitigation(); mmio_select_mitigation(); rfds_select_mitigation(); From bdd7fce7a8168cebd400926d6352d2fbc1ac9f79 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:07 -0500 Subject: [PATCH 247/331] x86/bugs: Restructure TAA mitigation Restructure TAA mitigation to use select/update/apply functions to create consistent vulnerability handling. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-3-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 98 ++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index f697e6b41afd..5db21d2ee87d 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -65,6 +65,8 @@ static void __init mds_apply_mitigation(void); static void __init md_clear_update_mitigation(void); static void __init md_clear_select_mitigation(void); static void __init taa_select_mitigation(void); +static void __init taa_update_mitigation(void); +static void __init taa_apply_mitigation(void); static void __init mmio_select_mitigation(void); static void __init srbds_select_mitigation(void); static void __init l1d_flush_select_mitigation(void); @@ -194,6 +196,7 @@ void __init cpu_select_mitigations(void) ssb_select_mitigation(); l1tf_select_mitigation(); mds_select_mitigation(); + taa_select_mitigation(); md_clear_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); @@ -210,8 +213,10 @@ void __init cpu_select_mitigations(void) * choices. */ mds_update_mitigation(); + taa_update_mitigation(); mds_apply_mitigation(); + taa_apply_mitigation(); } /* @@ -397,6 +402,11 @@ static const char * const taa_strings[] = { [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", }; +static bool __init taa_vulnerable(void) +{ + return boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM); +} + static void __init taa_select_mitigation(void) { if (!boot_cpu_has_bug(X86_BUG_TAA)) { @@ -410,48 +420,63 @@ static void __init taa_select_mitigation(void) return; } - if (cpu_mitigations_off()) { + if (cpu_mitigations_off()) taa_mitigation = TAA_MITIGATION_OFF; + + /* Microcode will be checked in taa_update_mitigation(). */ + if (taa_mitigation == TAA_MITIGATION_AUTO) + taa_mitigation = TAA_MITIGATION_VERW; + + if (taa_mitigation != TAA_MITIGATION_OFF) + verw_clear_cpu_buf_mitigation_selected = true; +} + +static void __init taa_update_mitigation(void) +{ + if (!taa_vulnerable() || cpu_mitigations_off()) return; + + if (verw_clear_cpu_buf_mitigation_selected) + taa_mitigation = TAA_MITIGATION_VERW; + + if (taa_mitigation == TAA_MITIGATION_VERW) { + /* Check if the requisite ucode is available. */ + if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) + taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; + + /* + * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. + * A microcode update fixes this behavior to clear CPU buffers. It also + * adds support for MSR_IA32_TSX_CTRL which is enumerated by the + * ARCH_CAP_TSX_CTRL_MSR bit. + * + * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode + * update is required. + */ + if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && + !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) + taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; } - /* - * TAA mitigation via VERW is turned off if both - * tsx_async_abort=off and mds=off are specified. - */ - if (taa_mitigation == TAA_MITIGATION_OFF && - mds_mitigation == MDS_MITIGATION_OFF) - return; + pr_info("%s\n", taa_strings[taa_mitigation]); +} - if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) - taa_mitigation = TAA_MITIGATION_VERW; - else - taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; +static void __init taa_apply_mitigation(void) +{ + if (taa_mitigation == TAA_MITIGATION_VERW || + taa_mitigation == TAA_MITIGATION_UCODE_NEEDED) { + /* + * TSX is enabled, select alternate mitigation for TAA which is + * the same as MDS. Enable MDS static branch to clear CPU buffers. + * + * For guests that can't determine whether the correct microcode is + * present on host, enable the mitigation for UCODE_NEEDED as well. + */ + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); - /* - * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. - * A microcode update fixes this behavior to clear CPU buffers. It also - * adds support for MSR_IA32_TSX_CTRL which is enumerated by the - * ARCH_CAP_TSX_CTRL_MSR bit. - * - * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode - * update is required. - */ - if ( (x86_arch_cap_msr & ARCH_CAP_MDS_NO) && - !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)) - taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; - - /* - * TSX is enabled, select alternate mitigation for TAA which is - * the same as MDS. Enable MDS static branch to clear CPU buffers. - * - * For guests that can't determine whether the correct microcode is - * present on host, enable the mitigation for UCODE_NEEDED as well. - */ - setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); - - if (taa_nosmt || cpu_mitigations_auto_nosmt()) - cpu_smt_disable(false); + if (taa_nosmt || cpu_mitigations_auto_nosmt()) + cpu_smt_disable(false); + } } static int __init tsx_async_abort_parse_cmdline(char *str) @@ -660,7 +685,6 @@ out: static void __init md_clear_select_mitigation(void) { - taa_select_mitigation(); mmio_select_mitigation(); rfds_select_mitigation(); From 4a5a04e61d7f8f26472f93287f6dcb669f0cf22f Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:08 -0500 Subject: [PATCH 248/331] x86/bugs: Restructure MMIO mitigation Restructure MMIO mitigation to use select/update/apply functions to create consistent vulnerability handling. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-4-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 74 +++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 5db21d2ee87d..bc74c22e927c 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -68,6 +68,8 @@ static void __init taa_select_mitigation(void); static void __init taa_update_mitigation(void); static void __init taa_apply_mitigation(void); static void __init mmio_select_mitigation(void); +static void __init mmio_update_mitigation(void); +static void __init mmio_apply_mitigation(void); static void __init srbds_select_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); @@ -197,6 +199,7 @@ void __init cpu_select_mitigations(void) l1tf_select_mitigation(); mds_select_mitigation(); taa_select_mitigation(); + mmio_select_mitigation(); md_clear_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); @@ -214,9 +217,11 @@ void __init cpu_select_mitigations(void) */ mds_update_mitigation(); taa_update_mitigation(); + mmio_update_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); + mmio_apply_mitigation(); } /* @@ -520,25 +525,62 @@ static void __init mmio_select_mitigation(void) return; } + /* Microcode will be checked in mmio_update_mitigation(). */ + if (mmio_mitigation == MMIO_MITIGATION_AUTO) + mmio_mitigation = MMIO_MITIGATION_VERW; + if (mmio_mitigation == MMIO_MITIGATION_OFF) return; /* * Enable CPU buffer clear mitigation for host and VMM, if also affected - * by MDS or TAA. Otherwise, enable mitigation for VMM only. + * by MDS or TAA. */ - if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && - boot_cpu_has(X86_FEATURE_RTM))) - setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); + if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable()) + verw_clear_cpu_buf_mitigation_selected = true; +} + +static void __init mmio_update_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || cpu_mitigations_off()) + return; + + if (verw_clear_cpu_buf_mitigation_selected) + mmio_mitigation = MMIO_MITIGATION_VERW; + + if (mmio_mitigation == MMIO_MITIGATION_VERW) { + /* + * Check if the system has the right microcode. + * + * CPU Fill buffer clear mitigation is enumerated by either an explicit + * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS + * affected systems. + */ + if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || + (boot_cpu_has(X86_FEATURE_MD_CLEAR) && + boot_cpu_has(X86_FEATURE_FLUSH_L1D) && + !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))) + mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; + } + + pr_info("%s\n", mmio_strings[mmio_mitigation]); +} + +static void __init mmio_apply_mitigation(void) +{ + if (mmio_mitigation == MMIO_MITIGATION_OFF) + return; /* - * X86_FEATURE_CLEAR_CPU_BUF could be enabled by other VERW based - * mitigations, disable KVM-only mitigation in that case. + * Only enable the VMM mitigation if the CPU buffer clear mitigation is + * not being used. */ - if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) + if (verw_clear_cpu_buf_mitigation_selected) { + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); static_branch_disable(&cpu_buf_vm_clear); - else + } else { static_branch_enable(&cpu_buf_vm_clear); + } /* * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can @@ -548,21 +590,6 @@ static void __init mmio_select_mitigation(void) if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) static_branch_enable(&mds_idle_clear); - /* - * Check if the system has the right microcode. - * - * CPU Fill buffer clear mitigation is enumerated by either an explicit - * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS - * affected systems. - */ - if ((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) || - (boot_cpu_has(X86_FEATURE_MD_CLEAR) && - boot_cpu_has(X86_FEATURE_FLUSH_L1D) && - !(x86_arch_cap_msr & ARCH_CAP_MDS_NO))) - mmio_mitigation = MMIO_MITIGATION_VERW; - else - mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; - if (mmio_nosmt || cpu_mitigations_auto_nosmt()) cpu_smt_disable(false); } @@ -685,7 +712,6 @@ out: static void __init md_clear_select_mitigation(void) { - mmio_select_mitigation(); rfds_select_mitigation(); /* From 203d81f8e167a9e82747a14dace40e0abbd5c791 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:09 -0500 Subject: [PATCH 249/331] x86/bugs: Restructure RFDS mitigation Restructure RFDS mitigation to use select/update/apply functions to create consistent vulnerability handling. [ bp: Rename the oneline helper to what it checks. ] Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-5-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 43 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index bc74c22e927c..2705105d9a5e 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -70,6 +70,9 @@ static void __init taa_apply_mitigation(void); static void __init mmio_select_mitigation(void); static void __init mmio_update_mitigation(void); static void __init mmio_apply_mitigation(void); +static void __init rfds_select_mitigation(void); +static void __init rfds_update_mitigation(void); +static void __init rfds_apply_mitigation(void); static void __init srbds_select_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); @@ -200,6 +203,7 @@ void __init cpu_select_mitigations(void) mds_select_mitigation(); taa_select_mitigation(); mmio_select_mitigation(); + rfds_select_mitigation(); md_clear_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); @@ -218,10 +222,12 @@ void __init cpu_select_mitigations(void) mds_update_mitigation(); taa_update_mitigation(); mmio_update_mitigation(); + rfds_update_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); + rfds_apply_mitigation(); } /* @@ -624,22 +630,48 @@ static const char * const rfds_strings[] = { [RFDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", }; +static inline bool __init verw_clears_cpu_reg_file(void) +{ + return (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR); +} + static void __init rfds_select_mitigation(void) { if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) { rfds_mitigation = RFDS_MITIGATION_OFF; return; } - if (rfds_mitigation == RFDS_MITIGATION_OFF) - return; if (rfds_mitigation == RFDS_MITIGATION_AUTO) rfds_mitigation = RFDS_MITIGATION_VERW; - if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR) + if (rfds_mitigation == RFDS_MITIGATION_OFF) + return; + + if (verw_clears_cpu_reg_file()) + verw_clear_cpu_buf_mitigation_selected = true; +} + +static void __init rfds_update_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) + return; + + if (verw_clear_cpu_buf_mitigation_selected) + rfds_mitigation = RFDS_MITIGATION_VERW; + + if (rfds_mitigation == RFDS_MITIGATION_VERW) { + if (!verw_clears_cpu_reg_file()) + rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; + } + + pr_info("%s\n", rfds_strings[rfds_mitigation]); +} + +static void __init rfds_apply_mitigation(void) +{ + if (rfds_mitigation == RFDS_MITIGATION_VERW) setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); - else - rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED; } static __init int rfds_parse_cmdline(char *str) @@ -712,7 +744,6 @@ out: static void __init md_clear_select_mitigation(void) { - rfds_select_mitigation(); /* * As these mitigations are inter-related and rely on VERW instruction From 6f0960a760eb926d8a2b9fe6fc7a1086cba14dd1 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Sun, 27 Apr 2025 17:12:41 +0200 Subject: [PATCH 250/331] x86/bugs: Remove md_clear_*_mitigation() The functionality in md_clear_update_mitigation() and md_clear_select_mitigation() is now integrated into the select/update functions for the MDS, TAA, MMIO, and RFDS vulnerabilities. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-6-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 65 -------------------------------------- 1 file changed, 65 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 2705105d9a5e..98476b8ba4fd 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -62,8 +62,6 @@ static void __init l1tf_select_mitigation(void); static void __init mds_select_mitigation(void); static void __init mds_update_mitigation(void); static void __init mds_apply_mitigation(void); -static void __init md_clear_update_mitigation(void); -static void __init md_clear_select_mitigation(void); static void __init taa_select_mitigation(void); static void __init taa_update_mitigation(void); static void __init taa_apply_mitigation(void); @@ -204,7 +202,6 @@ void __init cpu_select_mitigations(void) taa_select_mitigation(); mmio_select_mitigation(); rfds_select_mitigation(); - md_clear_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); @@ -691,68 +688,6 @@ static __init int rfds_parse_cmdline(char *str) } early_param("reg_file_data_sampling", rfds_parse_cmdline); -#undef pr_fmt -#define pr_fmt(fmt) "" fmt - -static void __init md_clear_update_mitigation(void) -{ - if (cpu_mitigations_off()) - return; - - if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) - goto out; - - /* - * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO - * Stale Data mitigation, if necessary. - */ - if (mds_mitigation == MDS_MITIGATION_OFF && - boot_cpu_has_bug(X86_BUG_MDS)) { - mds_mitigation = MDS_MITIGATION_FULL; - mds_select_mitigation(); - } - if (taa_mitigation == TAA_MITIGATION_OFF && - boot_cpu_has_bug(X86_BUG_TAA)) { - taa_mitigation = TAA_MITIGATION_VERW; - taa_select_mitigation(); - } - /* - * MMIO_MITIGATION_OFF is not checked here so that cpu_buf_vm_clear - * gets updated correctly as per X86_FEATURE_CLEAR_CPU_BUF state. - */ - if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { - mmio_mitigation = MMIO_MITIGATION_VERW; - mmio_select_mitigation(); - } - if (rfds_mitigation == RFDS_MITIGATION_OFF && - boot_cpu_has_bug(X86_BUG_RFDS)) { - rfds_mitigation = RFDS_MITIGATION_VERW; - rfds_select_mitigation(); - } -out: - if (boot_cpu_has_bug(X86_BUG_MDS)) - pr_info("MDS: %s\n", mds_strings[mds_mitigation]); - if (boot_cpu_has_bug(X86_BUG_TAA)) - pr_info("TAA: %s\n", taa_strings[taa_mitigation]); - if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) - pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); - else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) - pr_info("MMIO Stale Data: Unknown: No mitigations\n"); - if (boot_cpu_has_bug(X86_BUG_RFDS)) - pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]); -} - -static void __init md_clear_select_mitigation(void) -{ - - /* - * As these mitigations are inter-related and rely on VERW instruction - * to clear the microarchitural buffers, update and print their status - * after mitigation selection is done for each of these vulnerabilities. - */ - md_clear_update_mitigation(); -} - #undef pr_fmt #define pr_fmt(fmt) "SRBDS: " fmt From 2178ac58e176d8e1e4529b02647f5e549bb88405 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:11 -0500 Subject: [PATCH 251/331] x86/bugs: Restructure SRBDS mitigation Restructure SRBDS to use select/apply functions to create consistent vulnerability handling. Define new AUTO mitigation for SRBDS. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-7-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 98476b8ba4fd..25b74a78caa3 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -72,6 +72,7 @@ static void __init rfds_select_mitigation(void); static void __init rfds_update_mitigation(void); static void __init rfds_apply_mitigation(void); static void __init srbds_select_mitigation(void); +static void __init srbds_apply_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); static void __init gds_select_mitigation(void); @@ -225,6 +226,7 @@ void __init cpu_select_mitigations(void) taa_apply_mitigation(); mmio_apply_mitigation(); rfds_apply_mitigation(); + srbds_apply_mitigation(); } /* @@ -693,6 +695,7 @@ early_param("reg_file_data_sampling", rfds_parse_cmdline); enum srbds_mitigations { SRBDS_MITIGATION_OFF, + SRBDS_MITIGATION_AUTO, SRBDS_MITIGATION_UCODE_NEEDED, SRBDS_MITIGATION_FULL, SRBDS_MITIGATION_TSX_OFF, @@ -700,7 +703,7 @@ enum srbds_mitigations { }; static enum srbds_mitigations srbds_mitigation __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_FULL : SRBDS_MITIGATION_OFF; + IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_AUTO : SRBDS_MITIGATION_OFF; static const char * const srbds_strings[] = { [SRBDS_MITIGATION_OFF] = "Vulnerable", @@ -751,8 +754,13 @@ void update_srbds_msr(void) static void __init srbds_select_mitigation(void) { - if (!boot_cpu_has_bug(X86_BUG_SRBDS)) + if (!boot_cpu_has_bug(X86_BUG_SRBDS) || cpu_mitigations_off()) { + srbds_mitigation = SRBDS_MITIGATION_OFF; return; + } + + if (srbds_mitigation == SRBDS_MITIGATION_AUTO) + srbds_mitigation = SRBDS_MITIGATION_FULL; /* * Check to see if this is one of the MDS_NO systems supporting TSX that @@ -766,13 +774,17 @@ static void __init srbds_select_mitigation(void) srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; - else if (cpu_mitigations_off() || srbds_off) + else if (srbds_off) srbds_mitigation = SRBDS_MITIGATION_OFF; - update_srbds_msr(); pr_info("%s\n", srbds_strings[srbds_mitigation]); } +static void __init srbds_apply_mitigation(void) +{ + update_srbds_msr(); +} + static int __init srbds_parse_cmdline(char *str) { if (!str) From 9dcad2fb31bd9b9b860c515859625a065dd6e656 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:12 -0500 Subject: [PATCH 252/331] x86/bugs: Restructure GDS mitigation Restructure GDS mitigation to use select/apply functions to create consistent vulnerability handling. Define new AUTO mitigation for GDS. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-8-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 43 +++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 25b74a78caa3..b070ad7be049 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -76,6 +76,7 @@ static void __init srbds_apply_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); static void __init gds_select_mitigation(void); +static void __init gds_apply_mitigation(void); /* The base value of the SPEC_CTRL MSR without task-specific bits set */ u64 x86_spec_ctrl_base; @@ -227,6 +228,7 @@ void __init cpu_select_mitigations(void) mmio_apply_mitigation(); rfds_apply_mitigation(); srbds_apply_mitigation(); + gds_apply_mitigation(); } /* @@ -831,6 +833,7 @@ early_param("l1d_flush", l1d_flush_parse_cmdline); enum gds_mitigations { GDS_MITIGATION_OFF, + GDS_MITIGATION_AUTO, GDS_MITIGATION_UCODE_NEEDED, GDS_MITIGATION_FORCE, GDS_MITIGATION_FULL, @@ -839,7 +842,7 @@ enum gds_mitigations { }; static enum gds_mitigations gds_mitigation __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_FULL : GDS_MITIGATION_OFF; + IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_AUTO : GDS_MITIGATION_OFF; static const char * const gds_strings[] = { [GDS_MITIGATION_OFF] = "Vulnerable", @@ -880,6 +883,7 @@ void update_gds_msr(void) case GDS_MITIGATION_FORCE: case GDS_MITIGATION_UCODE_NEEDED: case GDS_MITIGATION_HYPERVISOR: + case GDS_MITIGATION_AUTO: return; } @@ -903,26 +907,21 @@ static void __init gds_select_mitigation(void) if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { gds_mitigation = GDS_MITIGATION_HYPERVISOR; - goto out; + return; } if (cpu_mitigations_off()) gds_mitigation = GDS_MITIGATION_OFF; /* Will verify below that mitigation _can_ be disabled */ + if (gds_mitigation == GDS_MITIGATION_AUTO) + gds_mitigation = GDS_MITIGATION_FULL; + /* No microcode */ if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) { - if (gds_mitigation == GDS_MITIGATION_FORCE) { - /* - * This only needs to be done on the boot CPU so do it - * here rather than in update_gds_msr() - */ - setup_clear_cpu_cap(X86_FEATURE_AVX); - pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); - } else { + if (gds_mitigation != GDS_MITIGATION_FORCE) gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; - } - goto out; + return; } /* Microcode has mitigation, use it */ @@ -943,9 +942,25 @@ static void __init gds_select_mitigation(void) */ gds_mitigation = GDS_MITIGATION_FULL_LOCKED; } +} + +static void __init gds_apply_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_GDS)) + return; + + /* Microcode is present */ + if (x86_arch_cap_msr & ARCH_CAP_GDS_CTRL) + update_gds_msr(); + else if (gds_mitigation == GDS_MITIGATION_FORCE) { + /* + * This only needs to be done on the boot CPU so do it + * here rather than in update_gds_msr() + */ + setup_clear_cpu_cap(X86_FEATURE_AVX); + pr_warn("Microcode update needed! Disabling AVX as mitigation.\n"); + } - update_gds_msr(); -out: pr_info("%s\n", gds_strings[gds_mitigation]); } From 46d5925b8eb8c7a8d634147c23db24669cfc2f76 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:13 -0500 Subject: [PATCH 253/331] x86/bugs: Restructure spectre_v1 mitigation Restructure spectre_v1 to use select/apply functions to create consistent vulnerability handling. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-9-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index b070ad7be049..1a42abb289ac 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -54,6 +54,7 @@ */ static void __init spectre_v1_select_mitigation(void); +static void __init spectre_v1_apply_mitigation(void); static void __init spectre_v2_select_mitigation(void); static void __init retbleed_select_mitigation(void); static void __init spectre_v2_user_select_mitigation(void); @@ -223,6 +224,7 @@ void __init cpu_select_mitigations(void) mmio_update_mitigation(); rfds_update_mitigation(); + spectre_v1_apply_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); @@ -1021,10 +1023,14 @@ static bool smap_works_speculatively(void) static void __init spectre_v1_select_mitigation(void) { - if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { + if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; +} + +static void __init spectre_v1_apply_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) return; - } if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { /* From 83d4b19331f3a5d5829d338a0a64b69c9c28b36e Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:14 -0500 Subject: [PATCH 254/331] x86/bugs: Allow retbleed=stuff only on Intel The retbleed=stuff mitigation is only applicable for Intel CPUs affected by retbleed. If this option is selected for another vendor, print a warning and fall back to the AUTO option. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-10-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 1a42abb289ac..7edf42912cdf 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -1191,6 +1191,10 @@ static void __init retbleed_select_mitigation(void) case RETBLEED_CMD_STUFF: if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) && spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { + pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); + goto do_cmd_auto; + } retbleed_mitigation = RETBLEED_MITIGATION_STUFF; } else { From c0a62eadb6fd158e4d6d4d47d806109e7ae32e8b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 11:30:06 -0700 Subject: [PATCH 255/331] x86/microcode/AMD: Use sha256() instead of init/update/final Just call sha256() instead of doing the init/update/final sequence. No functional changes. Signed-off-by: Eric Biggers Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250428183006.782501-1-ebiggers@kernel.org --- arch/x86/kernel/cpu/microcode/amd.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 57bd61f9c69b..b2bbfc4ef3dc 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -211,7 +211,6 @@ static bool verify_sha256_digest(u32 patch_id, u32 cur_rev, const u8 *data, unsi { struct patch_digest *pd = NULL; u8 digest[SHA256_DIGEST_SIZE]; - struct sha256_state s; int i; if (x86_family(bsp_cpuid_1_eax) < 0x17 || @@ -230,9 +229,7 @@ static bool verify_sha256_digest(u32 patch_id, u32 cur_rev, const u8 *data, unsi return false; } - sha256_init(&s); - sha256_update(&s, data, len); - sha256_final(&s, digest); + sha256(data, len, digest); if (memcmp(digest, pd->sha256, sizeof(digest))) { pr_err("Patch 0x%x SHA256 digest mismatch!\n", patch_id); From e3b78a7ad5ea718ea1dbaeb02ba9a6aa2aee9324 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:15 -0500 Subject: [PATCH 256/331] x86/bugs: Restructure retbleed mitigation Restructure retbleed mitigation to use select/update/apply functions to create consistent vulnerability handling. The retbleed_update_mitigation() simplifies the dependency between spectre_v2 and retbleed. The command line options now directly select a preferred mitigation which simplifies the logic. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-11-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 225 +++++++++++++++++++------------------ 1 file changed, 113 insertions(+), 112 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 7edf42912cdf..207a472d1a6e 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -57,6 +57,8 @@ static void __init spectre_v1_select_mitigation(void); static void __init spectre_v1_apply_mitigation(void); static void __init spectre_v2_select_mitigation(void); static void __init retbleed_select_mitigation(void); +static void __init retbleed_update_mitigation(void); +static void __init retbleed_apply_mitigation(void); static void __init spectre_v2_user_select_mitigation(void); static void __init ssb_select_mitigation(void); static void __init l1tf_select_mitigation(void); @@ -187,11 +189,6 @@ void __init cpu_select_mitigations(void) /* Select the proper CPU mitigations before patching alternatives: */ spectre_v1_select_mitigation(); spectre_v2_select_mitigation(); - /* - * retbleed_select_mitigation() relies on the state set by - * spectre_v2_select_mitigation(); specifically it wants to know about - * spectre_v2=ibrs. - */ retbleed_select_mitigation(); /* * spectre_v2_user_select_mitigation() relies on the state set by @@ -219,12 +216,14 @@ void __init cpu_select_mitigations(void) * After mitigations are selected, some may need to update their * choices. */ + retbleed_update_mitigation(); mds_update_mitigation(); taa_update_mitigation(); mmio_update_mitigation(); rfds_update_mitigation(); spectre_v1_apply_mitigation(); + retbleed_apply_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); @@ -1085,6 +1084,7 @@ enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; enum retbleed_mitigation { RETBLEED_MITIGATION_NONE, + RETBLEED_MITIGATION_AUTO, RETBLEED_MITIGATION_UNRET, RETBLEED_MITIGATION_IBPB, RETBLEED_MITIGATION_IBRS, @@ -1092,14 +1092,6 @@ enum retbleed_mitigation { RETBLEED_MITIGATION_STUFF, }; -enum retbleed_mitigation_cmd { - RETBLEED_CMD_OFF, - RETBLEED_CMD_AUTO, - RETBLEED_CMD_UNRET, - RETBLEED_CMD_IBPB, - RETBLEED_CMD_STUFF, -}; - static const char * const retbleed_strings[] = { [RETBLEED_MITIGATION_NONE] = "Vulnerable", [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", @@ -1110,9 +1102,7 @@ static const char * const retbleed_strings[] = { }; static enum retbleed_mitigation retbleed_mitigation __ro_after_init = - RETBLEED_MITIGATION_NONE; -static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_CMD_AUTO : RETBLEED_CMD_OFF; + IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE; static int __ro_after_init retbleed_nosmt = false; @@ -1129,15 +1119,15 @@ static int __init retbleed_parse_cmdline(char *str) } if (!strcmp(str, "off")) { - retbleed_cmd = RETBLEED_CMD_OFF; + retbleed_mitigation = RETBLEED_MITIGATION_NONE; } else if (!strcmp(str, "auto")) { - retbleed_cmd = RETBLEED_CMD_AUTO; + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; } else if (!strcmp(str, "unret")) { - retbleed_cmd = RETBLEED_CMD_UNRET; + retbleed_mitigation = RETBLEED_MITIGATION_UNRET; } else if (!strcmp(str, "ibpb")) { - retbleed_cmd = RETBLEED_CMD_IBPB; + retbleed_mitigation = RETBLEED_MITIGATION_IBPB; } else if (!strcmp(str, "stuff")) { - retbleed_cmd = RETBLEED_CMD_STUFF; + retbleed_mitigation = RETBLEED_MITIGATION_STUFF; } else if (!strcmp(str, "nosmt")) { retbleed_nosmt = true; } else if (!strcmp(str, "force")) { @@ -1158,76 +1148,109 @@ early_param("retbleed", retbleed_parse_cmdline); static void __init retbleed_select_mitigation(void) { - bool mitigate_smt = false; - - if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) + if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) { + retbleed_mitigation = RETBLEED_MITIGATION_NONE; return; - - switch (retbleed_cmd) { - case RETBLEED_CMD_OFF: - return; - - case RETBLEED_CMD_UNRET: - if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { - retbleed_mitigation = RETBLEED_MITIGATION_UNRET; - } else { - pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); - goto do_cmd_auto; - } - break; - - case RETBLEED_CMD_IBPB: - if (!boot_cpu_has(X86_FEATURE_IBPB)) { - pr_err("WARNING: CPU does not support IBPB.\n"); - goto do_cmd_auto; - } else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { - retbleed_mitigation = RETBLEED_MITIGATION_IBPB; - } else { - pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); - goto do_cmd_auto; - } - break; - - case RETBLEED_CMD_STUFF: - if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) && - spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { - pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); - goto do_cmd_auto; - } - retbleed_mitigation = RETBLEED_MITIGATION_STUFF; - - } else { - if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) - pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); - else - pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); - - goto do_cmd_auto; - } - break; - -do_cmd_auto: - case RETBLEED_CMD_AUTO: - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) - retbleed_mitigation = RETBLEED_MITIGATION_UNRET; - else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && - boot_cpu_has(X86_FEATURE_IBPB)) - retbleed_mitigation = RETBLEED_MITIGATION_IBPB; - } - - /* - * The Intel mitigation (IBRS or eIBRS) was already selected in - * spectre_v2_select_mitigation(). 'retbleed_mitigation' will - * be set accordingly below. - */ - - break; } switch (retbleed_mitigation) { + case RETBLEED_MITIGATION_UNRET: + if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) { + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n"); + } + break; + case RETBLEED_MITIGATION_IBPB: + if (!boot_cpu_has(X86_FEATURE_IBPB)) { + pr_err("WARNING: CPU does not support IBPB.\n"); + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { + pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } + break; + case RETBLEED_MITIGATION_STUFF: + if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { + pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n"); + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { + pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n"); + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } + break; + default: + break; + } + + if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO) + return; + + /* Intel mitigation selected in retbleed_update_mitigation() */ + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || + boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { + if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) + retbleed_mitigation = RETBLEED_MITIGATION_UNRET; + else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) && + boot_cpu_has(X86_FEATURE_IBPB)) + retbleed_mitigation = RETBLEED_MITIGATION_IBPB; + else + retbleed_mitigation = RETBLEED_MITIGATION_NONE; + } +} + +static void __init retbleed_update_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) + return; + + if (retbleed_mitigation == RETBLEED_MITIGATION_NONE) + goto out; + + /* + * retbleed=stuff is only allowed on Intel. If stuffing can't be used + * then a different mitigation will be selected below. + */ + if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) { + if (spectre_v2_enabled != SPECTRE_V2_RETPOLINE) { + pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); + retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } + } + /* + * Let IBRS trump all on Intel without affecting the effects of the + * retbleed= cmdline option except for call depth based stuffing + */ + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { + switch (spectre_v2_enabled) { + case SPECTRE_V2_IBRS: + retbleed_mitigation = RETBLEED_MITIGATION_IBRS; + break; + case SPECTRE_V2_EIBRS: + case SPECTRE_V2_EIBRS_RETPOLINE: + case SPECTRE_V2_EIBRS_LFENCE: + retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; + break; + default: + if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) + pr_err(RETBLEED_INTEL_MSG); + } + /* If nothing has set the mitigation yet, default to NONE. */ + if (retbleed_mitigation == RETBLEED_MITIGATION_AUTO) + retbleed_mitigation = RETBLEED_MITIGATION_NONE; + } +out: + pr_info("%s\n", retbleed_strings[retbleed_mitigation]); +} + + +static void __init retbleed_apply_mitigation(void) +{ + bool mitigate_smt = false; + + switch (retbleed_mitigation) { + case RETBLEED_MITIGATION_NONE: + return; + case RETBLEED_MITIGATION_UNRET: setup_force_cpu_cap(X86_FEATURE_RETHUNK); setup_force_cpu_cap(X86_FEATURE_UNRET); @@ -1277,28 +1300,6 @@ do_cmd_auto: if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && (retbleed_nosmt || cpu_mitigations_auto_nosmt())) cpu_smt_disable(false); - - /* - * Let IBRS trump all on Intel without affecting the effects of the - * retbleed= cmdline option except for call depth based stuffing - */ - if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { - switch (spectre_v2_enabled) { - case SPECTRE_V2_IBRS: - retbleed_mitigation = RETBLEED_MITIGATION_IBRS; - break; - case SPECTRE_V2_EIBRS: - case SPECTRE_V2_EIBRS_RETPOLINE: - case SPECTRE_V2_EIBRS_LFENCE: - retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; - break; - default: - if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) - pr_err(RETBLEED_INTEL_MSG); - } - } - - pr_info("%s\n", retbleed_strings[retbleed_mitigation]); } #undef pr_fmt @@ -1855,8 +1856,8 @@ static void __init spectre_v2_select_mitigation(void) if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && boot_cpu_has_bug(X86_BUG_RETBLEED) && - retbleed_cmd != RETBLEED_CMD_OFF && - retbleed_cmd != RETBLEED_CMD_STUFF && + retbleed_mitigation != RETBLEED_MITIGATION_NONE && + retbleed_mitigation != RETBLEED_MITIGATION_STUFF && boot_cpu_has(X86_FEATURE_IBRS) && boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { mode = SPECTRE_V2_IBRS; @@ -1964,7 +1965,7 @@ static void __init spectre_v2_select_mitigation(void) (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { - if (retbleed_cmd != RETBLEED_CMD_IBPB) { + if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) { setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); pr_info("Enabling Speculation Barrier for firmware calls\n"); } From ddfca9430a617780c8ad9691bf44660ae49e2a35 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:16 -0500 Subject: [PATCH 257/331] x86/bugs: Restructure spectre_v2_user mitigation Restructure spectre_v2_user to use select/update/apply functions to create consistent vulnerability handling. The IBPB/STIBP choices are first decided based on the spectre_v2_user command line but can be modified by the spectre_v2 command line option as well. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-12-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 174 +++++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 77 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 207a472d1a6e..dc75195760ca 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -60,6 +60,8 @@ static void __init retbleed_select_mitigation(void); static void __init retbleed_update_mitigation(void); static void __init retbleed_apply_mitigation(void); static void __init spectre_v2_user_select_mitigation(void); +static void __init spectre_v2_user_update_mitigation(void); +static void __init spectre_v2_user_apply_mitigation(void); static void __init ssb_select_mitigation(void); static void __init l1tf_select_mitigation(void); static void __init mds_select_mitigation(void); @@ -190,11 +192,6 @@ void __init cpu_select_mitigations(void) spectre_v1_select_mitigation(); spectre_v2_select_mitigation(); retbleed_select_mitigation(); - /* - * spectre_v2_user_select_mitigation() relies on the state set by - * retbleed_select_mitigation(); specifically the STIBP selection is - * forced for UNRET or IBPB. - */ spectre_v2_user_select_mitigation(); ssb_select_mitigation(); l1tf_select_mitigation(); @@ -217,6 +214,13 @@ void __init cpu_select_mitigations(void) * choices. */ retbleed_update_mitigation(); + + /* + * spectre_v2_user_update_mitigation() depends on + * retbleed_update_mitigation(), specifically the STIBP + * selection is forced for UNRET or IBPB. + */ + spectre_v2_user_update_mitigation(); mds_update_mitigation(); taa_update_mitigation(); mmio_update_mitigation(); @@ -224,6 +228,7 @@ void __init cpu_select_mitigations(void) spectre_v1_apply_mitigation(); retbleed_apply_mitigation(); + spectre_v2_user_apply_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); @@ -1379,6 +1384,8 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_IBRS, }; +static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = SPECTRE_V2_CMD_AUTO; + enum spectre_v2_user_cmd { SPECTRE_V2_USER_CMD_NONE, SPECTRE_V2_USER_CMD_AUTO, @@ -1417,31 +1424,18 @@ static void __init spec_v2_user_print_cond(const char *reason, bool secure) pr_info("spectre_v2_user=%s forced on command line.\n", reason); } -static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; - -static enum spectre_v2_user_cmd __init -spectre_v2_parse_user_cmdline(void) +static enum spectre_v2_user_cmd __init spectre_v2_parse_user_cmdline(void) { - enum spectre_v2_user_cmd mode; char arg[20]; int ret, i; - mode = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ? - SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE; - - switch (spectre_v2_cmd) { - case SPECTRE_V2_CMD_NONE: + if (cpu_mitigations_off() || !IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2)) return SPECTRE_V2_USER_CMD_NONE; - case SPECTRE_V2_CMD_FORCE: - return SPECTRE_V2_USER_CMD_FORCE; - default: - break; - } ret = cmdline_find_option(boot_command_line, "spectre_v2_user", arg, sizeof(arg)); if (ret < 0) - return mode; + return SPECTRE_V2_USER_CMD_AUTO; for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { if (match_option(arg, ret, v2_user_options[i].option)) { @@ -1452,7 +1446,7 @@ spectre_v2_parse_user_cmdline(void) } pr_err("Unknown user space protection option (%s). Switching to default\n", arg); - return mode; + return SPECTRE_V2_USER_CMD_AUTO; } static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) @@ -1460,60 +1454,72 @@ static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; } -static void __init -spectre_v2_user_select_mitigation(void) +static void __init spectre_v2_user_select_mitigation(void) { - enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; - enum spectre_v2_user_cmd cmd; - if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) return; - cmd = spectre_v2_parse_user_cmdline(); - switch (cmd) { + switch (spectre_v2_parse_user_cmdline()) { case SPECTRE_V2_USER_CMD_NONE: - goto set_mode; + return; case SPECTRE_V2_USER_CMD_FORCE: - mode = SPECTRE_V2_USER_STRICT; + spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; + spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; break; case SPECTRE_V2_USER_CMD_AUTO: case SPECTRE_V2_USER_CMD_PRCTL: + spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; + spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; + break; case SPECTRE_V2_USER_CMD_PRCTL_IBPB: - mode = SPECTRE_V2_USER_PRCTL; + spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; + spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; break; case SPECTRE_V2_USER_CMD_SECCOMP: - case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: if (IS_ENABLED(CONFIG_SECCOMP)) - mode = SPECTRE_V2_USER_SECCOMP; + spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP; else - mode = SPECTRE_V2_USER_PRCTL; + spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL; + spectre_v2_user_stibp = spectre_v2_user_ibpb; + break; + case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: + spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; + if (IS_ENABLED(CONFIG_SECCOMP)) + spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP; + else + spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL; break; } - /* Initialize Indirect Branch Prediction Barrier */ - if (boot_cpu_has(X86_FEATURE_IBPB)) { - static_branch_enable(&switch_vcpu_ibpb); + /* + * At this point, an STIBP mode other than "off" has been set. + * If STIBP support is not being forced, check if STIBP always-on + * is preferred. + */ + if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || + spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) && + boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) + spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; - spectre_v2_user_ibpb = mode; - switch (cmd) { - case SPECTRE_V2_USER_CMD_NONE: - break; - case SPECTRE_V2_USER_CMD_FORCE: - case SPECTRE_V2_USER_CMD_PRCTL_IBPB: - case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: - static_branch_enable(&switch_mm_always_ibpb); - spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; - break; - case SPECTRE_V2_USER_CMD_PRCTL: - case SPECTRE_V2_USER_CMD_AUTO: - case SPECTRE_V2_USER_CMD_SECCOMP: - static_branch_enable(&switch_mm_cond_ibpb); - break; - } + if (!boot_cpu_has(X86_FEATURE_IBPB)) + spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; - pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", - static_key_enabled(&switch_mm_always_ibpb) ? - "always-on" : "conditional"); + if (!boot_cpu_has(X86_FEATURE_STIBP)) + spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; +} + +static void __init spectre_v2_user_update_mitigation(void) +{ + if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) + return; + + /* The spectre_v2 cmd line can override spectre_v2_user options */ + if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) { + spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE; + spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; + } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) { + spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; + spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT; } /* @@ -1531,30 +1537,44 @@ spectre_v2_user_select_mitigation(void) if (!boot_cpu_has(X86_FEATURE_STIBP) || !cpu_smt_possible() || (spectre_v2_in_eibrs_mode(spectre_v2_enabled) && - !boot_cpu_has(X86_FEATURE_AUTOIBRS))) + !boot_cpu_has(X86_FEATURE_AUTOIBRS))) { + spectre_v2_user_stibp = SPECTRE_V2_USER_NONE; return; - - /* - * At this point, an STIBP mode other than "off" has been set. - * If STIBP support is not being forced, check if STIBP always-on - * is preferred. - */ - if (mode != SPECTRE_V2_USER_STRICT && - boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) - mode = SPECTRE_V2_USER_STRICT_PREFERRED; - - if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || - retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { - if (mode != SPECTRE_V2_USER_STRICT && - mode != SPECTRE_V2_USER_STRICT_PREFERRED) - pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); - mode = SPECTRE_V2_USER_STRICT_PREFERRED; } - spectre_v2_user_stibp = mode; + if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE && + (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || + retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) { + if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT && + spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED) + pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); + spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED; + } + pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]); +} -set_mode: - pr_info("%s\n", spectre_v2_user_strings[mode]); +static void __init spectre_v2_user_apply_mitigation(void) +{ + /* Initialize Indirect Branch Prediction Barrier */ + if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) { + static_branch_enable(&switch_vcpu_ibpb); + + switch (spectre_v2_user_ibpb) { + case SPECTRE_V2_USER_STRICT: + static_branch_enable(&switch_mm_always_ibpb); + break; + case SPECTRE_V2_USER_PRCTL: + case SPECTRE_V2_USER_SECCOMP: + static_branch_enable(&switch_mm_cond_ibpb); + break; + default: + break; + } + + pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", + static_key_enabled(&switch_mm_always_ibpb) ? + "always-on" : "conditional"); + } } static const char * const spectre_v2_strings[] = { From efe313827c98c81156dea9b004877db4ca728b1a Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:17 -0500 Subject: [PATCH 258/331] x86/bugs: Restructure BHI mitigation Restructure BHI mitigation to use select/update/apply functions to create consistent vulnerability handling. BHI mitigation was previously selected from within spectre_v2_select_mitigation() and now is selected from cpu_select_mitigation() like with all others. Define new AUTO mitigation for BHI. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-13-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index dc75195760ca..100a32044385 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -82,6 +82,9 @@ static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); static void __init gds_select_mitigation(void); static void __init gds_apply_mitigation(void); +static void __init bhi_select_mitigation(void); +static void __init bhi_update_mitigation(void); +static void __init bhi_apply_mitigation(void); /* The base value of the SPEC_CTRL MSR without task-specific bits set */ u64 x86_spec_ctrl_base; @@ -208,6 +211,7 @@ void __init cpu_select_mitigations(void) */ srso_select_mitigation(); gds_select_mitigation(); + bhi_select_mitigation(); /* * After mitigations are selected, some may need to update their @@ -225,6 +229,7 @@ void __init cpu_select_mitigations(void) taa_update_mitigation(); mmio_update_mitigation(); rfds_update_mitigation(); + bhi_update_mitigation(); spectre_v1_apply_mitigation(); retbleed_apply_mitigation(); @@ -235,6 +240,7 @@ void __init cpu_select_mitigations(void) rfds_apply_mitigation(); srbds_apply_mitigation(); gds_apply_mitigation(); + bhi_apply_mitigation(); } /* @@ -1794,12 +1800,13 @@ static bool __init spec_ctrl_bhi_dis(void) enum bhi_mitigations { BHI_MITIGATION_OFF, + BHI_MITIGATION_AUTO, BHI_MITIGATION_ON, BHI_MITIGATION_VMEXIT_ONLY, }; static enum bhi_mitigations bhi_mitigation __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_ON : BHI_MITIGATION_OFF; + IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_AUTO : BHI_MITIGATION_OFF; static int __init spectre_bhi_parse_cmdline(char *str) { @@ -1820,6 +1827,25 @@ static int __init spectre_bhi_parse_cmdline(char *str) early_param("spectre_bhi", spectre_bhi_parse_cmdline); static void __init bhi_select_mitigation(void) +{ + if (!boot_cpu_has(X86_BUG_BHI) || cpu_mitigations_off()) + bhi_mitigation = BHI_MITIGATION_OFF; + + if (bhi_mitigation == BHI_MITIGATION_AUTO) + bhi_mitigation = BHI_MITIGATION_ON; +} + +static void __init bhi_update_mitigation(void) +{ + if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) + bhi_mitigation = BHI_MITIGATION_OFF; + + if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && + spectre_v2_cmd == SPECTRE_V2_CMD_AUTO) + bhi_mitigation = BHI_MITIGATION_OFF; +} + +static void __init bhi_apply_mitigation(void) { if (bhi_mitigation == BHI_MITIGATION_OFF) return; @@ -1961,9 +1987,6 @@ static void __init spectre_v2_select_mitigation(void) mode == SPECTRE_V2_RETPOLINE) spec_ctrl_disable_kernel_rrsba(); - if (boot_cpu_has(X86_BUG_BHI)) - bhi_select_mitigation(); - spectre_v2_enabled = mode; pr_info("%s\n", spectre_v2_strings[mode]); From 480e803dacf8be92b1104ca65f2be4cb0e191375 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:18 -0500 Subject: [PATCH 259/331] x86/bugs: Restructure spectre_v2 mitigation Restructure spectre_v2 to use select/update/apply functions to create consistent vulnerability handling. The spectre_v2 mitigation may be updated based on the selected retbleed mitigation. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-14-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 133 ++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 100a32044385..93d07438eea7 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -56,6 +56,8 @@ static void __init spectre_v1_select_mitigation(void); static void __init spectre_v1_apply_mitigation(void); static void __init spectre_v2_select_mitigation(void); +static void __init spectre_v2_update_mitigation(void); +static void __init spectre_v2_apply_mitigation(void); static void __init retbleed_select_mitigation(void); static void __init retbleed_update_mitigation(void); static void __init retbleed_apply_mitigation(void); @@ -217,6 +219,12 @@ void __init cpu_select_mitigations(void) * After mitigations are selected, some may need to update their * choices. */ + spectre_v2_update_mitigation(); + /* + * retbleed_update_mitigation() relies on the state set by + * spectre_v2_update_mitigation(); specifically it wants to know about + * spectre_v2=ibrs. + */ retbleed_update_mitigation(); /* @@ -232,6 +240,7 @@ void __init cpu_select_mitigations(void) bhi_update_mitigation(); spectre_v1_apply_mitigation(); + spectre_v2_apply_mitigation(); retbleed_apply_mitigation(); spectre_v2_user_apply_mitigation(); mds_apply_mitigation(); @@ -1878,75 +1887,80 @@ static void __init bhi_apply_mitigation(void) static void __init spectre_v2_select_mitigation(void) { - enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); - enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; + spectre_v2_cmd = spectre_v2_parse_cmdline(); - /* - * If the CPU is not affected and the command line mode is NONE or AUTO - * then nothing to do. - */ if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && - (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) + (spectre_v2_cmd == SPECTRE_V2_CMD_NONE || spectre_v2_cmd == SPECTRE_V2_CMD_AUTO)) return; - switch (cmd) { + switch (spectre_v2_cmd) { case SPECTRE_V2_CMD_NONE: return; case SPECTRE_V2_CMD_FORCE: case SPECTRE_V2_CMD_AUTO: if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { - mode = SPECTRE_V2_EIBRS; + spectre_v2_enabled = SPECTRE_V2_EIBRS; break; } + spectre_v2_enabled = spectre_v2_select_retpoline(); + break; + + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: + pr_err(SPECTRE_V2_LFENCE_MSG); + spectre_v2_enabled = SPECTRE_V2_LFENCE; + break; + + case SPECTRE_V2_CMD_RETPOLINE_GENERIC: + spectre_v2_enabled = SPECTRE_V2_RETPOLINE; + break; + + case SPECTRE_V2_CMD_RETPOLINE: + spectre_v2_enabled = spectre_v2_select_retpoline(); + break; + + case SPECTRE_V2_CMD_IBRS: + spectre_v2_enabled = SPECTRE_V2_IBRS; + break; + + case SPECTRE_V2_CMD_EIBRS: + spectre_v2_enabled = SPECTRE_V2_EIBRS; + break; + + case SPECTRE_V2_CMD_EIBRS_LFENCE: + spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE; + break; + + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: + spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE; + break; + } +} + +static void __init spectre_v2_update_mitigation(void) +{ + if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO) { if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && boot_cpu_has_bug(X86_BUG_RETBLEED) && retbleed_mitigation != RETBLEED_MITIGATION_NONE && retbleed_mitigation != RETBLEED_MITIGATION_STUFF && boot_cpu_has(X86_FEATURE_IBRS) && boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { - mode = SPECTRE_V2_IBRS; - break; + spectre_v2_enabled = SPECTRE_V2_IBRS; } - - mode = spectre_v2_select_retpoline(); - break; - - case SPECTRE_V2_CMD_RETPOLINE_LFENCE: - pr_err(SPECTRE_V2_LFENCE_MSG); - mode = SPECTRE_V2_LFENCE; - break; - - case SPECTRE_V2_CMD_RETPOLINE_GENERIC: - mode = SPECTRE_V2_RETPOLINE; - break; - - case SPECTRE_V2_CMD_RETPOLINE: - mode = spectre_v2_select_retpoline(); - break; - - case SPECTRE_V2_CMD_IBRS: - mode = SPECTRE_V2_IBRS; - break; - - case SPECTRE_V2_CMD_EIBRS: - mode = SPECTRE_V2_EIBRS; - break; - - case SPECTRE_V2_CMD_EIBRS_LFENCE: - mode = SPECTRE_V2_EIBRS_LFENCE; - break; - - case SPECTRE_V2_CMD_EIBRS_RETPOLINE: - mode = SPECTRE_V2_EIBRS_RETPOLINE; - break; } - if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) + if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && !cpu_mitigations_off()) + pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]); +} + +static void __init spectre_v2_apply_mitigation(void) +{ + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); - if (spectre_v2_in_ibrs_mode(mode)) { + if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); } else { @@ -1955,8 +1969,10 @@ static void __init spectre_v2_select_mitigation(void) } } - switch (mode) { + switch (spectre_v2_enabled) { case SPECTRE_V2_NONE: + return; + case SPECTRE_V2_EIBRS: break; @@ -1982,15 +1998,12 @@ static void __init spectre_v2_select_mitigation(void) * JMPs gets protection against BHI and Intramode-BTI, but RET * prediction from a non-RSB predictor is still a risk. */ - if (mode == SPECTRE_V2_EIBRS_LFENCE || - mode == SPECTRE_V2_EIBRS_RETPOLINE || - mode == SPECTRE_V2_RETPOLINE) + if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE || + spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE || + spectre_v2_enabled == SPECTRE_V2_RETPOLINE) spec_ctrl_disable_kernel_rrsba(); - spectre_v2_enabled = mode; - pr_info("%s\n", spectre_v2_strings[mode]); - - spectre_v2_select_rsb_mitigation(mode); + spectre_v2_select_rsb_mitigation(spectre_v2_enabled); /* * Retpoline protects the kernel, but doesn't protect firmware. IBRS @@ -1998,10 +2011,10 @@ static void __init spectre_v2_select_mitigation(void) * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't * otherwise enabled. * - * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because - * the user might select retpoline on the kernel command line and if - * the CPU supports Enhanced IBRS, kernel might un-intentionally not - * enable IBRS around firmware calls. + * Use "spectre_v2_enabled" to check Enhanced IBRS instead of + * boot_cpu_has(), because the user might select retpoline on the kernel + * command line and if the CPU supports Enhanced IBRS, kernel might + * un-intentionally not enable IBRS around firmware calls. */ if (boot_cpu_has_bug(X86_BUG_RETBLEED) && boot_cpu_has(X86_FEATURE_IBPB) && @@ -2013,13 +2026,11 @@ static void __init spectre_v2_select_mitigation(void) pr_info("Enabling Speculation Barrier for firmware calls\n"); } - } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { + } else if (boot_cpu_has(X86_FEATURE_IBRS) && + !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) { setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } - - /* Set up IBPB and STIBP depending on the general spectre V2 command */ - spectre_v2_cmd = cmd; } static void update_stibp_msr(void * __unused) From 5ece59a2fca6e1467558467a05cf742b7e52d1b7 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:19 -0500 Subject: [PATCH 260/331] x86/bugs: Restructure SSB mitigation Restructure SSB to use select/apply functions to create consistent vulnerability handling. Remove __ssb_select_mitigation() and split the functionality between the select/apply functions. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-15-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 93d07438eea7..fbb4f1378de1 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -65,6 +65,7 @@ static void __init spectre_v2_user_select_mitigation(void); static void __init spectre_v2_user_update_mitigation(void); static void __init spectre_v2_user_apply_mitigation(void); static void __init ssb_select_mitigation(void); +static void __init ssb_apply_mitigation(void); static void __init l1tf_select_mitigation(void); static void __init mds_select_mitigation(void); static void __init mds_update_mitigation(void); @@ -243,6 +244,7 @@ void __init cpu_select_mitigations(void) spectre_v2_apply_mitigation(); retbleed_apply_mitigation(); spectre_v2_user_apply_mitigation(); + ssb_apply_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); @@ -2219,19 +2221,18 @@ static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) return cmd; } -static enum ssb_mitigation __init __ssb_select_mitigation(void) +static void __init ssb_select_mitigation(void) { - enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; enum ssb_mitigation_cmd cmd; if (!boot_cpu_has(X86_FEATURE_SSBD)) - return mode; + goto out; cmd = ssb_parse_cmdline(); if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && (cmd == SPEC_STORE_BYPASS_CMD_NONE || cmd == SPEC_STORE_BYPASS_CMD_AUTO)) - return mode; + return; switch (cmd) { case SPEC_STORE_BYPASS_CMD_SECCOMP: @@ -2240,28 +2241,35 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) * enabled. */ if (IS_ENABLED(CONFIG_SECCOMP)) - mode = SPEC_STORE_BYPASS_SECCOMP; + ssb_mode = SPEC_STORE_BYPASS_SECCOMP; else - mode = SPEC_STORE_BYPASS_PRCTL; + ssb_mode = SPEC_STORE_BYPASS_PRCTL; break; case SPEC_STORE_BYPASS_CMD_ON: - mode = SPEC_STORE_BYPASS_DISABLE; + ssb_mode = SPEC_STORE_BYPASS_DISABLE; break; case SPEC_STORE_BYPASS_CMD_AUTO: case SPEC_STORE_BYPASS_CMD_PRCTL: - mode = SPEC_STORE_BYPASS_PRCTL; + ssb_mode = SPEC_STORE_BYPASS_PRCTL; break; case SPEC_STORE_BYPASS_CMD_NONE: break; } +out: + if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) + pr_info("%s\n", ssb_strings[ssb_mode]); +} + +static void __init ssb_apply_mitigation(void) +{ /* * We have three CPU feature flags that are in play here: * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation */ - if (mode == SPEC_STORE_BYPASS_DISABLE) { + if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) { setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); /* * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may @@ -2275,16 +2283,6 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) update_spec_ctrl(x86_spec_ctrl_base); } } - - return mode; -} - -static void ssb_select_mitigation(void) -{ - ssb_mode = __ssb_select_mitigation(); - - if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) - pr_info("%s\n", ssb_strings[ssb_mode]); } #undef pr_fmt From d43ba2dc8eeeca21811fd9b30e3bd15bb35caaec Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:20 -0500 Subject: [PATCH 261/331] x86/bugs: Restructure L1TF mitigation Restructure L1TF to use select/apply functions to create consistent vulnerability handling. Define new AUTO mitigation for L1TF. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-16-david.kaplan@amd.com --- arch/x86/include/asm/processor.h | 1 + arch/x86/kernel/cpu/bugs.c | 25 +++++++++++++++++++------ arch/x86/kvm/vmx/vmx.c | 2 ++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 5d2f7e5aff26..0973bed22172 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -734,6 +734,7 @@ void store_cpu_caps(struct cpuinfo_x86 *info); enum l1tf_mitigations { L1TF_MITIGATION_OFF, + L1TF_MITIGATION_AUTO, L1TF_MITIGATION_FLUSH_NOWARN, L1TF_MITIGATION_FLUSH, L1TF_MITIGATION_FLUSH_NOSMT, diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index fbb4f1378de1..25d84e2a4f2b 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -67,6 +67,7 @@ static void __init spectre_v2_user_apply_mitigation(void); static void __init ssb_select_mitigation(void); static void __init ssb_apply_mitigation(void); static void __init l1tf_select_mitigation(void); +static void __init l1tf_apply_mitigation(void); static void __init mds_select_mitigation(void); static void __init mds_update_mitigation(void); static void __init mds_apply_mitigation(void); @@ -245,6 +246,7 @@ void __init cpu_select_mitigations(void) retbleed_apply_mitigation(); spectre_v2_user_apply_mitigation(); ssb_apply_mitigation(); + l1tf_apply_mitigation(); mds_apply_mitigation(); taa_apply_mitigation(); mmio_apply_mitigation(); @@ -2538,7 +2540,7 @@ EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); /* Default mitigation for L1TF-affected CPUs */ enum l1tf_mitigations l1tf_mitigation __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_FLUSH : L1TF_MITIGATION_OFF; + IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_AUTO : L1TF_MITIGATION_OFF; #if IS_ENABLED(CONFIG_KVM_INTEL) EXPORT_SYMBOL_GPL(l1tf_mitigation); #endif @@ -2585,23 +2587,34 @@ static void override_cache_bits(struct cpuinfo_x86 *c) } static void __init l1tf_select_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_L1TF) || cpu_mitigations_off()) { + l1tf_mitigation = L1TF_MITIGATION_OFF; + return; + } + + if (l1tf_mitigation == L1TF_MITIGATION_AUTO) { + if (cpu_mitigations_auto_nosmt()) + l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; + else + l1tf_mitigation = L1TF_MITIGATION_FLUSH; + } +} + +static void __init l1tf_apply_mitigation(void) { u64 half_pa; if (!boot_cpu_has_bug(X86_BUG_L1TF)) return; - if (cpu_mitigations_off()) - l1tf_mitigation = L1TF_MITIGATION_OFF; - else if (cpu_mitigations_auto_nosmt()) - l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; - override_cache_bits(&boot_cpu_data); switch (l1tf_mitigation) { case L1TF_MITIGATION_OFF: case L1TF_MITIGATION_FLUSH_NOWARN: case L1TF_MITIGATION_FLUSH: + case L1TF_MITIGATION_AUTO: break; case L1TF_MITIGATION_FLUSH_NOSMT: case L1TF_MITIGATION_FULL: diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a1754f7ba889..0aba4719ae0a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -273,6 +273,7 @@ static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) case L1TF_MITIGATION_OFF: l1tf = VMENTER_L1D_FLUSH_NEVER; break; + case L1TF_MITIGATION_AUTO: case L1TF_MITIGATION_FLUSH_NOWARN: case L1TF_MITIGATION_FLUSH: case L1TF_MITIGATION_FLUSH_NOSMT: @@ -7704,6 +7705,7 @@ int vmx_vm_init(struct kvm *kvm) case L1TF_MITIGATION_FLUSH_NOWARN: /* 'I explicitly don't care' is set */ break; + case L1TF_MITIGATION_AUTO: case L1TF_MITIGATION_FLUSH: case L1TF_MITIGATION_FLUSH_NOSMT: case L1TF_MITIGATION_FULL: From 1f4bb068b498a544ae913764a797449463ef620c Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 18 Apr 2025 11:17:21 -0500 Subject: [PATCH 262/331] x86/bugs: Restructure SRSO mitigation Restructure SRSO to use select/update/apply functions to create consistent vulnerability handling. Like with retbleed, the command line options directly select mitigations which can later be modified. While at it, remove a comment which doesn't apply anymore due to the changed mitigation detection flow. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/20250418161721.1855190-17-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 217 +++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 115 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 25d84e2a4f2b..a4f3b1d03406 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -84,6 +84,8 @@ static void __init srbds_select_mitigation(void); static void __init srbds_apply_mitigation(void); static void __init l1d_flush_select_mitigation(void); static void __init srso_select_mitigation(void); +static void __init srso_update_mitigation(void); +static void __init srso_apply_mitigation(void); static void __init gds_select_mitigation(void); static void __init gds_apply_mitigation(void); static void __init bhi_select_mitigation(void); @@ -208,11 +210,6 @@ void __init cpu_select_mitigations(void) rfds_select_mitigation(); srbds_select_mitigation(); l1d_flush_select_mitigation(); - - /* - * srso_select_mitigation() depends and must run after - * retbleed_select_mitigation(). - */ srso_select_mitigation(); gds_select_mitigation(); bhi_select_mitigation(); @@ -240,6 +237,8 @@ void __init cpu_select_mitigations(void) mmio_update_mitigation(); rfds_update_mitigation(); bhi_update_mitigation(); + /* srso_update_mitigation() depends on retbleed_update_mitigation(). */ + srso_update_mitigation(); spectre_v1_apply_mitigation(); spectre_v2_apply_mitigation(); @@ -252,6 +251,7 @@ void __init cpu_select_mitigations(void) mmio_apply_mitigation(); rfds_apply_mitigation(); srbds_apply_mitigation(); + srso_apply_mitigation(); gds_apply_mitigation(); bhi_apply_mitigation(); } @@ -2674,6 +2674,7 @@ early_param("l1tf", l1tf_cmdline); enum srso_mitigation { SRSO_MITIGATION_NONE, + SRSO_MITIGATION_AUTO, SRSO_MITIGATION_UCODE_NEEDED, SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, SRSO_MITIGATION_MICROCODE, @@ -2683,14 +2684,6 @@ enum srso_mitigation { SRSO_MITIGATION_BP_SPEC_REDUCE, }; -enum srso_mitigation_cmd { - SRSO_CMD_OFF, - SRSO_CMD_MICROCODE, - SRSO_CMD_SAFE_RET, - SRSO_CMD_IBPB, - SRSO_CMD_IBPB_ON_VMEXIT, -}; - static const char * const srso_strings[] = { [SRSO_MITIGATION_NONE] = "Vulnerable", [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", @@ -2702,8 +2695,7 @@ static const char * const srso_strings[] = { [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" }; -static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE; -static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET; +static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; static int __init srso_parse_cmdline(char *str) { @@ -2711,15 +2703,15 @@ static int __init srso_parse_cmdline(char *str) return -EINVAL; if (!strcmp(str, "off")) - srso_cmd = SRSO_CMD_OFF; + srso_mitigation = SRSO_MITIGATION_NONE; else if (!strcmp(str, "microcode")) - srso_cmd = SRSO_CMD_MICROCODE; + srso_mitigation = SRSO_MITIGATION_MICROCODE; else if (!strcmp(str, "safe-ret")) - srso_cmd = SRSO_CMD_SAFE_RET; + srso_mitigation = SRSO_MITIGATION_SAFE_RET; else if (!strcmp(str, "ibpb")) - srso_cmd = SRSO_CMD_IBPB; + srso_mitigation = SRSO_MITIGATION_IBPB; else if (!strcmp(str, "ibpb-vmexit")) - srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT; + srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; else pr_err("Ignoring unknown SRSO option (%s).", str); @@ -2731,132 +2723,83 @@ early_param("spec_rstack_overflow", srso_parse_cmdline); static void __init srso_select_mitigation(void) { - bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); + bool has_microcode; - if (!boot_cpu_has_bug(X86_BUG_SRSO) || - cpu_mitigations_off() || - srso_cmd == SRSO_CMD_OFF) { - if (boot_cpu_has(X86_FEATURE_SBPB)) - x86_pred_cmd = PRED_CMD_SBPB; - goto out; - } + if (!boot_cpu_has_bug(X86_BUG_SRSO) || cpu_mitigations_off()) + srso_mitigation = SRSO_MITIGATION_NONE; + if (srso_mitigation == SRSO_MITIGATION_NONE) + return; + + if (srso_mitigation == SRSO_MITIGATION_AUTO) + srso_mitigation = SRSO_MITIGATION_SAFE_RET; + + has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); if (has_microcode) { /* * Zen1/2 with SMT off aren't vulnerable after the right * IBPB microcode has been applied. - * - * Zen1/2 don't have SBPB, no need to try to enable it here. */ if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { setup_force_cpu_cap(X86_FEATURE_SRSO_NO); - goto out; - } - - if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { - srso_mitigation = SRSO_MITIGATION_IBPB; - goto out; + srso_mitigation = SRSO_MITIGATION_NONE; + return; } } else { pr_warn("IBPB-extending microcode not applied!\n"); pr_warn(SRSO_NOTICE); - - /* may be overwritten by SRSO_CMD_SAFE_RET below */ - srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; } - switch (srso_cmd) { - case SRSO_CMD_MICROCODE: - if (has_microcode) { - srso_mitigation = SRSO_MITIGATION_MICROCODE; - pr_warn(SRSO_NOTICE); - } - break; - - case SRSO_CMD_SAFE_RET: - if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) + switch (srso_mitigation) { + case SRSO_MITIGATION_SAFE_RET: + if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO)) { + srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; goto ibpb_on_vmexit; + } - if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) { - /* - * Enable the return thunk for generated code - * like ftrace, static_call, etc. - */ - setup_force_cpu_cap(X86_FEATURE_RETHUNK); - setup_force_cpu_cap(X86_FEATURE_UNRET); - - if (boot_cpu_data.x86 == 0x19) { - setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); - x86_return_thunk = srso_alias_return_thunk; - } else { - setup_force_cpu_cap(X86_FEATURE_SRSO); - x86_return_thunk = srso_return_thunk; - } - if (has_microcode) - srso_mitigation = SRSO_MITIGATION_SAFE_RET; - else - srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; - } else { + if (!IS_ENABLED(CONFIG_MITIGATION_SRSO)) { pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n"); + srso_mitigation = SRSO_MITIGATION_NONE; } + + if (!has_microcode) + srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; break; - - case SRSO_CMD_IBPB: - if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { - if (has_microcode) { - setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); - setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); - srso_mitigation = SRSO_MITIGATION_IBPB; - - /* - * IBPB on entry already obviates the need for - * software-based untraining so clear those in case some - * other mitigation like Retbleed has selected them. - */ - setup_clear_cpu_cap(X86_FEATURE_UNRET); - setup_clear_cpu_cap(X86_FEATURE_RETHUNK); - - /* - * There is no need for RSB filling: write_ibpb() ensures - * all predictions, including the RSB, are invalidated, - * regardless of IBPB implementation. - */ - setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); - } - } else { - pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); - } - break; - ibpb_on_vmexit: - case SRSO_CMD_IBPB_ON_VMEXIT: + case SRSO_MITIGATION_IBPB_ON_VMEXIT: if (boot_cpu_has(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) { pr_notice("Reducing speculation to address VM/HV SRSO attack vector.\n"); srso_mitigation = SRSO_MITIGATION_BP_SPEC_REDUCE; break; } - - if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { - if (has_microcode) { - setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); - srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; - - /* - * There is no need for RSB filling: write_ibpb() ensures - * all predictions, including the RSB, are invalidated, - * regardless of IBPB implementation. - */ - setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); - } - } else { + fallthrough; + case SRSO_MITIGATION_IBPB: + if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) { pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n"); + srso_mitigation = SRSO_MITIGATION_NONE; } + + if (!has_microcode) + srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; break; default: break; } +} -out: +static void __init srso_update_mitigation(void) +{ + /* If retbleed is using IBPB, that works for SRSO as well */ + if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB && + boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) + srso_mitigation = SRSO_MITIGATION_IBPB; + + if (boot_cpu_has_bug(X86_BUG_SRSO) && !cpu_mitigations_off()) + pr_info("%s\n", srso_strings[srso_mitigation]); +} + +static void __init srso_apply_mitigation(void) +{ /* * Clear the feature flag if this mitigation is not selected as that * feature flag controls the BpSpecReduce MSR bit toggling in KVM. @@ -2864,8 +2807,52 @@ out: if (srso_mitigation != SRSO_MITIGATION_BP_SPEC_REDUCE) setup_clear_cpu_cap(X86_FEATURE_SRSO_BP_SPEC_REDUCE); - if (srso_mitigation != SRSO_MITIGATION_NONE) - pr_info("%s\n", srso_strings[srso_mitigation]); + if (srso_mitigation == SRSO_MITIGATION_NONE) { + if (boot_cpu_has(X86_FEATURE_SBPB)) + x86_pred_cmd = PRED_CMD_SBPB; + return; + } + + switch (srso_mitigation) { + case SRSO_MITIGATION_SAFE_RET: + case SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED: + /* + * Enable the return thunk for generated code + * like ftrace, static_call, etc. + */ + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_UNRET); + + if (boot_cpu_data.x86 == 0x19) { + setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); + x86_return_thunk = srso_alias_return_thunk; + } else { + setup_force_cpu_cap(X86_FEATURE_SRSO); + x86_return_thunk = srso_return_thunk; + } + break; + case SRSO_MITIGATION_IBPB: + setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); + /* + * IBPB on entry already obviates the need for + * software-based untraining so clear those in case some + * other mitigation like Retbleed has selected them. + */ + setup_clear_cpu_cap(X86_FEATURE_UNRET); + setup_clear_cpu_cap(X86_FEATURE_RETHUNK); + fallthrough; + case SRSO_MITIGATION_IBPB_ON_VMEXIT: + setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); + /* + * There is no need for RSB filling: entry_ibpb() ensures + * all predictions, including the RSB, are invalidated, + * regardless of IBPB implementation. + */ + setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT); + break; + default: + break; + } } #undef pr_fmt From b43dc4ab097859c24e2a6993119c927cffc856aa Mon Sep 17 00:00:00 2001 From: Annie Li Date: Wed, 30 Apr 2025 05:34:24 +0000 Subject: [PATCH 263/331] x86/microcode/AMD: Do not return error when microcode update is not necessary After 6f059e634dcd("x86/microcode: Clarify the late load logic"), if the load is up-to-date, the AMD side returns UCODE_OK which leads to load_late_locked() returning -EBADFD. Handle UCODE_OK in the switch case to avoid this error. [ bp: Massage commit message. ] Fixes: 6f059e634dcd ("x86/microcode: Clarify the late load logic") Signed-off-by: Annie Li Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250430053424.77438-1-jiayanli@google.com --- arch/x86/kernel/cpu/microcode/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index b3658d11e7b6..2309321cf6a1 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -686,6 +686,8 @@ static int load_late_locked(void) return load_late_stop_cpus(true); case UCODE_NFOUND: return -ENOENT; + case UCODE_OK: + return 0; default: return -EBADFD; } From cafb22242bfb7fad10fde6f9b99853fc924e691a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 22 Apr 2025 18:48:26 -0500 Subject: [PATCH 264/331] Documentation: Add AMD Zen debugging document Debugging issues on AMD hardware can be challenging for users without proper documentation and tools. Introduce a document that includes techniques for debugging s2idle issues. It will be expanded for debugging other issues later. Signed-off-by: Mario Limonciello Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/20250422234830.2840784-2-superm1@kernel.org --- Documentation/arch/x86/amd-debugging.rst | 320 +++++++++++++++++++++++ Documentation/arch/x86/index.rst | 1 + Documentation/arch/x86/resume.svg | 4 + Documentation/arch/x86/suspend.svg | 4 + 4 files changed, 329 insertions(+) create mode 100644 Documentation/arch/x86/amd-debugging.rst create mode 100644 Documentation/arch/x86/resume.svg create mode 100644 Documentation/arch/x86/suspend.svg diff --git a/Documentation/arch/x86/amd-debugging.rst b/Documentation/arch/x86/amd-debugging.rst new file mode 100644 index 000000000000..d3290f2e4be7 --- /dev/null +++ b/Documentation/arch/x86/amd-debugging.rst @@ -0,0 +1,320 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Debugging AMD Zen systems ++++++++++++++++++++++++++ + +Introduction +============ + +This document describes techniques that are useful for debugging issues with +AMD Zen systems. It is intended for use by developers and technical users +to help identify and resolve issues. + +S3 vs s2idle +============ + +On AMD systems, it's not possible to simultaneously support suspend-to-RAM (S3) +and suspend-to-idle (s2idle). To confirm which mode your system supports you +can look at ``cat /sys/power/mem_sleep``. If it shows ``s2idle [deep]`` then +*S3* is supported. If it shows ``[s2idle]`` then *s2idle* is +supported. + +On systems that support *S3*, the firmware will be utilized to put all hardware into +the appropriate low power state. + +On systems that support *s2idle*, the kernel will be responsible for transitioning devices +into the appropriate low power state. When all devices are in the appropriate low +power state, the hardware will transition into a hardware sleep state. + +After a suspend cycle you can tell how much time was spent in a hardware sleep +state by looking at ``cat /sys/power/suspend_stats/last_hw_sleep``. + +This flowchart explains how the AMD s2idle suspend flow works. + +.. kernel-figure:: suspend.svg + +This flowchart explains how the amd s2idle resume flow works. + +.. kernel-figure:: resume.svg + +s2idle debugging tool +===================== + +As there are a lot of places that problems can occur, a debugging tool has been +created at +`amd-debug-tools `_ +that can help test for common problems and offer suggestions. + +If you have an s2idle issue, it's best to start with this and follow instructions +from its findings. If you continue to have an issue, raise a bug with the +report generated from this script to +`drm/amd gitlab `_. + +Spurious s2idle wakeups from an IRQ +=================================== +Spurious wakeups will generally have an IRQ set to ``/sys/power/pm_wakeup_irq``. +This can be matched to ``/proc/interrupts`` to determine what device woke the system. + +If this isn't enough to debug the problem, then the following sysfs files +can be set to add more verbosity to the wakeup process: :: + + # echo 1 | sudo tee /sys/power/pm_debug_messages + # echo 1 | sudo tee /sys/power/pm_print_times + +After making those changes, the kernel will display messages that can +be traced back to kernel s2idle loop code as well as display any active +GPIO sources while waking up. + +If the wakeup is caused by the ACPI SCI, additional ACPI debugging may be +needed. These commands can enable additional trace data: :: + + # echo enable | sudo tee /sys/module/acpi/parameters/trace_state + # echo 1 | sudo tee /sys/module/acpi/parameters/aml_debug_output + # echo 0x0800000f | sudo tee /sys/module/acpi/parameters/debug_level + # echo 0xffff0000 | sudo tee /sys/module/acpi/parameters/debug_layer + +Spurious s2idle wakeups from a GPIO +=================================== + +If a GPIO is active when waking up the system ideally you would look at the +schematic to determine what device it is associated with. If the schematic +is not available, another tactic is to look at the ACPI _EVT() entry +to determine what device is notified when that GPIO is active. + +For a hypothetical example, say that GPIO 59 woke up the system. You can +look at the SSDT to determine what device is notified when GPIO 59 is active. + +First convert the GPIO number into hex. :: + + $ python3 -c "print(hex(59))" + 0x3b + +Next determine which ACPI table has the ``_EVT`` entry. For example: :: + + $ sudo grep EVT /sys/firmware/acpi/tables/SSDT* + grep: /sys/firmware/acpi/tables/SSDT27: binary file matches + +Decode this table:: + + $ sudo cp /sys/firmware/acpi/tables/SSDT27 . + $ sudo iasl -d SSDT27 + +Then look at the table and find the matching entry for GPIO 0x3b. :: + + Case (0x3B) + { + M000 (0x393B) + M460 (" Notify (\\_SB.PCI0.GP17.XHC1, 0x02)\n", Zero, Zero, Zero, Zero, Zero, Zero) + Notify (\_SB.PCI0.GP17.XHC1, 0x02) // Device Wake + } + +You can see in this case that the device ``\_SB.PCI0.GP17.XHC1`` is notified +when GPIO 59 is active. It's obvious this is an XHCI controller, but to go a +step further you can figure out which XHCI controller it is by matching it to +ACPI.:: + + $ grep "PCI0.GP17.XHC1" /sys/bus/acpi/devices/*/path + /sys/bus/acpi/devices/device:2d/path:\_SB_.PCI0.GP17.XHC1 + /sys/bus/acpi/devices/device:2e/path:\_SB_.PCI0.GP17.XHC1.RHUB + /sys/bus/acpi/devices/device:2f/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1 + /sys/bus/acpi/devices/device:30/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1.CAM0 + /sys/bus/acpi/devices/device:31/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1.CAM1 + /sys/bus/acpi/devices/device:32/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT2 + /sys/bus/acpi/devices/LNXPOWER:0d/path:\_SB_.PCI0.GP17.XHC1.PWRS + +Here you can see it matches to ``device:2d``. Look at the ``physical_node`` +to determine what PCI device that actually is. :: + + $ ls -l /sys/bus/acpi/devices/device:2d/physical_node + lrwxrwxrwx 1 root root 0 Feb 12 13:22 /sys/bus/acpi/devices/device:2d/physical_node -> ../../../../../pci0000:00/0000:00:08.1/0000:c2:00.4 + +So there you have it: the PCI device associated with this GPIO wakeup was ``0000:c2:00.4``. + +The ``amd_s2idle.py`` script will capture most of these artifacts for you. + +s2idle PM debug messages +======================== +During the s2idle flow on AMD systems, the ACPI LPS0 driver is responsible +to check all uPEP constraints. Failing uPEP constraints does not prevent +s0i3 entry. This means that if some constraints are not met, it is possible +the kernel may attempt to enter s2idle even if there are some known issues. + +To activate PM debugging, either specify ``pm_debug_messagess`` kernel +command-line option at boot or write to ``/sys/power/pm_debug_messages``. +Unmet constraints will be displayed in the kernel log and can be +viewed by logging tools that process kernel ring buffer like ``dmesg`` or +``journalctl``." + +If the system freezes on entry/exit before these messages are flushed, a +useful debugging tactic is to unbind the ``amd_pmc`` driver to prevent +notification to the platform to start s0i3 entry. This will stop the +system from freezing on entry or exit and let you view all the failed +constraints. :: + + cd /sys/bus/platform/drivers/amd_pmc + ls | grep AMD | sudo tee unbind + +After doing this, run the suspend cycle and look specifically for errors around: :: + + ACPI: LPI: Constraint not met; min power state:%s current power state:%s + +Historical examples of s2idle issues +==================================== +To help understand the types of issues that can occur and how to debug them, +here are some historical examples of s2idle issues that have been resolved. + +Core offlining +-------------- +An end user had reported that taking a core offline would prevent the system +from properly entering s0i3. This was debugged using internal AMD tools +to capture and display a stream of metrics from the hardware showing what changed +when a core was offlined. It was determined that the hardware didn't get +notification the offline cores were in the deepest state, and so it prevented +CPU from going into the deepest state. The issue was debugged to a missing +command to put cores into C3 upon offline. + +`commit d6b88ce2eb9d2 ("ACPI: processor idle: Allow playing dead in C3 state") `_ + +Corruption after resume +----------------------- +A big problem that occurred with Rembrandt was that there was graphical +corruption after resume. This happened because of a misalignment of PSP +and driver responsibility. The PSP will save and restore DMCUB, but the +driver assumed it needed to reset DMCUB on resume. +This actually was a misalignment for earlier silicon as well, but was not +observed. + +`commit 79d6b9351f086 ("drm/amd/display: Don't reinitialize DMCUB on s0ix resume") `_ + +Back to Back suspends fail +-------------------------- +When using a wakeup source that triggers the IRQ to wakeup, a bug in the +pinctrl-amd driver may capture the wrong state of the IRQ and prevent the +system going back to sleep properly. + +`commit b8c824a869f22 ("pinctrl: amd: Don't save/restore interrupt status and wake status bits") `_ + +Spurious timer based wakeup after 5 minutes +------------------------------------------- +The HPET was being used to program the wakeup source for the system, however +this was causing a spurious wakeup after 5 minutes. The correct alarm to use +was the ACPI alarm. + +`commit 3d762e21d5637 ("rtc: cmos: Use ACPI alarm for non-Intel x86 systems too") `_ + +Disk disappears after resume +---------------------------- +After resuming from s2idle, the NVME disk would disappear. This was due to the +BIOS not specifying the _DSD StorageD3Enable property. This caused the NVME +driver not to put the disk into the expected state at suspend and to fail +on resume. + +`commit e79a10652bbd3 ("ACPI: x86: Force StorageD3Enable on more products") `_ + +Spurious IRQ1 +------------- +A number of Renoir, Lucienne, Cezanne, & Barcelo platforms have a +platform firmware bug where IRQ1 is triggered during s0i3 resume. + +This was fixed in the platform firmware, but a number of systems didn't +receive any more platform firmware updates. + +`commit 8e60615e89321 ("platform/x86/amd: pmc: Disable IRQ1 wakeup for RN/CZN") `_ + +Hardware timeout +---------------- +The hardware performs many actions besides accepting the values from +amd-pmc driver. As the communication path with the hardware is a mailbox, +it's possible that it might not respond quickly enough. +This issue manifested as a failure to suspend: :: + + PM: dpm_run_callback(): acpi_subsys_suspend_noirq+0x0/0x50 returns -110 + amd_pmc AMDI0005:00: PM: failed to suspend noirq: error -110 + +The timing problem was identified by comparing the values of the idle mask. + +`commit 3c3c8e88c8712 ("platform/x86: amd-pmc: Increase the response register timeout") `_ + +Failed to reach hardware sleep state with panel on +-------------------------------------------------- +On some Strix systems certain panels were observed to block the system from +entering a hardware sleep state if the internal panel was on during the sequence. + +Even though the panel got turned off during suspend it exposed a timing problem +where an interrupt caused the display hardware to wake up and block low power +state entry. + +`commit 40b8c14936bd2 ("drm/amd/display: Disable unneeded hpd interrupts during dm_init") `_ + +Runtime power consumption issues +================================ +Runtime power consumption is influenced by many factors, including but not +limited to the configuration of the PCIe Active State Power Management (ASPM), +the display brightness, the EPP policy of the CPU, and the power management +of the devices. + +ASPM +---- +For the best runtime power consumption, ASPM should be programmed as intended +by the BIOS from the hardware vendor. To accomplish this the Linux kernel +should be compiled with ``CONFIG_PCIEASPM_DEFAULT`` set to ``y`` and the +sysfs file ``/sys/module/pcie_aspm/parameters/policy`` should not be modified. + +Most notably, if L1.2 is not configured properly for any devices, the SoC +will not be able to enter the deepest idle state. + +EPP Policy +---------- +The ``energy_performance_preference`` sysfs file can be used to set a bias +of efficiency or performance for a CPU. This has a direct relationship on +the battery life when more heavily biased towards performance. + + +BIOS debug messages +=================== +Most OEM machines don't have a serial UART for outputting kernel or BIOS +debug messages. However BIOS debug messages are useful for understanding +both BIOS bugs and bugs with the Linux kernel drivers that call BIOS AML. + +As the BIOS on most OEM AMD systems are based off an AMD reference BIOS, +the infrastructure used for exporting debugging messages is often the same +as AMD reference BIOS. + +Manually Parsing +---------------- +There is generally an ACPI method ``\M460`` that different paths of the AML +will call to emit a message to the BIOS serial log. This method takes +7 arguments, with the first being a string and the rest being optional +integers:: + + Method (M460, 7, Serialized) + +Here is an example of a string that BIOS AML may call out using ``\M460``:: + + M460 (" OEM-ASL-PCIe Address (0x%X)._REG (%d %d) PCSA = %d\n", DADR, Arg0, Arg1, PCSA, Zero, Zero) + +Normally when executed, the ``\M460`` method would populate the additional +arguments into the string. In order to get these messages from the Linux +kernel a hook has been added into ACPICA that can capture the *arguments* +sent to ``\M460`` and print them to the kernel ring buffer. +For example the following message could be emitted into kernel ring buffer:: + + extrace-0174 ex_trace_args : " OEM-ASL-PCIe Address (0x%X)._REG (%d %d) PCSA = %d\n", ec106000, 2, 1, 1, 0, 0 + +In order to get these messages, you need to compile with ``CONFIG_ACPI_DEBUG`` +and then turn on the following ACPICA tracing parameters. +This can be done either on the kernel command line or at runtime: + +* ``acpi.trace_method_name=\M460`` +* ``acpi.trace_state=method`` + +NOTE: These can be very noisy at bootup. If you turn these parameters on +the kernel command, please also consider turning up ``CONFIG_LOG_BUF_SHIFT`` +to a larger size such as 17 to avoid losing early boot messages. + +Tool assisted Parsing +--------------------- +As mentioned above, parsing by hand can be tedious, especially with a lot of +messages. To help with this, a tool has been created at +`amd-debug-tools `_ +to help parse the messages. diff --git a/Documentation/arch/x86/index.rst b/Documentation/arch/x86/index.rst index 8ac64d7de4dc..58a006525ae8 100644 --- a/Documentation/arch/x86/index.rst +++ b/Documentation/arch/x86/index.rst @@ -25,6 +25,7 @@ x86-specific Documentation shstk iommu intel_txt + amd-debugging amd-memory-encryption amd_hsmp tdx diff --git a/Documentation/arch/x86/resume.svg b/Documentation/arch/x86/resume.svg new file mode 100644 index 000000000000..ad8839f762bf --- /dev/null +++ b/Documentation/arch/x86/resume.svg @@ -0,0 +1,4 @@ + + + +
Wakeup event occurs
MP1 hands off control to OS
OS Moves one core out of ACPI C3
MP0/MP1 boot process
OS checks all wake sources
no
yes
ACPI fixed
event active
no
yes
IRQ other
than ACPI SCI active
no
yes
GPIO
IRQ shared
with SCI
no
yes
no
Any PM
wakeup event
pending
Kernel resumes system
uPEP driver removes OS_HINT
yes
no
Any GPIO
w/ WAKESTS
active
Check for ACPI Notify() events
yes
Any GPE
pending
OS moves active
core back to
ACPI C3
MP1 puts system back to sleep
 \ No newline at end of file diff --git a/Documentation/arch/x86/suspend.svg b/Documentation/arch/x86/suspend.svg new file mode 100644 index 000000000000..a69073c018d5 --- /dev/null +++ b/Documentation/arch/x86/suspend.svg @@ -0,0 +1,4 @@ + + + +
SFH driver notifies MP2 to stop all sensor collection
no
Abort suspend; details logged in dmesg
Failures?
yes
no
yes
All devices go into deepest D-state or F-state
Failures?
no
yes
GPIO driver suspends non-wake GPIOs
Suspend initiated from userspace
GPU driver shuts down clocks and sends SMU messages
Failures?
ACPI s2idle driver notifies EC using _DSM
uPEP driver (amd-pmc) sends OS_HINT
Put all x86 CPU cores into ACPI C3
s2idle loop waiting for IRQ
to wake
no
Failures?
yes
 \ No newline at end of file From 1b3f2bd04d90f61e1f291b5e365b9bc4ce0ea7c7 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 29 Apr 2025 19:46:21 -0700 Subject: [PATCH 265/331] x86/devmem: Remove duplicate range_is_allowed() definition 17 years ago, Venki suggested [1] "A future improvement would be to avoid the range_is_allowed duplication". The only thing preventing a common implementation is that phys_mem_access_prot_allowed() expects the range check to exit immediately when PAT is disabled [2]. I.e. there is no cache conflict to manage in that case. This cleanup was noticed on the path to considering changing range_is_allowed() policy to blanket deny /dev/mem for private (confidential computing) memory. Note, however that phys_mem_access_prot_allowed() has long since stopped being relevant for managing cache-type validation due to [3], and [4]. Commit 0124cecfc85a ("x86, PAT: disable /dev/mem mmap RAM with PAT") [1] Commit 9e41bff2708e ("x86: fix /dev/mem mmap breakage when PAT is disabled") [2] Commit 1886297ce0c8 ("x86/mm/pat: Fix BUG_ON() in mmap_mem() on QEMU/i386") [3] Commit 0c3c8a18361a ("x86, PAT: Remove duplicate memtype reserve in devmem mmap") [4] Signed-off-by: Dan Williams Signed-off-by: Dave Hansen Reviewed-by: Nikolay Borisov Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/all/20250430024622.1134277-2-dan.j.williams%40intel.com --- arch/x86/mm/pat/memtype.c | 31 ++++--------------------------- drivers/char/mem.c | 18 ------------------ include/linux/io.h | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 45 deletions(-) diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 72d8cbc61158..c97b6598f187 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -773,38 +774,14 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, return vma_prot; } -#ifdef CONFIG_STRICT_DEVMEM -/* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM */ -static inline int range_is_allowed(unsigned long pfn, unsigned long size) -{ - return 1; -} -#else -/* This check is needed to avoid cache aliasing when PAT is enabled */ -static inline int range_is_allowed(unsigned long pfn, unsigned long size) -{ - u64 from = ((u64)pfn) << PAGE_SHIFT; - u64 to = from + size; - u64 cursor = from; - - if (!pat_enabled()) - return 1; - - while (cursor < to) { - if (!devmem_is_allowed(pfn)) - return 0; - cursor += PAGE_SIZE; - pfn++; - } - return 1; -} -#endif /* CONFIG_STRICT_DEVMEM */ - int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, unsigned long size, pgprot_t *vma_prot) { enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB; + if (!pat_enabled()) + return 1; + if (!range_is_allowed(pfn, size)) return 0; diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 169eed162a7f..48839958b0b1 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -61,29 +61,11 @@ static inline int page_is_allowed(unsigned long pfn) { return devmem_is_allowed(pfn); } -static inline int range_is_allowed(unsigned long pfn, unsigned long size) -{ - u64 from = ((u64)pfn) << PAGE_SHIFT; - u64 to = from + size; - u64 cursor = from; - - while (cursor < to) { - if (!devmem_is_allowed(pfn)) - return 0; - cursor += PAGE_SIZE; - pfn++; - } - return 1; -} #else static inline int page_is_allowed(unsigned long pfn) { return 1; } -static inline int range_is_allowed(unsigned long pfn, unsigned long size) -{ - return 1; -} #endif static inline bool should_stop_iteration(void) diff --git a/include/linux/io.h b/include/linux/io.h index 6a6bc4d46d0a..0642c7ee41db 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -183,4 +183,25 @@ static inline void arch_io_free_memtype_wc(resource_size_t base, int devm_arch_io_reserve_memtype_wc(struct device *dev, resource_size_t start, resource_size_t size); +#ifdef CONFIG_STRICT_DEVMEM +static inline int range_is_allowed(unsigned long pfn, unsigned long size) +{ + u64 from = ((u64)pfn) << PAGE_SHIFT; + u64 to = from + size; + u64 cursor = from; + + while (cursor < to) { + if (!devmem_is_allowed(pfn)) + return 0; + cursor += PAGE_SIZE; + pfn++; + } + return 1; +} +#else +static inline int range_is_allowed(unsigned long pfn, unsigned long size) +{ + return 1; +} +#endif #endif /* _LINUX_IO_H */ From 76deb5452e79ab86eac699ae3b6a44ab9c437ffe Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 2 May 2025 10:07:59 +0200 Subject: [PATCH 266/331] x86/msr: Improve the comments of the DECLARE_ARGS()/EAX_EDX_VAL()/EAX_EDX_RET() facility Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Peter Zijlstra Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/msr.h | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 2ccc78ebc3d7..27bc0b54d0cb 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -37,20 +37,22 @@ struct saved_msrs { }; /* - * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" - * constraint has different meanings. For i386, "A" means exactly - * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, - * it means rax *or* rdx. + * Both i386 and x86_64 returns 64-bit values in edx:eax for certain + * instructions, but GCC's "A" constraint has different meanings. + * For i386, "A" means exactly edx:eax, while for x86_64 it + * means rax *or* rdx. + * + * These helpers wrapping these semantic differences save one instruction + * clearing the high half of 'low': */ #ifdef CONFIG_X86_64 -/* Using 64-bit values saves one instruction clearing the high half of low */ -#define DECLARE_ARGS(val, low, high) unsigned long low, high -#define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) -#define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) +# define DECLARE_ARGS(val, low, high) unsigned long low, high +# define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) +# define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) #else -#define DECLARE_ARGS(val, low, high) u64 val -#define EAX_EDX_VAL(val, low, high) (val) -#define EAX_EDX_RET(val, low, high) "=A" (val) +# define DECLARE_ARGS(val, low, high) u64 val +# define EAX_EDX_VAL(val, low, high) (val) +# define EAX_EDX_RET(val, low, high) "=A" (val) #endif /* From c9d8ea9d53d4ddb80f2ad2bca5b9e9e40fcb9b16 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 2 May 2025 10:08:42 +0200 Subject: [PATCH 267/331] x86/msr: Rename DECLARE_ARGS() to EAX_EDX_DECLARE_ARGS DECLARE_ARGS() is way too generic of a name that says very little about why these args are declared in that fashion - use the EAX_EDX_ prefix to create a common prefix between the three helper methods: EAX_EDX_DECLARE_ARGS() EAX_EDX_VAL() EAX_EDX_RET() Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Peter Zijlstra Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/msr.h | 14 +++++++------- arch/x86/kernel/cpu/mce/core.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 27bc0b54d0cb..d57a94c5e595 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -46,11 +46,11 @@ struct saved_msrs { * clearing the high half of 'low': */ #ifdef CONFIG_X86_64 -# define DECLARE_ARGS(val, low, high) unsigned long low, high +# define EAX_EDX_DECLARE_ARGS(val, low, high) unsigned long low, high # define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) # define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) #else -# define DECLARE_ARGS(val, low, high) u64 val +# define EAX_EDX_DECLARE_ARGS(val, low, high) u64 val # define EAX_EDX_VAL(val, low, high) (val) # define EAX_EDX_RET(val, low, high) "=A" (val) #endif @@ -83,7 +83,7 @@ static inline void do_trace_rdpmc(u32 msr, u64 val, int failed) {} */ static __always_inline u64 __rdmsr(u32 msr) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); asm volatile("1: rdmsr\n" "2:\n" @@ -129,7 +129,7 @@ static inline u64 native_read_msr(u32 msr) static inline u64 native_read_msr_safe(u32 msr, int *err) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); asm volatile("1: rdmsr ; xor %[err],%[err]\n" "2:\n\t" @@ -182,7 +182,7 @@ extern int wrmsr_safe_regs(u32 regs[8]); */ static __always_inline u64 rdtsc(void) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); @@ -199,7 +199,7 @@ static __always_inline u64 rdtsc(void) */ static __always_inline u64 rdtsc_ordered(void) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); /* * The RDTSC instruction is not ordered relative to memory @@ -227,7 +227,7 @@ static __always_inline u64 rdtsc_ordered(void) static inline u64 native_read_pmc(int counter) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); if (tracepoint_enabled(rdpmc)) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 255927f0284e..7b9908cb5d69 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -390,7 +390,7 @@ void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) /* MSR access wrappers used for error injection */ noinstr u64 mce_rdmsrq(u32 msr) { - DECLARE_ARGS(val, low, high); + EAX_EDX_DECLARE_ARGS(val, low, high); if (__this_cpu_read(injectm.finished)) { int offset; From bdfda83a6b5988f1ac62cd0eaceb6c3b44dc2a31 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 2 May 2025 10:13:53 +0200 Subject: [PATCH 268/331] x86/msr: Move the EAX_EDX_*() methods from to We are going to use them from multiple headers, and in any case, such register access wrapper macros are better in anyway. Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Peter Zijlstra Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/asm.h | 19 +++++++++++++++++++ arch/x86/include/asm/msr.h | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index cc2881576c2c..206e1342fc03 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -243,5 +243,24 @@ register unsigned long current_stack_pointer asm(_ASM_SP); #define _ASM_EXTABLE_FAULT(from, to) \ _ASM_EXTABLE_TYPE(from, to, EX_TYPE_FAULT) +/* + * Both i386 and x86_64 returns 64-bit values in edx:eax for certain + * instructions, but GCC's "A" constraint has different meanings. + * For i386, "A" means exactly edx:eax, while for x86_64 it + * means rax *or* rdx. + * + * These helpers wrapping these semantic differences save one instruction + * clearing the high half of 'low': + */ +#ifdef CONFIG_X86_64 +# define EAX_EDX_DECLARE_ARGS(val, low, high) unsigned long low, high +# define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) +# define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) +#else +# define EAX_EDX_DECLARE_ARGS(val, low, high) u64 val +# define EAX_EDX_VAL(val, low, high) (val) +# define EAX_EDX_RET(val, low, high) "=A" (val) +#endif + #endif /* __KERNEL__ */ #endif /* _ASM_X86_ASM_H */ diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index d57a94c5e595..856d660c926d 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -36,25 +36,6 @@ struct saved_msrs { struct saved_msr *array; }; -/* - * Both i386 and x86_64 returns 64-bit values in edx:eax for certain - * instructions, but GCC's "A" constraint has different meanings. - * For i386, "A" means exactly edx:eax, while for x86_64 it - * means rax *or* rdx. - * - * These helpers wrapping these semantic differences save one instruction - * clearing the high half of 'low': - */ -#ifdef CONFIG_X86_64 -# define EAX_EDX_DECLARE_ARGS(val, low, high) unsigned long low, high -# define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32) -# define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) -#else -# define EAX_EDX_DECLARE_ARGS(val, low, high) u64 val -# define EAX_EDX_VAL(val, low, high) (val) -# define EAX_EDX_RET(val, low, high) "=A" (val) -#endif - /* * Be very careful with includes. This header is prone to include loops. */ From efef7f184f2eaf29a1ca676712d0e6e851cd0191 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Wed, 30 Apr 2025 22:42:41 -0700 Subject: [PATCH 269/331] x86/msr: Add explicit includes of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For historic reasons there are some TSC-related functions in the header, even though there's an header. To facilitate the relocation of rdtsc{,_ordered}() from to and to eventually eliminate the inclusion of in , add an explicit dependency to the source files that reference definitions from . [ mingo: Clarified the changelog. ] Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Acked-by: Ilpo Järvinen Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Peter Zijlstra Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250501054241.1245648-1-xin@zytor.com --- arch/x86/coco/sev/core.c | 1 + arch/x86/events/amd/core.c | 1 + arch/x86/events/amd/ibs.c | 1 + arch/x86/events/amd/iommu.c | 2 ++ arch/x86/events/amd/lbr.c | 1 + arch/x86/events/amd/power.c | 1 + arch/x86/events/core.c | 1 + arch/x86/events/intel/bts.c | 1 + arch/x86/events/intel/core.c | 1 + arch/x86/events/intel/cstate.c | 1 + arch/x86/events/intel/ds.c | 1 + arch/x86/events/intel/knc.c | 1 + arch/x86/events/intel/p4.c | 1 + arch/x86/events/intel/p6.c | 1 + arch/x86/events/intel/pt.c | 1 + arch/x86/events/intel/uncore.c | 1 + arch/x86/events/intel/uncore_discovery.c | 1 + arch/x86/events/intel/uncore_nhmex.c | 1 + arch/x86/events/intel/uncore_snb.c | 1 + arch/x86/events/intel/uncore_snbep.c | 1 + arch/x86/events/msr.c | 2 ++ arch/x86/events/perf_event.h | 1 + arch/x86/events/probe.c | 2 ++ arch/x86/events/rapl.c | 1 + arch/x86/events/utils.c | 1 + arch/x86/events/zhaoxin/core.c | 1 + arch/x86/hyperv/hv_apic.c | 1 + arch/x86/hyperv/hv_init.c | 1 + arch/x86/hyperv/hv_spinlock.c | 1 + arch/x86/hyperv/hv_vtl.c | 1 + arch/x86/hyperv/ivm.c | 1 + arch/x86/include/asm/fred.h | 1 + arch/x86/include/asm/kvm_host.h | 1 + arch/x86/include/asm/microcode.h | 2 ++ arch/x86/include/asm/mshyperv.h | 1 + arch/x86/include/asm/msr.h | 1 + arch/x86/include/asm/resctrl.h | 2 ++ arch/x86/include/asm/suspend_32.h | 1 + arch/x86/include/asm/suspend_64.h | 1 + arch/x86/include/asm/switch_to.h | 2 ++ arch/x86/kernel/acpi/sleep.c | 1 + arch/x86/kernel/apic/apic.c | 1 + arch/x86/kernel/apic/apic_numachip.c | 1 + arch/x86/kernel/cet.c | 1 + arch/x86/kernel/cpu/amd.c | 1 + arch/x86/kernel/cpu/aperfmperf.c | 1 + arch/x86/kernel/cpu/bus_lock.c | 1 + arch/x86/kernel/cpu/feat_ctl.c | 1 + arch/x86/kernel/cpu/hygon.c | 1 + arch/x86/kernel/cpu/mce/inject.c | 1 + arch/x86/kernel/cpu/microcode/core.c | 1 + arch/x86/kernel/cpu/mshyperv.c | 1 + arch/x86/kernel/cpu/resctrl/core.c | 1 + arch/x86/kernel/cpu/resctrl/monitor.c | 1 + arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 1 + arch/x86/kernel/cpu/resctrl/rdtgroup.c | 1 + arch/x86/kernel/cpu/sgx/main.c | 1 + arch/x86/kernel/cpu/topology.c | 1 + arch/x86/kernel/cpu/topology_amd.c | 1 + arch/x86/kernel/cpu/tsx.c | 1 + arch/x86/kernel/cpu/zhaoxin.c | 1 + arch/x86/kernel/fpu/core.c | 1 + arch/x86/kernel/fpu/xstate.c | 1 + arch/x86/kernel/fpu/xstate.h | 1 + arch/x86/kernel/fred.c | 1 + arch/x86/kernel/hpet.c | 1 + arch/x86/kernel/kvm.c | 1 + arch/x86/kernel/paravirt.c | 1 + arch/x86/kernel/process.c | 1 + arch/x86/kernel/process_64.c | 1 + arch/x86/kernel/trace_clock.c | 2 +- arch/x86/kernel/traps.c | 1 + arch/x86/kernel/tsc.c | 1 + arch/x86/kernel/tsc_sync.c | 1 + arch/x86/kvm/svm/avic.c | 1 + arch/x86/kvm/svm/sev.c | 1 + arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/vmx/nested.c | 1 + arch/x86/kvm/vmx/pmu_intel.c | 1 + arch/x86/kvm/vmx/sgx.c | 1 + arch/x86/kvm/vmx/vmx.c | 1 + arch/x86/lib/insn-eval.c | 1 + arch/x86/lib/kaslr.c | 2 +- arch/x86/mm/tlb.c | 1 + arch/x86/pci/mmconfig-shared.c | 5 +++-- arch/x86/power/cpu.c | 1 + arch/x86/realmode/init.c | 1 + arch/x86/virt/svm/sev.c | 1 + arch/x86/xen/enlighten_pv.c | 1 + arch/x86/xen/pmu.c | 1 + arch/x86/xen/suspend.c | 1 + drivers/accel/habanalabs/common/habanalabs_ioctl.c | 2 -- drivers/acpi/acpi_extlog.c | 1 + drivers/acpi/processor_perflib.c | 1 + drivers/acpi/processor_throttling.c | 6 +++++- drivers/char/agp/nvidia-agp.c | 1 + drivers/cpufreq/amd-pstate-ut.c | 2 ++ drivers/cpufreq/elanfreq.c | 1 - drivers/cpufreq/sc520_freq.c | 1 - drivers/crypto/ccp/sev-dev.c | 1 + drivers/edac/ie31200_edac.c | 1 + drivers/edac/mce_amd.c | 1 + drivers/hwmon/hwmon-vid.c | 4 ++++ drivers/idle/intel_idle.c | 1 + drivers/misc/cs5535-mfgpt.c | 1 + drivers/net/vmxnet3/vmxnet3_drv.c | 4 ++++ drivers/platform/x86/intel/ifs/core.c | 1 + drivers/platform/x86/intel/ifs/load.c | 1 + drivers/platform/x86/intel/ifs/runtest.c | 1 + drivers/platform/x86/intel/pmc/cnp.c | 1 + drivers/platform/x86/intel/speed_select_if/isst_if_common.c | 1 + .../platform/x86/intel/speed_select_if/isst_if_mbox_msr.c | 1 + drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c | 1 + drivers/platform/x86/intel/turbo_max_3.c | 1 + .../platform/x86/intel/uncore-frequency/uncore-frequency.c | 1 + drivers/powercap/intel_rapl_common.c | 1 + drivers/powercap/intel_rapl_msr.c | 1 + .../intel/int340x_thermal/processor_thermal_device.c | 1 + drivers/thermal/intel/intel_tcc_cooling.c | 1 + drivers/thermal/intel/x86_pkg_temp_thermal.c | 1 + drivers/video/fbdev/geode/display_gx.c | 1 + drivers/video/fbdev/geode/gxfb_core.c | 1 + drivers/video/fbdev/geode/lxfb_ops.c | 1 + 123 files changed, 139 insertions(+), 9 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index b18a33fe8dd3..85b16a0ee417 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -43,6 +43,7 @@ #include #include #include +#include #define DR7_RESET_VALUE 0x400 diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index cb62b6d12691..79e8453dd051 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "../perf_event.h" diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 82fa755d1b12..20877927b021 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -15,6 +15,7 @@ #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c index f8228d8243f7..a721da9987dd 100644 --- a/arch/x86/events/amd/iommu.c +++ b/arch/x86/events/amd/iommu.c @@ -16,6 +16,8 @@ #include #include +#include + #include "../perf_event.h" #include "iommu.h" diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 198851985bb7..d24da377df77 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include "../perf_event.h" diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c index 598a727d823a..dad42790cf7d 100644 --- a/arch/x86/events/amd/power.c +++ b/arch/x86/events/amd/power.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../perf_event.h" /* Event code: LSB 8 bits, passed in attr->config any other bit is reserved. */ diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 64943e5ba685..7cfd3764a582 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c index a95e6c91c4d7..ca9f57437d8b 100644 --- a/arch/x86/events/intel/bts.c +++ b/arch/x86/events/intel/bts.c @@ -17,6 +17,7 @@ #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 9ef3577d4b35..3d80ffb2c374 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 56b1c391ccc7..ec753e39b007 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -111,6 +111,7 @@ #include #include #include +#include #include "../perf_event.h" #include "../probe.h" diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index b6b486d8530d..00a0a859f177 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "../perf_event.h" diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index 425f6e6eed89..38904a558128 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -5,6 +5,7 @@ #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index 24d811a9608a..aa5202126752 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c index 35917a776bec..6e41de355bd8 100644 --- a/arch/x86/events/intel/p6.c +++ b/arch/x86/events/intel/p6.c @@ -3,6 +3,7 @@ #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index d579f1092357..f37cce231266 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "../perf_event.h" #include "pt.h" diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index d6070529c58e..c24d21932c91 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -3,6 +3,7 @@ #include #include +#include #include "uncore.h" #include "uncore_discovery.h" diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 4fc3eec325f6..18a3022f26a0 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -5,6 +5,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include "uncore.h" #include "uncore_discovery.h" diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index bef9c782c781..8962e7cb21e3 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Nehalem-EX/Westmere-EX uncore support */ #include +#include #include "uncore.h" /* NHM-EX event control */ diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index afc8ef02a7a9..a1a96833e30e 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */ +#include #include "uncore.h" #include "uncore_discovery.h" diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index a330053407b8..2824dc9950be 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* SandyBridge-EP/IvyTown uncore support */ #include +#include #include "uncore.h" #include "uncore_discovery.h" diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 8970ecef87c5..7f5007a4752a 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -3,6 +3,8 @@ #include #include #include +#include + #include "probe.h" enum perf_msr_id { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index a5166fa9339b..a8d4e82e3589 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -17,6 +17,7 @@ #include #include #include +#include /* To enable MSR tracing please use the generic trace points. */ diff --git a/arch/x86/events/probe.c b/arch/x86/events/probe.c index fda35cf25528..bb719d0d3f0b 100644 --- a/arch/x86/events/probe.c +++ b/arch/x86/events/probe.c @@ -2,6 +2,8 @@ #include #include #include + +#include #include "probe.h" static umode_t diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index 7ff52c23d7a1..defd86137f12 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -65,6 +65,7 @@ #include #include #include +#include #include "perf_event.h" #include "probe.h" diff --git a/arch/x86/events/utils.c b/arch/x86/events/utils.c index dab4ed199227..77fd00b3305e 100644 --- a/arch/x86/events/utils.c +++ b/arch/x86/events/utils.c @@ -2,6 +2,7 @@ #include #include +#include #include "perf_event.h" static int decode_branch_type(struct insn *insn) diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index e299364eb889..91443aba4c7d 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "../perf_event.h" diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index c450e67cb0a4..a079a1427091 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -28,6 +28,7 @@ #include #include #include +#include #include diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index ed89867b6fd0..5d27194a2efa 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 626f6d4d6253..81b006601370 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -15,6 +15,7 @@ #include #include #include +#include static bool hv_pvspin __initdata = true; diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index 13242ed8ff16..079b276e5f30 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include <../kernel/smpboot.h> diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 1b8a2415183b..8209de792388 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #ifdef CONFIG_AMD_MEM_ENCRYPT diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h index 2a29e5216881..12b34d5b2953 100644 --- a/arch/x86/include/asm/fred.h +++ b/arch/x86/include/asm/fred.h @@ -9,6 +9,7 @@ #include #include +#include #include /* diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3131abcac4f1..9c971f846108 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 263ea3dd0001..107a1aaa211b 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_MICROCODE_H #define _ASM_X86_MICROCODE_H +#include + struct cpu_signature { unsigned int sig; unsigned int pf; diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index bab5ccfc60a7..15d00dace70f 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -8,6 +8,7 @@ #include #include #include +#include #include /* diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 856d660c926d..35a78d2721a3 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -12,6 +12,7 @@ #include #include +#include #include struct msr_info { diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index 011bf67a1866..bd6afe805cf6 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -9,6 +9,8 @@ #include #include +#include + /* * This value can never be a valid CLOSID, and is used when mapping a * (closid, rmid) pair to an index and back. On x86 only the RMID is diff --git a/arch/x86/include/asm/suspend_32.h b/arch/x86/include/asm/suspend_32.h index d8416b3bf832..e8e5aab06255 100644 --- a/arch/x86/include/asm/suspend_32.h +++ b/arch/x86/include/asm/suspend_32.h @@ -9,6 +9,7 @@ #include #include +#include /* image of the saved processor state */ struct saved_context { diff --git a/arch/x86/include/asm/suspend_64.h b/arch/x86/include/asm/suspend_64.h index 54df06687d83..b512f9665f78 100644 --- a/arch/x86/include/asm/suspend_64.h +++ b/arch/x86/include/asm/suspend_64.h @@ -9,6 +9,7 @@ #include #include +#include /* * Image of the saved processor state, used by the low level ACPI suspend to diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h index 75248546403d..4f21df7af715 100644 --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -52,6 +52,8 @@ do { \ } while (0) #ifdef CONFIG_X86_32 +#include + static inline void refresh_sysenter_cs(struct thread_struct *thread) { /* Only happens when SEP is enabled, no need to test "SEP"arately: */ diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 6dfecb27b846..91fa262f0e30 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index a05871c85183..d73ba5a7b623 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index fcfef701c17a..e272bc7fdc8e 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -14,6 +14,7 @@ #include #include +#include #include #include diff --git a/arch/x86/kernel/cet.c b/arch/x86/kernel/cet.c index d897aadd1d44..99444409c026 100644 --- a/arch/x86/kernel/cet.c +++ b/arch/x86/kernel/cet.c @@ -2,6 +2,7 @@ #include #include +#include #include enum cp_error_code { diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1f7925e45b46..e153cd1d36fe 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index e99892aad628..a315b0627dfb 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index a18d0f2ea832..981f8b1f0792 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -10,6 +10,7 @@ #include #include #include +#include enum split_lock_detect_state { sld_off = 0, diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 441174844e01..d69757246bde 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 21541e310c2c..2154f12766fb 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 338aeee95bd2..e13f533e31e6 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index b3658d11e7b6..793e2927d0fa 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include "internal.h" diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index b924befe8d6e..c78f860419d6 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 280d6900726b..d987b11c168c 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -22,6 +22,7 @@ #include #include +#include #include #include "internal.h" diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index f73a74945ffa..591b0b44d260 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -23,6 +23,7 @@ #include #include +#include #include #include "internal.h" diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 2a82eb6a0376..26c354bdea07 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "../../events/perf_event.h" /* For X86_CONFIG() */ #include "internal.h" diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index f4a2ee2a6404..f724b8f6a6a4 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -28,6 +28,7 @@ #include +#include #include #include "internal.h" diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 40967d8f995a..6722b2fc82cf 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "driver.h" #include "encl.h" diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 6e1885dece0f..e35ccdc84910 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index 535dcf511096..f78d38510027 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -3,6 +3,7 @@ #include #include +#include #include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/tsx.c b/arch/x86/kernel/cpu/tsx.c index b0a9c9e9d029..49782724a943 100644 --- a/arch/x86/kernel/cpu/tsx.c +++ b/arch/x86/kernel/cpu/tsx.c @@ -12,6 +12,7 @@ #include #include +#include #include "cpu.h" diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c index 90eba7eb5335..89b1c8a70fe8 100644 --- a/arch/x86/kernel/cpu/zhaoxin.c +++ b/arch/x86/kernel/cpu/zhaoxin.c @@ -4,6 +4,7 @@ #include #include +#include #include "cpu.h" diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 985dfffe28c1..e92d27324d9a 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 2bd87b788630..86d690afb63c 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kernel/fpu/xstate.h b/arch/x86/kernel/fpu/xstate.h index 5e5d35027f13..f705bd355ea2 100644 --- a/arch/x86/kernel/fpu/xstate.h +++ b/arch/x86/kernel/fpu/xstate.h @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 DECLARE_PER_CPU(u64, xfd_state); diff --git a/arch/x86/kernel/fred.c b/arch/x86/kernel/fred.c index 10b0169f3fc1..816187da3a47 100644 --- a/arch/x86/kernel/fred.c +++ b/arch/x86/kernel/fred.c @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index cc5d12232216..c9982a7c9536 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -12,6 +12,7 @@ #include #include #include +#include #undef pr_fmt #define pr_fmt(fmt) "hpet: " fmt diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 44a45df7200a..27f7192e1c61 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 1ccd05d8999f..015bf298434f 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -33,6 +33,7 @@ #include #include #include +#include /* stub always returning 0. */ DEFINE_ASM_FUNC(paravirt_ret0, "xor %eax,%eax", .entry.text); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index c168f99b5f0b..bd50249cff50 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include "process.h" diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 24e1ccf22912..cfa9c031de91 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -57,6 +57,7 @@ #include #include #include +#include #ifdef CONFIG_IA32_EMULATION /* Not included via unistd.h */ #include diff --git a/arch/x86/kernel/trace_clock.c b/arch/x86/kernel/trace_clock.c index b8e7abe00b06..708d61743d15 100644 --- a/arch/x86/kernel/trace_clock.c +++ b/arch/x86/kernel/trace_clock.c @@ -4,7 +4,7 @@ */ #include #include -#include +#include /* * trace_clock_x86_tsc(): A clock that is just the cycle counter. diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 823410aaf429..ca43eb5a02a3 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -68,6 +68,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 #include diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 160fef71b9a3..5d3a764ba77c 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index f1c7a86dbf49..ec3aa340d351 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -21,6 +21,7 @@ #include #include #include +#include #include struct tsc_adjust { diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index d0f28f6880e2..067f8e3f5a0d 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -20,6 +20,7 @@ #include #include +#include #include "trace.h" #include "lapic.h" diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index a4aabd666665..4b607cc377c9 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "mmu.h" diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 67657b3a36ce..c23f620989ed 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a7fea622e204..d268224227f0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6,6 +6,7 @@ #include #include +#include #include "x86.h" #include "cpuid.h" diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 5e0bb821c7bc..231a9633359c 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "x86.h" #include "cpuid.h" diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 949864259ee6..df1d0cf76947 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include "x86.h" diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cd0d6c1fcf9c..d8412cfdb18e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index da5af3cc25b1..dbe0fbf0037f 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #undef pr_fmt diff --git a/arch/x86/lib/kaslr.c b/arch/x86/lib/kaslr.c index a58f451a7dd3..b5893928d55c 100644 --- a/arch/x86/lib/kaslr.c +++ b/arch/x86/lib/kaslr.c @@ -8,7 +8,7 @@ */ #include #include -#include +#include #include #include #include diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 7c35b500ce49..33645dde7f3b 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 39255f0eb14d..1f4522325920 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -22,9 +22,10 @@ #include #include #include -#include -#include #include +#include +#include +#include /* Indicate if the ECAM resources have been placed into the resource table */ static bool pci_mmcfg_running_state; diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index d5a7b3bc2453..916441f5e85c 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #ifdef CONFIG_X86_32 diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 263787b4800c..ed5c63c0b4e5 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -9,6 +9,7 @@ #include #include #include +#include #include struct real_mode_header *real_mode_header; diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 334177c89df0..76926f75e9bf 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -30,6 +30,7 @@ #include #include #include +#include /* * The RMP entry information as returned by the RMPREAD instruction. diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 846b5737d320..8ddd9e535f99 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index f06987b0efc3..3cb566d4aaad 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c index 7bb3ac2d5ac8..ba2f17e64321 100644 --- a/arch/x86/xen/suspend.c +++ b/arch/x86/xen/suspend.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "xen-ops.h" diff --git a/drivers/accel/habanalabs/common/habanalabs_ioctl.c b/drivers/accel/habanalabs/common/habanalabs_ioctl.c index 8729a0c57d78..dc80ca921d90 100644 --- a/drivers/accel/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/accel/habanalabs/common/habanalabs_ioctl.c @@ -17,8 +17,6 @@ #include #include -#include - /* make sure there is space for all the signed info */ static_assert(sizeof(struct cpucp_info) <= SEC_DEV_INFO_BUF_SZ); diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c index 8465822b6672..f6b9562779de 100644 --- a/drivers/acpi/acpi_extlog.c +++ b/drivers/acpi/acpi_extlog.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "apei/apei-internal.h" #include diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index 53996f1a2d80..64b8d1e19594 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -20,6 +20,7 @@ #include #ifdef CONFIG_X86 #include +#include #endif #define ACPI_PROCESSOR_FILE_PERFORMANCE "performance" diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c index 00d045e5f524..ecd7fe256153 100644 --- a/drivers/acpi/processor_throttling.c +++ b/drivers/acpi/processor_throttling.c @@ -18,9 +18,13 @@ #include #include #include +#include #include #include -#include +#include +#ifdef CONFIG_X86 +#include +#endif /* ignore_tpc: * 0 -> acpi processor driver doesn't ignore _TPC values diff --git a/drivers/char/agp/nvidia-agp.c b/drivers/char/agp/nvidia-agp.c index e424360fb4a1..4787391bb6b4 100644 --- a/drivers/char/agp/nvidia-agp.c +++ b/drivers/char/agp/nvidia-agp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "agp.h" /* NVIDIA registers */ diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 707fa81c749f..c8d031b297d2 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -31,6 +31,8 @@ #include +#include + #include "amd-pstate.h" diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c index 36494b855e41..fc5a58088b35 100644 --- a/drivers/cpufreq/elanfreq.c +++ b/drivers/cpufreq/elanfreq.c @@ -21,7 +21,6 @@ #include #include -#include #include #include diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c index 103d2519dff7..b360f03a116f 100644 --- a/drivers/cpufreq/sc520_freq.c +++ b/drivers/cpufreq/sc520_freq.c @@ -21,7 +21,6 @@ #include #include -#include #define MMCR_BASE 0xfffef000 /* The default base address */ #define OFFS_CPUCTL 0x2 /* CPU Control Register */ diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index bb8a25ef5b43..ec8b37a7f40c 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "psp-dev.h" #include "sev-dev.h" diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 204834149579..5ddd83dc94ba 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -52,6 +52,7 @@ #include #include +#include #include "edac_module.h" #define EDAC_MOD_STR "ie31200_edac" diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 50d74d3bf0f5..af3c12284a1e 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -3,6 +3,7 @@ #include #include +#include #include "mce_amd.h" diff --git a/drivers/hwmon/hwmon-vid.c b/drivers/hwmon/hwmon-vid.c index 6d1175a51832..2df4956296ed 100644 --- a/drivers/hwmon/hwmon-vid.c +++ b/drivers/hwmon/hwmon-vid.c @@ -15,6 +15,10 @@ #include #include +#ifdef CONFIG_X86 +#include +#endif + /* * Common code for decoding VID pins. * diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 517b28a85560..6a1712b50c7f 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -56,6 +56,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/misc/cs5535-mfgpt.c b/drivers/misc/cs5535-mfgpt.c index 18fc1aaa5cdd..2b6778d8d166 100644 --- a/drivers/misc/cs5535-mfgpt.c +++ b/drivers/misc/cs5535-mfgpt.c @@ -16,6 +16,7 @@ #include #include #include +#include #define DRV_NAME "cs5535-mfgpt" diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index 3df6aabc7e33..7edd0b5e0e77 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -27,6 +27,10 @@ #include #include +#ifdef CONFIG_X86 +#include +#endif + #include "vmxnet3_int.h" #include "vmxnet3_xdp.h" diff --git a/drivers/platform/x86/intel/ifs/core.c b/drivers/platform/x86/intel/ifs/core.c index c4328a7ae083..b73e582128c9 100644 --- a/drivers/platform/x86/intel/ifs/core.c +++ b/drivers/platform/x86/intel/ifs/core.c @@ -8,6 +8,7 @@ #include #include +#include #include "ifs.h" diff --git a/drivers/platform/x86/intel/ifs/load.c b/drivers/platform/x86/intel/ifs/load.c index 0289391eccde..50f1fdf7dfed 100644 --- a/drivers/platform/x86/intel/ifs/load.c +++ b/drivers/platform/x86/intel/ifs/load.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "ifs.h" diff --git a/drivers/platform/x86/intel/ifs/runtest.c b/drivers/platform/x86/intel/ifs/runtest.c index 6b6ed7be461a..dfc119d7354d 100644 --- a/drivers/platform/x86/intel/ifs/runtest.c +++ b/drivers/platform/x86/intel/ifs/runtest.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "ifs.h" diff --git a/drivers/platform/x86/intel/pmc/cnp.c b/drivers/platform/x86/intel/pmc/cnp.c index 547bdf1ab02d..efea4e1ba52b 100644 --- a/drivers/platform/x86/intel/pmc/cnp.c +++ b/drivers/platform/x86/intel/pmc/cnp.c @@ -10,6 +10,7 @@ #include #include +#include #include "core.h" /* Cannon Lake: PGD PFET Enable Ack Status Register(s) bitmap */ diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c index 44dcd165b4c0..8a5713593811 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c @@ -21,6 +21,7 @@ #include #include +#include #include "isst_if_common.h" diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c index 78989f649aea..22745b217c6f 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_if_mbox_msr.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "isst_if_common.h" diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 0b8ef0cfaf80..4d30d5360c8f 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "isst_tpmi_core.h" diff --git a/drivers/platform/x86/intel/turbo_max_3.c b/drivers/platform/x86/intel/turbo_max_3.c index 7e538bbd5b50..b5af3e91ba04 100644 --- a/drivers/platform/x86/intel/turbo_max_3.c +++ b/drivers/platform/x86/intel/turbo_max_3.c @@ -17,6 +17,7 @@ #include #include +#include #define MSR_OC_MAILBOX 0x150 #define MSR_OC_MAILBOX_CMD_OFFSET 32 diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c index 6f873765d2d1..96f854c21bb5 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "uncore-frequency-common.h" diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 5ab3feb29686..e3be40adc0d7 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -28,6 +28,7 @@ #include #include #include +#include /* bitmasks for RAPL MSRs, used by primitive access functions */ #define ENERGY_STATUS_MASK 0xffffffff diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 6d5853db17ad..8ad2115d65f6 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -24,6 +24,7 @@ #include #include +#include /* Local defines */ #define MSR_PLATFORM_POWER_LIMIT 0x0000065C diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index b0249468b844..57cf46f69669 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "int340x_thermal_zone.h" #include "processor_thermal_device.h" #include "../intel_soc_dts_iosf.h" diff --git a/drivers/thermal/intel/intel_tcc_cooling.c b/drivers/thermal/intel/intel_tcc_cooling.c index 0394897e83cf..f352ecafbedf 100644 --- a/drivers/thermal/intel/intel_tcc_cooling.c +++ b/drivers/thermal/intel/intel_tcc_cooling.c @@ -11,6 +11,7 @@ #include #include #include +#include #define TCC_PROGRAMMABLE BIT(30) #define TCC_LOCKED BIT(31) diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 496abf8e55e0..4894a26b1e4e 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -20,6 +20,7 @@ #include #include +#include #include "thermal_interrupt.h" diff --git a/drivers/video/fbdev/geode/display_gx.c b/drivers/video/fbdev/geode/display_gx.c index b5f25dffd274..099322cefce0 100644 --- a/drivers/video/fbdev/geode/display_gx.c +++ b/drivers/video/fbdev/geode/display_gx.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "gxfb.h" diff --git a/drivers/video/fbdev/geode/gxfb_core.c b/drivers/video/fbdev/geode/gxfb_core.c index 2b27d6540805..8d69be7c9d31 100644 --- a/drivers/video/fbdev/geode/gxfb_core.c +++ b/drivers/video/fbdev/geode/gxfb_core.c @@ -29,6 +29,7 @@ #include #include +#include #include #include "gxfb.h" diff --git a/drivers/video/fbdev/geode/lxfb_ops.c b/drivers/video/fbdev/geode/lxfb_ops.c index a27531b5de11..2e33da9849b0 100644 --- a/drivers/video/fbdev/geode/lxfb_ops.c +++ b/drivers/video/fbdev/geode/lxfb_ops.c @@ -11,6 +11,7 @@ #include #include +#include #include "lxfb.h" /* TODO From 288a4ff0ad29d1a9391b8a111a4b6da51da3aa85 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Fri, 2 May 2025 10:20:14 +0200 Subject: [PATCH 270/331] x86/msr: Move rdtsc{,_ordered}() to Relocate rdtsc{,_ordered}() from to . [ mingo: Do not remove the inclusion from just yet, to reduce -next breakages. We can do this later on, separately, shortly before the next -rc1. ] Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250427092027.1598740-3-xin@zytor.com --- arch/x86/include/asm/msr.h | 54 ------------------------------------- arch/x86/include/asm/tsc.h | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 35a78d2721a3..f5c0969fa3bb 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -153,60 +153,6 @@ native_write_msr_safe(u32 msr, u32 low, u32 high) extern int rdmsr_safe_regs(u32 regs[8]); extern int wrmsr_safe_regs(u32 regs[8]); -/** - * rdtsc() - returns the current TSC without ordering constraints - * - * rdtsc() returns the result of RDTSC as a 64-bit integer. The - * only ordering constraint it supplies is the ordering implied by - * "asm volatile": it will put the RDTSC in the place you expect. The - * CPU can and will speculatively execute that RDTSC, though, so the - * results can be non-monotonic if compared on different CPUs. - */ -static __always_inline u64 rdtsc(void) -{ - EAX_EDX_DECLARE_ARGS(val, low, high); - - asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); - - return EAX_EDX_VAL(val, low, high); -} - -/** - * rdtsc_ordered() - read the current TSC in program order - * - * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer. - * It is ordered like a load to a global in-memory counter. It should - * be impossible to observe non-monotonic rdtsc_unordered() behavior - * across multiple CPUs as long as the TSC is synced. - */ -static __always_inline u64 rdtsc_ordered(void) -{ - EAX_EDX_DECLARE_ARGS(val, low, high); - - /* - * The RDTSC instruction is not ordered relative to memory - * access. The Intel SDM and the AMD APM are both vague on this - * point, but empirically an RDTSC instruction can be - * speculatively executed before prior loads. An RDTSC - * immediately after an appropriate barrier appears to be - * ordered as a normal load, that is, it provides the same - * ordering guarantees as reading from a global memory location - * that some other imaginary CPU is updating continuously with a - * time stamp. - * - * Thus, use the preferred barrier on the respective CPU, aiming for - * RDTSCP as the default. - */ - asm volatile(ALTERNATIVE_2("rdtsc", - "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC, - "rdtscp", X86_FEATURE_RDTSCP) - : EAX_EDX_RET(val, low, high) - /* RDTSCP clobbers ECX with MSR_TSC_AUX. */ - :: "ecx"); - - return EAX_EDX_VAL(val, low, high); -} - static inline u64 native_read_pmc(int counter) { EAX_EDX_DECLARE_ARGS(val, low, high); diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index 94408a784c8e..4f7f09f50552 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -5,10 +5,65 @@ #ifndef _ASM_X86_TSC_H #define _ASM_X86_TSC_H +#include #include #include #include +/** + * rdtsc() - returns the current TSC without ordering constraints + * + * rdtsc() returns the result of RDTSC as a 64-bit integer. The + * only ordering constraint it supplies is the ordering implied by + * "asm volatile": it will put the RDTSC in the place you expect. The + * CPU can and will speculatively execute that RDTSC, though, so the + * results can be non-monotonic if compared on different CPUs. + */ +static __always_inline u64 rdtsc(void) +{ + EAX_EDX_DECLARE_ARGS(val, low, high); + + asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); + + return EAX_EDX_VAL(val, low, high); +} + +/** + * rdtsc_ordered() - read the current TSC in program order + * + * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer. + * It is ordered like a load to a global in-memory counter. It should + * be impossible to observe non-monotonic rdtsc_unordered() behavior + * across multiple CPUs as long as the TSC is synced. + */ +static __always_inline u64 rdtsc_ordered(void) +{ + EAX_EDX_DECLARE_ARGS(val, low, high); + + /* + * The RDTSC instruction is not ordered relative to memory + * access. The Intel SDM and the AMD APM are both vague on this + * point, but empirically an RDTSC instruction can be + * speculatively executed before prior loads. An RDTSC + * immediately after an appropriate barrier appears to be + * ordered as a normal load, that is, it provides the same + * ordering guarantees as reading from a global memory location + * that some other imaginary CPU is updating continuously with a + * time stamp. + * + * Thus, use the preferred barrier on the respective CPU, aiming for + * RDTSCP as the default. + */ + asm volatile(ALTERNATIVE_2("rdtsc", + "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC, + "rdtscp", X86_FEATURE_RDTSCP) + : EAX_EDX_RET(val, low, high) + /* RDTSCP clobbers ECX with MSR_TSC_AUX. */ + :: "ecx"); + + return EAX_EDX_VAL(val, low, high); +} + /* * Standard way to access the cycle counter. */ From 91882511ef905fdfb7d33be88f6ddb91012ee30f Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:15 -0700 Subject: [PATCH 271/331] x86/msr: Remove the unused rdpmc() method rdpmc() is not used anywhere anymore, remove it. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250427092027.1598740-4-xin@zytor.com --- arch/x86/include/asm/msr.h | 7 ------- arch/x86/include/asm/paravirt.h | 7 ------- 2 files changed, 14 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index f5c0969fa3bb..07d45fc374ef 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -217,13 +217,6 @@ static inline int rdmsrq_safe(u32 msr, u64 *p) return err; } -#define rdpmc(counter, low, high) \ -do { \ - u64 _l = native_read_pmc((counter)); \ - (low) = (u32)_l; \ - (high) = (u32)(_l >> 32); \ -} while (0) - #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) #endif /* !CONFIG_PARAVIRT_XXL */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 86a77528792d..c4dedb984735 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -244,13 +244,6 @@ static inline u64 paravirt_read_pmc(int counter) return PVOP_CALL1(u64, cpu.read_pmc, counter); } -#define rdpmc(counter, low, high) \ -do { \ - u64 _l = paravirt_read_pmc(counter); \ - low = (u32)_l; \ - high = _l >> 32; \ -} while (0) - #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) From 7d9ccde56bc023f9a968db537963a029aa41d902 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:16 -0700 Subject: [PATCH 272/331] x86/msr: Rename rdpmcl() to rdpmc() Now that rdpmc() is gone, rdpmcl() is the sole PMC read helper, simply rename rdpmcl() to rdpmc(). Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250427092027.1598740-5-xin@zytor.com --- arch/x86/events/amd/uncore.c | 2 +- arch/x86/events/core.c | 2 +- arch/x86/events/intel/core.c | 4 ++-- arch/x86/events/intel/ds.c | 2 +- arch/x86/include/asm/msr.h | 2 +- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 12 ++++++------ 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 2a3259df619a..42c833cf9d98 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -108,7 +108,7 @@ static void amd_uncore_read(struct perf_event *event) if (hwc->event_base_rdpmc < 0) rdmsrq(hwc->event_base, new); else - rdpmcl(hwc->event_base_rdpmc, new); + rdpmc(hwc->event_base_rdpmc, new); local64_set(&hwc->prev_count, new); delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 7cfd3764a582..600c0b61fec0 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -135,7 +135,7 @@ u64 x86_perf_event_update(struct perf_event *event) */ prev_raw_count = local64_read(&hwc->prev_count); do { - rdpmcl(hwc->event_base_rdpmc, new_raw_count); + rdpmc(hwc->event_base_rdpmc, new_raw_count); } while (!local64_try_cmpxchg(&hwc->prev_count, &prev_raw_count, new_raw_count)); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 3d80ffb2c374..e8eec162da24 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2725,12 +2725,12 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end, if (!val) { /* read Fixed counter 3 */ - rdpmcl((3 | INTEL_PMC_FIXED_RDPMC_BASE), slots); + rdpmc((3 | INTEL_PMC_FIXED_RDPMC_BASE), slots); if (!slots) return 0; /* read PERF_METRICS */ - rdpmcl(INTEL_PMC_FIXED_RDPMC_METRICS, metrics); + rdpmc(INTEL_PMC_FIXED_RDPMC_METRICS, metrics); } else { slots = val[0]; metrics = val[1]; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 00a0a859f177..346db2f132d7 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2277,7 +2277,7 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count) WARN_ON(this_cpu_read(cpu_hw_events.enabled)); prev_raw_count = local64_read(&hwc->prev_count); - rdpmcl(hwc->event_base_rdpmc, new_raw_count); + rdpmc(hwc->event_base_rdpmc, new_raw_count); local64_set(&hwc->prev_count, new_raw_count); /* diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 07d45fc374ef..435a07bda3a1 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -217,7 +217,7 @@ static inline int rdmsrq_safe(u32 msr, u64 *p) return err; } -#define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) +#define rdpmc(counter, val) ((val) = native_read_pmc(counter)) #endif /* !CONFIG_PARAVIRT_XXL */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index c4dedb984735..faa0713553b1 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -244,7 +244,7 @@ static inline u64 paravirt_read_pmc(int counter) return PVOP_CALL1(u64, cpu.read_pmc, counter); } -#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) +#define rdpmc(counter, val) ((val) = paravirt_read_pmc(counter)) static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) { diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 26c354bdea07..15ff62d83bd8 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -1019,8 +1019,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * used in L1 cache, second to capture accurate value that does not * include cache misses incurred because of instruction loads. */ - rdpmcl(hit_pmcnum, hits_before); - rdpmcl(miss_pmcnum, miss_before); + rdpmc(hit_pmcnum, hits_before); + rdpmc(miss_pmcnum, miss_before); /* * From SDM: Performing back-to-back fast reads are not guaranteed * to be monotonic. @@ -1028,8 +1028,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * before proceeding. */ rmb(); - rdpmcl(hit_pmcnum, hits_before); - rdpmcl(miss_pmcnum, miss_before); + rdpmc(hit_pmcnum, hits_before); + rdpmc(miss_pmcnum, miss_before); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. @@ -1051,8 +1051,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * before proceeding. */ rmb(); - rdpmcl(hit_pmcnum, hits_after); - rdpmcl(miss_pmcnum, miss_after); + rdpmc(hit_pmcnum, hits_after); + rdpmc(miss_pmcnum, miss_after); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. From 795ada52875fe61469f635f226d19a4cd733d1e8 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:17 -0700 Subject: [PATCH 273/331] x86/msr: Convert the rdpmc() macro to an __always_inline function Functions offer type safety and better readability compared to macros. Additionally, always inline functions can match the performance of macros. Converting the rdpmc() macro into an always inline function is simple and straightforward, so just make the change. Moreover, the read result is now the returned value, further enhancing readability. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250427092027.1598740-6-xin@zytor.com --- arch/x86/events/amd/uncore.c | 2 +- arch/x86/events/core.c | 2 +- arch/x86/events/intel/core.c | 4 ++-- arch/x86/events/intel/ds.c | 2 +- arch/x86/include/asm/msr.h | 5 ++++- arch/x86/include/asm/paravirt.h | 4 +--- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 12 ++++++------ 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 42c833cf9d98..13c4cea545c5 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -108,7 +108,7 @@ static void amd_uncore_read(struct perf_event *event) if (hwc->event_base_rdpmc < 0) rdmsrq(hwc->event_base, new); else - rdpmc(hwc->event_base_rdpmc, new); + new = rdpmc(hwc->event_base_rdpmc); local64_set(&hwc->prev_count, new); delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 600c0b61fec0..bc92eba7631f 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -135,7 +135,7 @@ u64 x86_perf_event_update(struct perf_event *event) */ prev_raw_count = local64_read(&hwc->prev_count); do { - rdpmc(hwc->event_base_rdpmc, new_raw_count); + new_raw_count = rdpmc(hwc->event_base_rdpmc); } while (!local64_try_cmpxchg(&hwc->prev_count, &prev_raw_count, new_raw_count)); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index e8eec162da24..33f3fd2b8b88 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2725,12 +2725,12 @@ static u64 intel_update_topdown_event(struct perf_event *event, int metric_end, if (!val) { /* read Fixed counter 3 */ - rdpmc((3 | INTEL_PMC_FIXED_RDPMC_BASE), slots); + slots = rdpmc(3 | INTEL_PMC_FIXED_RDPMC_BASE); if (!slots) return 0; /* read PERF_METRICS */ - rdpmc(INTEL_PMC_FIXED_RDPMC_METRICS, metrics); + metrics = rdpmc(INTEL_PMC_FIXED_RDPMC_METRICS); } else { slots = val[0]; metrics = val[1]; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 346db2f132d7..4f52719cd2c0 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2277,7 +2277,7 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count) WARN_ON(this_cpu_read(cpu_hw_events.enabled)); prev_raw_count = local64_read(&hwc->prev_count); - rdpmc(hwc->event_base_rdpmc, new_raw_count); + new_raw_count = rdpmc(hwc->event_base_rdpmc); local64_set(&hwc->prev_count, new_raw_count); /* diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index 435a07bda3a1..fbeb313ccad2 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -217,7 +217,10 @@ static inline int rdmsrq_safe(u32 msr, u64 *p) return err; } -#define rdpmc(counter, val) ((val) = native_read_pmc(counter)) +static __always_inline u64 rdpmc(int counter) +{ + return native_read_pmc(counter); +} #endif /* !CONFIG_PARAVIRT_XXL */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index faa0713553b1..f272c4bd3d5b 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -239,13 +239,11 @@ static inline int rdmsrq_safe(unsigned msr, u64 *p) return err; } -static inline u64 paravirt_read_pmc(int counter) +static __always_inline u64 rdpmc(int counter) { return PVOP_CALL1(u64, cpu.read_pmc, counter); } -#define rdpmc(counter, val) ((val) = paravirt_read_pmc(counter)) - static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) { PVOP_VCALL2(cpu.alloc_ldt, ldt, entries); diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 15ff62d83bd8..61d762555a79 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -1019,8 +1019,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * used in L1 cache, second to capture accurate value that does not * include cache misses incurred because of instruction loads. */ - rdpmc(hit_pmcnum, hits_before); - rdpmc(miss_pmcnum, miss_before); + hits_before = rdpmc(hit_pmcnum); + miss_before = rdpmc(miss_pmcnum); /* * From SDM: Performing back-to-back fast reads are not guaranteed * to be monotonic. @@ -1028,8 +1028,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * before proceeding. */ rmb(); - rdpmc(hit_pmcnum, hits_before); - rdpmc(miss_pmcnum, miss_before); + hits_before = rdpmc(hit_pmcnum); + miss_before = rdpmc(miss_pmcnum); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. @@ -1051,8 +1051,8 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * before proceeding. */ rmb(); - rdpmc(hit_pmcnum, hits_after); - rdpmc(miss_pmcnum, miss_after); + hits_after = rdpmc(hit_pmcnum); + miss_after = rdpmc(miss_pmcnum); /* * Use LFENCE to ensure all previous instructions are retired * before proceeding. From 5afa4cf54518b26e18d7b7ce2e9d9724f9cb9324 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:18 -0700 Subject: [PATCH 274/331] x86/xen/msr: Return u64 consistently in Xen PMC xen_*_read functions The pv_ops PMC read API is defined as: u64 (*read_pmc)(int counter); But Xen PMC read functions return 'unsigned long long', make them return u64 consistently. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Acked-by: Peter Zijlstra (Intel) Cc: Stefano Stabellini Cc: Andy Lutomirski Cc: Brian Gerst Cc: Juergen Gross Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Kees Cook Cc: Josh Poimboeuf Cc: Uros Bizjak Link: https://lore.kernel.org/r/20250427092027.1598740-7-xin@zytor.com --- arch/x86/xen/pmu.c | 6 +++--- arch/x86/xen/xen-ops.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 3cb566d4aaad..3c01fbab7a2c 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -347,7 +347,7 @@ bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) return true; } -static unsigned long long xen_amd_read_pmc(int counter) +static u64 xen_amd_read_pmc(int counter) { struct xen_pmu_amd_ctxt *ctxt; uint64_t *counter_regs; @@ -367,7 +367,7 @@ static unsigned long long xen_amd_read_pmc(int counter) return counter_regs[counter]; } -static unsigned long long xen_intel_read_pmc(int counter) +static u64 xen_intel_read_pmc(int counter) { struct xen_pmu_intel_ctxt *ctxt; uint64_t *fixed_counters; @@ -397,7 +397,7 @@ static unsigned long long xen_intel_read_pmc(int counter) return arch_cntr_pair[counter].counter; } -unsigned long long xen_read_pmc(int counter) +u64 xen_read_pmc(int counter) { if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return xen_amd_read_pmc(counter); diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 25e318ef27d6..dc886c3cc24d 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -274,7 +274,7 @@ static inline void xen_pmu_finish(int cpu) {} bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err); bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err); int pmu_apic_update(uint32_t reg); -unsigned long long xen_read_pmc(int counter); +u64 xen_read_pmc(int counter); #ifdef CONFIG_SMP From 519be7da37b955164c59aacaee6d6ac89f4bbe15 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:19 -0700 Subject: [PATCH 275/331] x86/msr: Convert __wrmsr() uses to native_wrmsr{,q}() uses __wrmsr() is the lowest level MSR write API, with native_wrmsr() and native_wrmsrq() serving as higher-level wrappers around it: #define native_wrmsr(msr, low, high) \ __wrmsr(msr, low, high) #define native_wrmsrl(msr, val) \ __wrmsr((msr), (u32)((u64)(val)), \ (u32)((u64)(val) >> 32)) However, __wrmsr() continues to be utilized in various locations. MSR APIs are designed for different scenarios, such as native or pvops, with or without trace, and safe or non-safe. Unfortunately, the current MSR API names do not adequately reflect these factors, making it challenging to select the most appropriate API for various situations. To pave the way for improving MSR API names, convert __wrmsr() uses to native_wrmsr{,q}() to ensure consistent usage. Later, these APIs can be renamed to better reflect their implications, such as native or pvops, with or without trace, and safe or non-safe. No functional change intended. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-8-xin@zytor.com --- arch/x86/events/amd/brs.c | 2 +- arch/x86/include/asm/apic.h | 2 +- arch/x86/include/asm/msr.h | 6 ++++-- arch/x86/kernel/cpu/mce/core.c | 2 +- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 6 +++--- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index ec4e8a4cace4..3f5ecfd80d1e 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -44,7 +44,7 @@ static inline unsigned int brs_to(int idx) static __always_inline void set_debug_extn_cfg(u64 val) { /* bits[4:3] must always be set to 11b */ - __wrmsr(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3, val >> 32); + native_wrmsrq(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3); } static __always_inline u64 get_debug_extn_cfg(void) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 1c136f54651c..0174dd548327 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -214,7 +214,7 @@ static inline void native_apic_msr_write(u32 reg, u32 v) static inline void native_apic_msr_eoi(void) { - __wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0); + native_wrmsrq(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK); } static inline u32 native_apic_msr_read(u32 reg) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index fbeb313ccad2..e5f95a17124f 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -127,10 +127,12 @@ static inline u64 native_read_msr_safe(u32 msr, int *err) static inline void notrace native_write_msr(u32 msr, u32 low, u32 high) { - __wrmsr(msr, low, high); + u64 val = (u64)high << 32 | low; + + native_wrmsrq(msr, val); if (tracepoint_enabled(write_msr)) - do_trace_write_msr(msr, ((u64)high << 32 | low), 0); + do_trace_write_msr(msr, val, 0); } /* Can be uninlined because referenced by paravirt */ diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 7b9908cb5d69..96db2fd03e66 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1306,7 +1306,7 @@ static noinstr bool mce_check_crashing_cpu(void) } if (mcgstatus & MCG_STATUS_RIPV) { - __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); + native_wrmsrq(MSR_IA32_MCG_STATUS, 0); return true; } } diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 61d762555a79..6e5edd76086e 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -483,7 +483,7 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) * cache. */ saved_msr = __rdmsr(MSR_MISC_FEATURE_CONTROL); - __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + native_wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); closid_p = this_cpu_read(pqr_state.cur_closid); rmid_p = this_cpu_read(pqr_state.cur_rmid); mem_r = plr->kmem; @@ -495,7 +495,7 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) * pseudo-locked followed by reading of kernel memory to load it * into the cache. */ - __wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, plr->closid); + native_wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, plr->closid); /* * Cache was flushed earlier. Now access kernel memory to read it @@ -532,7 +532,7 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) * Critical section end: restore closid with capacity bitmask that * does not overlap with pseudo-locked region. */ - __wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, closid_p); + native_wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, closid_p); /* Re-enable the hardware prefetcher(s) */ wrmsrq(MSR_MISC_FEATURE_CONTROL, saved_msr); From ed56a309f7e1b05c15702ea1275172d507a606f4 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:20 -0700 Subject: [PATCH 276/331] x86/msr: Add the native_rdmsrq() helper __rdmsr() is the lowest-level primitive MSR read API, implemented in assembly code and returning an MSR value in a u64 integer, on top of which a convenience wrapper native_rdmsr() is defined to return an MSR value in two u32 integers. For some reason, native_rdmsrq() is not defined and __rdmsr() is directly used when it needs to return an MSR value in a u64 integer. Add the native_rdmsrq() helper, which is simply an alias of __rdmsr(), to make native_rdmsr() and native_rdmsrq() a pair of MSR read APIs. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-9-xin@zytor.com --- arch/x86/include/asm/msr.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index e5f95a17124f..adef3aeaee93 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -90,6 +90,11 @@ do { \ (void)((val2) = (u32)(__val >> 32)); \ } while (0) +static __always_inline u64 native_rdmsrq(u32 msr) +{ + return __rdmsr(msr); +} + #define native_wrmsr(msr, low, high) \ __wrmsr(msr, low, high) From 3204877d05ca17162270890e3f28552741a1b1a2 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:21 -0700 Subject: [PATCH 277/331] x86/msr: Convert __rdmsr() uses to native_rdmsrq() uses __rdmsr() is the lowest level MSR write API, with native_rdmsr() and native_rdmsrq() serving as higher-level wrappers around it. #define native_rdmsr(msr, val1, val2) \ do { \ u64 __val = __rdmsr((msr)); \ (void)((val1) = (u32)__val); \ (void)((val2) = (u32)(__val >> 32)); \ } while (0) static __always_inline u64 native_rdmsrq(u32 msr) { return __rdmsr(msr); } However, __rdmsr() continues to be utilized in various locations. MSR APIs are designed for different scenarios, such as native or pvops, with or without trace, and safe or non-safe. Unfortunately, the current MSR API names do not adequately reflect these factors, making it challenging to select the most appropriate API for various situations. To pave the way for improving MSR API names, convert __rdmsr() uses to native_rdmsrq() to ensure consistent usage. Later, these APIs can be renamed to better reflect their implications, such as native or pvops, with or without trace, and safe or non-safe. No functional change intended. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-10-xin@zytor.com --- arch/x86/coco/sev/core.c | 2 +- arch/x86/events/amd/brs.c | 2 +- arch/x86/hyperv/hv_vtl.c | 4 ++-- arch/x86/hyperv/ivm.c | 2 +- arch/x86/include/asm/mshyperv.h | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/mce/core.c | 4 ++-- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 2 +- arch/x86/kvm/vmx/vmx.c | 4 ++-- arch/x86/mm/mem_encrypt_identity.c | 4 ++-- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 85b16a0ee417..ff82151f7718 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -277,7 +277,7 @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) static inline u64 sev_es_rd_ghcb_msr(void) { - return __rdmsr(MSR_AMD64_SEV_ES_GHCB); + return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB); } static __always_inline void sev_es_wr_ghcb_msr(u64 val) diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index 3f5ecfd80d1e..06f35a6b58a5 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -49,7 +49,7 @@ static __always_inline void set_debug_extn_cfg(u64 val) static __always_inline u64 get_debug_extn_cfg(void) { - return __rdmsr(MSR_AMD_DBG_EXTN_CFG); + return native_rdmsrq(MSR_AMD_DBG_EXTN_CFG); } static bool __init amd_brs_detect(void) diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index 079b276e5f30..4580936dcb03 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -150,11 +150,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) input->vp_context.rip = rip; input->vp_context.rsp = rsp; input->vp_context.rflags = 0x0000000000000002; - input->vp_context.efer = __rdmsr(MSR_EFER); + input->vp_context.efer = native_rdmsrq(MSR_EFER); input->vp_context.cr0 = native_read_cr0(); input->vp_context.cr3 = __native_read_cr3(); input->vp_context.cr4 = native_read_cr4(); - input->vp_context.msr_cr_pat = __rdmsr(MSR_IA32_CR_PAT); + input->vp_context.msr_cr_pat = native_rdmsrq(MSR_IA32_CR_PAT); input->vp_context.idtr.limit = idt_ptr.size; input->vp_context.idtr.base = idt_ptr.address; input->vp_context.gdtr.limit = gdt_ptr.size; diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 8209de792388..09a165a3c41e 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -111,7 +111,7 @@ u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size) static inline u64 rd_ghcb_msr(void) { - return __rdmsr(MSR_AMD64_SEV_ES_GHCB); + return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB); } static inline void wr_ghcb_msr(u64 val) diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 15d00dace70f..778444310cfb 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -305,7 +305,7 @@ void hv_set_non_nested_msr(unsigned int reg, u64 value); static __always_inline u64 hv_raw_get_msr(unsigned int reg) { - return __rdmsr(reg); + return native_rdmsrq(reg); } #else /* CONFIG_HYPERV */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 079ded4eeb86..cefc99990bde 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -164,7 +164,7 @@ static void ppin_init(struct cpuinfo_x86 *c) /* Is the enable bit set? */ if (val & 2UL) { - c->ppin = __rdmsr(info->msr_ppin); + c->ppin = native_rdmsrq(info->msr_ppin); set_cpu_cap(c, info->feature); return; } diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 96db2fd03e66..e9b3c5d4a52e 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -121,7 +121,7 @@ void mce_prep_record_common(struct mce *m) { m->cpuid = cpuid_eax(1); m->cpuvendor = boot_cpu_data.x86_vendor; - m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); + m->mcgcap = native_rdmsrq(MSR_IA32_MCG_CAP); /* need the internal __ version to avoid deadlocks */ m->time = __ktime_get_real_seconds(); } @@ -1298,7 +1298,7 @@ static noinstr bool mce_check_crashing_cpu(void) (crashing_cpu != -1 && crashing_cpu != cpu)) { u64 mcgstatus; - mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); + mcgstatus = native_rdmsrq(MSR_IA32_MCG_STATUS); if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { if (mcgstatus & MCG_STATUS_LMCES) diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 6e5edd76086e..324bd4919300 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -482,7 +482,7 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) * the buffer and evict pseudo-locked memory read earlier from the * cache. */ - saved_msr = __rdmsr(MSR_MISC_FEATURE_CONTROL); + saved_msr = native_rdmsrq(MSR_MISC_FEATURE_CONTROL); native_wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); closid_p = this_cpu_read(pqr_state.cur_closid); rmid_p = this_cpu_read(pqr_state.cur_rmid); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d8412cfdb18e..63de5f6051e5 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -381,7 +381,7 @@ static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) if (!vmx->disable_fb_clear) return; - msr = __rdmsr(MSR_IA32_MCU_OPT_CTRL); + msr = native_rdmsrq(MSR_IA32_MCU_OPT_CTRL); msr |= FB_CLEAR_DIS; native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, msr); /* Cache the MSR value to avoid reading it later */ @@ -7308,7 +7308,7 @@ void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, return; if (flags & VMX_RUN_SAVE_SPEC_CTRL) - vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); + vmx->spec_ctrl = native_rdmsrq(MSR_IA32_SPEC_CTRL); /* * If the guest/host SPEC_CTRL values differ, restore the host value. diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 5eecdd92da10..25f6677e8575 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -526,7 +526,7 @@ void __head sme_enable(struct boot_params *bp) me_mask = 1UL << (ebx & 0x3f); /* Check the SEV MSR whether SEV or SME is enabled */ - RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV); + RIP_REL_REF(sev_status) = msr = native_rdmsrq(MSR_AMD64_SEV); feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT; /* @@ -557,7 +557,7 @@ void __head sme_enable(struct boot_params *bp) return; /* For SME, check the SYSCFG MSR */ - msr = __rdmsr(MSR_AMD64_SYSCFG); + msr = native_rdmsrq(MSR_AMD64_SYSCFG); if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) return; } From 0cb6f4128a7d8dca9c9bb45f9ecbddcd5cf377aa Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:22 -0700 Subject: [PATCH 278/331] x86/xen/msr: Remove calling native_{read,write}_msr{,_safe}() in pmu_msr_{read,write}() hpa found that pmu_msr_write() is actually a completely pointless function: https://lore.kernel.org/lkml/0ec48b84-d158-47c6-b14c-3563fd14bcc4@zytor.com/ all it does is shuffle some arguments, then calls pmu_msr_chk_emulated() and if it returns true AND the emulated flag is clear then does *exactly the same thing* that the calling code would have done if pmu_msr_write() itself had returned true. And pmu_msr_read() does the equivalent stupidity. Remove the calls to native_{read,write}_msr{,_safe}() within pmu_msr_{read,write}(). Instead reuse the existing calling code that decides whether to call native_{read,write}_msr{,_safe}() based on the return value from pmu_msr_{read,write}(). Consequently, eliminate the need to pass an error pointer to pmu_msr_{read,write}(). While at it, refactor pmu_msr_write() to take the MSR value as a u64 argument, replacing the current dual u32 arguments, because the dual u32 arguments were only used to call native_write_msr{,_safe}(), which has now been removed. Suggested-by: H. Peter Anvin (Intel) Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-11-xin@zytor.com --- arch/x86/xen/enlighten_pv.c | 8 ++++++-- arch/x86/xen/pmu.c | 27 ++++----------------------- arch/x86/xen/xen-ops.h | 4 ++-- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 8ddd9e535f99..530b59b0589c 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1091,7 +1091,7 @@ static u64 xen_do_read_msr(unsigned int msr, int *err) { u64 val = 0; /* Avoid uninitialized value for safe variant. */ - if (pmu_msr_read(msr, &val, err)) + if (pmu_msr_read_emulated(msr, &val)) return val; if (err) @@ -1133,6 +1133,8 @@ static void set_seg(unsigned int which, unsigned int low, unsigned int high, static void xen_do_write_msr(unsigned int msr, unsigned int low, unsigned int high, int *err) { + u64 val; + switch (msr) { case MSR_FS_BASE: set_seg(SEGBASE_FS, low, high, err); @@ -1159,7 +1161,9 @@ static void xen_do_write_msr(unsigned int msr, unsigned int low, break; default: - if (!pmu_msr_write(msr, low, high, err)) { + val = (u64)high << 32 | low; + + if (!pmu_msr_write_emulated(msr, val)) { if (err) *err = native_write_msr_safe(msr, low, high); else diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 3c01fbab7a2c..0062dbbd6661 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -314,37 +314,18 @@ static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read, return true; } -bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) +bool pmu_msr_read_emulated(u32 msr, u64 *val) { bool emulated; - if (!pmu_msr_chk_emulated(msr, val, true, &emulated)) - return false; - - if (!emulated) { - *val = err ? native_read_msr_safe(msr, err) - : native_read_msr(msr); - } - - return true; + return pmu_msr_chk_emulated(msr, val, true, &emulated) && emulated; } -bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) +bool pmu_msr_write_emulated(u32 msr, u64 val) { - uint64_t val = ((uint64_t)high << 32) | low; bool emulated; - if (!pmu_msr_chk_emulated(msr, &val, false, &emulated)) - return false; - - if (!emulated) { - if (err) - *err = native_write_msr_safe(msr, low, high); - else - native_write_msr(msr, low, high); - } - - return true; + return pmu_msr_chk_emulated(msr, &val, false, &emulated) && emulated; } static u64 xen_amd_read_pmc(int counter) diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index dc886c3cc24d..2de30618aa5e 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -271,8 +271,8 @@ void xen_pmu_finish(int cpu); static inline void xen_pmu_init(int cpu) {} static inline void xen_pmu_finish(int cpu) {} #endif -bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err); -bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err); +bool pmu_msr_read_emulated(u32 msr, u64 *val); +bool pmu_msr_write_emulated(u32 msr, u64 val); int pmu_apic_update(uint32_t reg); u64 xen_read_pmc(int counter); From f7998621db691b6fad4d84ad7d8ecc1a0a9b703f Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:23 -0700 Subject: [PATCH 279/331] x86/xen/msr: Remove pmu_msr_{read,write}() As pmu_msr_{read,write}() are now wrappers of pmu_msr_chk_emulated(), remove them and use pmu_msr_chk_emulated() directly. As pmu_msr_chk_emulated() could easily return false in the cases where it would set *emul to false, remove the "emul" argument and use the return value instead. While at it, convert the data type of MSR index to u32 in functions called in pmu_msr_chk_emulated(). Suggested-by: H. Peter Anvin (Intel) Suggested-by: Juergen Gross Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-12-xin@zytor.com --- arch/x86/xen/enlighten_pv.c | 15 ++++++++------- arch/x86/xen/pmu.c | 33 ++++++++------------------------- arch/x86/xen/xen-ops.h | 3 +-- 3 files changed, 17 insertions(+), 34 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 530b59b0589c..719370d36622 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1091,7 +1091,7 @@ static u64 xen_do_read_msr(unsigned int msr, int *err) { u64 val = 0; /* Avoid uninitialized value for safe variant. */ - if (pmu_msr_read_emulated(msr, &val)) + if (pmu_msr_chk_emulated(msr, &val, true)) return val; if (err) @@ -1163,12 +1163,13 @@ static void xen_do_write_msr(unsigned int msr, unsigned int low, default: val = (u64)high << 32 | low; - if (!pmu_msr_write_emulated(msr, val)) { - if (err) - *err = native_write_msr_safe(msr, low, high); - else - native_write_msr(msr, low, high); - } + if (pmu_msr_chk_emulated(msr, &val, false)) + return; + + if (err) + *err = native_write_msr_safe(msr, low, high); + else + native_write_msr(msr, low, high); } } diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 0062dbbd6661..043d72b060ed 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -129,7 +129,7 @@ static inline uint32_t get_fam15h_addr(u32 addr) return addr; } -static inline bool is_amd_pmu_msr(unsigned int msr) +static bool is_amd_pmu_msr(u32 msr) { if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) @@ -195,8 +195,7 @@ static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index) } } -static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type, - int index, bool is_read) +static bool xen_intel_pmu_emulate(u32 msr, u64 *val, int type, int index, bool is_read) { uint64_t *reg = NULL; struct xen_pmu_intel_ctxt *ctxt; @@ -258,7 +257,7 @@ static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type, return false; } -static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) +static bool xen_amd_pmu_emulate(u32 msr, u64 *val, bool is_read) { uint64_t *reg = NULL; int i, off = 0; @@ -299,33 +298,17 @@ static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) return false; } -static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read, - bool *emul) +bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read) { int type, index = 0; if (is_amd_pmu_msr(msr)) - *emul = xen_amd_pmu_emulate(msr, val, is_read); - else if (is_intel_pmu_msr(msr, &type, &index)) - *emul = xen_intel_pmu_emulate(msr, val, type, index, is_read); - else - return false; + return xen_amd_pmu_emulate(msr, val, is_read); - return true; -} + if (is_intel_pmu_msr(msr, &type, &index)) + return xen_intel_pmu_emulate(msr, val, type, index, is_read); -bool pmu_msr_read_emulated(u32 msr, u64 *val) -{ - bool emulated; - - return pmu_msr_chk_emulated(msr, val, true, &emulated) && emulated; -} - -bool pmu_msr_write_emulated(u32 msr, u64 val) -{ - bool emulated; - - return pmu_msr_chk_emulated(msr, &val, false, &emulated) && emulated; + return false; } static u64 xen_amd_read_pmc(int counter) diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 2de30618aa5e..090349baec09 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -271,8 +271,7 @@ void xen_pmu_finish(int cpu); static inline void xen_pmu_init(int cpu) {} static inline void xen_pmu_finish(int cpu) {} #endif -bool pmu_msr_read_emulated(u32 msr, u64 *val); -bool pmu_msr_write_emulated(u32 msr, u64 val); +bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read); int pmu_apic_update(uint32_t reg); u64 xen_read_pmc(int counter); From 2b7e25301c5418059b013c14af9022d892466398 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:24 -0700 Subject: [PATCH 280/331] x86/xen/msr: Remove the error pointer argument from set_seg() set_seg() is used to write the following MSRs on Xen: MSR_FS_BASE MSR_KERNEL_GS_BASE MSR_GS_BASE But none of these MSRs are written using any MSR write safe API. Therefore there is no need to pass an error pointer argument to set_seg() for returning an error code to be used in MSR safe APIs. Remove the error pointer argument. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-13-xin@zytor.com --- arch/x86/xen/enlighten_pv.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 719370d36622..97f7894d3136 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1111,17 +1111,11 @@ static u64 xen_do_read_msr(unsigned int msr, int *err) return val; } -static void set_seg(unsigned int which, unsigned int low, unsigned int high, - int *err) +static void set_seg(u32 which, u32 low, u32 high) { u64 base = ((u64)high << 32) | low; - if (HYPERVISOR_set_segment_base(which, base) == 0) - return; - - if (err) - *err = -EIO; - else + if (HYPERVISOR_set_segment_base(which, base)) WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base); } @@ -1137,15 +1131,15 @@ static void xen_do_write_msr(unsigned int msr, unsigned int low, switch (msr) { case MSR_FS_BASE: - set_seg(SEGBASE_FS, low, high, err); + set_seg(SEGBASE_FS, low, high); break; case MSR_KERNEL_GS_BASE: - set_seg(SEGBASE_GS_USER, low, high, err); + set_seg(SEGBASE_GS_USER, low, high); break; case MSR_GS_BASE: - set_seg(SEGBASE_GS_KERNEL, low, high, err); + set_seg(SEGBASE_GS_KERNEL, low, high); break; case MSR_STAR: From 0c2678efed6c6a2908dfe14af1e93a6bebc78e79 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:25 -0700 Subject: [PATCH 281/331] x86/pvops/msr: Refactor pv_cpu_ops.write_msr{,_safe}() An MSR value is represented as a 64-bit unsigned integer, with existing MSR instructions storing it in EDX:EAX as two 32-bit segments. The new immediate form MSR instructions, however, utilize a 64-bit general-purpose register to store the MSR value. To unify the usage of all MSR instructions, let the default MSR access APIs accept an MSR value as a single 64-bit argument instead of two 32-bit segments. The dual 32-bit APIs are still available as convenient wrappers over the APIs that handle an MSR value as a single 64-bit argument. The following illustrates the updated derivation of the MSR write APIs: __wrmsrq(u32 msr, u64 val) / \ / \ native_wrmsrq(msr, val) native_wrmsr(msr, low, high) | | native_write_msr(msr, val) / \ / \ wrmsrq(msr, val) wrmsr(msr, low, high) When CONFIG_PARAVIRT is enabled, wrmsrq() and wrmsr() are defined on top of paravirt_write_msr(): paravirt_write_msr(u32 msr, u64 val) / \ / \ wrmsrq(msr, val) wrmsr(msr, low, high) paravirt_write_msr() invokes cpu.write_msr(msr, val), an indirect layer of pv_ops MSR write call: If on native: cpu.write_msr = native_write_msr If on Xen: cpu.write_msr = xen_write_msr Therefore, refactor pv_cpu_ops.write_msr{_safe}() to accept an MSR value in a single u64 argument, replacing the current dual u32 arguments. No functional change intended. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-14-xin@zytor.com --- arch/x86/include/asm/msr.h | 35 ++++++++++++--------------- arch/x86/include/asm/paravirt.h | 27 +++++++++++---------- arch/x86/include/asm/paravirt_types.h | 4 +-- arch/x86/kernel/kvmclock.c | 2 +- arch/x86/kvm/svm/svm.c | 15 +++--------- arch/x86/xen/enlighten_pv.c | 30 +++++++++-------------- 6 files changed, 46 insertions(+), 67 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index adef3aeaee93..b2440765ba04 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -75,12 +75,12 @@ static __always_inline u64 __rdmsr(u32 msr) return EAX_EDX_VAL(val, low, high); } -static __always_inline void __wrmsr(u32 msr, u32 low, u32 high) +static __always_inline void __wrmsrq(u32 msr, u64 val) { asm volatile("1: wrmsr\n" "2:\n" _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) - : : "c" (msr), "a"(low), "d" (high) : "memory"); + : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32)) : "memory"); } #define native_rdmsr(msr, val1, val2) \ @@ -96,11 +96,10 @@ static __always_inline u64 native_rdmsrq(u32 msr) } #define native_wrmsr(msr, low, high) \ - __wrmsr(msr, low, high) + __wrmsrq((msr), (u64)(high) << 32 | (low)) #define native_wrmsrq(msr, val) \ - __wrmsr((msr), (u32)((u64)(val)), \ - (u32)((u64)(val) >> 32)) + __wrmsrq((msr), (val)) static inline u64 native_read_msr(u32 msr) { @@ -129,11 +128,8 @@ static inline u64 native_read_msr_safe(u32 msr, int *err) } /* Can be uninlined because referenced by paravirt */ -static inline void notrace -native_write_msr(u32 msr, u32 low, u32 high) +static inline void notrace native_write_msr(u32 msr, u64 val) { - u64 val = (u64)high << 32 | low; - native_wrmsrq(msr, val); if (tracepoint_enabled(write_msr)) @@ -141,8 +137,7 @@ native_write_msr(u32 msr, u32 low, u32 high) } /* Can be uninlined because referenced by paravirt */ -static inline int notrace -native_write_msr_safe(u32 msr, u32 low, u32 high) +static inline int notrace native_write_msr_safe(u32 msr, u64 val) { int err; @@ -150,10 +145,10 @@ native_write_msr_safe(u32 msr, u32 low, u32 high) "2:\n\t" _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err]) : [err] "=a" (err) - : "c" (msr), "0" (low), "d" (high) + : "c" (msr), "0" ((u32)val), "d" ((u32)(val >> 32)) : "memory"); if (tracepoint_enabled(write_msr)) - do_trace_write_msr(msr, ((u64)high << 32 | low), err); + do_trace_write_msr(msr, val, err); return err; } @@ -189,7 +184,7 @@ do { \ static inline void wrmsr(u32 msr, u32 low, u32 high) { - native_write_msr(msr, low, high); + native_write_msr(msr, (u64)high << 32 | low); } #define rdmsrq(msr, val) \ @@ -197,13 +192,13 @@ static inline void wrmsr(u32 msr, u32 low, u32 high) static inline void wrmsrq(u32 msr, u64 val) { - native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); + native_write_msr(msr, val); } /* wrmsr with exception handling */ -static inline int wrmsr_safe(u32 msr, u32 low, u32 high) +static inline int wrmsrq_safe(u32 msr, u64 val) { - return native_write_msr_safe(msr, low, high); + return native_write_msr_safe(msr, val); } /* rdmsr with exception handling */ @@ -247,11 +242,11 @@ static __always_inline void wrmsrns(u32 msr, u64 val) } /* - * 64-bit version of wrmsr_safe(): + * Dual u32 version of wrmsrq_safe(): */ -static inline int wrmsrq_safe(u32 msr, u64 val) +static inline int wrmsr_safe(u32 msr, u32 low, u32 high) { - return wrmsr_safe(msr, (u32)val, (u32)(val >> 32)); + return wrmsrq_safe(msr, (u64)high << 32 | low); } struct msr __percpu *msrs_alloc(void); diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index f272c4bd3d5b..edf23bde367e 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -180,10 +180,9 @@ static inline u64 paravirt_read_msr(unsigned msr) return PVOP_CALL1(u64, cpu.read_msr, msr); } -static inline void paravirt_write_msr(unsigned msr, - unsigned low, unsigned high) +static inline void paravirt_write_msr(u32 msr, u64 val) { - PVOP_VCALL3(cpu.write_msr, msr, low, high); + PVOP_VCALL2(cpu.write_msr, msr, val); } static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) @@ -191,10 +190,9 @@ static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err); } -static inline int paravirt_write_msr_safe(unsigned msr, - unsigned low, unsigned high) +static inline int paravirt_write_msr_safe(u32 msr, u64 val) { - return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high); + return PVOP_CALL2(int, cpu.write_msr_safe, msr, val); } #define rdmsr(msr, val1, val2) \ @@ -204,22 +202,25 @@ do { \ val2 = _l >> 32; \ } while (0) -#define wrmsr(msr, val1, val2) \ -do { \ - paravirt_write_msr(msr, val1, val2); \ -} while (0) +static __always_inline void wrmsr(u32 msr, u32 low, u32 high) +{ + paravirt_write_msr(msr, (u64)high << 32 | low); +} #define rdmsrq(msr, val) \ do { \ val = paravirt_read_msr(msr); \ } while (0) -static inline void wrmsrq(unsigned msr, u64 val) +static inline void wrmsrq(u32 msr, u64 val) { - wrmsr(msr, (u32)val, (u32)(val>>32)); + paravirt_write_msr(msr, val); } -#define wrmsr_safe(msr, a, b) paravirt_write_msr_safe(msr, a, b) +static inline int wrmsrq_safe(u32 msr, u64 val) +{ + return paravirt_write_msr_safe(msr, val); +} /* rdmsr with exception handling */ #define rdmsr_safe(msr, a, b) \ diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 631c306ce1ff..78777b78da12 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -92,14 +92,14 @@ struct pv_cpu_ops { /* Unsafe MSR operations. These will warn or panic on failure. */ u64 (*read_msr)(unsigned int msr); - void (*write_msr)(unsigned int msr, unsigned low, unsigned high); + void (*write_msr)(u32 msr, u64 val); /* * Safe MSR operations. * read sets err to 0 or -EIO. write returns 0 or -EIO. */ u64 (*read_msr_safe)(unsigned int msr, int *err); - int (*write_msr_safe)(unsigned int msr, unsigned low, unsigned high); + int (*write_msr_safe)(u32 msr, u64 val); u64 (*read_pmc)(int counter); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 0af797930ccb..ca0a49eeac4a 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -196,7 +196,7 @@ static void kvm_setup_secondary_clock(void) void kvmclock_disable(void) { if (msr_kvm_system_time) - native_write_msr(msr_kvm_system_time, 0, 0); + native_write_msr(msr_kvm_system_time, 0); } static void __init kvmclock_init_mem(void) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c23f620989ed..131f4852c6bb 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -476,7 +476,6 @@ static void svm_inject_exception(struct kvm_vcpu *vcpu) static void svm_init_erratum_383(void) { - u32 low, high; int err; u64 val; @@ -490,10 +489,7 @@ static void svm_init_erratum_383(void) val |= (1ULL << 47); - low = lower_32_bits(val); - high = upper_32_bits(val); - - native_write_msr_safe(MSR_AMD64_DC_CFG, low, high); + native_write_msr_safe(MSR_AMD64_DC_CFG, val); erratum_383_found = true; } @@ -2168,17 +2164,12 @@ static bool is_erratum_383(void) /* Clear MCi_STATUS registers */ for (i = 0; i < 6; ++i) - native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0); + native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0); value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); if (!err) { - u32 low, high; - value &= ~(1ULL << 2); - low = lower_32_bits(value); - high = upper_32_bits(value); - - native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high); + native_write_msr_safe(MSR_IA32_MCG_STATUS, value); } /* Flush tlb to evict multi-match entries */ diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 97f7894d3136..4fbe0bd393a1 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1111,10 +1111,8 @@ static u64 xen_do_read_msr(unsigned int msr, int *err) return val; } -static void set_seg(u32 which, u32 low, u32 high) +static void set_seg(u32 which, u64 base) { - u64 base = ((u64)high << 32) | low; - if (HYPERVISOR_set_segment_base(which, base)) WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base); } @@ -1124,22 +1122,19 @@ static void set_seg(u32 which, u32 low, u32 high) * With err == NULL write_msr() semantics are selected. * Supplying an err pointer requires err to be pre-initialized with 0. */ -static void xen_do_write_msr(unsigned int msr, unsigned int low, - unsigned int high, int *err) +static void xen_do_write_msr(u32 msr, u64 val, int *err) { - u64 val; - switch (msr) { case MSR_FS_BASE: - set_seg(SEGBASE_FS, low, high); + set_seg(SEGBASE_FS, val); break; case MSR_KERNEL_GS_BASE: - set_seg(SEGBASE_GS_USER, low, high); + set_seg(SEGBASE_GS_USER, val); break; case MSR_GS_BASE: - set_seg(SEGBASE_GS_KERNEL, low, high); + set_seg(SEGBASE_GS_KERNEL, val); break; case MSR_STAR: @@ -1155,15 +1150,13 @@ static void xen_do_write_msr(unsigned int msr, unsigned int low, break; default: - val = (u64)high << 32 | low; - if (pmu_msr_chk_emulated(msr, &val, false)) return; if (err) - *err = native_write_msr_safe(msr, low, high); + *err = native_write_msr_safe(msr, val); else - native_write_msr(msr, low, high); + native_write_msr(msr, val); } } @@ -1172,12 +1165,11 @@ static u64 xen_read_msr_safe(unsigned int msr, int *err) return xen_do_read_msr(msr, err); } -static int xen_write_msr_safe(unsigned int msr, unsigned int low, - unsigned int high) +static int xen_write_msr_safe(u32 msr, u64 val) { int err = 0; - xen_do_write_msr(msr, low, high, &err); + xen_do_write_msr(msr, val, &err); return err; } @@ -1189,11 +1181,11 @@ static u64 xen_read_msr(unsigned int msr) return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL); } -static void xen_write_msr(unsigned int msr, unsigned low, unsigned high) +static void xen_write_msr(u32 msr, u64 val) { int err; - xen_do_write_msr(msr, low, high, xen_msr_safe ? &err : NULL); + xen_do_write_msr(msr, val, xen_msr_safe ? &err : NULL); } /* This is called once we have the cpu_possible_mask */ From 444b46a128ccd0883f83ffa2e6b4a1f64ea4ca1c Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:26 -0700 Subject: [PATCH 282/331] x86/msr: Replace wrmsr(msr, low, 0) with wrmsrq(msr, low) The third argument in wrmsr(msr, low, 0) is unnecessary. Instead, use wrmsrq(msr, low), which automatically sets the higher 32 bits of the MSR value to 0. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-15-xin@zytor.com --- arch/x86/hyperv/hv_apic.c | 6 +++--- arch/x86/include/asm/apic.h | 2 +- arch/x86/include/asm/switch_to.h | 2 +- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/common.c | 8 ++++---- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 4 ++-- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 2 +- arch/x86/kernel/cpu/umwait.c | 4 ++-- arch/x86/kernel/kvm.c | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index a079a1427091..bfde0a3498b9 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -76,10 +76,10 @@ static void hv_apic_write(u32 reg, u32 val) { switch (reg) { case APIC_EOI: - wrmsr(HV_X64_MSR_EOI, val, 0); + wrmsrq(HV_X64_MSR_EOI, val); break; case APIC_TASKPRI: - wrmsr(HV_X64_MSR_TPR, val, 0); + wrmsrq(HV_X64_MSR_TPR, val); break; default: native_apic_mem_write(reg, val); @@ -93,7 +93,7 @@ static void hv_apic_eoi_write(void) if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1)) return; - wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0); + wrmsrq(HV_X64_MSR_EOI, APIC_EOI_ACK); } static bool cpu_is_self(int cpu) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 0174dd548327..68e10e30fe9b 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -209,7 +209,7 @@ static inline void native_apic_msr_write(u32 reg, u32 v) reg == APIC_LVR) return; - wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0); + wrmsrq(APIC_BASE_MSR + (reg >> 4), v); } static inline void native_apic_msr_eoi(void) diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h index 4f21df7af715..499b1c15cc8b 100644 --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -61,7 +61,7 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread) return; this_cpu_write(cpu_tss_rw.x86_tss.ss1, thread->sysenter_cs); - wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); + wrmsrq(MSR_IA32_SYSENTER_CS, thread->sysenter_cs); } #endif diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index e153cd1d36fe..d8365c4d9057 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1207,7 +1207,7 @@ void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr) if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask) return; - wrmsr(amd_msr_dr_addr_masks[dr], mask, 0); + wrmsrq(amd_msr_dr_addr_masks[dr], mask); per_cpu(amd_dr_addr_mask, cpu)[dr] = mask; } diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index cefc99990bde..ef9751d577c3 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1982,9 +1982,9 @@ void enable_sep_cpu(void) */ tss->x86_tss.ss1 = __KERNEL_CS; - wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0); - wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0); - wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0); + wrmsrq(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1); + wrmsrq(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1)); + wrmsrq(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32); put_cpu(); } @@ -2198,7 +2198,7 @@ static inline void setup_getcpu(int cpu) struct desc_struct d = { }; if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID)) - wrmsr(MSR_TSC_AUX, cpudata, 0); + wrmsrq(MSR_TSC_AUX, cpudata); /* Store CPU and node number in limit. */ d.limit0 = cpudata; diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 324bd4919300..1190c48a16b2 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -905,7 +905,7 @@ int resctrl_arch_measure_cycles_lat_fn(void *_plr) * Disable hardware prefetchers. */ rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); - wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); mem_r = READ_ONCE(plr->kmem); /* * Dummy execute of the time measurement to load the needed @@ -1001,7 +1001,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, * Disable hardware prefetchers. */ rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); - wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); /* Initialize rest of local variables */ /* diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index f724b8f6a6a4..c85ace29ea3a 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1708,7 +1708,7 @@ void resctrl_arch_mon_event_config_write(void *_config_info) pr_warn_once("Invalid event id %d\n", config_info->evtid); return; } - wrmsr(MSR_IA32_EVT_CFG_BASE + index, config_info->mon_config, 0); + wrmsrq(MSR_IA32_EVT_CFG_BASE + index, config_info->mon_config); } static void mbm_config_write_domain(struct rdt_resource *r, diff --git a/arch/x86/kernel/cpu/umwait.c b/arch/x86/kernel/cpu/umwait.c index 0050eae153bb..933fcd7ff250 100644 --- a/arch/x86/kernel/cpu/umwait.c +++ b/arch/x86/kernel/cpu/umwait.c @@ -33,7 +33,7 @@ static DEFINE_MUTEX(umwait_lock); static void umwait_update_control_msr(void * unused) { lockdep_assert_irqs_disabled(); - wrmsr(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached), 0); + wrmsrq(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached)); } /* @@ -71,7 +71,7 @@ static int umwait_cpu_offline(unsigned int cpu) * the original control MSR value in umwait_init(). So there * is no race condition here. */ - wrmsr(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached, 0); + wrmsrq(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached); return 0; } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 27f7192e1c61..f3642226e0a5 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -400,7 +400,7 @@ static void kvm_disable_steal_time(void) if (!has_steal_clock) return; - wrmsr(MSR_KVM_STEAL_TIME, 0, 0); + wrmsrq(MSR_KVM_STEAL_TIME, 0); } static u64 kvm_steal_clock(int cpu) From 502ad6e5a6196840976c4c84b2ea2f9769942fbe Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sun, 27 Apr 2025 02:20:27 -0700 Subject: [PATCH 283/331] x86/msr: Change the function type of native_read_msr_safe() Modify the function type of native_read_msr_safe() to: int native_read_msr_safe(u32 msr, u64 *val) This change makes the function return an error code instead of the MSR value, aligning it with the type of native_write_msr_safe(). Consequently, their callers can check the results in the same way. While at it, convert leftover MSR data type "unsigned int" to u32. Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Brian Gerst Cc: David Woodhouse Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Juergen Gross Cc: Kees Cook Cc: Linus Torvalds Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Stefano Stabellini Cc: Uros Bizjak Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20250427092027.1598740-16-xin@zytor.com --- arch/x86/include/asm/msr.h | 21 +++++++++++---------- arch/x86/include/asm/paravirt.h | 19 ++++++++----------- arch/x86/include/asm/paravirt_types.h | 6 +++--- arch/x86/kvm/svm/svm.c | 19 +++++++------------ arch/x86/xen/enlighten_pv.c | 13 ++++++++----- arch/x86/xen/pmu.c | 14 ++++++++------ 6 files changed, 45 insertions(+), 47 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index b2440765ba04..a9ce56fc8785 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -113,18 +113,22 @@ static inline u64 native_read_msr(u32 msr) return val; } -static inline u64 native_read_msr_safe(u32 msr, int *err) +static inline int native_read_msr_safe(u32 msr, u64 *p) { + int err; EAX_EDX_DECLARE_ARGS(val, low, high); asm volatile("1: rdmsr ; xor %[err],%[err]\n" "2:\n\t" _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err]) - : [err] "=r" (*err), EAX_EDX_RET(val, low, high) + : [err] "=r" (err), EAX_EDX_RET(val, low, high) : "c" (msr)); if (tracepoint_enabled(read_msr)) - do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); - return EAX_EDX_VAL(val, low, high); + do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), err); + + *p = EAX_EDX_VAL(val, low, high); + + return err; } /* Can be uninlined because referenced by paravirt */ @@ -204,8 +208,8 @@ static inline int wrmsrq_safe(u32 msr, u64 val) /* rdmsr with exception handling */ #define rdmsr_safe(msr, low, high) \ ({ \ - int __err; \ - u64 __val = native_read_msr_safe((msr), &__err); \ + u64 __val; \ + int __err = native_read_msr_safe((msr), &__val); \ (*low) = (u32)__val; \ (*high) = (u32)(__val >> 32); \ __err; \ @@ -213,10 +217,7 @@ static inline int wrmsrq_safe(u32 msr, u64 val) static inline int rdmsrq_safe(u32 msr, u64 *p) { - int err; - - *p = native_read_msr_safe(msr, &err); - return err; + return native_read_msr_safe(msr, p); } static __always_inline u64 rdpmc(int counter) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index edf23bde367e..03f680d1057a 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -175,7 +175,7 @@ static inline void __write_cr4(unsigned long x) PVOP_VCALL1(cpu.write_cr4, x); } -static inline u64 paravirt_read_msr(unsigned msr) +static inline u64 paravirt_read_msr(u32 msr) { return PVOP_CALL1(u64, cpu.read_msr, msr); } @@ -185,9 +185,9 @@ static inline void paravirt_write_msr(u32 msr, u64 val) PVOP_VCALL2(cpu.write_msr, msr, val); } -static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) +static inline int paravirt_read_msr_safe(u32 msr, u64 *val) { - return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err); + return PVOP_CALL2(int, cpu.read_msr_safe, msr, val); } static inline int paravirt_write_msr_safe(u32 msr, u64 val) @@ -225,19 +225,16 @@ static inline int wrmsrq_safe(u32 msr, u64 val) /* rdmsr with exception handling */ #define rdmsr_safe(msr, a, b) \ ({ \ - int _err; \ - u64 _l = paravirt_read_msr_safe(msr, &_err); \ + u64 _l; \ + int _err = paravirt_read_msr_safe((msr), &_l); \ (*a) = (u32)_l; \ - (*b) = _l >> 32; \ + (*b) = (u32)(_l >> 32); \ _err; \ }) -static inline int rdmsrq_safe(unsigned msr, u64 *p) +static __always_inline int rdmsrq_safe(u32 msr, u64 *p) { - int err; - - *p = paravirt_read_msr_safe(msr, &err); - return err; + return paravirt_read_msr_safe(msr, p); } static __always_inline u64 rdpmc(int counter) diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 78777b78da12..b08b9d3122d6 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -91,14 +91,14 @@ struct pv_cpu_ops { unsigned int *ecx, unsigned int *edx); /* Unsafe MSR operations. These will warn or panic on failure. */ - u64 (*read_msr)(unsigned int msr); + u64 (*read_msr)(u32 msr); void (*write_msr)(u32 msr, u64 val); /* * Safe MSR operations. - * read sets err to 0 or -EIO. write returns 0 or -EIO. + * Returns 0 or -EIO. */ - u64 (*read_msr_safe)(unsigned int msr, int *err); + int (*read_msr_safe)(u32 msr, u64 *val); int (*write_msr_safe)(u32 msr, u64 val); u64 (*read_pmc)(int counter); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 131f4852c6bb..4c2a843780bf 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -476,15 +476,13 @@ static void svm_inject_exception(struct kvm_vcpu *vcpu) static void svm_init_erratum_383(void) { - int err; u64 val; if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) return; /* Use _safe variants to not break nested virtualization */ - val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err); - if (err) + if (native_read_msr_safe(MSR_AMD64_DC_CFG, &val)) return; val |= (1ULL << 47); @@ -649,13 +647,12 @@ static int svm_enable_virtualization_cpu(void) * erratum is present everywhere). */ if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) { - uint64_t len, status = 0; + u64 len, status = 0; int err; - len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err); + err = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &len); if (!err) - status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, - &err); + err = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, &status); if (err) osvw_status = osvw_len = 0; @@ -2146,14 +2143,13 @@ static int ac_interception(struct kvm_vcpu *vcpu) static bool is_erratum_383(void) { - int err, i; + int i; u64 value; if (!erratum_383_found) return false; - value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err); - if (err) + if (native_read_msr_safe(MSR_IA32_MC0_STATUS, &value)) return false; /* Bit 62 may or may not be set for this mce */ @@ -2166,8 +2162,7 @@ static bool is_erratum_383(void) for (i = 0; i < 6; ++i) native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0); - value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err); - if (!err) { + if (!native_read_msr_safe(MSR_IA32_MCG_STATUS, &value)) { value &= ~(1ULL << 2); native_write_msr_safe(MSR_IA32_MCG_STATUS, value); } diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 4fbe0bd393a1..3be38350f044 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1087,7 +1087,7 @@ static void xen_write_cr4(unsigned long cr4) native_write_cr4(cr4); } -static u64 xen_do_read_msr(unsigned int msr, int *err) +static u64 xen_do_read_msr(u32 msr, int *err) { u64 val = 0; /* Avoid uninitialized value for safe variant. */ @@ -1095,7 +1095,7 @@ static u64 xen_do_read_msr(unsigned int msr, int *err) return val; if (err) - val = native_read_msr_safe(msr, err); + *err = native_read_msr_safe(msr, &val); else val = native_read_msr(msr); @@ -1160,9 +1160,12 @@ static void xen_do_write_msr(u32 msr, u64 val, int *err) } } -static u64 xen_read_msr_safe(unsigned int msr, int *err) +static int xen_read_msr_safe(u32 msr, u64 *val) { - return xen_do_read_msr(msr, err); + int err; + + *val = xen_do_read_msr(msr, &err); + return err; } static int xen_write_msr_safe(u32 msr, u64 val) @@ -1174,7 +1177,7 @@ static int xen_write_msr_safe(u32 msr, u64 val) return err; } -static u64 xen_read_msr(unsigned int msr) +static u64 xen_read_msr(u32 msr) { int err; diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 043d72b060ed..8f89ce0b67e3 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -319,11 +319,12 @@ static u64 xen_amd_read_pmc(int counter) uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { - uint32_t msr; - int err; + u32 msr; + u64 val; msr = amd_counters_base + (counter * amd_msr_step); - return native_read_msr_safe(msr, &err); + native_read_msr_safe(msr, &val); + return val; } ctxt = &xenpmu_data->pmu.c.amd; @@ -340,15 +341,16 @@ static u64 xen_intel_read_pmc(int counter) uint8_t xenpmu_flags = get_xenpmu_flags(); if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { - uint32_t msr; - int err; + u32 msr; + u64 val; if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff); else msr = MSR_IA32_PERFCTR0 + counter; - return native_read_msr_safe(msr, &err); + native_read_msr_safe(msr, &val); + return val; } ctxt = &xenpmu_data->pmu.c.intel; From 4804f5ad5d63cf7ddad148132a3ecea11410dfa9 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Fri, 2 May 2025 09:33:57 +0700 Subject: [PATCH 284/331] x86/cpu: Add "Old Microcode" docs to hw-vuln toctree Sphinx reports missing toctree entry warning: Documentation/admin-guide/hw-vuln/old_microcode.rst: WARNING: document isn't included in any toctree Add entry for "Old Microcode" docs to fix the warning. Fixes: 4e2c719782a847 ("x86/cpu: Help users notice when running old Intel microcode") Signed-off-by: Bagas Sanjaya Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250502023358.14846-1-bagasdotme%40gmail.com --- Documentation/admin-guide/hw-vuln/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/admin-guide/hw-vuln/index.rst b/Documentation/admin-guide/hw-vuln/index.rst index 451874b8135d..cf1511145927 100644 --- a/Documentation/admin-guide/hw-vuln/index.rst +++ b/Documentation/admin-guide/hw-vuln/index.rst @@ -23,3 +23,4 @@ are configurable at compile, boot or run time. gather_data_sampling reg-file-data-sampling rsb + old_microcode From 730faa15a069f4025a0f8c2a5244c3067da7ecbe Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:38:30 +0200 Subject: [PATCH 285/331] x86/fpu: Simplify the switch_fpu_prepare() + switch_fpu_finish() logic Now that switch_fpu_finish() doesn't load the FPU state, it makes more sense to fold it into switch_fpu_prepare() renamed to switch_fpu(), and more importantly, use the "prev_p" task as a target for TIF_NEED_FPU_LOAD. It doesn't make any sense to delay set_tsk_thread_flag(TIF_NEED_FPU_LOAD) until "prev_p" is scheduled again. There is no worry about the very first context switch, fpu_clone() must always set TIF_NEED_FPU_LOAD. Also, shift the test_tsk_thread_flag(TIF_NEED_FPU_LOAD) from the callers to switch_fpu(). Note that the "PF_KTHREAD | PF_USER_WORKER" check can be removed but this deserves a separate patch which can change more functions, say, kernel_fpu_begin_mask(). Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143830.GA8982@redhat.com --- arch/x86/include/asm/fpu/sched.h | 34 +++++++++----------------------- arch/x86/kernel/process_32.c | 5 +---- arch/x86/kernel/process_64.c | 5 +---- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index 5fd12634bcc4..c060549c6c94 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -18,31 +18,25 @@ extern void fpu_flush_thread(void); /* * FPU state switching for scheduling. * - * This is a two-stage process: + * switch_fpu() saves the old state and sets TIF_NEED_FPU_LOAD if + * TIF_NEED_FPU_LOAD is not set. This is done within the context + * of the old process. * - * - switch_fpu_prepare() saves the old state. - * This is done within the context of the old process. - * - * - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state - * will get loaded on return to userspace, or when the kernel needs it. - * - * If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers - * are saved in the current thread's FPU register state. - * - * If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not - * hold current()'s FPU registers. It is required to load the + * Once TIF_NEED_FPU_LOAD is set, it is required to load the * registers before returning to userland or using the content * otherwise. * * The FPU context is only stored/restored for a user task and * PF_KTHREAD is used to distinguish between kernel and user threads. */ -static inline void switch_fpu_prepare(struct task_struct *old, int cpu) +static inline void switch_fpu(struct task_struct *old, int cpu) { - if (cpu_feature_enabled(X86_FEATURE_FPU) && + if (!test_tsk_thread_flag(old, TIF_NEED_FPU_LOAD) && + cpu_feature_enabled(X86_FEATURE_FPU) && !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) { struct fpu *old_fpu = x86_task_fpu(old); + set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD); save_fpregs_to_fpstate(old_fpu); /* * The save operation preserved register state, so the @@ -50,7 +44,7 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu) * current CPU number in @old_fpu, so the next return * to user space can avoid the FPU register restore * when is returns on the same CPU and still owns the - * context. + * context. See fpregs_restore_userregs(). */ old_fpu->last_cpu = cpu; @@ -58,14 +52,4 @@ static inline void switch_fpu_prepare(struct task_struct *old, int cpu) } } -/* - * Delay loading of the complete FPU state until the return to userland. - * PKRU is handled separately. - */ -static inline void switch_fpu_finish(struct task_struct *new) -{ - if (cpu_feature_enabled(X86_FEATURE_FPU)) - set_tsk_thread_flag(new, TIF_NEED_FPU_LOAD); -} - #endif /* _ASM_X86_FPU_SCHED_H */ diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 4636ef359973..9bd4fa694da5 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -160,8 +160,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */ - if (!test_tsk_thread_flag(prev_p, TIF_NEED_FPU_LOAD)) - switch_fpu_prepare(prev_p, cpu); + switch_fpu(prev_p, cpu); /* * Save away %gs. No need to save %fs, as it was saved on the @@ -208,8 +207,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) raw_cpu_write(current_task, next_p); - switch_fpu_finish(next_p); - /* Load the Intel cache allocation PQR MSR. */ resctrl_sched_in(next_p); diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 7196ca7048be..d55310d3133c 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -616,8 +616,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && this_cpu_read(hardirq_stack_inuse)); - if (!test_tsk_thread_flag(prev_p, TIF_NEED_FPU_LOAD)) - switch_fpu_prepare(prev_p, cpu); + switch_fpu(prev_p, cpu); /* We must save %fs and %gs before load_TLS() because * %fs and %gs may be cleared by load_TLS(). @@ -671,8 +670,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) raw_cpu_write(current_task, next_p); raw_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p)); - switch_fpu_finish(next_p); - /* Reload sp0. */ update_task_stack(next_p); From 392bbe11c7cf90e65cba32e90af3b969a981c4fe Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:38:37 +0200 Subject: [PATCH 286/331] x86/fpu: Remove x86_init_fpu It is not actually used after: 55bc30f2e34d ("x86/fpu: Remove the thread::fpu pointer") Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143837.GA8985@redhat.com --- arch/x86/kernel/fpu/init.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 2d9b5e677559..6bb3e35c40e2 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -71,13 +71,9 @@ static bool __init fpu__probe_without_cpuid(void) return fsw == 0 && (fcw & 0x103f) == 0x003f; } -static struct fpu x86_init_fpu __attribute__ ((aligned (64))) __read_mostly; - static void __init fpu__init_system_early_generic(void) { - fpstate_reset(&x86_init_fpu); set_thread_flag(TIF_NEED_FPU_LOAD); - x86_init_fpu.last_cpu = -1; if (!boot_cpu_has(X86_FEATURE_CPUID) && !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { From 8e269c030ecafbfebf4f55e24fb336fd7b489708 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:38:43 +0200 Subject: [PATCH 287/331] x86/fpu: Remove DEFINE_EVENT(x86_fpu, x86_fpu_copy_src) trace_x86_fpu_copy_src() has no users after: 22aafe3bcb67 ("x86/fpu: Remove init_task FPU state dependencies, add debugging warning for PF_KTHREAD tasks") Remove the event. Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143843.GA8989@redhat.com --- arch/x86/include/asm/trace/fpu.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/x86/include/asm/trace/fpu.h b/arch/x86/include/asm/trace/fpu.h index 4645a6334063..0454d5e60e5d 100644 --- a/arch/x86/include/asm/trace/fpu.h +++ b/arch/x86/include/asm/trace/fpu.h @@ -74,11 +74,6 @@ DEFINE_EVENT(x86_fpu, x86_fpu_dropped, TP_ARGS(fpu) ); -DEFINE_EVENT(x86_fpu, x86_fpu_copy_src, - TP_PROTO(struct fpu *fpu), - TP_ARGS(fpu) -); - DEFINE_EVENT(x86_fpu, x86_fpu_copy_dst, TP_PROTO(struct fpu *fpu), TP_ARGS(fpu) From 2d299e3d773d519ee93e5aaa3ffddd4a6276b005 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:38:50 +0200 Subject: [PATCH 288/331] x86/fpu: Always use memcpy_and_pad() in arch_dup_task_struct() It makes no sense to copy the bytes after sizeof(struct task_struct), FPU state will be initialized in fpu_clone(). A plain memcpy(dst, src, sizeof(struct task_struct)) should work too, but "_and_pad" looks safer. [ mingo: Simplify it a bit more. ] Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143850.GA8997@redhat.com --- arch/x86/kernel/process.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 7a1bfb61d86f..9e6180777565 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -93,11 +93,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid); */ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - /* init_task is not dynamically sized (incomplete FPU state) */ - if (unlikely(src == &init_task)) - memcpy_and_pad(dst, arch_task_struct_size, src, sizeof(init_task), 0); - else - memcpy(dst, src, arch_task_struct_size); + /* fpu_clone() will initialize the "dst_fpu" memory */ + memcpy_and_pad(dst, arch_task_struct_size, src, sizeof(*dst), 0); #ifdef CONFIG_VM86 dst->thread.vm86 = NULL; From 016a2e6f8ae5ed544ba8fb2b6d78f64ddfd9d01b Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:38:56 +0200 Subject: [PATCH 289/331] x86/fpu: Check TIF_NEED_FPU_LOAD instead of PF_KTHREAD|PF_USER_WORKER in fpu__drop() PF_KTHREAD|PF_USER_WORKER tasks should never clear TIF_NEED_FPU_LOAD, so the TIF_NEED_FPU_LOAD check should equally filter them out. And this way an exiting userspace task can avoid the unnecessary "fwait" if it does context_switch() at least once on its way to exit_thread(). Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143856.GA9009@redhat.com --- arch/x86/kernel/fpu/core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 8d674435f173..fa131299c7da 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -693,8 +693,7 @@ void fpu__drop(struct task_struct *tsk) { struct fpu *fpu; - /* PF_KTHREAD tasks do not use the FPU context area: */ - if (tsk->flags & (PF_KTHREAD | PF_USER_WORKER)) + if (test_tsk_thread_flag(tsk, TIF_NEED_FPU_LOAD)) return; fpu = x86_task_fpu(tsk); From 46c158e3ad0fc633007802c338c409c188ec0a12 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sat, 3 May 2025 16:39:02 +0200 Subject: [PATCH 290/331] x86/fpu: Shift fpregs_assert_state_consistent() from arch_exit_work() to its caller If CONFIG_X86_DEBUG_FPU=Y, arch_exit_to_user_mode_prepare() calls arch_exit_work() even if ti_work == 0. There only reason is that we want to call fpregs_assert_state_consistent() if TIF_NEED_FPU_LOAD is not set. This looks confusing. arch_exit_to_user_mode_prepare() can just call fpregs_assert_state_consistent() unconditionally, it depends on CONFIG_X86_DEBUG_FPU and checks TIF_NEED_FPU_LOAD itself. Signed-off-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Chang S . Bae Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250503143902.GA9012@redhat.com --- arch/x86/include/asm/entry-common.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h index 77d20555e04d..d535a97c7284 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -53,7 +53,6 @@ static inline void arch_exit_work(unsigned long ti_work) if (unlikely(ti_work & _TIF_IO_BITMAP)) tss_update_io_bitmap(); - fpregs_assert_state_consistent(); if (unlikely(ti_work & _TIF_NEED_FPU_LOAD)) switch_fpu_return(); } @@ -61,7 +60,9 @@ static inline void arch_exit_work(unsigned long ti_work) static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, unsigned long ti_work) { - if (IS_ENABLED(CONFIG_X86_DEBUG_FPU) || unlikely(ti_work)) + fpregs_assert_state_consistent(); + + if (unlikely(ti_work)) arch_exit_work(ti_work); fred_update_rsp0(); From bd4a58beaaf1f4aff025282c6e8b130bdb4a29e4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:31 +0200 Subject: [PATCH 291/331] x86/boot: Move early_setup_gdt() back into head64.c Move early_setup_gdt() out of the startup code that is callable from the 1:1 mapping - this is not needed, and instead, it is better to expose the helper that does reside in __head directly. This reduces the amount of code that needs special checks for 1:1 execution suitability. In particular, it avoids dealing with the GHCB page (and its physical address) in startup code, which runs from the 1:1 mapping, making physical to virtual translations ambiguous. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-26-ardb+git@google.com --- arch/x86/boot/startup/gdt_idt.c | 15 +-------------- arch/x86/include/asm/setup.h | 1 + arch/x86/kernel/head64.c | 12 ++++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c index 7e34d0b426b1..a3112a69b06a 100644 --- a/arch/x86/boot/startup/gdt_idt.c +++ b/arch/x86/boot/startup/gdt_idt.c @@ -24,7 +24,7 @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; /* This may run while still in the direct mapping */ -static void __head startup_64_load_idt(void *vc_handler) +void __head startup_64_load_idt(void *vc_handler) { struct desc_ptr desc = { .address = (unsigned long)rip_rel_ptr(bringup_idt_table), @@ -43,19 +43,6 @@ static void __head startup_64_load_idt(void *vc_handler) native_load_idt(&desc); } -/* This is used when running on kernel addresses */ -void early_setup_idt(void) -{ - void *handler = NULL; - - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - setup_ghcb(); - handler = vc_boot_ghcb; - } - - startup_64_load_idt(handler); -} - /* * Setup boot CPU state needed before kernel switches to virtual addresses. */ diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index ad9212df0ec0..6324f4c6c545 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -52,6 +52,7 @@ extern void reserve_standard_io_resources(void); extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long p2v_offset, struct boot_params *bp); extern void startup_64_setup_gdt_idt(void); +extern void startup_64_load_idt(void *vc_handler); extern void early_setup_idt(void); extern void __init do_early_exception(struct pt_regs *regs, int trapnr); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 6b68a206fa7f..29226f3ac064 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -303,3 +303,15 @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data) start_kernel(); } + +void early_setup_idt(void) +{ + void *handler = NULL; + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + setup_ghcb(); + handler = vc_boot_ghcb; + } + + startup_64_load_idt(handler); +} From b3464a36f7f2499d517e8334e07ddd6eefcd67c1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:32 +0200 Subject: [PATCH 292/331] x86/boot: Disregard __supported_pte_mask in __startup_64() __supported_pte_mask is statically initialized to U64_MAX and never assigned until long after the startup code executes that creates the initial page tables. So applying the mask is unnecessary, and can be avoided. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-27-ardb+git@google.com --- arch/x86/boot/startup/map_kernel.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c index 0eac3f17dbd3..099ae2559336 100644 --- a/arch/x86/boot/startup/map_kernel.c +++ b/arch/x86/boot/startup/map_kernel.c @@ -179,8 +179,6 @@ unsigned long __head __startup_64(unsigned long p2v_offset, pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; - /* Filter out unsupported __PAGE_KERNEL_* bits: */ - pmd_entry &= __supported_pte_mask; pmd_entry += sme_get_me_mask(); pmd_entry += physaddr; From fae89bbfdd9d692e3cb6eace89f4994177731b8c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:34 +0200 Subject: [PATCH 293/331] x86/sev: Make sev_snp_enabled() a static function sev_snp_enabled() is no longer used outside of the source file that defines it, so make it static and drop the extern declarations. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-29-ardb+git@google.com --- arch/x86/boot/compressed/sev.c | 2 +- arch/x86/boot/compressed/sev.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 2ccad0a35e2d..f4b7f1784bea 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -164,7 +164,7 @@ int svsm_perform_call_protocol(struct svsm_call *call) return ret; } -bool sev_snp_enabled(void) +static bool sev_snp_enabled(void) { return sev_status & MSR_AMD64_SEV_SNP_ENABLED; } diff --git a/arch/x86/boot/compressed/sev.h b/arch/x86/boot/compressed/sev.h index d3900384b8ab..e87af54fab2f 100644 --- a/arch/x86/boot/compressed/sev.h +++ b/arch/x86/boot/compressed/sev.h @@ -10,14 +10,12 @@ #ifdef CONFIG_AMD_MEM_ENCRYPT -bool sev_snp_enabled(void); void snp_accept_memory(phys_addr_t start, phys_addr_t end); u64 sev_get_status(void); bool early_is_sevsnp_guest(void); #else -static inline bool sev_snp_enabled(void) { return false; } static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { } static inline u64 sev_get_status(void) { return 0; } static inline bool early_is_sevsnp_guest(void) { return false; } From ae862964cbc562582576da91b0b92742a574c437 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:35 +0200 Subject: [PATCH 294/331] x86/sev: Move instruction decoder into separate source file As a first step towards disentangling the SEV #VC handling code -which is shared between the decompressor and the core kernel- from the SEV startup code, move the decompressor's copy of the instruction decoder into a separate source file. Code movement only - no functional change intended. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-30-ardb+git@google.com --- arch/x86/boot/compressed/Makefile | 6 +-- arch/x86/boot/compressed/misc.h | 7 ++++ arch/x86/boot/compressed/sev-handle-vc.c | 51 ++++++++++++++++++++++++ arch/x86/boot/compressed/sev.c | 39 +----------------- 4 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 arch/x86/boot/compressed/sev-handle-vc.c diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 0fcad7b7e007..f4f7b22d8113 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -44,10 +44,10 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(call cc-option,-Wa$(comma)-mrelax-relocations=no) KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h -# sev.c indirectly includes inat-table.h which is generated during +# sev-decode-insn.c indirectly includes inat-table.c which is generated during # compilation and stored in $(objtree). Add the directory to the includes so # that the compiler finds it even with out-of-tree builds (make O=/some/path). -CFLAGS_sev.o += -I$(objtree)/arch/x86/lib/ +CFLAGS_sev-handle-vc.o += -I$(objtree)/arch/x86/lib/ KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ @@ -96,7 +96,7 @@ ifdef CONFIG_X86_64 vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o vmlinux-objs-y += $(obj)/pgtable_64.o - vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o + vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o $(obj)/sev-handle-vc.o endif vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index dd8d1a85f671..04de48c8d4cf 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -136,6 +136,9 @@ static inline void console_init(void) #endif #ifdef CONFIG_AMD_MEM_ENCRYPT +struct es_em_ctxt; +struct insn; + void sev_enable(struct boot_params *bp); void snp_check_features(void); void sev_es_shutdown_ghcb(void); @@ -143,6 +146,10 @@ extern bool sev_es_check_ghcb_fault(unsigned long address); void snp_set_page_private(unsigned long paddr); void snp_set_page_shared(unsigned long paddr); void sev_prep_identity_maps(unsigned long top_level_pgt); + +enum es_result vc_decode_insn(struct es_em_ctxt *ctxt); +bool insn_has_rep_prefix(struct insn *insn); +void sev_insn_decode_init(void); #else static inline void sev_enable(struct boot_params *bp) { diff --git a/arch/x86/boot/compressed/sev-handle-vc.c b/arch/x86/boot/compressed/sev-handle-vc.c new file mode 100644 index 000000000000..b1aa073b732c --- /dev/null +++ b/arch/x86/boot/compressed/sev-handle-vc.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "misc.h" + +#include +#include +#include +#include +#include +#include + +#define __BOOT_COMPRESSED + +/* Basic instruction decoding support needed */ +#include "../../lib/inat.c" +#include "../../lib/insn.c" + +/* + * Copy a version of this function here - insn-eval.c can't be used in + * pre-decompression code. + */ +bool insn_has_rep_prefix(struct insn *insn) +{ + insn_byte_t p; + int i; + + insn_get_prefixes(insn); + + for_each_insn_prefix(insn, i, p) { + if (p == 0xf2 || p == 0xf3) + return true; + } + + return false; +} + +enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + int ret; + + memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); + + ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); + if (ret < 0) + return ES_DECODE_FAILED; + + return ES_OK; +} + +extern void sev_insn_decode_init(void) __alias(inat_init_tables); diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index f4b7f1784bea..6fe7e85ade6b 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -29,25 +29,6 @@ static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); struct ghcb *boot_ghcb; -/* - * Copy a version of this function here - insn-eval.c can't be used in - * pre-decompression code. - */ -static bool insn_has_rep_prefix(struct insn *insn) -{ - insn_byte_t p; - int i; - - insn_get_prefixes(insn); - - for_each_insn_prefix(insn, i, p) { - if (p == 0xf2 || p == 0xf3) - return true; - } - - return false; -} - /* * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and * doesn't use segments. @@ -74,20 +55,6 @@ static inline void sev_es_wr_ghcb_msr(u64 val) boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m); } -static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) -{ - char buffer[MAX_INSN_SIZE]; - int ret; - - memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); - - ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); - if (ret < 0) - return ES_DECODE_FAILED; - - return ES_OK; -} - static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, void *dst, char *buf, size_t size) { @@ -122,10 +89,6 @@ static bool fault_in_kernel_space(unsigned long address) #define __BOOT_COMPRESSED -/* Basic instruction decoding support needed */ -#include "../../lib/inat.c" -#include "../../lib/insn.c" - extern struct svsm_ca *boot_svsm_caa; extern u64 boot_svsm_caa_pa; @@ -230,7 +193,7 @@ static bool early_setup_ghcb(void) boot_ghcb = &boot_ghcb_page; /* Initialize lookup tables for the instruction decoder */ - inat_init_tables(); + sev_insn_decode_init(); /* SNP guest requires the GHCB GPA must be registered */ if (sev_snp_enabled()) From f932adcc8650eb169efd0fd19f2a450c260d306c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:43 +0200 Subject: [PATCH 295/331] x86/linkage: Add SYM_PIC_ALIAS() macro helper to emit symbol aliases Startup code that may execute from the early 1:1 mapping of memory will be confined into its own address space, and only be permitted to access ordinary kernel symbols if this is known to be safe. Introduce a macro helper SYM_PIC_ALIAS() that emits a __pi_ prefixed alias for a symbol, which allows startup code to access it. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-38-ardb+git@google.com --- arch/x86/include/asm/linkage.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index b51d8a4673f5..9d38ae744a2e 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -141,5 +141,15 @@ #define SYM_FUNC_START_WEAK_NOALIGN(name) \ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) +/* + * Expose 'sym' to the startup code in arch/x86/boot/startup/, by emitting an + * alias prefixed with __pi_ + */ +#ifdef __ASSEMBLER__ +#define SYM_PIC_ALIAS(sym) SYM_ALIAS(__pi_ ## sym, sym, SYM_L_GLOBAL) +#else +#define SYM_PIC_ALIAS(sym) extern typeof(sym) __PASTE(__pi_, sym) __alias(sym) +#endif + #endif /* _ASM_X86_LINKAGE_H */ From 419cbaf6a56a6e4b7e6d2278302c197f55dec830 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:44 +0200 Subject: [PATCH 296/331] x86/boot: Add a bunch of PIC aliases Add aliases for all the data objects that the startup code references - this is needed so that this code can be moved into its own confined area where it can only access symbols that have a __pi_ prefix. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-39-ardb+git@google.com --- arch/x86/coco/core.c | 2 ++ arch/x86/kernel/cpu/common.c | 1 + arch/x86/kernel/head64.c | 4 ++++ arch/x86/kernel/head_64.S | 8 ++++++++ arch/x86/kernel/setup.c | 1 + arch/x86/kernel/vmlinux.lds.S | 4 ++++ arch/x86/lib/memcpy_64.S | 1 + arch/x86/lib/memset_64.S | 1 + arch/x86/lib/retpoline.S | 2 ++ arch/x86/mm/mem_encrypt_amd.c | 2 ++ arch/x86/mm/pgtable.c | 1 + tools/objtool/arch/x86/decode.c | 6 ++++-- 12 files changed, 31 insertions(+), 2 deletions(-) diff --git a/arch/x86/coco/core.c b/arch/x86/coco/core.c index 9a0ddda3aa69..d4610af68114 100644 --- a/arch/x86/coco/core.c +++ b/arch/x86/coco/core.c @@ -18,7 +18,9 @@ #include enum cc_vendor cc_vendor __ro_after_init = CC_VENDOR_NONE; +SYM_PIC_ALIAS(cc_vendor); u64 cc_mask __ro_after_init; +SYM_PIC_ALIAS(cc_mask); static struct cc_attr_flags { __u64 host_sev_snp : 1, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 12126adbc3a9..f0f85482a73b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -242,6 +242,7 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { #endif } }; EXPORT_PER_CPU_SYMBOL_GPL(gdt_page); +SYM_PIC_ALIAS(gdt_page); #ifdef CONFIG_X86_64 static int __init x86_nopcid_setup(char *s) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 29226f3ac064..510fb41f55fc 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -48,6 +48,7 @@ */ extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; unsigned int __initdata next_early_pgt; +SYM_PIC_ALIAS(next_early_pgt); pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX); #ifdef CONFIG_X86_5LEVEL @@ -61,10 +62,13 @@ EXPORT_SYMBOL(ptrs_per_p4d); #ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4; EXPORT_SYMBOL(page_offset_base); +SYM_PIC_ALIAS(page_offset_base); unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4; EXPORT_SYMBOL(vmalloc_base); +SYM_PIC_ALIAS(vmalloc_base); unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); +SYM_PIC_ALIAS(vmemmap_base); #endif /* Wipe all early page tables except for the kernel symbol map */ diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index fefe2a25cf02..069420853304 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -573,6 +573,7 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) /* Pure iret required here - don't use INTERRUPT_RETURN */ iretq SYM_CODE_END(vc_no_ghcb) +SYM_PIC_ALIAS(vc_no_ghcb); #endif #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION @@ -604,10 +605,12 @@ SYM_DATA_START_PTI_ALIGNED(early_top_pgt) .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC .fill PTI_USER_PGD_FILL,8,0 SYM_DATA_END(early_top_pgt) +SYM_PIC_ALIAS(early_top_pgt) SYM_DATA_START_PAGE_ALIGNED(early_dynamic_pgts) .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0 SYM_DATA_END(early_dynamic_pgts) +SYM_PIC_ALIAS(early_dynamic_pgts); SYM_DATA(early_recursion_flag, .long 0) @@ -651,6 +654,7 @@ SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt) .fill 511,8,0 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC SYM_DATA_END(level4_kernel_pgt) +SYM_PIC_ALIAS(level4_kernel_pgt) #endif SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt) @@ -659,6 +663,7 @@ SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt) .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC SYM_DATA_END(level3_kernel_pgt) +SYM_PIC_ALIAS(level3_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt) /* @@ -676,6 +681,7 @@ SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt) */ PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE) SYM_DATA_END(level2_kernel_pgt) +SYM_PIC_ALIAS(level2_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt) .fill (512 - 4 - FIXMAP_PMD_NUM),8,0 @@ -688,6 +694,7 @@ SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt) /* 6 MB reserved space + a 2MB hole */ .fill 4,8,0 SYM_DATA_END(level2_fixmap_pgt) +SYM_PIC_ALIAS(level2_fixmap_pgt) SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt) .rept (FIXMAP_PMD_NUM) @@ -703,6 +710,7 @@ SYM_DATA(smpboot_control, .long 0) .align 16 /* This must match the first entry in level2_kernel_pgt */ SYM_DATA(phys_base, .quad 0x0) +SYM_PIC_ALIAS(phys_base); EXPORT_SYMBOL(phys_base) #include "../xen/xen-head.S" diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 9d2a13b37833..e0cf1595a0ab 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -134,6 +134,7 @@ struct ist_info ist_info; struct cpuinfo_x86 boot_cpu_data __read_mostly; EXPORT_SYMBOL(boot_cpu_data); +SYM_PIC_ALIAS(boot_cpu_data); #if !defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64) __visible unsigned long mmu_cr4_features __ro_after_init; diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index ccdc45e5b759..9340c74b680d 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -79,11 +79,13 @@ const_cpu_current_top_of_stack = cpu_current_top_of_stack; #define BSS_DECRYPTED \ . = ALIGN(PMD_SIZE); \ __start_bss_decrypted = .; \ + __pi___start_bss_decrypted = .; \ *(.bss..decrypted); \ . = ALIGN(PAGE_SIZE); \ __start_bss_decrypted_unused = .; \ . = ALIGN(PMD_SIZE); \ __end_bss_decrypted = .; \ + __pi___end_bss_decrypted = .; \ #else @@ -128,6 +130,7 @@ SECTIONS /* Text and read-only data */ .text : AT(ADDR(.text) - LOAD_OFFSET) { _text = .; + __pi__text = .; _stext = .; ALIGN_ENTRY_TEXT_BEGIN *(.text..__x86.rethunk_untrain) @@ -391,6 +394,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); /* keep VO_INIT_SIZE page aligned */ _end = .; + __pi__end = .; #ifdef CONFIG_AMD_MEM_ENCRYPT /* diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 0ae2e1712e2e..12a23fa7c44c 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -41,6 +41,7 @@ SYM_FUNC_END(__memcpy) EXPORT_SYMBOL(__memcpy) SYM_FUNC_ALIAS_MEMFUNC(memcpy, __memcpy) +SYM_PIC_ALIAS(memcpy) EXPORT_SYMBOL(memcpy) SYM_FUNC_START_LOCAL(memcpy_orig) diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index d66b710d628f..fb5a03cf5ab7 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S @@ -42,6 +42,7 @@ SYM_FUNC_END(__memset) EXPORT_SYMBOL(__memset) SYM_FUNC_ALIAS_MEMFUNC(memset, __memset) +SYM_PIC_ALIAS(memset) EXPORT_SYMBOL(memset) SYM_FUNC_START_LOCAL(memset_orig) diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index a26c43abd47d..9f3116609c8c 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -40,6 +40,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \ __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE) +SYM_PIC_ALIAS(__x86_indirect_thunk_\reg) .endm @@ -394,6 +395,7 @@ SYM_CODE_START(__x86_return_thunk) #endif int3 SYM_CODE_END(__x86_return_thunk) +SYM_PIC_ALIAS(__x86_return_thunk) EXPORT_SYMBOL(__x86_return_thunk) #endif /* CONFIG_MITIGATION_RETHUNK */ diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index 7490ff6d83b1..faf3a13fb6ba 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -40,7 +40,9 @@ * section is later cleared. */ u64 sme_me_mask __section(".data") = 0; +SYM_PIC_ALIAS(sme_me_mask); u64 sev_status __section(".data") = 0; +SYM_PIC_ALIAS(sev_status); u64 sev_check_data __section(".data") = 0; EXPORT_SYMBOL(sme_me_mask); diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index f7ae44d3dd9e..e62af72923e8 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -10,6 +10,7 @@ #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; EXPORT_SYMBOL(physical_mask); +SYM_PIC_ALIAS(physical_mask); #endif pgtable_t pte_alloc_one(struct mm_struct *mm) diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index 3ce7b54003c2..331b9a744410 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -842,12 +842,14 @@ int arch_decode_hint_reg(u8 sp_reg, int *base) bool arch_is_retpoline(struct symbol *sym) { - return !strncmp(sym->name, "__x86_indirect_", 15); + return !strncmp(sym->name, "__x86_indirect_", 15) || + !strncmp(sym->name, "__pi___x86_indirect_", 20); } bool arch_is_rethunk(struct symbol *sym) { - return !strcmp(sym->name, "__x86_return_thunk"); + return !strcmp(sym->name, "__x86_return_thunk") || + !strcmp(sym->name, "__pi___x86_return_thunk"); } bool arch_is_embedded_insn(struct symbol *sym) From 5297886f0cc45db5f4a804caf359e6e7874ee864 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:45 +0200 Subject: [PATCH 297/331] x86/boot: Provide __pti_set_user_pgtbl() to startup code The SME encryption startup code populates page tables using the ordinary set_pXX() helpers, and in a PTI build, these will call out to __pti_set_user_pgtbl() to manipulate the shadow copy of the page tables for user space. This is unneeded for the startup code, which only manipulates the swapper page tables, and so this call could be avoided in this particular case. So instead of exposing the ordinary __pti_set_user_pgtblt() to the startup code after its gets confined into its own symbol space, provide an alternative which just returns pgd, which is always correct in the startup context. Annotate it as __weak for now, this will be dropped in a subsequent patch. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-40-ardb+git@google.com --- arch/x86/boot/startup/sme.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c index 5738b31c8e60..753cd2094080 100644 --- a/arch/x86/boot/startup/sme.c +++ b/arch/x86/boot/startup/sme.c @@ -564,3 +564,12 @@ void __head sme_enable(struct boot_params *bp) cc_vendor = CC_VENDOR_AMD; cc_set_mask(me_mask); } + +#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION +/* Local version for startup code, which never operates on user page tables */ +__weak +pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) +{ + return pgd; +} +#endif From ed4d95d033e359f9445e85bf5a768a5859a5830b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 4 May 2025 11:52:36 +0200 Subject: [PATCH 298/331] x86/sev: Disentangle #VC handling code from startup code Most of the SEV support code used to reside in a single C source file that was included in two places: the core kernel, and the decompressor. The code that is actually shared with the decompressor was moved into a separate, shared source file under startup/, on the basis that the decompressor also executes from the early 1:1 mapping of memory. However, while the elaborate #VC handling and instruction decoding that it involves is also performed by the decompressor, it does not actually occur in the core kernel at early boot, and therefore, does not need to be part of the confined early startup code. So split off the #VC handling code and move it back into arch/x86/coco where it came from, into another C source file that is included from both the decompressor and the core kernel. Code movement only - no functional change intended. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Arnd Bergmann Cc: David Woodhouse Cc: Dionna Amalie Glaze Cc: H. Peter Anvin Cc: Kees Cook Cc: Kevin Loughlin Cc: Len Brown Cc: Linus Torvalds Cc: Rafael J. Wysocki Cc: Tom Lendacky Cc: linux-efi@vger.kernel.org Link: https://lore.kernel.org/r/20250504095230.2932860-31-ardb+git@google.com --- arch/x86/boot/compressed/misc.h | 1 + arch/x86/boot/compressed/sev-handle-vc.c | 83 ++ arch/x86/boot/compressed/sev.c | 96 +- arch/x86/boot/compressed/sev.h | 19 + arch/x86/boot/startup/sev-shared.c | 519 +---------- arch/x86/boot/startup/sev-startup.c | 1022 --------------------- arch/x86/coco/sev/Makefile | 2 +- arch/x86/coco/sev/vc-handle.c | 1061 ++++++++++++++++++++++ arch/x86/coco/sev/vc-shared.c | 504 ++++++++++ arch/x86/include/asm/sev-internal.h | 8 - arch/x86/include/asm/sev.h | 22 + 11 files changed, 1694 insertions(+), 1643 deletions(-) create mode 100644 arch/x86/coco/sev/vc-handle.c create mode 100644 arch/x86/coco/sev/vc-shared.c diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 04de48c8d4cf..db1048621ea2 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -150,6 +150,7 @@ void sev_prep_identity_maps(unsigned long top_level_pgt); enum es_result vc_decode_insn(struct es_em_ctxt *ctxt); bool insn_has_rep_prefix(struct insn *insn); void sev_insn_decode_init(void); +bool early_setup_ghcb(void); #else static inline void sev_enable(struct boot_params *bp) { diff --git a/arch/x86/boot/compressed/sev-handle-vc.c b/arch/x86/boot/compressed/sev-handle-vc.c index b1aa073b732c..89dd02de2a0f 100644 --- a/arch/x86/boot/compressed/sev-handle-vc.c +++ b/arch/x86/boot/compressed/sev-handle-vc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include "misc.h" +#include "sev.h" #include #include @@ -8,6 +9,9 @@ #include #include #include +#include +#include +#include #define __BOOT_COMPRESSED @@ -49,3 +53,82 @@ enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) } extern void sev_insn_decode_init(void) __alias(inat_init_tables); + +/* + * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and + * doesn't use segments. + */ +static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) +{ + return 0UL; +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + void *dst, char *buf, size_t size) +{ + memcpy(dst, buf, size); + + return ES_OK; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + void *src, char *buf, size_t size) +{ + memcpy(buf, src, size); + + return ES_OK; +} + +static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) +{ + return ES_OK; +} + +static bool fault_in_kernel_space(unsigned long address) +{ + return false; +} + +#define sev_printk(fmt, ...) + +#include "../../coco/sev/vc-shared.c" + +void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) +{ + struct es_em_ctxt ctxt; + enum es_result result; + + if (!boot_ghcb && !early_setup_ghcb()) + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); + + vc_ghcb_invalidate(boot_ghcb); + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result != ES_OK) + goto finish; + + result = vc_check_opcode_bytes(&ctxt, exit_code); + if (result != ES_OK) + goto finish; + + switch (exit_code) { + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code); + break; + case SVM_EXIT_IOIO: + result = vc_handle_ioio(boot_ghcb, &ctxt); + break; + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(boot_ghcb, &ctxt); + break; + default: + result = ES_UNSUPPORTED; + break; + } + +finish: + if (result == ES_OK) + vc_finish_insn(&ctxt); + else if (result != ES_RETRY) + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); +} diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 6fe7e85ade6b..612b443296d3 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -24,63 +24,11 @@ #include #include "error.h" -#include "../msr.h" +#include "sev.h" static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); struct ghcb *boot_ghcb; -/* - * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and - * doesn't use segments. - */ -static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) -{ - return 0UL; -} - -static inline u64 sev_es_rd_ghcb_msr(void) -{ - struct msr m; - - boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m); - - return m.q; -} - -static inline void sev_es_wr_ghcb_msr(u64 val) -{ - struct msr m; - - m.q = val; - boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m); -} - -static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, - void *dst, char *buf, size_t size) -{ - memcpy(dst, buf, size); - - return ES_OK; -} - -static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, - void *src, char *buf, size_t size) -{ - memcpy(buf, src, size); - - return ES_OK; -} - -static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) -{ - return ES_OK; -} - -static bool fault_in_kernel_space(unsigned long address) -{ - return false; -} - #undef __init #define __init @@ -182,7 +130,7 @@ void snp_set_page_shared(unsigned long paddr) __page_state_change(paddr, SNP_PAGE_STATE_SHARED); } -static bool early_setup_ghcb(void) +bool early_setup_ghcb(void) { if (set_page_decrypted((unsigned long)&boot_ghcb_page)) return false; @@ -266,46 +214,6 @@ bool sev_es_check_ghcb_fault(unsigned long address) return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page); } -void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) -{ - struct es_em_ctxt ctxt; - enum es_result result; - - if (!boot_ghcb && !early_setup_ghcb()) - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); - - vc_ghcb_invalidate(boot_ghcb); - result = vc_init_em_ctxt(&ctxt, regs, exit_code); - if (result != ES_OK) - goto finish; - - result = vc_check_opcode_bytes(&ctxt, exit_code); - if (result != ES_OK) - goto finish; - - switch (exit_code) { - case SVM_EXIT_RDTSC: - case SVM_EXIT_RDTSCP: - result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code); - break; - case SVM_EXIT_IOIO: - result = vc_handle_ioio(boot_ghcb, &ctxt); - break; - case SVM_EXIT_CPUID: - result = vc_handle_cpuid(boot_ghcb, &ctxt); - break; - default: - result = ES_UNSUPPORTED; - break; - } - -finish: - if (result == ES_OK) - vc_finish_insn(&ctxt); - else if (result != ES_RETRY) - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); -} - /* * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need * guest side implementation for proper functioning of the guest. If any diff --git a/arch/x86/boot/compressed/sev.h b/arch/x86/boot/compressed/sev.h index e87af54fab2f..92f79c21939c 100644 --- a/arch/x86/boot/compressed/sev.h +++ b/arch/x86/boot/compressed/sev.h @@ -10,10 +10,29 @@ #ifdef CONFIG_AMD_MEM_ENCRYPT +#include "../msr.h" + void snp_accept_memory(phys_addr_t start, phys_addr_t end); u64 sev_get_status(void); bool early_is_sevsnp_guest(void); +static inline u64 sev_es_rd_ghcb_msr(void) +{ + struct msr m; + + boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m); + + return m.q; +} + +static inline void sev_es_wr_ghcb_msr(u64 val) +{ + struct msr m; + + m.q = val; + boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m); +} + #else static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { } diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c index 173f3d1f777a..7a706db87b93 100644 --- a/arch/x86/boot/startup/sev-shared.c +++ b/arch/x86/boot/startup/sev-shared.c @@ -14,13 +14,9 @@ #ifndef __BOOT_COMPRESSED #define error(v) pr_err(v) #define has_cpuflag(f) boot_cpu_has(f) -#define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__) -#define sev_printk_rtl(fmt, ...) printk_ratelimited(fmt, ##__VA_ARGS__) #else #undef WARN #define WARN(condition, format...) (!!(condition)) -#define sev_printk(fmt, ...) -#define sev_printk_rtl(fmt, ...) #undef vc_forward_exception #define vc_forward_exception(c) panic("SNP: Hypervisor requested exception\n") #endif @@ -36,16 +32,6 @@ struct svsm_ca *boot_svsm_caa __ro_after_init; u64 boot_svsm_caa_pa __ro_after_init; -/* I/O parameters for CPUID-related helpers */ -struct cpuid_leaf { - u32 fn; - u32 subfn; - u32 eax; - u32 ebx; - u32 ecx; - u32 edx; -}; - /* * Since feature negotiation related variables are set early in the boot * process they must reside in the .data section so as not to be zeroed @@ -151,33 +137,6 @@ bool sev_es_negotiate_protocol(void) return true; } -static bool vc_decoding_needed(unsigned long exit_code) -{ - /* Exceptions don't require to decode the instruction */ - return !(exit_code >= SVM_EXIT_EXCP_BASE && - exit_code <= SVM_EXIT_LAST_EXCP); -} - -static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, - struct pt_regs *regs, - unsigned long exit_code) -{ - enum es_result ret = ES_OK; - - memset(ctxt, 0, sizeof(*ctxt)); - ctxt->regs = regs; - - if (vc_decoding_needed(exit_code)) - ret = vc_decode_insn(ctxt); - - return ret; -} - -static void vc_finish_insn(struct es_em_ctxt *ctxt) -{ - ctxt->regs->ip += ctxt->insn.length; -} - static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { u32 ret; @@ -627,7 +586,7 @@ snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt, * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value * should be treated as fatal by caller. */ -static int __head +int __head snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) { const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); @@ -737,391 +696,6 @@ fail: sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); } -static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, - unsigned long address, - bool write) -{ - if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = X86_PF_USER; - ctxt->fi.cr2 = address; - if (write) - ctxt->fi.error_code |= X86_PF_WRITE; - - return ES_EXCEPTION; - } - - return ES_OK; -} - -static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, - void *src, char *buf, - unsigned int data_size, - unsigned int count, - bool backwards) -{ - int i, b = backwards ? -1 : 1; - unsigned long address = (unsigned long)src; - enum es_result ret; - - ret = vc_insn_string_check(ctxt, address, false); - if (ret != ES_OK) - return ret; - - for (i = 0; i < count; i++) { - void *s = src + (i * data_size * b); - char *d = buf + (i * data_size); - - ret = vc_read_mem(ctxt, s, d, data_size); - if (ret != ES_OK) - break; - } - - return ret; -} - -static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, - void *dst, char *buf, - unsigned int data_size, - unsigned int count, - bool backwards) -{ - int i, s = backwards ? -1 : 1; - unsigned long address = (unsigned long)dst; - enum es_result ret; - - ret = vc_insn_string_check(ctxt, address, true); - if (ret != ES_OK) - return ret; - - for (i = 0; i < count; i++) { - void *d = dst + (i * data_size * s); - char *b = buf + (i * data_size); - - ret = vc_write_mem(ctxt, d, b, data_size); - if (ret != ES_OK) - break; - } - - return ret; -} - -#define IOIO_TYPE_STR BIT(2) -#define IOIO_TYPE_IN 1 -#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) -#define IOIO_TYPE_OUT 0 -#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) - -#define IOIO_REP BIT(3) - -#define IOIO_ADDR_64 BIT(9) -#define IOIO_ADDR_32 BIT(8) -#define IOIO_ADDR_16 BIT(7) - -#define IOIO_DATA_32 BIT(6) -#define IOIO_DATA_16 BIT(5) -#define IOIO_DATA_8 BIT(4) - -#define IOIO_SEG_ES (0 << 10) -#define IOIO_SEG_DS (3 << 10) - -static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) -{ - struct insn *insn = &ctxt->insn; - size_t size; - u64 port; - - *exitinfo = 0; - - switch (insn->opcode.bytes[0]) { - /* INS opcodes */ - case 0x6c: - case 0x6d: - *exitinfo |= IOIO_TYPE_INS; - *exitinfo |= IOIO_SEG_ES; - port = ctxt->regs->dx & 0xffff; - break; - - /* OUTS opcodes */ - case 0x6e: - case 0x6f: - *exitinfo |= IOIO_TYPE_OUTS; - *exitinfo |= IOIO_SEG_DS; - port = ctxt->regs->dx & 0xffff; - break; - - /* IN immediate opcodes */ - case 0xe4: - case 0xe5: - *exitinfo |= IOIO_TYPE_IN; - port = (u8)insn->immediate.value & 0xffff; - break; - - /* OUT immediate opcodes */ - case 0xe6: - case 0xe7: - *exitinfo |= IOIO_TYPE_OUT; - port = (u8)insn->immediate.value & 0xffff; - break; - - /* IN register opcodes */ - case 0xec: - case 0xed: - *exitinfo |= IOIO_TYPE_IN; - port = ctxt->regs->dx & 0xffff; - break; - - /* OUT register opcodes */ - case 0xee: - case 0xef: - *exitinfo |= IOIO_TYPE_OUT; - port = ctxt->regs->dx & 0xffff; - break; - - default: - return ES_DECODE_FAILED; - } - - *exitinfo |= port << 16; - - switch (insn->opcode.bytes[0]) { - case 0x6c: - case 0x6e: - case 0xe4: - case 0xe6: - case 0xec: - case 0xee: - /* Single byte opcodes */ - *exitinfo |= IOIO_DATA_8; - size = 1; - break; - default: - /* Length determined by instruction parsing */ - *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 - : IOIO_DATA_32; - size = (insn->opnd_bytes == 2) ? 2 : 4; - } - - switch (insn->addr_bytes) { - case 2: - *exitinfo |= IOIO_ADDR_16; - break; - case 4: - *exitinfo |= IOIO_ADDR_32; - break; - case 8: - *exitinfo |= IOIO_ADDR_64; - break; - } - - if (insn_has_rep_prefix(insn)) - *exitinfo |= IOIO_REP; - - return vc_ioio_check(ctxt, (u16)port, size); -} - -static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct pt_regs *regs = ctxt->regs; - u64 exit_info_1, exit_info_2; - enum es_result ret; - - ret = vc_ioio_exitinfo(ctxt, &exit_info_1); - if (ret != ES_OK) - return ret; - - if (exit_info_1 & IOIO_TYPE_STR) { - - /* (REP) INS/OUTS */ - - bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); - unsigned int io_bytes, exit_bytes; - unsigned int ghcb_count, op_count; - unsigned long es_base; - u64 sw_scratch; - - /* - * For the string variants with rep prefix the amount of in/out - * operations per #VC exception is limited so that the kernel - * has a chance to take interrupts and re-schedule while the - * instruction is emulated. - */ - io_bytes = (exit_info_1 >> 4) & 0x7; - ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; - - op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; - exit_info_2 = min(op_count, ghcb_count); - exit_bytes = exit_info_2 * io_bytes; - - es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); - - /* Read bytes of OUTS into the shared buffer */ - if (!(exit_info_1 & IOIO_TYPE_IN)) { - ret = vc_insn_string_read(ctxt, - (void *)(es_base + regs->si), - ghcb->shared_buffer, io_bytes, - exit_info_2, df); - if (ret) - return ret; - } - - /* - * Issue an VMGEXIT to the HV to consume the bytes from the - * shared buffer or to have it write them into the shared buffer - * depending on the instruction: OUTS or INS. - */ - sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); - ghcb_set_sw_scratch(ghcb, sw_scratch); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, - exit_info_1, exit_info_2); - if (ret != ES_OK) - return ret; - - /* Read bytes from shared buffer into the guest's destination. */ - if (exit_info_1 & IOIO_TYPE_IN) { - ret = vc_insn_string_write(ctxt, - (void *)(es_base + regs->di), - ghcb->shared_buffer, io_bytes, - exit_info_2, df); - if (ret) - return ret; - - if (df) - regs->di -= exit_bytes; - else - regs->di += exit_bytes; - } else { - if (df) - regs->si -= exit_bytes; - else - regs->si += exit_bytes; - } - - if (exit_info_1 & IOIO_REP) - regs->cx -= exit_info_2; - - ret = regs->cx ? ES_RETRY : ES_OK; - - } else { - - /* IN/OUT into/from rAX */ - - int bits = (exit_info_1 & 0x70) >> 1; - u64 rax = 0; - - if (!(exit_info_1 & IOIO_TYPE_IN)) - rax = lower_bits(regs->ax, bits); - - ghcb_set_rax(ghcb, rax); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); - if (ret != ES_OK) - return ret; - - if (exit_info_1 & IOIO_TYPE_IN) { - if (!ghcb_rax_is_valid(ghcb)) - return ES_VMM_ERROR; - regs->ax = lower_bits(ghcb->save.rax, bits); - } - } - - return ret; -} - -static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct pt_regs *regs = ctxt->regs; - struct cpuid_leaf leaf; - int ret; - - leaf.fn = regs->ax; - leaf.subfn = regs->cx; - ret = snp_cpuid(ghcb, ctxt, &leaf); - if (!ret) { - regs->ax = leaf.eax; - regs->bx = leaf.ebx; - regs->cx = leaf.ecx; - regs->dx = leaf.edx; - } - - return ret; -} - -static enum es_result vc_handle_cpuid(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct pt_regs *regs = ctxt->regs; - u32 cr4 = native_read_cr4(); - enum es_result ret; - int snp_cpuid_ret; - - snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); - if (!snp_cpuid_ret) - return ES_OK; - if (snp_cpuid_ret != -EOPNOTSUPP) - return ES_VMM_ERROR; - - ghcb_set_rax(ghcb, regs->ax); - ghcb_set_rcx(ghcb, regs->cx); - - if (cr4 & X86_CR4_OSXSAVE) - /* Safe to read xcr0 */ - ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); - else - /* xgetbv will cause #GP - use reset value for xcr0 */ - ghcb_set_xcr0(ghcb, 1); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); - if (ret != ES_OK) - return ret; - - if (!(ghcb_rax_is_valid(ghcb) && - ghcb_rbx_is_valid(ghcb) && - ghcb_rcx_is_valid(ghcb) && - ghcb_rdx_is_valid(ghcb))) - return ES_VMM_ERROR; - - regs->ax = ghcb->save.rax; - regs->bx = ghcb->save.rbx; - regs->cx = ghcb->save.rcx; - regs->dx = ghcb->save.rdx; - - return ES_OK; -} - -static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, - struct es_em_ctxt *ctxt, - unsigned long exit_code) -{ - bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); - enum es_result ret; - - /* - * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure - * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP - * instructions are being intercepted. If this should occur and Secure - * TSC is enabled, guest execution should be terminated as the guest - * cannot rely on the TSC value provided by the hypervisor. - */ - if (sev_status & MSR_AMD64_SNP_SECURE_TSC) - return ES_VMM_ERROR; - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); - if (ret != ES_OK) - return ret; - - if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && - (!rdtscp || ghcb_rcx_is_valid(ghcb)))) - return ES_VMM_ERROR; - - ctxt->regs->ax = ghcb->save.rax; - ctxt->regs->dx = ghcb->save.rdx; - if (rdtscp) - ctxt->regs->cx = ghcb->save.rcx; - - return ES_OK; -} - struct cc_setup_data { struct setup_data header; u32 cc_blob_address; @@ -1238,97 +812,6 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, } } -static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt, - unsigned long exit_code) -{ - unsigned int opcode = (unsigned int)ctxt->insn.opcode.value; - u8 modrm = ctxt->insn.modrm.value; - - switch (exit_code) { - - case SVM_EXIT_IOIO: - case SVM_EXIT_NPF: - /* handled separately */ - return ES_OK; - - case SVM_EXIT_CPUID: - if (opcode == 0xa20f) - return ES_OK; - break; - - case SVM_EXIT_INVD: - if (opcode == 0x080f) - return ES_OK; - break; - - case SVM_EXIT_MONITOR: - /* MONITOR and MONITORX instructions generate the same error code */ - if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa)) - return ES_OK; - break; - - case SVM_EXIT_MWAIT: - /* MWAIT and MWAITX instructions generate the same error code */ - if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb)) - return ES_OK; - break; - - case SVM_EXIT_MSR: - /* RDMSR */ - if (opcode == 0x320f || - /* WRMSR */ - opcode == 0x300f) - return ES_OK; - break; - - case SVM_EXIT_RDPMC: - if (opcode == 0x330f) - return ES_OK; - break; - - case SVM_EXIT_RDTSC: - if (opcode == 0x310f) - return ES_OK; - break; - - case SVM_EXIT_RDTSCP: - if (opcode == 0x010f && modrm == 0xf9) - return ES_OK; - break; - - case SVM_EXIT_READ_DR7: - if (opcode == 0x210f && - X86_MODRM_REG(ctxt->insn.modrm.value) == 7) - return ES_OK; - break; - - case SVM_EXIT_VMMCALL: - if (opcode == 0x010f && modrm == 0xd9) - return ES_OK; - - break; - - case SVM_EXIT_WRITE_DR7: - if (opcode == 0x230f && - X86_MODRM_REG(ctxt->insn.modrm.value) == 7) - return ES_OK; - break; - - case SVM_EXIT_WBINVD: - if (opcode == 0x90f) - return ES_OK; - break; - - default: - break; - } - - sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n", - opcode, exit_code, ctxt->regs->ip); - - return ES_UNSUPPORTED; -} - /* * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM * services needed when not running in VMPL0. diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c index f901ce9680e6..435853a55768 100644 --- a/arch/x86/boot/startup/sev-startup.c +++ b/arch/x86/boot/startup/sev-startup.c @@ -9,7 +9,6 @@ #define pr_fmt(fmt) "SEV: " fmt -#include /* For show_regs() */ #include #include #include @@ -112,315 +111,6 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) return ghcb; } -static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, - unsigned char *buffer) -{ - return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); -} - -static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) -{ - char buffer[MAX_INSN_SIZE]; - int insn_bytes; - - insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); - if (insn_bytes == 0) { - /* Nothing could be copied */ - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; - } else if (insn_bytes == -EINVAL) { - /* Effective RIP could not be calculated */ - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - ctxt->fi.cr2 = 0; - return ES_EXCEPTION; - } - - if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) - return ES_DECODE_FAILED; - - if (ctxt->insn.immediate.got) - return ES_OK; - else - return ES_DECODE_FAILED; -} - -static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) -{ - char buffer[MAX_INSN_SIZE]; - int res, ret; - - res = vc_fetch_insn_kernel(ctxt, buffer); - if (res) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = X86_PF_INSTR; - ctxt->fi.cr2 = ctxt->regs->ip; - return ES_EXCEPTION; - } - - ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); - if (ret < 0) - return ES_DECODE_FAILED; - else - return ES_OK; -} - -static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) -{ - if (user_mode(ctxt->regs)) - return __vc_decode_user_insn(ctxt); - else - return __vc_decode_kern_insn(ctxt); -} - -static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, - char *dst, char *buf, size_t size) -{ - unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; - - /* - * This function uses __put_user() independent of whether kernel or user - * memory is accessed. This works fine because __put_user() does no - * sanity checks of the pointer being accessed. All that it does is - * to report when the access failed. - * - * Also, this function runs in atomic context, so __put_user() is not - * allowed to sleep. The page-fault handler detects that it is running - * in atomic context and will not try to take mmap_sem and handle the - * fault, so additional pagefault_enable()/disable() calls are not - * needed. - * - * The access can't be done via copy_to_user() here because - * vc_write_mem() must not use string instructions to access unsafe - * memory. The reason is that MOVS is emulated by the #VC handler by - * splitting the move up into a read and a write and taking a nested #VC - * exception on whatever of them is the MMIO access. Using string - * instructions here would cause infinite nesting. - */ - switch (size) { - case 1: { - u8 d1; - u8 __user *target = (u8 __user *)dst; - - memcpy(&d1, buf, 1); - if (__put_user(d1, target)) - goto fault; - break; - } - case 2: { - u16 d2; - u16 __user *target = (u16 __user *)dst; - - memcpy(&d2, buf, 2); - if (__put_user(d2, target)) - goto fault; - break; - } - case 4: { - u32 d4; - u32 __user *target = (u32 __user *)dst; - - memcpy(&d4, buf, 4); - if (__put_user(d4, target)) - goto fault; - break; - } - case 8: { - u64 d8; - u64 __user *target = (u64 __user *)dst; - - memcpy(&d8, buf, 8); - if (__put_user(d8, target)) - goto fault; - break; - } - default: - WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); - return ES_UNSUPPORTED; - } - - return ES_OK; - -fault: - if (user_mode(ctxt->regs)) - error_code |= X86_PF_USER; - - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = error_code; - ctxt->fi.cr2 = (unsigned long)dst; - - return ES_EXCEPTION; -} - -static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, - char *src, char *buf, size_t size) -{ - unsigned long error_code = X86_PF_PROT; - - /* - * This function uses __get_user() independent of whether kernel or user - * memory is accessed. This works fine because __get_user() does no - * sanity checks of the pointer being accessed. All that it does is - * to report when the access failed. - * - * Also, this function runs in atomic context, so __get_user() is not - * allowed to sleep. The page-fault handler detects that it is running - * in atomic context and will not try to take mmap_sem and handle the - * fault, so additional pagefault_enable()/disable() calls are not - * needed. - * - * The access can't be done via copy_from_user() here because - * vc_read_mem() must not use string instructions to access unsafe - * memory. The reason is that MOVS is emulated by the #VC handler by - * splitting the move up into a read and a write and taking a nested #VC - * exception on whatever of them is the MMIO access. Using string - * instructions here would cause infinite nesting. - */ - switch (size) { - case 1: { - u8 d1; - u8 __user *s = (u8 __user *)src; - - if (__get_user(d1, s)) - goto fault; - memcpy(buf, &d1, 1); - break; - } - case 2: { - u16 d2; - u16 __user *s = (u16 __user *)src; - - if (__get_user(d2, s)) - goto fault; - memcpy(buf, &d2, 2); - break; - } - case 4: { - u32 d4; - u32 __user *s = (u32 __user *)src; - - if (__get_user(d4, s)) - goto fault; - memcpy(buf, &d4, 4); - break; - } - case 8: { - u64 d8; - u64 __user *s = (u64 __user *)src; - if (__get_user(d8, s)) - goto fault; - memcpy(buf, &d8, 8); - break; - } - default: - WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); - return ES_UNSUPPORTED; - } - - return ES_OK; - -fault: - if (user_mode(ctxt->regs)) - error_code |= X86_PF_USER; - - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.error_code = error_code; - ctxt->fi.cr2 = (unsigned long)src; - - return ES_EXCEPTION; -} - -static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, - unsigned long vaddr, phys_addr_t *paddr) -{ - unsigned long va = (unsigned long)vaddr; - unsigned int level; - phys_addr_t pa; - pgd_t *pgd; - pte_t *pte; - - pgd = __va(read_cr3_pa()); - pgd = &pgd[pgd_index(va)]; - pte = lookup_address_in_pgd(pgd, va, &level); - if (!pte) { - ctxt->fi.vector = X86_TRAP_PF; - ctxt->fi.cr2 = vaddr; - ctxt->fi.error_code = 0; - - if (user_mode(ctxt->regs)) - ctxt->fi.error_code |= X86_PF_USER; - - return ES_EXCEPTION; - } - - if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) - /* Emulated MMIO to/from encrypted memory not supported */ - return ES_UNSUPPORTED; - - pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; - pa |= va & ~page_level_mask(level); - - *paddr = pa; - - return ES_OK; -} - -static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) -{ - BUG_ON(size > 4); - - if (user_mode(ctxt->regs)) { - struct thread_struct *t = ¤t->thread; - struct io_bitmap *iobm = t->io_bitmap; - size_t idx; - - if (!iobm) - goto fault; - - for (idx = port; idx < port + size; ++idx) { - if (test_bit(idx, iobm->bitmap)) - goto fault; - } - } - - return ES_OK; - -fault: - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - - return ES_EXCEPTION; -} - -static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) -{ - long error_code = ctxt->fi.error_code; - int trapnr = ctxt->fi.vector; - - ctxt->regs->orig_ax = ctxt->fi.error_code; - - switch (trapnr) { - case X86_TRAP_GP: - exc_general_protection(ctxt->regs, error_code); - break; - case X86_TRAP_UD: - exc_invalid_op(ctxt->regs); - break; - case X86_TRAP_PF: - write_cr2(ctxt->fi.cr2); - exc_page_fault(ctxt->regs, error_code); - break; - case X86_TRAP_AC: - exc_alignment_check(ctxt->regs, error_code); - break; - default: - pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); - BUG(); - } -} - /* Include code shared with pre-decompression boot stage */ #include "sev-shared.c" @@ -563,718 +253,6 @@ void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); } -/* Writes to the SVSM CAA MSR are ignored */ -static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) -{ - if (write) - return ES_OK; - - regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); - regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); - - return ES_OK; -} - -/* - * TSC related accesses should not exit to the hypervisor when a guest is - * executing with Secure TSC enabled, so special handling is required for - * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ. - */ -static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write) -{ - u64 tsc; - - /* - * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled. - * Terminate the SNP guest when the interception is enabled. - */ - if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ) - return ES_VMM_ERROR; - - /* - * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC - * to return undefined values, so ignore all writes. - * - * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use - * the value returned by rdtsc_ordered(). - */ - if (write) { - WARN_ONCE(1, "TSC MSR writes are verboten!\n"); - return ES_OK; - } - - tsc = rdtsc_ordered(); - regs->ax = lower_32_bits(tsc); - regs->dx = upper_32_bits(tsc); - - return ES_OK; -} - -static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct pt_regs *regs = ctxt->regs; - enum es_result ret; - bool write; - - /* Is it a WRMSR? */ - write = ctxt->insn.opcode.bytes[1] == 0x30; - - switch (regs->cx) { - case MSR_SVSM_CAA: - return __vc_handle_msr_caa(regs, write); - case MSR_IA32_TSC: - case MSR_AMD64_GUEST_TSC_FREQ: - if (sev_status & MSR_AMD64_SNP_SECURE_TSC) - return __vc_handle_secure_tsc_msrs(regs, write); - break; - default: - break; - } - - ghcb_set_rcx(ghcb, regs->cx); - if (write) { - ghcb_set_rax(ghcb, regs->ax); - ghcb_set_rdx(ghcb, regs->dx); - } - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); - - if ((ret == ES_OK) && !write) { - regs->ax = ghcb->save.rax; - regs->dx = ghcb->save.rdx; - } - - return ret; -} - -static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) -{ - int trapnr = ctxt->fi.vector; - - if (trapnr == X86_TRAP_PF) - native_write_cr2(ctxt->fi.cr2); - - ctxt->regs->orig_ax = ctxt->fi.error_code; - do_early_exception(ctxt->regs, trapnr); -} - -static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) -{ - long *reg_array; - int offset; - - reg_array = (long *)ctxt->regs; - offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); - - if (offset < 0) - return NULL; - - offset /= sizeof(long); - - return reg_array + offset; -} -static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, - unsigned int bytes, bool read) -{ - u64 exit_code, exit_info_1, exit_info_2; - unsigned long ghcb_pa = __pa(ghcb); - enum es_result res; - phys_addr_t paddr; - void __user *ref; - - ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); - if (ref == (void __user *)-1L) - return ES_UNSUPPORTED; - - exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; - - res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); - if (res != ES_OK) { - if (res == ES_EXCEPTION && !read) - ctxt->fi.error_code |= X86_PF_WRITE; - - return res; - } - - exit_info_1 = paddr; - /* Can never be greater than 8 */ - exit_info_2 = bytes; - - ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); - - return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); -} - -/* - * The MOVS instruction has two memory operands, which raises the - * problem that it is not known whether the access to the source or the - * destination caused the #VC exception (and hence whether an MMIO read - * or write operation needs to be emulated). - * - * Instead of playing games with walking page-tables and trying to guess - * whether the source or destination is an MMIO range, split the move - * into two operations, a read and a write with only one memory operand. - * This will cause a nested #VC exception on the MMIO address which can - * then be handled. - * - * This implementation has the benefit that it also supports MOVS where - * source _and_ destination are MMIO regions. - * - * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a - * rare operation. If it turns out to be a performance problem the split - * operations can be moved to memcpy_fromio() and memcpy_toio(). - */ -static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, - unsigned int bytes) -{ - unsigned long ds_base, es_base; - unsigned char *src, *dst; - unsigned char buffer[8]; - enum es_result ret; - bool rep; - int off; - - ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); - es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); - - if (ds_base == -1L || es_base == -1L) { - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; - } - - src = ds_base + (unsigned char *)ctxt->regs->si; - dst = es_base + (unsigned char *)ctxt->regs->di; - - ret = vc_read_mem(ctxt, src, buffer, bytes); - if (ret != ES_OK) - return ret; - - ret = vc_write_mem(ctxt, dst, buffer, bytes); - if (ret != ES_OK) - return ret; - - if (ctxt->regs->flags & X86_EFLAGS_DF) - off = -bytes; - else - off = bytes; - - ctxt->regs->si += off; - ctxt->regs->di += off; - - rep = insn_has_rep_prefix(&ctxt->insn); - if (rep) - ctxt->regs->cx -= 1; - - if (!rep || ctxt->regs->cx == 0) - return ES_OK; - else - return ES_RETRY; -} - -static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - struct insn *insn = &ctxt->insn; - enum insn_mmio_type mmio; - unsigned int bytes = 0; - enum es_result ret; - u8 sign_byte; - long *reg_data; - - mmio = insn_decode_mmio(insn, &bytes); - if (mmio == INSN_MMIO_DECODE_FAILED) - return ES_DECODE_FAILED; - - if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { - reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); - if (!reg_data) - return ES_DECODE_FAILED; - } - - if (user_mode(ctxt->regs)) - return ES_UNSUPPORTED; - - switch (mmio) { - case INSN_MMIO_WRITE: - memcpy(ghcb->shared_buffer, reg_data, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); - break; - case INSN_MMIO_WRITE_IMM: - memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); - break; - case INSN_MMIO_READ: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Zero-extend for 32-bit operation */ - if (bytes == 4) - *reg_data = 0; - - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_READ_ZERO_EXTEND: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Zero extend based on operand size */ - memset(reg_data, 0, insn->opnd_bytes); - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_READ_SIGN_EXTEND: - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - if (bytes == 1) { - u8 *val = (u8 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x80) ? 0xff : 0x00; - } else { - u16 *val = (u16 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x8000) ? 0xff : 0x00; - } - - /* Sign extend based on operand size */ - memset(reg_data, sign_byte, insn->opnd_bytes); - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - case INSN_MMIO_MOVS: - ret = vc_handle_mmio_movs(ctxt, bytes); - break; - default: - ret = ES_UNSUPPORTED; - break; - } - - return ret; -} - -static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct sev_es_runtime_data *data = this_cpu_read(runtime_data); - long val, *reg = vc_insn_get_rm(ctxt); - enum es_result ret; - - if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) - return ES_VMM_ERROR; - - if (!reg) - return ES_DECODE_FAILED; - - val = *reg; - - /* Upper 32 bits must be written as zeroes */ - if (val >> 32) { - ctxt->fi.vector = X86_TRAP_GP; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; - } - - /* Clear out other reserved bits and set bit 10 */ - val = (val & 0xffff23ffL) | BIT(10); - - /* Early non-zero writes to DR7 are not supported */ - if (!data && (val & ~DR7_RESET_VALUE)) - return ES_UNSUPPORTED; - - /* Using a value of 0 for ExitInfo1 means RAX holds the value */ - ghcb_set_rax(ghcb, val); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); - if (ret != ES_OK) - return ret; - - if (data) - data->dr7 = val; - - return ES_OK; -} - -static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct sev_es_runtime_data *data = this_cpu_read(runtime_data); - long *reg = vc_insn_get_rm(ctxt); - - if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) - return ES_VMM_ERROR; - - if (!reg) - return ES_DECODE_FAILED; - - if (data) - *reg = data->dr7; - else - *reg = DR7_RESET_VALUE; - - return ES_OK; -} - -static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); -} - -static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) -{ - enum es_result ret; - - ghcb_set_rcx(ghcb, ctxt->regs->cx); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); - if (ret != ES_OK) - return ret; - - if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) - return ES_VMM_ERROR; - - ctxt->regs->ax = ghcb->save.rax; - ctxt->regs->dx = ghcb->save.rdx; - - return ES_OK; -} - -static enum es_result vc_handle_monitor(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* - * Treat it as a NOP and do not leak a physical address to the - * hypervisor. - */ - return ES_OK; -} - -static enum es_result vc_handle_mwait(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* Treat the same as MONITOR/MONITORX */ - return ES_OK; -} - -static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - enum es_result ret; - - ghcb_set_rax(ghcb, ctxt->regs->ax); - ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); - - if (x86_platform.hyper.sev_es_hcall_prepare) - x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); - - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); - if (ret != ES_OK) - return ret; - - if (!ghcb_rax_is_valid(ghcb)) - return ES_VMM_ERROR; - - ctxt->regs->ax = ghcb->save.rax; - - /* - * Call sev_es_hcall_finish() after regs->ax is already set. - * This allows the hypervisor handler to overwrite it again if - * necessary. - */ - if (x86_platform.hyper.sev_es_hcall_finish && - !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) - return ES_VMM_ERROR; - - return ES_OK; -} - -static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - /* - * Calling ecx_alignment_check() directly does not work, because it - * enables IRQs and the GHCB is active. Forward the exception and call - * it later from vc_forward_exception(). - */ - ctxt->fi.vector = X86_TRAP_AC; - ctxt->fi.error_code = 0; - return ES_EXCEPTION; -} - -static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, - struct ghcb *ghcb, - unsigned long exit_code) -{ - enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); - - if (result != ES_OK) - return result; - - switch (exit_code) { - case SVM_EXIT_READ_DR7: - result = vc_handle_dr7_read(ghcb, ctxt); - break; - case SVM_EXIT_WRITE_DR7: - result = vc_handle_dr7_write(ghcb, ctxt); - break; - case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: - result = vc_handle_trap_ac(ghcb, ctxt); - break; - case SVM_EXIT_RDTSC: - case SVM_EXIT_RDTSCP: - result = vc_handle_rdtsc(ghcb, ctxt, exit_code); - break; - case SVM_EXIT_RDPMC: - result = vc_handle_rdpmc(ghcb, ctxt); - break; - case SVM_EXIT_INVD: - pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); - result = ES_UNSUPPORTED; - break; - case SVM_EXIT_CPUID: - result = vc_handle_cpuid(ghcb, ctxt); - break; - case SVM_EXIT_IOIO: - result = vc_handle_ioio(ghcb, ctxt); - break; - case SVM_EXIT_MSR: - result = vc_handle_msr(ghcb, ctxt); - break; - case SVM_EXIT_VMMCALL: - result = vc_handle_vmmcall(ghcb, ctxt); - break; - case SVM_EXIT_WBINVD: - result = vc_handle_wbinvd(ghcb, ctxt); - break; - case SVM_EXIT_MONITOR: - result = vc_handle_monitor(ghcb, ctxt); - break; - case SVM_EXIT_MWAIT: - result = vc_handle_mwait(ghcb, ctxt); - break; - case SVM_EXIT_NPF: - result = vc_handle_mmio(ghcb, ctxt); - break; - default: - /* - * Unexpected #VC exception - */ - result = ES_UNSUPPORTED; - } - - return result; -} - -static __always_inline bool is_vc2_stack(unsigned long sp) -{ - return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); -} - -static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) -{ - unsigned long sp, prev_sp; - - sp = (unsigned long)regs; - prev_sp = regs->sp; - - /* - * If the code was already executing on the VC2 stack when the #VC - * happened, let it proceed to the normal handling routine. This way the - * code executing on the VC2 stack can cause #VC exceptions to get handled. - */ - return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); -} - -static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) -{ - struct ghcb_state state; - struct es_em_ctxt ctxt; - enum es_result result; - struct ghcb *ghcb; - bool ret = true; - - ghcb = __sev_get_ghcb(&state); - - vc_ghcb_invalidate(ghcb); - result = vc_init_em_ctxt(&ctxt, regs, error_code); - - if (result == ES_OK) - result = vc_handle_exitcode(&ctxt, ghcb, error_code); - - __sev_put_ghcb(&state); - - /* Done - now check the result */ - switch (result) { - case ES_OK: - vc_finish_insn(&ctxt); - break; - case ES_UNSUPPORTED: - pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_VMM_ERROR: - pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_DECODE_FAILED: - pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", - error_code, regs->ip); - ret = false; - break; - case ES_EXCEPTION: - vc_forward_exception(&ctxt); - break; - case ES_RETRY: - /* Nothing to do */ - break; - default: - pr_emerg("Unknown result in %s():%d\n", __func__, result); - /* - * Emulating the instruction which caused the #VC exception - * failed - can't continue so print debug information - */ - BUG(); - } - - return ret; -} - -static __always_inline bool vc_is_db(unsigned long error_code) -{ - return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; -} - -/* - * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode - * and will panic when an error happens. - */ -DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) -{ - irqentry_state_t irq_state; - - /* - * With the current implementation it is always possible to switch to a - * safe stack because #VC exceptions only happen at known places, like - * intercepted instructions or accesses to MMIO areas/IO ports. They can - * also happen with code instrumentation when the hypervisor intercepts - * #DB, but the critical paths are forbidden to be instrumented, so #DB - * exceptions currently also only happen in safe places. - * - * But keep this here in case the noinstr annotations are violated due - * to bug elsewhere. - */ - if (unlikely(vc_from_invalid_context(regs))) { - instrumentation_begin(); - panic("Can't handle #VC exception from unsupported context\n"); - instrumentation_end(); - } - - /* - * Handle #DB before calling into !noinstr code to avoid recursive #DB. - */ - if (vc_is_db(error_code)) { - exc_debug(regs); - return; - } - - irq_state = irqentry_nmi_enter(regs); - - instrumentation_begin(); - - if (!vc_raw_handle_exception(regs, error_code)) { - /* Show some debug info */ - show_regs(regs); - - /* Ask hypervisor to sev_es_terminate */ - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); - - /* If that fails and we get here - just panic */ - panic("Returned from Terminate-Request to Hypervisor\n"); - } - - instrumentation_end(); - irqentry_nmi_exit(regs, irq_state); -} - -/* - * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode - * and will kill the current task with SIGBUS when an error happens. - */ -DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) -{ - /* - * Handle #DB before calling into !noinstr code to avoid recursive #DB. - */ - if (vc_is_db(error_code)) { - noist_exc_debug(regs); - return; - } - - irqentry_enter_from_user_mode(regs); - instrumentation_begin(); - - if (!vc_raw_handle_exception(regs, error_code)) { - /* - * Do not kill the machine if user-space triggered the - * exception. Send SIGBUS instead and let user-space deal with - * it. - */ - force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); - } - - instrumentation_end(); - irqentry_exit_to_user_mode(regs); -} - -bool __init handle_vc_boot_ghcb(struct pt_regs *regs) -{ - unsigned long exit_code = regs->orig_ax; - struct es_em_ctxt ctxt; - enum es_result result; - - vc_ghcb_invalidate(boot_ghcb); - - result = vc_init_em_ctxt(&ctxt, regs, exit_code); - if (result == ES_OK) - result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); - - /* Done - now check the result */ - switch (result) { - case ES_OK: - vc_finish_insn(&ctxt); - break; - case ES_UNSUPPORTED: - early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_VMM_ERROR: - early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_DECODE_FAILED: - early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", - exit_code, regs->ip); - goto fail; - case ES_EXCEPTION: - vc_early_forward_exception(&ctxt); - break; - case ES_RETRY: - /* Nothing to do */ - break; - default: - BUG(); - } - - return true; - -fail: - show_regs(regs); - - sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); -} - /* * Initial set up of SNP relies on information provided by the * Confidential Computing blob, which can be passed to the kernel diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile index 2919dcfc4b02..db3255b979bd 100644 --- a/arch/x86/coco/sev/Makefile +++ b/arch/x86/coco/sev/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += core.o sev-nmi.o +obj-y += core.o sev-nmi.o vc-handle.o # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining UBSAN_SANITIZE_sev-nmi.o := n diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c new file mode 100644 index 000000000000..b4895c648024 --- /dev/null +++ b/arch/x86/coco/sev/vc-handle.c @@ -0,0 +1,1061 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Memory Encryption Support + * + * Copyright (C) 2019 SUSE + * + * Author: Joerg Roedel + */ + +#define pr_fmt(fmt) "SEV: " fmt + +#include /* For show_regs() */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned long vaddr, phys_addr_t *paddr) +{ + unsigned long va = (unsigned long)vaddr; + unsigned int level; + phys_addr_t pa; + pgd_t *pgd; + pte_t *pte; + + pgd = __va(read_cr3_pa()); + pgd = &pgd[pgd_index(va)]; + pte = lookup_address_in_pgd(pgd, va, &level); + if (!pte) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.cr2 = vaddr; + ctxt->fi.error_code = 0; + + if (user_mode(ctxt->regs)) + ctxt->fi.error_code |= X86_PF_USER; + + return ES_EXCEPTION; + } + + if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) + /* Emulated MMIO to/from encrypted memory not supported */ + return ES_UNSUPPORTED; + + pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; + pa |= va & ~page_level_mask(level); + + *paddr = pa; + + return ES_OK; +} + +static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) +{ + BUG_ON(size > 4); + + if (user_mode(ctxt->regs)) { + struct thread_struct *t = ¤t->thread; + struct io_bitmap *iobm = t->io_bitmap; + size_t idx; + + if (!iobm) + goto fault; + + for (idx = port; idx < port + size; ++idx) { + if (test_bit(idx, iobm->bitmap)) + goto fault; + } + } + + return ES_OK; + +fault: + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + + return ES_EXCEPTION; +} + +void vc_forward_exception(struct es_em_ctxt *ctxt) +{ + long error_code = ctxt->fi.error_code; + int trapnr = ctxt->fi.vector; + + ctxt->regs->orig_ax = ctxt->fi.error_code; + + switch (trapnr) { + case X86_TRAP_GP: + exc_general_protection(ctxt->regs, error_code); + break; + case X86_TRAP_UD: + exc_invalid_op(ctxt->regs); + break; + case X86_TRAP_PF: + write_cr2(ctxt->fi.cr2); + exc_page_fault(ctxt->regs, error_code); + break; + case X86_TRAP_AC: + exc_alignment_check(ctxt->regs, error_code); + break; + default: + pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); + BUG(); + } +} + +static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, + unsigned char *buffer) +{ + return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); +} + +static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + int insn_bytes; + + insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); + if (insn_bytes == 0) { + /* Nothing could be copied */ + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } else if (insn_bytes == -EINVAL) { + /* Effective RIP could not be calculated */ + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + ctxt->fi.cr2 = 0; + return ES_EXCEPTION; + } + + if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) + return ES_DECODE_FAILED; + + if (ctxt->insn.immediate.got) + return ES_OK; + else + return ES_DECODE_FAILED; +} + +static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) +{ + char buffer[MAX_INSN_SIZE]; + int res, ret; + + res = vc_fetch_insn_kernel(ctxt, buffer); + if (res) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_INSTR; + ctxt->fi.cr2 = ctxt->regs->ip; + return ES_EXCEPTION; + } + + ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); + if (ret < 0) + return ES_DECODE_FAILED; + else + return ES_OK; +} + +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) +{ + if (user_mode(ctxt->regs)) + return __vc_decode_user_insn(ctxt); + else + return __vc_decode_kern_insn(ctxt); +} + +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, + char *dst, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; + + /* + * This function uses __put_user() independent of whether kernel or user + * memory is accessed. This works fine because __put_user() does no + * sanity checks of the pointer being accessed. All that it does is + * to report when the access failed. + * + * Also, this function runs in atomic context, so __put_user() is not + * allowed to sleep. The page-fault handler detects that it is running + * in atomic context and will not try to take mmap_sem and handle the + * fault, so additional pagefault_enable()/disable() calls are not + * needed. + * + * The access can't be done via copy_to_user() here because + * vc_write_mem() must not use string instructions to access unsafe + * memory. The reason is that MOVS is emulated by the #VC handler by + * splitting the move up into a read and a write and taking a nested #VC + * exception on whatever of them is the MMIO access. Using string + * instructions here would cause infinite nesting. + */ + switch (size) { + case 1: { + u8 d1; + u8 __user *target = (u8 __user *)dst; + + memcpy(&d1, buf, 1); + if (__put_user(d1, target)) + goto fault; + break; + } + case 2: { + u16 d2; + u16 __user *target = (u16 __user *)dst; + + memcpy(&d2, buf, 2); + if (__put_user(d2, target)) + goto fault; + break; + } + case 4: { + u32 d4; + u32 __user *target = (u32 __user *)dst; + + memcpy(&d4, buf, 4); + if (__put_user(d4, target)) + goto fault; + break; + } + case 8: { + u64 d8; + u64 __user *target = (u64 __user *)dst; + + memcpy(&d8, buf, 8); + if (__put_user(d8, target)) + goto fault; + break; + } + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)dst; + + return ES_EXCEPTION; +} + +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, + char *src, char *buf, size_t size) +{ + unsigned long error_code = X86_PF_PROT; + + /* + * This function uses __get_user() independent of whether kernel or user + * memory is accessed. This works fine because __get_user() does no + * sanity checks of the pointer being accessed. All that it does is + * to report when the access failed. + * + * Also, this function runs in atomic context, so __get_user() is not + * allowed to sleep. The page-fault handler detects that it is running + * in atomic context and will not try to take mmap_sem and handle the + * fault, so additional pagefault_enable()/disable() calls are not + * needed. + * + * The access can't be done via copy_from_user() here because + * vc_read_mem() must not use string instructions to access unsafe + * memory. The reason is that MOVS is emulated by the #VC handler by + * splitting the move up into a read and a write and taking a nested #VC + * exception on whatever of them is the MMIO access. Using string + * instructions here would cause infinite nesting. + */ + switch (size) { + case 1: { + u8 d1; + u8 __user *s = (u8 __user *)src; + + if (__get_user(d1, s)) + goto fault; + memcpy(buf, &d1, 1); + break; + } + case 2: { + u16 d2; + u16 __user *s = (u16 __user *)src; + + if (__get_user(d2, s)) + goto fault; + memcpy(buf, &d2, 2); + break; + } + case 4: { + u32 d4; + u32 __user *s = (u32 __user *)src; + + if (__get_user(d4, s)) + goto fault; + memcpy(buf, &d4, 4); + break; + } + case 8: { + u64 d8; + u64 __user *s = (u64 __user *)src; + if (__get_user(d8, s)) + goto fault; + memcpy(buf, &d8, 8); + break; + } + default: + WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); + return ES_UNSUPPORTED; + } + + return ES_OK; + +fault: + if (user_mode(ctxt->regs)) + error_code |= X86_PF_USER; + + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = error_code; + ctxt->fi.cr2 = (unsigned long)src; + + return ES_EXCEPTION; +} + +#define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__) + +#include "vc-shared.c" + +/* Writes to the SVSM CAA MSR are ignored */ +static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) +{ + if (write) + return ES_OK; + + regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); + regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); + + return ES_OK; +} + +/* + * TSC related accesses should not exit to the hypervisor when a guest is + * executing with Secure TSC enabled, so special handling is required for + * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ. + */ +static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write) +{ + u64 tsc; + + /* + * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled. + * Terminate the SNP guest when the interception is enabled. + */ + if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ) + return ES_VMM_ERROR; + + /* + * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC + * to return undefined values, so ignore all writes. + * + * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use + * the value returned by rdtsc_ordered(). + */ + if (write) { + WARN_ONCE(1, "TSC MSR writes are verboten!\n"); + return ES_OK; + } + + tsc = rdtsc_ordered(); + regs->ax = lower_32_bits(tsc); + regs->dx = upper_32_bits(tsc); + + return ES_OK; +} + +static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + enum es_result ret; + bool write; + + /* Is it a WRMSR? */ + write = ctxt->insn.opcode.bytes[1] == 0x30; + + switch (regs->cx) { + case MSR_SVSM_CAA: + return __vc_handle_msr_caa(regs, write); + case MSR_IA32_TSC: + case MSR_AMD64_GUEST_TSC_FREQ: + if (sev_status & MSR_AMD64_SNP_SECURE_TSC) + return __vc_handle_secure_tsc_msrs(regs, write); + break; + default: + break; + } + + ghcb_set_rcx(ghcb, regs->cx); + if (write) { + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rdx(ghcb, regs->dx); + } + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); + + if ((ret == ES_OK) && !write) { + regs->ax = ghcb->save.rax; + regs->dx = ghcb->save.rdx; + } + + return ret; +} + +static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) +{ + int trapnr = ctxt->fi.vector; + + if (trapnr == X86_TRAP_PF) + native_write_cr2(ctxt->fi.cr2); + + ctxt->regs->orig_ax = ctxt->fi.error_code; + do_early_exception(ctxt->regs, trapnr); +} + +static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) +{ + long *reg_array; + int offset; + + reg_array = (long *)ctxt->regs; + offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); + + if (offset < 0) + return NULL; + + offset /= sizeof(long); + + return reg_array + offset; +} +static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, + unsigned int bytes, bool read) +{ + u64 exit_code, exit_info_1, exit_info_2; + unsigned long ghcb_pa = __pa(ghcb); + enum es_result res; + phys_addr_t paddr; + void __user *ref; + + ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); + if (ref == (void __user *)-1L) + return ES_UNSUPPORTED; + + exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; + + res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); + if (res != ES_OK) { + if (res == ES_EXCEPTION && !read) + ctxt->fi.error_code |= X86_PF_WRITE; + + return res; + } + + exit_info_1 = paddr; + /* Can never be greater than 8 */ + exit_info_2 = bytes; + + ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); + + return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); +} + +/* + * The MOVS instruction has two memory operands, which raises the + * problem that it is not known whether the access to the source or the + * destination caused the #VC exception (and hence whether an MMIO read + * or write operation needs to be emulated). + * + * Instead of playing games with walking page-tables and trying to guess + * whether the source or destination is an MMIO range, split the move + * into two operations, a read and a write with only one memory operand. + * This will cause a nested #VC exception on the MMIO address which can + * then be handled. + * + * This implementation has the benefit that it also supports MOVS where + * source _and_ destination are MMIO regions. + * + * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a + * rare operation. If it turns out to be a performance problem the split + * operations can be moved to memcpy_fromio() and memcpy_toio(). + */ +static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, + unsigned int bytes) +{ + unsigned long ds_base, es_base; + unsigned char *src, *dst; + unsigned char buffer[8]; + enum es_result ret; + bool rep; + int off; + + ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + if (ds_base == -1L || es_base == -1L) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + src = ds_base + (unsigned char *)ctxt->regs->si; + dst = es_base + (unsigned char *)ctxt->regs->di; + + ret = vc_read_mem(ctxt, src, buffer, bytes); + if (ret != ES_OK) + return ret; + + ret = vc_write_mem(ctxt, dst, buffer, bytes); + if (ret != ES_OK) + return ret; + + if (ctxt->regs->flags & X86_EFLAGS_DF) + off = -bytes; + else + off = bytes; + + ctxt->regs->si += off; + ctxt->regs->di += off; + + rep = insn_has_rep_prefix(&ctxt->insn); + if (rep) + ctxt->regs->cx -= 1; + + if (!rep || ctxt->regs->cx == 0) + return ES_OK; + else + return ES_RETRY; +} + +static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct insn *insn = &ctxt->insn; + enum insn_mmio_type mmio; + unsigned int bytes = 0; + enum es_result ret; + u8 sign_byte; + long *reg_data; + + mmio = insn_decode_mmio(insn, &bytes); + if (mmio == INSN_MMIO_DECODE_FAILED) + return ES_DECODE_FAILED; + + if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { + reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); + if (!reg_data) + return ES_DECODE_FAILED; + } + + if (user_mode(ctxt->regs)) + return ES_UNSUPPORTED; + + switch (mmio) { + case INSN_MMIO_WRITE: + memcpy(ghcb->shared_buffer, reg_data, bytes); + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + case INSN_MMIO_WRITE_IMM: + memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); + ret = vc_do_mmio(ghcb, ctxt, bytes, false); + break; + case INSN_MMIO_READ: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero-extend for 32-bit operation */ + if (bytes == 4) + *reg_data = 0; + + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_ZERO_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + /* Zero extend based on operand size */ + memset(reg_data, 0, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_READ_SIGN_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + if (bytes == 1) { + u8 *val = (u8 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + u16 *val = (u16 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + + /* Sign extend based on operand size */ + memset(reg_data, sign_byte, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case INSN_MMIO_MOVS: + ret = vc_handle_mmio_movs(ctxt, bytes); + break; + default: + ret = ES_UNSUPPORTED; + break; + } + + return ret; +} + +static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long val, *reg = vc_insn_get_rm(ctxt); + enum es_result ret; + + if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) + return ES_VMM_ERROR; + + if (!reg) + return ES_DECODE_FAILED; + + val = *reg; + + /* Upper 32 bits must be written as zeroes */ + if (val >> 32) { + ctxt->fi.vector = X86_TRAP_GP; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; + } + + /* Clear out other reserved bits and set bit 10 */ + val = (val & 0xffff23ffL) | BIT(10); + + /* Early non-zero writes to DR7 are not supported */ + if (!data && (val & ~DR7_RESET_VALUE)) + return ES_UNSUPPORTED; + + /* Using a value of 0 for ExitInfo1 means RAX holds the value */ + ghcb_set_rax(ghcb, val); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); + if (ret != ES_OK) + return ret; + + if (data) + data->dr7 = val; + + return ES_OK; +} + +static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct sev_es_runtime_data *data = this_cpu_read(runtime_data); + long *reg = vc_insn_get_rm(ctxt); + + if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) + return ES_VMM_ERROR; + + if (!reg) + return ES_DECODE_FAILED; + + if (data) + *reg = data->dr7; + else + *reg = DR7_RESET_VALUE; + + return ES_OK; +} + +static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); +} + +static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rcx(ghcb, ctxt->regs->cx); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + + return ES_OK; +} + +static enum es_result vc_handle_monitor(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Treat it as a NOP and do not leak a physical address to the + * hypervisor. + */ + return ES_OK; +} + +static enum es_result vc_handle_mwait(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* Treat the same as MONITOR/MONITORX */ + return ES_OK; +} + +static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + enum es_result ret; + + ghcb_set_rax(ghcb, ctxt->regs->ax); + ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); + + if (x86_platform.hyper.sev_es_hcall_prepare) + x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); + if (ret != ES_OK) + return ret; + + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + + /* + * Call sev_es_hcall_finish() after regs->ax is already set. + * This allows the hypervisor handler to overwrite it again if + * necessary. + */ + if (x86_platform.hyper.sev_es_hcall_finish && + !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) + return ES_VMM_ERROR; + + return ES_OK; +} + +static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + /* + * Calling ecx_alignment_check() directly does not work, because it + * enables IRQs and the GHCB is active. Forward the exception and call + * it later from vc_forward_exception(). + */ + ctxt->fi.vector = X86_TRAP_AC; + ctxt->fi.error_code = 0; + return ES_EXCEPTION; +} + +static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, + struct ghcb *ghcb, + unsigned long exit_code) +{ + enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); + + if (result != ES_OK) + return result; + + switch (exit_code) { + case SVM_EXIT_READ_DR7: + result = vc_handle_dr7_read(ghcb, ctxt); + break; + case SVM_EXIT_WRITE_DR7: + result = vc_handle_dr7_write(ghcb, ctxt); + break; + case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: + result = vc_handle_trap_ac(ghcb, ctxt); + break; + case SVM_EXIT_RDTSC: + case SVM_EXIT_RDTSCP: + result = vc_handle_rdtsc(ghcb, ctxt, exit_code); + break; + case SVM_EXIT_RDPMC: + result = vc_handle_rdpmc(ghcb, ctxt); + break; + case SVM_EXIT_INVD: + pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); + result = ES_UNSUPPORTED; + break; + case SVM_EXIT_CPUID: + result = vc_handle_cpuid(ghcb, ctxt); + break; + case SVM_EXIT_IOIO: + result = vc_handle_ioio(ghcb, ctxt); + break; + case SVM_EXIT_MSR: + result = vc_handle_msr(ghcb, ctxt); + break; + case SVM_EXIT_VMMCALL: + result = vc_handle_vmmcall(ghcb, ctxt); + break; + case SVM_EXIT_WBINVD: + result = vc_handle_wbinvd(ghcb, ctxt); + break; + case SVM_EXIT_MONITOR: + result = vc_handle_monitor(ghcb, ctxt); + break; + case SVM_EXIT_MWAIT: + result = vc_handle_mwait(ghcb, ctxt); + break; + case SVM_EXIT_NPF: + result = vc_handle_mmio(ghcb, ctxt); + break; + default: + /* + * Unexpected #VC exception + */ + result = ES_UNSUPPORTED; + } + + return result; +} + +static __always_inline bool is_vc2_stack(unsigned long sp) +{ + return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); +} + +static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) +{ + unsigned long sp, prev_sp; + + sp = (unsigned long)regs; + prev_sp = regs->sp; + + /* + * If the code was already executing on the VC2 stack when the #VC + * happened, let it proceed to the normal handling routine. This way the + * code executing on the VC2 stack can cause #VC exceptions to get handled. + */ + return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); +} + +static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) +{ + struct ghcb_state state; + struct es_em_ctxt ctxt; + enum es_result result; + struct ghcb *ghcb; + bool ret = true; + + ghcb = __sev_get_ghcb(&state); + + vc_ghcb_invalidate(ghcb); + result = vc_init_em_ctxt(&ctxt, regs, error_code); + + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, ghcb, error_code); + + __sev_put_ghcb(&state); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_VMM_ERROR: + pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_DECODE_FAILED: + pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + error_code, regs->ip); + ret = false; + break; + case ES_EXCEPTION: + vc_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + pr_emerg("Unknown result in %s():%d\n", __func__, result); + /* + * Emulating the instruction which caused the #VC exception + * failed - can't continue so print debug information + */ + BUG(); + } + + return ret; +} + +static __always_inline bool vc_is_db(unsigned long error_code) +{ + return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; +} + +/* + * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode + * and will panic when an error happens. + */ +DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) +{ + irqentry_state_t irq_state; + + /* + * With the current implementation it is always possible to switch to a + * safe stack because #VC exceptions only happen at known places, like + * intercepted instructions or accesses to MMIO areas/IO ports. They can + * also happen with code instrumentation when the hypervisor intercepts + * #DB, but the critical paths are forbidden to be instrumented, so #DB + * exceptions currently also only happen in safe places. + * + * But keep this here in case the noinstr annotations are violated due + * to bug elsewhere. + */ + if (unlikely(vc_from_invalid_context(regs))) { + instrumentation_begin(); + panic("Can't handle #VC exception from unsupported context\n"); + instrumentation_end(); + } + + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (vc_is_db(error_code)) { + exc_debug(regs); + return; + } + + irq_state = irqentry_nmi_enter(regs); + + instrumentation_begin(); + + if (!vc_raw_handle_exception(regs, error_code)) { + /* Show some debug info */ + show_regs(regs); + + /* Ask hypervisor to sev_es_terminate */ + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); + + /* If that fails and we get here - just panic */ + panic("Returned from Terminate-Request to Hypervisor\n"); + } + + instrumentation_end(); + irqentry_nmi_exit(regs, irq_state); +} + +/* + * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode + * and will kill the current task with SIGBUS when an error happens. + */ +DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) +{ + /* + * Handle #DB before calling into !noinstr code to avoid recursive #DB. + */ + if (vc_is_db(error_code)) { + noist_exc_debug(regs); + return; + } + + irqentry_enter_from_user_mode(regs); + instrumentation_begin(); + + if (!vc_raw_handle_exception(regs, error_code)) { + /* + * Do not kill the machine if user-space triggered the + * exception. Send SIGBUS instead and let user-space deal with + * it. + */ + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); + } + + instrumentation_end(); + irqentry_exit_to_user_mode(regs); +} + +bool __init handle_vc_boot_ghcb(struct pt_regs *regs) +{ + unsigned long exit_code = regs->orig_ax; + struct es_em_ctxt ctxt; + enum es_result result; + + vc_ghcb_invalidate(boot_ghcb); + + result = vc_init_em_ctxt(&ctxt, regs, exit_code); + if (result == ES_OK) + result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); + + /* Done - now check the result */ + switch (result) { + case ES_OK: + vc_finish_insn(&ctxt); + break; + case ES_UNSUPPORTED: + early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_VMM_ERROR: + early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_DECODE_FAILED: + early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", + exit_code, regs->ip); + goto fail; + case ES_EXCEPTION: + vc_early_forward_exception(&ctxt); + break; + case ES_RETRY: + /* Nothing to do */ + break; + default: + BUG(); + } + + return true; + +fail: + show_regs(regs); + + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); +} + diff --git a/arch/x86/coco/sev/vc-shared.c b/arch/x86/coco/sev/vc-shared.c new file mode 100644 index 000000000000..2c0ab0fdc060 --- /dev/null +++ b/arch/x86/coco/sev/vc-shared.c @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: GPL-2.0 + +static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt, + unsigned long exit_code) +{ + unsigned int opcode = (unsigned int)ctxt->insn.opcode.value; + u8 modrm = ctxt->insn.modrm.value; + + switch (exit_code) { + + case SVM_EXIT_IOIO: + case SVM_EXIT_NPF: + /* handled separately */ + return ES_OK; + + case SVM_EXIT_CPUID: + if (opcode == 0xa20f) + return ES_OK; + break; + + case SVM_EXIT_INVD: + if (opcode == 0x080f) + return ES_OK; + break; + + case SVM_EXIT_MONITOR: + /* MONITOR and MONITORX instructions generate the same error code */ + if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa)) + return ES_OK; + break; + + case SVM_EXIT_MWAIT: + /* MWAIT and MWAITX instructions generate the same error code */ + if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb)) + return ES_OK; + break; + + case SVM_EXIT_MSR: + /* RDMSR */ + if (opcode == 0x320f || + /* WRMSR */ + opcode == 0x300f) + return ES_OK; + break; + + case SVM_EXIT_RDPMC: + if (opcode == 0x330f) + return ES_OK; + break; + + case SVM_EXIT_RDTSC: + if (opcode == 0x310f) + return ES_OK; + break; + + case SVM_EXIT_RDTSCP: + if (opcode == 0x010f && modrm == 0xf9) + return ES_OK; + break; + + case SVM_EXIT_READ_DR7: + if (opcode == 0x210f && + X86_MODRM_REG(ctxt->insn.modrm.value) == 7) + return ES_OK; + break; + + case SVM_EXIT_VMMCALL: + if (opcode == 0x010f && modrm == 0xd9) + return ES_OK; + + break; + + case SVM_EXIT_WRITE_DR7: + if (opcode == 0x230f && + X86_MODRM_REG(ctxt->insn.modrm.value) == 7) + return ES_OK; + break; + + case SVM_EXIT_WBINVD: + if (opcode == 0x90f) + return ES_OK; + break; + + default: + break; + } + + sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n", + opcode, exit_code, ctxt->regs->ip); + + return ES_UNSUPPORTED; +} + +static bool vc_decoding_needed(unsigned long exit_code) +{ + /* Exceptions don't require to decode the instruction */ + return !(exit_code >= SVM_EXIT_EXCP_BASE && + exit_code <= SVM_EXIT_LAST_EXCP); +} + +static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, + struct pt_regs *regs, + unsigned long exit_code) +{ + enum es_result ret = ES_OK; + + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->regs = regs; + + if (vc_decoding_needed(exit_code)) + ret = vc_decode_insn(ctxt); + + return ret; +} + +static void vc_finish_insn(struct es_em_ctxt *ctxt) +{ + ctxt->regs->ip += ctxt->insn.length; +} + +static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, + unsigned long address, + bool write) +{ + if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { + ctxt->fi.vector = X86_TRAP_PF; + ctxt->fi.error_code = X86_PF_USER; + ctxt->fi.cr2 = address; + if (write) + ctxt->fi.error_code |= X86_PF_WRITE; + + return ES_EXCEPTION; + } + + return ES_OK; +} + +static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, + void *src, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, b = backwards ? -1 : 1; + unsigned long address = (unsigned long)src; + enum es_result ret; + + ret = vc_insn_string_check(ctxt, address, false); + if (ret != ES_OK) + return ret; + + for (i = 0; i < count; i++) { + void *s = src + (i * data_size * b); + char *d = buf + (i * data_size); + + ret = vc_read_mem(ctxt, s, d, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} + +static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, + void *dst, char *buf, + unsigned int data_size, + unsigned int count, + bool backwards) +{ + int i, s = backwards ? -1 : 1; + unsigned long address = (unsigned long)dst; + enum es_result ret; + + ret = vc_insn_string_check(ctxt, address, true); + if (ret != ES_OK) + return ret; + + for (i = 0; i < count; i++) { + void *d = dst + (i * data_size * s); + char *b = buf + (i * data_size); + + ret = vc_write_mem(ctxt, d, b, data_size); + if (ret != ES_OK) + break; + } + + return ret; +} + +#define IOIO_TYPE_STR BIT(2) +#define IOIO_TYPE_IN 1 +#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) +#define IOIO_TYPE_OUT 0 +#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) + +#define IOIO_REP BIT(3) + +#define IOIO_ADDR_64 BIT(9) +#define IOIO_ADDR_32 BIT(8) +#define IOIO_ADDR_16 BIT(7) + +#define IOIO_DATA_32 BIT(6) +#define IOIO_DATA_16 BIT(5) +#define IOIO_DATA_8 BIT(4) + +#define IOIO_SEG_ES (0 << 10) +#define IOIO_SEG_DS (3 << 10) + +static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) +{ + struct insn *insn = &ctxt->insn; + size_t size; + u64 port; + + *exitinfo = 0; + + switch (insn->opcode.bytes[0]) { + /* INS opcodes */ + case 0x6c: + case 0x6d: + *exitinfo |= IOIO_TYPE_INS; + *exitinfo |= IOIO_SEG_ES; + port = ctxt->regs->dx & 0xffff; + break; + + /* OUTS opcodes */ + case 0x6e: + case 0x6f: + *exitinfo |= IOIO_TYPE_OUTS; + *exitinfo |= IOIO_SEG_DS; + port = ctxt->regs->dx & 0xffff; + break; + + /* IN immediate opcodes */ + case 0xe4: + case 0xe5: + *exitinfo |= IOIO_TYPE_IN; + port = (u8)insn->immediate.value & 0xffff; + break; + + /* OUT immediate opcodes */ + case 0xe6: + case 0xe7: + *exitinfo |= IOIO_TYPE_OUT; + port = (u8)insn->immediate.value & 0xffff; + break; + + /* IN register opcodes */ + case 0xec: + case 0xed: + *exitinfo |= IOIO_TYPE_IN; + port = ctxt->regs->dx & 0xffff; + break; + + /* OUT register opcodes */ + case 0xee: + case 0xef: + *exitinfo |= IOIO_TYPE_OUT; + port = ctxt->regs->dx & 0xffff; + break; + + default: + return ES_DECODE_FAILED; + } + + *exitinfo |= port << 16; + + switch (insn->opcode.bytes[0]) { + case 0x6c: + case 0x6e: + case 0xe4: + case 0xe6: + case 0xec: + case 0xee: + /* Single byte opcodes */ + *exitinfo |= IOIO_DATA_8; + size = 1; + break; + default: + /* Length determined by instruction parsing */ + *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 + : IOIO_DATA_32; + size = (insn->opnd_bytes == 2) ? 2 : 4; + } + + switch (insn->addr_bytes) { + case 2: + *exitinfo |= IOIO_ADDR_16; + break; + case 4: + *exitinfo |= IOIO_ADDR_32; + break; + case 8: + *exitinfo |= IOIO_ADDR_64; + break; + } + + if (insn_has_rep_prefix(insn)) + *exitinfo |= IOIO_REP; + + return vc_ioio_check(ctxt, (u16)port, size); +} + +static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u64 exit_info_1, exit_info_2; + enum es_result ret; + + ret = vc_ioio_exitinfo(ctxt, &exit_info_1); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_STR) { + + /* (REP) INS/OUTS */ + + bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); + unsigned int io_bytes, exit_bytes; + unsigned int ghcb_count, op_count; + unsigned long es_base; + u64 sw_scratch; + + /* + * For the string variants with rep prefix the amount of in/out + * operations per #VC exception is limited so that the kernel + * has a chance to take interrupts and re-schedule while the + * instruction is emulated. + */ + io_bytes = (exit_info_1 >> 4) & 0x7; + ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; + + op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; + exit_info_2 = min(op_count, ghcb_count); + exit_bytes = exit_info_2 * io_bytes; + + es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); + + /* Read bytes of OUTS into the shared buffer */ + if (!(exit_info_1 & IOIO_TYPE_IN)) { + ret = vc_insn_string_read(ctxt, + (void *)(es_base + regs->si), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + } + + /* + * Issue an VMGEXIT to the HV to consume the bytes from the + * shared buffer or to have it write them into the shared buffer + * depending on the instruction: OUTS or INS. + */ + sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); + ghcb_set_sw_scratch(ghcb, sw_scratch); + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, + exit_info_1, exit_info_2); + if (ret != ES_OK) + return ret; + + /* Read bytes from shared buffer into the guest's destination. */ + if (exit_info_1 & IOIO_TYPE_IN) { + ret = vc_insn_string_write(ctxt, + (void *)(es_base + regs->di), + ghcb->shared_buffer, io_bytes, + exit_info_2, df); + if (ret) + return ret; + + if (df) + regs->di -= exit_bytes; + else + regs->di += exit_bytes; + } else { + if (df) + regs->si -= exit_bytes; + else + regs->si += exit_bytes; + } + + if (exit_info_1 & IOIO_REP) + regs->cx -= exit_info_2; + + ret = regs->cx ? ES_RETRY : ES_OK; + + } else { + + /* IN/OUT into/from rAX */ + + int bits = (exit_info_1 & 0x70) >> 1; + u64 rax = 0; + + if (!(exit_info_1 & IOIO_TYPE_IN)) + rax = lower_bits(regs->ax, bits); + + ghcb_set_rax(ghcb, rax); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); + if (ret != ES_OK) + return ret; + + if (exit_info_1 & IOIO_TYPE_IN) { + if (!ghcb_rax_is_valid(ghcb)) + return ES_VMM_ERROR; + regs->ax = lower_bits(ghcb->save.rax, bits); + } + } + + return ret; +} + +static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + struct cpuid_leaf leaf; + int ret; + + leaf.fn = regs->ax; + leaf.subfn = regs->cx; + ret = snp_cpuid(ghcb, ctxt, &leaf); + if (!ret) { + regs->ax = leaf.eax; + regs->bx = leaf.ebx; + regs->cx = leaf.ecx; + regs->dx = leaf.edx; + } + + return ret; +} + +static enum es_result vc_handle_cpuid(struct ghcb *ghcb, + struct es_em_ctxt *ctxt) +{ + struct pt_regs *regs = ctxt->regs; + u32 cr4 = native_read_cr4(); + enum es_result ret; + int snp_cpuid_ret; + + snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); + if (!snp_cpuid_ret) + return ES_OK; + if (snp_cpuid_ret != -EOPNOTSUPP) + return ES_VMM_ERROR; + + ghcb_set_rax(ghcb, regs->ax); + ghcb_set_rcx(ghcb, regs->cx); + + if (cr4 & X86_CR4_OSXSAVE) + /* Safe to read xcr0 */ + ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); + else + /* xgetbv will cause #GP - use reset value for xcr0 */ + ghcb_set_xcr0(ghcb, 1); + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && + ghcb_rbx_is_valid(ghcb) && + ghcb_rcx_is_valid(ghcb) && + ghcb_rdx_is_valid(ghcb))) + return ES_VMM_ERROR; + + regs->ax = ghcb->save.rax; + regs->bx = ghcb->save.rbx; + regs->cx = ghcb->save.rcx; + regs->dx = ghcb->save.rdx; + + return ES_OK; +} + +static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + unsigned long exit_code) +{ + bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); + enum es_result ret; + + /* + * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure + * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP + * instructions are being intercepted. If this should occur and Secure + * TSC is enabled, guest execution should be terminated as the guest + * cannot rely on the TSC value provided by the hypervisor. + */ + if (sev_status & MSR_AMD64_SNP_SECURE_TSC) + return ES_VMM_ERROR; + + ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); + if (ret != ES_OK) + return ret; + + if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && + (!rdtscp || ghcb_rcx_is_valid(ghcb)))) + return ES_VMM_ERROR; + + ctxt->regs->ax = ghcb->save.rax; + ctxt->regs->dx = ghcb->save.rdx; + if (rdtscp) + ctxt->regs->cx = ghcb->save.rcx; + + return ES_OK; +} diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h index a78f97208a39..b7232081f8f7 100644 --- a/arch/x86/include/asm/sev-internal.h +++ b/arch/x86/include/asm/sev-internal.h @@ -3,7 +3,6 @@ #define DR7_RESET_VALUE 0x400 extern struct ghcb boot_ghcb_page; -extern struct ghcb *boot_ghcb; extern u64 sev_hv_features; extern u64 sev_secrets_pa; @@ -59,8 +58,6 @@ DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa); void early_set_pages_state(unsigned long vaddr, unsigned long paddr, unsigned long npages, enum psc_op op); -void __noreturn sev_es_terminate(unsigned int set, unsigned int reason); - DECLARE_PER_CPU(struct svsm_ca *, svsm_caa); DECLARE_PER_CPU(u64, svsm_caa_pa); @@ -100,11 +97,6 @@ static __always_inline void sev_es_wr_ghcb_msr(u64 val) native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); } -enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, - struct es_em_ctxt *ctxt, - u64 exit_code, u64 exit_info_1, - u64 exit_info_2); - void snp_register_ghcb_early(unsigned long paddr); bool sev_es_negotiate_protocol(void); bool sev_es_check_cpu_features(void); diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index a8661dfc9a9a..6158893786d6 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -521,6 +521,28 @@ static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } +void vc_forward_exception(struct es_em_ctxt *ctxt); + +/* I/O parameters for CPUID-related helpers */ +struct cpuid_leaf { + u32 fn; + u32 subfn; + u32 eax; + u32 ebx; + u32 ecx; + u32 edx; +}; + +int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf); + +void __noreturn sev_es_terminate(unsigned int set, unsigned int reason); +enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2); + +extern struct ghcb *boot_ghcb; + #else /* !CONFIG_AMD_MEM_ENCRYPT */ #define snp_vmpl 0 From 960bc2bcba5987a82530b9756e1f602a894cffa4 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sun, 4 May 2025 15:30:38 -0700 Subject: [PATCH 299/331] x86/fpu: Restore fpu_thread_struct_whitelist() to fix CONFIG_HARDENED_USERCOPY=y crash Borislav Petkov reported the following boot crash on x86-32, with CONFIG_HARDENED_USERCOPY=y: | usercopy: Kernel memory overwrite attempt detected to SLUB object 'task_struct' (offset 2112, size 160)! | ... | kernel BUG at mm/usercopy.c:102! So the useroffset and usersize arguments are what control the allowed window of copying in/out of the "task_struct" kmem cache: /* create a slab on which task_structs can be allocated */ task_struct_whitelist(&useroffset, &usersize); task_struct_cachep = kmem_cache_create_usercopy("task_struct", arch_task_struct_size, align, SLAB_PANIC|SLAB_ACCOUNT, useroffset, usersize, NULL); task_struct_whitelist() positions this window based on the location of the thread_struct within task_struct, and gets the arch-specific details via arch_thread_struct_whitelist(offset, size): static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size) { /* Fetch thread_struct whitelist for the architecture. */ arch_thread_struct_whitelist(offset, size); /* * Handle zero-sized whitelist or empty thread_struct, otherwise * adjust offset to position of thread_struct in task_struct. */ if (unlikely(*size == 0)) *offset = 0; else *offset += offsetof(struct task_struct, thread); } Commit cb7ca40a3882 ("x86/fpu: Make task_struct::thread constant size") removed the logic for the window, leaving: static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) { *offset = 0; *size = 0; } So now there is no window that usercopy hardening will allow to be copied in/out of task_struct. But as reported above, there *is* a copy in copy_uabi_to_xstate(). (It seems there are several, actually.) int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf) { return copy_uabi_to_xstate(x86_task_fpu(tsk)->fpstate, NULL, ubuf, &tsk->thread.pkru); } This appears to be writing into x86_task_fpu(tsk)->fpstate. With or without CONFIG_X86_DEBUG_FPU, this resolves to: ((struct fpu *)((void *)(task) + sizeof(*(task)))) i.e. the memory "after task_struct" is cast to "struct fpu", and the uses the "fpstate" pointer. How that pointer gets set looks to be variable, but I think the one we care about here is: fpu->fpstate = &fpu->__fpstate; And struct fpu::__fpstate says: struct fpstate __fpstate; /* * WARNING: '__fpstate' is dynamically-sized. Do not put * anything after it here. */ So we're still dealing with a dynamically sized thing, even if it's not within the literal struct task_struct -- it's still in the kmem cache, though. Looking at the kmem cache size, it has allocated "arch_task_struct_size" bytes, which is calculated in fpu__init_task_struct_size(): int task_size = sizeof(struct task_struct); task_size += sizeof(struct fpu); /* * Subtract off the static size of the register state. * It potentially has a bunch of padding. */ task_size -= sizeof(union fpregs_state); /* * Add back the dynamically-calculated register state * size. */ task_size += fpu_kernel_cfg.default_size; /* * We dynamically size 'struct fpu', so we require that * 'state' be at the end of 'it: */ CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate); arch_task_struct_size = task_size; So, this is still copying out of the kmem cache for task_struct, and the window seems unchanged (still fpu regs). This is what the window was before: void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) { *offset = offsetof(struct thread_struct, fpu.__fpstate.regs); *size = fpu_kernel_cfg.default_size; } And the same commit I mentioned above removed it. I think the misunderstanding is here: | The fpu_thread_struct_whitelist() quirk to hardened usercopy can be removed, | now that the FPU structure is not embedded in the task struct anymore, which | reduces text footprint a bit. Yes, FPU is no longer in task_struct, but it IS in the kmem cache named "task_struct", since the fpstate is still being allocated there. Partially revert the earlier mentioned commit, along with a recalculation of the fpstate regs location. Fixes: cb7ca40a3882 ("x86/fpu: Make task_struct::thread constant size") Reported-by: Borislav Petkov (AMD) Tested-by: Borislav Petkov (AMD) Signed-off-by: Kees Cook Signed-off-by: Ingo Molnar Cc: Chang S. Bae Cc: Gustavo A. R. Silva Cc: H. Peter Anvin Cc: Andy Lutomirski Cc: Brian Gerst Cc: Linus Torvalds Cc: linux-hardening@vger.kernel.org Link: https://lore.kernel.org/all/20250409211127.3544993-1-mingo@kernel.org/ # Discussion #1 Link: https://lore.kernel.org/r/202505041418.F47130C4C8@keescook # Discussion #2 --- arch/x86/include/asm/processor.h | 12 +++++------- arch/x86/kernel/fpu/core.c | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index eaa7214d6953..ad33903dc92a 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -522,14 +522,12 @@ extern struct fpu *x86_task_fpu(struct task_struct *task); # define x86_task_fpu(task) ((struct fpu *)((void *)(task) + sizeof(*(task)))) #endif -/* - * X86 doesn't need any embedded-FPU-struct quirks: - */ -static inline void -arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) +extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size); + +static inline void arch_thread_struct_whitelist(unsigned long *offset, + unsigned long *size) { - *offset = 0; - *size = 0; + fpu_thread_struct_whitelist(offset, size); } static inline void diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index fa131299c7da..105b1b80d88d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -680,6 +680,20 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, return 0; } +/* + * While struct fpu is no longer part of struct thread_struct, it is still + * allocated after struct task_struct in the "task_struct" kmem cache. But + * since FPU is expected to be part of struct thread_struct, we have to + * adjust for it here. + */ +void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) +{ + /* The allocation follows struct task_struct. */ + *offset = sizeof(struct task_struct) - offsetof(struct task_struct, thread); + *offset += offsetof(struct fpu, __fpstate.regs); + *size = fpu_kernel_cfg.default_size; +} + /* * Drops current FPU state: deactivates the fpregs and * the fpstate. NOTE: it still leaves previous contents From ab8131028710d009ab93d6bffd2a2749ade909b0 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Tue, 22 Apr 2025 18:48:30 -0500 Subject: [PATCH 300/331] x86/CPU/AMD: Print the reason for the last reset The following register contains bits that indicate the cause for the previous reset. PMx000000C0 (FCH::PM::S5_RESET_STATUS) This is useful for debug. The reasons for reset are broken into 6 high level categories. Decode it by category and print during boot. Specifics within a category are split off into debugging documentation. The register is accessed indirectly through a "PM" port in the FCH. Use MMIO access in order to avoid restrictions with legacy port access. Use a late_initcall() to ensure that MMIO has been set up before trying to access the register. This register was introduced with AMD Family 17h, so avoid access on older families. There is no CPUID feature bit for this register. [ bp: Simplify the reason dumping loop. - merge a fix to not access an array element after the last one: https://lore.kernel.org/r/20250505133609.83933-1-superm1@kernel.org Reported-by: James Dutton ] [ mingo: - Use consistent .rst formatting - Fix 'Sleep' class field to 'ACPI-State' - Standardize pin messages around the 'tripped' verbiage - Remove reference to ring-buffer printing & simplify the wording - Use curly braces for multi-line conditional statements ] Signed-off-by: Yazen Ghannam Co-developed-by: Mario Limonciello Signed-off-by: Mario Limonciello Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250422234830.2840784-6-superm1@kernel.org --- Documentation/arch/x86/amd-debugging.rst | 48 +++++++++++++++++++++ arch/x86/include/asm/amd/fch.h | 1 + arch/x86/kernel/cpu/amd.c | 54 ++++++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/Documentation/arch/x86/amd-debugging.rst b/Documentation/arch/x86/amd-debugging.rst index d3290f2e4be7..d92bf59d62c7 100644 --- a/Documentation/arch/x86/amd-debugging.rst +++ b/Documentation/arch/x86/amd-debugging.rst @@ -52,6 +52,7 @@ report generated from this script to Spurious s2idle wakeups from an IRQ =================================== + Spurious wakeups will generally have an IRQ set to ``/sys/power/pm_wakeup_irq``. This can be matched to ``/proc/interrupts`` to determine what device woke the system. @@ -134,6 +135,7 @@ The ``amd_s2idle.py`` script will capture most of these artifacts for you. s2idle PM debug messages ======================== + During the s2idle flow on AMD systems, the ACPI LPS0 driver is responsible to check all uPEP constraints. Failing uPEP constraints does not prevent s0i3 entry. This means that if some constraints are not met, it is possible @@ -160,6 +162,7 @@ After doing this, run the suspend cycle and look specifically for errors around: Historical examples of s2idle issues ==================================== + To help understand the types of issues that can occur and how to debug them, here are some historical examples of s2idle issues that have been resolved. @@ -248,6 +251,7 @@ state entry. Runtime power consumption issues ================================ + Runtime power consumption is influenced by many factors, including but not limited to the configuration of the PCIe Active State Power Management (ASPM), the display brightness, the EPP policy of the CPU, and the power management @@ -272,6 +276,7 @@ the battery life when more heavily biased towards performance. BIOS debug messages =================== + Most OEM machines don't have a serial UART for outputting kernel or BIOS debug messages. However BIOS debug messages are useful for understanding both BIOS bugs and bugs with the Linux kernel drivers that call BIOS AML. @@ -318,3 +323,46 @@ As mentioned above, parsing by hand can be tedious, especially with a lot of messages. To help with this, a tool has been created at `amd-debug-tools `_ to help parse the messages. + +Random reboot issues +==================== + +When a random reboot occurs, the high-level reason for the reboot is stored +in a register that will persist onto the next boot. + +There are 6 classes of reasons for the reboot: + * Software induced + * Power state transition + * Pin induced + * Hardware induced + * Remote reset + * Internal CPU event + +.. csv-table:: + :header: "Bit", "Type", "Reason" + :align: left + + "0", "Pin", "thermal pin BP_THERMTRIP_L was tripped" + "1", "Pin", "power button was pressed for 4 seconds" + "2", "Pin", "shutdown pin was tripped" + "4", "Remote", "remote ASF power off command was received" + "9", "Internal", "internal CPU thermal limit was tripped" + "16", "Pin", "system reset pin BP_SYS_RST_L was tripped" + "17", "Software", "software issued PCI reset" + "18", "Software", "software wrote 0x4 to reset control register 0xCF9" + "19", "Software", "software wrote 0x6 to reset control register 0xCF9" + "20", "Software", "software wrote 0xE to reset control register 0xCF9" + "21", "ACPI-state", "ACPI power state transition occurred" + "22", "Pin", "keyboard reset pin KB_RST_L was tripped" + "23", "Internal", "internal CPU shutdown event occurred" + "24", "Hardware", "system failed to boot before failed boot timer expired" + "25", "Hardware", "hardware watchdog timer expired" + "26", "Remote", "remote ASF reset command was received" + "27", "Internal", "an uncorrected error caused a data fabric sync flood event" + "29", "Internal", "FCH and MP1 failed warm reset handshake" + "30", "Internal", "a parity error occurred" + "31", "Internal", "a software sync flood event occurred" + +This information is read by the kernel at bootup and printed into +the syslog. When a random reboot occurs this message can be helpful +to determine the next component to debug. diff --git a/arch/x86/include/asm/amd/fch.h b/arch/x86/include/asm/amd/fch.h index 01ee15bcd25f..2cf5153edbc2 100644 --- a/arch/x86/include/asm/amd/fch.h +++ b/arch/x86/include/asm/amd/fch.h @@ -8,5 +8,6 @@ #define FCH_PM_DECODEEN 0x00 #define FCH_PM_DECODEEN_SMBUS0SEL GENMASK(20, 19) #define FCH_PM_SCRATCH 0x80 +#define FCH_PM_S5_RESET_STATUS 0xC0 #endif /* _ASM_X86_AMD_FCH_H_ */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 2b36379ff675..ded4a2a38016 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1237,3 +1238,56 @@ void amd_check_microcode(void) if (cpu_feature_enabled(X86_FEATURE_ZEN2)) on_each_cpu(zenbleed_check_cpu, NULL, 1); } + +static const char * const s5_reset_reason_txt[] = { + [0] = "thermal pin BP_THERMTRIP_L was tripped", + [1] = "power button was pressed for 4 seconds", + [2] = "shutdown pin was tripped", + [4] = "remote ASF power off command was received", + [9] = "internal CPU thermal limit was tripped", + [16] = "system reset pin BP_SYS_RST_L was tripped", + [17] = "software issued PCI reset", + [18] = "software wrote 0x4 to reset control register 0xCF9", + [19] = "software wrote 0x6 to reset control register 0xCF9", + [20] = "software wrote 0xE to reset control register 0xCF9", + [21] = "ACPI power state transition occurred", + [22] = "keyboard reset pin KB_RST_L was tripped", + [23] = "internal CPU shutdown event occurred", + [24] = "system failed to boot before failed boot timer expired", + [25] = "hardware watchdog timer expired", + [26] = "remote ASF reset command was received", + [27] = "an uncorrected error caused a data fabric sync flood event", + [29] = "FCH and MP1 failed warm reset handshake", + [30] = "a parity error occurred", + [31] = "a software sync flood event occurred", +}; + +static __init int print_s5_reset_status_mmio(void) +{ + unsigned long value; + void __iomem *addr; + int i; + + if (!cpu_feature_enabled(X86_FEATURE_ZEN)) + return 0; + + addr = ioremap(FCH_PM_BASE + FCH_PM_S5_RESET_STATUS, sizeof(value)); + if (!addr) + return 0; + + value = ioread32(addr); + iounmap(addr); + + for (i = 0; i < ARRAY_SIZE(s5_reset_reason_txt); i++) { + if (!(value & BIT(i))) + continue; + + if (s5_reset_reason_txt[i]) { + pr_info("x86/amd: Previous system reset reason [0x%08lx]: %s\n", + value, s5_reset_reason_txt[i]); + } + } + + return 0; +} +late_initcall(print_s5_reset_status_mmio); From 43c2df7e2b08db6d65ce9707e4090f1f0c61f2f6 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Mon, 5 May 2025 15:16:46 +0200 Subject: [PATCH 301/331] x86/alternative: Remove unused header #defines Remove some unfortunately-named unused macros which could potentially result in weird build failures. Fortunately, they are under an #ifdef __ASSEMBLER__ which has kept them from causing problems so far. [ dhansen: subject and changelog tweaks ] Fixes: 1a6ade825079 ("x86/alternative: Convert the asm ALTERNATIVE_3() macro") Signed-off-by: Juergen Gross Signed-off-by: Dave Hansen Link: https://lore.kernel.org/all/20250505131646.29288-1-jgross%40suse.com --- arch/x86/include/asm/alternative.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index ef84739a77f5..e18cdaa1573c 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -341,11 +341,6 @@ void nop_func(void); __ALTERNATIVE(\oldinstr, \newinstr, \ft_flags) .endm -#define old_len 141b-140b -#define new_len1 144f-143f -#define new_len2 145f-144f -#define new_len3 146f-145f - /* * Same as ALTERNATIVE macro above but for two alternatives. If CPU * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has From 49394b5af45cc368c587371592a7c5f71834557e Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Tue, 6 May 2025 07:04:12 +0200 Subject: [PATCH 302/331] tools/x86/kcpuid: Update bitfields to x86-cpuid-db v2.4 Update kcpuid's CSV file to version 2.4, as generated by x86-cpuid-db. Summary of the v2.4 changes: * Mark CPUID(0x80000001) EDX:23 bit, 'e_mmx', as not exclusive to Transmeta since it is supported by AMD as well. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: H. Peter Anvin Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://gitlab.com/x86-cpuid.org/x86-cpuid-db/-/blob/v2.4/CHANGELOG.rst Link: https://lore.kernel.org/r/20250506050437.10264-2-darwi@linutronix.de --- tools/arch/x86/kcpuid/cpuid.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/arch/x86/kcpuid/cpuid.csv b/tools/arch/x86/kcpuid/cpuid.csv index 8d25b0b49f3b..8d925ce9750f 100644 --- a/tools/arch/x86/kcpuid/cpuid.csv +++ b/tools/arch/x86/kcpuid/cpuid.csv @@ -1,5 +1,5 @@ # SPDX-License-Identifier: CC0-1.0 -# Generator: x86-cpuid-db v2.3 +# Generator: x86-cpuid-db v2.4 # # Auto-generated file. @@ -689,7 +689,7 @@ 0x80000001, 0, edx, 19, mp , Out-of-spec AMD Multiprocessing bit 0x80000001, 0, edx, 20, nx , No-execute page protection 0x80000001, 0, edx, 22, mmxext , AMD MMX extensions -0x80000001, 0, edx, 23, e_mmx , MMX instructions (Transmeta) +0x80000001, 0, edx, 23, e_mmx , MMX instructions 0x80000001, 0, edx, 24, e_fxsr , FXSAVE and FXRSTOR instructions 0x80000001, 0, edx, 25, fxsr_opt , FXSAVE and FXRSTOR optimizations 0x80000001, 0, edx, 26, pdpe1gb , 1-GB large page support From cc663ba3fe383a628a812f893cc98aafff39ab04 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Tue, 6 May 2025 07:04:13 +0200 Subject: [PATCH 303/331] x86/cpu: Sanitize CPUID(0x80000000) output CPUID(0x80000000).EAX returns the max extended CPUID leaf available. On x86-32 machines without an extended CPUID range, a CPUID(0x80000000) query will just repeat the output of the last valid standard CPUID leaf on the CPU; i.e., a garbage values. Current tip:x86/cpu code protects against this by doing: eax = cpuid_eax(0x80000000); c->extended_cpuid_level = eax; if ((eax & 0xffff0000) == 0x80000000) { // CPU has an extended CPUID range. Check for 0x80000001 if (eax >= 0x80000001) { cpuid(0x80000001, ...); } } This is correct so far. Afterwards though, the same possibly broken EAX value is used to check the availability of other extended CPUID leaves: if (c->extended_cpuid_level >= 0x80000007) ... if (c->extended_cpuid_level >= 0x80000008) ... if (c->extended_cpuid_level >= 0x8000000a) ... if (c->extended_cpuid_level >= 0x8000001f) ... which is invalid. Fix this by immediately setting the CPU's max extended CPUID leaf to zero if CPUID(0x80000000).EAX doesn't indicate a valid CPUID extended range. While at it, add a comment, similar to kernel/head_32.S, clarifying the CPUID(0x80000000) sanity check. References: 8a50e5135af0 ("x86-32: Use symbolic constants, safer CPUID when enabling EFER.NX") Fixes: 3da99c977637 ("x86: make (early)_identify_cpu more the same between 32bit and 64 bit") Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: H. Peter Anvin Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250506050437.10264-3-darwi@linutronix.de --- arch/x86/kernel/cpu/common.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 4ada55f126ae..e5734df3b4a1 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1005,17 +1005,18 @@ void get_cpu_cap(struct cpuinfo_x86 *c) c->x86_capability[CPUID_D_1_EAX] = eax; } - /* AMD-defined flags: level 0x80000001 */ + /* + * Check if extended CPUID leaves are implemented: Max extended + * CPUID leaf must be in the 0x80000001-0x8000ffff range. + */ eax = cpuid_eax(0x80000000); - c->extended_cpuid_level = eax; + c->extended_cpuid_level = ((eax & 0xffff0000) == 0x80000000) ? eax : 0; - if ((eax & 0xffff0000) == 0x80000000) { - if (eax >= 0x80000001) { - cpuid(0x80000001, &eax, &ebx, &ecx, &edx); + if (c->extended_cpuid_level >= 0x80000001) { + cpuid(0x80000001, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_8000_0001_ECX] = ecx; - c->x86_capability[CPUID_8000_0001_EDX] = edx; - } + c->x86_capability[CPUID_8000_0001_ECX] = ecx; + c->x86_capability[CPUID_8000_0001_EDX] = edx; } if (c->extended_cpuid_level >= 0x80000007) { From 7f9958230d8a79d474829bee25ec9426397335ce Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 30 Apr 2025 10:11:54 +0200 Subject: [PATCH 304/331] x86/mm: Fix false positive warning in switch_mm_irqs_off() Multiple testers reported the following new warning: WARNING: CPU: 0 PID: 0 at arch/x86/mm/tlb.c:795 Which corresponds to: if (IS_ENABLED(CONFIG_DEBUG_VM) && WARN_ON_ONCE(prev != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(next)))) cpumask_set_cpu(cpu, mm_cpumask(next)); So the problem is that unuse_temporary_mm() explicitly clears that bit; and it has to, because otherwise the flush_tlb_mm_range() in __text_poke() will try sending IPIs, which are not at all needed. See also: https://lore.kernel.org/all/20241113095550.GBZzR3pg-RhJKPDazS@fat_crate.local/ Notably, the whole {,un}use_temporary_mm() thing requires preemption to be disabled across it with the express purpose of keeping all TLB nonsense CPU local, such that invalidations can also stay local etc. However, as a side-effect, we violate this above WARN(), which sorta makes sense for the normal case, but very much doesn't make sense here. Change unuse_temporary_mm() to mark the mm_struct such that a further exception (beyond init_mm) can be grafted, to keep the warning for all the other cases. Reported-by: Chaitanya Kumar Borah Reported-by: Jani Nikula Signed-off-by: Peter Zijlstra Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: Andy Lutomirski Cc: Brian Gerst Cc: H. Peter Anvin Cc: Juergen Gross Cc: Linus Torvalds Cc: Rik van Riel Link: https://lore.kernel.org/r/20250430081154.GH4439@noisy.programming.kicks-ass.net --- arch/x86/include/asm/mmu.h | 4 ++-- arch/x86/include/asm/mmu_context.h | 10 ++++++++++ arch/x86/mm/init.c | 3 +++ arch/x86/mm/tlb.c | 3 ++- arch/x86/platform/efi/efi_64.c | 1 + 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 8b8055a8eb9e..0fe9c569d171 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -16,6 +16,8 @@ #define MM_CONTEXT_LOCK_LAM 2 /* Allow LAM and SVA coexisting */ #define MM_CONTEXT_FORCE_TAGGED_SVA 3 +/* Tracks mm_cpumask */ +#define MM_CONTEXT_NOTRACK 4 /* * x86 has arch-specific MMU state beyond what lives in mm_struct. @@ -44,9 +46,7 @@ typedef struct { struct ldt_struct *ldt; #endif -#ifdef CONFIG_X86_64 unsigned long flags; -#endif #ifdef CONFIG_ADDRESS_MASKING /* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */ diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index c511f8584ae4..73bf3b1b44e8 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -247,6 +247,16 @@ static inline bool is_64bit_mm(struct mm_struct *mm) } #endif +static inline bool is_notrack_mm(struct mm_struct *mm) +{ + return test_bit(MM_CONTEXT_NOTRACK, &mm->context.flags); +} + +static inline void set_notrack_mm(struct mm_struct *mm) +{ + set_bit(MM_CONTEXT_NOTRACK, &mm->context.flags); +} + /* * We only want to enforce protection keys on the current process * because we effectively have no access to PKRU for other diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index f8c74d19bebb..aa56d9ac0b8f 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -28,6 +28,7 @@ #include #include #include +#include /* * We need to define the tracepoints somewhere, and tlb.c @@ -830,6 +831,8 @@ void __init poking_init(void) /* Xen PV guests need the PGD to be pinned. */ paravirt_enter_mmap(text_poke_mm); + set_notrack_mm(text_poke_mm); + /* * Randomize the poking address, but make sure that the following page * will be mapped at the same PMD. We need 2 pages, so find space for 3, diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 39761c7765bd..f5b990e46d7b 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -847,7 +847,8 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, * mm_cpumask. The TLB shootdown code can figure out from * cpu_tlbstate_shared.is_lazy whether or not to send an IPI. */ - if (IS_ENABLED(CONFIG_DEBUG_VM) && WARN_ON_ONCE(prev != &init_mm && + if (IS_ENABLED(CONFIG_DEBUG_VM) && + WARN_ON_ONCE(prev != &init_mm && !is_notrack_mm(prev) && !cpumask_test_cpu(cpu, mm_cpumask(next)))) cpumask_set_cpu(cpu, mm_cpumask(next)); diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index a5d3496d32a5..ce4c08adca88 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -89,6 +89,7 @@ int __init efi_alloc_page_tables(void) efi_mm.pgd = efi_pgd; mm_init_cpumask(&efi_mm); init_new_context(NULL, &efi_mm); + set_notrack_mm(&efi_mm); return 0; From d8414603b29f25191fcceafb72e74b67ac2e92b1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 6 May 2025 17:36:06 +0800 Subject: [PATCH 305/331] x86/fpu/xstate: Always preserve non-user xfeatures/flags in __state_perm When granting userspace or a KVM guest access to an xfeature, preserve the entity's existing supervisor and software-defined permissions as tracked by __state_perm, i.e. use __state_perm to track *all* permissions even though all supported supervisor xfeatures are granted to all FPUs and FPU_GUEST_PERM_LOCKED disallows changing permissions. Effectively clobbering supervisor permissions results in inconsistent behavior, as xstate_get_group_perm() will report supervisor features for process that do NOT request access to dynamic user xfeatures, whereas any and all supervisor features will be absent from the set of permissions for any process that is granted access to one or more dynamic xfeatures (which right now means AMX). The inconsistency isn't problematic because fpu_xstate_prctl() already strips out everything except user xfeatures: case ARCH_GET_XCOMP_PERM: /* * Lockless snapshot as it can also change right after the * dropping the lock. */ permitted = xstate_get_host_group_perm(); permitted &= XFEATURE_MASK_USER_SUPPORTED; return put_user(permitted, uptr); case ARCH_GET_XCOMP_GUEST_PERM: permitted = xstate_get_guest_group_perm(); permitted &= XFEATURE_MASK_USER_SUPPORTED; return put_user(permitted, uptr); and similarly KVM doesn't apply the __state_perm to supervisor states (kvm_get_filtered_xcr0() incorporates xstate_get_guest_group_perm()): case 0xd: { u64 permitted_xcr0 = kvm_get_filtered_xcr0(); u64 permitted_xss = kvm_caps.supported_xss; But if KVM in particular were to ever change, dropping supervisor permissions would result in subtle bugs in KVM's reporting of supported CPUID settings. And the above behavior also means that having supervisor xfeatures in __state_perm is correctly handled by all users. Dropping supervisor permissions also creates another landmine for KVM. If more dynamic user xfeatures are ever added, requesting access to multiple xfeatures in separate ARCH_REQ_XCOMP_GUEST_PERM calls will result in the second invocation of __xstate_request_perm() computing the wrong ksize, as as the mask passed to xstate_calculate_size() would not contain *any* supervisor features. Commit 781c64bfcb73 ("x86/fpu/xstate: Handle supervisor states in XSTATE permissions") fudged around the size issue for userspace FPUs, but for reasons unknown skipped guest FPUs. Lack of a fix for KVM "works" only because KVM doesn't yet support virtualizing features that have supervisor xfeatures, i.e. as of today, KVM guest FPUs will never need the relevant xfeatures. Simply extending the hack-a-fix for guests would temporarily solve the ksize issue, but wouldn't address the inconsistency issue and would leave another lurking pitfall for KVM. KVM support for virtualizing CET will likely add CET_KERNEL as a guest-only xfeature, i.e. CET_KERNEL will not be set in xfeatures_mask_supervisor() and would again be dropped when granting access to dynamic xfeatures. Note, the existing clobbering behavior is rather subtle. The @permitted parameter to __xstate_request_perm() comes from: permitted = xstate_get_group_perm(guest); which is either fpu->guest_perm.__state_perm or fpu->perm.__state_perm, where __state_perm is initialized to: fpu->perm.__state_perm = fpu_kernel_cfg.default_features; and copied to the guest side of things: /* Same defaults for guests */ fpu->guest_perm = fpu->perm; fpu_kernel_cfg.default_features contains everything except the dynamic xfeatures, i.e. everything except XFEATURE_MASK_XTILE_DATA: fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features; fpu_kernel_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC; When __xstate_request_perm() restricts the local "mask" variable to compute the user state size: mask &= XFEATURE_MASK_USER_SUPPORTED; usize = xstate_calculate_size(mask, false); it subtly overwrites the target __state_perm with "mask" containing only user xfeatures: perm = guest ? &fpu->guest_perm : &fpu->perm; /* Pairs with the READ_ONCE() in xstate_get_group_perm() */ WRITE_ONCE(perm->__state_perm, mask); Signed-off-by: Sean Christopherson Signed-off-by: Yang Weijiang Signed-off-by: Chao Gao Signed-off-by: Ingo Molnar Reviewed-by: Maxim Levitsky Reviewed-by: Rick Edgecombe Reviewed-by: Chang S. Bae Acked-by: Dave Hansen Cc: Andy Lutomirski Cc: David Woodhouse Cc: H. Peter Anvin Cc: John Allen Cc: Linus Torvalds Cc: Mitchell Levy Cc: Oleg Nesterov Cc: Paolo Bonzini Cc: Samuel Holland Cc: Sohil Mehta Cc: Vignesh Balasubramanian Cc: Vitaly Kuznetsov Cc: Xin Li Cc: kvm@vger.kernel.org Link: https://lore.kernel.org/all/ZTqgzZl-reO1m01I@google.com Link: https://lore.kernel.org/r/20250506093740.2864458-2-chao.gao@intel.com --- arch/x86/include/asm/fpu/types.h | 8 +++++--- arch/x86/kernel/fpu/xstate.c | 18 +++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h index 97310df3ea13..e64db0eb9d27 100644 --- a/arch/x86/include/asm/fpu/types.h +++ b/arch/x86/include/asm/fpu/types.h @@ -416,9 +416,11 @@ struct fpu_state_perm { /* * @__state_perm: * - * This bitmap indicates the permission for state components, which - * are available to a thread group. The permission prctl() sets the - * enabled state bits in thread_group_leader()->thread.fpu. + * This bitmap indicates the permission for state components + * available to a thread group, including both user and supervisor + * components and software-defined bits like FPU_GUEST_PERM_LOCKED. + * The permission prctl() sets the enabled state bits in + * thread_group_leader()->thread.fpu. * * All run time operations use the per thread information in the * currently active fpu.fpstate which contains the xfeature masks diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 8b14c9d3a1df..1c8410b68108 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1656,16 +1656,20 @@ static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) if ((permitted & requested) == requested) return 0; - /* Calculate the resulting kernel state size */ + /* + * Calculate the resulting kernel state size. Note, @permitted also + * contains supervisor xfeatures even though supervisor are always + * permitted for kernel and guest FPUs, and never permitted for user + * FPUs. + */ mask = permitted | requested; - /* Take supervisor states into account on the host */ - if (!guest) - mask |= xfeatures_mask_supervisor(); ksize = xstate_calculate_size(mask, compacted); - /* Calculate the resulting user state size */ - mask &= XFEATURE_MASK_USER_SUPPORTED; - usize = xstate_calculate_size(mask, false); + /* + * Calculate the resulting user state size. Take care not to clobber + * the supervisor xfeatures in the new mask! + */ + usize = xstate_calculate_size(mask & XFEATURE_MASK_USER_SUPPORTED, false); if (!guest) { ret = validate_sigaltstack(usize); From 32d5fa804dc9bd7cf6651a1378ba616d332e7444 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Tue, 6 May 2025 17:36:07 +0800 Subject: [PATCH 306/331] x86/fpu: Drop @perm from guest pseudo FPU container Remove @perm from the guest pseudo FPU container. The field is initialized during allocation and never used later. Rename fpu_init_guest_permissions() to show that its sole purpose is to lock down guest permissions. Suggested-by: Maxim Levitsky Signed-off-by: Chao Gao Signed-off-by: Ingo Molnar Reviewed-by: Chang S. Bae Cc: Andy Lutomirski Cc: David Woodhouse Cc: Eric Biggers Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Cc: Mitchell Levy Cc: Oleg Nesterov Cc: Paolo Bonzini Cc: Samuel Holland Cc: Sean Christopherson Cc: Vitaly Kuznetsov Link: https://lore.kernel.org/kvm/af972fe5981b9e7101b64de43c7be0a8cc165323.camel@redhat.com/ Link: https://lore.kernel.org/r/20250506093740.2864458-3-chao.gao@intel.com --- arch/x86/include/asm/fpu/types.h | 7 ------- arch/x86/kernel/fpu/core.c | 7 ++----- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h index e64db0eb9d27..1c94121acd3d 100644 --- a/arch/x86/include/asm/fpu/types.h +++ b/arch/x86/include/asm/fpu/types.h @@ -535,13 +535,6 @@ struct fpu_guest { */ u64 xfeatures; - /* - * @perm: xfeature bitmap of features which are - * permitted to be enabled for the guest - * vCPU. - */ - u64 perm; - /* * @xfd_err: Save the guest value. */ diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 105b1b80d88d..1cda5b78540b 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -212,7 +212,7 @@ void fpu_reset_from_exception_fixup(void) #if IS_ENABLED(CONFIG_KVM) static void __fpstate_reset(struct fpstate *fpstate, u64 xfd); -static void fpu_init_guest_permissions(struct fpu_guest *gfpu) +static void fpu_lock_guest_permissions(void) { struct fpu_state_perm *fpuperm; u64 perm; @@ -228,8 +228,6 @@ static void fpu_init_guest_permissions(struct fpu_guest *gfpu) WRITE_ONCE(fpuperm->__state_perm, perm | FPU_GUEST_PERM_LOCKED); spin_unlock_irq(¤t->sighand->siglock); - - gfpu->perm = perm & ~FPU_GUEST_PERM_LOCKED; } bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) @@ -250,7 +248,6 @@ bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) gfpu->fpstate = fpstate; gfpu->xfeatures = fpu_kernel_cfg.default_features; - gfpu->perm = fpu_kernel_cfg.default_features; /* * KVM sets the FP+SSE bits in the XSAVE header when copying FPU state @@ -265,7 +262,7 @@ bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) if (WARN_ON_ONCE(fpu_user_cfg.default_size > gfpu->uabi_size)) gfpu->uabi_size = fpu_user_cfg.default_size; - fpu_init_guest_permissions(gfpu); + fpu_lock_guest_permissions(); return true; } From ca698ec2f07873a448d53c580795c4e023c75393 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 28 Apr 2025 10:48:10 +0900 Subject: [PATCH 307/331] x86/insn: Fix opcode map (!REX2) superscript tags Commit: 159039af8c07 ("x86/insn: x86/insn: Add support for REX2 prefix to the instruction decoder opcode map") added (!REX2) superscript with a space, but the correct format requires ',' for concatination with other superscript tags. Add ',' to generate correct insn attribute tables. I confirmed with following command: arch/x86/lib/x86-opcode-map.txt | grep e8 | head -n 1 [0xe8] = INAT_MAKE_IMM(INAT_IMM_VWORD32) | INAT_FORCE64 | INAT_NO_REX2, Fixes: 159039af8c07 ("x86/insn: x86/insn: Add support for REX2 prefix to the instruction decoder opcode map") Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Adrian Hunter Cc: Peter Zijlstra Link: https://lore.kernel.org/r/174580489027.388420.15539375184727726142.stgit@devnote2 --- arch/x86/lib/x86-opcode-map.txt | 50 +++++++++++++-------------- tools/arch/x86/lib/x86-opcode-map.txt | 50 +++++++++++++-------------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index f5dd84eb55dc..cd3fd5155f6e 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -35,7 +35,7 @@ # - (!F3) : the last prefix is not 0xF3 (including non-last prefix case) # - (66&F2): Both 0x66 and 0xF2 prefixes are specified. # -# REX2 Prefix +# REX2 Prefix Superscripts # - (!REX2): REX2 is not allowed # - (REX2): REX2 variant e.g. JMPABS @@ -286,10 +286,10 @@ df: ESC # Note: "forced64" is Intel CPU behavior: they ignore 0x66 prefix # in 64-bit mode. AMD CPUs accept 0x66 prefix, it causes RIP truncation # to 16 bits. In 32-bit mode, 0x66 is accepted by both Intel and AMD. -e0: LOOPNE/LOOPNZ Jb (f64) (!REX2) -e1: LOOPE/LOOPZ Jb (f64) (!REX2) -e2: LOOP Jb (f64) (!REX2) -e3: JrCXZ Jb (f64) (!REX2) +e0: LOOPNE/LOOPNZ Jb (f64),(!REX2) +e1: LOOPE/LOOPZ Jb (f64),(!REX2) +e2: LOOP Jb (f64),(!REX2) +e3: JrCXZ Jb (f64),(!REX2) e4: IN AL,Ib (!REX2) e5: IN eAX,Ib (!REX2) e6: OUT Ib,AL (!REX2) @@ -298,10 +298,10 @@ e7: OUT Ib,eAX (!REX2) # in "near" jumps and calls is 16-bit. For CALL, # push of return address is 16-bit wide, RSP is decremented by 2 # but is not truncated to 16 bits, unlike RIP. -e8: CALL Jz (f64) (!REX2) -e9: JMP-near Jz (f64) (!REX2) -ea: JMP-far Ap (i64) (!REX2) -eb: JMP-short Jb (f64) (!REX2) +e8: CALL Jz (f64),(!REX2) +e9: JMP-near Jz (f64),(!REX2) +ea: JMP-far Ap (i64),(!REX2) +eb: JMP-short Jb (f64),(!REX2) ec: IN AL,DX (!REX2) ed: IN eAX,DX (!REX2) ee: OUT DX,AL (!REX2) @@ -478,22 +478,22 @@ AVXcode: 1 7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqa32/64 Wx,Vx (66),(evo) | vmovdqu Wx,Vx (F3) | vmovdqu32/64 Wx,Vx (F3),(evo) | vmovdqu8/16 Wx,Vx (F2),(ev) # 0x0f 0x80-0x8f # Note: "forced64" is Intel CPU behavior (see comment about CALL insn). -80: JO Jz (f64) (!REX2) -81: JNO Jz (f64) (!REX2) -82: JB/JC/JNAE Jz (f64) (!REX2) -83: JAE/JNB/JNC Jz (f64) (!REX2) -84: JE/JZ Jz (f64) (!REX2) -85: JNE/JNZ Jz (f64) (!REX2) -86: JBE/JNA Jz (f64) (!REX2) -87: JA/JNBE Jz (f64) (!REX2) -88: JS Jz (f64) (!REX2) -89: JNS Jz (f64) (!REX2) -8a: JP/JPE Jz (f64) (!REX2) -8b: JNP/JPO Jz (f64) (!REX2) -8c: JL/JNGE Jz (f64) (!REX2) -8d: JNL/JGE Jz (f64) (!REX2) -8e: JLE/JNG Jz (f64) (!REX2) -8f: JNLE/JG Jz (f64) (!REX2) +80: JO Jz (f64),(!REX2) +81: JNO Jz (f64),(!REX2) +82: JB/JC/JNAE Jz (f64),(!REX2) +83: JAE/JNB/JNC Jz (f64),(!REX2) +84: JE/JZ Jz (f64),(!REX2) +85: JNE/JNZ Jz (f64),(!REX2) +86: JBE/JNA Jz (f64),(!REX2) +87: JA/JNBE Jz (f64),(!REX2) +88: JS Jz (f64),(!REX2) +89: JNS Jz (f64),(!REX2) +8a: JP/JPE Jz (f64),(!REX2) +8b: JNP/JPO Jz (f64),(!REX2) +8c: JL/JNGE Jz (f64),(!REX2) +8d: JNL/JGE Jz (f64),(!REX2) +8e: JLE/JNG Jz (f64),(!REX2) +8f: JNLE/JG Jz (f64),(!REX2) # 0x0f 0x90-0x9f 90: SETO Eb | kmovw/q Vk,Wk | kmovb/d Vk,Wk (66) 91: SETNO Eb | kmovw/q Mv,Vk | kmovb/d Mv,Vk (66) diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index f5dd84eb55dc..cd3fd5155f6e 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -35,7 +35,7 @@ # - (!F3) : the last prefix is not 0xF3 (including non-last prefix case) # - (66&F2): Both 0x66 and 0xF2 prefixes are specified. # -# REX2 Prefix +# REX2 Prefix Superscripts # - (!REX2): REX2 is not allowed # - (REX2): REX2 variant e.g. JMPABS @@ -286,10 +286,10 @@ df: ESC # Note: "forced64" is Intel CPU behavior: they ignore 0x66 prefix # in 64-bit mode. AMD CPUs accept 0x66 prefix, it causes RIP truncation # to 16 bits. In 32-bit mode, 0x66 is accepted by both Intel and AMD. -e0: LOOPNE/LOOPNZ Jb (f64) (!REX2) -e1: LOOPE/LOOPZ Jb (f64) (!REX2) -e2: LOOP Jb (f64) (!REX2) -e3: JrCXZ Jb (f64) (!REX2) +e0: LOOPNE/LOOPNZ Jb (f64),(!REX2) +e1: LOOPE/LOOPZ Jb (f64),(!REX2) +e2: LOOP Jb (f64),(!REX2) +e3: JrCXZ Jb (f64),(!REX2) e4: IN AL,Ib (!REX2) e5: IN eAX,Ib (!REX2) e6: OUT Ib,AL (!REX2) @@ -298,10 +298,10 @@ e7: OUT Ib,eAX (!REX2) # in "near" jumps and calls is 16-bit. For CALL, # push of return address is 16-bit wide, RSP is decremented by 2 # but is not truncated to 16 bits, unlike RIP. -e8: CALL Jz (f64) (!REX2) -e9: JMP-near Jz (f64) (!REX2) -ea: JMP-far Ap (i64) (!REX2) -eb: JMP-short Jb (f64) (!REX2) +e8: CALL Jz (f64),(!REX2) +e9: JMP-near Jz (f64),(!REX2) +ea: JMP-far Ap (i64),(!REX2) +eb: JMP-short Jb (f64),(!REX2) ec: IN AL,DX (!REX2) ed: IN eAX,DX (!REX2) ee: OUT DX,AL (!REX2) @@ -478,22 +478,22 @@ AVXcode: 1 7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqa32/64 Wx,Vx (66),(evo) | vmovdqu Wx,Vx (F3) | vmovdqu32/64 Wx,Vx (F3),(evo) | vmovdqu8/16 Wx,Vx (F2),(ev) # 0x0f 0x80-0x8f # Note: "forced64" is Intel CPU behavior (see comment about CALL insn). -80: JO Jz (f64) (!REX2) -81: JNO Jz (f64) (!REX2) -82: JB/JC/JNAE Jz (f64) (!REX2) -83: JAE/JNB/JNC Jz (f64) (!REX2) -84: JE/JZ Jz (f64) (!REX2) -85: JNE/JNZ Jz (f64) (!REX2) -86: JBE/JNA Jz (f64) (!REX2) -87: JA/JNBE Jz (f64) (!REX2) -88: JS Jz (f64) (!REX2) -89: JNS Jz (f64) (!REX2) -8a: JP/JPE Jz (f64) (!REX2) -8b: JNP/JPO Jz (f64) (!REX2) -8c: JL/JNGE Jz (f64) (!REX2) -8d: JNL/JGE Jz (f64) (!REX2) -8e: JLE/JNG Jz (f64) (!REX2) -8f: JNLE/JG Jz (f64) (!REX2) +80: JO Jz (f64),(!REX2) +81: JNO Jz (f64),(!REX2) +82: JB/JC/JNAE Jz (f64),(!REX2) +83: JAE/JNB/JNC Jz (f64),(!REX2) +84: JE/JZ Jz (f64),(!REX2) +85: JNE/JNZ Jz (f64),(!REX2) +86: JBE/JNA Jz (f64),(!REX2) +87: JA/JNBE Jz (f64),(!REX2) +88: JS Jz (f64),(!REX2) +89: JNS Jz (f64),(!REX2) +8a: JP/JPE Jz (f64),(!REX2) +8b: JNP/JPO Jz (f64),(!REX2) +8c: JL/JNGE Jz (f64),(!REX2) +8d: JNL/JGE Jz (f64),(!REX2) +8e: JLE/JNG Jz (f64),(!REX2) +8f: JNLE/JG Jz (f64),(!REX2) # 0x0f 0x90-0x9f 90: SETO Eb | kmovw/q Vk,Wk | kmovb/d Vk,Wk (66) 91: SETNO Eb | kmovw/q Mv,Vk | kmovb/d Mv,Vk (66) From 4b626015e1bf119cd31d7e62f9bd9eb1412fce7b Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 28 Apr 2025 10:48:20 +0900 Subject: [PATCH 308/331] x86/insn: Stop decoding i64 instructions in x86-64 mode at opcode In commit 2e044911be75 ("x86/traps: Decode 0xEA instructions as #UD") FineIBT starts using 0xEA as an invalid instruction like UD2. But insn decoder always returns the length of "0xea" instruction as 7 because it does not check the (i64) superscript. The x86 instruction decoder should also decode 0xEA on x86-64 as a one-byte invalid instruction by decoding the "(i64)" superscript tag. This stops decoding instruction which has (i64) but does not have (o64) superscript in 64-bit mode at opcode and skips other fields. With this change, insn_decoder_test says 0xea is 1 byte length if x86-64 (-y option means 64-bit): $ printf "0:\tea\t\n" | insn_decoder_test -y -v insn_decoder_test: success: Decoded and checked 1 instructions Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Ingo Molnar Cc: Adrian Hunter Cc: H. Peter Anvin Cc: Peter Zijlstra Link: https://lore.kernel.org/r/174580490000.388420.5225447607417115496.stgit@devnote2 --- arch/x86/include/asm/inat.h | 6 ++++++ arch/x86/lib/insn.c | 7 ++++++- arch/x86/lib/x86-opcode-map.txt | 6 +++--- arch/x86/tools/gen-insn-attr-x86.awk | 7 +++++++ tools/arch/x86/include/asm/inat.h | 6 ++++++ tools/arch/x86/lib/insn.c | 7 ++++++- tools/arch/x86/lib/x86-opcode-map.txt | 6 +++--- tools/arch/x86/tools/gen-insn-attr-x86.awk | 7 +++++++ 8 files changed, 44 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h index 53e4015242b4..97f341777db5 100644 --- a/arch/x86/include/asm/inat.h +++ b/arch/x86/include/asm/inat.h @@ -82,6 +82,7 @@ #define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) #define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) #define INAT_EVEX_SCALABLE (1 << (INAT_FLAG_OFFS + 10)) +#define INAT_INV64 (1 << (INAT_FLAG_OFFS + 11)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -242,4 +243,9 @@ static inline int inat_evex_scalable(insn_attr_t attr) { return attr & INAT_EVEX_SCALABLE; } + +static inline int inat_is_invalid64(insn_attr_t attr) +{ + return attr & INAT_INV64; +} #endif diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 6ffb931b9fb1..149a57e334ab 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -324,6 +324,11 @@ int insn_get_opcode(struct insn *insn) } insn->attr = inat_get_opcode_attribute(op); + if (insn->x86_64 && inat_is_invalid64(insn->attr)) { + /* This instruction is invalid, like UD2. Stop decoding. */ + insn->attr &= INAT_INV64; + } + while (inat_is_escape(insn->attr)) { /* Get escaped opcode */ op = get_next(insn_byte_t, insn); @@ -337,6 +342,7 @@ int insn_get_opcode(struct insn *insn) insn->attr = 0; return -EINVAL; } + end: opcode->got = 1; return 0; @@ -658,7 +664,6 @@ int insn_get_immediate(struct insn *insn) } if (!inat_has_immediate(insn->attr)) - /* no immediates */ goto done; switch (inat_immediate_size(insn->attr)) { diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index cd3fd5155f6e..262f7ca1fb95 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -147,7 +147,7 @@ AVXcode: # 0x60 - 0x6f 60: PUSHA/PUSHAD (i64) 61: POPA/POPAD (i64) -62: BOUND Gv,Ma (i64) | EVEX (Prefix) +62: BOUND Gv,Ma (i64) | EVEX (Prefix),(o64) 63: ARPL Ew,Gw (i64) | MOVSXD Gv,Ev (o64) 64: SEG=FS (Prefix) 65: SEG=GS (Prefix) @@ -253,8 +253,8 @@ c0: Grp2 Eb,Ib (1A) c1: Grp2 Ev,Ib (1A) c2: RETN Iw (f64) c3: RETN -c4: LES Gz,Mp (i64) | VEX+2byte (Prefix) -c5: LDS Gz,Mp (i64) | VEX+1byte (Prefix) +c4: LES Gz,Mp (i64) | VEX+2byte (Prefix),(o64) +c5: LDS Gz,Mp (i64) | VEX+1byte (Prefix),(o64) c6: Grp11A Eb,Ib (1A) c7: Grp11B Ev,Iz (1A) c8: ENTER Iw,Ib diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index 5770c8097f32..2c19d7fc8a85 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -64,6 +64,8 @@ BEGIN { modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])" force64_expr = "\\([df]64\\)" + invalid64_expr = "\\(i64\\)" + only64_expr = "\\(o64\\)" rex_expr = "^((REX(\\.[XRWB]+)+)|(REX$))" rex2_expr = "\\(REX2\\)" no_rex2_expr = "\\(!REX2\\)" @@ -319,6 +321,11 @@ function convert_operands(count,opnd, i,j,imm,mod) if (match(ext, force64_expr)) flags = add_flags(flags, "INAT_FORCE64") + # check invalid in 64-bit (and no only64) + if (match(ext, invalid64_expr) && + !match($0, only64_expr)) + flags = add_flags(flags, "INAT_INV64") + # check REX2 not allowed if (match(ext, no_rex2_expr)) flags = add_flags(flags, "INAT_NO_REX2") diff --git a/tools/arch/x86/include/asm/inat.h b/tools/arch/x86/include/asm/inat.h index 253690eb3c26..183aa662b165 100644 --- a/tools/arch/x86/include/asm/inat.h +++ b/tools/arch/x86/include/asm/inat.h @@ -82,6 +82,7 @@ #define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) #define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) #define INAT_EVEX_SCALABLE (1 << (INAT_FLAG_OFFS + 10)) +#define INAT_INV64 (1 << (INAT_FLAG_OFFS + 11)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -242,4 +243,9 @@ static inline int inat_evex_scalable(insn_attr_t attr) { return attr & INAT_EVEX_SCALABLE; } + +static inline int inat_is_invalid64(insn_attr_t attr) +{ + return attr & INAT_INV64; +} #endif diff --git a/tools/arch/x86/lib/insn.c b/tools/arch/x86/lib/insn.c index e91d4c4e1c16..bce69c6bfa69 100644 --- a/tools/arch/x86/lib/insn.c +++ b/tools/arch/x86/lib/insn.c @@ -324,6 +324,11 @@ int insn_get_opcode(struct insn *insn) } insn->attr = inat_get_opcode_attribute(op); + if (insn->x86_64 && inat_is_invalid64(insn->attr)) { + /* This instruction is invalid, like UD2. Stop decoding. */ + insn->attr &= INAT_INV64; + } + while (inat_is_escape(insn->attr)) { /* Get escaped opcode */ op = get_next(insn_byte_t, insn); @@ -337,6 +342,7 @@ int insn_get_opcode(struct insn *insn) insn->attr = 0; return -EINVAL; } + end: opcode->got = 1; return 0; @@ -658,7 +664,6 @@ int insn_get_immediate(struct insn *insn) } if (!inat_has_immediate(insn->attr)) - /* no immediates */ goto done; switch (inat_immediate_size(insn->attr)) { diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index cd3fd5155f6e..262f7ca1fb95 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -147,7 +147,7 @@ AVXcode: # 0x60 - 0x6f 60: PUSHA/PUSHAD (i64) 61: POPA/POPAD (i64) -62: BOUND Gv,Ma (i64) | EVEX (Prefix) +62: BOUND Gv,Ma (i64) | EVEX (Prefix),(o64) 63: ARPL Ew,Gw (i64) | MOVSXD Gv,Ev (o64) 64: SEG=FS (Prefix) 65: SEG=GS (Prefix) @@ -253,8 +253,8 @@ c0: Grp2 Eb,Ib (1A) c1: Grp2 Ev,Ib (1A) c2: RETN Iw (f64) c3: RETN -c4: LES Gz,Mp (i64) | VEX+2byte (Prefix) -c5: LDS Gz,Mp (i64) | VEX+1byte (Prefix) +c4: LES Gz,Mp (i64) | VEX+2byte (Prefix),(o64) +c5: LDS Gz,Mp (i64) | VEX+1byte (Prefix),(o64) c6: Grp11A Eb,Ib (1A) c7: Grp11B Ev,Iz (1A) c8: ENTER Iw,Ib diff --git a/tools/arch/x86/tools/gen-insn-attr-x86.awk b/tools/arch/x86/tools/gen-insn-attr-x86.awk index 5770c8097f32..2c19d7fc8a85 100644 --- a/tools/arch/x86/tools/gen-insn-attr-x86.awk +++ b/tools/arch/x86/tools/gen-insn-attr-x86.awk @@ -64,6 +64,8 @@ BEGIN { modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])" force64_expr = "\\([df]64\\)" + invalid64_expr = "\\(i64\\)" + only64_expr = "\\(o64\\)" rex_expr = "^((REX(\\.[XRWB]+)+)|(REX$))" rex2_expr = "\\(REX2\\)" no_rex2_expr = "\\(!REX2\\)" @@ -319,6 +321,11 @@ function convert_operands(count,opnd, i,j,imm,mod) if (match(ext, force64_expr)) flags = add_flags(flags, "INAT_FORCE64") + # check invalid in 64-bit (and no only64) + if (match(ext, invalid64_expr) && + !match($0, only64_expr)) + flags = add_flags(flags, "INAT_INV64") + # check REX2 not allowed if (match(ext, no_rex2_expr)) flags = add_flags(flags, "INAT_NO_REX2") From c1ab4ce3cb759f69fb9085a060e568b73e8f5cd8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 6 May 2025 13:41:43 +0200 Subject: [PATCH 309/331] tools/arch/x86: Move the header to Synchronize with what we did with the kernel side header in: 3846389c03a8 ("x86/platform/amd: Move the header to ") Reported-by: Stephen Rothwell Signed-off-by: Ingo Molnar Cc: linux-kernel@vger.kernel.org --- tools/arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tools/arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} (99%) diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd/ibs.h similarity index 99% rename from tools/arch/x86/include/asm/amd-ibs.h rename to tools/arch/x86/include/asm/amd/ibs.h index cb1740bc3da2..300b6e0765b2 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd/ibs.h @@ -4,7 +4,7 @@ * 55898 Rev 0.35 - Feb 5, 2021 */ -#include "msr-index.h" +#include "../msr-index.h" /* IBS_OP_DATA2 DataSrc */ #define IBS_DATA_SRC_LOC_CACHE 2 From 9cf78722003178b09c409df9aafe9d79e5b9a74e Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 7 May 2025 20:25:59 +0200 Subject: [PATCH 310/331] accel/habanalabs: Don't build the driver on UML The following commit: 288a4ff0ad29 ("x86/msr: Move rdtsc{,_ordered}() to ") removed the include from the accel/habanalabs driver, which broke the build on UML: drivers/accel/habanalabs/common/habanalabs_ioctl.c:326:23: error: call to undeclared function 'rdtsc'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] Make the driver depend on 'X86 && X86_64', instead of just 'X86_64', thus it won't be built on UML. Suggested-by: Johannes Berg Reported-by: kernel test robot Signed-off-by: Ingo Molnar Reviewed-by: Johannes Berg Cc: Ofir Bitton Cc: Oded Gabbay Link: https://lore.kernel.org/r/202505080003.0t7ewxGp-lkp@intel.com --- drivers/accel/habanalabs/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/accel/habanalabs/Kconfig b/drivers/accel/habanalabs/Kconfig index be85336107f9..1919fbb169c7 100644 --- a/drivers/accel/habanalabs/Kconfig +++ b/drivers/accel/habanalabs/Kconfig @@ -6,7 +6,7 @@ config DRM_ACCEL_HABANALABS tristate "HabanaLabs AI accelerators" depends on DRM_ACCEL - depends on X86_64 + depends on X86 && X86_64 depends on PCI && HAS_IOMEM select GENERIC_ALLOCATOR select HWMON From 891d3b8be32a69ae94d32e3f1e1ee01359020d49 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Tue, 13 May 2025 13:04:05 +0200 Subject: [PATCH 311/331] x86/bugs: Fix SRSO reporting on Zen1/2 with SMT disabled 1f4bb068b498 ("x86/bugs: Restructure SRSO mitigation") does this: if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { setup_force_cpu_cap(X86_FEATURE_SRSO_NO); srso_mitigation = SRSO_MITIGATION_NONE; return; } and, in particular, sets srso_mitigation to NONE. This leads to reporting Speculative Return Stack Overflow: Vulnerable on Zen2 machines. There's a far bigger confusion with what SRSO_NO means and how it is used in the code but this will be a matter of future fixes and restructuring to how the SRSO mitigation gets determined. Fix the reporting issue for now. Signed-off-by: Borislav Petkov (AMD) Reviewed-by: David Kaplan Link: https://lore.kernel.org/20250513110405.15872-1-bp@kernel.org --- arch/x86/kernel/cpu/bugs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 47c74c4ae206..dd8b50b4ceaa 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -2942,7 +2942,9 @@ static void __init srso_update_mitigation(void) boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) srso_mitigation = SRSO_MITIGATION_IBPB; - if (boot_cpu_has_bug(X86_BUG_SRSO) && !cpu_mitigations_off()) + if (boot_cpu_has_bug(X86_BUG_SRSO) && + !cpu_mitigations_off() && + !boot_cpu_has(X86_FEATURE_SRSO_NO)) pr_info("%s\n", srso_strings[srso_mitigation]); } From 64797551baec252f953fa8234051f88b0c368ed5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 13 May 2025 13:11:59 +0200 Subject: [PATCH 312/331] x86/boot: Defer initialization of VM space related global variables The global pseudo-constants 'page_offset_base', 'vmalloc_base' and 'vmemmap_base' are not used extremely early during the boot, and cannot be used safely until after the KASLR memory randomization code in kernel_randomize_memory() executes, which may update their values. So there is no point in setting these variables extremely early, and it can wait until after the kernel itself is mapped and running from its permanent virtual mapping. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250513111157.717727-9-ardb+git@google.com --- arch/x86/boot/startup/map_kernel.c | 3 --- arch/x86/kernel/head64.c | 9 ++++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c index 099ae2559336..905e8734b5a3 100644 --- a/arch/x86/boot/startup/map_kernel.c +++ b/arch/x86/boot/startup/map_kernel.c @@ -29,9 +29,6 @@ static inline bool check_la57_support(void) __pgtable_l5_enabled = 1; pgdir_shift = 48; ptrs_per_p4d = 512; - page_offset_base = __PAGE_OFFSET_BASE_L5; - vmalloc_base = __VMALLOC_BASE_L5; - vmemmap_base = __VMEMMAP_BASE_L5; return true; } diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 510fb41f55fc..14f7dda20954 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -62,13 +62,10 @@ EXPORT_SYMBOL(ptrs_per_p4d); #ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4; EXPORT_SYMBOL(page_offset_base); -SYM_PIC_ALIAS(page_offset_base); unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4; EXPORT_SYMBOL(vmalloc_base); -SYM_PIC_ALIAS(vmalloc_base); unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); -SYM_PIC_ALIAS(vmemmap_base); #endif /* Wipe all early page tables except for the kernel symbol map */ @@ -244,6 +241,12 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode /* Kill off the identity-map trampoline */ reset_early_page_tables(); + if (pgtable_l5_enabled()) { + page_offset_base = __PAGE_OFFSET_BASE_L5; + vmalloc_base = __VMALLOC_BASE_L5; + vmemmap_base = __VMEMMAP_BASE_L5; + } + clear_bss(); /* From 25219c2578b32c96087569d074e32c8ae634d602 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 14 May 2025 12:42:46 +0200 Subject: [PATCH 313/331] x86/asm-offsets: Export certain 'struct cpuinfo_x86' fields for 64-bit asm use too Expose certain 'struct cpuinfo_x86' fields via asm-offsets for x86_64 too, so that it will be possible to set CPU capabilities from 64-bit asm code. 32-bit already used these fields, so simply move those offset exports into the unified asm-offsets.c file. Signed-off-by: Ard Biesheuvel Signed-off-by: Ingo Molnar Cc: Brian Gerst Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250514104242.1275040-12-ardb+git@google.com --- arch/x86/kernel/asm-offsets.c | 8 ++++++++ arch/x86/kernel/asm-offsets_32.c | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index ad4ea6fb3b6c..6259b474073b 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -33,6 +33,14 @@ static void __used common(void) { + OFFSET(CPUINFO_x86, cpuinfo_x86, x86); + OFFSET(CPUINFO_x86_vendor, cpuinfo_x86, x86_vendor); + OFFSET(CPUINFO_x86_model, cpuinfo_x86, x86_model); + OFFSET(CPUINFO_x86_stepping, cpuinfo_x86, x86_stepping); + OFFSET(CPUINFO_cpuid_level, cpuinfo_x86, cpuid_level); + OFFSET(CPUINFO_x86_capability, cpuinfo_x86, x86_capability); + OFFSET(CPUINFO_x86_vendor_id, cpuinfo_x86, x86_vendor_id); + BLANK(); OFFSET(TASK_threadsp, task_struct, thread.sp); #ifdef CONFIG_STACKPROTECTOR diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index 2b411cd00a4e..e0a292db97b2 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c @@ -12,15 +12,6 @@ void foo(void); void foo(void) { - OFFSET(CPUINFO_x86, cpuinfo_x86, x86); - OFFSET(CPUINFO_x86_vendor, cpuinfo_x86, x86_vendor); - OFFSET(CPUINFO_x86_model, cpuinfo_x86, x86_model); - OFFSET(CPUINFO_x86_stepping, cpuinfo_x86, x86_stepping); - OFFSET(CPUINFO_cpuid_level, cpuinfo_x86, cpuid_level); - OFFSET(CPUINFO_x86_capability, cpuinfo_x86, x86_capability); - OFFSET(CPUINFO_x86_vendor_id, cpuinfo_x86, x86_vendor_id); - BLANK(); - OFFSET(PT_EBX, pt_regs, bx); OFFSET(PT_ECX, pt_regs, cx); OFFSET(PT_EDX, pt_regs, dx); From 1adf711919de7c9e1b281a4d9cd9e3a81f3a70f7 Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 30 Apr 2025 11:29:59 +0000 Subject: [PATCH 314/331] x86/mm: Fix kernel-doc descriptions of various pgtable methods So 'make W=1' complains about a couple of kernel-doc descriptions in our MM primitives in pgtable.c: arch/x86/mm/pgtable.c:623: warning: Function parameter or struct member 'reserve' not described in 'reserve_top_address' arch/x86/mm/pgtable.c:672: warning: Function parameter or struct member 'p4d' not described in 'p4d_set_huge' arch/x86/mm/pgtable.c:672: warning: Function parameter or struct member 'addr' not described in 'p4d_set_huge' ... so on Fix them all up, add missing parameter documentation, and fix various spelling inconsistencies while at it. [ mingo: Harmonize kernel-doc annotations some more. ] Signed-off-by: Shivank Garg Signed-off-by: Ingo Molnar Cc: Dave Hansen Cc: Andrew Cooper Cc: Andy Lutomirski Cc: Brian Gerst Cc: H. Peter Anvin Cc: Juergen Gross Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Rik van Riel Link: https://lore.kernel.org/r/20250514062637.3287779-1-shivankg@amd.com --- arch/x86/mm/pgtable.c | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 7c253deb46e9..59c42dec7076 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -543,11 +543,11 @@ pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, #endif /** - * reserve_top_address - reserves a hole in the top of kernel address space - * @reserve - size of hole to reserve + * reserve_top_address - Reserve a hole in the top of the kernel address space + * @reserve: Size of hole to reserve * * Can be used to relocate the fixmap area and poke a hole in the top - * of kernel address space to make room for a hypervisor. + * of the kernel address space to make room for a hypervisor. */ void __init reserve_top_address(unsigned long reserve) { @@ -594,7 +594,10 @@ void native_set_fixmap(unsigned /* enum fixed_addresses */ idx, #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP #ifdef CONFIG_X86_5LEVEL /** - * p4d_set_huge - setup kernel P4D mapping + * p4d_set_huge - Set up kernel P4D mapping + * @p4d: Pointer to the P4D entry + * @addr: Virtual address associated with the P4D entry + * @prot: Protection bits to use * * No 512GB pages yet -- always return 0 */ @@ -604,9 +607,10 @@ int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) } /** - * p4d_clear_huge - clear kernel P4D mapping when it is set + * p4d_clear_huge - Clear kernel P4D mapping when it is set + * @p4d: Pointer to the P4D entry to clear * - * No 512GB pages yet -- always return 0 + * No 512GB pages yet -- do nothing */ void p4d_clear_huge(p4d_t *p4d) { @@ -614,7 +618,10 @@ void p4d_clear_huge(p4d_t *p4d) #endif /** - * pud_set_huge - setup kernel PUD mapping + * pud_set_huge - Set up kernel PUD mapping + * @pud: Pointer to the PUD entry + * @addr: Virtual address associated with the PUD entry + * @prot: Protection bits to use * * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this * function sets up a huge page only if the complete range has the same MTRR @@ -645,7 +652,10 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) } /** - * pmd_set_huge - setup kernel PMD mapping + * pmd_set_huge - Set up kernel PMD mapping + * @pmd: Pointer to the PMD entry + * @addr: Virtual address associated with the PMD entry + * @prot: Protection bits to use * * See text over pud_set_huge() above. * @@ -674,7 +684,8 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) } /** - * pud_clear_huge - clear kernel PUD mapping when it is set + * pud_clear_huge - Clear kernel PUD mapping when it is set + * @pud: Pointer to the PUD entry to clear. * * Returns 1 on success and 0 on failure (no PUD map is found). */ @@ -689,7 +700,8 @@ int pud_clear_huge(pud_t *pud) } /** - * pmd_clear_huge - clear kernel PMD mapping when it is set + * pmd_clear_huge - Clear kernel PMD mapping when it is set + * @pmd: Pointer to the PMD entry to clear. * * Returns 1 on success and 0 on failure (no PMD map is found). */ @@ -705,11 +717,11 @@ int pmd_clear_huge(pmd_t *pmd) #ifdef CONFIG_X86_64 /** - * pud_free_pmd_page - Clear pud entry and free pmd page. - * @pud: Pointer to a PUD. - * @addr: Virtual address associated with pud. + * pud_free_pmd_page - Clear PUD entry and free PMD page + * @pud: Pointer to a PUD + * @addr: Virtual address associated with PUD * - * Context: The pud range has been unmapped and TLB purged. + * Context: The PUD range has been unmapped and TLB purged. * Return: 1 if clearing the entry succeeded. 0 otherwise. * * NOTE: Callers must allow a single page allocation. @@ -752,11 +764,11 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) } /** - * pmd_free_pte_page - Clear pmd entry and free pte page. - * @pmd: Pointer to a PMD. - * @addr: Virtual address associated with pmd. + * pmd_free_pte_page - Clear PMD entry and free PTE page. + * @pmd: Pointer to the PMD + * @addr: Virtual address associated with PMD * - * Context: The pmd range has been unmapped and TLB purged. + * Context: The PMD range has been unmapped and TLB purged. * Return: 1 if clearing the entry succeeded. 0 otherwise. */ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) @@ -778,7 +790,7 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) /* * Disable free page handling on x86-PAE. This assures that ioremap() - * does not update sync'd pmd entries. See vmalloc_sync_one(). + * does not update sync'd PMD entries. See vmalloc_sync_one(). */ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) { From baad9190e6465fdbe458f59cf04c2b2032ec4797 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 15 May 2025 17:49:16 +0200 Subject: [PATCH 315/331] x86/msr: Add rdmsrl_on_cpu() compatibility wrapper Add a simple rdmsrl_on_cpu() compatibility wrapper for rdmsrq_on_cpu(), to make life in -next easier, where the PM tree recently grew more uses of the old API. Reported-by: Stephen Rothwell Signed-off-by: Ingo Molnar Acked-by: Mario Limonciello Cc: "H. Peter Anvin" Cc: Juergen Gross Cc: Dave Hansen Cc: Xin Li Link: https://lore.kernel.org/r/20250512145517.6e0666e3@canb.auug.org.au --- arch/x86/include/asm/msr.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index a9ce56fc8785..4096b8af4ba7 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -329,6 +329,7 @@ static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) /* Compatibility wrappers: */ #define rdmsrl(msr, val) rdmsrq(msr, val) #define wrmsrl(msr, val) wrmsrq(msr, val) +#define rdmsrl_on_cpu(cpu, msr, q) rdmsrq_on_cpu(cpu, msr, q) #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_MSR_H */ From cdc8be31cb324a0c52529f192e39a44abcfff513 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:30 +0200 Subject: [PATCH 316/331] x86/cpuid: Move CPUID(0x2) APIs into Move all of the CPUID(0x2) APIs at into , in order centralize all CPUID APIs into the latter. While at it, separate the different CPUID leaf parsing APIs using header comments like "CPUID(0xN) parsing: ". Suggested-by: Ingo Molnar Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Cooper Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-2-darwi@linutronix.de --- arch/x86/include/asm/cpuid.h | 1 - arch/x86/include/asm/cpuid/api.h | 75 ++++++++++++++++++++++- arch/x86/include/asm/cpuid/leaf_0x2_api.h | 73 ---------------------- arch/x86/include/asm/cpuid/types.h | 3 +- 4 files changed, 75 insertions(+), 77 deletions(-) delete mode 100644 arch/x86/include/asm/cpuid/leaf_0x2_api.h diff --git a/arch/x86/include/asm/cpuid.h b/arch/x86/include/asm/cpuid.h index 585819331dc6..d5749b25fa10 100644 --- a/arch/x86/include/asm/cpuid.h +++ b/arch/x86/include/asm/cpuid.h @@ -4,6 +4,5 @@ #define _ASM_X86_CPUID_H #include -#include #endif /* _ASM_X86_CPUID_H */ diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index bf76a1706d02..ff8891a0b6c8 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -160,6 +160,10 @@ static inline void __cpuid_read_reg(u32 leaf, u32 subleaf, __cpuid_read_reg(leaf, 0, regidx, (u32 *)(reg)); \ } +/* + * Hypervisor-related APIs: + */ + static __always_inline bool cpuid_function_is_indexed(u32 function) { switch (function) { @@ -208,7 +212,76 @@ static inline u32 hypervisor_cpuid_base(const char *sig, u32 leaves) } /* - * CPUID(0x80000006) parsing helpers + * CPUID(0x2) parsing: + */ + +/** + * cpuid_get_leaf_0x2_regs() - Return sanitized leaf 0x2 register output + * @regs: Output parameter + * + * Query CPUID leaf 0x2 and store its output in @regs. Force set any + * invalid 1-byte descriptor returned by the hardware to zero (the NULL + * cache/TLB descriptor) before returning it to the caller. + * + * Use for_each_leaf_0x2_entry() to iterate over the register output in + * parsed form. + */ +static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) +{ + cpuid_leaf(0x2, regs); + + /* + * All Intel CPUs must report an iteration count of 1. In case + * of bogus hardware, treat all returned descriptors as NULL. + */ + if (regs->desc[0] != 0x01) { + for (int i = 0; i < 4; i++) + regs->regv[i] = 0; + return; + } + + /* + * The most significant bit (MSB) of each register must be clear. + * If a register is invalid, replace its descriptors with NULL. + */ + for (int i = 0; i < 4; i++) { + if (regs->reg[i].invalid) + regs->regv[i] = 0; + } +} + +/** + * for_each_leaf_0x2_entry() - Iterator for parsed leaf 0x2 descriptors + * @regs: Leaf 0x2 register output, returned by cpuid_get_leaf_0x2_regs() + * @__ptr: u8 pointer, for macro internal use only + * @entry: Pointer to parsed descriptor information at each iteration + * + * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers + * @regs. Provide the parsed information for each descriptor through @entry. + * + * To handle cache-specific descriptors, switch on @entry->c_type. For TLB + * descriptors, switch on @entry->t_type. + * + * Example usage for cache descriptors:: + * + * const struct leaf_0x2_table *entry; + * union leaf_0x2_regs regs; + * u8 *ptr; + * + * cpuid_get_leaf_0x2_regs(®s); + * for_each_leaf_0x2_entry(regs, ptr, entry) { + * switch (entry->c_type) { + * ... + * } + * } + */ +#define for_each_leaf_0x2_entry(regs, __ptr, entry) \ + for (__ptr = &(regs).desc[1]; \ + __ptr < &(regs).desc[16] && (entry = &cpuid_0x2_table[*__ptr]); \ + __ptr++) + +/* + * CPUID(0x80000006) parsing: */ static inline bool cpuid_amd_hygon_has_l3_cache(void) diff --git a/arch/x86/include/asm/cpuid/leaf_0x2_api.h b/arch/x86/include/asm/cpuid/leaf_0x2_api.h deleted file mode 100644 index 09fa3070b271..000000000000 --- a/arch/x86/include/asm/cpuid/leaf_0x2_api.h +++ /dev/null @@ -1,73 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_CPUID_LEAF_0x2_API_H -#define _ASM_X86_CPUID_LEAF_0x2_API_H - -#include -#include - -/** - * cpuid_get_leaf_0x2_regs() - Return sanitized leaf 0x2 register output - * @regs: Output parameter - * - * Query CPUID leaf 0x2 and store its output in @regs. Force set any - * invalid 1-byte descriptor returned by the hardware to zero (the NULL - * cache/TLB descriptor) before returning it to the caller. - * - * Use for_each_leaf_0x2_entry() to iterate over the register output in - * parsed form. - */ -static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) -{ - cpuid_leaf(0x2, regs); - - /* - * All Intel CPUs must report an iteration count of 1. In case - * of bogus hardware, treat all returned descriptors as NULL. - */ - if (regs->desc[0] != 0x01) { - for (int i = 0; i < 4; i++) - regs->regv[i] = 0; - return; - } - - /* - * The most significant bit (MSB) of each register must be clear. - * If a register is invalid, replace its descriptors with NULL. - */ - for (int i = 0; i < 4; i++) { - if (regs->reg[i].invalid) - regs->regv[i] = 0; - } -} - -/** - * for_each_leaf_0x2_entry() - Iterator for parsed leaf 0x2 descriptors - * @regs: Leaf 0x2 register output, returned by cpuid_get_leaf_0x2_regs() - * @__ptr: u8 pointer, for macro internal use only - * @entry: Pointer to parsed descriptor information at each iteration - * - * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers - * @regs. Provide the parsed information for each descriptor through @entry. - * - * To handle cache-specific descriptors, switch on @entry->c_type. For TLB - * descriptors, switch on @entry->t_type. - * - * Example usage for cache descriptors:: - * - * const struct leaf_0x2_table *entry; - * union leaf_0x2_regs regs; - * u8 *ptr; - * - * cpuid_get_leaf_0x2_regs(®s); - * for_each_leaf_0x2_entry(regs, ptr, entry) { - * switch (entry->c_type) { - * ... - * } - * } - */ -#define for_each_leaf_0x2_entry(regs, __ptr, entry) \ - for (__ptr = &(regs).desc[1]; \ - __ptr < &(regs).desc[16] && (entry = &cpuid_0x2_table[*__ptr]); \ - __ptr++) - -#endif /* _ASM_X86_CPUID_LEAF_0x2_API_H */ diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h index c95fee66e148..8a00364b79de 100644 --- a/arch/x86/include/asm/cpuid/types.h +++ b/arch/x86/include/asm/cpuid/types.h @@ -31,8 +31,7 @@ enum cpuid_regs_idx { #define CPUID_LEAF_TILE 0x1d /* - * Types for CPUID(0x2) parsing - * Check + * Types for CPUID(0x2) parsing: */ struct leaf_0x2_reg { From 968e3000680713f712bcf02c51c4d7bb7d4d7685 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:31 +0200 Subject: [PATCH 317/331] x86/cpuid: Set as the main CPUID header The main CPUID header was originally a storefront for the headers: Now that the latter CPUID(0x2) header has been merged into the former, there is no practical difference between and . Migrate all users to the header, in preparation of the removal of . Don't remove just yet, in case some new code in -next started using it. Suggested-by: Ingo Molnar Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: H. Peter Anvin Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-3-darwi@linutronix.de --- arch/x86/boot/compressed/sev.c | 2 +- arch/x86/boot/startup/sev-startup.c | 2 +- arch/x86/coco/sev/core.c | 2 +- arch/x86/coco/sev/vc-handle.c | 2 +- arch/x86/events/intel/pt.c | 2 +- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/acpi/cstate.c | 2 +- arch/x86/kernel/amd_nb.c | 2 +- arch/x86/kernel/cpu/cacheinfo.c | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/cpu/intel.c | 2 +- arch/x86/kernel/fpu/xstate.c | 2 +- arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/smpboot.c | 2 +- arch/x86/kernel/tsc.c | 2 +- arch/x86/kvm/cpuid.c | 2 +- arch/x86/virt/svm/sev.c | 2 +- arch/x86/xen/enlighten_pv.c | 2 +- drivers/acpi/acpi_pad.c | 2 +- drivers/dma/ioat/dca.c | 2 +- drivers/idle/intel_idle.c | 2 +- drivers/platform/x86/intel/pmc/core.c | 2 +- sound/soc/intel/avs/tgl.c | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 612b443296d3..fd1b67dfea22 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include "error.h" #include "sev.h" diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c index 435853a55768..0b7e3b950183 100644 --- a/arch/x86/boot/startup/sev-startup.c +++ b/arch/x86/boot/startup/sev-startup.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include /* For early boot hypervisor communication in SEV-ES enabled guests */ diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index b40c159b64e4..5678c3fdcb06 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c index b4895c648024..0989d98da130 100644 --- a/arch/x86/coco/sev/vc-handle.c +++ b/arch/x86/coco/sev/vc-handle.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, unsigned long vaddr, phys_addr_t *paddr) diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index f37cce231266..9dbc688a7343 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 50d34698036d..bde58f6510ac 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -16,7 +16,7 @@ struct vm86; #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index d5ac34186555..8698d66563ed 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 7c79c9f9f89d..c1acead6227a 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -15,7 +15,7 @@ #include #include -#include +#include static u32 *flush_words; diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index f866d94352fb..6d61f7dff9e7 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 114aaaf6ae8a..c14db8d8f183 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -29,7 +29,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 584dd55bf739..7f8ca29c8ac4 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 3e477a553401..9aa9ac8399ae 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index c9982a7c9536..d6387dde3ff9 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 4b668bc683c4..c1d2dac72b9c 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index d6cf1e23c2a3..d7d61b3de2bf 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -64,7 +64,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 5d3a764ba77c..87e749106dda 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 571c906ffcbf..7f43d8d24fbe 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include "cpuid.h" #include "lapic.h" #include "mmu.h" diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 76926f75e9bf..942372e69b4d 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 3be38350f044..7f9ded1bc707 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -49,7 +49,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c index 3fde4496f8a2..6f8bbe1247a5 100644 --- a/drivers/acpi/acpi_pad.c +++ b/drivers/acpi/acpi_pad.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index c9aba2304de7..5d3c0ae6b342 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* either a kernel change is needed, or we need something like this in kernel */ #ifndef CONFIG_SMP diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 6a1712b50c7f..433d858b7be1 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -51,7 +51,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c index ff7f64df7c09..9f678c753dfa 100644 --- a/drivers/platform/x86/intel/pmc/core.c +++ b/drivers/platform/x86/intel/pmc/core.c @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include #include diff --git a/sound/soc/intel/avs/tgl.c b/sound/soc/intel/avs/tgl.c index 56905f2b9eb2..9dbb3ad0954a 100644 --- a/sound/soc/intel/avs/tgl.c +++ b/sound/soc/intel/avs/tgl.c @@ -47,7 +47,7 @@ static int avs_tgl_config_basefw(struct avs_dev *adev) #ifdef CONFIG_X86 unsigned int ecx; -#include +#include ecx = cpuid_ecx(CPUID_TSC_LEAF); if (ecx) { ret = avs_ipc_set_fw_config(adev, 1, AVS_FW_CFG_XTAL_FREQ_HZ, sizeof(ecx), &ecx); From 2f924ca36d2f788d40a57ea48825ff51cba4e700 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:32 +0200 Subject: [PATCH 318/331] x86/cpuid: Rename have_cpuid_p() to cpuid_feature() In order to let all the APIs under have a shared "cpuid_" namespace, rename have_cpuid_p() to cpuid_feature(). Adjust all call-sites accordingly. Suggested-by: Ingo Molnar Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Cooper Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-4-darwi@linutronix.de --- arch/x86/include/asm/cpuid/api.h | 4 ++-- arch/x86/kernel/cpu/common.c | 10 +++++----- arch/x86/kernel/cpu/microcode/core.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index ff8891a0b6c8..c0211fcdc706 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -14,9 +14,9 @@ */ #ifdef CONFIG_X86_32 -bool have_cpuid_p(void); +bool cpuid_feature(void); #else -static inline bool have_cpuid_p(void) +static inline bool cpuid_feature(void) { return true; } diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index c14db8d8f183..8feb8fd2957a 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -322,7 +322,7 @@ static int __init cachesize_setup(char *str) __setup("cachesize=", cachesize_setup); /* Probe for the CPUID instruction */ -bool have_cpuid_p(void) +bool cpuid_feature(void) { return flag_is_changeable_p(X86_EFLAGS_ID); } @@ -1711,11 +1711,11 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) memset(&c->x86_capability, 0, sizeof(c->x86_capability)); c->extended_cpuid_level = 0; - if (!have_cpuid_p()) + if (!cpuid_feature()) identify_cpu_without_cpuid(c); /* cyrix could have cpuid enabled via c_identify()*/ - if (have_cpuid_p()) { + if (cpuid_feature()) { cpu_detect(c); get_cpu_vendor(c); intel_unlock_cpuid_leafs(c); @@ -1875,11 +1875,11 @@ static void generic_identify(struct cpuinfo_x86 *c) { c->extended_cpuid_level = 0; - if (!have_cpuid_p()) + if (!cpuid_feature()) identify_cpu_without_cpuid(c); /* cyrix could have cpuid enabled via c_identify()*/ - if (!have_cpuid_p()) + if (!cpuid_feature()) return; cpu_detect(c); diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index a3bcb6d43f5c..fe50eb5b7c4a 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -118,7 +118,7 @@ bool __init microcode_loader_disabled(void) * 3) Certain AMD patch levels are not allowed to be * overwritten. */ - if (!have_cpuid_p() || + if (!cpuid_feature() || native_cpuid_ecx(1) & BIT(31) || amd_check_current_patch_level()) dis_ucode_ldr = true; From e7df7289f1481993c9f326aea801323a1d3d0c5f Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:34 +0200 Subject: [PATCH 319/331] x86/cpuid: Rename cpuid_get_leaf_0x2_regs() to cpuid_leaf_0x2() Rename the CPUID(0x2) register accessor function: cpuid_get_leaf_0x2_regs(regs) to: cpuid_leaf_0x2(regs) for consistency with other accessors that return full CPUID registers outputs like: cpuid_leaf(regs) cpuid_subleaf(regs) In the same vein, rename the CPUID(0x2) iteration macro: for_each_leaf_0x2_entry() to: for_each_cpuid_0x2_desc() to include "cpuid" in the macro name, and since what is iterated upon is CPUID(0x2) cache and TLB "descriptos", not "entries". Prefix an underscore to that iterator macro parameters, so that the newly renamed 'desc' parameter do not get mixed with "union leaf_0x2_regs :: desc[]" in the macro's implementation. Adjust all the affected call-sites accordingly. While at it, use "CPUID(0x2)" instead of "CPUID leaf 0x2" as this is the recommended style. No change in functionality intended. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Cooper Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-6-darwi@linutronix.de --- arch/x86/include/asm/cpuid/api.h | 40 ++++++++++++++++---------------- arch/x86/kernel/cpu/cacheinfo.c | 4 ++-- arch/x86/kernel/cpu/intel.c | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index c0211fcdc706..ccf20c62b89f 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -216,17 +216,17 @@ static inline u32 hypervisor_cpuid_base(const char *sig, u32 leaves) */ /** - * cpuid_get_leaf_0x2_regs() - Return sanitized leaf 0x2 register output + * cpuid_leaf_0x2() - Return sanitized CPUID(0x2) register output * @regs: Output parameter * - * Query CPUID leaf 0x2 and store its output in @regs. Force set any + * Query CPUID(0x2) and store its output in @regs. Force set any * invalid 1-byte descriptor returned by the hardware to zero (the NULL * cache/TLB descriptor) before returning it to the caller. * - * Use for_each_leaf_0x2_entry() to iterate over the register output in + * Use for_each_cpuid_0x2_desc() to iterate over the register output in * parsed form. */ -static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) +static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs) { cpuid_leaf(0x2, regs); @@ -251,34 +251,34 @@ static inline void cpuid_get_leaf_0x2_regs(union leaf_0x2_regs *regs) } /** - * for_each_leaf_0x2_entry() - Iterator for parsed leaf 0x2 descriptors - * @regs: Leaf 0x2 register output, returned by cpuid_get_leaf_0x2_regs() - * @__ptr: u8 pointer, for macro internal use only - * @entry: Pointer to parsed descriptor information at each iteration + * for_each_cpuid_0x2_desc() - Iterator for parsed CPUID(0x2) descriptors + * @_regs: CPUID(0x2) register output, as returned by cpuid_leaf_0x2() + * @_ptr: u8 pointer, for macro internal use only + * @_desc: Pointer to the parsed CPUID(0x2) descriptor at each iteration * - * Loop over the 1-byte descriptors in the passed leaf 0x2 output registers - * @regs. Provide the parsed information for each descriptor through @entry. + * Loop over the 1-byte descriptors in the passed CPUID(0x2) output registers + * @_regs. Provide the parsed information for each descriptor through @_desc. * - * To handle cache-specific descriptors, switch on @entry->c_type. For TLB - * descriptors, switch on @entry->t_type. + * To handle cache-specific descriptors, switch on @_desc->c_type. For TLB + * descriptors, switch on @_desc->t_type. * * Example usage for cache descriptors:: * - * const struct leaf_0x2_table *entry; + * const struct leaf_0x2_table *desc; * union leaf_0x2_regs regs; * u8 *ptr; * - * cpuid_get_leaf_0x2_regs(®s); - * for_each_leaf_0x2_entry(regs, ptr, entry) { - * switch (entry->c_type) { + * cpuid_leaf_0x2(®s); + * for_each_cpuid_0x2_desc(regs, ptr, desc) { + * switch (desc->c_type) { * ... * } * } */ -#define for_each_leaf_0x2_entry(regs, __ptr, entry) \ - for (__ptr = &(regs).desc[1]; \ - __ptr < &(regs).desc[16] && (entry = &cpuid_0x2_table[*__ptr]); \ - __ptr++) +#define for_each_cpuid_0x2_desc(_regs, _ptr, _desc) \ + for (_ptr = &(_regs).desc[1]; \ + _ptr < &(_regs).desc[16] && (_desc = &cpuid_0x2_table[*_ptr]); \ + _ptr++) /* * CPUID(0x80000006) parsing: diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 6d61f7dff9e7..b6349c1792dd 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -388,8 +388,8 @@ static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) if (c->cpuid_level < 2) return; - cpuid_get_leaf_0x2_regs(®s); - for_each_leaf_0x2_entry(regs, ptr, entry) { + cpuid_leaf_0x2(®s); + for_each_cpuid_0x2_desc(regs, ptr, entry) { switch (entry->c_type) { case CACHE_L1_INST: l1i += entry->c_size; break; case CACHE_L1_DATA: l1d += entry->c_size; break; diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 7f8ca29c8ac4..f8141b5d3e30 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -716,8 +716,8 @@ static void intel_detect_tlb(struct cpuinfo_x86 *c) if (c->cpuid_level < 2) return; - cpuid_get_leaf_0x2_regs(®s); - for_each_leaf_0x2_entry(regs, ptr, entry) + cpuid_leaf_0x2(®s); + for_each_cpuid_0x2_desc(regs, ptr, entry) intel_tlb_lookup(entry); } From 4b21e71ad6cc93d39d78176de269a0dc9a318fc6 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:35 +0200 Subject: [PATCH 320/331] x86/cacheinfo: Rename CPUID(0x2) descriptors iterator parameter The CPUID(0x2) descriptors iterator has been renamed from: for_each_leaf_0x2_entry() to: for_each_cpuid_0x2_desc() since it iterates over CPUID(0x2) cache and TLB "descriptors", not "entries". In the macro's x86/cacheinfo call-site, rename the parameter denoting the parsed descriptor at each iteration from 'entry' to 'desc'. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Cooper Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-7-darwi@linutronix.de --- arch/x86/kernel/cpu/cacheinfo.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index b6349c1792dd..adfa7e8bb865 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -381,7 +381,7 @@ static void intel_cacheinfo_done(struct cpuinfo_x86 *c, unsigned int l3, static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) { unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0; - const struct leaf_0x2_table *entry; + const struct leaf_0x2_table *desc; union leaf_0x2_regs regs; u8 *ptr; @@ -389,12 +389,12 @@ static void intel_cacheinfo_0x2(struct cpuinfo_x86 *c) return; cpuid_leaf_0x2(®s); - for_each_cpuid_0x2_desc(regs, ptr, entry) { - switch (entry->c_type) { - case CACHE_L1_INST: l1i += entry->c_size; break; - case CACHE_L1_DATA: l1d += entry->c_size; break; - case CACHE_L2: l2 += entry->c_size; break; - case CACHE_L3: l3 += entry->c_size; break; + for_each_cpuid_0x2_desc(regs, ptr, desc) { + switch (desc->c_type) { + case CACHE_L1_INST: l1i += desc->c_size; break; + case CACHE_L1_DATA: l1d += desc->c_size; break; + case CACHE_L2: l2 += desc->c_size; break; + case CACHE_L3: l3 += desc->c_size; break; } } From 119deb95b0bc2793d4b002549444ce0aec346b4f Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 8 May 2025 17:02:36 +0200 Subject: [PATCH 321/331] x86/cpu/intel: Rename CPUID(0x2) descriptors iterator parameter The CPUID(0x2) descriptors iterator has been renamed from: for_each_leaf_0x2_entry() to: for_each_cpuid_0x2_desc() since it iterates over CPUID(0x2) cache and TLB "descriptors", not "entries". In the macro's x86/cpu call-site, rename the parameter denoting the parsed descriptor at each iteration from 'entry' to 'desc'. Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Cc: Andrew Cooper Cc: John Ogness Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/20250508150240.172915-8-darwi@linutronix.de --- arch/x86/kernel/cpu/intel.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index f8141b5d3e30..076eaa41b8c8 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -648,11 +648,11 @@ static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) } #endif -static void intel_tlb_lookup(const struct leaf_0x2_table *entry) +static void intel_tlb_lookup(const struct leaf_0x2_table *desc) { - short entries = entry->entries; + short entries = desc->entries; - switch (entry->t_type) { + switch (desc->t_type) { case STLB_4K: tlb_lli_4k = max(tlb_lli_4k, entries); tlb_lld_4k = max(tlb_lld_4k, entries); @@ -709,7 +709,7 @@ static void intel_tlb_lookup(const struct leaf_0x2_table *entry) static void intel_detect_tlb(struct cpuinfo_x86 *c) { - const struct leaf_0x2_table *entry; + const struct leaf_0x2_table *desc; union leaf_0x2_regs regs; u8 *ptr; @@ -717,8 +717,8 @@ static void intel_detect_tlb(struct cpuinfo_x86 *c) return; cpuid_leaf_0x2(®s); - for_each_cpuid_0x2_desc(regs, ptr, entry) - intel_tlb_lookup(entry); + for_each_cpuid_0x2_desc(regs, ptr, desc) + intel_tlb_lookup(desc); } static const struct cpu_dev intel_cpu_dev = { From 3bf8ce828419810f45a272948805cf9a2b685529 Mon Sep 17 00:00:00 2001 From: "Ahmed S. Darwish" Date: Thu, 15 May 2025 22:28:59 +0200 Subject: [PATCH 322/331] x86/cpuid: Rename hypervisor_cpuid_base()/for_each_possible_hypervisor_cpuid_base() to cpuid_base_hypervisor()/for_each_possible_cpuid_base_hypervisor() In order to let all the APIs under have a shared "cpuid_" namespace, rename hypervisor_cpuid_base() to cpuid_base_hypervisor(). To align with the new style, also rename: for_each_possible_hypervisor_cpuid_base(function) to: for_each_possible_cpuid_base_hypervisor(function) Adjust call-sites accordingly. Suggested-by: Ingo Molnar Signed-off-by: Ahmed S. Darwish Signed-off-by: Ingo Molnar Cc: Andrew Cooper Cc: Dave Hansen Cc: David Woodhouse Cc: H. Peter Anvin Cc: John Ogness Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Vitaly Kuznetsov Cc: x86-cpuid@lists.linux.dev Link: https://lore.kernel.org/r/aCZOi0Oohc7DpgTo@lx-t490 --- arch/x86/include/asm/acrn.h | 2 +- arch/x86/include/asm/cpuid/api.h | 6 +++--- arch/x86/include/asm/xen/hypervisor.h | 2 +- arch/x86/kernel/jailhouse.c | 2 +- arch/x86/kernel/kvm.c | 2 +- arch/x86/kvm/cpuid.c | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h index 1dd14381bcb6..fab11192c60a 100644 --- a/arch/x86/include/asm/acrn.h +++ b/arch/x86/include/asm/acrn.h @@ -25,7 +25,7 @@ void acrn_remove_intr_handler(void); static inline u32 acrn_cpuid_base(void) { if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) - return hypervisor_cpuid_base("ACRNACRNACRN", 0); + return cpuid_base_hypervisor("ACRNACRNACRN", 0); return 0; } diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h index ccf20c62b89f..44fa82e1267c 100644 --- a/arch/x86/include/asm/cpuid/api.h +++ b/arch/x86/include/asm/cpuid/api.h @@ -188,14 +188,14 @@ static __always_inline bool cpuid_function_is_indexed(u32 function) return false; } -#define for_each_possible_hypervisor_cpuid_base(function) \ +#define for_each_possible_cpuid_base_hypervisor(function) \ for (function = 0x40000000; function < 0x40010000; function += 0x100) -static inline u32 hypervisor_cpuid_base(const char *sig, u32 leaves) +static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves) { u32 base, eax, signature[3]; - for_each_possible_hypervisor_cpuid_base(base) { + for_each_possible_cpuid_base_hypervisor(base) { cpuid(base, &eax, &signature[0], &signature[1], &signature[2]); /* diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index bd0fc69a10a7..c2fc7869b996 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -43,7 +43,7 @@ extern struct start_info *xen_start_info; static inline uint32_t xen_cpuid_base(void) { - return hypervisor_cpuid_base(XEN_SIGNATURE, 2); + return cpuid_base_hypervisor(XEN_SIGNATURE, 2); } struct pci_dev; diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c index cd8ed1edbf9e..9e9a591a5fec 100644 --- a/arch/x86/kernel/jailhouse.c +++ b/arch/x86/kernel/jailhouse.c @@ -49,7 +49,7 @@ static uint32_t jailhouse_cpuid_base(void) !boot_cpu_has(X86_FEATURE_HYPERVISOR)) return 0; - return hypervisor_cpuid_base("Jailhouse\0\0\0", 0); + return cpuid_base_hypervisor("Jailhouse\0\0\0", 0); } static uint32_t __init jailhouse_detect(void) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index f3642226e0a5..921c1c783bc1 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -875,7 +875,7 @@ static noinline uint32_t __kvm_cpuid_base(void) return 0; /* So we don't blow up on old processors */ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) - return hypervisor_cpuid_base(KVM_SIGNATURE, 0); + return cpuid_base_hypervisor(KVM_SIGNATURE, 0); return 0; } diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 7f43d8d24fbe..ecd85f4801cc 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -236,7 +236,7 @@ static struct kvm_hypervisor_cpuid kvm_get_hypervisor_cpuid(struct kvm_vcpu *vcp struct kvm_cpuid_entry2 *entry; u32 base; - for_each_possible_hypervisor_cpuid_base(base) { + for_each_possible_cpuid_base_hypervisor(base) { entry = kvm_find_cpuid_entry(vcpu, base); if (entry) { From a0f3fe547eb35a2dc298a78da73da73304f41ce1 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Fri, 16 May 2025 16:31:38 +0200 Subject: [PATCH 323/331] x86/bugs: Fix indentation due to ITS merge No functional changes. Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/bugs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index dd8b50b4ceaa..d1a03ffd4b9d 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -2975,10 +2975,10 @@ static void __init srso_apply_mitigation(void) if (boot_cpu_data.x86 == 0x19) { setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); - set_return_thunk(srso_alias_return_thunk); + set_return_thunk(srso_alias_return_thunk); } else { setup_force_cpu_cap(X86_FEATURE_SRSO); - set_return_thunk(srso_return_thunk); + set_return_thunk(srso_return_thunk); } break; case SRSO_MITIGATION_IBPB: From 1bffe6f6890cb40a8d26aec1ffe5f95e2bd09ac2 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Fri, 16 May 2025 15:33:03 +0300 Subject: [PATCH 324/331] x86/mm/64: Always use dynamic memory layout Dynamic memory layout is used by KASLR and 5-level paging. CONFIG_X86_5LEVEL is going to be removed, making 5-level paging support unconditional which requires unconditional support of dynamic memory layout. Remove CONFIG_DYNAMIC_MEMORY_LAYOUT. Signed-off-by: Kirill A. Shutemov Signed-off-by: Ingo Molnar Reviewed-by: Ard Biesheuvel Reviewed-by: Borislav Petkov (AMD) Cc: Jan Kiszka Cc: Kieran Bingham Cc: H. Peter Anvin Cc: Dave Hansen Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250516123306.3812286-2-kirill.shutemov@linux.intel.com --- arch/x86/Kconfig | 8 -------- arch/x86/include/asm/page_64_types.h | 4 ---- arch/x86/include/asm/pgtable_64_types.h | 6 ------ arch/x86/kernel/head64.c | 2 -- scripts/gdb/linux/pgtable.py | 4 +--- 5 files changed, 1 insertion(+), 23 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 90570e3582cc..22c60be0eb37 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1467,7 +1467,6 @@ config X86_PAE config X86_5LEVEL bool "Enable 5-level page tables support" default y - select DYNAMIC_MEMORY_LAYOUT select SPARSEMEM_VMEMMAP depends on X86_64 help @@ -2168,17 +2167,10 @@ config PHYSICAL_ALIGN Don't change this unless you know what you are doing. -config DYNAMIC_MEMORY_LAYOUT - bool - help - This option makes base addresses of vmalloc and vmemmap as well as - __PAGE_OFFSET movable during boot. - config RANDOMIZE_MEMORY bool "Randomize the kernel memory sections" depends on X86_64 depends on RANDOMIZE_BASE - select DYNAMIC_MEMORY_LAYOUT default RANDOMIZE_BASE help Randomizes the base virtual address of kernel memory sections diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 1faa8f88850a..6b8c8169c71d 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -41,11 +41,7 @@ #define __PAGE_OFFSET_BASE_L5 _AC(0xff11000000000000, UL) #define __PAGE_OFFSET_BASE_L4 _AC(0xffff888000000000, UL) -#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT #define __PAGE_OFFSET page_offset_base -#else -#define __PAGE_OFFSET __PAGE_OFFSET_BASE_L4 -#endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */ #define __START_KERNEL_map _AC(0xffffffff80000000, UL) diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index e83721db18c9..eee06f77b245 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -128,15 +128,9 @@ extern unsigned int ptrs_per_p4d; #define __VMEMMAP_BASE_L4 0xffffea0000000000UL #define __VMEMMAP_BASE_L5 0xffd4000000000000UL -#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT # define VMALLOC_START vmalloc_base # define VMALLOC_SIZE_TB (pgtable_l5_enabled() ? VMALLOC_SIZE_TB_L5 : VMALLOC_SIZE_TB_L4) # define VMEMMAP_START vmemmap_base -#else -# define VMALLOC_START __VMALLOC_BASE_L4 -# define VMALLOC_SIZE_TB VMALLOC_SIZE_TB_L4 -# define VMEMMAP_START __VMEMMAP_BASE_L4 -#endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */ #ifdef CONFIG_RANDOMIZE_MEMORY # define DIRECT_MAP_PHYSMEM_END direct_map_physmem_end diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 14f7dda20954..9f617be64fa9 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -59,14 +59,12 @@ unsigned int ptrs_per_p4d __ro_after_init = 1; EXPORT_SYMBOL(ptrs_per_p4d); #endif -#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4; EXPORT_SYMBOL(page_offset_base); unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4; EXPORT_SYMBOL(vmalloc_base); unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4; EXPORT_SYMBOL(vmemmap_base); -#endif /* Wipe all early page tables except for the kernel symbol map */ static void __init reset_early_page_tables(void) diff --git a/scripts/gdb/linux/pgtable.py b/scripts/gdb/linux/pgtable.py index 30d837f3dfae..09aac2421fb8 100644 --- a/scripts/gdb/linux/pgtable.py +++ b/scripts/gdb/linux/pgtable.py @@ -29,11 +29,9 @@ def page_mask(level=1): raise Exception(f'Unknown page level: {level}') -#page_offset_base in case CONFIG_DYNAMIC_MEMORY_LAYOUT is disabled -POB_NO_DYNAMIC_MEM_LAYOUT = '0xffff888000000000' def _page_offset_base(): pob_symbol = gdb.lookup_global_symbol('page_offset_base') - pob = pob_symbol.name if pob_symbol else POB_NO_DYNAMIC_MEM_LAYOUT + pob = pob_symbol.name return gdb.parse_and_eval(pob) From cba5d9b3e99d6268d7909a65c2bd78f4d195aead Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Fri, 16 May 2025 15:33:04 +0300 Subject: [PATCH 325/331] x86/mm/64: Make SPARSEMEM_VMEMMAP the only memory model 5-level paging only supports SPARSEMEM_VMEMMAP. CONFIG_X86_5LEVEL is being phased out, making 5-level paging support mandatory. Make CONFIG_SPARSEMEM_VMEMMAP mandatory for x86-64 and eliminate any associated conditional statements. Signed-off-by: Kirill A. Shutemov Signed-off-by: Ingo Molnar Reviewed-by: Ard Biesheuvel Reviewed-by: Borislav Petkov (AMD) Cc: Dave Hansen Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250516123306.3812286-3-kirill.shutemov@linux.intel.com --- arch/x86/Kconfig | 2 +- arch/x86/mm/init_64.c | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 22c60be0eb37..27f5d879e107 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1467,7 +1467,6 @@ config X86_PAE config X86_5LEVEL bool "Enable 5-level page tables support" default y - select SPARSEMEM_VMEMMAP depends on X86_64 help 5-level paging enables access to larger address space: @@ -1579,6 +1578,7 @@ config ARCH_SPARSEMEM_ENABLE def_bool y select SPARSEMEM_STATIC if X86_32 select SPARSEMEM_VMEMMAP_ENABLE if X86_64 + select SPARSEMEM_VMEMMAP if X86_64 config ARCH_SPARSEMEM_DEFAULT def_bool X86_64 || (NUMA && X86_32) diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index bf45c7aed336..66330fe4e18c 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -833,7 +833,6 @@ void __init paging_init(void) zone_sizes_init(); } -#ifdef CONFIG_SPARSEMEM_VMEMMAP #define PAGE_UNUSED 0xFD /* @@ -932,7 +931,6 @@ static void __meminit vmemmap_use_new_sub_pmd(unsigned long start, unsigned long if (!IS_ALIGNED(end, PMD_SIZE)) unused_pmd_start = end; } -#endif /* * Memory hotplug specific functions @@ -1152,16 +1150,13 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end, pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); pages++; - } -#ifdef CONFIG_SPARSEMEM_VMEMMAP - else if (vmemmap_pmd_is_unused(addr, next)) { + } else if (vmemmap_pmd_is_unused(addr, next)) { free_hugepage_table(pmd_page(*pmd), altmap); spin_lock(&init_mm.page_table_lock); pmd_clear(pmd); spin_unlock(&init_mm.page_table_lock); } -#endif continue; } @@ -1500,7 +1495,6 @@ unsigned long memory_block_size_bytes(void) return memory_block_size_probed; } -#ifdef CONFIG_SPARSEMEM_VMEMMAP /* * Initialise the sparsemem vmemmap using huge-pages at the PMD level. */ @@ -1647,4 +1641,3 @@ void __meminit vmemmap_populate_print_last(void) node_start = 0; } } -#endif From 7212b58d6d7133e4cd3c2295e1fb54febe284156 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Fri, 16 May 2025 15:33:05 +0300 Subject: [PATCH 326/331] x86/mm/64: Make 5-level paging support unconditional Both Intel and AMD CPUs support 5-level paging, which is expected to become more widely adopted in the future. All major x86 Linux distributions have the feature enabled. Remove CONFIG_X86_5LEVEL and related #ifdeffery for it to make it more readable. Suggested-by: Borislav Petkov Signed-off-by: Kirill A. Shutemov Signed-off-by: Ingo Molnar Reviewed-by: Ard Biesheuvel Reviewed-by: Borislav Petkov (AMD) Cc: Dave Hansen Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250516123306.3812286-4-kirill.shutemov@linux.intel.com --- Documentation/arch/x86/cpuinfo.rst | 8 +++---- .../arch/x86/x86_64/5level-paging.rst | 9 -------- arch/x86/Kconfig | 22 +------------------ arch/x86/Kconfig.cpufeatures | 4 ---- arch/x86/boot/compressed/pgtable_64.c | 11 ++-------- arch/x86/boot/header.S | 4 ---- arch/x86/boot/startup/map_kernel.c | 5 +---- arch/x86/entry/vsyscall/vsyscall_64.c | 2 -- arch/x86/include/asm/page_64.h | 2 -- arch/x86/include/asm/page_64_types.h | 7 ------ arch/x86/include/asm/pgtable_64.h | 2 -- arch/x86/include/asm/pgtable_64_types.h | 18 --------------- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/head64.c | 2 -- arch/x86/kernel/head_64.S | 2 -- arch/x86/mm/init.c | 4 ---- arch/x86/mm/pgtable.c | 2 +- arch/x86/xen/mmu_pv.c | 4 ---- drivers/firmware/efi/libstub/x86-5lvl.c | 2 +- 19 files changed, 10 insertions(+), 102 deletions(-) diff --git a/Documentation/arch/x86/cpuinfo.rst b/Documentation/arch/x86/cpuinfo.rst index f80e2a558d2a..dd8b7806944e 100644 --- a/Documentation/arch/x86/cpuinfo.rst +++ b/Documentation/arch/x86/cpuinfo.rst @@ -173,10 +173,10 @@ For example, when an old kernel is running on new hardware. The kernel disabled support for it at compile-time -------------------------------------------------- -For example, if 5-level-paging is not enabled when building (i.e., -CONFIG_X86_5LEVEL is not selected) the flag "la57" will not show up [#f1]_. +For example, if Linear Address Masking (LAM) is not enabled when building (i.e., +CONFIG_ADDRESS_MASKING is not selected) the flag "lam" will not show up. Even though the feature will still be detected via CPUID, the kernel disables -it by clearing via setup_clear_cpu_cap(X86_FEATURE_LA57). +it by clearing via setup_clear_cpu_cap(X86_FEATURE_LAM). The feature is disabled at boot-time ------------------------------------ @@ -200,5 +200,3 @@ missing at runtime. For example, AVX flags will not show up if XSAVE feature is disabled since they depend on XSAVE feature. Another example would be broken CPUs and them missing microcode patches. Due to that, the kernel decides not to enable a feature. - -.. [#f1] 5-level paging uses linear address of 57 bits. diff --git a/Documentation/arch/x86/x86_64/5level-paging.rst b/Documentation/arch/x86/x86_64/5level-paging.rst index 71f882f4a173..ad7ddc13f79d 100644 --- a/Documentation/arch/x86/x86_64/5level-paging.rst +++ b/Documentation/arch/x86/x86_64/5level-paging.rst @@ -22,15 +22,6 @@ QEMU 2.9 and later support 5-level paging. Virtual memory layout for 5-level paging is described in Documentation/arch/x86/x86_64/mm.rst - -Enabling 5-level paging -======================= -CONFIG_X86_5LEVEL=y enables the feature. - -Kernel with CONFIG_X86_5LEVEL=y still able to boot on 4-level hardware. -In this case additional page table level -- p4d -- will be folded at -runtime. - User-space and large virtual address space ========================================== On x86, 5-level paging enables 56-bit userspace virtual address space. diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 27f5d879e107..bae3e97482f1 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -427,8 +427,7 @@ config DYNAMIC_PHYSICAL_MASK config PGTABLE_LEVELS int - default 5 if X86_5LEVEL - default 4 if X86_64 + default 5 if X86_64 default 3 if X86_PAE default 2 @@ -1464,25 +1463,6 @@ config X86_PAE has the cost of more pagetable lookup overhead, and also consumes more pagetable space per process. -config X86_5LEVEL - bool "Enable 5-level page tables support" - default y - depends on X86_64 - help - 5-level paging enables access to larger address space: - up to 128 PiB of virtual address space and 4 PiB of - physical address space. - - It will be supported by future Intel CPUs. - - A kernel with the option enabled can be booted on machines that - support 4- or 5-level paging. - - See Documentation/arch/x86/x86_64/5level-paging.rst for more - information. - - Say N if unsure. - config X86_DIRECT_GBPAGES def_bool y depends on X86_64 diff --git a/arch/x86/Kconfig.cpufeatures b/arch/x86/Kconfig.cpufeatures index e12d5b7e39a2..250c10627ab3 100644 --- a/arch/x86/Kconfig.cpufeatures +++ b/arch/x86/Kconfig.cpufeatures @@ -132,10 +132,6 @@ config X86_DISABLED_FEATURE_OSPKE def_bool y depends on !X86_INTEL_MEMORY_PROTECTION_KEYS -config X86_DISABLED_FEATURE_LA57 - def_bool y - depends on !X86_5LEVEL - config X86_DISABLED_FEATURE_PTI def_bool y depends on !MITIGATION_PAGE_TABLE_ISOLATION diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 5a6c7a190e5b..bdd26050dff7 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -10,12 +10,10 @@ #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ -#ifdef CONFIG_X86_5LEVEL /* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */ unsigned int __section(".data") __pgtable_l5_enabled; unsigned int __section(".data") pgdir_shift = 39; unsigned int __section(".data") ptrs_per_p4d = 1; -#endif /* Buffer to preserve trampoline memory */ static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; @@ -114,18 +112,13 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * Check if LA57 is desired and supported. * * There are several parts to the check: - * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y * - if user asked to disable 5-level paging: no5lvl in cmdline * - if the machine supports 5-level paging: * + CPUID leaf 7 is supported * + the leaf has the feature bit set - * - * That's substitute for boot_cpu_has() in early boot code. */ - if (IS_ENABLED(CONFIG_X86_5LEVEL) && - !cmdline_find_option_bool("no5lvl") && - native_cpuid_eax(0) >= 7 && - (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) { + if (!cmdline_find_option_bool("no5lvl") && + native_cpuid_eax(0) >= 7 && (native_cpuid_ecx(7) & BIT(16))) { l5_required = true; /* Initialize variables for 5-level paging */ diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9cb91421b4e4..e30649e44d8f 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -361,12 +361,8 @@ xloadflags: #endif #ifdef CONFIG_X86_64 -#ifdef CONFIG_X86_5LEVEL #define XLF56 (XLF_5LEVEL|XLF_5LEVEL_ENABLED) #else -#define XLF56 XLF_5LEVEL -#endif -#else #define XLF56 0 #endif diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c index 905e8734b5a3..332dbe6688c4 100644 --- a/arch/x86/boot/startup/map_kernel.c +++ b/arch/x86/boot/startup/map_kernel.c @@ -16,9 +16,6 @@ extern unsigned int next_early_pgt; static inline bool check_la57_support(void) { - if (!IS_ENABLED(CONFIG_X86_5LEVEL)) - return false; - /* * 5-level paging is detected and enabled at kernel decompression * stage. Only check if it has been enabled there. @@ -129,7 +126,7 @@ unsigned long __head __startup_64(unsigned long p2v_offset, pgd = rip_rel_ptr(early_top_pgt); pgd[pgd_index(__START_KERNEL_map)] += load_delta; - if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) { + if (la57) { p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); p4d[MAX_PTRS_PER_P4D - 1] += load_delta; diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 2fb7d53cf333..c9103a6fa06e 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -341,9 +341,7 @@ void __init set_vsyscall_pgtable_user_bits(pgd_t *root) pgd = pgd_offset_pgd(root, VSYSCALL_ADDR); set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER)); p4d = p4d_offset(pgd, VSYSCALL_ADDR); -#if CONFIG_PGTABLE_LEVELS >= 5 set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER)); -#endif pud = pud_offset(p4d, VSYSCALL_ADDR); set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER)); pmd = pmd_offset(pud, VSYSCALL_ADDR); diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index d3aab6f4e59a..015d23f3e01f 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -62,7 +62,6 @@ static inline void clear_page(void *page) void copy_page(void *to, void *from); KCFI_REFERENCE(copy_page); -#ifdef CONFIG_X86_5LEVEL /* * User space process size. This is the first address outside the user range. * There are a few constraints that determine this: @@ -93,7 +92,6 @@ static __always_inline unsigned long task_size_max(void) return ret; } -#endif /* CONFIG_X86_5LEVEL */ #endif /* !__ASSEMBLER__ */ diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 6b8c8169c71d..7400dab373fe 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -48,14 +48,7 @@ /* See Documentation/arch/x86/x86_64/mm.rst for a description of the memory map. */ #define __PHYSICAL_MASK_SHIFT 52 - -#ifdef CONFIG_X86_5LEVEL #define __VIRTUAL_MASK_SHIFT (pgtable_l5_enabled() ? 56 : 47) -/* See task_size_max() in */ -#else -#define __VIRTUAL_MASK_SHIFT 47 -#define task_size_max() ((_AC(1,UL) << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE) -#endif #define TASK_SIZE_MAX task_size_max() #define DEFAULT_MAP_WINDOW ((1UL << 47) - PAGE_SIZE) diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index b89f8f1194a9..f06e5d6a2747 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -41,11 +41,9 @@ static inline void sync_initial_page_table(void) { } pr_err("%s:%d: bad pud %p(%016lx)\n", \ __FILE__, __LINE__, &(e), pud_val(e)) -#if CONFIG_PGTABLE_LEVELS >= 5 #define p4d_ERROR(e) \ pr_err("%s:%d: bad p4d %p(%016lx)\n", \ __FILE__, __LINE__, &(e), p4d_val(e)) -#endif #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %p(%016lx)\n", \ diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index eee06f77b245..4604f924d8b8 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -23,7 +23,6 @@ typedef struct { pmdval_t pmd; } pmd_t; extern unsigned int __pgtable_l5_enabled; -#ifdef CONFIG_X86_5LEVEL #ifdef USE_EARLY_PGTABLE_L5 /* * cpu_feature_enabled() is not available in early boot code. @@ -37,17 +36,11 @@ static inline bool pgtable_l5_enabled(void) #define pgtable_l5_enabled() cpu_feature_enabled(X86_FEATURE_LA57) #endif /* USE_EARLY_PGTABLE_L5 */ -#else -#define pgtable_l5_enabled() 0 -#endif /* CONFIG_X86_5LEVEL */ - extern unsigned int pgdir_shift; extern unsigned int ptrs_per_p4d; #endif /* !__ASSEMBLER__ */ -#ifdef CONFIG_X86_5LEVEL - /* * PGDIR_SHIFT determines what a top-level page table entry can map */ @@ -65,17 +58,6 @@ extern unsigned int ptrs_per_p4d; #define MAX_POSSIBLE_PHYSMEM_BITS 52 -#else /* CONFIG_X86_5LEVEL */ - -/* - * PGDIR_SHIFT determines what a top-level page table entry can map - */ -#define PGDIR_SHIFT 39 -#define PTRS_PER_PGD 512 -#define MAX_PTRS_PER_P4D 1 - -#endif /* CONFIG_X86_5LEVEL */ - /* * 3rd level page */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 2385528792b2..f3c68b586a95 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -590,7 +590,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, DPRINTK(ALT, "alt table %px, -> %px", start, end); /* - * In the case CONFIG_X86_5LEVEL=y, KASAN_SHADOW_START is defined using + * KASAN_SHADOW_START is defined using * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here. * During the process, KASAN becomes confused seeing partial LA57 * conversion and triggers a false-positive out-of-bound report. diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 9f617be64fa9..533fcf5636fc 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -51,13 +51,11 @@ unsigned int __initdata next_early_pgt; SYM_PIC_ALIAS(next_early_pgt); pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX); -#ifdef CONFIG_X86_5LEVEL unsigned int __pgtable_l5_enabled __ro_after_init; unsigned int pgdir_shift __ro_after_init = 39; EXPORT_SYMBOL(pgdir_shift); unsigned int ptrs_per_p4d __ro_after_init = 1; EXPORT_SYMBOL(ptrs_per_p4d); -#endif unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4; EXPORT_SYMBOL(page_offset_base); diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 069420853304..3e9b3a3bd039 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -649,13 +649,11 @@ SYM_DATA_START_PTI_ALIGNED(init_top_pgt) SYM_DATA_END(init_top_pgt) #endif -#ifdef CONFIG_X86_5LEVEL SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt) .fill 511,8,0 .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC SYM_DATA_END(level4_kernel_pgt) SYM_PIC_ALIAS(level4_kernel_pgt) -#endif SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt) .fill L3_START_KERNEL,8,0 diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index aa56d9ac0b8f..7456df985d96 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -174,11 +174,7 @@ __ref void *alloc_low_pages(unsigned int num) * randomization is enabled. */ -#ifndef CONFIG_X86_5LEVEL -#define INIT_PGD_PAGE_TABLES 3 -#else #define INIT_PGD_PAGE_TABLES 4 -#endif #ifndef CONFIG_RANDOMIZE_MEMORY #define INIT_PGD_PAGE_COUNT (2 * INIT_PGD_PAGE_TABLES) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 59c42dec7076..62777ba4de1a 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -592,7 +592,7 @@ void native_set_fixmap(unsigned /* enum fixed_addresses */ idx, } #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP -#ifdef CONFIG_X86_5LEVEL +#if CONFIG_PGTABLE_LEVELS > 4 /** * p4d_set_huge - Set up kernel P4D mapping * @p4d: Pointer to the P4D entry diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 38971c6dcd4b..2a4a8deaf612 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -578,7 +578,6 @@ static void xen_set_p4d(p4d_t *ptr, p4d_t val) xen_mc_issue(XEN_LAZY_MMU); } -#if CONFIG_PGTABLE_LEVELS >= 5 __visible p4dval_t xen_p4d_val(p4d_t p4d) { return pte_mfn_to_pfn(p4d.p4d); @@ -592,7 +591,6 @@ __visible p4d_t xen_make_p4d(p4dval_t p4d) return native_make_p4d(p4d); } PV_CALLEE_SAVE_REGS_THUNK(xen_make_p4d); -#endif /* CONFIG_PGTABLE_LEVELS >= 5 */ static void xen_pmd_walk(struct mm_struct *mm, pmd_t *pmd, void (*func)(struct mm_struct *mm, struct page *, @@ -2222,10 +2220,8 @@ static const typeof(pv_ops) xen_mmu_ops __initconst = { .alloc_pud = xen_alloc_pmd_init, .release_pud = xen_release_pmd_init, -#if CONFIG_PGTABLE_LEVELS >= 5 .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), -#endif .enter_mmap = xen_enter_mmap, .exit_mmap = xen_exit_mmap, diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c index 77359e802181..f1c5fb45d5f7 100644 --- a/drivers/firmware/efi/libstub/x86-5lvl.c +++ b/drivers/firmware/efi/libstub/x86-5lvl.c @@ -62,7 +62,7 @@ efi_status_t efi_setup_5level_paging(void) void efi_5level_switch(void) { - bool want_la57 = IS_ENABLED(CONFIG_X86_5LEVEL) && !efi_no5lvl; + bool want_la57 = !efi_no5lvl; bool have_la57 = native_read_cr4() & X86_CR4_LA57; bool need_toggle = want_la57 ^ have_la57; u64 *pgt = (void *)la57_toggle + PAGE_SIZE; From 09230b7554824c9db1712324efcf3595c67fd326 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Fri, 16 May 2025 15:33:06 +0300 Subject: [PATCH 327/331] x86/paravirt: Restrict PARAVIRT_XXL to 64-bit only PARAVIRT_XXL is exclusively utilized by XEN_PV, which is only compatible with 64-bit machines. Clearly designate PARAVIRT_XXL as 64-bit only and remove ifdefs to support CONFIG_PGTABLE_LEVELS < 5. Signed-off-by: Kirill A. Shutemov Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Cc: Dave Hansen Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250516123306.3812286-5-kirill.shutemov@linux.intel.com --- arch/x86/Kconfig | 1 + arch/x86/include/asm/paravirt.h | 4 ---- arch/x86/include/asm/paravirt_types.h | 2 -- arch/x86/kernel/paravirt.c | 2 -- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bae3e97482f1..121f9f03bd5c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -799,6 +799,7 @@ config PARAVIRT config PARAVIRT_XXL bool + depends on X86_64 config PARAVIRT_DEBUG bool "paravirt-ops debugging" diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 03f680d1057a..b5e59a7ba0d0 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -463,8 +463,6 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) PVOP_VCALL2(mmu.set_p4d, p4dp, val); } -#if CONFIG_PGTABLE_LEVELS >= 5 - static inline p4d_t __p4d(p4dval_t val) { p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val, @@ -496,8 +494,6 @@ static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) set_pgd(pgdp, native_make_pgd(0)); \ } while (0) -#endif /* CONFIG_PGTABLE_LEVELS == 5 */ - static inline void p4d_clear(p4d_t *p4dp) { set_p4d(p4dp, native_make_p4d(0)); diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index b08b9d3122d6..37a8627d8277 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -189,12 +189,10 @@ struct pv_mmu_ops { void (*set_p4d)(p4d_t *p4dp, p4d_t p4dval); -#if CONFIG_PGTABLE_LEVELS >= 5 struct paravirt_callee_save p4d_val; struct paravirt_callee_save make_p4d; void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval); -#endif /* CONFIG_PGTABLE_LEVELS >= 5 */ struct pv_lazy_ops lazy_mode; diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 015bf298434f..ab3e172dcc69 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -211,12 +211,10 @@ struct paravirt_patch_template pv_ops = { .mmu.set_p4d = native_set_p4d, -#if CONFIG_PGTABLE_LEVELS >= 5 .mmu.p4d_val = PTE_IDENT, .mmu.make_p4d = PTE_IDENT, .mmu.set_pgd = native_set_pgd, -#endif /* CONFIG_PGTABLE_LEVELS >= 5 */ .mmu.pte_val = PTE_IDENT, .mmu.pgd_val = PTE_IDENT, From 9220aa8a6779b586ef11bcd5473d103f7cf60756 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Mon, 12 May 2025 01:45:50 -0700 Subject: [PATCH 328/331] x86/msr: Remove a superfluous inclusion of The following commit: efef7f184f2e ("x86/msr: Add explicit includes of ") added a superfluous inclusion of to drivers/acpi/processor_throttling.c. Remove it. Fixes: efef7f184f2e ("x86/msr: Add explicit includes of ") Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Cc: H. Peter Anvin Link: https://lore.kernel.org/r/20250512084552.1586883-2-xin@zytor.com --- drivers/acpi/processor_throttling.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c index ecd7fe256153..d1541a386fbc 100644 --- a/drivers/acpi/processor_throttling.c +++ b/drivers/acpi/processor_throttling.c @@ -21,7 +21,6 @@ #include #include #include -#include #ifdef CONFIG_X86 #include #endif From e95534e107d2e9e136aa4d7cbededb3827e80074 Mon Sep 17 00:00:00 2001 From: "Xin Li (Intel)" Date: Sat, 17 May 2025 09:57:12 -0700 Subject: [PATCH 329/331] x86/xen/msr: Fix uninitialized variable 'err' xen_read_msr_safe() currently passes an uninitialized argument 'err' to xen_do_read_msr(). But as xen_do_read_msr() may not set the argument, xen_read_msr_safe() could return err with an unpredictable value. To ensure correctness, initialize err to 0 (representing success) in xen_read_msr_safe(). Do the same in xen_read_msr(), even err is not used after being passed to xen_do_read_msr(). Closes: https://lore.kernel.org/xen-devel/aBxNI_Q0-MhtBSZG@stanley.mountain/ Reported-by: Dan Carpenter Signed-off-by: Xin Li (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Juergen Gross Cc: H. Peter Anvin Link: https://lore.kernel.org/r/20250517165713.935384-1-xin@zytor.com --- arch/x86/xen/enlighten_pv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 7f9ded1bc707..26bbaf4b7330 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1162,7 +1162,7 @@ static void xen_do_write_msr(u32 msr, u64 val, int *err) static int xen_read_msr_safe(u32 msr, u64 *val) { - int err; + int err = 0; *val = xen_do_read_msr(msr, &err); return err; @@ -1179,7 +1179,7 @@ static int xen_write_msr_safe(u32 msr, u64 val) static u64 xen_read_msr(u32 msr) { - int err; + int err = 0; return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL); } From 61ab72c2c6bf24f28b3dbfd3126e984d5afa8424 Mon Sep 17 00:00:00 2001 From: David Kaplan Date: Fri, 16 May 2025 14:32:11 -0500 Subject: [PATCH 330/331] x86/bugs: Restructure ITS mitigation Restructure the ITS mitigation to use select/update/apply functions like the other mitigations. There is a particularly complex interaction between ITS and Retbleed as CDT (Call Depth Tracking) is a mitigation for both, and either its=stuff or retbleed=stuff will attempt to enable CDT. retbleed_update_mitigation() runs first and will check the necessary pre-conditions for CDT if either ITS or Retbleed stuffing is selected. If checks pass and ITS stuffing is selected, it will select stuffing for Retbleed as well. its_update_mitigation() runs after and will either select stuffing if retbleed stuffing was enabled, or fall back to the default (aligned thunks) if stuffing could not be enabled. Enablement of CDT is done exclusively in retbleed_apply_mitigation(). its_apply_mitigation() is only used to enable aligned thunks. Signed-off-by: David Kaplan Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/20250516193212.128782-1-david.kaplan@amd.com --- arch/x86/kernel/cpu/bugs.c | 177 ++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 82 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index d1a03ffd4b9d..3d5796d25f78 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -92,6 +92,8 @@ static void __init bhi_select_mitigation(void); static void __init bhi_update_mitigation(void); static void __init bhi_apply_mitigation(void); static void __init its_select_mitigation(void); +static void __init its_update_mitigation(void); +static void __init its_apply_mitigation(void); /* The base value of the SPEC_CTRL MSR without task-specific bits set */ u64 x86_spec_ctrl_base; @@ -235,6 +237,11 @@ void __init cpu_select_mitigations(void) * spectre_v2=ibrs. */ retbleed_update_mitigation(); + /* + * its_update_mitigation() depends on spectre_v2_update_mitigation() + * and retbleed_update_mitigation(). + */ + its_update_mitigation(); /* * spectre_v2_user_update_mitigation() depends on @@ -263,6 +270,7 @@ void __init cpu_select_mitigations(void) srbds_apply_mitigation(); srso_apply_mitigation(); gds_apply_mitigation(); + its_apply_mitigation(); bhi_apply_mitigation(); } @@ -1115,6 +1123,17 @@ enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; #undef pr_fmt #define pr_fmt(fmt) "RETBleed: " fmt +enum its_mitigation { + ITS_MITIGATION_OFF, + ITS_MITIGATION_AUTO, + ITS_MITIGATION_VMEXIT_ONLY, + ITS_MITIGATION_ALIGNED_THUNKS, + ITS_MITIGATION_RETPOLINE_STUFF, +}; + +static enum its_mitigation its_mitigation __ro_after_init = + IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_MITIGATION_AUTO : ITS_MITIGATION_OFF; + enum retbleed_mitigation { RETBLEED_MITIGATION_NONE, RETBLEED_MITIGATION_AUTO, @@ -1242,11 +1261,19 @@ static void __init retbleed_update_mitigation(void) /* * retbleed=stuff is only allowed on Intel. If stuffing can't be used * then a different mitigation will be selected below. + * + * its=stuff will also attempt to enable stuffing. */ - if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) { + if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF || + its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF) { if (spectre_v2_enabled != SPECTRE_V2_RETPOLINE) { pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); retbleed_mitigation = RETBLEED_MITIGATION_AUTO; + } else { + if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) + pr_info("Retbleed mitigation updated to stuffing\n"); + + retbleed_mitigation = RETBLEED_MITIGATION_STUFF; } } /* @@ -1338,20 +1365,6 @@ static void __init retbleed_apply_mitigation(void) #undef pr_fmt #define pr_fmt(fmt) "ITS: " fmt -enum its_mitigation_cmd { - ITS_CMD_OFF, - ITS_CMD_ON, - ITS_CMD_VMEXIT, - ITS_CMD_RSB_STUFF, -}; - -enum its_mitigation { - ITS_MITIGATION_OFF, - ITS_MITIGATION_VMEXIT_ONLY, - ITS_MITIGATION_ALIGNED_THUNKS, - ITS_MITIGATION_RETPOLINE_STUFF, -}; - static const char * const its_strings[] = { [ITS_MITIGATION_OFF] = "Vulnerable", [ITS_MITIGATION_VMEXIT_ONLY] = "Mitigation: Vulnerable, KVM: Not affected", @@ -1359,11 +1372,6 @@ static const char * const its_strings[] = { [ITS_MITIGATION_RETPOLINE_STUFF] = "Mitigation: Retpolines, Stuffing RSB", }; -static enum its_mitigation its_mitigation __ro_after_init = ITS_MITIGATION_ALIGNED_THUNKS; - -static enum its_mitigation_cmd its_cmd __ro_after_init = - IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_CMD_ON : ITS_CMD_OFF; - static int __init its_parse_cmdline(char *str) { if (!str) @@ -1375,16 +1383,16 @@ static int __init its_parse_cmdline(char *str) } if (!strcmp(str, "off")) { - its_cmd = ITS_CMD_OFF; + its_mitigation = ITS_MITIGATION_OFF; } else if (!strcmp(str, "on")) { - its_cmd = ITS_CMD_ON; + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; } else if (!strcmp(str, "force")) { - its_cmd = ITS_CMD_ON; + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; setup_force_cpu_bug(X86_BUG_ITS); } else if (!strcmp(str, "vmexit")) { - its_cmd = ITS_CMD_VMEXIT; + its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; } else if (!strcmp(str, "stuff")) { - its_cmd = ITS_CMD_RSB_STUFF; + its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; } else { pr_err("Ignoring unknown indirect_target_selection option (%s).", str); } @@ -1395,85 +1403,90 @@ early_param("indirect_target_selection", its_parse_cmdline); static void __init its_select_mitigation(void) { - enum its_mitigation_cmd cmd = its_cmd; - if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) { its_mitigation = ITS_MITIGATION_OFF; return; } - /* Retpoline+CDT mitigates ITS, bail out */ - if (boot_cpu_has(X86_FEATURE_RETPOLINE) && - boot_cpu_has(X86_FEATURE_CALL_DEPTH)) { - its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; - goto out; - } + if (its_mitigation == ITS_MITIGATION_AUTO) + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; + + if (its_mitigation == ITS_MITIGATION_OFF) + return; - /* Exit early to avoid irrelevant warnings */ - if (cmd == ITS_CMD_OFF) { - its_mitigation = ITS_MITIGATION_OFF; - goto out; - } - if (spectre_v2_enabled == SPECTRE_V2_NONE) { - pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); - its_mitigation = ITS_MITIGATION_OFF; - goto out; - } if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) { pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n"); its_mitigation = ITS_MITIGATION_OFF; - goto out; + return; } + if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) { pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n"); its_mitigation = ITS_MITIGATION_OFF; - goto out; + return; } - if (boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) { + + if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && + !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) { + pr_err("RSB stuff mitigation not supported, using default\n"); + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; + } + + if (its_mitigation == ITS_MITIGATION_VMEXIT_ONLY && + !boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) + its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; +} + +static void __init its_update_mitigation(void) +{ + if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) + return; + + switch (spectre_v2_enabled) { + case SPECTRE_V2_NONE: + pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n"); + its_mitigation = ITS_MITIGATION_OFF; + break; + case SPECTRE_V2_RETPOLINE: + /* Retpoline+CDT mitigates ITS */ + if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) + its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; + break; + case SPECTRE_V2_LFENCE: + case SPECTRE_V2_EIBRS_LFENCE: pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n"); - its_mitigation = ITS_MITIGATION_OFF; - goto out; - } - - if (cmd == ITS_CMD_RSB_STUFF && - (!boot_cpu_has(X86_FEATURE_RETPOLINE) || !IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING))) { - pr_err("RSB stuff mitigation not supported, using default\n"); - cmd = ITS_CMD_ON; - } - - switch (cmd) { - case ITS_CMD_OFF: its_mitigation = ITS_MITIGATION_OFF; break; - case ITS_CMD_VMEXIT: - if (boot_cpu_has_bug(X86_BUG_ITS_NATIVE_ONLY)) { - its_mitigation = ITS_MITIGATION_VMEXIT_ONLY; - goto out; - } - fallthrough; - case ITS_CMD_ON: + default: + break; + } + + /* + * retbleed_update_mitigation() will try to do stuffing if its=stuff. + * If it can't, such as if spectre_v2!=retpoline, then fall back to + * aligned thunks. + */ + if (its_mitigation == ITS_MITIGATION_RETPOLINE_STUFF && + retbleed_mitigation != RETBLEED_MITIGATION_STUFF) its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS; - if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) - setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); - setup_force_cpu_cap(X86_FEATURE_RETHUNK); - set_return_thunk(its_return_thunk); - break; - case ITS_CMD_RSB_STUFF: - its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF; - setup_force_cpu_cap(X86_FEATURE_RETHUNK); - setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); - set_return_thunk(call_depth_return_thunk); - if (retbleed_mitigation == RETBLEED_MITIGATION_NONE) { - retbleed_mitigation = RETBLEED_MITIGATION_STUFF; - pr_info("Retbleed mitigation updated to stuffing\n"); - } - break; - } -out: + pr_info("%s\n", its_strings[its_mitigation]); } +static void __init its_apply_mitigation(void) +{ + /* its=stuff forces retbleed stuffing and is enabled there. */ + if (its_mitigation != ITS_MITIGATION_ALIGNED_THUNKS) + return; + + if (!boot_cpu_has(X86_FEATURE_RETPOLINE)) + setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS); + + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + set_return_thunk(its_return_thunk); +} + #undef pr_fmt #define pr_fmt(fmt) "Spectre V2 : " fmt From 6a7c3c2606105a41dde81002c0037420bc1ddf00 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Tue, 20 May 2025 22:35:20 -0700 Subject: [PATCH 331/331] x86/bugs: Fix spectre_v2 mitigation default on Intel Commit 480e803dacf8 ("x86/bugs: Restructure spectre_v2 mitigation") inadvertently changed the spectre-v2 mitigation default from eIBRS to IBRS on Intel. While splitting the spectre_v2 mitigation in select/update/apply functions, eIBRS and IBRS selection logic was separated in select and update. This caused IBRS selection to not consider that eIBRS mitigation is already selected, fix it. Fixes: 480e803dacf8 ("x86/bugs: Restructure spectre_v2 mitigation") Signed-off-by: Pawan Gupta Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/20250520-eibrs-fix-v1-1-91bacd35ed09@linux.intel.com --- arch/x86/kernel/cpu/bugs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 3d5796d25f78..7f94e6a5497d 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -2105,7 +2105,8 @@ static void __init spectre_v2_select_mitigation(void) static void __init spectre_v2_update_mitigation(void) { - if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO) { + if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO && + !spectre_v2_in_eibrs_mode(spectre_v2_enabled)) { if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) && boot_cpu_has_bug(X86_BUG_RETBLEED) && retbleed_mitigation != RETBLEED_MITIGATION_NONE &&