-
Notifications
You must be signed in to change notification settings - Fork 251
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add code to generate the SVE and NEON routines for ARM
- Loading branch information
Showing
8 changed files
with
1,641 additions
and
789 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,293 @@ | ||
| // Copyright 2024, Klaus Post/Minio Inc. See LICENSE for details. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "regexp" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| avxtwo2sve "github.com/fwessels/avxTwo2sve" | ||
| sve_as "github.com/fwessels/sve-as" | ||
| ) | ||
|
|
||
| func patchLabel(line string) string { | ||
| return strings.ReplaceAll(line, "AvxTwo", "Sve") | ||
| } | ||
|
|
||
| func extractRoutine(filename, routine string) (lines []string, err error) { | ||
| file, err := os.Open(filename) | ||
| if err != nil { | ||
| return | ||
| } | ||
| defer file.Close() | ||
|
|
||
| // Create a scanner to read the file line by line | ||
| scanner := bufio.NewScanner(file) | ||
|
|
||
| // Iterate over each line | ||
| collect := false | ||
| for scanner.Scan() { | ||
| line := scanner.Text() | ||
| if strings.HasPrefix(line, routine) { | ||
| collect = true | ||
| } | ||
| if collect { | ||
| lines = append(lines, line) | ||
| } | ||
| if collect && strings.HasSuffix(line, "RET") { | ||
| collect = false | ||
| } | ||
| } | ||
|
|
||
| // Check for any errors that occurred during scanning | ||
| err = scanner.Err() | ||
| return | ||
| } | ||
|
|
||
| func addArmInitializations(instructions []string) (processed []string) { | ||
| for _, instr := range instructions { | ||
| processed = append(processed, instr) | ||
| if strings.HasPrefix(instr, "TEXT ·") { | ||
| sve := "ptrue p0.d" | ||
| opcode, err := sve_as.Assemble(sve) | ||
| if err != nil { | ||
| processed = append(processed, fmt.Sprintf(" WORD $0x00000000 // %-44s\n", sve)) | ||
| } else { | ||
| processed = append(processed, fmt.Sprintf(" WORD $0x%08x // %-44s\n", opcode, sve)) | ||
| } | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // Expand #defines | ||
| func expandHashDefines(instructions []string) (processed []string) { | ||
| for _, instr := range instructions { | ||
| if strings.Contains(instr, "XOR3WAY") { | ||
| f := strings.Fields(instr) | ||
| if len(f) >= 3 { | ||
| dst := strings.ReplaceAll(f[len(f)-1], ")", "") | ||
| b := strings.ReplaceAll(f[len(f)-2], ",", "") | ||
| a := strings.ReplaceAll(f[len(f)-3], ",", "") | ||
|
|
||
| processed = append(processed, fmt.Sprintf("VPXOR %s, %s, %s", a, dst, dst)) | ||
| processed = append(processed, fmt.Sprintf("VPXOR %s, %s, %s", b, dst, dst)) | ||
| } else { | ||
| log.Fatalf("Not enough arguments for 'XOR3WAY' macro: %d", len(f)) | ||
| } | ||
| } else if !strings.Contains(instr, "VZEROUPPER") { | ||
| processed = append(processed, instr) | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func convertRoutine(asmBuf *bytes.Buffer, instructions []string) { | ||
|
|
||
| asmF := func(format string, args ...interface{}) { | ||
| (*asmBuf).WriteString(fmt.Sprintf(format, args...)) | ||
| } | ||
|
|
||
| wordOpcode := regexp.MustCompile(`WORD \$0x[0-9a-f]{8}`) | ||
|
|
||
| for _, instr := range instructions { | ||
| instr = strings.TrimSpace(instr) | ||
| if instr == "" { | ||
| asmF("\n") | ||
| } else if strings.HasPrefix(instr, "TEXT ") { // function header | ||
| asmF("%s\n", patchLabel(instr)) | ||
| } else if wordOpcode.MatchString(instr) { // arm code | ||
| asmF(" %s\n", instr) | ||
| } else if strings.HasPrefix(instr, "//") { // comment | ||
| asmF(" %s\n", instr) | ||
| } else if strings.HasSuffix(instr, ":") { // label | ||
| asmF("%s\n", patchLabel(instr)) | ||
| } else { | ||
| sve, plan9, err := avxtwo2sve.AvxTwo2Sve(instr, patchLabel) | ||
| if err != nil { | ||
| panic(err) | ||
| } else if !plan9 { | ||
| opcode, err := sve_as.Assemble(sve) | ||
| if err != nil { | ||
| asmF(" WORD $0x00000000 // %-44s\n", sve) | ||
| } else { | ||
| asmF(" WORD $0x%08x // %-44s\n", opcode, sve) | ||
| } | ||
| } else { | ||
| asmF(" %s\n", sve) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func fromAvx2ToSve() { | ||
| asmOut, goOut := &bytes.Buffer{}, &bytes.Buffer{} | ||
|
|
||
| goOut.WriteString(`// Code generated by command: go generate ` + os.Getenv("GOFILE") + `. DO NOT EDIT.` + "\n\n") | ||
| goOut.WriteString("//go:build !noasm && !appengine && !gccgo && !nopshufb\n\n") | ||
| goOut.WriteString("package reedsolomon\n\n") | ||
|
|
||
| const input = 10 | ||
| const AVX2_CODE = "../galois_gen_amd64.s" | ||
|
|
||
| // Processing 64 bytes variants | ||
| for output := 1; output <= 3; output++ { | ||
| for op := ""; len(op) <= 3; op += "Xor" { | ||
| templName := fmt.Sprintf("mulAvxTwo_%dx%d_64%s", input, output, op) | ||
| funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "AvxTwo", "Sve")) | ||
|
|
||
| // asm first | ||
| lines, err := extractRoutine(AVX2_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName)) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| lines = expandHashDefines(lines) | ||
|
|
||
| convertRoutine(asmOut, lines) | ||
|
|
||
| // add newline after RET | ||
| asmOut.WriteString("\n") | ||
|
|
||
| // golang declaration | ||
| goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n\n", funcDef)) | ||
| } | ||
| } | ||
|
|
||
| // Processing 32 bytes variants | ||
| for output := 4; output <= 10; output++ { | ||
| for op := ""; len(op) <= 3; op += "Xor" { | ||
| templName := fmt.Sprintf("mulAvxTwo_%dx%d%s", input, output, op) | ||
| funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", strings.ReplaceAll(templName, "AvxTwo", "Sve")) | ||
|
|
||
| // asm first | ||
| lines, err := extractRoutine(AVX2_CODE, fmt.Sprintf("TEXT ·%s(SB)", templName)) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| lines = expandHashDefines(lines) | ||
|
|
||
| // add additional initialization for SVE | ||
| // (for predicated loads and stores in | ||
| // case of register shortage) | ||
| lines = addArmInitializations(lines) | ||
|
|
||
| convertRoutine(asmOut, lines) | ||
|
|
||
| // add newline after RET | ||
| asmOut.WriteString("\n") | ||
|
|
||
| // golang declaration | ||
| goOut.WriteString(fmt.Sprintf("//go:noescape\n%s\n\n", funcDef)) | ||
| } | ||
| } | ||
|
|
||
| if err := os.WriteFile("../galois_gen_arm64.s", asmOut.Bytes(), 0644); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| if err := os.WriteFile("../galois_gen_arm64.go", goOut.Bytes(), 0644); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| func insertEarlyExit(lines []string, funcName string, outputs int) (processed []string) { | ||
|
|
||
| const reg = "R16" | ||
| label := funcName + "_store" | ||
|
|
||
| reComment := regexp.MustCompile(fmt.Sprintf(`// Load and process \d* bytes from input (\d*) to %d outputs`, outputs)) | ||
| reLoop := regexp.MustCompile(`^` + strings.ReplaceAll(label, "store", "loop") + `:`) | ||
| reStore := regexp.MustCompile(fmt.Sprintf(`// Store %d outputs`, outputs)) | ||
|
|
||
| for _, line := range lines { | ||
| if matches := reLoop.FindAllStringSubmatch(line, -1); len(matches) == 1 { | ||
| lastline := processed[len(processed)-1] | ||
| processed = processed[:len(processed)-1] | ||
| processed = append(processed, "") | ||
| processed = append(processed, fmt.Sprintf(" // Load number of input shards")) | ||
| processed = append(processed, fmt.Sprintf(" MOVD in_len+32(FP), %s", reg)) | ||
| processed = append(processed, lastline) | ||
| } | ||
|
|
||
| if matches := reComment.FindAllStringSubmatch(line, -1); len(matches) == 1 { | ||
| if inputs, err := strconv.Atoi(matches[0][1]); err != nil { | ||
| panic(err) | ||
| } else { | ||
| if inputs > 0 && inputs < 10 { | ||
| lastline := processed[len(processed)-1] | ||
| processed = processed[:len(processed)-1] | ||
| processed = append(processed, fmt.Sprintf(" // Check for early termination")) | ||
| processed = append(processed, fmt.Sprintf(" CMP $%d, %s", inputs, reg)) | ||
| processed = append(processed, fmt.Sprintf(" BEQ %s", label)) | ||
| processed = append(processed, lastline) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if matches := reStore.FindAllStringSubmatch(line, -1); len(matches) == 1 { | ||
| processed = append(processed, fmt.Sprintf("%s:", label)) | ||
| } | ||
|
|
||
| processed = append(processed, line) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func addEarlyExit(arch string) { | ||
| const filename = "../galois_gen_arm64.s" | ||
| asmOut := &bytes.Buffer{} | ||
|
|
||
| asmOut.WriteString(`// Code generated by command: go generate ` + os.Getenv("GOFILE") + `. DO NOT EDIT.` + "\n\n") | ||
| asmOut.WriteString("//go:build !appengine && !noasm && !nogen && !nopshufb && gc\n\n") | ||
| asmOut.WriteString(`#include "textflag.h"` + "\n\n") | ||
|
|
||
| input := 10 | ||
| for outputs := 1; outputs <= 3; outputs++ { | ||
| for op := ""; len(op) <= 3; op += "Xor" { | ||
| funcName := fmt.Sprintf("mul%s_%dx%d_64%s", arch, input, outputs, op) | ||
| funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", funcName) | ||
|
|
||
| lines, _ := extractRoutine(filename, fmt.Sprintf("TEXT ·%s(SB)", funcName)) | ||
|
|
||
| // prepend output with commented out function definition and comment | ||
| asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef)) | ||
| asmOut.WriteString("// Requires: SVE\n") | ||
|
|
||
| lines = insertEarlyExit(lines, funcName, outputs) | ||
|
|
||
| asmOut.WriteString(strings.Join(lines, "\n")) | ||
| asmOut.WriteString("\n\n") | ||
| } | ||
| } | ||
|
|
||
| for outputs := 4; outputs <= 10; outputs++ { | ||
| for op := ""; len(op) <= 3; op += "Xor" { | ||
| funcName := fmt.Sprintf("mul%s_%dx%d%s", arch, input, outputs, op) | ||
| funcDef := fmt.Sprintf("func %s(matrix []byte, in [][]byte, out [][]byte, start int, n int)", funcName) | ||
|
|
||
| lines, _ := extractRoutine(filename, fmt.Sprintf("TEXT ·%s(SB)", funcName)) | ||
|
|
||
| // prepend output with commented out function definition and comment | ||
| asmOut.WriteString(fmt.Sprintf("// %s\n", funcDef)) | ||
| asmOut.WriteString("// Requires: SVE\n") | ||
|
|
||
| lines = insertEarlyExit(lines, funcName, outputs) | ||
| asmOut.WriteString(strings.Join(lines, "\n")) | ||
| asmOut.WriteString("\n\n") | ||
| } | ||
| } | ||
|
|
||
| if err := os.WriteFile("../galois_gen_arm64.s", asmOut.Bytes(), 0644); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| func genArmSve() { | ||
| fromAvx2ToSve() | ||
| addEarlyExit("Sve") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.