all: wasip2 support

Author: dgryski

.gitignore

@@ -1,3 +1,6 @@
+go.work
+go.work.sum
+
 docs/_build
 src/device/avr/*.go
 src/device/avr/*.ld
@@ -17,7 +20,7 @@ src/device/kendryte/*.go
 src/device/kendryte/*.s
 src/device/rp/*.go
 src/device/rp/*.s
-vendor
+./vendor
 llvm-build
 llvm-project
 build/*

.gitmodules

@@ -39,3 +39,9 @@
 	path = src/net
 	url = https://github.com/tinygo-org/net.git
 	branch = dev
+[submodule "lib/wasi-cli"]
+	path = lib/wasi-cli
+	url = https://github.com/WebAssembly/wasi-cli
+[submodule "src/vendor/github.com/ydnar/wasm-tools-go"]
+	path = src/vendor/github.com/ydnar/wasm-tools-go
+	url = https://github.com/ydnar/wasm-tools-go.git

GNUmakefile

@@ -268,6 +268,11 @@ lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a:
 	@if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n  git submodule update --init"; exit 1; fi
 	cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM)
 
+# Generate WASI syscall bindings
+.PHONY: wasi-syscall
+wasi-syscall:
+	wit-bindgen-go generate -o ./src/syscall -p syscall --versioned ./lib/wasi-cli/wit
+
 # Check for Node.js used during WASM tests.
 NODEJS_VERSION := $(word 1,$(subst ., ,$(shell node -v | cut -c 2-)))
 MIN_NODEJS_VERSION=18
@@ -434,11 +439,38 @@ tinygo-test-wasi-fast:
 	$(TINYGO) test -target wasip1 $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
 tinygo-test-wasip1-fast:
 	GOOS=wasip1 GOARCH=wasm $(TINYGO) test $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
-tinygo-bench-wasi:
+
+tinygo-test-wasip2-slow:
+	$(TINYGO) test -target=wasip2 $(TEST_PACKAGES_SLOW)
+tinygo-test-wasip2-fast:
+	$(TINYGO) test -target=wasip2 $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
+
+tinygo-test-wasip2-wip:
+	$(TINYGO) test -target wasip2 -x -v $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
+tinygo-test-wasip2-dev:
+	$(TINYGO) test -target wasip2 -wit-package $$(tinygo env TINYGOROOT)/lib/wasi-cli/wit/ -wit-world wasi:cli/command -x -work encoding/csv
+tinygo-test-wasip2-sum-slow:
+	TINYGO=$(TINYGO) \
+	TARGET=wasip2 \
+	TESTOPTS="-x -work" \
+	PACKAGES="$(TEST_PACKAGES_SLOW)" \
+	gotestsum --raw-command -- ./tools/tgtestjson.sh
+tinygo-test-wasip2-sum-fast:
+	TINYGO=$(TINYGO) \
+	TARGET=wasip2 \
+	TESTOPTS="-x -work" \
+	PACKAGES="$(TEST_PACKAGES_FAST)" \
+	gotestsum --raw-command -- ./tools/tgtestjson.sh
+tinygo-bench-wasip1:
 	$(TINYGO) test -target wasip1 -bench . $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW)
-tinygo-bench-wasi-fast:
+tinygo-bench-wasip1-fast:
 	$(TINYGO) test -target wasip1 -bench . $(TEST_PACKAGES_FAST)
 
+tinygo-bench-wasip2:
+	$(TINYGO) test -target wasip2 -bench . $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW)
+tinygo-bench-wasip2-fast:
+	$(TINYGO) test -target wasip2 -bench . $(TEST_PACKAGES_FAST)
+
 # Test external packages in a large corpus.
 test-corpus:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml

builder/build.go

@@ -832,7 +832,11 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 					"--output", result.Executable,
 				)
 
-				cmd := exec.Command(goenv.Get("WASMOPT"), args...)
+				wasmopt := goenv.Get("WASMOPT")
+				if config.Options.PrintCommands != nil {
+					config.Options.PrintCommands(wasmopt, args...)
+				}
+				cmd := exec.Command(wasmopt, args...)
 				cmd.Stdout = os.Stdout
 				cmd.Stderr = os.Stderr
 
@@ -842,6 +846,80 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 				}
 			}
 
+			// Run wasm-tools for component-model binaries
+			witPackage := strings.ReplaceAll(config.Target.WitPackage, "{root}", goenv.Get("TINYGOROOT"))
+			if config.Options.WitPackage != "" {
+				witPackage = config.Options.WitPackage
+			}
+			witWorld := config.Target.WitWorld
+			if config.Options.WitWorld != "" {
+				witWorld = config.Options.WitWorld
+			}
+			if witPackage != "" && witWorld != "" {
+
+				// wasm-tools component embed -w wasi:cli/command
+				// 		$$(tinygo env TINYGOROOT)/lib/wasi-cli/wit/ main.wasm -o embedded.wasm
+				args := []string{
+					"component",
+					"embed",
+					"-w", witWorld,
+					witPackage,
+					result.Executable,
+					"-o", result.Executable,
+				}
+
+				wasmtools := goenv.Get("WASMTOOLS")
+				if config.Options.PrintCommands != nil {
+					config.Options.PrintCommands(wasmtools, args...)
+				}
+				cmd := exec.Command(wasmtools, args...)
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err := cmd.Run()
+				if err != nil {
+					// Preserve a copy of the failure if -work option is used
+					if config.Options.Work {
+						dest := filepath.Join(
+							filepath.Dir(result.Executable),
+							fmt.Sprintf("%s.embed-fail.wasm", strings.Replace(pkgName, "/", "_", -1)),
+						)
+						copyCmd := exec.Command("cp", "-f", result.Executable, dest)
+						copyCmd.Run()
+					}
+					return fmt.Errorf("wasm-tools failed: %w", err)
+				}
+
+				// wasm-tools component new embedded.wasm -o component.wasm
+				args = []string{
+					"component",
+					"new",
+					result.Executable,
+					"-o", result.Executable,
+				}
+
+				if config.Options.PrintCommands != nil {
+					config.Options.PrintCommands(wasmtools, args...)
+				}
+				cmd = exec.Command(wasmtools, args...)
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err = cmd.Run()
+				if err != nil {
+					// Preserve a copy of the failure if -work option is used
+					if config.Options.Work {
+						dest := filepath.Join(
+							filepath.Dir(result.Executable),
+							fmt.Sprintf("%s.comp-fail.wasm", strings.Replace(pkgName, "/", "_", -1)),
+						)
+						copyCmd := exec.Command("cp", "-f", result.Executable, dest)
+						copyCmd.Run()
+					}
+					return fmt.Errorf("wasm-tools failed: %w", err)
+				}
+			}
+
 			// Print code size if requested.
 			if config.Options.PrintSizes == "short" || config.Options.PrintSizes == "full" {
 				packagePathMap := make(map[string]string, len(lprogram.Packages))

compileopts/options.go

@@ -53,6 +53,8 @@ type Options struct {
 	Monitor         bool
 	BaudRate        int
 	Timeout         time.Duration
+	WitPackage      string // pass through to wasm-tools component embed invocation
+	WitWorld        string // pass through to wasm-tools component embed -w option
 }
 
 // Verify performs a validation on the given options, raising an error if options are not valid.

compileopts/target.go

@@ -62,6 +62,8 @@ type TargetSpec struct {
 	JLinkDevice      string   `json:"jlink-device,omitempty"`
 	CodeModel        string   `json:"code-model,omitempty"`
 	RelocationModel  string   `json:"relocation-model,omitempty"`
+	WitPackage       string   `json:"wit-package,omitempty"`
+	WitWorld         string   `json:"wit-world,omitempty"`
 }
 
 // overrideProperties overrides all properties that are set in child into itself using reflection.

compiler/symbol.go

@@ -346,7 +346,7 @@ func (c *compilerContext) parsePragmas(info *functionInfo, f *ssa.Function) {
 // The list of allowed types is based on this proposal:
 // https://github.com/golang/go/issues/59149
 func (c *compilerContext) checkWasmImport(f *ssa.Function, pragma string) {
-	if c.pkg.Path() == "runtime" || c.pkg.Path() == "syscall/js" {
+	if c.pkg.Path() == "runtime" || c.pkg.Path() == "syscall/js" || c.pkg.Path() == "syscall" {
 		// The runtime is a special case. Allow all kinds of parameters
 		// (importantly, including pointers).
 		return
@@ -375,19 +375,26 @@ func (c *compilerContext) checkWasmImport(f *ssa.Function, pragma string) {
 
 // Check whether the type maps directly to a WebAssembly type, according to:
 // https://github.com/golang/go/issues/59149
+// TODO(ydnar): document why we relaxed this for WASI Preview 2.
 func isValidWasmType(typ types.Type, isReturn bool) bool {
 	switch typ := typ.Underlying().(type) {
 	case *types.Basic:
 		switch typ.Kind() {
-		case types.Int32, types.Uint32, types.Int64, types.Uint64:
+		case types.Bool:
+			return true
+		case types.Int8, types.Uint8, types.Int16, types.Uint16, types.Int32, types.Uint32, types.Int64, types.Uint64:
 			return true
 		case types.Float32, types.Float64:
 			return true
-		case types.UnsafePointer:
-			if !isReturn {
-				return true
-			}
+		case types.Uintptr, types.UnsafePointer:
+			return true
+		case types.String:
+			return true
 		}
+	case *types.Struct:
+		return true
+	case *types.Pointer:
+		return isValidWasmType(typ.Elem(), isReturn)
 	}
 	return false
 }

compiler/testdata/errors.go

@@ -16,14 +16,6 @@ type Uint uint32
 //go:wasmimport modulename validparam
 func validparam(a int32, b uint64, c float64, d unsafe.Pointer, e Uint)
 
-// ERROR: //go:wasmimport modulename invalidparam: unsupported parameter type int
-// ERROR: //go:wasmimport modulename invalidparam: unsupported parameter type string
-// ERROR: //go:wasmimport modulename invalidparam: unsupported parameter type []byte
-// ERROR: //go:wasmimport modulename invalidparam: unsupported parameter type *int32
-//
-//go:wasmimport modulename invalidparam
-func invalidparam(a int, b string, c []byte, d *int32)
-
 //go:wasmimport modulename validreturn
 func validreturn() int32
 
@@ -36,8 +28,3 @@ func manyreturns() (int32, int32)
 //
 //go:wasmimport modulename invalidreturn
 func invalidreturn() int
-
-// ERROR: //go:wasmimport modulename invalidUnsafePointerReturn: unsupported result type unsafe.Pointer
-//
-//go:wasmimport modulename invalidUnsafePointerReturn
-func invalidUnsafePointerReturn() unsafe.Pointer

goenv/goenv.go

@@ -159,6 +159,11 @@ func Get(name string) string {
 		}
 
 		return findWasmOpt()
+	case "WASMTOOLS":
+		if path := os.Getenv("WASMTOOLS"); path != "" {
+			return path
+		}
+		return "wasm-tools"
 	default:
 		return ""
 	}

lib/wasi-cli

@@ -240,6 +240,7 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
 		"internal/fuzz/":        false,
 		"internal/reflectlite/": false,
 		"internal/task/":        false,
+		"internal/wasm/":        false,
 		"machine/":              false,
 		"net/":                  true,
 		"net/http/":             false,
@@ -249,6 +250,8 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
 		"runtime/":              false,
 		"sync/":                 true,
 		"testing/":              true,
+		"vendor/":               true,
+		"vendor/github.com/":    false,
 	}
 
 	if goMinor >= 19 {
@@ -259,6 +262,7 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
 
 	if needsSyscallPackage {
 		paths["syscall/"] = true // include syscall/js
+		paths["syscall/wasi/"] = false
 	}
 	return paths
 }

loader/goroot.go

@@ -306,16 +306,25 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 			// reads any files.
 			//
 			// Ex. run --dir=.. --dir=../.. --dir=../../..
-			dirs := dirsToModuleRoot(result.MainDir, result.ModuleRoot)
+			var dirs []string
+			switch config.GOOS() {
+			case "wasip1":
+				dirs = dirsToModuleRootRel(result.MainDir, result.ModuleRoot)
+			case "wasip2", "linux":
+				dirs = dirsToModuleRootAbs(result.MainDir, result.ModuleRoot)
+			default:
+				return fmt.Errorf("unknown GOOS target: %v", config.GOOS())
+			}
+
 			args := []string{"run"}
-			for _, d := range dirs[1:] {
+			for _, d := range dirs {
 				args = append(args, "--dir="+d)
 			}
 
-			// The below re-organizes the arguments so that the current
-			// directory is added last.
+			args = append(args, "--env=PWD="+cmd.Dir)
+
 			args = append(args, cmd.Args[1:]...)
-			cmd.Args = append(cmd.Args[:1:1], args...)
+			cmd.Args = args
 		}
 
 		// Run the test.
@@ -356,7 +365,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 	return passed, err
 }
 
-func dirsToModuleRoot(maindir, modroot string) []string {
+func dirsToModuleRootRel(maindir, modroot string) []string {
 	var dirs = []string{"."}
 	last := ".."
 	// strip off path elements until we hit the module root
@@ -369,6 +378,19 @@ func dirsToModuleRoot(maindir, modroot string) []string {
 	return dirs
 }
 
+func dirsToModuleRootAbs(maindir, modroot string) []string {
+	var dirs = []string{maindir}
+	last := filepath.Join(maindir, "..")
+	// strip off path elements until we hit the module root
+	// adding `..`, `../..`, `../../..` until we're done
+	for maindir != modroot {
+		dirs = append(dirs, last)
+		last = filepath.Join(last, "..")
+		maindir = filepath.Dir(maindir)
+	}
+	return dirs
+}
+
 // Flash builds and flashes the built binary to the given serial port.
 func Flash(pkgName, port string, options *compileopts.Options) error {
 	config, err := builder.NewConfig(options)
@@ -820,7 +842,6 @@ func buildAndRun(pkgName string, config *compileopts.Config, stdout io.Writer, c
 	} else if config.EmulatorName() == "wasmtime" {
 		// Wasmtime needs some special flags to pass environment variables
 		// and allow reading from the current directory.
-		emuArgs = append(emuArgs, "--dir=.")
 		for _, v := range environmentVars {
 			emuArgs = append(emuArgs, "--env", v)
 		}
@@ -1464,6 +1485,12 @@ func main() {
 		flag.StringVar(&outpath, "o", "", "output filename")
 	}
 
+	var witPackage, witWorld string
+	if command == "help" || command == "build" || command == "test" {
+		flag.StringVar(&witPackage, "wit-package", "", "wit package for wasm component embedding")
+		flag.StringVar(&witWorld, "wit-world", "", "wit world for wasm component embedding")
+	}
+
 	var testConfig compileopts.TestConfig
 	if command == "help" || command == "test" {
 		flag.BoolVar(&testConfig.CompileOnly, "c", false, "compile the test binary but do not run it")
@@ -1543,6 +1570,8 @@ func main() {
 		Monitor:         *monitor,
 		BaudRate:        *baudrate,
 		Timeout:         *timeout,
+		WitPackage:      witPackage,
+		WitWorld:        witWorld,
 	}
 	if *printCommands {
 		options.PrintCommands = printCommand

main.go

@@ -1,4 +1,4 @@
-//go:build darwin || tinygo.wasm
+//go:build darwin || wasip1 || wasip2
 
 // This implementation of crypto/rand uses the arc4random_buf function
 // (available on both MacOS and WASI) to generate random numbers.

src/crypto/rand/rand_arc4random.go

@@ -1,4 +1,4 @@
-//go:build linux && !baremetal && !wasip1
+//go:build linux && !baremetal && !wasip1 && !wasip2
 
 // This implementation of crypto/rand uses the /dev/urandom pseudo-file to
 // generate random numbers.