runtime,cmd/ld: on darwin, create theads using libc

Replace thread creation with calls to the pthread
library in libc.

Update #17490

Change-Id: I1e19965c45255deb849b059231252fc6a7861d6c
Reviewed-on: https://go-review.googlesource.com/108679
Run-TryBot: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/cmd/link/internal/amd64/asm.go

@@ -243,7 +243,12 @@ func adddynrel(ctxt *ld.Link, s *sym.Symbol, r *sym.Reloc) bool {
 			// nothing to do, the relocation will be laid out in reloc
 			return true
 		}
-		// for both ELF and Mach-O
+		if ctxt.LinkMode == ld.LinkExternal {
+			// External linker will do this relocation.
+			return true
+		}
+		// Internal linking, for both ELF and Mach-O.
+		// Build a PLT entry and change the relocation target to that entry.
 		addpltsym(ctxt, targ)
 		r.Sym = ctxt.Syms.Lookup(".plt", 0)
 		r.Add = int64(targ.Plt)
@@ -432,7 +437,7 @@ func machoreloc1(arch *sys.Arch, out *ld.OutBuf, s *sym.Symbol, r *sym.Reloc, se
 
 	rs := r.Xsym
 
-	if rs.Type == sym.SHOSTOBJ || r.Type == objabi.R_PCREL || r.Type == objabi.R_GOTPCREL {
+	if rs.Type == sym.SHOSTOBJ || r.Type == objabi.R_PCREL || r.Type == objabi.R_GOTPCREL || r.Type == objabi.R_CALL {
 		if rs.Dynid < 0 {
 			ld.Errorf(s, "reloc %d (%s) to non-macho symbol %s type=%d (%s)", r.Type, sym.RelocName(arch, r.Type), rs.Name, rs.Type, rs.Type)
 			return false

src/cmd/link/internal/ld/data.go

@@ -159,8 +159,8 @@ func relocsym(ctxt *Link, s *sym.Symbol) {
 		}
 
 		// We need to be able to reference dynimport symbols when linking against
-		// shared libraries, and Solaris needs it always
-		if ctxt.HeadType != objabi.Hsolaris && r.Sym != nil && r.Sym.Type == sym.SDYNIMPORT && !ctxt.DynlinkingGo() && !r.Sym.Attr.SubSymbol() {
+		// shared libraries, and Solaris and Darwin need it always
+		if ctxt.HeadType != objabi.Hsolaris && ctxt.HeadType != objabi.Hdarwin && r.Sym != nil && r.Sym.Type == sym.SDYNIMPORT && !ctxt.DynlinkingGo() && !r.Sym.Attr.SubSymbol() {
 			if !(ctxt.Arch.Family == sys.PPC64 && ctxt.LinkMode == LinkExternal && r.Sym.Name == ".TOC.") {
 				Errorf(s, "unhandled relocation for %s (type %d (%s) rtype %d (%s))", r.Sym.Name, r.Sym.Type, r.Sym.Type, r.Type, sym.RelocName(ctxt.Arch, r.Type))
 			}
@@ -402,10 +402,22 @@ func relocsym(ctxt *Link, s *sym.Symbol) {
 					}
 				} else if ctxt.HeadType == objabi.Hdarwin {
 					if r.Type == objabi.R_CALL {
-						if rs.Type != sym.SHOSTOBJ {
-							o += int64(uint64(Symaddr(rs)) - rs.Sect.Vaddr)
+						if ctxt.LinkMode == LinkExternal && rs.Type == sym.SDYNIMPORT {
+							switch ctxt.Arch.Family {
+							case sys.AMD64:
+								// AMD64 dynamic relocations are relative to the end of the relocation.
+								o += int64(r.Siz)
+							case sys.I386:
+								// I386 dynamic relocations are relative to the start of the section.
+								o -= int64(r.Off)                         // offset in symbol
+								o -= int64(s.Value - int64(s.Sect.Vaddr)) // offset of symbol in section
+							}
+						} else {
+							if rs.Type != sym.SHOSTOBJ {
+								o += int64(uint64(Symaddr(rs)) - rs.Sect.Vaddr)
+							}
+							o -= int64(r.Off) // relative to section offset, not symbol
 						}
-						o -= int64(r.Off) // relative to section offset, not symbol
 					} else if ctxt.Arch.Family == sys.ARM {
 						// see ../arm/asm.go:/machoreloc1
 						o += Symaddr(rs) - s.Value - int64(r.Off)

src/cmd/link/internal/ld/lib.go

@@ -387,7 +387,7 @@ func (ctxt *Link) loadlib() {
 		toc.Type = sym.SDYNIMPORT
 	}
 
-	if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil {
+	if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) {
 		// This indicates a user requested -linkmode=external.
 		// The startup code uses an import of runtime/cgo to decide
 		// whether to initialize the TLS.  So give it one. This could
@@ -1824,10 +1824,11 @@ func stkcheck(ctxt *Link, up *chain, depth int) int {
 		// should never be called directly.
 		// onlyctxt.Diagnose the direct caller.
 		// TODO(mwhudson): actually think about this.
+		// TODO(khr): disabled for now. Calls to external functions can only happen on the g0 stack.
+		// See the trampolines in src/runtime/sys_darwin_$ARCH.go.
 		if depth == 1 && s.Type != sym.SXREF && !ctxt.DynlinkingGo() &&
 			ctxt.BuildMode != BuildModeCArchive && ctxt.BuildMode != BuildModePIE && ctxt.BuildMode != BuildModeCShared && ctxt.BuildMode != BuildModePlugin {
-
-			Errorf(s, "call to external function")
+			//Errorf(s, "call to external function")
 		}
 		return -1
 	}

src/cmd/link/internal/x86/asm.go

@@ -297,6 +297,10 @@ func adddynrel(ctxt *ld.Link, s *sym.Symbol, r *sym.Reloc) bool {
 	switch r.Type {
 	case objabi.R_CALL,
 		objabi.R_PCREL:
+		if ctxt.LinkMode == ld.LinkExternal {
+			// External linker will do this relocation.
+			return true
+		}
 		addpltsym(ctxt, targ)
 		r.Sym = ctxt.Syms.Lookup(".plt", 0)
 		r.Add = int64(targ.Plt)
@@ -409,7 +413,7 @@ func machoreloc1(arch *sys.Arch, out *ld.OutBuf, s *sym.Symbol, r *sym.Reloc, se
 
 	rs := r.Xsym
 
-	if rs.Type == sym.SHOSTOBJ {
+	if rs.Type == sym.SHOSTOBJ || r.Type == objabi.R_CALL {
 		if rs.Dynid < 0 {
 			ld.Errorf(s, "reloc %d (%s) to non-macho symbol %s type=%d (%s)", r.Type, sym.RelocName(arch, r.Type), rs.Name, rs.Type, rs.Type)
 			return false

src/cmd/vet/all/whitelist/darwin_386.txt

@@ -3,7 +3,6 @@
 // Ok
 
 runtime/sys_darwin_386.s: [386] now: function now missing Go declaration
-runtime/sys_darwin_386.s: [386] bsdthread_start: function bsdthread_start missing Go declaration
 runtime/sys_darwin_386.s: [386] sysenter: function sysenter missing Go declaration
 runtime/sys_darwin_386.s: [386] setldt: function setldt missing Go declaration
 runtime/sys_darwin_386.s: [386] cannot check cross-package assembly function: now is in package time

src/cmd/vet/all/whitelist/darwin_amd64.txt

 // darwin/amd64-specific vet whitelist. See readme.txt for details.
 
-runtime/sys_darwin_amd64.s: [amd64] bsdthread_start: function bsdthread_start missing Go declaration
 runtime/sys_darwin_amd64.s: [amd64] settls: function settls missing Go declaration
 runtime/sys_darwin_amd64.s: [amd64] cannot check cross-package assembly function: now is in package time

src/runtime/defs_darwin.go

@@ -23,6 +23,7 @@ package runtime
 #include <signal.h>
 #include <sys/event.h>
 #include <sys/mman.h>
+#include <pthread.h>
 */
 import "C"
 
@@ -142,6 +143,8 @@ const (
 	EV_EOF       = C.EV_EOF
 	EVFILT_READ  = C.EVFILT_READ
 	EVFILT_WRITE = C.EVFILT_WRITE
+
+	PTHREAD_CREATE_DETACHED = C.PTHREAD_CREATE_DETACHED
 )
 
 type MachBody C.mach_msg_body_t
@@ -178,3 +181,6 @@ type Mcontext32 C.struct_mcontext32
 type Ucontext C.struct_ucontext
 
 type Kevent C.struct_kevent
+
+type Pthread C.pthread_t
+type PthreadAttr C.pthread_attr_t

src/runtime/defs_darwin_386.go

@@ -121,6 +121,8 @@ const (
 	_EV_EOF       = 0x8000
 	_EVFILT_READ  = -0x1
 	_EVFILT_WRITE = -0x2
+
+	_PTHREAD_CREATE_DETACHED = 0x2
 )
 
 type machbody struct {
@@ -387,3 +389,9 @@ type keventt struct {
 	data   int32
 	udata  *byte
 }
+
+type pthread uintptr
+type pthreadattr struct {
+	X__sig    int32
+	X__opaque [36]int8
+}

src/runtime/defs_darwin_amd64.go

@@ -121,6 +121,8 @@ const (
 	_EV_EOF       = 0x8000
 	_EVFILT_READ  = -0x1
 	_EVFILT_WRITE = -0x2
+
+	_PTHREAD_CREATE_DETACHED = 0x2
 )
 
 type machbody struct {
@@ -390,3 +392,9 @@ type keventt struct {
 	data   int64
 	udata  *byte
 }
+
+type pthread uintptr
+type pthreadattr struct {
+	X__sig    int64
+	X__opaque [56]int8
+}

src/runtime/defs_darwin_arm.go

@@ -123,6 +123,8 @@ const (
 	_EV_EOF       = 0x8000
 	_EVFILT_READ  = -0x1
 	_EVFILT_WRITE = -0x2
+
+	_PTHREAD_CREATE_DETACHED = 0x2
 )
 
 type machbody struct {
@@ -250,3 +252,9 @@ type keventt struct {
 	data   int32
 	udata  *byte
 }
+
+type pthread uintptr
+type pthreadattr struct {
+	X__sig    int32
+	X__opaque [36]int8
+}

src/runtime/defs_darwin_arm64.go

@@ -121,6 +121,8 @@ const (
 	_EV_EOF       = 0x8000
 	_EVFILT_READ  = -0x1
 	_EVFILT_WRITE = -0x2
+
+	_PTHREAD_CREATE_DETACHED = 0x2
 )
 
 type machbody struct {
@@ -253,3 +255,9 @@ type keventt struct {
 	data   int64
 	udata  *byte
 }
+
+type pthread uintptr
+type pthreadattr struct {
+	X__sig    int64
+	X__opaque [56]int8
+}

src/runtime/os3_solaris.go

@@ -140,7 +140,7 @@ func tstart_sysvicall(newm *m) uint32
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, _ unsafe.Pointer) {
+func newosproc(mp *m) {
 	var (
 		attr pthreadattr
 		oset sigset
@@ -152,9 +152,11 @@ func newosproc(mp *m, _ unsafe.Pointer) {
 	if pthread_attr_init(&attr) != 0 {
 		throw("pthread_attr_init")
 	}
+	// Allocate a new 2MB stack.
 	if pthread_attr_setstack(&attr, 0, 0x200000) != 0 {
 		throw("pthread_attr_setstack")
 	}
+	// Read back the allocated stack.
 	if pthread_attr_getstack(&attr, unsafe.Pointer(&mp.g0.stack.hi), &size) != 0 {
 		throw("pthread_attr_getstack")
 	}

src/runtime/os_darwin.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build 386 amd64
+
 package runtime
 
 import "unsafe"
@@ -13,9 +15,6 @@ type mOS struct {
 
 var darwinVersion int
 
-func bsdthread_create(stk, arg unsafe.Pointer, fn uintptr) int32
-func bsdthread_register() int32
-
 //go:noescape
 func mach_msg_trap(h unsafe.Pointer, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32
 
@@ -48,7 +47,7 @@ func semacreate(mp *m) {
 
 // BSD interface for threading.
 func osinit() {
-	// bsdthread_register delayed until end of goenvs so that we
+	// pthread_create delayed until end of goenvs so that we
 	// can look at the environment first.
 
 	ncpu = getncpu()
@@ -117,43 +116,56 @@ func getRandomData(r []byte) {
 
 func goenvs() {
 	goenvs_unix()
-
-	// Register our thread-creation callback (see sys_darwin_{amd64,386}.s)
-	// but only if we're not using cgo. If we are using cgo we need
-	// to let the C pthread library install its own thread-creation callback.
-	if !iscgo {
-		if bsdthread_register() != 0 {
-			if gogetenv("DYLD_INSERT_LIBRARIES") != "" {
-				throw("runtime: bsdthread_register error (unset DYLD_INSERT_LIBRARIES)")
-			}
-			throw("runtime: bsdthread_register error")
-		}
-	}
 }
 
 // May run with m.p==nil, so write barriers are not allowed.
-//go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+//go:nowritebarrierrec
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	if false {
 		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n")
 	}
 
+	// Initialize an attribute object.
+	var attr pthreadattr
+	var err int32
+	err = pthread_attr_init(&attr)
+	if err != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Set the stack we want to use.
+	if pthread_attr_setstack(&attr, unsafe.Pointer(mp.g0.stack.lo), mp.g0.stack.hi-mp.g0.stack.lo) != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Tell the pthread library we won't join with this thread.
+	if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Finally, create the thread. It starts at mstart_stub, which does some low-level
+	// setup and then calls mstart.
 	var oset sigset
 	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
-	errno := bsdthread_create(stk, unsafe.Pointer(mp), funcPC(mstart))
+	_, err = pthread_create(&attr, funcPC(mstart_stub), unsafe.Pointer(mp))
 	sigprocmask(_SIG_SETMASK, &oset, nil)
-
-	if errno < 0 {
-		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -errno, ")\n")
-		throw("runtime.newosproc")
+	if err != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
 	}
 }
 
+// glue code to call mstart from pthread_create.
+func mstart_stub()
+
 // newosproc0 is a version of newosproc that can be called before the runtime
 // is initialized.
 //
-// As Go uses bsdthread_register when running without cgo, this function is
-// not safe to use after initialization as it does not pass an M as fnarg.
+// This function is not safe to use after initialization as it does not pass an M as fnarg.
 //
 //go:nosplit
 func newosproc0(stacksize uintptr, fn uintptr) {
@@ -162,14 +174,36 @@ func newosproc0(stacksize uintptr, fn uintptr) {
 		write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack)))
 		exit(1)
 	}
-	stk := unsafe.Pointer(uintptr(stack) + stacksize)
 
+	// Initialize an attribute object.
+	var attr pthreadattr
+	var err int32
+	err = pthread_attr_init_trampoline(&attr)
+	if err != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Set the stack we want to use.
+	if pthread_attr_setstack_trampoline(&attr, stack, stacksize) != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Tell the pthread library we won't join with this thread.
+	if pthread_attr_setdetachstate_trampoline(&attr, _PTHREAD_CREATE_DETACHED) != 0 {
+		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
+		exit(1)
+	}
+
+	// Finally, create the thread. It starts at mstart_stub, which does some low-level
+	// setup and then calls mstart.
 	var oset sigset
 	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
-	errno := bsdthread_create(stk, nil, fn)
+	var t pthread
+	err = pthread_create_trampoline(&t, &attr, fn, nil)
 	sigprocmask(_SIG_SETMASK, &oset, nil)
-
-	if errno < 0 {
+	if err != 0 {
 		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
 		exit(1)
 	}

src/runtime/os_dragonfly.go

@@ -129,7 +129,8 @@ func lwp_start(uintptr)
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	if false {
 		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " lwp_start=", funcPC(lwp_start), " id=", mp.id, " ostk=", &mp, "\n")
 	}

src/runtime/os_freebsd.go

@@ -177,7 +177,8 @@ func thr_start()
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	if false {
 		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " thr_start=", funcPC(thr_start), " id=", mp.id, " ostk=", &mp, "\n")
 	}

src/runtime/os_linux.go

@@ -142,7 +142,8 @@ func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	/*
 	 * note: strace gets confused if we use CLONE_PTRACE here.
 	 */

src/runtime/os_nacl.go

@@ -158,7 +158,8 @@ func mstart_nacl()
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	mp.tls[0] = uintptr(unsafe.Pointer(mp.g0))
 	mp.tls[1] = uintptr(unsafe.Pointer(mp))
 	ret := nacl_thread_create(funcPC(mstart_nacl), stk, unsafe.Pointer(&mp.tls[2]), nil)

src/runtime/os_netbsd.go

@@ -174,7 +174,8 @@ func semawakeup(mp *m) {
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	if false {
 		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n")
 	}

src/runtime/os_openbsd.go

@@ -159,7 +159,8 @@ func semawakeup(mp *m) {
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
+	stk := unsafe.Pointer(mp.g0.stack.hi)
 	if false {
 		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n")
 	}

src/runtime/os_plan9.go

@@ -408,7 +408,7 @@ func exit(e int32) {
 
 // May run with m.p==nil, so write barriers are not allowed.
 //go:nowritebarrier
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
 	if false {
 		print("newosproc mp=", mp, " ostk=", &mp, "\n")
 	}

src/runtime/os_windows.go

@@ -618,7 +618,7 @@ func semacreate(mp *m) {
 // operate without stack guards.
 //go:nowritebarrierrec
 //go:nosplit
-func newosproc(mp *m, stk unsafe.Pointer) {
+func newosproc(mp *m) {
 	const _STACK_SIZE_PARAM_IS_A_RESERVATION = 0x00010000
 	// stackSize must match SizeOfStackReserve in cmd/link/internal/ld/pe.go.
 	const stackSize = 0x00200000*_64bit + 0x00100000*(1-_64bit)
@@ -649,7 +649,10 @@ func newosproc(mp *m, stk unsafe.Pointer) {
 //go:nowritebarrierrec
 //go:nosplit
 func newosproc0(mp *m, stk unsafe.Pointer) {
-	newosproc(mp, stk)
+	// TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s)
+	// are stacksize and function, not *m and stack.
+	// Check os_linux.go for an implemention that might actually work.
+	throw("bad newosproc0")
 }
 
 func exitThread(wait *uint32) {

src/runtime/proc.go

@@ -1878,7 +1878,7 @@ func newm1(mp *m) {
 		return
 	}
 	execLock.rlock() // Prevent process clone.
-	newosproc(mp, unsafe.Pointer(mp.g0.stack.hi))
+	newosproc(mp)
 	execLock.runlock()
 }
 
@@ -3746,7 +3746,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
 		// Normal traceback is impossible or has failed.
 		// See if it falls into several common cases.
 		n = 0
-		if (GOOS == "windows" || GOOS == "solaris") && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
+		if (GOOS == "windows" || GOOS == "solaris" || GOOS == "darwin") && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
 			// Libcall, i.e. runtime syscall on windows.
 			// Collect Go stack that leads to the call.
 			n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0)

src/runtime/sys_darwin.go

+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import "unsafe"
+
+// The *_trampoline functions convert from the Go calling convention to the C calling convention
+// and then call the underlying libc function.  They are defined in sys_darwin_$ARCH.s.
+
+//go:nowritebarrier
+func pthread_attr_init(attr *pthreadattr) (errno int32) {
+	systemstack(func() {
+		errno = pthread_attr_init_trampoline(attr)
+	})
+	return
+}
+
+//go:noescape
+func pthread_attr_init_trampoline(attr *pthreadattr) int32
+
+//go:nowritebarrier
+func pthread_attr_setstack(attr *pthreadattr, addr unsafe.Pointer, size uintptr) (errno int32) {
+	systemstack(func() {
+		errno = pthread_attr_setstack_trampoline(attr, addr, size)
+	})
+	return
+}
+
+//go:noescape
+func pthread_attr_setstack_trampoline(attr *pthreadattr, addr unsafe.Pointer, size uintptr) int32
+
+//go:nowritebarrier
+func pthread_attr_setdetachstate(attr *pthreadattr, state int) (errno int32) {
+	systemstack(func() {
+		errno = pthread_attr_setdetachstate_trampoline(attr, state)
+	})
+	return
+}
+
+//go:noescape
+func pthread_attr_setdetachstate_trampoline(attr *pthreadattr, state int) int32
+
+//go:nowritebarrier
+func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) (t pthread, errno int32) {
+	systemstack(func() {
+		errno = pthread_create_trampoline(&t, attr, start, arg)
+	})
+	return
+}
+
+//go:noescape
+func pthread_create_trampoline(t *pthread, attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32
+
+// Tell the linker that the libc_* functions are to be found
+// in a system library, with the libc_ prefix missing.
+
+//go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "/usr/lib/libSystem.B.dylib"
+//go:cgo_import_dynamic libc_pthread_attr_setstack pthread_attr_setstack "/usr/lib/libSystem.B.dylib"
+//go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "/usr/lib/libSystem.B.dylib"
+//go:cgo_import_dynamic libc_pthread_create pthread_create "/usr/lib/libSystem.B.dylib"
+
+// Magic incantation to get libSystem actually dynamically linked.
+// TODO: Why does the code require this?  See cmd/compile/internal/ld/go.go:210
+//go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib"

src/runtime/sys_darwin_386.s

@@ -370,86 +370,6 @@ TEXT runtime·usleep(SB),NOSPLIT,$32
 	INT	$0x80
 	RET
 
-// func bsdthread_create(stk, arg unsafe.Pointer, fn uintptr) int32
-// System call args are: func arg stack pthread flags.
-TEXT runtime·bsdthread_create(SB),NOSPLIT,$32
-	MOVL	$360, AX
-	// 0(SP) is where the caller PC would be; kernel skips it
-	MOVL	fn+8(FP), BX
-	MOVL	BX, 4(SP)	// func
-	MOVL	arg+4(FP), BX
-	MOVL	BX, 8(SP)	// arg
-	MOVL	stk+0(FP), BX
-	MOVL	BX, 12(SP)	// stack
-	MOVL    $0, 16(SP)      // pthread
-	MOVL	$0x1000000, 20(SP)	// flags = PTHREAD_START_CUSTOM
-	INT	$0x80
-	JAE	4(PC)
-	NEGL	AX
-	MOVL	AX, ret+12(FP)
-	RET
-	MOVL	$0, AX
-	MOVL	AX, ret+12(FP)
-	RET
-
-// The thread that bsdthread_create creates starts executing here,
-// because we registered this function using bsdthread_register
-// at startup.
-//	AX = "pthread" (= 0x0)
-//	BX = mach thread port
-//	CX = "func" (= fn)
-//	DX = "arg" (= m)
-//	DI = stack top
-//	SI = flags (= 0x1000000)
-//	SP = stack - C_32_STK_ALIGN
-TEXT runtime·bsdthread_start(SB),NOSPLIT,$0
-	// set up ldt 7+id to point at m->tls.
-	LEAL	m_tls(DX), BP
-	MOVL	m_id(DX), DI
-	ADDL	$7, DI	// m0 is LDT#7. count up.
-	// setldt(tls#, &tls, sizeof tls)
-	PUSHAL	// save registers
-	PUSHL	$32	// sizeof tls
-	PUSHL	BP	// &tls
-	PUSHL	DI	// tls #
-	CALL	runtime·setldt(SB)
-	POPL	AX
-	POPL	AX
-	POPL	AX
-	POPAL
-
-	// Now segment is established. Initialize m, g.
-	get_tls(BP)
-	MOVL    m_g0(DX), AX
-	MOVL	AX, g(BP)
-	MOVL	DX, g_m(AX)
-	MOVL	BX, m_procid(DX)	// m->procid = thread port (for debuggers)
-	CALL	runtime·stackcheck(SB)		// smashes AX
-	CALL	CX	// fn()
-	CALL	exit1<>(SB)
-	RET
-
-// func bsdthread_register() int32
-// registers callbacks for threadstart (see bsdthread_create above
-// and wqthread and pthsize (not used).  returns 0 on success.
-TEXT runtime·bsdthread_register(SB),NOSPLIT,$40
-	MOVL	$366, AX
-	// 0(SP) is where kernel expects caller PC; ignored
-	MOVL	$runtime·bsdthread_start(SB), 4(SP)	// threadstart
-	MOVL	$0, 8(SP)	// wqthread, not used by us
-	MOVL	$0, 12(SP)	// pthsize, not used by us
-	MOVL	$0, 16(SP)	// dummy_value [sic]
-	MOVL	$0, 20(SP)	// targetconc_ptr
-	MOVL	$0, 24(SP)	// dispatchqueue_offset
-	INT	$0x80
-	JAE	4(PC)
-	NEGL	AX
-	MOVL	AX, ret+0(FP)
-	RET
-	MOVL	$0, AX
-	MOVL	AX, ret+0(FP)
-	RET
-
 // Invoke Mach system call.
 // Assumes system call number in AX,
 // caller PC on stack, caller's caller PC next,
@@ -592,3 +512,121 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$32
 	JAE	2(PC)
 	NEGL	AX
 	RET
+
+// mstart_stub is the first function executed on a new thread started by pthread_create.
+// It just does some low-level setup and then calls mstart.
+// Note: called with the C calling convention.
+TEXT runtime·mstart_stub(SB),NOSPLIT,$0
+	// The value at SP+4 points to the m.
+	// We are already on m's g0 stack.
+
+	MOVL	SP, AX       // hide argument read from vet (vet thinks this function is using the Go calling convention)
+	MOVL	4(AX), DI    // m
+	MOVL	m_g0(DI), DX // g
+
+	// Initialize TLS entry.
+	// See cmd/link/internal/ld/sym.go:computeTLSOffset.
+	MOVL	DX, 0x18(GS)
+
+	// Someday the convention will be D is always cleared.
+	CLD
+
+	CALL	runtime·stackcheck(SB) // just in case
+	CALL	runtime·mstart(SB)
+
+	// mstart shouldn't ever return, and if it does, we shouldn't ever join to this thread
+	// to get its return status. But tell pthread everything is ok, just in case.
+	XORL	AX, AX
+	RET
+
+TEXT runtime·pthread_attr_init_trampoline(SB),NOSPLIT,$0-8
+	// move args into registers
+	MOVL	attr+0(FP), AX
+
+	// save SP, BP
+	PUSHL	BP
+	MOVL	SP, BP
+
+	// allocate space for args
+	SUBL	$4, SP
+
+	// align stack to 16 bytes
+	ANDL	$~15, SP
+
+	// call libc function
+	MOVL	AX, 0(SP)
+	CALL	libc_pthread_attr_init(SB)
+
+	// restore BP, SP
+	MOVL	BP, SP
+	POPL	BP
+
+	// save result.
+	MOVL	AX, ret+4(FP)
+	RET
+
+TEXT runtime·pthread_attr_setstack_trampoline(SB),NOSPLIT,$0-16
+	MOVL	attr+0(FP), AX
+	MOVL	addr+4(FP), CX
+	MOVL	size+8(FP), DX
+
+	PUSHL	BP
+	MOVL	SP, BP
+
+	SUBL	$12, SP
+	ANDL	$~15, SP
+
+	MOVL	AX, 0(SP)
+	MOVL	CX, 4(SP)
+	MOVL	DX, 8(SP)
+	CALL	libc_pthread_attr_setstack(SB)
+
+	MOVL	BP, SP
+	POPL	BP
+
+	MOVL	AX, ret+12(FP)
+	RET
+
+TEXT runtime·pthread_attr_setdetachstate_trampoline(SB),NOSPLIT,$0-12
+	MOVL	attr+0(FP), AX
+	MOVL	state+4(FP), CX
+
+	PUSHL	BP
+	MOVL	SP, BP
+
+	SUBL	$8, SP
+	ANDL	$~15, SP
+
+	MOVL	AX, 0(SP)
+	MOVL	CX, 4(SP)
+	CALL	libc_pthread_attr_setdetachstate(SB)
+
+	MOVL	BP, SP
+	POPL	BP
+
+	MOVL	AX, ret+8(FP)
+	RET
+
+TEXT runtime·pthread_create_trampoline(SB),NOSPLIT,$0-20
+	MOVL	t+0(FP), AX
+	MOVL	attr+4(FP), CX
+	MOVL	start+8(FP), DX
+	MOVL	arg+12(FP), BX
+
+	PUSHL	BP
+	MOVL	SP, BP
+
+	SUBL	$16, SP
+	ANDL	$~15, SP
+
+	MOVL	AX, 0(SP)
+	MOVL	CX, 4(SP)
+	MOVL	DX, 8(SP)
+	MOVL	BX, 12(SP)
+	CALL	libc_pthread_create(SB)
+
+	MOVL	BP, SP
+	POPL	BP
+
+	MOVL	AX, ret+16(FP)
+	RET

src/runtime/sys_darwin_amd64.s

@@ -417,82 +417,6 @@ TEXT runtime·usleep(SB),NOSPLIT,$16
 	SYSCALL
 	RET
 
-// func bsdthread_create(stk, arg unsafe.Pointer, fn uintptr) int32
-TEXT runtime·bsdthread_create(SB),NOSPLIT,$0
-	// Set up arguments to bsdthread_create system call.
-	// The ones in quotes pass through to the thread callback
-	// uninterpreted, so we can put whatever we want there.
-	MOVQ	fn+16(FP),   DI
-	MOVQ	arg+8(FP),  SI
-	MOVQ	stk+0(FP),   DX
-	MOVQ	$0x01000000, R8  // flags = PTHREAD_START_CUSTOM
-	MOVQ	$0,          R9  // paranoia
-	MOVQ	$0,          R10 // paranoia, "pthread"
-	MOVQ	$(0x2000000+360), AX	// bsdthread_create
-	SYSCALL
-	JCC 4(PC)
-	NEGQ	AX
-	MOVL	AX, ret+24(FP)
-	RET
-	MOVL	$0, AX
-	MOVL	AX, ret+24(FP)
-	RET
-
-// The thread that bsdthread_create creates starts executing here,
-// because we registered this function using bsdthread_register
-// at startup.
-//	DI = "pthread"
-//	SI = mach thread port
-//	DX = "func" (= fn)
-//	CX = "arg" (= m)
-//	R8 = stack
-//	R9 = flags (= 0)
-//	SP = stack - C_64_REDZONE_LEN (= stack - 128)
-TEXT runtime·bsdthread_start(SB),NOSPLIT,$0
-	MOVQ	R8, SP		// empirically, SP is very wrong but R8 is right
-
-	PUSHQ	DX
-	PUSHQ	CX
-	PUSHQ	SI
-
-	// set up thread local storage pointing at m->tls.
-	LEAQ	m_tls(CX), DI
-	CALL	runtime·settls(SB)
-
-	POPQ	SI
-	POPQ	CX
-	POPQ	DX
-
-	get_tls(BX)
-	MOVQ	SI, m_procid(CX)	// thread port is m->procid
-	MOVQ	m_g0(CX), AX
-	MOVQ	AX, g(BX)
-	MOVQ	CX, g_m(AX)
-	CALL	runtime·stackcheck(SB)	// smashes AX, CX
-	CALL	DX	// fn
-	CALL	exit1<>(SB)
-	RET
-
-// func bsdthread_register() int32
-// registers callbacks for threadstart (see bsdthread_create above
-// and wqthread and pthsize (not used).  returns 0 on success.
-TEXT runtime·bsdthread_register(SB),NOSPLIT,$0
-	MOVQ	$runtime·bsdthread_start(SB), DI	// threadstart
-	MOVQ	$0, SI	// wqthread, not used by us
-	MOVQ	$0, DX	// pthsize, not used by us
-	MOVQ	$0, R10	// dummy_value [sic]
-	MOVQ	$0, R8	// targetconc_ptr
-	MOVQ	$0, R9	// dispatchqueue_offset
-	MOVQ	$(0x2000000+366), AX	// bsdthread_register
-	SYSCALL
-	JCC 4(PC)
-	NEGQ	AX
-	MOVL	AX, ret+0(FP)
-	RET
-	MOVL	$0, AX
-	MOVL	AX, ret+0(FP)
-	RET
-
 // Mach system calls use 0x1000000 instead of the BSD's 0x2000000.
 
 // func mach_msg_trap(h unsafe.Pointer, op int32, send_size, rcv_size, rcv_name, timeout, notify uint32) int32
@@ -630,3 +554,78 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0
 	MOVL	$(0x2000000+92), AX  // fcntl
 	SYSCALL
 	RET
+
+// mstart_stub is the first function executed on a new thread started by pthread_create.
+// It just does some low-level setup and then calls mstart.
+// Note: called with the C calling convention.
+TEXT runtime·mstart_stub(SB),NOSPLIT,$0
+	// DI points to the m.
+	// We are already on m's g0 stack.
+
+	MOVQ	m_g0(DI), DX // g
+
+	// Initialize TLS entry.
+	// See cmd/link/internal/ld/sym.go:computeTLSOffset.
+	MOVQ	DX, 0x30(GS)
+
+	// Someday the convention will be D is always cleared.
+	CLD
+
+	CALL	runtime·stackcheck(SB) // just in case
+	CALL	runtime·mstart(SB)
+
+	// mstart shouldn't ever return, and if it does, we shouldn't ever join to this thread
+	// to get its return status. But tell pthread everything is ok, just in case.
+	XORL	AX, AX
+	RET
+
+// These trampolines convert from Go calling convention to C calling convention.
+TEXT runtime·pthread_attr_init_trampoline(SB),NOSPLIT,$0-12
+	MOVQ	attr+0(FP), DI
+	PUSHQ	BP     // save BP
+	MOVQ	SP, BP // save SP
+	ANDQ	$~15, SP // align stack to 16 bytes
+	CALL	libc_pthread_attr_init(SB)
+	MOVQ	BP, SP // restore SP
+	POPQ	BP     // restore BP
+	MOVL	AX, ret+8(FP)
+	RET
+
+TEXT runtime·pthread_attr_setstack_trampoline(SB),NOSPLIT,$0-28
+	MOVQ	attr+0(FP), DI
+	MOVQ	addr+8(FP), SI
+	MOVQ	size+16(FP), DX
+	PUSHQ	BP
+	MOVQ	SP, BP
+	ANDQ	$~15, SP
+	CALL	libc_pthread_attr_setstack(SB)
+	MOVQ	BP, SP
+	POPQ	BP
+	MOVL	AX, ret+24(FP)
+	RET
+
+TEXT runtime·pthread_attr_setdetachstate_trampoline(SB),NOSPLIT,$0-20
+	MOVQ	attr+0(FP), DI
+	MOVQ	state+8(FP), SI
+	PUSHQ	BP
+	MOVQ	SP, BP
+	ANDQ	$~15, SP
+	CALL	libc_pthread_attr_setdetachstate(SB)
+	MOVQ	BP, SP
+	POPQ	BP
+	MOVL	AX, ret+16(FP)
+	RET
+
+TEXT runtime·pthread_create_trampoline(SB),NOSPLIT,$0-36
+	MOVQ	t+0(FP), DI
+	MOVQ	attr+8(FP), SI
+	MOVQ	start+16(FP), DX
+	MOVQ	arg+24(FP), CX
+	PUSHQ	BP
+	MOVQ	SP, BP
+	ANDQ	$~15, SP
+	CALL	libc_pthread_create(SB)
+	MOVQ	BP, SP
+	POPQ	BP
+	MOVL	AX, ret+32(FP)
+	RET