6l, 8l: support for linking ELF and Mach-O .o files

More support for references to dynamic symbols,
including full GOT and PLT for ELF objects.

For Mach-O everything ends up in the GOT: dealing
with the real lazy PLT is too hard for now so we punt.

R=iant, iant2
CC=golang-dev
https://golang.org/cl/3491042

src/cmd/6l/Makefile

@@ -14,6 +14,8 @@ OFILES=\
 	elf.$O\
 	enam.$O\
 	go.$O\
+	ldelf.$O\
+	ldmacho.$O\
 	lib.$O\
 	list.$O\
 	macho.$O\

src/cmd/6l/l.h

@@ -84,7 +84,7 @@ struct	Reloc
 {
 	int32	off;
 	uchar	siz;
-	uchar	type;
+	int32	type;
 	int64	add;
 	Sym*	sym;
 };
@@ -128,10 +128,14 @@ struct	Sym
 	uchar	reachable;
 	uchar	dynexport;
 	uchar	special;
+	int32	dynid;
 	int32	sig;
+	int32	plt;
+	int32	got;
 	Sym*	hash;	// in hash table
 	Sym*	next;	// in text or data list
 	Sym*	sub;	// in SSUB list
+	Sym*	outer;	// container of sub
 	vlong	value;
 	vlong	size;
 	Sym*	gotype;
@@ -174,14 +178,20 @@ enum
 	/* order here is order in output file */
 	STEXT		= 1,
 	SELFDATA,
+	SMACHOPLT,
 	SRODATA,
 	SDATA,
-	SMACHO,
+	SMACHOGOT,
 	SBSS,
 
 	SXREF,
+	SMACHODYNSTR,
+	SMACHODYNSYM,
+	SMACHOINDIRECTPLT,
+	SMACHOINDIRECTGOT,
 	SFILE,
 	SCONST,
+	SDYNIMPORT,
 	SSUB	= 1<<8,
 
 	NHASH		= 10007,

src/cmd/8l/8.out.h

@@ -500,6 +500,8 @@ enum
 	D_CONST2 = D_INDIR+D_INDIR,
 	D_SIZE,	/* 8l internal */
 	D_PCREL,
+	D_GOTOFF,
+	D_GOTPCREL,
 
 	T_TYPE		= 1<<0,
 	T_INDEX		= 1<<1,

src/cmd/8l/Makefile

@@ -14,6 +14,8 @@ OFILES=\
 	elf.$O\
 	enam.$O\
 	go.$O\
+	ldelf.$O\
+	ldmacho.$O\
 	lib.$O\
 	list.$O\
 	macho.$O\

src/cmd/8l/l.h

@@ -84,7 +84,7 @@ struct	Reloc
 {
 	int32	off;
 	uchar	siz;
-	uchar	type;
+	int32	type;
 	int32	add;
 	Sym*	sym;
 };
@@ -130,9 +130,13 @@ struct	Sym
 	int32	value;
 	int32	size;
 	int32	sig;
+	int32	dynid;
+	int32	plt;
+	int32	got;
 	Sym*	hash;	// in hash table
 	Sym*	next;	// in text or data list
 	Sym*	sub;	// in sub list
+	Sym*	outer;	// container of sub
 	Sym*	gotype;
 	char*	file;
 	char*	dynimpname;
@@ -165,16 +169,23 @@ enum
 	/* order here is order in output file */
 	STEXT,
 	SELFDATA,
+	SMACHOPLT,
 	SRODATA,
 	SDATA,
 	SMACHO,	/* Mach-O __nl_symbol_ptr */
+	SMACHOGOT,
 	SWINDOWS,
 	SBSS,
 
 	SXREF,
+	SMACHODYNSTR,
+	SMACHODYNSYM,
+	SMACHOINDIRECTPLT,
+	SMACHOINDIRECTGOT,
 	SFILE,
 	SCONST,
-	
+	SDYNIMPORT,
+
 	SSUB = 1<<8,	/* sub-symbol, linked from parent via ->sub list */
 
 	NHASH		= 10007,
@@ -317,7 +328,7 @@ EXTERN	int	dtype;
 EXTERN	int	tlsoffset;
 EXTERN	Sym*	adrgotype;	// type symbol on last Adr read
 EXTERN	Sym*	fromgotype;	// type symbol on last p->from read
-
+EXTERN	int	elftextsh;
 
 extern	Optab	optab[];
 extern	char*	anames[];

src/cmd/ld/data.c

@@ -35,6 +35,8 @@
 #include	"../ld/elf.h"
 #include	"../ld/pe.h"
 
+void	dynreloc(void);
+
 /*
  * divide-and-conquer list-link
  * sort of Sym* structures.
@@ -147,27 +149,49 @@ relocsym(Sym *s)
 	vlong o;
 	uchar *cast;
 	
+	cursym = s;
 	memset(&p, 0, sizeof p);
 	for(r=s->r; r<s->r+s->nr; r++) {
 		off = r->off;
 		siz = r->siz;
+		if(off < 0 || off+(siz&~Rbig) > s->np) {
+			diag("%s: invalid relocation %d+%d not in [%d,%d)", s->name, off, siz&~Rbig, 0, s->np);
+			continue;
+		}
+		if(r->sym != S && (r->sym->type == 0 || r->sym->type == SXREF)) {
+			diag("%s: not defined", r->sym->name);
+			continue;
+		}
+		if(r->type >= 256)
+			continue;
+
+		if(r->sym != S && (r->sym->type == SDYNIMPORT || r->sym->dynimpname != nil))
+			diag("unhandled relocation for %s (rtype %d)", r->sym->name, r->type);
+
+		if(r->sym != S && !r->sym->reachable)
+			diag("unreachable sym in relocation: %s %s", s->name, r->sym->name);
+
 		switch(r->type) {
 		default:
-			diag("unknown reloc %d", r->type);
+			o = 0;
+			if(archreloc(r, s, &o) < 0)
+				diag("unknown reloc %d", r->type);
+			break;
 		case D_ADDR:
-			o = symaddr(r->sym);
+			o = symaddr(r->sym) + r->add;
 			break;
 		case D_PCREL:
-			o = symaddr(r->sym) - (s->value + r->off + r->siz);
+			o = symaddr(r->sym) + r->add - (s->value + r->off + r->siz);
 			break;
 		case D_SIZE:
-			o = r->sym->size;
+			o = r->sym->size + r->add;
 			break;
 		}
-		o += r->add;
+//print("relocate %s %p %s => %p %p %p %p [%p]\n", s->name, s->value+off, r->sym ? r->sym->name : "<nil>", (void*)symaddr(r->sym), (void*)s->value, (void*)r->off, (void*)r->siz, (void*)o);
 		switch(siz) {
 		default:
-			diag("bad reloc size %#ux", siz);
+			cursym = s;
+			diag("bad reloc size %#ux for %s", siz, r->sym->name);
 		case 4 + Rbig:
 			fl = o;
 			s->p[off] = fl>>24;
@@ -212,6 +236,33 @@ reloc(void)
 		relocsym(s);
 }
 
+void
+dynrelocsym(Sym *s)
+{
+	Reloc *r;
+
+	for(r=s->r; r<s->r+s->nr; r++)
+		if(r->sym->type == SDYNIMPORT || r->type >= 256)
+			adddynrel(s, r);
+}
+
+void
+dynreloc(void)
+{
+	Sym *s;
+	
+	if(debug['v'])
+		Bprint(&bso, "%5.2f reloc\n", cputime());
+	Bflush(&bso);
+
+	for(s=textp; s!=S; s=s->next)
+		dynrelocsym(s);
+	for(s=datap; s!=S; s=s->next)
+		dynrelocsym(s);
+	if(iself)
+		elfdynhash();
+}
+
 void
 symgrow(Sym *s, int32 siz)
 {
@@ -240,7 +291,7 @@ savedata(Sym *s, Prog *p)
 	uchar *cast;
 	vlong o;
 	Reloc *r;
-	
+
 	off = p->from.offset;
 	siz = p->datasize;
 	symgrow(s, off+siz);
@@ -324,11 +375,13 @@ blk(Sym *allsym, int32 addr, int32 size)
 	uchar *p, *ep;
 
 	for(sym = allsym; sym != nil; sym = sym->next)
-		if(sym->value >= addr)
+		if(!(sym->type&SSUB) && sym->value >= addr)
 			break;
 
 	eaddr = addr+size;
 	for(; sym != nil; sym = sym->next) {
+		if(sym->type&SSUB)
+			continue;
 		if(sym->value >= eaddr)
 			break;
 		if(sym->value < addr) {
@@ -411,7 +464,7 @@ codeblk(int32 addr, int32 size)
 			continue;
 		}
 			
-		Bprint(&bso, "%.6llux\t%-20s | %P\n", (vlong)addr, sym->name, p);
+		Bprint(&bso, "%.6llux\t%-20s | %P\n", sym->value, sym->name, p);
 		for(p = p->link; p != P; p = p->link) {
 			if(p->link != P)
 				epc = p->link->pc;
@@ -522,6 +575,7 @@ adduintxx(Sym *s, uint64 v, int wid)
 	r = s->size;
 	s->size += wid;
 	symgrow(s, s->size);
+	assert(r+wid <= s->size);
 	fl = v;
 	cast = (uchar*)&fl;
 	switch(wid) {
@@ -571,7 +625,7 @@ adduint64(Sym *s, uint64 v)
 }
 
 vlong
-addaddr(Sym *s, Sym *t)
+addaddrplus(Sym *s, Sym *t, int32 add)
 {
 	vlong i;
 	Reloc *r;
@@ -587,9 +641,37 @@ addaddr(Sym *s, Sym *t)
 	r->off = i;
 	r->siz = PtrSize;
 	r->type = D_ADDR;
+	r->add = add;
+	return i;
+}
+
+vlong
+addpcrelplus(Sym *s, Sym *t, int32 add)
+{
+	vlong i;
+	Reloc *r;
+	
+	if(s->type == 0)
+		s->type = SDATA;
+	s->reachable = 1;
+	i = s->size;
+	s->size += 4;
+	symgrow(s, s->size);
+	r = addrel(s);
+	r->sym = t;
+	r->off = i;
+	r->add = add;
+	r->type = D_PCREL;
+	r->siz = 4;
 	return i;
 }
 
+vlong
+addaddr(Sym *s, Sym *t)
+{
+	return addaddrplus(s, t, 0);
+}
+
 vlong
 addsize(Sym *s, Sym *t)
 {
@@ -615,7 +697,7 @@ dodata(void)
 {
 	int32 h, t, datsize;
 	Section *sect;
-	Sym *s, *last;
+	Sym *s, *last, **l;
 
 	if(debug['v'])
 		Bprint(&bso, "%5.2f dodata\n", cputime());
@@ -645,6 +727,24 @@ dodata(void)
 			diag("%s: initialize bounds (%lld < %d)",
 				s->name, s->size, s->np);
 	}
+	
+	/*
+	 * now that we have the datap list, but before we start
+	 * to assign addresses, record all the necessary
+	 * dynamic relocations.  these will grow the relocation
+	 * symbol, which is itself data.
+	 */
+	dynreloc();
+	
+	/* some symbols may no longer belong in datap (Mach-O) */
+	for(l=&datap; (s=*l) != nil; ) {
+		if(s->type <= STEXT || SXREF <= s->type)
+			*l = s->next;
+		else
+			l = &s->next;
+	}
+	*l = nil;
+
 	datap = datsort(datap);
 
 	/*
@@ -714,6 +814,40 @@ dodata(void)
 	sect->len = datsize - sect->vaddr;
 }
 
+// assign addresses to text
+void
+textaddress(void)
+{
+	uvlong va;
+	Prog *p;
+	Section *sect;
+	Sym *sym, *sub;
+
+	addsection(&segtext, ".text", 05);
+
+	// Assign PCs in text segment.
+	// Could parallelize, by assigning to text 
+	// and then letting threads copy down, but probably not worth it.
+	sect = segtext.sect;
+	va = INITTEXT;
+	sect->vaddr = va;
+	for(sym = textp; sym != nil; sym = sym->next) {
+		if(sym->type & SSUB)
+			continue;
+		sym->value = 0;
+		for(sub = sym; sub != S; sub = sub->sub) {
+			sub->value += va;
+			for(p = sub->text; p != P; p = p->link)
+				p->pc += sub->value;
+		}
+		if(sym->size == 0 && sym->sub != S) {
+			cursym = sym;
+		}
+		va += sym->size;
+	}
+	sect->len = va - sect->vaddr;
+}
+
 // assign addresses
 void
 address(void)
@@ -723,7 +857,7 @@ address(void)
 	uvlong va;
 
 	va = INITTEXT;
-	segtext.rwx = 05;
+	segtext.rwx = 07;
 	segtext.vaddr = va;
 	segtext.fileoff = HEADR;
 	for(s=segtext.sect; s != nil; s=s->next) {

src/cmd/ld/elf.c

@@ -164,8 +164,7 @@ newElfPhdr(void)
 {
 	ElfPhdr *e;
 
-	e = malloc(sizeof *e);
-	memset(e, 0, sizeof *e);
+	e = mal(sizeof *e);
 	if (hdr.phnum >= NSECT)
 		diag("too many phdrs");
 	else
@@ -189,8 +188,7 @@ newElfShdr(vlong name)
 {
 	ElfShdr *e;
 
-	e = malloc(sizeof *e);
-	memset(e, 0, sizeof *e);
+	e = mal(sizeof *e);
 	e->name = name;
 	if (hdr.shnum >= NSECT) {
 		diag("too many shdrs");
@@ -332,17 +330,25 @@ elfinterp(ElfShdr *sh, uint64 startva, char *p)
 	sh->size = n;
 }
 
+extern int nelfsym;
+
 void
-elfdynhash(int nsym)
+elfdynhash(void)
 {
 	Sym *s, *sy;
 	int i, h, nbucket, b;
 	uchar *pc;
 	uint32 hc, g;
 	uint32 *chain, *buckets;
+	int nsym;
+	char *name;
+	
+	if(!iself)
+		return;
 
+	nsym = nelfsym;
 	s = lookup(".hash", 0);
-	s->type = SELFDATA;	// TODO: rodata
+	s->type = SELFDATA;
 	s->reachable = 1;
 
 	i = nsym;
@@ -353,17 +359,24 @@ elfdynhash(int nsym)
 	}
 
 	chain = malloc(nsym * sizeof(uint32));
-	memset(chain, 0, nsym * sizeof(uint32));
 	buckets = malloc(nbucket * sizeof(uint32));
+	if(chain == nil || buckets == nil) {
+		cursym = nil;
+		diag("out of memory");
+		errorexit();
+	}
+	memset(chain, 0, nsym * sizeof(uint32));
 	memset(buckets, 0, nbucket * sizeof(uint32));
-	i = 1;
 	for(h = 0; h<NHASH; h++) {
 		for(sy=hash[h]; sy!=S; sy=sy->hash) {
-			if (!sy->reachable || (sy->type != STEXT && sy->type != SDATA && sy->type != SBSS) || sy->dynimpname == nil)
+			if (sy->dynid <= 0)
 				continue;
 
 			hc = 0;
-			for(pc = (uchar*)sy->dynimpname; *pc; pc++) {
+			name = sy->dynimpname;
+			if(name == nil)
+				name = sy->name;
+			for(pc = (uchar*)name; *pc; pc++) {
 				hc = (hc<<4) + *pc;
 				g = hc & 0xf0000000;
 				hc ^= g >> 24;
@@ -371,9 +384,8 @@ elfdynhash(int nsym)
 			}
 
 			b = hc % nbucket;
-			chain[i] = buckets[b];
-			buckets[b] = i;
-			i++;
+			chain[sy->dynid] = buckets[b];
+			buckets[b] = sy->dynid;
 		}
 	}
 
@@ -386,6 +398,8 @@ elfdynhash(int nsym)
 
 	free(chain);
 	free(buckets);
+
+	elfwritedynent(lookup(".dynamic", 0), DT_NULL, 0);
 }
 
 ElfPhdr*

src/cmd/ld/go.c

@@ -266,6 +266,10 @@ expandpkg(char *t0, char *pkg)
 
 	// use malloc, not mal, so that caller can free
 	w0 = malloc(strlen(t0) + strlen(pkg)*n);
+	if(w0 == nil) {
+		diag("out of memory");
+		errorexit();
+	}
 	w = w0;
 	for(p=t=t0; (p=strstr(p, "\"\".")) != nil; p=t) {
 		memmove(w, t, p - t);
@@ -442,12 +446,21 @@ loaddynimport(char *file, char *pkg, char *p, int n)
 		// successful parse: now can edit the line
 		*strchr(name, ' ') = 0;
 		*strchr(def, ' ') = 0;
+		
+		if(strcmp(name, "_") == 0 && strcmp(def, "_") == 0) {
+			// allow #pragma dynimport _ _ "foo.so"
+			// to force a link of foo.so.
+			adddynlib(lib);
+			continue;
+		}
 
 		name = expandpkg(name, pkg);
-
 		s = lookup(name, 0);
-		s->dynimplib = lib;
-		s->dynimpname = def;
+		if(s->type == 0 || s->type == SXREF) {
+			s->dynimplib = lib;
+			s->dynimpname = def;
+			s->type = SDYNIMPORT;
+		}
 	}
 	return;
 
@@ -547,6 +560,10 @@ mark(Sym *s)
 		mark(s->r[i].sym);
 	if(s->gotype)
 		mark(s->gotype);
+	if(s->sub)
+		mark(s->sub);
+	if(s->outer)
+		mark(s->outer);
 }
 
 static char*

src/cmd/ld/lib.c

@@ -374,32 +374,38 @@ out:
 void
 ldobj(Biobuf *f, char *pkg, int64 len, char *pn, int whence)
 {
-	static int files;
-	static char **filen;
-	char **nfilen, *line;
-	int i, n, c1, c2, c3;
+	char *line;
+	int n, c1, c2, c3, c4;
+	uint32 magic;
 	vlong import0, import1, eof;
 	char src[1024];
 
 	eof = Boffset(f) + len;
 	src[0] = '\0';
 
-	// don't load individual object more than once.
-	// happens with import of .6 files because of loop in xresolv.
-	// doesn't happen with .a because SYMDEF is consulted
-	// first to decide whether each individual object file is needed.
-	for(i=0; i<files; i++)
-		if(strcmp(filen[i], pn) == 0)
-			return;
+	pn = strdup(pn);
+	
+	USED(c4);
+	USED(magic);
 
-	if((files&15) == 0){
-		nfilen = malloc((files+16)*sizeof(char*));
-		memmove(nfilen, filen, files*sizeof(char*));
-		free(filen);
-		filen = nfilen;
+	c1 = Bgetc(f);
+	c2 = Bgetc(f);
+	c3 = Bgetc(f);
+	c4 = Bgetc(f);
+	Bungetc(f);
+	Bungetc(f);
+	Bungetc(f);
+	Bungetc(f);
+	
+	magic = c1<<24 | c2<<16 | c3<<8 | c4;
+	if(magic == 0x7f454c46) {	// \x7F E L F
+		ldelf(f, pkg, len, pn);
+		return;
+	}
+	if((magic&~1) == 0xfeedface || (magic&~0x01000000) == 0xcefaedfe) {
+		ldmacho(f, pkg, len, pn);
+		return;
 	}
-	pn = strdup(pn);
-	filen[files++] = pn;
 
 	/* check the header */
 	line = Brdline(f, '\n');
@@ -471,6 +477,9 @@ lookup(char *symb, int v)
 	if(debug['v'] > 1)
 		Bprint(&bso, "lookup %s\n", symb);
 
+	s->dynid = -1;
+	s->plt = -1;
+	s->got = -1;
 	s->name = mal(l + 1);
 	memmove(s->name, symb, l);
 
@@ -766,11 +775,19 @@ mal(uint32 n)
 	n = (n+7)&~7;
 	if(n > NHUNK) {
 		v = malloc(n);
+		if(v == nil) {
+			diag("out of memory");
+			errorexit();
+		}
 		memset(v, 0, n);
 		return v;
 	}
 	if(n > nhunk) {
 		hunk = malloc(NHUNK);
+		if(hunk == nil) {
+			diag("out of memory");
+			errorexit();
+		}
 		nhunk = NHUNK;
 	}
 
@@ -1013,3 +1030,42 @@ mkfwd(void)
 		}
 	}
 }
+
+uint16
+le16(uchar *b)
+{
+	return b[0] | b[1]<<8;
+}
+
+uint32
+le32(uchar *b)
+{
+	return b[0] | b[1]<<8 | b[2]<<16 | b[3]<<24;
+}
+
+uint64
+le64(uchar *b)
+{
+	return le32(b) | (uint64)le32(b+4)<<32;
+}
+
+uint16
+be16(uchar *b)
+{
+	return b[0]<<8 | b[1];
+}
+
+uint32
+be32(uchar *b)
+{
+	return b[0]<<24 | b[1]<<16 | b[2]<<8 | b[3];
+}
+
+uint64
+be64(uchar *b)
+{
+	return (uvlong)be32(b)<<32 | be32(b+4);
+}
+
+Endian be = { be16, be32, be64 };
+Endian le = { le16, le32, le64 };

src/cmd/ld/lib.h

@@ -125,8 +125,11 @@ void	pclntab(void);
 void	symtab(void);
 void	Lflag(char *arg);
 void	usage(void);
+void	adddynrel(Sym*, Reloc*);
 void	ldobj1(Biobuf *f, char*, int64 len, char *pn);
 void	ldobj(Biobuf*, char*, int64, char*, int);
+void	ldelf(Biobuf*, char*, int64, char*);
+void	ldmacho(Biobuf*, char*, int64, char*);
 void	ldpkg(Biobuf*, char*, int64, char*, int);
 void	mark(Sym *s);
 void	mkfwd(void);
@@ -144,6 +147,8 @@ vlong	addstring(Sym*, char*);
 vlong	adduint32(Sym*, uint32);
 vlong	adduint64(Sym*, uint64);
 vlong	addaddr(Sym*, Sym*);
+vlong	addaddrplus(Sym*, Sym*, int32);
+vlong	addpcrelplus(Sym*, Sym*, int32);
 vlong	addsize(Sym*, Sym*);
 vlong	adduint8(Sym*, uint8);
 vlong	adduint16(Sym*, uint16);
@@ -152,13 +157,34 @@ void	asmelfsym64(void);
 void	strnput(char*, int);
 void	dodata(void);
 void	address(void);
+void	textaddress(void);
 void	genasmsym(void (*put)(Sym*, char*, int, vlong, vlong, int, Sym*));
 vlong	datoff(vlong);
+void	adddynlib(char*);
+int	archreloc(Reloc*, Sym*, vlong*);
 
 int	pathchar(void);
 void*	mal(uint32);
 void	unmal(void*, uint32);
 void	mywhatsys(void);
+int	rbyoff(const void*, const void*);
+
+uint16	le16(uchar*);
+uint32	le32(uchar*);
+uint64	le64(uchar*);
+uint16	be16(uchar*);
+uint32	be32(uchar*);
+uint64	be64(uchar*);
+
+typedef struct Endian Endian;
+struct Endian
+{
+	uint16	(*e16)(uchar*);
+	uint32	(*e32)(uchar*);
+	uint64	(*e64)(uchar*);
+};
+
+extern Endian be, le;
 
 // relocation size bits
 enum {

src/cmd/ld/macho.h

@@ -18,6 +18,8 @@ struct MachoSect {
 	uint32	reloc;
 	uint32	nreloc;
 	uint32	flag;
+	uint32	res1;
+	uint32	res2;
 };
 
 typedef struct MachoSeg MachoSeg;
@@ -70,6 +72,20 @@ enum {
 
 	MACHO32SYMSIZE = 12,
 	MACHO64SYMSIZE = 16,
+	
+	MACHO_X86_64_RELOC_UNSIGNED = 0,
+	MACHO_X86_64_RELOC_SIGNED = 1,
+	MACHO_X86_64_RELOC_BRANCH = 2,
+	MACHO_X86_64_RELOC_GOT_LOAD = 3,
+	MACHO_X86_64_RELOC_GOT = 4,
+	MACHO_X86_64_RELOC_SUBTRACTOR = 5,
+	MACHO_X86_64_RELOC_SIGNED_1 = 6,
+	MACHO_X86_64_RELOC_SIGNED_2 = 7,
+	MACHO_X86_64_RELOC_SIGNED_4 = 8,
+	
+	MACHO_GENERIC_RELOC_VANILLA = 0,
+	
+	MACHO_FAKE_GOTPCREL = 100,
 };
 
 void	domacho(void);