xgbgen: process <doc> elements

Most of XCB documentation now ends up in Go sources,
although the end result is of mixed quality.

nexgb/xgbgen/context.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"sort"
+	"strings"
 )
 
 // Context represents the protocol we're converting to Go, and a writer
@@ -34,6 +35,13 @@ func (c *Context) Put(format string, v ...interface{}) {
 	}
 }
 
+// PutComment writes each line of comment commented out to 'out'.
+func (c *Context) PutComment(comment string) {
+	for _, line := range strings.Split(comment, "\n") {
+		c.Putln("// %s", line)
+	}
+}
+
 // Morph is the big daddy of them all. It takes in an XML byte slice,
 // parse it, transforms the XML types into more usable types,
 // and writes Go code to the 'out' buffer.

nexgb/xgbgen/field.go

@@ -105,6 +105,7 @@ type SingleField struct {
 	srcName string
 	xmlName string
 	Type    Type
+	Comment string
 }
 
 func (f *SingleField) Initialize(p *Protocol) {
@@ -259,6 +260,8 @@ type ValueField struct {
 	MaskType    Type
 	MaskName    string
 	ListName    string
+	MaskComment string
+	ListComment string
 }
 
 func (f *ValueField) SrcName() string {
@@ -324,6 +327,7 @@ type SwitchField struct {
 	MaskName string
 	Expr     Expression
 	Bitcases []*Bitcase
+	Comment  string
 }
 
 func (f *SwitchField) SrcName() string {

nexgb/xgbgen/go.go

@@ -168,7 +168,13 @@ func (f *ExprField) Write(c *Context, prefix string) {
 
 // Value field
 func (f *ValueField) Define(c *Context) {
+	if f.MaskComment != "" {
+		c.PutComment(f.MaskComment)
+	}
 	c.Putln("%s %s", f.MaskName, f.SrcType())
+	if f.ListComment != "" {
+		c.PutComment(f.ListComment)
+	}
 	c.Putln("%s []uint32", f.ListName)
 }
 
@@ -197,6 +203,9 @@ func (f *ValueField) Write(c *Context, prefix string) {
 
 // Switch field
 func (f *SwitchField) Define(c *Context) {
+	if f.Comment != "" {
+		c.PutComment(f.Comment)
+	}
 	c.Putln("%s []uint32", f.Name)
 }
 

nexgb/xgbgen/go_request_reply.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 )
 
@@ -12,6 +13,33 @@ func (r *Request) Define(c *Context) {
 	c.Putln("*xgb.Cookie")
 	c.Putln("}")
 	c.Putln("")
+
+	if r.Doc.Description != "" {
+		c.PutComment(r.Doc.Description)
+		c.Putln("//")
+	}
+
+	allErrors := make([]string, 0, len(r.Doc.Errors))
+	for kind := range r.Doc.Errors {
+		allErrors = append(allErrors, kind)
+	}
+	sort.Strings(allErrors)
+
+	undocErrors := make([]string, 0)
+	for _, kind := range allErrors {
+		if desc := r.Doc.Errors[kind]; desc == "" {
+			undocErrors = append(undocErrors, kind)
+		} else {
+			c.PutComment(fmt.Sprintf("May return a %s error if %s%s", kind,
+				strings.ToLower(desc[:1]), desc[1:]))
+			c.Putln("//")
+		}
+	}
+	if len(undocErrors) > 0 {
+		c.Putln("// May return %s errors.", strings.Join(undocErrors, ", "))
+		c.Putln("//")
+	}
+
 	if r.Reply != nil {
 		c.Putln("// %s sends a checked request.", r.SrcName())
 		c.Putln("// If an error occurs, it will be returned with the reply "+

nexgb/xgbgen/go_single_field.go

@@ -6,6 +6,9 @@ import (
 )
 
 func (f *SingleField) Define(c *Context) {
+	if f.Comment != "" {
+		c.PutComment(f.Comment)
+	}
 	c.Putln("%s %s", f.SrcName(), f.Type.SrcName())
 }
 

nexgb/xgbgen/request_reply.go

@@ -6,6 +6,22 @@ import (
 	"unicode"
 )
 
+// Doc contains any documentation, if present. Example C code is excluded.
+type Doc struct {
+	Brief       string            // short description
+	Description string            // long description
+	Fields      map[string]string // from field name to description
+	Errors      map[string]string // from error type to description
+}
+
+// DescribeField is an accessor that supports nil receivers.
+func (d *Doc) DescribeField(name string) string {
+	if d == nil {
+		return ""
+	}
+	return d.Fields[name]
+}
+
 // Request represents all XML 'request' nodes.
 // If the request doesn't have a reply, Reply is nil.
 type Request struct {
@@ -15,6 +31,7 @@ type Request struct {
 	Combine bool    // Not currently used.
 	Fields  []Field // All fields in the request.
 	Reply   *Reply  // A reply, if one exists for this request.
+	Doc     Doc     // Documentation.
 }
 
 type Requests []*Request
@@ -126,6 +143,7 @@ func (r *Request) Size(c *Context) Size {
 // Reply encapsulates the fields associated with a 'reply' element.
 type Reply struct {
 	Fields []Field
+	Doc    Doc
 }
 
 // Size gets the number of bytes in this request's reply.

nexgb/xgbgen/translation.go

@@ -7,13 +7,11 @@ the XML type into our "better" representation.
 i.e., the representation of Fields and Expressions is just too general.
 We end up losing a lot of the advantages of static typing if we keep
 the types that encoding/xml forces us into.
-
-Please see 'representation.go' for the type definitions that we're
-translating to.
 */
 
 import (
 	"log"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -138,12 +136,10 @@ func (x *XMLEvent) Translate() *Event {
 		Number:     x.Number,
 		NoSequence: x.NoSequence,
 		Fields:     make([]Field, 0, len(x.Fields)),
+		Doc:        x.Doc.Translate(),
 	}
 	for _, field := range x.Fields {
-		if field.XMLName.Local == "doc" {
+		ev.Fields = append(ev.Fields, field.Translate(ev, &ev.Doc))
-			continue
-		}
-		ev.Fields = append(ev.Fields, field.Translate(ev))
 	}
 	return ev
 }
@@ -163,7 +159,7 @@ func (x *XMLError) Translate() *Error {
 		Fields:  make([]Field, len(x.Fields)),
 	}
 	for i, field := range x.Fields {
-		err.Fields[i] = field.Translate(err)
+		err.Fields[i] = field.Translate(err, nil)
 	}
 	return err
 }
@@ -176,13 +172,82 @@ func (x *XMLErrorCopy) Translate() *ErrorCopy {
 	}
 }
 
+// XCB documentation is positively stuffed with TODOs. We'd like to make it
+// look a bit less shitty, so remove those as they don't convey information.
+//
+//  ^TODO
+//  ^TODO:
+//  ^TODO: question?
+//  ^TODO: Some words
+//  ^TODO: some words
+//  ^TODO: some words with full stop.
+//  ^TODO: some words with full stop. and a question?
+//  ... (TODO),
+//  ... (TODO).
+//  ... (TODO: a question?).
+//  ... TODO: a question?
+//  ... (word TODO) ...
+var todoRE = regexp.MustCompile(`(?m:^TODO.*| \([^)]*TODO[^)]*\)| TODO:.*)`)
+var paraRE = regexp.MustCompile(`\n{3,}`)
+
+var backticksRE = regexp.MustCompile("`(?:xcb_|XCB_)?(.*?)(?:_t)?`")
+
+// fixDocumentation tries to translate XCB documentation to match XGB.
+// Note that <doc> blocks only appear in xproto, so this doesn't have that much
+// of a value--users still need to read Xlib or X11 protocol docs.
+// Despite that, it's better to have something than nothing.
+//
+// We don't attempt to add proper prefixes to enum values or guess at
+// specific XCB_NONE types (the latter is undecidable).
+//
+// We can't decide whether `fields_len` should be converted to len(Fields)
+// or FieldsLen because e.g. StringLen is retained in ImageText8/16 but
+// PointsLen is implied by the length of the Points slice in PolyLine.
+func fixDocumentation(xcb string) string {
+	last, result := 0, make([]byte, 0, len(xcb))
+	for _, m := range backticksRE.FindAllStringSubmatchIndex(xcb, -1) {
+		result = append(result, xcb[last:m[0]]...)
+		inner := xcb[m[2]:m[3]]
+		last = m[1]
+
+		// Do not convert atom names to identifiers, mainly _NET_WM_*.
+		if strings.Contains(inner, "WM_") {
+			result = append(result, inner...)
+		} else {
+			result = append(result, splitAndTitle(inner)...)
+		}
+	}
+	result = todoRE.ReplaceAllLiteral(append(result, xcb[last:]...), nil)
+	result = paraRE.ReplaceAllLiteral(result, []byte("\n\n"))
+	return strings.TrimSpace(string(result))
+}
+
+func (x *XMLDoc) Translate() Doc {
+	if x == nil {
+		return Doc{}
+	}
+	d := Doc{
+		Brief:       fixDocumentation(x.Brief),
+		Description: fixDocumentation(x.Description),
+		Fields:      make(map[string]string),
+		Errors:      make(map[string]string),
+	}
+	for _, x := range x.Fields {
+		d.Fields[x.Name] = fixDocumentation(x.Description)
+	}
+	for _, x := range x.Errors {
+		d.Errors[x.Type] = fixDocumentation(x.Description)
+	}
+	return d
+}
+
 func (x *XMLStruct) Translate() *Struct {
 	s := &Struct{
 		xmlName: x.Name,
 		Fields:  make([]Field, len(x.Fields)),
 	}
 	for i, field := range x.Fields {
-		s.Fields[i] = field.Translate(s)
+		s.Fields[i] = field.Translate(s, nil)
 	}
 	return s
 }
@@ -193,7 +258,7 @@ func (x *XMLUnion) Translate() *Union {
 		Fields:  make([]Field, len(x.Fields)),
 	}
 	for i, field := range x.Fields {
-		u.Fields[i] = field.Translate(u)
+		u.Fields[i] = field.Translate(u, nil)
 	}
 	return u
 }
@@ -205,12 +270,13 @@ func (x *XMLRequest) Translate() *Request {
 		Combine: x.Combine,
 		Fields:  make([]Field, 0, len(x.Fields)),
 		Reply:   x.Reply.Translate(),
+		Doc:     x.Doc.Translate(),
 	}
 	for _, field := range x.Fields {
-		if field.XMLName.Local == "doc" || field.XMLName.Local == "fd" {
+		if field.XMLName.Local == "fd" {
 			continue
 		}
-		r.Fields = append(r.Fields, field.Translate(r))
+		r.Fields = append(r.Fields, field.Translate(r, &r.Doc))
 	}
 
 	// Address bug (or legacy code) in QueryTextExtents.
@@ -236,12 +302,13 @@ func (x *XMLReply) Translate() *Reply {
 
 	r := &Reply{
 		Fields: make([]Field, 0, len(x.Fields)),
+		Doc:    x.Doc.Translate(),
 	}
 	for _, field := range x.Fields {
-		if field.XMLName.Local == "doc" || field.XMLName.Local == "fd" {
+		if field.XMLName.Local == "fd" {
 			continue
 		}
-		r.Fields = append(r.Fields, field.Translate(r))
+		r.Fields = append(r.Fields, field.Translate(r, &r.Doc))
 	}
 	return r
 }
@@ -320,7 +387,7 @@ func (x *XMLExpression) Translate() Expression {
 	panic("unreachable")
 }
 
-func (x *XMLField) Translate(parent interface{}) Field {
+func (x *XMLField) Translate(parent interface{}, doc *Doc) Field {
 	switch x.XMLName.Local {
 	case "pad":
 		return &PadField{
@@ -331,6 +398,7 @@ func (x *XMLField) Translate(parent interface{}) Field {
 		s := &SingleField{
 			xmlName: x.Name,
 			Type:    newTranslation(x.Type),
+			Comment: doc.DescribeField(x.Name),
 		}
 		return s
 	case "list":
@@ -356,12 +424,15 @@ func (x *XMLField) Translate(parent interface{}) Field {
 			MaskType:    newTranslation(x.ValueMaskType),
 			MaskName:    x.ValueMaskName,
 			ListName:    x.ValueListName,
+			MaskComment: doc.DescribeField(x.ValueMaskName),
+			ListComment: doc.DescribeField(x.ValueListName),
 		}
 	case "switch":
 		swtch := &SwitchField{
 			Name:     x.Name,
 			Expr:     x.Expr.Translate(),
 			Bitcases: make([]*Bitcase, len(x.Bitcases)),
+			Comment:  doc.DescribeField(x.Name),
 		}
 		for i, bitcase := range x.Bitcases {
 			swtch.Bitcases[i] = bitcase.Translate()
@@ -381,7 +452,7 @@ func (x *XMLBitcase) Translate() *Bitcase {
 		Fields: make([]Field, len(x.Fields)),
 	}
 	for i, field := range x.Fields {
-		b.Fields[i] = field.Translate(b)
+		b.Fields[i] = field.Translate(b, nil)
 	}
 	return b
 }

nexgb/xgbgen/type.go

@@ -190,6 +190,7 @@ type Event struct {
 	Number     int
 	NoSequence bool
 	Fields     []Field
+	Doc        Doc
 }
 
 func (e *Event) SrcName() string {

nexgb/xgbgen/xml.go

@@ -92,6 +92,24 @@ type XMLErrorCopy struct {
 	Ref    string `xml:"ref,attr"`
 }
 
+type XMLDocField struct {
+	Name        string `xml:"name,attr"`
+	Description string `xml:",chardata"`
+}
+
+type XMLDocError struct {
+	Type        string `xml:"type,attr"`
+	Description string `xml:",chardata"`
+}
+
+type XMLDoc struct {
+	Brief       string         `xml:"brief"`
+	Description string         `xml:"description"`
+	Example     string         `xml:"example"`
+	Fields      []*XMLDocField `xml:"field"`
+	Errors      []*XMLDocError `xml:"error"`
+}
+
 type XMLStruct struct {
 	Name   string      `xml:"name,attr"`
 	Fields []*XMLField `xml:",any"`
@@ -108,10 +126,12 @@ type XMLRequest struct {
 	Combine bool        `xml:"combine-adjacent,attr"`
 	Fields  []*XMLField `xml:",any"`
 	Reply   *XMLReply   `xml:"reply"`
+	Doc     *XMLDoc     `xml:"doc"`
 }
 
 type XMLReply struct {
 	Fields []*XMLField `xml:",any"`
+	Doc    *XMLDoc     `xml:"doc"`
 }
 
 type XMLEvent struct {
@@ -119,6 +139,7 @@ type XMLEvent struct {
 	Number     int         `xml:"number,attr"`
 	NoSequence bool        `xml:"no-sequence-number,attr"`
 	Fields     []*XMLField `xml:",any"`
+	Doc        *XMLDoc     `xml:"doc"`
 }
 
 type XMLError struct {