From 70c4a5f5f06b2ceb230e551340dd9ba81d393507 Mon Sep 17 00:00:00 2001 From: Yu Huang Date: Sat, 22 May 2021 10:14:59 -0400 Subject: [PATCH 1/3] add parser --- parser/parser.go | 184 ++++++++++++++++++++++++++++++++++++++++++ parser/parser_test.go | 39 +++++++++ 2 files changed, 223 insertions(+) create mode 100644 parser/parser.go create mode 100644 parser/parser_test.go diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..28ca6ff --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,184 @@ +package beeParser + +import ( + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" +) + +// type to default value +var builtInTypeMap = map[string]interface{}{ + "string": "", + "int": 0, + "int64": 0, + "int32": 0, + "uint": 0, + "uint32": 0, + "uint64": 0, + "bool": false, + // @todo add more type +} + +type StructField struct { + Name string + Type string + NestedType *StructNode + Comment string + Doc string + Tag string +} + +func (sf *StructField) IsBuiltInType() bool { + _, found := builtInTypeMap[sf.Type] + return found +} + +func (sf *StructField) Key() string { + return sf.Name +} + +func (sf *StructField) Value() interface{} { + val, found := builtInTypeMap[sf.Type] + if found { + return val + } + + return sf.NestedType.ToKV() +} + +type StructNode struct { + Name string + Fields []*StructField +} + +func (sn *StructNode) ToKV() map[string]interface{} { + value := map[string]interface{}{} + for _, field := range sn.Fields { + value[field.Key()] = field.Value() + } + return value +} + +type ConfigGenerator struct { + StructMap map[string]*StructNode // @todo key = {package}+{struct name} + RootStruct string //match with the key of StructMap +} + +func NewConfigGenerator(filePath string, src interface{}, rootStruct string) (*ConfigGenerator, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, src, parser.ParseComments) + if err != nil { + return nil, err + } + + structMap := map[string]*StructNode{} + + ast.Inspect(f, func(n ast.Node) bool { + // ast.Print(nil, n) + ts, ok := n.(*ast.TypeSpec) + if !ok || ts.Type == nil { + return true + } + + structName := ts.Name.Name + if structName != rootStruct { + return true + } + + s, ok := ts.Type.(*ast.StructType) + if !ok { + return true + } + + structMap[structName] = ParseStruct(structName, s) + + return false + }) + + if _, found := structMap[rootStruct]; !found { + return nil, errors.New("non-exist root struct") + } + + return &ConfigGenerator{ + StructMap: structMap, + RootStruct: rootStruct, + }, nil +} + +func (cg *ConfigGenerator) ToJSON() ([]byte, error) { + rootStruct := cg.StructMap[cg.RootStruct] + value := rootStruct.ToKV() + return json.MarshalIndent(value, "", " ") +} + +func ParseField(field *ast.Field) *StructField { + fieldName := field.Names[0].Name + fieldType := fmt.Sprint(field.Type) + + fieldTag := "" + if field.Tag != nil { + fieldTag = field.Tag.Value + } + fieldComment := "" + if field.Comment != nil { + fieldComment = field.Comment.Text() + } + fieldDoc := "" + if field.Doc != nil { + fieldDoc = field.Doc.Text() + } + + switch field.Type.(type) { + case *ast.Ident: // built-in or nested + isNested := (field.Type.(*ast.Ident).Obj != nil) + if !isNested { + return &StructField{ + Name: fieldName, + Type: fieldType, + Tag: fieldTag, + Comment: fieldComment, + Doc: fieldDoc, + } + } + ts, ok := field.Type.(*ast.Ident).Obj.Decl.(*ast.TypeSpec) + if !ok || ts.Type == nil { + return nil + } + + s, ok := ts.Type.(*ast.StructType) + if !ok { + return nil + } + return &StructField{ + Name: fieldName, + Type: fieldType, + Tag: fieldTag, + Comment: fieldComment, + Doc: fieldDoc, + NestedType: ParseStruct(ts.Name.Name, s), + } + case *ast.ArrayType: + case *ast.MapType: + case *ast.SelectorExpr: // third party + } + + return nil +} + +func ParseStruct(structName string, s *ast.StructType) *StructNode { + fields := []*StructField{} + for _, field := range s.Fields.List { + parsedField := ParseField(field) + if parsedField != nil { + fields = append(fields, parsedField) + } + } + + return &StructNode{ + Name: structName, + Fields: fields, + } +} diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 0000000..af1161a --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,39 @@ +package beeParser + +import ( + "fmt" + "log" +) + +func ExampleConfigGenerator() { + const src = ` +package p +import "http" + +type StructB struct { + Field1 string +} +type StructA struct { + Field1 string + Field2 StructB + Field3 []string + Field4 map[string]string + Field5 http.SameSite +} +` + cg, err := NewConfigGenerator("./sample.go", src, "StructA") + if err != nil { + log.Fatal(err) + } + + b, err := cg.ToJSON() + if err != nil { + log.Fatal(err) + } + + fmt.Println(string(b)) + + // Output: + // { + // } +} From c3bb85f7b10660bd9a1347eee9c5e15736e88dfb Mon Sep 17 00:00:00 2001 From: Yu Huang Date: Sat, 22 May 2021 10:26:33 -0400 Subject: [PATCH 2/3] ignore other type --- parser/parser.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 28ca6ff..dcc2b28 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -24,7 +24,7 @@ var builtInTypeMap = map[string]interface{}{ type StructField struct { Name string - Type string + Type ast.Expr NestedType *StructNode Comment string Doc string @@ -32,7 +32,7 @@ type StructField struct { } func (sf *StructField) IsBuiltInType() bool { - _, found := builtInTypeMap[sf.Type] + _, found := builtInTypeMap[fmt.Sprint(sf.Type)] return found } @@ -41,12 +41,19 @@ func (sf *StructField) Key() string { } func (sf *StructField) Value() interface{} { - val, found := builtInTypeMap[sf.Type] - if found { - return val + switch sf.Type.(type) { + case *ast.Ident: + val, found := builtInTypeMap[fmt.Sprint(sf.Type)] + if found { + return val + } + return sf.NestedType.ToKV() + case *ast.ArrayType: + case *ast.MapType: + case *ast.SelectorExpr: // third party } - return sf.NestedType.ToKV() + return "" } type StructNode struct { @@ -116,7 +123,7 @@ func (cg *ConfigGenerator) ToJSON() ([]byte, error) { func ParseField(field *ast.Field) *StructField { fieldName := field.Names[0].Name - fieldType := fmt.Sprint(field.Type) + fieldType := field.Type fieldTag := "" if field.Tag != nil { @@ -165,7 +172,13 @@ func ParseField(field *ast.Field) *StructField { case *ast.SelectorExpr: // third party } - return nil + return &StructField{ + Name: fieldName, + Type: fieldType, + Tag: fieldTag, + Comment: fieldComment, + Doc: fieldDoc, + } } func ParseStruct(structName string, s *ast.StructType) *StructNode { From aa69764e9f65755aebbe10c56287d2de05c2a80f Mon Sep 17 00:00:00 2001 From: Yu Huang Date: Sat, 22 May 2021 22:34:39 -0400 Subject: [PATCH 3/3] supports inline struct and optimized code --- parser/parser.go | 140 +++++++++++++++++++----------------------- parser/parser_test.go | 29 +++++++-- 2 files changed, 89 insertions(+), 80 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index dcc2b28..2b93e3a 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -3,64 +3,45 @@ package beeParser import ( "encoding/json" "errors" - "fmt" "go/ast" + "go/importer" "go/parser" "go/token" + "go/types" ) -// type to default value -var builtInTypeMap = map[string]interface{}{ - "string": "", - "int": 0, - "int64": 0, - "int32": 0, - "uint": 0, - "uint32": 0, - "uint64": 0, - "bool": false, - // @todo add more type -} - +// StructField defines struct field type StructField struct { Name string - Type ast.Expr + Type types.Type NestedType *StructNode Comment string Doc string Tag string } -func (sf *StructField) IsBuiltInType() bool { - _, found := builtInTypeMap[fmt.Sprint(sf.Type)] - return found -} - +// Key returns the key of the field func (sf *StructField) Key() string { return sf.Name } +// Value returns the value of the field +// if the field contains nested struct, it will return a nested result func (sf *StructField) Value() interface{} { - switch sf.Type.(type) { - case *ast.Ident: - val, found := builtInTypeMap[fmt.Sprint(sf.Type)] - if found { - return val - } + if sf.NestedType != nil { return sf.NestedType.ToKV() - case *ast.ArrayType: - case *ast.MapType: - case *ast.SelectorExpr: // third party } return "" } +// StructNode defines struct node type StructNode struct { Name string Fields []*StructField } +// ToKV transfers struct to key value pair func (sn *StructNode) ToKV() map[string]interface{} { value := map[string]interface{}{} for _, field := range sn.Fields { @@ -69,19 +50,39 @@ func (sn *StructNode) ToKV() map[string]interface{} { return value } -type ConfigGenerator struct { - StructMap map[string]*StructNode // @todo key = {package}+{struct name} - RootStruct string //match with the key of StructMap +// StructParser parses structs in given file or string +type StructParser struct { + MainStruct *StructNode + Info types.Info } -func NewConfigGenerator(filePath string, src interface{}, rootStruct string) (*ConfigGenerator, error) { +// NewStructParser is the constructor of StructParser +// filePath and src follow the same rule with go/parser.ParseFile +// If src != nil, ParseFile parses the source from src and the filename is only used when recording position information. The type of the argument for the src parameter must be string, []byte, or io.Reader. If src == nil, ParseFile parses the file specified by filename. +// rootStruct is the root struct we want to use +func NewStructParser(filePath string, src interface{}, rootStruct string) (*StructParser, error) { fset := token.NewFileSet() f, err := parser.ParseFile(fset, filePath, src, parser.ParseComments) if err != nil { return nil, err } - structMap := map[string]*StructNode{} + info := types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + conf := types.Config{ + Importer: importer.ForCompiler(fset, "source", nil), + } + _, err = conf.Check("src", fset, []*ast.File{f}, &info) + if err != nil { + return nil, err + } + + cg := &StructParser{ + Info: info, + } ast.Inspect(f, func(n ast.Node) bool { // ast.Print(nil, n) @@ -100,30 +101,27 @@ func NewConfigGenerator(filePath string, src interface{}, rootStruct string) (*C return true } - structMap[structName] = ParseStruct(structName, s) - + cg.MainStruct = cg.ParseStruct(structName, s) return false }) - if _, found := structMap[rootStruct]; !found { + if cg.MainStruct == nil { return nil, errors.New("non-exist root struct") } - return &ConfigGenerator{ - StructMap: structMap, - RootStruct: rootStruct, - }, nil + return cg, nil } -func (cg *ConfigGenerator) ToJSON() ([]byte, error) { - rootStruct := cg.StructMap[cg.RootStruct] - value := rootStruct.ToKV() +func (cg *StructParser) ToJSON() ([]byte, error) { + value := cg.MainStruct.ToKV() return json.MarshalIndent(value, "", " ") } -func ParseField(field *ast.Field) *StructField { +// ParseField parses struct field in nested way +func (cg *StructParser) ParseField(field *ast.Field) *StructField { + // ast.Print(nil, field) fieldName := field.Names[0].Name - fieldType := field.Type + fieldType := cg.Info.TypeOf(field.Type) fieldTag := "" if field.Tag != nil { @@ -138,18 +136,12 @@ func ParseField(field *ast.Field) *StructField { fieldDoc = field.Doc.Text() } - switch field.Type.(type) { - case *ast.Ident: // built-in or nested - isNested := (field.Type.(*ast.Ident).Obj != nil) - if !isNested { - return &StructField{ - Name: fieldName, - Type: fieldType, - Tag: fieldTag, - Comment: fieldComment, - Doc: fieldDoc, - } - } + var nestedStruct *StructNode + if s, isInlineStruct := field.Type.(*ast.StructType); isInlineStruct { + nestedStruct = cg.ParseStruct("", s) + } + + if _, isNamedStructorBasic := field.Type.(*ast.Ident); isNamedStructorBasic && field.Type.(*ast.Ident).Obj != nil { ts, ok := field.Type.(*ast.Ident).Obj.Decl.(*ast.TypeSpec) if !ok || ts.Type == nil { return nil @@ -159,32 +151,28 @@ func ParseField(field *ast.Field) *StructField { if !ok { return nil } - return &StructField{ - Name: fieldName, - Type: fieldType, - Tag: fieldTag, - Comment: fieldComment, - Doc: fieldDoc, - NestedType: ParseStruct(ts.Name.Name, s), - } - case *ast.ArrayType: - case *ast.MapType: - case *ast.SelectorExpr: // third party + nestedStruct = cg.ParseStruct(ts.Name.Name, s) } + // fieldType.(*types.Basic) // basic type + // *ast.ArrayType: + // *ast.MapType: + // *ast.SelectorExpr: // third party return &StructField{ - Name: fieldName, - Type: fieldType, - Tag: fieldTag, - Comment: fieldComment, - Doc: fieldDoc, + Name: fieldName, + Type: fieldType, + Tag: fieldTag, + Comment: fieldComment, + Doc: fieldDoc, + NestedType: nestedStruct, } } -func ParseStruct(structName string, s *ast.StructType) *StructNode { +// ParseStruct parses struct in nested way +func (cg *StructParser) ParseStruct(structName string, s *ast.StructType) *StructNode { fields := []*StructField{} for _, field := range s.Fields.List { - parsedField := ParseField(field) + parsedField := cg.ParseField(field) if parsedField != nil { fields = append(fields, parsedField) } diff --git a/parser/parser_test.go b/parser/parser_test.go index af1161a..4ba578a 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -5,23 +5,32 @@ import ( "log" ) -func ExampleConfigGenerator() { +func ExampleStructParser() { const src = ` package p -import "http" + +import ( + "net/http" +) type StructB struct { Field1 string } type StructA struct { Field1 string - Field2 StructB + Field2 struct{ + a string + b string + } Field3 []string Field4 map[string]string Field5 http.SameSite + Field6 func(int) + Field7 StructB } + ` - cg, err := NewConfigGenerator("./sample.go", src, "StructA") + cg, err := NewStructParser("src.go", src, "StructA") if err != nil { log.Fatal(err) } @@ -35,5 +44,17 @@ type StructA struct { // Output: // { + // "Field1": "", + // "Field2": { + // "a": "", + // "b": "" + // }, + // "Field3": "", + // "Field4": "", + // "Field5": "", + // "Field6": "", + // "Field7": { + // "Field1": "" + // } // } }