From e79ba634661baba562fa0ab9245040bd37c29f4c Mon Sep 17 00:00:00 2001 From: Jeff Carr Date: Thu, 9 Jan 2025 12:54:04 -0600 Subject: rename dir --- argv.go | 2 +- auto.proto | 9 +++-- example/Makefile | 25 ++++++++++++ example/auto.proto | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ example/fruit.New.go | 7 ++++ example/fruit.proto | 46 ++++++++++++++++++++++ example/main.go | 21 ++++++++++ newsort.go | 13 ++++--- protoParse.go | 10 ----- sort.go | 60 ++++++++++++++--------------- testfiles/Makefile | 23 ----------- testfiles/auto.proto | 99 ----------------------------------------------- testfiles/main.go | 22 ----------- 13 files changed, 248 insertions(+), 196 deletions(-) create mode 100644 example/Makefile create mode 100644 example/auto.proto create mode 100644 example/fruit.New.go create mode 100644 example/fruit.proto create mode 100644 example/main.go delete mode 100644 testfiles/Makefile delete mode 100644 testfiles/auto.proto delete mode 100644 testfiles/main.go diff --git a/argv.go b/argv.go index 57a8ffc..6925956 100644 --- a/argv.go +++ b/argv.go @@ -18,7 +18,7 @@ type args struct { Marshal []string `arg:"--marshal" help:"what to marshal on"` NoMarshal bool `arg:"--no-marshal" help:"do not make a marshal.pb.go file"` NoSort bool `arg:"--no-sort" help:"do not make a sort.pb.go file"` - Mutex bool `arg:"--mutex" help:"try mutex hack (breaks proto.Marshal()"` + Mutex bool `arg:"--mutex" help:"try mutex hack (breaks proto.Marshal() ???)"` DryRun bool `arg:"--dry-run" help:"show what would be run"` GoSrc string `arg:"--go-src" help:"default is ~/go/src. could be set to your go.work path"` GoPath string `arg:"--gopath" help:"the gopath of this repo"` diff --git a/auto.proto b/auto.proto index a44b63f..a19afa6 100644 --- a/auto.proto +++ b/auto.proto @@ -66,10 +66,11 @@ message MsgName { string name = 1; // the name of the message aka struct. for this example: "Shelf" string lockname = 2; // ShelfMU bool doMarshal = 3; // if "Shelf" should have Marshal & Unmarshal functions - bool doMutex = 4; // an experiment to insert a mutex into the protoc generated msg struct (bad idea?) - bool mutexFound = 5; // an experiment to insert a mutex into the protoc generated msg struct (bad idea?) - repeated string sort = 6; // "Book", "Picture", etc - repeated string unique = 7; // if the fields should have AppendUnique() functions + bool doMutex = 4; // true if a mutex is needed for the message struct + bool doProtocMutex = 5; // an experiment to insert a mutex into the protoc generated msg struct (bad idea?) + bool mutexFound = 6; // true if the mutex was added to the protoc pb.go file + repeated string sort = 7; // "Book", "Picture", etc + repeated string unique = 8; // if the fields should have AppendUnique() functions } message File { // `autogenpb:nomarshal` diff --git a/example/Makefile b/example/Makefile new file mode 100644 index 0000000..7558b5e --- /dev/null +++ b/example/Makefile @@ -0,0 +1,25 @@ +VERSION = $(shell git describe --tags) +BUILDTIME = $(shell date +%Y.%m.%d_%H%M) + +full: clean goimports auto vet build + ./testfiles + +vet: + @GO111MODULE=off go vet + @echo this go binary package should build okay + +build: + rm -f fruit.newsort.pb.go + GO111MODULE=off go build + ./testfiles + +auto: + ../autogenpb --proto fruit.proto --package main + +goimports: + goimports -w *.go + +clean: + -rm -f go.* + -rm -f *.pb.go + -rm -f testfiles diff --git a/example/auto.proto b/example/auto.proto new file mode 100644 index 0000000..d65ccd0 --- /dev/null +++ b/example/auto.proto @@ -0,0 +1,107 @@ +syntax = "proto3"; + +// here are some docs, but probably it's just easier to run +// autogenpb on this file and see what gets autogenerated +// in this directory. All autogenerated files are named *.pb.go + +// the 'uuid' standard at the end is an experiment +// establish a way to identify arbitrary .pb files + +// You can generate Marshal & Unmarshal for any struct (message) you want +// You can generate SortBy and Append functions ONLY FOR 'repeated ' +// Also, those structs must be defined in the same file +// Additionally, you must use `autogenpb:mutex` on the parent struct. +// The autogenerated code requires a RW mutex and autogenpb will insert it into the struct + +package main; + +message Apple { // `autogenpb:marshal` + string name = 1; // `autogenpb:unique` // generates SortByxxx() and AppendUnique() functions + string genus = 2; // `autogenpb:unique` // generates same thing here but SortByGenus() +} + +message Apples { // `autogenpb:marshal` `autogenpb:mutex` + string uuid = 1; // `autogenpb:default:b2a2de35-07b6-443b-8188-709e27bee8a7` + string version = 2; // `autogenpb:default:2` + repeated Apple apples = 3; // `autogenpb:sort` `autogenpb:unique` + repeated Pear pears = 4; // `autogenpb:sort` `autogenpb:unique` + repeated Pear more = 5; // `autogenpb:sort` `autogenpb:unique` // not supported. 'More' can only be the string 'Pears' + repeated string color = 6; // `autogenpb:sort` `autogenpb:unique` +} + +message Pear { + string name = 1; // + string favorite = 2; // `autogenpb:sort` +} + +// above is an example + +// +// below are the actual structs autogen uses +// autogen parses the .proto file and then store the information +// it needs in these protobuf files, then it processes the +// protobuf files to write out *.sort.pb.go and *.marshal.pb.go files +// +message MsgName { + // If you have: + // + // "Shelf" for msgname + // "Books" for name + // + // Then in the proto file, that would mean it would look like: + // + // message Shelf { + // and then + // repeated string Books = 42; + // + // autogenpb will then generate sort functions for each 'name' + // things like: + // + // for _, b := range all.Book { + // + // and sort functions like: + // + // func (a ShelfBook) Less(i, j int) bool { return a[i].Book < a[j].Book } + // + + string name = 1; // the name of the message aka struct. for this example: "Shelf" + bool marshal = 2; // if "Shelf" should have Marshal & Unmarshal functions + bool mutex = 3; // an experiment to insert a mutex into the protoc generated msg struct (bad idea?) + repeated string sort = 4; // "Book", "Picture", etc + repeated string aq = 5; // if the fields should have AppendUnique() functions + repeated string uniq = 6; // the non-repeating fields that should be unique +} + +message Unique { + string name = 1; // the variable name of the repeatable struct that must be unique + string msgName = 2; // the struct that is repeated + repeated string keys = 3; // the variables in that structure to check are unique +} + +message File { // `autogenpb:nomarshal` + string name = 1; // for this one: autogen.proto + string uuid = 2; // the uuid to use in a func NewMsgName() + int64 version = 3; // the version to use in a func NewMsgName() + + // in this proto file, this would have "Apple", "Apples", ... "File", etc... + repeated MsgName msgNames = 4; // `autogenpb:unique` // in this file +} + +// I know, I know, the whole point of using protobuf +// is so you don't need a uuid or versions because it's +// inherently forward compatable. nonetheless, a simple stubbed out +// trivial and empty protobuf message can marshal and identify all the files +// also, this could be used to modify /usr/bin/file /usr/share/magic to identify the files +// maybe this is already been done and is pointless, but it seems like a good idea +message Files { // `autogenpb:marshal` + string uuid = 1; // `autogenpb:uuid:fakeuuid` + string version = 2; // `autogenpb:id:42` + repeated File Files = 3; // an array of each .proto file in the working directory +} + +// this generic message is used by autogen to identify and +// then dump the uuid and version from any arbitrary .pb file +message Identify { // `autogenpb:marshal` + string uuid = 1; // + int64 version = 2; // +} diff --git a/example/fruit.New.go b/example/fruit.New.go new file mode 100644 index 0000000..c00f981 --- /dev/null +++ b/example/fruit.New.go @@ -0,0 +1,7 @@ +package main + +func NewFruits() *Fruits { + x := new(Fruits) + x.Uuid = "test" + return x +} diff --git a/example/fruit.proto b/example/fruit.proto new file mode 100644 index 0000000..821ae6c --- /dev/null +++ b/example/fruit.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +// this file is called "fruit.proto" +// +// for autogenpb to work, you must have: +// +// "Fruit" must exist. you can put anything in it +// +// and +// +// "Fruits" MUST EXIST and start exactly this way +// It must be "Fruit" + 's' and must match the name of this file: "fruit.proto" + +package fruit; + +import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp + +message Apple { // `autogenpb:marshal` + string name = 1; // `autogenpb:unique` // generates SortByxxx() and AppendUnique() functions + string genus = 2; // `autogenpb:unique` // generates same thing here but SortByGenus() + google.protobuf.Timestamp ctime = 3; // when the apple was born +} + +message Pear { + string name = 1; // + string favorite = 2; // `autogenpb:sort` +} + +// "Fruit" must exist. you can put anything in it +message Fruit { // `autogenpb:marshal` + string brand = 1; // `autogenpb:unique` + repeated Apple apples = 2; + repeated Pear pears = 3; + string UPC = 4; // `autogenpb:sort` `autogenpb:unique` + string city = 5; // `autogenpb:sort` +} + +// "Fruits" MUST EXIST and start exactly this way +// It must be "Fruit" + 's' and must match the name of this file: "fruit.proto" +message Fruits { // `autogenpb:marshal` `autogenpb:mutex` + string uuid = 1; // `autogenpb:uuid:be926ad9-f07f-484c-adf2-d96eeabf3079` + string version = 2; // `autogenpb:version:v0.0.1` + repeated Fruit Fruits = 3; // THIS MUST BE "Fruit" and then "Fruit" + "s" + // you can add additional things here but the three lines above must conform to the standard above + int64 cost = 4; +} diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..47dad4f --- /dev/null +++ b/example/main.go @@ -0,0 +1,21 @@ +//go:build go1.20 +// +build go1.20 + +package main + +import "go.wit.com/log" + +// sent via -ldflags +var VERSION string +var BUILDTIME string + +var sortmap map[string]string +var marshalKeys []string +var uniqueKeys []string + +var pb *Fruits + +func main() { + pb = NewFruits() + log.Info("did nothing yet", pb) +} diff --git a/newsort.go b/newsort.go index aa6a8c2..1fba9be 100644 --- a/newsort.go +++ b/newsort.go @@ -14,16 +14,19 @@ func (pb *Files) makeNewSortfile(pf *File) { for _, msg := range pf.MsgNames { if msg.DoMutex { - pf.syncLock(f, msg.Lockname) - pf.iterTop(f, msg.Name) + msg.syncLock(f, msg.Lockname) + for _, key := range msg.Sort { + pf.iterTop(f, key) + pf.iterNext(f, key) + } } else { log.Info("Skipping syncLock() for", msg.Name, "DoMutex = false") } + pf.appendUnique(f, msg, sortmap) // Append() enforce no unique keys } - iterNext(f, sortmap) - iterAppend(f, sortmap) // Append() enforce no unique keys - iterSortAll(f, sortmap) + return + // iterSortAll(f, sortmap) if argv.Append != "" { sortmap["append"] = string(argv.Append) diff --git a/protoParse.go b/protoParse.go index 8c36e57..31e2f6d 100644 --- a/protoParse.go +++ b/protoParse.go @@ -120,16 +120,6 @@ func (pb *Files) protoParse(f *File) error { // don't sort anything (don't make foo.sort.pb.go) argv.NoSort = true } - if strings.Contains(line, "autogenpb:mutex") { - // try the mutex hack - argv.Mutex = true - } - if strings.Contains(line, "autogenpb:gover:") { - // todo: parse the output here - parts := strings.Split(line, "autogenpb:gover:") - log.Info("found gover:", parts[1]) - argv.Mutex = true - } } return nil } diff --git a/sort.go b/sort.go index b098897..8fa8e79 100644 --- a/sort.go +++ b/sort.go @@ -11,10 +11,10 @@ import ( func (pb *Files) makeSortfile(pf *File) { f, _ := os.OpenFile(pf.Filebase+".sort.pb.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - header(f, pf) + // header(f, pf) - pf.iterTop(f, sortmap["base"]) - iterNext(f, sortmap) + // pf.iterTop(f, sortmap["base"]) + // iterNext(f, sortmap) iterAppend(f, sortmap) // Append() enforce no unique keys iterSortAll(f, sortmap) @@ -55,8 +55,8 @@ func (pb *Files) makeSortfile(pf *File) { iterEnd(f, sortmap) } -func (pf *File) syncLock(w io.Writer, lock string) { - var LOCK string = lock +func (msg *MsgName) syncLock(w io.Writer, s string) { + var LOCK string = msg.Name fmt.Fprintln(w, "// bad global lock until modifying the .pb.go file is tested") fmt.Fprintln(w, "// sync.RWMutex or sync.Mutex?") @@ -64,9 +64,7 @@ func (pf *File) syncLock(w io.Writer, lock string) { fmt.Fprintln(w, "") } -func (pf *File) iterTop(w io.Writer, base string) { - var BASE string = base - +func (pf *File) iterTop(w io.Writer, BASE string) { fmt.Fprintln(w, "type "+BASE+"Iterator struct {") fmt.Fprintln(w, " sync.RWMutex") fmt.Fprintln(w, "") @@ -96,9 +94,9 @@ func (pf *File) iterTop(w io.Writer, base string) { fmt.Fprintln(w, "") } -func iterNext(w io.Writer, names map[string]string) { +func (pf *File) iterNext(w io.Writer, BASE string) { fmt.Fprintln(w, "// Next() returns the next thing in the array") - fmt.Fprintln(w, "func (it *"+names["Base"]+"Iterator) Next() *"+names["Base"]+" {") + fmt.Fprintln(w, "func (it *"+BASE+"Iterator) Next() *"+BASE+" {") fmt.Fprintln(w, " if it.things[it.index-1] == nil {") fmt.Fprintln(w, " for i, d := range it.things {") fmt.Fprintln(w, " fmt.Println(\"i =\", i, d)") @@ -175,34 +173,32 @@ func iterEnd(w io.Writer, names map[string]string) { } func iterAppend(w io.Writer, names map[string]string) { - if names["append"] == "" { - fmt.Fprintln(w, "// does not enforce any unique fields") - } else { - fmt.Fprintln(w, "// enforces "+names["append"]+" is unique") - } - if names["append"] == "" { - fmt.Fprintln(w, "func (all *"+names["Bases"]+") Append(newP *"+names["Base"]+") bool {") - } else { - // fmt.Fprintln(w, "func (all *"+names["Bases"]+") Append(newP *"+names["Base"]+") bool { // todo: make unique name here") - fmt.Fprintln(w, "func (all *"+names["Bases"]+") AppendUnique"+names["append"]+"(newP *"+names["Base"]+") bool {") - } - if sortmap["lock"] == "all" { - fmt.Fprintln(w, " "+names["lock"]+".Lock.RLock()") - fmt.Fprintln(w, " defer "+names["lock"]+".Lock.RUnlock()") +} + +func (pf *File) appendUnique(w io.Writer, msg *MsgName, names map[string]string) { + var MSG string = msg.Name // msg.Name + var BASE string = names["Base"] + var LOCK string = names["lock"] + if argv.Mutex { + LOCK = "Lock" } else { - fmt.Fprintln(w, " "+names["lock"]+".RLock()") - fmt.Fprintln(w, " defer "+names["lock"]+".RUnlock()") + LOCK = names["lock"] + ".Lock" } + + fmt.Fprintln(w, "// enforces "+BASE+" is unique") + fmt.Fprintln(w, "func (all *"+MSG+") AppendUnique(newP *"+BASE+") bool {") + fmt.Fprintln(w, " "+LOCK+".RLock()") + fmt.Fprintln(w, " defer "+LOCK+".RUnlock()") fmt.Fprintln(w, "") - if names["append"] != "" { - fmt.Fprintln(w, " for _, p := range all."+names["Bases"]+" {") - fmt.Fprintln(w, " if p."+names["append"]+" == newP."+names["append"]+" {") + fmt.Fprintln(w, " for _, p := range all."+MSG+" {") + for _, KEY := range msg.Unique { + fmt.Fprintln(w, " if p."+KEY+" == newP."+KEY+" {") fmt.Fprintln(w, " return false") fmt.Fprintln(w, " }") - fmt.Fprintln(w, " }") - fmt.Fprintln(w, "") } - fmt.Fprintln(w, " all."+names["Bases"]+" = append(all."+names["Bases"]+", newP)") + fmt.Fprintln(w, " }") + fmt.Fprintln(w, "") + fmt.Fprintln(w, " all."+MSG+" = append(all."+MSG+", newP)") fmt.Fprintln(w, " return true") fmt.Fprintln(w, "}") fmt.Fprintln(w, "") diff --git a/testfiles/Makefile b/testfiles/Makefile deleted file mode 100644 index bc4405b..0000000 --- a/testfiles/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -VERSION = $(shell git describe --tags) -BUILDTIME = $(shell date +%Y.%m.%d_%H%M) - -full: clean goimports auto vet build - ./testfiles - -vet: - @GO111MODULE=off go vet - @echo this go binary package should build okay - -build: - GO111MODULE=off go build - -auto: - ../autogenpb --proto auto.proto --package main - -goimports: - goimports -w *.go - -clean: - -rm -f go.* - -rm -f *.pb.go - -rm -f testfiles diff --git a/testfiles/auto.proto b/testfiles/auto.proto deleted file mode 100644 index b2166e8..0000000 --- a/testfiles/auto.proto +++ /dev/null @@ -1,99 +0,0 @@ -syntax = "proto3"; - -// here are some docs, but probably it's just easier to run -// autogenpb on this file and see what gets autogenerated -// in this directory. All autogenerated files are named *.pb.go - -// the 'uuid' standard at the end is an experiment -// establish a way to identify arbitrary .pb files - -// You can generate Marshal & Unmarshal for any struct (message) you want -// You can generate SortBy and Append functions ONLY FOR 'repeated ' -// Also, those structs must be defined in the same file -// Additionally, you must use `autogenpb:mutex` on the parent struct. -// The autogenerated code requires a RW mutex and autogenpb will insert it into the struct - -package main; - -message Apple { // `autogenpb:marshal` - string name = 1; // `autogenpb:unique` // generates SortByxxx() and AppendUnique() functions - string genus = 2; // `autogenpb:unique` // generates same thing here but SortByGenus() -} - -message Apples { // `autogenpb:marshal` `autogenpb:mutex` - string uuid = 1; // `autogenpb:default:b2a2de35-07b6-443b-8188-709e27bee8a7` - string version = 2; // `autogenpb:default:2` - repeated Apple Apples = 3; // `autogenpb:sort` - repeated Pears More = 4; // `autogenpb:sort` - repeated string Color = 5; // `autogenpb:sort` `autogenpb:unique` -} - -message Pears { - string name = 1; // - string favorite = 2; // `autogenpb:sort` -} - -// above is an example - -// -// below are the actual structs autogen uses -// autogen parses the .proto file and then store the information -// it needs in these protobuf files, then it processes the -// protobuf files to write out *.sort.pb.go and *.marshal.pb.go files -// -message MsgName { - // If you have: - // - // "Shelf" for msgname - // "Books" for name - // - // Then in the proto file, that would mean it would look like: - // - // message Shelf { - // and then - // repeated string Books = 42; - // - // autogenpb will then generate sort functions for each 'name' - // things like: - // - // for _, b := range all.Book { - // - // and sort functions like: - // - // func (a ShelfBook) Less(i, j int) bool { return a[i].Book < a[j].Book } - // - - string name = 1; // the name of the message aka struct. for this example: "Shelf" - bool marshal = 2; // if "Shelf" should have Marshal & Unmarshal functions - bool mutex = 3; // an experiment to insert a mutex into the protoc generated msg struct (bad idea?) - repeated string sort = 4; // "Book", "Picture", etc - repeated string unique = 5; // if the fields should have AppendUnique() functions -} - -message File { // `autogenpb:nomarshal` - string name = 1; // for this one: autogen.proto - string uuid = 2; // the uuid to use in a func NewMsgName() - int64 version = 3; // the version to use in a func NewMsgName() - - // in this proto file, this would have "Apple", "Apples", ... "File", etc... - repeated MsgName msgNames = 4; // `autogenpb:unique` // in this file -} - -// I know, I know, the whole point of using protobuf -// is so you don't need a uuid or versions because it's -// inherently forward compatable. nonetheless, a simple stubbed out -// trivial and empty protobuf message can marshal and identify all the files -// also, this could be used to modify /usr/bin/file /usr/share/magic to identify the files -// maybe this is already been done and is pointless, but it seems like a good idea -message Files { // `autogenpb:marshal` - string uuid = 1; // `autogenpb:uuid:fakeuuid` - int64 version = 2; // `autogenpb:id:42` - repeated File Files = 3; // an array of each .proto file in the working directory -} - -// this generic message is used by autogen to identify and -// then dump the uuid and version from any arbitrary .pb file -message Identify { // `autogenpb:marshal` - string uuid = 1; // - int64 version = 2; // -} diff --git a/testfiles/main.go b/testfiles/main.go deleted file mode 100644 index b67f3d8..0000000 --- a/testfiles/main.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build go1.20 -// +build go1.20 - -package main - -import "go.wit.com/log" - -// sent via -ldflags -var VERSION string -var BUILDTIME string - -var sortmap map[string]string -var marshalKeys []string -var uniqueKeys []string - -var pb *Files - -func main() { - - pb = new(Files) - log.Info("did nothing yet") -} -- cgit v1.2.3