Modify existing yaml file and add new data and comments
You could create a new node and directly append to the contents, without deleting the previous node. The following example illustrates this point:
package main
import (
"fmt"
"log"
"gopkg.in/yaml.v3"
)
var (
sourceYaml = `version: 1
type: verbose
kind : bfr
# my list of applications
applications:
# First app
- name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
`
)
type Application struct {
Name string `yaml:"name,omitempty" json:"name,omitempty"`
Kind string `yaml:"kind,omitempty" json:"kind,omitempty"`
Path string `yaml:"path,omitempty" json:"path,omitempty"`
Exec struct {
Platforms string `yaml:"platforms,omitempty" json:"platforms,omitempty"`
Builder string `yaml:"builder,omitempty" json:"builder,omitempty"`
} `yaml:"exec,omitempty" json:"exec,omitempty"`
}
func newApplicationNode(
name string,
kind string,
path string,
platforms string,
builder string,
comment string) (*yaml.Node, error) {
app := Application{
Name: name,
Kind: kind,
Path: path,
Exec: struct {
Platforms string `yaml:"platforms,omitempty" json:"platforms,omitempty"`
Builder string `yaml:"builder,omitempty" json:"builder,omitempty"`
}{platforms, builder},
}
marshalledApp, err := yaml.Marshal(&app)
if err != nil {
return nil, err
}
node := yaml.Node{}
if err := yaml.Unmarshal(marshalledApp, &node); err != nil {
return nil, err
}
node.Content[0].HeadComment = comment
return &node, nil
}
func main() {
yamlNode := yaml.Node{}
err := yaml.Unmarshal([]byte(sourceYaml), &yamlNode)
if err != nil {
log.Fatalf("error: %v", err)
}
newApp, err := newApplicationNode("app2", "golang", "app2", "dockerh",
"test", "Second app")
if err != nil {
log.Fatalf("error: %v", err)
}
appIdx := -1
for i, k := range yamlNode.Content[0].Content {
if k.Value == "applications" {
appIdx = i + 1
break
}
}
yamlNode.Content[0].Content[appIdx].Content = append(
yamlNode.Content[0].Content[appIdx].Content, newApp.Content[0])
out, err := yaml.Marshal(&yamlNode)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(out))
}
Clearly instead of going the hacky way as I did in newApplicationNode you can unmarshal properly from your JSON. However, as stated in previous answers, it is important to notice that the key and actual value are in subsequent indexes inside Content, therefore you need to take this into account when modifying the document. (e.g., look up applications key but then consider the next index (appIdx = i + 1 in my example) for its contents.
Hope that helps!
First, let Me Start off by saying using yaml.Node does not produce valid yaml when marshalled from a valid yaml, given by the following example. Probably should file an issue.
package main
import (
"fmt"
"log"
"gopkg.in/yaml.v3"
)
var (
sourceYaml = `version: 1
type: verbose
kind : bfr
# my list of applications
applications:
# First app
- name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
`
)
func main() {
t := yaml.Node{}
err := yaml.Unmarshal([]byte(sourceYaml), &t)
if err != nil {
log.Fatalf("error: %v", err)
}
b, err := yaml.Marshal(&t)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(b))
}
Produces the following invalid yaml in go version go1.12.3 windows/amd64
version: 1
type: verbose
kind: bfr
# my list of applications
applications:
- # First app
name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
Secondly, using a struct such as
type VTS struct {
Version string `yaml:"version" json:"version"`
Types string `yaml:"type" json:"type"`
Kind string `yaml:"kind,omitempty" json:"kind,omitempty"`
Apps yaml.Node `yaml:"applications,omitempty" json:"applications,omitempty"`
}
From ubuntu's blog and the source documentation it made it seem that it would correctly identify fields within the struct that are nodes and build that tree separately, but that is not the case. When unmarshalled, it will give a correct node tree, but when remarshalled it will produce the following yaml with all of the fields that yaml.Node exposes. Sadly we cannot go this route, must find another way.
version: "1"
type: verbose
kind: bfr
applications:
kind: 2
style: 0
tag: '!!seq'
value: ""
anchor: ""
alias: null
content:
- # First app
name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
headcomment: ""
linecomment: ""
footcomment: ""
line: 9
column: 3
Overlooking the first issue and the marshal bug for yaml.Nodes in a struct (on gopkg.in/yaml.v3 v3.0.0-20190409140830-cdc409dda467) we can now go about manipulating the Nodes that the package exposes. Unfortunately, there is no abstraction that will add Nodes with ease, so uses might vary and identifying nodes can be a pain. Reflection might help here a bit, so I leave that as an exercise for you.
You will find comment spew.Dumps that dump the entire node Tree in a nice format, this helped with debugging when adding Nodes to the source tree.
You can certainly remove nodes as well, you will just need to identify which particular nodes that need to be removed. You just have to ensure that you remove the parent nodes if it were a map or sequence.
package main
import (
"encoding/json"
"fmt"
"log"
"gopkg.in/yaml.v3"
)
var (
sourceYaml = `version: 1
type: verbose
kind : bfr
# my list of applications
applications:
# First app
- name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
`
modifyJsonSource = `
[
{
"comment": "Second app",
"name": "app2",
"kind": "golang",
"path": "app2",
"exec": {
"platforms": "dockerh",
"builder": "test"
}
}
]
`
)
// VTS Need to Make Fields Public otherwise unmarshalling will not fill in the unexported fields.
type VTS struct {
Version string `yaml:"version" json:"version"`
Types string `yaml:"type" json:"type"`
Kind string `yaml:"kind,omitempty" json:"kind,omitempty"`
Apps Applications `yaml:"applications,omitempty" json:"applications,omitempty"`
}
type Applications []struct {
Name string `yaml:"name,omitempty" json:"name,omitempty"`
Kind string `yaml:"kind,omitempty" json:"kind,omitempty"`
Path string `yaml:"path,omitempty" json:"path,omitempty"`
Exec struct {
Platforms string `yaml:"platforms,omitempty" json:"platforms,omitempty"`
Builder string `yaml:"builder,omitempty" json:"builder,omitempty"`
} `yaml:"exec,omitempty" json:"exec,omitempty"`
Comment string `yaml:"comment,omitempty" json:"comment,omitempty"`
}
func main() {
t := yaml.Node{}
err := yaml.Unmarshal([]byte(sourceYaml), &t)
if err != nil {
log.Fatalf("error: %v", err)
}
// Look for the Map Node with the seq array of items
applicationNode := iterateNode(&t, "applications")
// spew.Dump(iterateNode(&t, "applications"))
var addFromJson Applications
err = json.Unmarshal([]byte(modifyJsonSource), &addFromJson)
if err != nil {
log.Fatalf("error: %v", err)
}
// Delete the Original Applications the following options:
// applicationNode.Content = []*yaml.Node{}
// deleteAllContents(applicationNode)
deleteApplication(applicationNode, "name", "app1")
for _, app := range addFromJson {
// Build New Map Node for new sequences coming in from json
mapNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
// Build Name, Kind, and Path Nodes
mapNode.Content = append(mapNode.Content, buildStringNodes("name", app.Name, app.Comment)...)
mapNode.Content = append(mapNode.Content, buildStringNodes("kind", app.Kind, "")...)
mapNode.Content = append(mapNode.Content, buildStringNodes("path", app.Path, "")...)
// Build the Exec Nodes and the Platform and Builder Nodes within it
keyMapNode, keyMapValuesNode := buildMapNodes("exec")
keyMapValuesNode.Content = append(keyMapValuesNode.Content, buildStringNodes("platform", app.Exec.Platforms, "")...)
keyMapValuesNode.Content = append(keyMapValuesNode.Content, buildStringNodes("builder", app.Exec.Builder, "")...)
// Add to parent map Node
mapNode.Content = append(mapNode.Content, keyMapNode, keyMapValuesNode)
// Add to applications Node
applicationNode.Content = append(applicationNode.Content, mapNode)
}
// spew.Dump(t)
b, err := yaml.Marshal(&t)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(b))
}
// iterateNode will recursive look for the node following the identifier Node,
// as go-yaml has a node for the key and the value itself
// we want to manipulate the value Node
func iterateNode(node *yaml.Node, identifier string) *yaml.Node {
returnNode := false
for _, n := range node.Content {
if n.Value == identifier {
returnNode = true
continue
}
if returnNode {
return n
}
if len(n.Content) > 0 {
ac_node := iterateNode(n, identifier)
if ac_node != nil {
return ac_node
}
}
}
return nil
}
// deleteAllContents will remove all the contents of a node
// Mark sure to pass the correct node in otherwise bad things will happen
func deleteAllContents(node *yaml.Node) {
node.Content = []*yaml.Node{}
}
// deleteApplication expects that a sequence Node with all the applications are present
// if the key value are not found it will not log any errors, and return silently
// this is expecting a map like structure for the applications
func deleteApplication(node *yaml.Node, key, value string) {
state := -1
indexRemove := -1
for index, parentNode := range node.Content {
for _, childNode := range parentNode.Content {
if key == childNode.Value && state == -1 {
state += 1
continue // found expected move onto next
}
if value == childNode.Value && state == 0 {
state += 1
indexRemove = index
break // found the target exit out of the loop
} else if state == 0 {
state = -1
}
}
}
if state == 1 {
// Remove node from contents
// node.Content = append(node.Content[:indexRemove], node.Content[indexRemove+1:]...)
// Don't Do this you might have a potential memory leak source: https://github.com/golang/go/wiki/SliceTricks
// Since the underlying nodes are pointers
length := len(node.Content)
copy(node.Content[indexRemove:], node.Content[indexRemove+1:])
node.Content[length-1] = nil
node.Content = node.Content[:length-1]
}
}
// buildStringNodes builds Nodes for a single key: value instance
func buildStringNodes(key, value, comment string) []*yaml.Node {
keyNode := &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: key,
HeadComment: comment,
}
valueNode := &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: value,
}
return []*yaml.Node{keyNode, valueNode}
}
// buildMapNodes builds Nodes for a key: map instance
func buildMapNodes(key string) (*yaml.Node, *yaml.Node) {
n1, n2 := &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: key,
}, &yaml.Node{Kind: yaml.MappingNode,
Tag: "!!map",
}
return n1, n2
}
Produces yaml
version: 1
type: verbose
kind: bfr
# my list of applications
applications:
- # First app
name: app1
kind: nodejs
path: app1
exec:
platforms: k8s
builder: test
- # Second app
name: app2
kind: golang
path: app2
exec:
platform: dockerh
builder: test