blob: b42a78872c435db9daa8090adaecc0f559f3d44b [file] [log] [blame]
Marcel van Lohuizenea40e662018-12-11 18:07:49 +01001// Copyright 2018 The CUE Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package cmd
16
17import (
18 "bytes"
19 "encoding/json"
20 "errors"
21 "fmt"
22 "io"
23 "io/ioutil"
24 "os"
25 "path/filepath"
26 "regexp"
27 "strconv"
28 "strings"
29 "unicode"
30
31 "cuelang.org/go/cue"
32 "cuelang.org/go/cue/ast"
33 "cuelang.org/go/cue/encoding"
34 "cuelang.org/go/cue/format"
35 "cuelang.org/go/cue/load"
36 "cuelang.org/go/cue/parser"
37 "cuelang.org/go/cue/token"
38 "cuelang.org/go/internal"
39 "cuelang.org/go/internal/third_party/yaml"
40 "github.com/spf13/cobra"
41 "golang.org/x/sync/errgroup"
42)
43
44// importCmd represents the import command
45var importCmd = &cobra.Command{
46 Use: "import",
47 Short: "convert other data formats to CUE files",
48 Long: `import converts other data formats, like JSON and YAML to CUE files
49
50The following file formats are currently supported:
51
52 Format Extensions
53 JSON .json .jsonl .ndjson
54 YAML .yaml .yml
55
56Files can either be specified explicitly, or inferred from the specified
57packages. In either case, the file extension is replaced with .cue. It will
58fail if the file already exists by default. The -f flag overrides this.
59
60Examples:
61
62 # Convert individual files:
63 $ cue import foo.json bar.json # create foo.yaml and bar.yaml
64
65 # Convert all json files in the indicated directories:
66 $ cue import ./... -type=json
67
68
69The -path flag
70
71By default the parsed files are included as emit values. This default can be
72overridden by specifying a sequence of labels as you would in a CUE file.
73An identifier or string label are interpreted as usual. A label expression is
74evaluated within the context of the imported file. label expressions may also
75refer to builtin packages, which will be implicitly imported.
76
77
78Handling multiple documents or streams
79
80To handle Multi-document files, such as concatenated JSON objects or
81YAML files with document separators (---) the user must specify either the
82-path, -list, or -files flag. The -path flag assign each element to a path
83(identical paths are treated as usual); -list concatenates the entries, and
84-files causes each entry to be written to a different file. The -files flag
85may only be used if files are explicitly imported. The -list flag may be
86used in combination with the -path flag, concatenating each entry to the
87mapped location.
88
89
90Examples:
91
92 $ cat <<EOF > foo.yaml
93 kind: Service
94 name: booster
95 EOF
96
97 # include the parsed file as an emit value:
98 $ cue import foo.yaml
99 $ cat foo.cue
100 {
101 kind: Service
102 name: booster
103 }
104
105 # include the parsed file at the root of the CUE file:
106 $ cue import -f -l "" foo.yaml
107 $ cat foo.cue
108 kind: Service
109 name: booster
110
111 # include the import config at the mystuff path
112 $ cue import -f -l mystuff foo.yaml
113 $ cat foo.cue
114 myStuff: {
115 kind: Service
116 name: booster
117 }
118
119 # append another object to the input file
120 $ cat <<EOF >> foo.yaml
121 ---
122 kind: Deployment
123 name: booster
124 replicas: 1
125
126 # base the path values on th input
127 $ cue import -f -l '"\(strings.ToLower(kind))" "\(x.name)"' foo.yaml
128 $ cat foo.cue
129 service booster: {
130 kind: "Service"
131 name: "booster"
132 }
133
134 deployment booster: {
135 kind: "Deployment"
136 name: "booster
137 replicas: 1
138 }
139
140 # base the path values on th input
141 $ cue import -f -list -foo.yaml
142 $ cat foo.cue
143 [{
144 kind: "Service"
145 name: "booster"
146 }, {
147 kind: "Deployment"
148 name: "booster
149 replicas: 1
150 }]
151
152 # base the path values on th input
153 $ cue import -f -list -l '"\(strings.ToLower(kind))"' foo.yaml
154 $ cat foo.cue
155 service: [{
156 kind: "Service"
157 name: "booster"
158 }
159 deployment: [{
160 kind: "Deployment"
161 name: "booster
162 replicas: 1
163 }]
164
165
166Embedded data files
167
168The --recursive or -R flag enables the parsing of fields that are string
169representations of data formats themselves. A field that can be parsed is
170replaced with a call encoding the data from a structured form that is placed
171in a sibling field.
172
173It is also possible to recursively hoist data formats:
174
175Example:
176 $ cat <<EOF > example.json
177 "a": {
178 "data": '{ "foo": 1, "bar": 2 }',
179 }
180 EOF
181
182 $ cue import -R example.json
183 $ cat example.cue
184 import "encode/json"
185
186 a: {
187 data: json.Encode(_data),
188 _data: {
189 foo: 1
190 bar: 2
191 }
192 }
193`,
194 RunE: runImport,
195}
196
197func init() {
198 RootCmd.AddCommand(importCmd)
199
200 out = importCmd.Flags().StringP("out", "o", "", "alternative output or - for stdout")
201 name = importCmd.Flags().StringP("name", "n", "", "glob filter for file names")
202 typ = importCmd.Flags().String("type", "", "only apply to files of this type")
203 force = importCmd.Flags().BoolP("force", "f", false, "force overwriting existing files")
204 dryrun = importCmd.Flags().Bool("dryrun", false, "force overwriting existing files")
205
206 node = importCmd.Flags().StringP("path", "l", "", "path to include root")
207 list = importCmd.Flags().Bool("list", false, "concatenate multiple objects into a list")
208 files = importCmd.Flags().Bool("files", false, "split multiple entries into different files")
209 parseStrings = importCmd.Flags().BoolP("recursive", "R", false, "recursively parse string values")
210
211 importCmd.Flags().String("fix", "", "apply given fix")
212}
213
214var (
215 force *bool
216 name *string
217 typ *string
218 node *string
219 out *string
220 dryrun *bool
221 list *bool
222 files *bool
223 parseStrings *bool
224)
225
226type importFunc func(path string, r io.Reader) ([]ast.Expr, error)
227
228type encodingInfo struct {
229 fn importFunc
230 typ string
231}
232
233var (
234 jsonEnc = &encodingInfo{handleJSON, "json"}
235 yamlEnc = &encodingInfo{handleYAML, "yaml"}
236)
237
238func getExtInfo(ext string) *encodingInfo {
239 enc := encoding.MapExtension(ext)
240 if enc == nil {
241 return nil
242 }
243 switch enc.Name() {
244 case "json":
245 return jsonEnc
246 case "yaml":
247 return yamlEnc
248 }
249 return nil
250}
251
252func runImport(cmd *cobra.Command, args []string) error {
253 log.SetOutput(cmd.OutOrStderr())
254
255 var group errgroup.Group
256
257 group.Go(func() error {
258 if len(args) > 0 && len(filepath.Ext(args[0])) > len(".") {
259 for _, a := range args {
260 group.Go(func() error { return handleFile(cmd, *fPackage, a) })
261 }
262 return nil
263 }
264
265 done := map[string]bool{}
266
267 inst := load.Instances(args, &load.Config{DataFiles: true})
268 for _, pkg := range inst {
269 pkgName := *fPackage
270 if pkgName == "" {
271 pkgName = pkg.PkgName
272 }
273 if pkgName == "" && len(inst) > 1 {
274 return fmt.Errorf("must specify package name with the -p flag")
275 }
276 dir := pkg.Dir
277 if err := pkg.Err; err != nil {
278 return err
279 }
280 if done[dir] {
281 continue
282 }
283 done[dir] = true
284
285 files, err := ioutil.ReadDir(dir)
286 if err != nil {
287 return err
288 }
289 for _, file := range files {
290 ext := filepath.Ext(file.Name())
291 if enc := getExtInfo(ext); enc == nil || (*typ != "" && *typ != enc.typ) {
292 continue
293 }
294 path := filepath.Join(dir, file.Name())
295 group.Go(func() error { return handleFile(cmd, pkgName, path) })
296 }
297 }
298 return nil
299 })
300
301 err := group.Wait()
302 if err != nil {
303 return fmt.Errorf("Import failed: %v", err)
304 }
305 return nil
306}
307
308func handleFile(cmd *cobra.Command, pkg, filename string) error {
309 re, err := regexp.Compile(*name)
310 if err != nil {
311 return err
312 }
313 if !re.MatchString(filepath.Base(filename)) {
314 return nil
315 }
316 f, err := os.Open(filename)
317 if err != nil {
318 return fmt.Errorf("error opening file: %v", err)
319 }
320 defer f.Close()
321
322 ext := filepath.Ext(filename)
323 handler := getExtInfo(ext)
324
325 if handler == nil {
326 return fmt.Errorf("unsupported extension %q", ext)
327 }
328 objs, err := handler.fn(filename, f)
329 if err != nil {
330 return err
331 }
332
333 if *files {
334 for i, f := range objs {
335 err := combineExpressions(cmd, pkg, newName(filename, i), f)
336 if err != nil {
337 return err
338 }
339 }
340 return nil
341 } else if len(objs) > 1 {
342 if !*list && *node == "" && !*files {
343 return fmt.Errorf("list, flag, or files flag needed to handle multiple objects in file %q", filename)
344 }
345 }
346 return combineExpressions(cmd, pkg, newName(filename, 0), objs...)
347}
348
349func combineExpressions(cmd *cobra.Command, pkg, cueFile string, objs ...ast.Expr) error {
350 if *out != "" {
351 cueFile = *out
352 }
353 if cueFile != "-" {
354 switch _, err := os.Stat(cueFile); {
355 case os.IsNotExist(err):
356 case err == nil:
357 if !*force {
358 log.Printf("skipping file %q: already exists", cueFile)
359 return nil
360 }
361 default:
362 return fmt.Errorf("error creating file: %v", err)
363 }
364 }
365
366 f := &ast.File{}
367 if pkg != "" {
368 f.Name = ast.NewIdent(pkg)
369 }
370
371 h := hoister{
372 fields: map[string]bool{},
373 altNames: map[string]*ast.Ident{},
374 }
375
376 index := newIndex()
377 for _, expr := range objs {
378 if *parseStrings {
379 h.hoist(expr)
380 }
381
382 // Compute a path different from root.
383 var pathElems []ast.Label
384
385 switch {
386 case *node != "":
387 inst, err := cue.FromExpr(nil, expr)
388 if err != nil {
389 return err
390 }
391
392 labels, err := parsePath(*node)
393 if err != nil {
394 return err
395 }
396 for _, l := range labels {
397 switch x := l.(type) {
398 case *ast.Interpolation:
399 v := inst.Eval(x)
400 if v.Kind() == cue.BottomKind {
401 return v.Err()
402 }
403 pathElems = append(pathElems, v.Syntax().(ast.Label))
404
405 case *ast.Ident, *ast.BasicLit:
406 pathElems = append(pathElems, x)
407
Marcel van Lohuizenea40e662018-12-11 18:07:49 +0100408 case *ast.TemplateLabel:
409 return fmt.Errorf("template labels not supported in path flag")
410 }
411 }
412 }
413
414 if *list {
415 idx := index
416 for _, e := range pathElems {
417 idx = idx.label(e)
418 }
419 if idx.field.Value == nil {
420 idx.field.Value = &ast.ListLit{
421 Lbrack: token.Pos(token.NoSpace),
422 Rbrack: token.Pos(token.NoSpace),
423 }
424 }
425 list := idx.field.Value.(*ast.ListLit)
426 list.Elts = append(list.Elts, expr)
427 } else if len(pathElems) == 0 {
428 obj, ok := expr.(*ast.StructLit)
429 if !ok {
430 return fmt.Errorf("cannot map non-struct to object root")
431 }
432 f.Decls = append(f.Decls, obj.Elts...)
433 } else {
434 field := &ast.Field{Label: pathElems[0]}
435 f.Decls = append(f.Decls, field)
436 for _, e := range pathElems[1:] {
437 newField := &ast.Field{Label: e}
438 newVal := &ast.StructLit{Elts: []ast.Decl{newField}}
439 field.Value = newVal
440 field = newField
441 }
442 field.Value = expr
443 }
444 }
445
446 if len(h.altNames) > 0 {
447 imports := &ast.ImportDecl{}
448
449 for _, enc := range encoding.All() {
450 if ident, ok := h.altNames[enc.Name()]; ok {
451 short := enc.Name()
452 name := h.uniqueName(short, "")
453 ident.Name = name
454 if name == short {
455 ident = nil
456 }
457
458 path := fmt.Sprintf(`"encoding/%s"`, short)
459 imports.Specs = append(imports.Specs, &ast.ImportSpec{
460 Name: ident,
461 Path: &ast.BasicLit{Kind: token.STRING, Value: path},
462 })
463 }
464 }
465 f.Decls = append([]ast.Decl{imports}, f.Decls...)
466 }
467
468 if *list {
469 switch x := index.field.Value.(type) {
470 case *ast.StructLit:
471 f.Decls = append(f.Decls, x.Elts...)
472 case *ast.ListLit:
473 f.Decls = append(f.Decls, &ast.EmitDecl{Expr: x})
474 default:
475 panic("unreachable")
476 }
477 }
478
479 var buf bytes.Buffer
480 err := format.Node(&buf, f, format.Simplify())
481 if err != nil {
482 return fmt.Errorf("error formatting file: %v", err)
483 }
484
485 if cueFile == "-" {
486 _, err := io.Copy(cmd.OutOrStdout(), &buf)
487 return err
488 }
489 return ioutil.WriteFile(cueFile, buf.Bytes(), 0644)
490}
491
492type listIndex struct {
493 index map[string]*listIndex
494 file *ast.File // top-level only
495 field *ast.Field
496}
497
498func newIndex() *listIndex {
499 return &listIndex{
500 index: map[string]*listIndex{},
501 field: &ast.Field{},
502 }
503}
504
505func newString(s string) *ast.BasicLit {
506 return &ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(s)}
507}
508
509func (x *listIndex) label(label ast.Label) *listIndex {
510 key := internal.DebugStr(label)
511 idx := x.index[key]
512 if idx == nil {
513 if x.field.Value == nil {
514 x.field.Value = &ast.StructLit{}
515 }
516 obj := x.field.Value.(*ast.StructLit)
517 newField := &ast.Field{Label: label}
518 obj.Elts = append(obj.Elts, newField)
519 idx = &listIndex{
520 index: map[string]*listIndex{},
521 field: newField,
522 }
523 x.index[key] = idx
524 }
525 return idx
526}
527
528func parsePath(exprs string) (p []ast.Label, err error) {
529 fset := token.NewFileSet()
530 f, err := parser.ParseFile(fset, "<path flag>", exprs+": _")
531 if err != nil {
532 return nil, fmt.Errorf("parser error in path %q: %v", exprs, err)
533 }
534
535 if len(f.Decls) != 1 {
536 return nil, errors.New("path flag must be a space-separated sequence of labels")
537 }
538
539 for d := f.Decls[0]; ; {
540 field, ok := d.(*ast.Field)
541 if !ok {
542 // This should never happen
543 return nil, errors.New("%q not a sequence of labels")
544 }
545
546 p = append(p, field.Label)
547
548 v, ok := field.Value.(*ast.StructLit)
549 if !ok {
550 break
551 }
552
553 if len(v.Elts) != 1 {
554 // This should never happen
555 return nil, errors.New("path value may not contain a struct")
556 }
557
558 d = v.Elts[0]
559 }
560 return p, nil
561}
562
563func newName(filename string, i int) string {
564 ext := filepath.Ext(filename)
565 filename = filename[:len(filename)-len(ext)]
566 if i > 0 {
567 filename += fmt.Sprintf("-%d", i)
568 }
569 filename += ".cue"
570 return filename
571}
572
573var fset = token.NewFileSet()
574
575func handleJSON(path string, r io.Reader) (objects []ast.Expr, err error) {
576 d := json.NewDecoder(r)
577
578 for {
579 var raw json.RawMessage
580 err := d.Decode(&raw)
581 if err == io.EOF {
582 break
583 }
584 if err != nil {
585 return nil, fmt.Errorf("could not parse JSON: %v", err)
586 }
587 expr, err := parser.ParseExpr(fset, path, []byte(raw))
588 if err != nil {
589 return nil, fmt.Errorf("invalid input: %v %q", err, raw)
590 }
591 objects = append(objects, expr)
592 }
593 return objects, nil
594}
595
596func handleYAML(path string, r io.Reader) (objects []ast.Expr, err error) {
597 d, err := yaml.NewDecoder(fset, path, r)
598 if err != nil {
599 return nil, err
600 }
601 for i := 0; ; i++ {
602 expr, err := d.Decode()
603 if err == io.EOF {
604 break
605 }
606 if err != nil {
607 return nil, err
608 }
609 objects = append(objects, expr)
610 }
611 return objects, nil
612}
613
614type hoister struct {
615 fields map[string]bool
616 altNames map[string]*ast.Ident
617}
618
619func (h *hoister) hoist(expr ast.Expr) {
620 ast.Walk(expr, nil, func(n ast.Node) {
621 name := ""
622 switch x := n.(type) {
623 case *ast.Field:
624 name, _ = ast.LabelName(x.Label)
625 case *ast.Alias:
626 name = x.Ident.Name
627 }
628 if name != "" {
629 h.fields[name] = true
630 }
631 })
632
633 ast.Walk(expr, func(n ast.Node) bool {
634 switch n.(type) {
635 case *ast.ComprehensionDecl:
636 return false
637 }
638 return true
639
640 }, func(n ast.Node) {
641 obj, ok := n.(*ast.StructLit)
642 if !ok {
643 return
644 }
645 for i := 0; i < len(obj.Elts); i++ {
646 f, ok := obj.Elts[i].(*ast.Field)
647 if !ok {
648 continue
649 }
650
651 name, ident := ast.LabelName(f.Label)
652 if name == "" || !ident {
653 continue
654 }
655
656 lit, ok := f.Value.(*ast.BasicLit)
657 if !ok || lit.Kind != token.STRING {
658 continue
659 }
660
661 str, err := cue.Unquote(lit.Value)
662 if err != nil {
663 continue
664 }
665
666 expr, enc := tryParse(str)
667 if expr == nil {
668 continue
669 }
670
671 if h.altNames[enc.typ] == nil {
672 h.altNames[enc.typ] = &ast.Ident{Name: "cue"} // set name later
673 }
674
675 // found a replacable string
676 dataField := h.uniqueName(name, "cue")
677
678 f.Value = &ast.CallExpr{
679 Fun: &ast.SelectorExpr{
680 X: h.altNames[enc.typ],
681 Sel: ast.NewIdent("Marshal"),
682 },
683 Args: []ast.Expr{
684 ast.NewIdent(dataField),
685 },
686 }
687
688 obj.Elts = append(obj.Elts, nil)
689 copy(obj.Elts[i+1:], obj.Elts[i:])
690
691 obj.Elts[i+1] = &ast.Alias{
692 Ident: ast.NewIdent(dataField),
693 Expr: expr,
694 }
695
696 h.hoist(expr)
697 }
698 })
699}
700
701func tryParse(str string) (s ast.Expr, format *encodingInfo) {
702 b := []byte(str)
703 fset := token.NewFileSet()
704 if json.Valid(b) {
705 expr, err := parser.ParseExpr(fset, "", b)
706 if err != nil {
707 // TODO: report error
708 return nil, nil
709 }
710 switch expr.(type) {
711 case *ast.StructLit, *ast.ListLit:
712 default:
713 return nil, nil
714 }
715 return expr, jsonEnc
716 }
717
718 if expr, err := yaml.Unmarshal(fset, "", b); err == nil {
719 switch expr.(type) {
720 case *ast.StructLit, *ast.ListLit:
721 default:
722 return nil, nil
723 }
724 return expr, yamlEnc
725 }
726
727 return nil, nil
728}
729
730func (h *hoister) uniqueName(base, typ string) string {
731 base = strings.Map(func(r rune) rune {
732 if unicode.In(r, unicode.L, unicode.N) {
733 return r
734 }
735 return '_'
736 }, base)
737
738 name := base
739 for {
740 if !h.fields[name] {
741 return name
742 }
743 name = typ + "_" + base
744 typ += "x"
745 }
746}