blob: fc8f6fdc68d748f785664e58ff358ac98bf461bc [file] [log] [blame]
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +01001// Copyright 2018 The CUE Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package load
16
17import (
18 "bytes"
19 "fmt"
20 "log"
Marcel van Lohuizen9ccf2732019-02-23 14:32:03 +010021 "os"
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +010022 pathpkg "path"
23 "path/filepath"
24 "sort"
25 "strconv"
26 "strings"
27 "unicode"
28 "unicode/utf8"
29
30 "cuelang.org/go/cue/ast"
31 build "cuelang.org/go/cue/build"
32 "cuelang.org/go/cue/encoding"
33 "cuelang.org/go/cue/parser"
34 "cuelang.org/go/cue/token"
35)
36
37// An importMode controls the behavior of the Import method.
38type importMode uint
39
40const (
41 // If findOnly is set, Import stops after locating the directory
42 // that should contain the sources for a package. It does not
43 // read any files in the directory.
44 findOnly importMode = 1 << iota
45
46 // If importComment is set, parse import comments on package statements.
47 // Import returns an error if it finds a comment it cannot understand
48 // or finds conflicting comments in multiple source files.
49 // See golang.org/s/go14customimport for more information.
50 importComment
51
52 allowAnonymous
53)
54
55// importPkg returns details about the CUE package named by the import path,
56// interpreting local import paths relative to the srcDir directory.
57// If the path is a local import path naming a package that can be imported
58// using a standard import path, the returned package will set p.ImportPath
59// to that path.
60//
61// In the directory and ancestor directories up to including one with a
62// cue.mod file, all .cue files are considered part of the package except for:
63//
64// - files starting with _ or . (likely editor temporary files)
65// - files with build constraints not satisfied by the context
66//
67// If an error occurs, importPkg sets the error in the returned instance,
68// which then may contain partial information.
69//
70func (l *loader) importPkg(path, srcDir string) *build.Instance {
71 l.stk.Push(path)
72 defer l.stk.Pop()
73
74 cfg := l.cfg
75 ctxt := &cfg.fileSystem
76
77 parentPath := path
78 if isLocalImport(path) {
Marcel van Lohuizen9ccf2732019-02-23 14:32:03 +010079 parentPath = filepath.Join(srcDir, filepath.FromSlash(path))
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +010080 }
81 p := cfg.Context.NewInstance(path, l.loadFunc(parentPath))
82 p.DisplayPath = path
83
84 isLocal := isLocalImport(path)
85 var modDir string
86 // var modErr error
87 if !isLocal {
88 // TODO(mpvl): support module lookup
89 }
90
91 p.Local = isLocal
92
93 if err := updateDirs(cfg, p, path, srcDir, 0); err != nil {
94 p.ReportError(err)
95 return p
96 }
97
98 if modDir == "" && path != cleanImport(path) {
99 report(p, l.errPkgf(nil,
100 "non-canonical import path: %q should be %q", path, pathpkg.Clean(path)))
101 p.Incomplete = true
102 return p
103 }
104
105 fp := newFileProcessor(cfg, p)
106
107 root := p.Dir
108
109 for dir := p.Dir; ctxt.isDir(dir); {
110 files, err := ctxt.readDir(dir)
111 if err != nil {
112 p.ReportError(err)
113 return p
114 }
115 rootFound := false
116 for _, f := range files {
117 if f.IsDir() {
118 continue
119 }
120 if fp.add(dir, f.Name(), importComment) {
121 root = dir
122 }
123 if f.Name() == "cue.mod" {
124 root = dir
125 rootFound = true
126 }
127 }
128
129 if rootFound || dir == p.Root || fp.pkg.PkgName == "" {
130 break
131 }
132
133 // From now on we just ignore files that do not belong to the same
134 // package.
135 fp.ignoreOther = true
136
137 parent, _ := filepath.Split(filepath.Clean(dir))
138 if parent == dir {
139 break
140 }
141 dir = parent
142 }
143
144 rewriteFiles(p, root, false)
145 if err := fp.finalize(); err != nil {
146 p.ReportError(err)
147 return p
148 }
149
150 for _, f := range p.CUEFiles {
151 if !filepath.IsAbs(f) {
152 f = filepath.Join(root, f)
153 }
154 p.AddFile(f, nil)
155 }
156 p.Complete()
157 return p
158}
159
160// loadFunc creates a LoadFunc that can be used to create new build.Instances.
161func (l *loader) loadFunc(parentPath string) build.LoadFunc {
162
163 return func(path string) *build.Instance {
164 cfg := l.cfg
165
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100166 if !isLocalImport(path) {
Marcel van Lohuizen9ccf2732019-02-23 14:32:03 +0100167 // is it a builtin?
168 if strings.IndexByte(strings.Split(path, "/")[0], '.') == -1 {
Marcel van Lohuizen57333362019-04-01 14:35:09 +0200169 if l.cfg.StdRoot != "" {
170 return l.importPkg(path, l.cfg.StdRoot)
171 }
Marcel van Lohuizen9ccf2732019-02-23 14:32:03 +0100172 return nil
173 }
174 if cfg.modRoot == "" {
175 i := cfg.newInstance(path)
176 report(i, l.errPkgf(nil,
177 "import %q not found in the pkg directory", path))
178 return i
179 }
180 return l.importPkg(path, filepath.Join(cfg.modRoot, "pkg"))
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100181 }
182
183 if strings.Contains(path, "@") {
184 i := cfg.newInstance(path)
185 report(i, l.errPkgf(nil,
186 "can only use path@version syntax with 'cue get'"))
187 return i
188 }
189
190 return l.importPkg(path, parentPath)
191 }
192}
193
194func updateDirs(c *Config, p *build.Instance, path, srcDir string, mode importMode) error {
195 ctxt := &c.fileSystem
196 // path := p.ImportPath
197 if path == "" {
198 return fmt.Errorf("import %q: invalid import path", path)
199 }
200
Marcel van Lohuizen9ccf2732019-02-23 14:32:03 +0100201 if ctxt.isAbsPath(path) || strings.HasPrefix(path, "/") {
202 return fmt.Errorf("load: absolute import path %q not allowed", path)
203 }
204
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100205 if isLocalImport(path) {
206 if srcDir == "" {
207 return fmt.Errorf("import %q: import relative to unknown directory", path)
208 }
209 if !ctxt.isAbsPath(path) {
210 p.Dir = ctxt.joinPath(srcDir, path)
211 }
212 return nil
Marcel van Lohuizen57333362019-04-01 14:35:09 +0200213 }
214 dir := ctxt.joinPath(srcDir, path)
215 info, err := os.Stat(filepath.Join(srcDir, path))
216 if err == nil && info.IsDir() {
217 p.Dir = dir
218 return nil
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100219 }
220
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100221 // package was not found
222 return fmt.Errorf("cannot find package %q", path)
223}
224
225func normPrefix(root, path string, isLocal bool) string {
226 root = filepath.Clean(root)
227 prefix := ""
228 if isLocal {
229 prefix = "." + string(filepath.Separator)
230 }
231 if !strings.HasSuffix(root, string(filepath.Separator)) &&
232 strings.HasPrefix(path, root) {
233 path = prefix + path[len(root)+1:]
234 }
235 return path
236}
237
238func rewriteFiles(p *build.Instance, root string, isLocal bool) {
239 p.Root = root
240 for i, path := range p.CUEFiles {
241 p.CUEFiles[i] = normPrefix(root, path, isLocal)
242 sortParentsFirst(p.CUEFiles)
243 }
244 for i, path := range p.TestCUEFiles {
245 p.TestCUEFiles[i] = normPrefix(root, path, isLocal)
246 sortParentsFirst(p.TestCUEFiles)
247 }
248 for i, path := range p.ToolCUEFiles {
249 p.ToolCUEFiles[i] = normPrefix(root, path, isLocal)
250 sortParentsFirst(p.ToolCUEFiles)
251 }
252 for i, path := range p.IgnoredCUEFiles {
253 if strings.HasPrefix(path, root) {
254 p.IgnoredCUEFiles[i] = normPrefix(root, path, isLocal)
255 }
256 }
257 for i, path := range p.InvalidCUEFiles {
258 p.InvalidCUEFiles[i] = normPrefix(root, path, isLocal)
259 sortParentsFirst(p.InvalidCUEFiles)
260 }
261}
262
263func sortParentsFirst(s []string) {
264 sort.Slice(s, func(i, j int) bool {
265 return len(filepath.Dir(s[i])) < len(filepath.Dir(s[j]))
266 })
267}
268
269type fileProcessor struct {
270 firstFile string
271 firstCommentFile string
272 imported map[string][]token.Position
273 allTags map[string]bool
274 allFiles bool
275 ignoreOther bool // ignore files from other packages
276
277 c *Config
278 pkg *build.Instance
279
280 err error
281}
282
283func newFileProcessor(c *Config, p *build.Instance) *fileProcessor {
284 return &fileProcessor{
285 imported: make(map[string][]token.Position),
286 allTags: make(map[string]bool),
287 c: c,
288 pkg: p,
289 }
290}
291
292func (fp *fileProcessor) finalize() error {
293 p := fp.pkg
294 if fp.err != nil {
295 return fp.err
296 }
297 if len(p.CUEFiles) == 0 && !fp.c.DataFiles {
298 return &noCUEError{Package: p, Dir: p.Dir, Ignored: len(p.IgnoredCUEFiles) > 0}
299 }
300
301 for tag := range fp.allTags {
302 p.AllTags = append(p.AllTags, tag)
303 }
304 sort.Strings(p.AllTags)
305
306 p.ImportPaths, _ = cleanImports(fp.imported)
307
308 return nil
309}
310
311func (fp *fileProcessor) add(root, path string, mode importMode) (added bool) {
312 fullPath := path
313 if !filepath.IsAbs(path) {
314 fullPath = filepath.Join(root, path)
315 }
316 name := filepath.Base(fullPath)
317 dir := filepath.Dir(fullPath)
318
319 fset := token.NewFileSet()
320 ext := nameExt(name)
321 p := fp.pkg
322
323 badFile := func(err error) bool {
324 if fp.err == nil {
325 fp.err = err
326 }
327 p.InvalidCUEFiles = append(p.InvalidCUEFiles, fullPath)
328 return true
329 }
330
331 match, data, filename, err := matchFile(fp.c, dir, name, true, fp.allFiles, fp.allTags)
332 if err != nil {
333 return badFile(err)
334 }
335 if !match {
336 if ext == cueSuffix {
337 p.IgnoredCUEFiles = append(p.IgnoredCUEFiles, fullPath)
338 } else if encoding.MapExtension(ext) != nil {
339 p.DataFiles = append(p.DataFiles, fullPath)
340 }
341 return false // don't mark as added
342 }
343
344 pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly, parser.ParseComments)
345 if err != nil {
346 return badFile(err)
347 }
348
349 pkg := ""
350 if pf.Name != nil {
351 pkg = pf.Name.Name
352 }
353 if pkg == "" && mode&allowAnonymous == 0 {
354 p.IgnoredCUEFiles = append(p.IgnoredCUEFiles, fullPath)
355 return false // don't mark as added
356 }
357
358 if fp.c.Package != "" {
359 if pkg != fp.c.Package {
360 if fp.ignoreOther {
361 p.IgnoredCUEFiles = append(p.IgnoredCUEFiles, fullPath)
362 return false
363 }
364 // TODO: package does not conform with requested.
365 return badFile(fmt.Errorf("%s: found package %q; want %q", filename, pkg, fp.c.Package))
366 }
367 } else if fp.firstFile == "" {
368 p.PkgName = pkg
369 fp.firstFile = name
370 } else if pkg != p.PkgName {
371 if fp.ignoreOther {
372 p.IgnoredCUEFiles = append(p.IgnoredCUEFiles, fullPath)
373 return false
374 }
375 return badFile(&multiplePackageError{
376 Dir: p.Dir,
377 Packages: []string{p.PkgName, pkg},
378 Files: []string{fp.firstFile, name},
379 })
380 }
381
382 isTest := strings.HasSuffix(name, "_test"+cueSuffix)
383 isTool := strings.HasSuffix(name, "_tool"+cueSuffix)
384
385 if mode&importComment != 0 {
386 qcom, line := findimportComment(data)
387 if line != 0 {
388 com, err := strconv.Unquote(qcom)
389 if err != nil {
390 badFile(fmt.Errorf("%s:%d: cannot parse import comment", filename, line))
391 } else if p.ImportComment == "" {
392 p.ImportComment = com
393 fp.firstCommentFile = name
394 } else if p.ImportComment != com {
395 badFile(fmt.Errorf("found import comments %q (%s) and %q (%s) in %s", p.ImportComment, fp.firstCommentFile, com, name, p.Dir))
396 }
397 }
398 }
399
400 for _, decl := range pf.Decls {
401 d, ok := decl.(*ast.ImportDecl)
402 if !ok {
403 continue
404 }
405 for _, spec := range d.Specs {
406 quoted := spec.Path.Value
407 path, err := strconv.Unquote(quoted)
408 if err != nil {
409 log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
410 }
411 if !isTest || fp.c.Tests {
412 fp.imported[path] = append(fp.imported[path], fset.Position(spec.Pos()))
413 }
414 }
415 }
416 switch {
417 case isTest:
418 p.TestCUEFiles = append(p.TestCUEFiles, fullPath)
419 case isTool:
Marcel van Lohuizenf23fbff2018-12-20 12:23:16 +0100420 p.ToolCUEFiles = append(p.ToolCUEFiles, fullPath)
Marcel van Lohuizenbc4d65d2018-12-10 15:40:02 +0100421 default:
422 p.CUEFiles = append(p.CUEFiles, fullPath)
423 }
424 return true
425}
426
427func nameExt(name string) string {
428 i := strings.LastIndex(name, ".")
429 if i < 0 {
430 return ""
431 }
432 return name[i:]
433}
434
435// hasCUEFiles reports whether dir contains any files with names ending in .go.
436// For a vendor check we must exclude directories that contain no .go files.
437// Otherwise it is not possible to vendor just a/b/c and still import the
438// non-vendored a/b. See golang.org/issue/13832.
439func hasCUEFiles(ctxt *fileSystem, dir string) bool {
440 ents, _ := ctxt.readDir(dir)
441 for _, ent := range ents {
442 if !ent.IsDir() && strings.HasSuffix(ent.Name(), cueSuffix) {
443 return true
444 }
445 }
446 return false
447}
448
449func findimportComment(data []byte) (s string, line int) {
450 // expect keyword package
451 word, data := parseWord(data)
452 if string(word) != "package" {
453 return "", 0
454 }
455
456 // expect package name
457 _, data = parseWord(data)
458
459 // now ready for import comment, a // or /* */ comment
460 // beginning and ending on the current line.
461 for len(data) > 0 && (data[0] == ' ' || data[0] == '\t' || data[0] == '\r') {
462 data = data[1:]
463 }
464
465 var comment []byte
466 switch {
467 case bytes.HasPrefix(data, slashSlash):
468 i := bytes.Index(data, newline)
469 if i < 0 {
470 i = len(data)
471 }
472 comment = data[2:i]
473 case bytes.HasPrefix(data, slashStar):
474 data = data[2:]
475 i := bytes.Index(data, starSlash)
476 if i < 0 {
477 // malformed comment
478 return "", 0
479 }
480 comment = data[:i]
481 if bytes.Contains(comment, newline) {
482 return "", 0
483 }
484 }
485 comment = bytes.TrimSpace(comment)
486
487 // split comment into `import`, `"pkg"`
488 word, arg := parseWord(comment)
489 if string(word) != "import" {
490 return "", 0
491 }
492
493 line = 1 + bytes.Count(data[:cap(data)-cap(arg)], newline)
494 return strings.TrimSpace(string(arg)), line
495}
496
497var (
498 slashSlash = []byte("//")
499 slashStar = []byte("/*")
500 starSlash = []byte("*/")
501 newline = []byte("\n")
502)
503
504// skipSpaceOrComment returns data with any leading spaces or comments removed.
505func skipSpaceOrComment(data []byte) []byte {
506 for len(data) > 0 {
507 switch data[0] {
508 case ' ', '\t', '\r', '\n':
509 data = data[1:]
510 continue
511 case '/':
512 if bytes.HasPrefix(data, slashSlash) {
513 i := bytes.Index(data, newline)
514 if i < 0 {
515 return nil
516 }
517 data = data[i+1:]
518 continue
519 }
520 if bytes.HasPrefix(data, slashStar) {
521 data = data[2:]
522 i := bytes.Index(data, starSlash)
523 if i < 0 {
524 return nil
525 }
526 data = data[i+2:]
527 continue
528 }
529 }
530 break
531 }
532 return data
533}
534
535// parseWord skips any leading spaces or comments in data
536// and then parses the beginning of data as an identifier or keyword,
537// returning that word and what remains after the word.
538func parseWord(data []byte) (word, rest []byte) {
539 data = skipSpaceOrComment(data)
540
541 // Parse past leading word characters.
542 rest = data
543 for {
544 r, size := utf8.DecodeRune(rest)
545 if unicode.IsLetter(r) || '0' <= r && r <= '9' || r == '_' {
546 rest = rest[size:]
547 continue
548 }
549 break
550 }
551
552 word = data[:len(data)-len(rest)]
553 if len(word) == 0 {
554 return nil, nil
555 }
556
557 return word, rest
558}
559
560func cleanImports(m map[string][]token.Position) ([]string, map[string][]token.Position) {
561 all := make([]string, 0, len(m))
562 for path := range m {
563 all = append(all, path)
564 }
565 sort.Strings(all)
566 return all, m
567}
568
569// // Import is shorthand for Default.Import.
570// func Import(path, srcDir string, mode ImportMode) (*Package, error) {
571// return Default.Import(path, srcDir, mode)
572// }
573
574// // ImportDir is shorthand for Default.ImportDir.
575// func ImportDir(dir string, mode ImportMode) (*Package, error) {
576// return Default.ImportDir(dir, mode)
577// }
578
579var slashslash = []byte("//")
580
581// isLocalImport reports whether the import path is
582// a local import path, like ".", "..", "./foo", or "../foo".
583func isLocalImport(path string) bool {
584 return path == "." || path == ".." ||
585 strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../")
586}