[oil-language] Start implementing the parse_config() function

Part of the config dialect #951.

It's statically typed, but doesn't run yet.

[doc] Oil vs Python updates

core/shell.py

@@ -41,6 +41,7 @@
 
 from oil_lang import expr_eval
 from oil_lang import builtin_oil
+from oil_lang import funcs
 from oil_lang import funcs_builtin
 
 from osh import builtin_assign
@@ -402,6 +403,9 @@ def Main(lang, arg_r, environ, login_shell, loader, line_input):
 
   funcs_builtin.Init2(mem, splitter, globber)
 
+  config_parser = funcs.ConfigParser(fd_state, parse_ctx, errfmt)
+  funcs_builtin.Init3(mem, config_parser)
+
   # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
   # stuffing too much into one, since a .json crash dump isn't a stream.
   crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')

core/shell_native.py

@@ -37,6 +37,7 @@
 from frontend import reader
 from frontend import parse_lib
 
+from oil_lang import funcs
 from osh import builtin_assign
 from osh import builtin_bracket
 from osh import builtin_meta
@@ -338,6 +339,8 @@ def Main(lang, arg_r, environ, login_shell, loader, line_input):
 
   splitter = split.SplitContext(mem)
 
+  config_parser = funcs.ConfigParser(fd_state, parse_ctx, errfmt)
+
   # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
   # stuffing too much into one, since a .json crash dump isn't a stream.
   crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')

core/state.py

@@ -307,6 +307,25 @@ def MakeOpts(mem, opt_hook):
   return parse_opts, exec_opts, mutable_opts
 
 
+def _SetGroup(opt0_array, opt_nums, b):
+  # type: (List[bool], List[int], bool) -> None
+  for opt_num in opt_nums:
+    b2 = not b if opt_num in consts.DEFAULT_TRUE else b
+    opt0_array[opt_num] = b2
+
+
+def MakeOilOpts():
+  # type: () -> optview.Parse
+  opt0_array = InitOpts()
+  _SetGroup(opt0_array, consts.OIL_ALL, True)
+
+  no_stack = None  # type: List[bool]
+  opt_stacks = [no_stack] * option_i.ARRAY_SIZE  # type: List[List[bool]]
+
+  parse_opts = optview.Parse(opt0_array, opt_stacks)
+  return parse_opts
+
+
 def _ShoptOptionNum(opt_name):
   # type: (str) -> option_t
   opt_num = match.MatchOption(opt_name)
@@ -546,30 +565,24 @@ def SetOption(self, opt_name, b):
         new_val = value.Str(':'.join(names))
         self.mem.InternalSetGlobal('SHELLOPTS', new_val)
 
-  def _SetGroup(self, opt_nums, b):
-    # type: (List[int], bool) -> None
-    for opt_num in opt_nums:
-      b2 = not b if opt_num in consts.DEFAULT_TRUE else b
-      self.opt0_array[opt_num] = b2
-
   def SetShoptOption(self, opt_name, b):
     # type: (str, bool) -> None
     """ For shopt -s/-u and sh -O/+O. """
 
     # shopt -s all:oil turns on all Oil options, which includes all strict #
     # options
     if opt_name == 'oil:basic':
-      self._SetGroup(consts.OIL_BASIC, b)
+      _SetGroup(self.opt0_array, consts.OIL_BASIC, b)
       self.SetDeferredErrExit(b)  # Special case
       return
 
     if opt_name == 'oil:all':
-      self._SetGroup(consts.OIL_ALL, b)
+      _SetGroup(self.opt0_array, consts.OIL_ALL, b)
       self.SetDeferredErrExit(b)  # Special case
       return
 
     if opt_name == 'strict:all':
-      self._SetGroup(consts.STRICT_ALL, b)
+      _SetGroup(self.opt0_array, consts.STRICT_ALL, b)
       return
 
     opt_num = _ShoptOptionNum(opt_name)

doc/oil-vs-python.md

@@ -53,18 +53,16 @@ Language](oil-language-tour.html).
 `%symbol` (used in eggex now, but could also be used as interned strings)
 -->
 
-### Not Supported
-
-- No tuple type for now.  We might want Go-like multiple return values.
-- Iterators.  Instead we have a fixed for loop.
-- List comprehensions and generator expressions
-- Lambdas
-
-<!--
-- Tuples (TODO): Does Oil have true tuples?
-  - Singleton tuples like `42,` are disallowed, in favor of the more explicit
-    `tup(42)`.
--->
+### Omitted
+
+- Iterators.
+  - Instead we have for loop that works on lists and dicts.
+  - It flexibly accepts up to 3 loop variables, taking the place of Python's
+    `enumerate()`, `keys()`, `values()`, and `items()`.
+- List comprehensions and generator expressions.  QTT over pipes should address
+  these use cases.
+- Lambdas.  Functions are often external and don't have lexical scope.
+- TODO: No tuple type for now.  We might want Go-like multiple return values.
 
 ## Operators
 

frontend/parse_lib.py

@@ -274,6 +274,13 @@ def MakeOshParser(self, line_reader, emit_comp_dummy=False):
     c_parser = cmd_parse.CommandParser(self, w_parser, lx, line_reader)
     return c_parser
 
+  def MakeConfigParser(self, line_reader):
+    # type: (_Reader) -> CommandParser
+    lx = self.MakeLexer(line_reader)
+    w_parser = word_parse.WordParser(self, lx, line_reader)
+    c_parser = cmd_parse.CommandParser(self, w_parser, lx, line_reader)
+    return c_parser
+
   def MakeWordParserForHereDoc(self, line_reader):
     # type: (_Reader) -> WordParser
     lx = self.MakeLexer(line_reader)

oil_lang/funcs.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python2
+"""
+funcs.py
+"""
+from __future__ import print_function
+
+from _devbuild.gen.runtime_asdl import value
+from _devbuild.gen.syntax_asdl import source
+from asdl import runtime
+from core import alloc
+from core import error
+from core import main_loop
+from core import state
+from core import ui
+from frontend import reader
+
+import posix_ as posix
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+  from _devbuild.gen.runtime_asdl import value_t
+  from core import process
+  from frontend import parse_lib
+
+
+class ConfigParser(object):
+  """ For parse_config()
+
+  """
+  def __init__(self, fd_state, parse_ctx, errfmt):
+    # type: (process.FdState, parse_lib.ParseContext, ui.ErrorFormatter) -> None
+    self.fd_state = fd_state
+    self.parse_ctx = parse_ctx
+    self.errfmt = errfmt
+
+  def ParseFile(self, path):
+    # type: (str) -> value_t
+
+    # TODO: need to close the file!
+    try:
+      f = self.fd_state.Open(path)
+    except (IOError, OSError) as e:
+      raise error.Expr("Couldn't open %r: %s" % (path, posix.strerror(e.errno)))
+
+    arena = self.parse_ctx.arena
+    line_reader = reader.FileLineReader(f, arena)
+
+    parse_opts = state.MakeOilOpts()
+    # Note: runtime needs these options and totally different memory
+
+    # TODO: CommandParser needs parse_opts
+    c_parser = self.parse_ctx.MakeConfigParser(line_reader)
+
+    call_spid = runtime.NO_SPID  # TODO: location info
+
+    # TODO: Should there be a separate config file source?
+    src = source.SourcedFile(path, call_spid)
+    try:
+      with alloc.ctx_Location(arena, src):
+        node = main_loop.ParseWholeFile(c_parser)
+    except error.Parse as e:
+      self.errfmt.PrettyPrintError(e)
+      return None
+
+    # Wrap in expr.Block?
+    return value.Block(node)

oil_lang/funcs_builtin.py

@@ -7,6 +7,7 @@
 from _devbuild.gen.runtime_asdl import value, scope_e
 from _devbuild.gen.syntax_asdl import sh_lhs_expr
 from core.pyerror import e_die, log
+from frontend import parse_lib
 from oil_lang import expr_eval
 
 from typing import Callable, Union, TYPE_CHECKING
@@ -124,18 +125,6 @@ def __call__(self, *args):
     return expr_eval.LookupVar(self.mem, name, scope_e.Dynamic)
 
 
-class _ParseConfig(object):
-  """ parse_config()
-
-  type: (str) -> command_t
-  """
-  def __init__(self, mem):
-    self.mem = mem
-
-  def __call__(self, *args):
-    raise NotImplementedError()
-
-
 class _EvalToDict(object):
   """ eval_to_dict() """
   def __init__(self, mem):
@@ -208,6 +197,12 @@ def Init2(mem, splitter, globber):
   SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))
 
 
+def Init3(mem, config_parser):
+  # type: (funcs.ConfigParser) -> None
+
+  SetGlobalFunc(mem, 'parse_config', lambda path: config_parser.ParseFile(path))
+
+
 def Init(mem):
   # type: (state.Mem) -> None
   """Populate the top level namespace with some builtin functions."""
@@ -228,10 +223,7 @@ def Init(mem):
 
   SetGlobalFunc(mem, 'shvar_get', _Shvar_get(mem))
 
-  # Takes a filename.  Note that parse_equals should be on.
-  SetGlobalFunc(mem, 'parse_config', _ParseConfig(mem))
-
-  # for top level namespace, and the default for lower case 'server' blocks
+  # For top level namespace, and the default for lower case 'server' blocks
   #
   # Security: You need a whole different VM?  User should not be able to modify
   # PATH or anything else.  Yeah you get a new state.Mem(), but you copy some

osh/builtin_meta.py

@@ -98,6 +98,7 @@ def Run(self, cmd_val):
     resolved = self.search_path.Lookup(path, exec_required=False)
     if resolved is None:
       resolved = path
+    # TODO: need to close the file!
     try:
       f = self.fd_state.Open(resolved)  # Shell can't use descriptors 3-9
     except (IOError, OSError) as e:

spec/oil-config.test.sh

@@ -100,6 +100,13 @@ shopt --set parse_equals {
   }
 }
 
+= config
+= block
+
+return
+
+# Why does this crash?
+
 json write (config)
 
 ## STDOUT:
@@ -123,7 +130,10 @@ json write (config)
 #
 # }
 
-const config = parse_config('spec/testdata/config/package-manager.oil')
+const path = "$REPO_ROOT/spec/testdata/config/package-manager.oil"
+#ls $path
+
+const config = parse_config(path)
 const block = eval_to_dict(%(package user))
 
 ## STDOUT:

types/osh-eval-manifest.txt

@@ -43,6 +43,7 @@
 ./frontend/typed_args.py
 ./oil_lang/expr_parse.py
 ./oil_lang/expr_to_ast.py
+./oil_lang/funcs.py
 ./oil_lang/objects.py
 ./oil_lang/regex_translate.py
 ./osh/arith_parse.py