~andyc/oil

0cf06ed2a5a4ba2eec7a87b6d6869a3347a42a08 — Andy C 2 months ago 019d835
[mycpp] Upgrade to MyPy 0.780, and rebuild containers

- [soil/deps-mycpp] Use hermetic Python 3.10 if available
  - Reproduced the type comment syntax error with MyPy 0.730!  Gah
- Changed mycpp code get it to work with MyPy 0.780 AST
- Build optimized CPython 3.10!  Otherwise MyPy runs slowly
- [soil] Rebuild and tag OCI images with a new version
  - images cpp and clang: they use MyPy, and thus now use layer-py3
    - This is for issue #1262
  - image pea: rebuilt because it already uses layer-py3
  - Tag those 3 images 'v-2022-08-05'
- Add soil/Dockerfile.test-image to reproduce the ensurepip failure in a
  smaller image
  - It was a missing zlib dependency
  - So factor out PY3_DEPS, which is used consistently

Minor:

- Limit number of compiler errors in Clang
- Fix for ninja: don't create output file unless it succeeds
- Remove unused build/prepare.sh; it's now in soil/deps-tar.sh
- Misc fixes to the Docker images, e.g. COPY --chown=uke
M build/dev.sh => build/dev.sh +3 -1
@@ 15,6 15,8 @@ REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
readonly REPO_ROOT

source build/common.sh  # for log, $CLANGXX
source soil/deps-apt.sh   # PY3_DEPS
# TODO: We could have the user run soil/deps-apt.sh directly

export PYTHONPATH='.:vendor/'



@@ 28,7 30,7 @@ ubuntu-deps() {
  set -x  # show what needs sudo
  sudo apt install \
    python-dev gawk libreadline-dev ninja-build cmake \
    python-pip python3-pip
    "${PY3_DEPS[@]}"
  set +x

  test/spec.sh install-shells

D build/prepare.sh => build/prepare.sh +0 -30
@@ 1,30 0,0 @@
#!/usr/bin/env bash
#
# Do a full CPython build out of tree, so we can walk dependencies dynamically.
#
# The 'app-deps' and 'runpy-deps' build steps require this.
#
# Usage:
#   build/prepare.sh <function name>
#
# Example:
#
#   build/prepare.sh configure
#   build/prepare.sh build-python

set -o nounset
set -o pipefail
set -o errexit

REPO_ROOT=$(cd $(dirname $0)/..; pwd)
readonly REPO_ROOT

source build/common.sh

# For uftrace.
cpython-instrumented() {
  configure _devbuild/cpython-instrumented
  build-python _devbuild/cpython-instrumented '-O0 -pg'
}

"$@"

M build/translate.sh => build/translate.sh +1 -1
@@ 30,7 30,7 @@ mycpp() {

    source $MYCPP_VENV/bin/activate
    time PYTHONPATH=$REPO_ROOT:$MYPY_REPO MYPYPATH=$REPO_ROOT:$REPO_ROOT/native \
      mycpp/mycpp_main.py "$@"
      $0 maybe-our-python3 mycpp/mycpp_main.py "$@"
  )
}


M cpp/NINJA-steps.sh => cpp/NINJA-steps.sh +1 -1
@@ 194,7 194,7 @@ compile_one() {

  # this flag is only valid in Clang, doesn't work in continuous build
  if test "$compiler" = 'clang'; then
    flags="$flags -ferror-limit=1000"
    flags="$flags -ferror-limit=10"
  fi

  setglobal_cxx $compiler

M mycpp/NINJA-steps.sh => mycpp/NINJA-steps.sh +9 -4
@@ 12,7 12,7 @@ set -o errexit
REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
readonly REPO_ROOT

source $REPO_ROOT/mycpp/common.sh
source $REPO_ROOT/mycpp/common.sh  # maybe-our-python3
source $REPO_ROOT/test/tsv-lib.sh  # time-tsv
source $REPO_ROOT/build/common.sh  # for CXX, BASE_CXXFLAGS, ASAN_SYMBOLIZER_PATH



@@ 86,13 86,18 @@ translate-mycpp() {
  # TODO: mylib_old.h should grow the GC API
  export GC=1

  local tmp=$out.tmp

  # NOTE: mycpp has to be run in the virtualenv, as well as with a different
  # PYTHONPATH.
  ( source $MYCPP_VENV/bin/activate
    # flags may be empty
    time PYTHONPATH=$REPO_ROOT:$MYPY_REPO MYPYPATH="$REPO_ROOT:$REPO_ROOT/mycpp" \
      mycpp/mycpp_main.py "$@" > $out
    # MYPYPATH set to find mylib.pyi
    time PYTHONPATH=$REPO_ROOT:$MYPY_REPO MYPYPATH="$REPO_ROOT/mycpp" \
      $0 maybe-our-python3 mycpp/mycpp_main.py "$@" > $tmp
  )

  # Don't create output unless it succeeds!
  mv $tmp $out
}

translate-pea() {

M mycpp/common.sh => mycpp/common.sh +16 -0
@@ 33,3 33,19 @@ run-test() {
  $bin > $log
}

maybe-our-python3() {
  ### Run a command line with Python 3

  # Use Python 3.10 from soil/deps-tar if available.  Otherwise use the sytsem
  # python3.

  local py3_ours='../oil_DEPS/python3'
  if test -f $py3_ours; then
    echo "*** Running $py3_ours $@" >& 2
    $py3_ours "$@"
  else
    # Use system copy
    python3 "$@"
  fi
}


M mycpp/const_pass.py => mycpp/const_pass.py +2 -2
@@ 87,7 87,7 @@ class Collect(ExpressionVisitor[T], StatementVisitor[None]):
    def visit_mypy_file(self, o: 'mypy.nodes.MypyFile') -> T:
        # Skip some stdlib stuff.  A lot of it is brought in by 'import
        # typing'.
        if o.fullname() in (
        if o.fullname in (
            '__future__', 'sys', 'types', 'typing', 'abc', '_ast', 'ast',
            '_weakrefset', 'collections', 'cStringIO', 're', 'builtins'):



@@ 355,7 355,7 @@ class Collect(ExpressionVisitor[T], StatementVisitor[None]):
    def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> T:
        # got the type here, nice!
        typ = o.type
        self.log('FuncDef %s :: %s', o.name(), typ)
        self.log('FuncDef %s :: %s', o.name, typ)
        #self.log('%s', type(typ))

        for t, name in zip(typ.arg_types, typ.arg_names):

M mycpp/cppgen_pass.py => mycpp/cppgen_pass.py +55 -36
@@ 11,7 11,7 @@ from typing import overload, Union, Optional, Any, Dict
from mypy.visitor import ExpressionVisitor, StatementVisitor
from mypy.types import (
    Type, AnyType, NoneTyp, TupleType, Instance, Overloaded, CallableType,
    UnionType, UninhabitedType, PartialType)
    UnionType, UninhabitedType, PartialType, TypeAliasType)
from mypy.nodes import (
    Expression, Statement, Block, NameExpr, IndexExpr, MemberExpr, TupleExpr,
    ExpressionStmt, AssignmentStmt, IfStmt, StrExpr, SliceExpr, FuncDef,


@@ 71,7 71,7 @@ def _GetContainsFunc(t):
  contains_func = None

  if isinstance(t, Instance):
    type_name = t.type.fullname()
    type_name = t.type.fullname

    if type_name == 'builtins.list':
      contains_func = 'list_contains'


@@ 97,7 97,7 @@ def _GetContainsFunc(t):

def IsStr(t):
  """Helper to check if a type is a string."""
  return isinstance(t, Instance) and t.type.fullname() == 'builtins.str'
  return isinstance(t, Instance) and t.type.fullname == 'builtins.str'


def _CheckConditionType(t):


@@ 106,7 106,7 @@ def _CheckConditionType(t):
  doesn't translate to C++.
  """
  if isinstance(t, Instance):
    type_name = t.type.fullname()
    type_name = t.type.fullname
    if type_name == 'builtins.str':
      return False



@@ 154,7 154,7 @@ def get_c_type(t, param=False, local=False):
  # mypyc/genops.py does?

  elif isinstance(t, Instance):
    type_name = t.type.fullname()
    type_name = t.type.fullname

    if type_name == 'builtins.int':
      c_type = 'int'


@@ 188,8 188,8 @@ def get_c_type(t, param=False, local=False):
      is_pointer = True

    else:
      # note: fullname() => 'parse.Lexer'; name() => 'Lexer'
      base_class_names = [b.type.fullname() for b in t.type.bases]
      # note: fullname => 'parse.Lexer'; name => 'Lexer'
      base_class_names = [b.type.fullname for b in t.type.bases]

      #log('** base_class_names %s', base_class_names)



@@ 202,7 202,7 @@ def get_c_type(t, param=False, local=False):
      if 'asdl.pybase.SimpleObj' not in base_class_names:
        is_pointer = True

      parts = t.type.fullname().split('.')
      parts = t.type.fullname.split('.')
      c_type = '%s::%s' % (parts[-2], parts[-1])

  elif isinstance(t, UninhabitedType):


@@ 236,6 236,17 @@ def get_c_type(t, param=False, local=False):
    arg_types = [get_c_type(typ) for typ in t.arg_types]
    c_type = '%s (*f)(%s)' % (ret_type, ', '.join(arg_types))

  elif isinstance(t, TypeAliasType):
    if 0:
      log('***')
      log('%s', t)
      log('%s', dir(t))
      log('%s', t.alias)
      log('%s', dir(t.alias))
      log('%s', t.alias.target)
      log('***')
    return get_c_type(t.alias.target)

  else:
    raise NotImplementedError('MyPy type: %s %s' % (type(t), t))



@@ 364,7 375,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
    def visit_mypy_file(self, o: 'mypy.nodes.MypyFile') -> T:
        # Skip some stdlib stuff.  A lot of it is brought in by 'import
        # typing'.
        if o.fullname() in (
        if o.fullname in (
            '__future__', 'sys', 'types', 'typing', 'abc', '_ast', 'ast',
            '_weakrefset', 'collections', 'cStringIO', 're', 'builtins'):



@@ 373,9 384,9 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
            return

        #self.log('')
        #self.log('mypyfile %s', o.fullname())
        #self.log('mypyfile %s', o.fullname)

        mod_parts = o.fullname().split('.')
        mod_parts = o.fullname.split('.')
        if self.forward_decl:
          comment = 'forward declare' 
        elif self.decl:


@@ 516,7 527,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
        if (callee_name not in ('str', 'bool', 'float') and
            isinstance(ret_type, Instance)):

          ret_type_name = ret_type.type.name()
          ret_type_name = ret_type.type.name

          # HACK: Const is the callee; expr__Const is the return type
          if (ret_type_name == callee_name or


@@ 743,8 754,8 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          self.log('*** %r', c_op)
          self.log('%s', o.left)
          self.log('%s', o.right)
          #self.log('t0 %r', t0.type.fullname())
          #self.log('t1 %r', t1.type.fullname())
          #self.log('t0 %r', t0.type.fullname)
          #self.log('t1 %r', t1.type.fullname)
          self.log('left_ctype %r', left_ctype)
          self.log('right_ctype %r', right_ctype)
          self.log('')


@@ 1367,7 1378,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
            over_type = self.types[seq]
            #self.log('  iterating over type %s', over_type)

            if over_type.type.fullname() == 'builtins.list':
            if over_type.type.fullname == 'builtins.list':
              c_type = get_c_type(over_type)
              assert c_type.endswith('*'), c_type
              c_iter_type = c_type.replace('List', 'ListIter', 1)[:-1]  # remove *


@@ 1469,6 1480,11 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          # Str* y = tup1->at1()

          rvalue_type = self.types[o.rvalue]

          # type alias upgrade for MyPy 0.780
          if isinstance(rvalue_type, TypeAliasType):
            rvalue_type = rvalue_type.alias.target

          c_type = get_c_type(rvalue_type)

          is_return = isinstance(o.rvalue, CallExpr)


@@ 1599,12 1615,15 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          iterated_over = o.expr

        over_type = self.types[iterated_over]
        if isinstance(over_type, TypeAliasType):
          over_type = over_type.alias.target

        #self.log('  iterating over type %s', over_type)
        #self.log('  iterating over type %s', over_type.type.fullname())
        #self.log('  iterating over type %s', over_type.type.fullname)

        over_dict = False

        if over_type.type.fullname() == 'builtins.list':
        if over_type.type.fullname == 'builtins.list':
          c_type = get_c_type(over_type)
          assert c_type.endswith('*'), c_type
          c_iter_type = c_type.replace('List', 'ListIter', 1)[:-1]  # remove *


@@ 1613,7 1632,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          if reverse:
            c_iter_type = 'Reverse' + c_iter_type

        elif over_type.type.fullname() == 'builtins.dict':
        elif over_type.type.fullname == 'builtins.dict':
          # Iterator
          c_type = get_c_type(over_type)
          assert c_type.endswith('*'), c_type


@@ 1623,7 1642,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):

          assert not reverse

        elif over_type.type.fullname() == 'builtins.str':
        elif over_type.type.fullname == 'builtins.str':
          c_iter_type = 'StrIter'
          assert not reverse  # can't reverse iterate over string yet



@@ 1891,7 1910,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          c_type = get_c_type(arg_type, param=False)
          #c_type = get_c_type(arg_type, param=True)

          arg_name = arg.variable.name()
          arg_name = arg.variable.name

          # C++ has implicit 'this'
          if arg_name == 'self':


@@ 1921,9 1940,9 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
      default_val = o.arguments[-1].initializer
      if default_val:  # e.g. osh/bool_parse.py has default val
        if self.decl or class_name is None:
          func_name = o.name()
          func_name = o.name
        else:
          func_name = '%s::%s' % (self.current_class_name, o.name())
          func_name = '%s::%s' % (self.current_class_name, o.name)
        self.write('\n')

        # Write _Next() with no args


@@ 1945,7 1964,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          self.write(' {\n')
          # return MakeOshParser()
          kw = '' if isinstance(ret_type, NoneTyp) else 'return '
          self.write('  %s%s(' % (kw, o.name()))
          self.write('  %s%s(' % (kw, o.name))

          # Don't write self or last optional argument
          first_arg_index = 0 if class_name is None else 1


@@ 1955,7 1974,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
            for i, arg in enumerate(pass_through):
              if i != 0:
                self.write(', ')
              self.write(arg.variable.name())
              self.write(arg.variable.name)
            self.write(', ')

          # Now write default value, e.g. lex_mode_e::DBracket


@@ 1964,12 1983,12 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          self.write('}\n')

    def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> T:
        if o.name() == '__repr__':  # Don't translate
        if o.name == '__repr__':  # Don't translate
          return

        # No function prototypes when forward declaring.
        if self.forward_decl:
          self.virtual.OnMethod(self.current_class_name, o.name())
          self.virtual.OnMethod(self.current_class_name, o.name)
          return

        # Hacky MANUAL LIST of functions and methods with OPTIONAL ARGUMENTS.


@@ 1984,7 2003,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):

        # TODO: restrict this
        class_name = self.current_class_name
        func_name = o.name()
        func_name = o.name
        ret_type = o.type.ret_type

        if (class_name in ('BoolParser', 'CommandParser') and


@@ 2038,17 2057,17 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          self.local_var_list = []  # Make a new instance to collect from
          self.local_vars[o] = self.local_var_list

          #log('Is Virtual? %s %s', self.current_class_name, o.name())
          if self.virtual.IsVirtual(self.current_class_name, o.name()):
          #log('Is Virtual? %s %s', self.current_class_name, o.name)
          if self.virtual.IsVirtual(self.current_class_name, o.name):
            virtual = 'virtual '

        if not self.decl and self.current_class_name:
          # definition looks like
          # void Type::foo(...);
          func_name = '%s::%s' % (self.current_class_name, o.name())
          func_name = '%s::%s' % (self.current_class_name, o.name)
        else:
          # declaration inside class { }
          func_name = o.name()
          func_name = o.name

        self.write('\n')



@@ 2071,7 2090,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):

        # Write local vars we collected in the 'decl' phase
        if not self.forward_decl and not self.decl:
          arg_names = [arg.variable.name() for arg in o.arguments]
          arg_names = [arg.variable.name for arg in o.arguments]
          #log('arg_names %s', arg_names)
          #log('local_vars %s', self.local_vars[o])
          self.prepend_to_block = [


@@ 2137,7 2156,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):

            # Constructor is named after class
            if isinstance(stmt, FuncDef):
              method_name = stmt.name()
              method_name = stmt.name
              if method_name == '__init__':
                self.decl_write_ind('%s(', o.name)
                self._WriteFuncParams(stmt.type.arg_types, stmt.arguments)


@@ 2197,7 2216,7 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
          if isinstance(stmt, FuncDef):
            # Collect __init__ calls within __init__, and turn them into
            # initializer lists.
            if stmt.name() == '__init__':
            if stmt.name == '__init__':
              self.write('\n')
              self.write_ind('%s::%s(', o.name, o.name)
              self._WriteFuncParams(stmt.type.arg_types, stmt.arguments)


@@ 2250,10 2269,10 @@ class Generate(ExpressionVisitor[T], StatementVisitor[None]):
              self.accept(stmt.body)
              continue

            if stmt.name() == '__enter__':
            if stmt.name == '__enter__':
              continue

            if stmt.name() == '__exit__':
            if stmt.name == '__exit__':
              self.decl_write('\n')
              self.decl_write_ind('%s::~%s()', o.name, o.name)
              self.accept(stmt.body)

M soil/Dockerfile.clang => soil/Dockerfile.clang +6 -3
@@ 5,10 5,7 @@ RUN apt-get update
# repo is /home/uke/{oil,oil_DEPS}
WORKDIR /home/uke/tmp

# Copy build scripts into the container and run them

COPY soil/deps-apt.sh /home/uke/tmp/soil/deps-apt.sh

RUN soil/deps-apt.sh layer-for-soil
RUN soil/deps-apt.sh clang



@@ 39,6 36,12 @@ RUN soil/deps-tar.sh build-re2c

# mypy deps: Installs from PyPI
# needed to measure coverage of mycpp/examples

# Run MyPy under Python 3.10
COPY --chown=uke _cache/Python-3.10.4.tar.xz \
  /home/uke/tmp/_cache/Python-3.10.4.tar.xz
RUN soil/deps-tar.sh layer-py3

COPY mycpp/common.sh /home/uke/tmp/mycpp/common.sh
COPY soil/deps-mycpp.sh /home/uke/tmp/soil/deps-mycpp.sh
RUN soil/deps-mycpp.sh layer-mycpp

M soil/Dockerfile.cpp => soil/Dockerfile.cpp +6 -5
@@ 19,17 19,18 @@ USER uke
# We're in /home/uke/tmp, so these will create /home/uke/oil_DEPS, which will be 
# a sibling of the runtime bind mount /home/uke/oil.

# Used by soil/deps-{binary,tar}.sh
# Used by soil/deps-tar.sh
COPY build/common.sh /home/uke/tmp/build/common.sh

# For Clang coverage
COPY soil/deps-binary.sh /home/uke/tmp/soil/deps-binary.sh
#RUN soil/deps-binary.sh layer-clang

# re2c
COPY soil/deps-tar.sh /home/uke/tmp/soil/deps-tar.sh
RUN soil/deps-tar.sh layer-re2c

# Run MyPy under Python 3.10
COPY --chown=uke _cache/Python-3.10.4.tar.xz \
  /home/uke/tmp/_cache/Python-3.10.4.tar.xz
RUN soil/deps-tar.sh layer-py3

# Installs from PyPI
COPY mycpp/common.sh /home/uke/tmp/mycpp/common.sh
COPY soil/deps-mycpp.sh /home/uke/tmp/soil/deps-mycpp.sh

M soil/Dockerfile.pea => soil/Dockerfile.pea +5 -4
@@ 10,16 10,17 @@ COPY soil/deps-apt.sh /home/uke/tmp/soil/deps-apt.sh
RUN soil/deps-apt.sh layer-for-soil
RUN soil/deps-apt.sh pea

RUN useradd --create-home uke && chown -R uke /home/uke
USER uke

# deps-tar.sh has a 'wget' step which we're skipping
COPY _cache/Python-3.10.4.tar.xz /home/uke/tmp/_cache/Python-3.10.4.tar.xz
COPY --chown=uke _cache/Python-3.10.4.tar.xz \
  /home/uke/tmp/_cache/Python-3.10.4.tar.xz

COPY build/common.sh /home/uke/tmp/build/common.sh
COPY soil/deps-tar.sh /home/uke/tmp/soil/deps-tar.sh
RUN soil/deps-tar.sh layer-py3

RUN useradd --create-home uke && chown -R uke /home/uke
USER uke

COPY soil/deps-py.sh /home/uke/tmp/soil/deps-py.sh
RUN soil/deps-py.sh pea


A soil/Dockerfile.test-image => soil/Dockerfile.test-image +36 -0
@@ 0,0 1,36 @@
# This image is only for TESTING

FROM debian:buster-slim

RUN apt-get update

WORKDIR /home/uke/tmp

# Copy build scripts into the container and run them

COPY soil/deps-apt.sh /home/uke/tmp/soil/deps-apt.sh
RUN soil/deps-apt.sh layer-for-soil

# Has build-essential and Python build deps, no system Python 3
RUN soil/deps-apt.sh test-image

RUN useradd --create-home uke && chown -R uke /home/uke
USER uke

COPY --chown=uke _cache/Python-3.10.4.tar.xz \
  /home/uke/tmp/_cache/Python-3.10.4.tar.xz

COPY build/common.sh /home/uke/tmp/build/common.sh
COPY soil/deps-tar.sh /home/uke/tmp/soil/deps-tar.sh
RUN soil/deps-tar.sh layer-py3

# Note: reproducing ensurepip failure here?
COPY mycpp/common.sh /home/uke/tmp/mycpp/common.sh
COPY soil/deps-mycpp.sh /home/uke/tmp/soil/deps-mycpp.sh
RUN soil/deps-mycpp.sh layer-mycpp

#RUN soil/deps-mycpp.sh git-clone

#RUN soil/deps-mycpp.sh pip-install

CMD ["sh", "-c", "echo 'hello from oilshell/soil-test-image'"]

M soil/deps-apt.sh => soil/deps-apt.sh +32 -7
@@ 1,12 1,21 @@
#!/usr/bin/env bash
#
# Usage:
#   ./deps-apt.sh <function name>
#   soil/deps-apt.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

# These are needed for bootstrapping pip in Python 3.10
# (Also used by build/dev.sh ubuntu-deps)
#
# For building Python 3.10 with working 'pip install'
#   libssl-dev: to download packages
#   libffi-dev: for working setuptools
#   zlib1g-dev: needed for 'import zlib'
declare -a PY3_DEPS=(libssl-dev libffi-dev zlib1g-dev)

layer-python-symlink() {
  ### A special layer for building CPython; done as root
  ln -s -f -v /usr/bin/python2 /usr/bin/python


@@ 50,9 59,16 @@ dev-minimal() {
}

pea() {
  # For installing MyPy
  apt-get install -y python3-pip
}

test-image() {
  ### Minimal build with custom Python 3

  apt-get install -y build-essential "${PY3_DEPS[@]}"
}

other-tests() {
  local -a packages=(
    libreadline-dev


@@ 77,9 93,13 @@ cpp() {
    libreadline-dev
    python2-dev

    # for type checking with MyPy binary
    python3
    python3-pip
    python3-venv  # MyPy virtualenv requirements.txt
    python3-pip  # TODO: remove
    python3-venv  # TODO: remove

    # for custom Python 3
    "${PY3_DEPS[@]}"

    ninja-build
    # to create _test/index.html


@@ 110,10 130,13 @@ clang() {

    ninja-build

    # for translating mycpp/examples
    # for type checking with MyPy binary
    python3
    python3-pip
    python3-venv  # MyPy virtualenv requirements.txt
    python3-pip  # TODO: try removing
    python3-venv  # TODO: try removing

    # for custom Python 3
    "${PY3_DEPS[@]}"
  )

  apt-get install -y "${packages[@]}"


@@ 147,4 170,6 @@ ovm-tarball() {
  apt-get install -y "${packages[@]}"
}

"$@"
if test $(basename $0) = 'deps-apt.sh'; then
  "$@"
fi

M soil/deps-mycpp.sh => soil/deps-mycpp.sh +15 -4
@@ 32,29 32,40 @@ REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
source $REPO_ROOT/mycpp/common.sh  # MYPY_REPO

git-clone() {
  ### Invoked by services/toil-worker
  ### Clone mypy at a specific branch

  local out=$MYPY_REPO
  mkdir -p $out
  git clone --recursive --depth=50 --branch=release-0.730 \
  git clone --recursive --depth=50 --branch=release-0.780 \
    https://github.com/python/mypy $out
}

create-venv() {
  local dir=$MYCPP_VENV
  python3 -m venv $dir

  maybe-our-python3 -m venv $dir

  ls -l $dir
  
  echo "Now run . $dir/bin/activate"
}

ensure-pip() {
  ### Special module to add pip to hermetic build

  # Weird that it's a bunch of wheels

  maybe-our-python3 -m ensurepip
}

# Do this inside the virtualenv
# Re-run this when UPGRADING MyPy.  10/2019: Upgraded from 0.670 to 0.730.
mypy-deps() {
  python3 -m pip install -r $MYPY_REPO/test-requirements.txt
  maybe-our-python3 -m pip install -r $MYPY_REPO/test-requirements.txt
}

pip-install() {
  ensure-pip
  create-venv

  set +o nounset

M soil/deps-tar.sh => soil/deps-tar.sh +1 -6
@@ 146,15 146,10 @@ readonly JOBS=$(( NPROC == 1 ? NPROC : NPROC-1 ))

build-python() {
  local dir=${1:-$PREPARE_DIR}
  local extra_cflags=${2:-'-O0'}

  pushd $dir
  make clean
  # Speed it up with -O0.
  # NOTE: CFLAGS clobbers some Python flags, so use EXTRA_CFLAGS.

  time make -j $JOBS EXTRA_CFLAGS="$extra_cflags"
  #time make -j 7 CFLAGS='-O0'
  time make -j $JOBS
  popd
}


M soil/host-shim.sh => soil/host-shim.sh +9 -2
@@ 15,8 15,15 @@ live-image-tag() {
  ### image ID -> Docker tag name
  local image_id=$1

  # TODO: can use a case statement
  echo 'v-2022-08-04'
  case $image_id in
    (clang|cpp|pea)
      # Updated with layer-py3
      echo 'v-2022-08-05'
      ;;
    (*)
      echo 'v-2022-08-04'
      ;;
  esac
}

make-soil-dir() {

M soil/images.sh => soil/images.sh +7 -5
@@ 1,6 1,6 @@
#!/usr/bin/env bash
#
# Manage container images for Toil
# Manage container images for Soil
#
# Usage:
#   soil/images.sh <function name>


@@ 46,15 46,17 @@ build() {
tag() {
  local name=${1:-dummy}

  local tag=v-2022-08-04
  local tag=v-2022-08-05
  sudo docker tag oilshell/soil-$name:latest oilshell/soil-$name:$tag 
}

list-images() {
  for name in soil/Dockerfile.*; do
    #echo $name
    local image=${name//'soil/Dockerfile.'/}
    echo $image
    local image_id=${name//'soil/Dockerfile.'/}
    if test "$image_id" = 'test-image'; then
      continue
    fi
    echo $image_id
  done
}