A => .gitignore +119 -0
@@ 1,119 @@
+*.swp
+*.swo
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
A => BEMfile +2 -0
@@ 1,2 @@
+[BEM]
+minimum_version = 0.0.1
A => Dockerfile +5 -0
@@ 1,5 @@
+FROM ubuntu:18.04
+
+RUN DPKG_FRONTEND=noninteractive apt update && apt upgrade --yes
+
+RUN apt install --yes xterm
A => bem/__init__.py +0 -0
A => bem/bem_cache.py +79 -0
@@ 1,79 @@
+import logging
+import pathlib
+import sqlite3
+import hashlib
+
+from . import bem_config
+
+# TODO: eithe ruse this or delete it
+
+"""
+
+def add_image(inifile, args, ini_dir, image_id, bemfile_hash):
+ """add_image
+
+ Add an image ID in the BEM cache. This is how BEM knows it does not need to
+ re-build the image.
+
+ DANGER: image_id and bemfile hash are not sanitized
+
+ :param inifile:
+ :param args:
+ :param ini_dir:
+ :param image_id:
+ :param bemfile_hash:
+ """
+
+ cachedir = pathlib.Path(bem_config.get_key(inifile, args, ini_dir, "cachedir"))
+ cachefile = cachedir / bem_config.get_key(inifile, args, ini_dir, "cachefile_name"))
+
+ db = sqlite3.connect(cachefile)
+ cursor = db.cursor()
+ cursor.execute("""
+CREATE TABLE IF NOT EXISTS images(filehash TEXT PRIMARY KEY, imageid TEXT);
+""")
+
+ cursor.execute("""
+INSERT OR REPLACE INTO images (filehash imageid)
+ VALUES(
+ '{}',
+ '{}',
+ );
+""".format(bemfile_hash, image_id))
+
+ db.commit()
+
+def get_images(inifile, args, ini_dir, bemfile_hash):
+ """get_images
+
+ Return a list of image IDs that are associated with the given image ID.
+
+ DANGER: image_id is not sanatized
+
+ :param inifile:
+ :param args:
+ :param ini_dir:
+ :param bemfile_hash:
+ """
+
+ cachedir = pathlib.Path(bem_config.get_key(inifile, args, ini_dir, "cachedir"))
+ cachefile = cachedir / bem_config.get_key(inifile, args, ini_dir, "cachefile_name"))
+
+ db = sqlite3.connect(cachefile)
+ cursor = db.cursor()
+ cursor.execute("""
+CREATE TABLE IF NOT EXISTS images(filehash TEXT PRIMARY KEY, imageid TEXT);
+""")
+
+ cursor.execute("SELECT imageid FROM images WHERE filehash == '{}'".format(bemfile_hash))
+
+ return list(cursor.fetchall())
+
+def hash_file(bemfile_path):
+ hasher = hashlib.sha256()
+
+ with open(bemfile_path, "rb") as f:
+ hasher.update(f.read())
+
+ return hasher.hexdigest()
+"""
A => bem/bem_cli.py +102 -0
@@ 1,102 @@
+import argparse
+import configparser
+import pathlib
+import logging
+import sys
+
+from . import bem_constants
+from . import bem_util
+from . import bem_config
+from . import bem_engine
+
+def bem_cli():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("--version", "-V", action='version',
+ version=bem_constants.version)
+
+ verbosity = parser.add_mutually_exclusive_group()
+
+ verbosity.add_argument("--quiet", "-q", default=False,
+ action="store_true", help="Only display errors and warnings.")
+
+ verbosity.add_argument("--silent", "-s", default=False,
+ action="store_true", help="Only display errors.")
+
+ verbosity.add_argument("--debug", "-D", default=False,
+ action="store_true", help="Display debug log messages.")
+
+ parser.add_argument("--bemfile", "-f", default=None, type=pathlib.Path,
+ help="Specify path to BEMfile to use.")
+
+ parser.add_argument("--dockerfile", "-d", default=None, type=pathlib.Path,
+ help="Override bem.project.dockerfile to use the specified " +
+ "path")
+
+ parser.add_argument("--logfile", "-o", default=None, type=pathlib.Path,
+ help="Specify log file (default: don't output a log)")
+
+ parser.add_argument("--projectdir", "-p", default=None, type=pathlib.Path,
+ help="Override the project dir (the default is the parent of " +
+ "the BEMfile).")
+
+ parser.add_argument("--noninteractive", "-n", default=None,
+ action='store_true', help="Run the command non-interactively")
+
+
+ parser.add_argument("command", nargs="+")
+
+ args = parser.parse_args()
+
+ if args.quiet:
+ bem_util.setup_logging("WARNING")
+ elif args.silent:
+ bem_util.setup_logging("ERROR")
+ elif args.debug:
+ bem_util.setup_logging("DEBUG")
+ else:
+ bem_util.setup_logging("INFO")
+
+ logging.debug("Starting BEM version {}".format(bem_constants.version))
+
+ if (args.logfile is not None) and (args.noninteractive is None):
+ logging.warning("--logfile has no effect in interactive mode")
+
+ bemfile = None
+
+ if args.bemfile is not None:
+ bemfile = args.bemfile
+ else:
+ try:
+ bemfile = bem_util.find_bemfile()
+ except FileNotFoundError:
+ logging.error("unable to locate a suitable BEMfile")
+ sys.exit(1)
+
+ logging.debug("using bemfile: '{}'".format(bemfile))
+
+ config = configparser.ConfigParser()
+ config.read(bemfile)
+
+ min_version = bem_config.get_key(config, args, bemfile.parent,
+ "minimum_version")
+ if not bem_util.compare_versions(bem_constants.version, min_version):
+ logging.error("BEM version '{}' too old, at least version '{}' is required"
+ .format(bem_constants.version, min_version))
+ sys.exit(1)
+
+ logging.debug("project dir: {}".format(
+ bem_config.get_key(config, args, bemfile.parent, "projectdir")))
+ logging.debug("dockerfile: {}".format(
+ bem_config.get_key(config, args, bemfile.parent, "dockerfile")))
+
+ try:
+ bem_engine.validate_config(config, args, bemfile.parent)
+ except Exception as e:
+ logging.error("Configuration error: {}".format(e))
+ sys.exit(1)
+
+ bem_engine.execute_command(config, args, bemfile.parent, args.command)
+
+if __name__ == "__main__":
+ bem_cli()
A => bem/bem_config.py +90 -0
@@ 1,90 @@
+import logging
+import configparser
+
+from . import bem_constants
+
+def apply_indirection(inifile, args, ini_dir, val):
+
+ # there isn't a good way to enumerate what keys are available, so we
+ # make a dummy that portends to be a dict, but really just calls get_key()
+ class DummyDict:
+ def __getitem__(this, key):
+ return get_key(inifile, args, ini_dir, key, True)
+
+ def __missing__(this, key):
+ logging.error("unknown indirect key '{}'".format(key))
+
+ dummy = DummyDict()
+ return str(val).format_map(dummy)
+
+def get_key(inifile, args, ini_dir, key, indirection=True):
+ """get_key
+
+ :param inifile: INI file as loaded by configparser.ConfigParser.read
+ :param args: args file from argparse
+ :param key: key to retrieve
+ :param indirection: set to True to allow indirection
+ :param ini_dir: parent directory of the loaded INI file
+
+ Retrieves the specified config key. Order of precedence is:
+
+ * runtime keys
+ * arguments
+ * INI file
+ * bem_constants.config_defaults
+
+ Returned value will always be a string. Throws a KeyError if there is no
+ such config value.
+ """
+
+ logging.debug("retrieving config key '{}'".format(key))
+
+
+ runtime = {
+ "bem_file_dir": ini_dir,
+ "version": bem_constants.version
+ }
+
+ result = None
+
+ # check values provided by arguments
+ arguments = {}
+
+ if args.dockerfile is not None:
+ arguments["dockerfile"] = args.dockerfile
+
+ if args.projectdir is not None:
+ arguments["projectdir"] = args.projectdir
+
+ if args.logfile is not None:
+ arguments["logfile"] = args.logfile
+
+ if args.noninteractive is not None:
+ arguments["noninteractive"] = args.noninteractive
+
+ if key in runtime:
+ logging.debug("\trequested key found in runtime: '{}'".format(runtime[key]))
+ result = runtime[key]
+
+ elif key in arguments:
+ result = arguments[key]
+ logging.debug("\tkey found in arguments: '{}'".format(result))
+
+ elif key in inifile["BEM"]:
+ logging.debug("\trequested key found in INI file: '{}'".format(inifile["BEM"][key]))
+ result = inifile["BEM"][key]
+
+ elif key in bem_constants.config_defaults:
+ logging.debug("\trequested key found in defaults: '{}'".format(bem_constants.config_defaults[key]))
+ result = bem_constants.config_defaults[key]
+
+ if result is not None:
+ if indirection:
+ return apply_indirection(inifile, args, ini_dir, result)
+ else:
+ return result
+
+ logging.error("No such config key: '{}'".format(key))
+ raise KeyError("No such config key: '{}'".format(key))
+
+
A => bem/bem_constants.py +15 -0
@@ 1,15 @@
+import appdirs
+
+version = "0.0.1-dev"
+
+config_defaults = {
+ "minimum_version": version,
+ "dockerfile": "{bem_file_dir}/Dockerfile",
+ "projectdir": "{bem_file_dir}",
+ "cachedir": appdirs.user_cache_dir(),
+ "cachefile_name": "bem_cache.db",
+ "noninteractive": "False",
+ "logfile": "/dev/null",
+ "flush_interval": "5",
+}
+
A => bem/bem_engine.py +166 -0
@@ 1,166 @@
+import docker
+import dockerpty
+import pathlib
+import logging
+import sys
+import datetime
+import os
+
+from . import bem_config
+from . import bem_util
+
+def validate_config(config, args, ini_dir):
+ """validate_config
+
+ Sanity check a given runtime configuration. Throw an exception if there is
+ a problem.
+
+ :param config:
+ :param args:
+ :param ini_dir:
+ """
+
+ logging.debug("validating config... ")
+
+ try:
+ dockerfile = pathlib.Path(bem_config.get_key(config, args, ini_dir, "dockerfile"))
+ except Exception as e:
+ raise Exception("unable to get key 'dockerfile', this may indicate a problem with your BEMfile; reason: {}".format(e))
+
+ try:
+ projectdir = pathlib.Path(bem_config.get_key(config, args, ini_dir, "projectdir"))
+ except Exception as e:
+ raise Exception("unable to get key 'projectdir', this may indicate a problem with your BEMfile; reason: {}".format(e))
+
+ try:
+ cachedir = pathlib.Path(bem_config.get_key(config, args, ini_dir, "cachedir"))
+ except Exception as e:
+ raise Exception("unable to get key 'cachedir', this may indicate a problem with your BEMfile; reason: {}".format(e))
+
+ if not dockerfile.is_file():
+ raise Exception("Specified Dockerfile '{}' does not exist or not a file"
+ .format(dockerfile))
+
+ if not projectdir.is_dir():
+ raise Exception("Specified project directory '{}' does not exist or not a directory"
+ .format(projectdir))
+
+ if not cachedir.is_dir():
+ raise Exception("Specified cachedir directory '{}' does not exist or not a directory"
+ .format(cachedir))
+
+ try:
+ bool(bem_config.get_key(config, args, ini_dir, "noninteractive"))
+ except Exception as e:
+ raise Exception("Invalid value for 'noninteractive'; reason: {}".format(e))
+
+ logging.debug("config looks ok")
+
+
+def ensure_image_built(config, args, ini_dir):
+ """ensure_image_built
+
+ Ensure the docker image for the particular project is built.
+
+ Return the image ID.
+
+ :param config:
+ :param args:
+ :param ini_dir:
+ """
+
+ logging.info("building image... ")
+
+ dockerfile = pathlib.Path(bem_config.get_key(config, args, ini_dir, "dockerfile"))
+ projectdir = pathlib.Path(bem_config.get_key(config, args, ini_dir, "projectdir"))
+
+ client = docker.from_env()
+ image = client.images.build(path=str(projectdir), dockerfile=str(dockerfile))
+
+ logging.info("image build completed.")
+ logging.debug("\t{}".format(image))
+
+ return image[0].id
+
+def execute_command(config, args, ini_dir, command):
+ image = ensure_image_built(config, args, ini_dir)
+ noninteractive = bem_util.parse_bool(bem_config.get_key(config, args, ini_dir, "noninteractive"))
+ logfile = str(bem_config.get_key(config, args, ini_dir, "logfile"))
+ flush_interval = int(bem_config.get_key(config, args, ini_dir, "flush_interval"))
+ projectdir = pathlib.Path(bem_config.get_key(config, args, ini_dir, "projectdir"))
+
+ client = docker.from_env()
+
+ logging.debug("executing command '{}' in image {}".format(command, image))
+
+ # TODO: should use a FUSE mount with -o allow_other to force I/O to happen
+ # as the real owner. In the interim, we map /etc/passwd and /etc/shadow.
+ #
+ # TODO: configurable X11 bind
+ #
+ # TODO: load extra binds from config in some way
+ #
+ # TODO: global binds from ~/.config (?)
+
+ volumes = {
+ str(projectdir): {
+ "bind": str(projectdir),
+ "mode": "rw"
+ },
+ "/etc/shadow": {
+ "bind": "/etc/shadow",
+ "mode": "ro",
+ },
+ "/etc/passwd": {
+ "bind": "/etc/passwd",
+ "mode": "ro",
+ },
+ "/etc/group": {
+ "bind": "/etc/group",
+ "mode": "ro",
+ },
+ "/tmp/.X11-unix": {
+ "bind": "/tmp/.X11-unix",
+ "mode": "rw",
+ }
+ }
+
+ logging.debug("volumes: {}".format(volumes))
+
+ environment = dict(os.environ)
+
+ environment["PS1"] = "[BEM:{}] $(whoami)@$(hostname) $(pwd) $ ".format(pathlib.Path(projectdir).name)
+
+ # TODO: if getcwd does not exist inside of the container (not a child of
+ # any volume), set to project dir instead
+
+ if noninteractive:
+ logging.debug("\trunning in non-interactive mode, log file is '{}'".format(logfile))
+ # TODO: bind mounts
+ # TODO: map user
+ # TODO: environment
+ container = client.containers.run(image, command, detach=True, volumes=volumes)
+ if logfile != "/dev/null":
+ lastflush = datetime.datetime.now()
+ with open(logfile, 'wb') as f:
+ for line in container.logs(stream=True):
+ f.write(line)
+ if (datetime.datetime.now() - lastflush).seconds > flush_interval:
+ f.flush()
+ lastflush = datetime.datetime.now()
+
+ else:
+ container = client.api.create_container(
+ image=image,
+ stdin_open = True,
+ tty=True,
+ command=command,
+ host_config=client.api.create_host_config(
+ binds=volumes,
+ ),
+ user=os.getuid(),
+ environment=environment,
+ working_dir=os.getcwd()
+ )
+
+ dockerpty.start(client.api, container)
A => bem/bem_util.py +100 -0
@@ 1,100 @@
+import logging
+import pathlib
+
+from . import bem_config
+
+def str2level(s):
+ """str2level
+
+ :param s: string level
+
+ Converts a string level into a logging level object.
+
+ Returns logging.DEBUG if the given string is unknown.
+ """
+
+ if s == "DEBUG":
+ return logging.DEBUG
+
+ elif s == "INFO":
+ return logging.INFO
+
+ elif s == "WARNING":
+ return logging.WARNING
+
+ elif s == "WARN":
+ return logging.WARNING
+
+ else:
+ logging.warning("unknown log level '{}', assuming DEBUG".format(s))
+ return logging.DEBUG
+
+def find_bemfile():
+ """find_project_dir
+
+ Traverse up the filesystem hierarchy looking for a BEMfile. Returns it's
+ path if it is found.
+
+ Raise a FileNotFoundError if no BEMfile found.
+ """
+
+ p = pathlib.Path("./BEMfile").resolve()
+
+ logging.debug("searching for a BEMfile at or above '{}'".format(pathlib.Path("./").resolve()))
+
+ while True:
+ if p.exists():
+ logging.debug("\tfound at '{}'".format(p))
+ return p
+
+ elif p.parent.resolve() == p.parent.parent.resolve():
+ logging.error("reached filesystem anchor without encountering a BEMfile (hint: are you in the right directory?)")
+ raise FileNotFoundError
+
+ else:
+ logging.debug("\ttraversing to parent... ")
+ p = (p.parent.parent.resolve() / "BEMfile").resolve()
+ logging.debug("\t\t'{}'".format(p))
+
+def setup_logging(level="INFO"):
+ logging.basicConfig(
+ level=str2level(level), format="%(levelname)s: %(message)s", datefmt="%H:%M:%S"
+ )
+
+
+def log_exception(e):
+ logging.error("Exception: {}".format(e))
+ logging.debug("".join(traceback.format_tb(e.__traceback__)))
+
+
+def log_pretty(logfunc, obj):
+ logfunc(pprint.pformat(obj))
+
+def compare_versions(v1, v2):
+ """compare_versions
+
+ Returns True if v1 is at least as new as or newer than v2.
+
+ :param v1:
+ :param v2:
+ """
+
+ v1 = str(v1).strip()
+ v2 = str(v2).strip()
+
+ v1 = v1.split("-")[0]
+ v1 = [int(x) for x in v1.split(".")]
+
+ v2 = v2.split("-")[0]
+ v2 = [int(x) for x in v2.split(".")]
+
+ for i in range(min([len(v1), len(v2)])):
+ if v1[i] > v2[i]:
+ return True
+ if v1[i] < v2[i]:
+ return False
+
+ return True
+
+def parse_bool(s):
+ return s.lower in ["true", "t", "yes", "1", "on", "one", "yep", "yup"]
A => setup.py +42 -0
@@ 1,42 @@
+from setuptools import setup
+from setuptools import find_packages
+
+from bem import bem_constants
+
+short_description = "Build Environment Manager"
+
+long_description = '''
+Build Environment Manager
+'''.lstrip() # remove leading newline
+
+classifiers = [
+ # see http://pypi.python.org/pypi?:action=list_classifiers
+ 'Development Status :: 3 - Alpha',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: POSIX',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 3',
+ ]
+
+setup(name="pretor",
+ version=bem_constants.version,
+ description=short_description,
+ long_description=long_description,
+ author="Charles Daniels",
+ author_email="charles@cdaniels.net",
+ url="TODO",
+ license='BSD',
+ classifiers=classifiers,
+ keywords=["grading", "computer science"],
+ packages=find_packages(),
+ entry_points={'console_scripts':
+ [
+ 'bem=bem.bem_cli:bem_cli'
+ ]},
+ package_dir={'bem': 'bem'},
+ platforms=['POSIX'],
+ install_requires=['docker', 'dockerpty']
+ )
+