Merge pull request #79 from digital-asset/meta-proto-fetch

meta: Change the way that Protobuf files are downloaded

Makefile

@@ -1,3 +1,34 @@
+cache_dir=.cache
+daml_proto_version=0.13.56-snapshot.20200408.3877.0.1ddcd3c0
+
+download_protos_zip := $(cache_dir)/download/protobufs-$(daml_proto_version).zip
+download_status_proto := $(cache_dir)/download/google/rpc/status.proto
+proto_dir := $(cache_dir)/protos
+proto_manifest := $(proto_dir)/manifest.json
+python := $(shell cd python && poetry env info -p)/bin/python3
+
+
+$(download_protos_zip):
+	@mkdir -p $(@D)
+	curl -sSL https://github.com/digital-asset/daml/releases/download/v$(daml_proto_version)/protobufs-$(daml_proto_version).zip -o $@
+
+
+$(download_status_proto):
+	@mkdir -p $(@D)
+	curl -sSL https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/status.proto -o $@
+
+
+$(proto_manifest): $(download_protos_zip) $(download_status_proto)
+	_build/unpack.py \
+	  -i $(download_protos_zip):protos-$(daml_proto_version) \
+	  -i $(download_status_proto):$(cache_dir)/download \
+	  -o $(@D) -m $@
+
+
+.PHONY: unpack-protos
+unpack-protos: $(cache_dir)/protos/manifest.json
+
+
 .PHONY: help
 help:	## Show list of available make targets
 	@cat Makefile | grep -e "^[a-zA-Z_\-]*: *.*## *" | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -6,24 +37,41 @@ help:	## Show list of available make targets
 deps:  ## Fetch all dependencies.
 	make -C python deps
 
-.PHONY:
+.PHONY: clean
 clean:  ## Clean everything.
+	rm -fr .cache
 	make -C python clean
 	make -C tests clean
 
-.PHONY:
+.PHONY: build
 build:  ## Build everything.
 	make -C python build
 
-.PHONY:
+.PHONY: test
 test:  ## Run all tests.
 	make -C python test
 	make -C tests test
 
-.PHONY:
+.PHONY: local-ci
 local-ci:  ## Run the build as if it were running on CI.
 	circleci local execute
 
-.PHONY:
+.PHONY: publish
 publish:  ## Publish everything.
 	make -C python publish
+
+
+.PHONY: gen-python
+gen-python: .cache/make/python.mk  ## Rebuild Python code-generated files.
+
+
+.PHONY: fetch-protos
+fetch-protos: .cache/protos/protobufs-$(daml_proto_version).zip
+
+
+.cache/make/python.mk: _build/make-template.py $(proto_manifest)
+	mkdir -p $(@D)
+	$^ > $@
+
+
+include .cache/make/python.mk

README.md

@@ -15,7 +15,8 @@ Rich Python bindings for accessing Ledger API-based applications.
 Requirements
 ------------
 * Python 3.6+
-* [Pipenv](https://pipenv.readthedocs.io/en/latest/)
+* [Poetry](https://python-poetry.org/)
+* GNU Make
 * Although not strictly required for building, you'll probably want the [DAML SDK](https://www.daml.com)
 
 Examples

_build/make-template.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent
+
+PY_GEN_DIR = 'python/dazl/_gen'
+REWRITE = '_build/rewrite_pb2.py'
+
+
+def main():
+    generate(sys.argv[1])
+
+
+def generate(manifest_file):
+    import json
+    from collections import defaultdict
+    manifest = defaultdict(set)
+    manifest[''].add('init')
+
+    with Path(manifest_file).open() as f:
+        for k, v in json.load(f).items():
+            manifest[k] = set(v)
+
+    sources = []
+
+    # enrich the manifest with 'init' entries for all directories, so that
+    # everything generated exists in a Python package
+    new_packages = set()
+    for proto_package in list(manifest):
+        # split the string on slash, and grab everything in the front
+        components = proto_package.split('/')[:-1]
+        while components:
+            pkg = '/'.join(components)
+            manifest[pkg].add('init')
+            components.pop()
+
+    for p, types in manifest.items():
+        if 'init' in types:
+            sources.append(f'{p}/__init__.py')
+        if 'pb' in types:
+            sources.append(f'{p}_pb2.py')
+        if 'grpc' in types:
+            sources.append(f'{p}_pb2_grpc.py')
+
+    print(f'# THIS FILE IS AUTOMATICALLY GENERATED. DO NOT MODIFY.')
+    print(f'py_gen_dir := {PY_GEN_DIR}')
+    print(f'py_gen_src := \\')
+    for i, p in enumerate(sorted(sources)):
+        if i < (len(sources) - 1):
+            print(f'  $(py_gen_dir)/{p} \\')
+        else:
+            print(f'  $(py_gen_dir)/{p}')
+    print(f'_py_gen_tmp_dir := $(cache_dir)/python')
+    print()
+    print('.PHONY: gen-python')
+    print('gen-python: $(py_gen_src)')
+
+    for p, types in manifest.items():
+        # init packages are directories that merely have a copyright file as
+        # their lone comment
+        if 'init' in types:
+            s = f'/{p}' if p else ''
+            print()
+            print(f'$(py_gen_dir){s}/__init__.py: COPYRIGHT')
+            print('\t@mkdir -p $(@D)')
+            print("\tsed -e 's/^/# /' < $< > $@")
+
+        # Protobuf files are firstly the raw output from grpc_tools; then they
+        # get their copyright notices added and absolute imports rewritten as
+        # relative imports
+        if 'pb' in types and 'grpc' not in types:
+            print()
+            print(f'$(py_gen_dir)/{p}_pb2.py: $(_py_gen_tmp_dir)/{p}_pb2.py COPYRIGHT')
+            print('\t@mkdir -p $(@D)')
+            print('\t$(python) $(REWRITE) $< .cache/python COPYRIGHT > $@')
+            print()
+            print(f'$(_py_gen_tmp_dir)/{p}_pb2.py: $(proto_dir)/{p}.proto')
+            print('\t@mkdir -p $(_py_gen_tmp_dir)')
+            print('\t$(python) -m grpc_tools.protoc -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) $<')
+
+        if 'grpc' in types:
+            print()
+            print(f'$(py_gen_dir)/{p}_pb2.py: $(_py_gen_tmp_dir)/{p}_pb2.py COPYRIGHT')
+            print('\t@mkdir -p $(@D)')
+            print('\t$(python) $(REWRITE) $< .cache/python COPYRIGHT > $@')
+            print()
+            print(f'$(py_gen_dir)/{p}_pb2_grpc.py: $(_py_gen_tmp_dir)/{p}_pb2_grpc.py COPYRIGHT')
+            print('\t@mkdir -p $(@D)')
+            print('\t$(python) $(REWRITE) $< .cache/python COPYRIGHT > $@')
+            print()
+            print(f'$(_py_gen_tmp_dir)/{p}_pb2.py $(_py_gen_tmp_dir)/{p}_pb2_grpc.py: $(proto_dir)/{p}.proto COPYRIGHT')
+            print('\t@mkdir -p $(_py_gen_tmp_dir)')
+            print('\t$(python) -m grpc_tools.protoc -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) --grpc_python_out=$(_py_gen_tmp_dir) $<')
+
+
+if __name__ == '__main__':
+    main()

_build/rewrite_pb2.py

@@ -0,0 +1,41 @@
+import re
+import sys
+from pathlib import Path
+
+
+FROM = re.compile(r'from ([\w.]+) import (\w+) as (\w+)')
+
+
+def main():
+    rewrite_file(sys.argv[1], sys.argv[2], sys.argv[3])
+
+
+def rewrite_file(input_file, root_path, copyright_file) -> str:
+    if not input_file.startswith(root_path):
+        raise Exception()
+
+    current_module = input_file[len(root_path) + 1:]
+    current_module = current_module.rpartition('/')[0].replace('/', '.')
+    print(current_module)
+    with Path(input_file).open('r', encoding='utf-8') as f:
+        for line in f.readlines():
+            print(rewrite_import(current_module, line).rstrip())
+
+
+def rewrite_import(parent_module: str, line: str) -> str:
+    result = FROM.match(line)
+    if result:
+        module_name = result.group(1)
+        identifier_name = result.group(2)
+        local_name = result.group(3)
+        if module_name.startswith('google.'):
+            # don't rewrite Google imports; simply do nothing
+            return line
+        if module_name == parent_module:
+            return f'from . import {identifier_name} as {local_name}'
+
+    return line
+
+
+if __name__ == '__main__':
+    main()

_build/unpack.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import json
+import logging
+from os import PathLike, fspath
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping, Optional, Sequence
+from zipfile import ZipFile
+from io import TextIOWrapper
+
+
+Manifest = Mapping[str, Sequence[str]]
+
+
+def main():
+    from argparse import ArgumentParser
+
+    logging.basicConfig()
+
+    parser = ArgumentParser(description='Unpack the DAML SDK protobuf package.')
+    parser.add_argument('--input', '-i', required=True, action='append')
+    parser.add_argument('--output', '-o', required=True)
+    parser.add_argument('--output-manifest', '-m')
+    args = parser.parse_args()
+
+    unpacker = Unpacker([PathSpec.parse(p) for p in args.input], Path(args.output))
+    manifest = unpacker.run()
+
+    if args.output_manifest:
+        with open(args.output_manifest, 'w') as f:
+            json.dump(manifest, f, indent='  ')
+
+
+@dataclass(frozen=True)
+class PathSpec:
+    """
+    An individual input file to unpack.
+
+    Attributes:
+        path:
+            Path to the file to read data from. Can be a .zip file or a
+            .proto file.
+        relative_root:
+            For .zip files, the name of a folder in the zip file to treat as the
+            Protobuf root; for other files, the directory on the file system to
+            be used as a root for ``path``.
+    """
+
+    @classmethod
+    def parse(cls, s: str):
+        p, _, r = s.partition(':')
+        return cls(Path(p), r)
+
+    path: Path
+    relative_root: 'Optional[str]' = None
+
+    def relative(self, p: PathLike) -> Optional[str]:
+        """
+        Evaluate the file path, and return the part of the path contained in this
+        directory, but only if the file exists in the relative root.
+        """
+        f = fspath(p)
+        print(f, self.relative_root)
+        if f is not None and self.relative_root is not None and f.startswith(self.relative_root):
+            return f[len(self.relative_root):].lstrip('/')
+        else:
+            return None
+
+    def as_proto_record(self) -> 'ProtoRecord':
+        return ProtoRecord()
+
+
+@dataclass(frozen=True)
+class ProtoRecord:
+    name: str
+    contents: str
+
+
+@dataclass(frozen=True)
+class Unpacker:
+    inputs: 'Sequence[PathSpec]'
+    output: 'Path'
+
+    def run(self) -> 'Manifest':
+        manifest = {}
+
+        for input_path_spec in self.inputs:
+            if input_path_spec.path.suffix == '.zip':
+                manifest.update(self._process_zip(input_path_spec))
+            elif input_path_spec.path.suffix == '.proto':
+                manifest.update(self._process_proto(input_path_spec))
+            else:
+                raise ValueError(
+                    f"don't know how to process {input_path_spec.path}")
+
+        return manifest
+
+    def _process_zip(self, zip_file_spec: 'PathSpec') -> 'Manifest':
+        proto_packages = {}  # type: Manifest
+        with ZipFile(zip_file_spec.path) as z:
+            for zi in z.infolist():
+                path = zip_file_spec.relative(zi.filename)
+                print(zi.filename, path)
+                if path is not None and not zi.is_dir():
+                    with z.open(zi) as f:
+                        with TextIOWrapper(f) as text_buf:
+                            contents = text_buf.read()
+                    proto_packages.update(self._process_proto(PathSpec(path, None), contents=contents))
+        return proto_packages
+
+    def _process_proto(self, proto_file_spec: 'PathSpec', contents: 'Optional[str]' = None) -> 'Manifest':
+        logging.info(proto_file_spec)
+        if contents is None:
+            contents = proto_file_spec.path.read_text()
+
+        name = proto_file_spec.relative(proto_file_spec.path) if proto_file_spec.relative_root is not None else proto_file_spec.path
+        if name is None:
+            print(proto_file_spec)
+            return {}
+
+        is_grpc = any(line.startswith('service') for line in contents.splitlines())
+
+        out_file = self.output / name
+        out_file.parent.mkdir(parents=True, exist_ok=True)
+        out_file.write_text(contents)
+
+        return { name.rpartition('.')[0]: ['pb', 'grpc'] if is_grpc else ['pb'] }
+
+
+if __name__ == '__main__':
+    main()

python/poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true