Proto support

.gitignore

@@ -1 +1,2 @@
 *.pyc
+bazel-*

README.md

@@ -1,7 +1,5 @@
 # pazel - generate Bazel BUILD files for Python
 
-[![Build Status](https://travis-ci.org/tuomasr/pazel.svg?branch=master)](https://travis-ci.org/tuomasr/pazel)
-
 ## Requirements
 
 ### pazel
@@ -13,7 +11,7 @@ Tested on Bazel 0.11.1. All recent versions are expected to work.
 ## Installation
 
 ```
-> git clone https://github.com/tuomasr/pazel.git
+> git clone https://github.com/gobeil/pazel.git
 > cd pazel
 > python setup.py install
 ```
@@ -44,6 +42,11 @@ Start from the `pazel` root directory.
 Generated BUILD files for <pazel_install_dir>/sample_app.
 ```
 
+### Bazel Rules Generated
+
+- [Python Conventions](pazel/languages/py/conventions.md)
+- [Protocol Buffer Conventions](pazel/languages/proto/conventions.md)
+
 ### Testing the generated BUILD files
 
 Now, we can build, test, and run the sample project by running the following invocations in the

pazel/BUILD

@@ -21,16 +21,24 @@ py_binary(
 py_library(
     name = "bazel_rules",
     srcs = ["bazel_rules.py"],
-    deps = [],
+)
+
+py_test(
+    name = "bazel_rules_test",
+    srcs = ["bazel_rules_test.py"],
+    size = "small",
+    deps = [":bazel_rules"],
 )
 
 py_library(
     name = "generate_rule",
     srcs = ["generate_rule.py"],
     deps = [
-        ":bazel_rules",
+        "//pazel/languages/py:py_deps",
+        "//pazel/languages/py:py_rules",
+        "//pazel/languages/proto:proto_deps",
+        "//pazel/languages/proto:proto_rules",
         ":parse_build",
-        ":parse_imports",
     ],
 )
 
@@ -52,12 +60,6 @@ py_library(
     deps = [":helpers"],
 )
 
-py_library(
-    name = "parse_imports",
-    srcs = ["parse_imports.py"],
-    deps = [":helpers"],
-)
-
 py_library(
     name = "pazel_extensions",
     srcs = ["pazel_extensions.py"],

pazel/app.py

@@ -7,10 +7,10 @@
 import argparse
 import os
 
-from pazel.generate_rule import parse_script_and_generate_rule
+from pazel.generate_rule import parse_script_and_generate_rules
 from pazel.helpers import get_build_file_path
 from pazel.helpers import is_ignored
-from pazel.helpers import is_python_file
+from pazel.helpers import has_extension
 from pazel.output_build import output_build_file
 from pazel.parse_build import get_ignored_rules
 from pazel.pazel_extensions import parse_pazel_extensions
@@ -31,14 +31,19 @@ def app(input_path, project_root, contains_pre_installed_packages, pazelrc_path)
         RuntimeError: input_path does is not a directory or a Python file.
     """
     # Parse user-defined extensions to pazel.
-    output_extension, custom_bazel_rules, custom_import_inference_rules, import_name_to_pip_name, \
+    output_extension, custom_bazel_rules, source_file_extensions, \
+        custom_import_inference_rules, import_name_to_pip_name, \
         local_import_name_to_dep, requirement_load = parse_pazel_extensions(pazelrc_path)
 
+    # TODO(gobeil): Make this part of native language registration
+    source_file_extensions.extend(['.py','.proto'])
+
     # Handle directories.
     if os.path.isdir(input_path):
         # Traverse the directory recursively.
         for dirpath, _, filenames in os.walk(input_path):
             build_source = ''
+            rule_types = set()
 
             # Parse ignored rules in an existing BUILD file, if any.
             build_file_path = get_build_file_path(dirpath)
@@ -49,27 +54,28 @@ def app(input_path, project_root, contains_pre_installed_packages, pazelrc_path)
 
                 # If a Python file is met and it is not in the list of ignored rules,
                 # generate a Bazel rule for it.
-                if is_python_file(path) and not is_ignored(path, ignored_rules):
-                    new_rule = parse_script_and_generate_rule(path, project_root,
-                                                              contains_pre_installed_packages,
-                                                              custom_bazel_rules,
-                                                              custom_import_inference_rules,
-                                                              import_name_to_pip_name,
-                                                              local_import_name_to_dep)
-
-                    # Add the new rule and a newline between it and any previous rules.
-                    if new_rule:
+                if has_extension(path, source_file_extensions) and not is_ignored(path, ignored_rules):
+                    new_rules, new_rule_types = parse_script_and_generate_rules(path, project_root,
+                                                               contains_pre_installed_packages,
+                                                               custom_bazel_rules,
+                                                               custom_import_inference_rules,
+                                                               import_name_to_pip_name,
+                                                               local_import_name_to_dep)
+                    rule_types.update(new_rule_types)
+
+                    # Add the new rules and a newline between them and any previous rules.
+                    for new_rule in new_rules:
                         if build_source:
                             build_source += 2*'\n'
 
                         build_source += new_rule
 
-            # If Python files were found, output the BUILD file.
+            # If source files were found, output the BUILD file.
             if build_source != '' or ignored_rules:
-                output_build_file(build_source, ignored_rules, output_extension, custom_bazel_rules,
+                output_build_file(build_source, ignored_rules, output_extension, list(rule_types),
                                   build_file_path, requirement_load)
-    # Handle single Python file.
-    elif is_python_file(input_path):
+    # Handle single source file.
+    elif has_extension(path, source_file_extensions):
         build_source = ''
 
         # Parse ignored rules in an existing BUILD file, if any.
@@ -78,16 +84,22 @@ def app(input_path, project_root, contains_pre_installed_packages, pazelrc_path)
 
         # Check that the script is not in the list of ignored rules.
         if not is_ignored(input_path, ignored_rules):
-            build_source = parse_script_and_generate_rule(input_path, project_root,
-                                                          contains_pre_installed_packages,
-                                                          custom_bazel_rules,
-                                                          custom_import_inference_rules,
-                                                          import_name_to_pip_name,
-                                                          local_import_name_to_dep)
-
-        # If Python files were found, output the BUILD file.
+            new_rules, rule_types = parse_script_and_generate_rules(input_path, project_root,
+                                                            contains_pre_installed_packages,
+                                                            custom_bazel_rules,
+                                                            custom_import_inference_rules,
+                                                            import_name_to_pip_name,
+                                                            local_import_name_to_dep)
+            # Add the new rules and a newline between them and any previous rules.
+            for new_rule in new_rules:
+                if build_source:
+                    build_source += 2*'\n'
+
+                build_source += new_rule
+
+        # If source files were found, output the BUILD file.
         if build_source != '' or ignored_rules:
-            output_build_file(build_source, ignored_rules, output_extension, custom_bazel_rules,
+            output_build_file(build_source, ignored_rules, output_extension, rule_types,
                               build_file_path, requirement_load)
     else:
         raise RuntimeError("Invalid input path %s." % input_path)

pazel/bazel_rules.py

@@ -1,37 +1,5 @@
-"""Classes for identifying Bazel rule type of a script and generating new rules to BUILD files."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
 import re
 
-# These templates will be filled and used to generate BUILD files.
-# Note that both 'data' and 'deps' can be empty in which case they are left out from the rules.
-PY_BINARY_TEMPLATE = """py_binary(
-    name = "{name}",
-    srcs = ["{name}.py"],
-    {data}
-    {deps}
-)"""
-
-PY_LIBRARY_TEMPLATE = """py_library(
-    name = "{name}",
-    srcs = ["{name}.py"],
-    {data}
-    {deps}
-)"""
-
-PY_TEST_TEMPLATE = """py_test(
-    name = "{name}",
-    srcs = ["{name}.py"],
-    size = "{size}",
-    {data}
-    {deps}
-)"""
-
-
 class BazelRule(object):
     """Base class defining the interface for parsing Bazel rules.
 
@@ -42,13 +10,14 @@ class BazelRule(object):
     is_test_rule = None
     template = None
     rule_identifier = None
+    replaces_rules = []
 
     @staticmethod
     def applies_to(script_name, script_source):
         """Check whether this rule applies to a given script.
 
         Args:
-            script_name (str): Name of a Python script without the .py suffix.
+            script_name (str): Name of a Python script without the file extension suffix.
             script_source (str): Source code of the script.
 
         Returns:
@@ -77,139 +46,3 @@ def find_existing(build_source, script_filename):
     def get_load_statement():
         """If the rule requires a special 'load' statement, return it, otherwise return None."""
         return None
-
-
-class PyBinaryRule(BazelRule):
-    """Class for representing Bazel-native py_binary."""
-
-    # Required class variables.
-    is_test_rule = False    # Is this a test rule?
-    template = PY_BINARY_TEMPLATE   # Filled version of this will be written to the BUILD file.
-    rule_identifier = 'py_binary'   # The name of the rule.
-
-    @staticmethod
-    def applies_to(script_name, script_source):
-        """Check whether this rule applies to a given script.
-
-        Args:
-            script_name (str): Name of a Python script without the .py suffix.
-            script_source (str): Source code of the script.
-
-        Returns:
-            applies (bool): Whether this Bazel rule can be used to represent the script.
-        """
-        # Check if there is indentation level 0 code that launches a function.
-        entrypoints = re.findall('\nif\s*__name__\s*==\s*["\']__main__["\']\s*:', script_source)
-        entrypoints += re.findall('\n\S+\([\S+]?\)', script_source)
-
-        # Rule out tests using unittest.
-        is_test = PyTestRule.applies_to(script_name, script_source)
-
-        applies = len(entrypoints) > 0 and not is_test
-
-        return applies
-
-
-class PyLibraryRule(BazelRule):
-    """Class for representing Bazel-native py_library."""
-
-    # Required class variables.
-    is_test_rule = False    # Is this a test rule?
-    template = PY_LIBRARY_TEMPLATE  # Filled version of this will be written to the BUILD file.
-    rule_identifier = 'py_library'  # The name of the rule.
-
-    @staticmethod
-    def applies_to(script_name, script_source):
-        """Check whether this rule applies to a given script.
-
-        Args:
-            script_name (str): Name of a Python script without the .py suffix.
-            script_source (str): Source code of the script.
-
-        Returns:
-            applies (bool): Whether this Bazel rule can be used to represent the script.
-        """
-        is_test = PyTestRule.applies_to(script_name, script_source)
-        is_binary = PyBinaryRule.applies_to(script_name, script_source)
-
-        applies = not (is_test or is_binary)
-
-        return applies
-
-
-class PyTestRule(BazelRule):
-    """Class for representing Bazel-native py_test."""
-
-    # Required class variables.
-    is_test_rule = True     # Is this a test rule?
-    template = PY_TEST_TEMPLATE     # Filled version of this will be written to the BUILD file.
-    rule_identifier = 'py_test'     # The name of the rule.
-
-    @staticmethod
-    def applies_to(script_name, script_source):
-        """Check whether this rule applies to a given script.
-
-        Args:
-            script_name (str): Name of a Python script without the .py suffix.
-            script_source (str): Source code of the script.
-
-        Returns:
-            applies (bool): Whether this Bazel rule can be used to represent the script.
-        """
-        imports_unittest = len(re.findall('import unittest', script_source)) > 0 or \
-            len(re.findall('from unittest', script_source)) > 0
-        uses_unittest = len(re.findall('unittest.TestCase', script_source)) > 0 or \
-            len(re.findall('TestCase', script_source)) > 0
-        test_filename = script_name.startswith('test_') or script_name.endswith('_test')
-
-        applies = test_filename and imports_unittest and uses_unittest
-
-        return applies
-
-
-def get_native_bazel_rules():
-    """Return a copy of the pazel-native classes implementing BazelRule."""
-    return [PyBinaryRule, PyLibraryRule, PyTestRule]    # No custom classes here.
-
-
-def infer_bazel_rule_type(script_path, script_source, custom_rules):
-    """Infer the Bazel rule type given the path to the script and its source code.
-
-    Args:
-        script_path (str): Path to a Python script.
-        script_source (str): Source code of the Python script.
-        custom_rules (list of BazelRule classes): User-defined classes implementing BazelRule.
-
-    Returns:
-        bazel_rule_type (BazelRule): Rule object representing the type of the Python script.
-
-    Raises:
-        RuntimeError: If zero or more than one Bazel rule is found for the current script.
-    """
-    script_name = os.path.basename(script_path).replace('.py', '')
-
-    bazel_rule_types = []
-
-    native_rules = get_native_bazel_rules()
-    registered_rules = native_rules + custom_rules
-
-    for bazel_rule in registered_rules:
-        if bazel_rule.applies_to(script_name, script_source):
-            bazel_rule_types.append(bazel_rule)
-
-    if not bazel_rule_types:
-        raise RuntimeError("No suitable Bazel rule type found for %s." % script_path)
-    elif len(bazel_rule_types) > 1:
-        # If the script is recognized by pazel native rule(s) and one exactly custom rule, then use
-        # the custom rule. This is because the pazel native rules may generate false positives.
-        is_custom = [rule not in native_rules for rule in bazel_rule_types]
-        one_custom = sum(is_custom) == 1
-
-        if one_custom:
-            custom_bazel_rule_idx = is_custom.index(True)
-            return bazel_rule_types[custom_bazel_rule_idx]
-        else:
-            raise RuntimeError("Multiple Bazel rule types (%s) found for %s."
-                               % (bazel_rule_types, script_path))
-
-    return bazel_rule_types[0]