Allow packwerk to scan for packages inside of engines

USAGE.md

@@ -72,15 +72,16 @@ Secondly, to enable Spring, first run `bin/spring binstub packwerk` which will "
 
 Packwerk reads from the `packwerk.yml` configuration file in the root directory. Packwerk will run with the default configuration if any of these settings are not specified.
 
-| Key                  | Default value                             | Description  |
-|----------------------|-------------------------------------------|--------------|
-| include              | **/*.{rb,rake,erb}                        | list of patterns for folder paths to include |
-| exclude              | {bin,node_modules,script,tmp,vendor}/**/* | list of patterns for folder paths to exclude |
-| package_paths        | **/                                       | a single pattern or a list of patterns to find package configuration files, see: [Defining packages](#Defining-packages) |
-| custom_associations  | N/A                                       | list of custom associations, if any |
-| parallel             | true                                      | when true, fork code parsing out to subprocesses |
-| cache                | false                                     | when true, caches the results of parsing files |
-| cache_directory      | tmp/cache/packwerk                        | the directory that will hold the packwerk cache |
+| Key                                 | Default value                             | Description                                                                                                              |
+|-------------------------------------|-------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|
+| include                             | **/*.{rb,rake,erb}                        | list of patterns for folder paths to include                                                                             |
+| exclude                             | {bin,node_modules,script,tmp,vendor}/**/* | list of patterns for folder paths to exclude                                                                             |
+| package_paths                       | **/                                       | a single pattern or a list of patterns to find package configuration files, see: [Defining packages](#Defining-packages) |
+| custom_associations                 | N/A                                       | list of custom associations, if any                                                                                      |
+| parallel                            | true                                      | when true, fork code parsing out to subprocesses                                                                         |
+| cache                               | false                                     | when true, caches the results of parsing files                                                                           |
+| cache_directory                     | tmp/cache/packwerk                        | the directory that will hold the packwerk cache                                                                          |
+| packages_outside_of_app_dir_enabled | false                                     | when true, packwerk will scan for packages in folders for locally published rails engines                                |
 
 ### Using a custom ERB parser
 

lib/packwerk.rb

@@ -27,6 +27,8 @@ module Packwerk
   autoload :Package
   autoload :PackageSet
   autoload :PackageTodo
+  autoload :PackagePaths
+  autoload :ParsedConstantDefinitions
   autoload :Parsers
   autoload :RailsLoadPaths
   autoload :Reference

lib/packwerk/configuration.rb

@@ -60,6 +60,9 @@ def from_packwerk_config(path)
     sig { returns(Pathname) }
     attr_reader(:cache_directory)
 
+    sig { returns(T::Boolean) }
+    attr_reader(:packages_outside_of_app_dir_enabled)
+
     sig { params(parallel: T::Boolean).returns(T::Boolean) }
     attr_writer(:parallel)
 
@@ -80,6 +83,7 @@ def initialize(configs = {}, config_path: nil)
       @cache_enabled = T.let(configs.key?("cache") ? configs["cache"] : false, T::Boolean)
       @cache_directory = T.let(Pathname.new(configs["cache_directory"] || "tmp/cache/packwerk"), Pathname)
       @config_path = config_path
+      @packages_outside_of_app_dir_enabled = configs["packages_outside_of_app_dir_enabled"] || false
 
       @offenses_formatter_identifier = T.let(
         configs["offenses_formatter"] || Formatters::DefaultOffensesFormatter::IDENTIFIER, String

lib/packwerk/package_paths.rb

@@ -0,0 +1,71 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "pathname"
+require "bundler"
+
+module Packwerk
+  class PackagePaths
+    PACKAGE_CONFIG_FILENAME = "package.yml"
+    PathSpec = T.type_alias { T.any(String, T::Array[String]) }
+
+    extend T::Sig
+    extend T::Generic
+
+    sig do
+      params(
+        root_path: String,
+        package_pathspec: T.nilable(PathSpec),
+        exclude_pathspec: T.nilable(PathSpec),
+        scan_for_packages_outside_of_app_dir: T.nilable(T::Boolean)
+      ).void
+    end
+    def initialize(root_path, package_pathspec, exclude_pathspec = nil, scan_for_packages_outside_of_app_dir = false)
+      @root_path = root_path
+      @package_pathspec = package_pathspec
+      @exclude_pathspec = exclude_pathspec
+      @scan_for_packages_outside_of_app_dir = scan_for_packages_outside_of_app_dir
+    end
+
+    sig {returns(T::Array[Pathname])}
+    def all_paths
+      exclude_pathspec = Array(@exclude_pathspec).dup
+        .push(Bundler.bundle_path.join("**").to_s)
+        .map { |glob| File.expand_path(glob) }
+
+      paths_to_scan = if @scan_for_packages_outside_of_app_dir
+        engine_paths_to_scan.push(@root_path)
+      else
+        [@root_path]
+      end
+
+      glob_patterns = paths_to_scan.product(Array(@package_pathspec)).map do |path, pathspec|
+        File.join(path, pathspec, PACKAGE_CONFIG_FILENAME)
+      end
+
+      Dir.glob(glob_patterns)
+        .map { |path| Pathname.new(path).cleanpath }
+        .reject { |path| exclude_path?(exclude_pathspec, path) }
+    end
+
+    private
+
+    sig { params(globs: T::Array[String], path: Pathname).returns(T::Boolean) }
+    def exclude_path?(globs, path)
+      globs.any? do |glob|
+        path.realpath.fnmatch(glob, File::FNM_EXTGLOB)
+      end
+    end
+
+    sig { returns(T::Array[String]) }
+    def engine_paths_to_scan
+      bundle_path_match = Bundler.bundle_path.join("**")
+
+      Rails.application.railties
+        .select { |r| r.is_a?(Rails::Engine) }
+        .map { |r| Pathname.new(r.root).expand_path }
+        .reject { |path| path.fnmatch(bundle_path_match.to_s) } # reject paths from vendored gems
+        .map(&:to_s)
+    end
+  end
+end

lib/packwerk/package_set.rb

@@ -5,7 +5,6 @@
 require "bundler"
 
 module Packwerk
-  PathSpec = T.type_alias { T.any(String, T::Array[String]) }
 
   # A set of {Packwerk::Package}s as well as methods to parse packages from the filesystem.
   class PackageSet
@@ -20,11 +19,14 @@ class PackageSet
     class << self
       extend T::Sig
 
-      sig { params(root_path: String, package_pathspec: T.nilable(PathSpec)).returns(PackageSet) }
-      def load_all_from(root_path, package_pathspec: nil)
-        package_paths = package_paths(root_path, package_pathspec || "**")
+      sig do
+        params(root_path: String, package_pathspec: T.nilable(PackagePaths::PathSpec),
+          scan_for_packages_outside_of_app_dir: T.nilable(T::Boolean)).returns(PackageSet)
+      end
+      def load_all_from(root_path, package_pathspec: nil, scan_for_packages_outside_of_app_dir: false)
+        package_paths = PackagePaths.new(root_path, package_pathspec || "**", nil, scan_for_packages_outside_of_app_dir)
 
-        packages = package_paths.map do |path|
+        packages = package_paths.all_paths.map do |path|
           root_relative = path.dirname.relative_path_from(root_path)
           Package.new(name: root_relative.to_s, config: YAML.load_file(path, fallback: nil))
         end
@@ -34,27 +36,6 @@ def load_all_from(root_path, package_pathspec: nil)
         new(packages)
       end
 
-      sig do
-        params(
-          root_path: String,
-          package_pathspec: PathSpec,
-          exclude_pathspec: T.nilable(PathSpec)
-        ).returns(T::Array[Pathname])
-      end
-      def package_paths(root_path, package_pathspec, exclude_pathspec = [])
-        exclude_pathspec = Array(exclude_pathspec).dup
-          .push(Bundler.bundle_path.join("**").to_s)
-          .map { |glob| File.expand_path(glob) }
-
-        glob_patterns = Array(package_pathspec).map do |pathspec|
-          File.join(root_path, pathspec, PACKAGE_CONFIG_FILENAME)
-        end
-
-        Dir.glob(glob_patterns)
-          .map { |path| Pathname.new(path).cleanpath }
-          .reject { |path| exclude_path?(exclude_pathspec, path) }
-      end
-
       private
 
       sig { params(packages: T::Array[Package]).void }
@@ -64,12 +45,7 @@ def create_root_package_if_none_in(packages)
         packages << Package.new(name: Package::ROOT_PACKAGE_NAME, config: nil)
       end
 
-      sig { params(globs: T::Array[String], path: Pathname).returns(T::Boolean) }
-      def exclude_path?(globs, path)
-        globs.any? do |glob|
-          path.realpath.fnmatch(glob, File::FNM_EXTGLOB)
-        end
-      end
+
     end
 
     sig { returns(T::Hash[String, Package]) }

lib/packwerk/rails_load_paths.rb

@@ -35,11 +35,9 @@ def extract_application_autoload_paths
       end
       def filter_relevant_paths(all_paths, bundle_path: Bundler.bundle_path, rails_root: Rails.root)
         bundle_path_match = bundle_path.join("**")
-        rails_root_match = rails_root.join("**")
 
         all_paths
           .transform_keys { |path| Pathname.new(path).expand_path }
-          .select { |path| path.fnmatch(rails_root_match.to_s) } # path needs to be in application directory
           .reject { |path| path.fnmatch(bundle_path_match.to_s) } # reject paths from vendored gems
       end
 

lib/packwerk/run_context.rb

@@ -21,6 +21,7 @@ def from_configuration(configuration)
           root_path: configuration.root_path,
           load_paths: configuration.load_paths,
           package_paths: configuration.package_paths,
+          packages_outside_of_app_dir_enabled: configuration.packages_outside_of_app_dir_enabled,
           inflector: inflector,
           custom_associations: configuration.custom_associations,
           cache_enabled: configuration.cache_enabled?,
@@ -41,6 +42,7 @@ def from_configuration(configuration)
         custom_associations: AssociationInspector::CustomAssociations,
         checkers: T::Array[Checker],
         cache_enabled: T::Boolean,
+        packages_outside_of_app_dir_enabled: T.nilable(T::Boolean),
       ).void
     end
     def initialize(
@@ -52,7 +54,8 @@ def initialize(
       package_paths: nil,
       custom_associations: [],
       checkers: Checker.all,
-      cache_enabled: false
+      cache_enabled: false,
+      packages_outside_of_app_dir_enabled: false
     )
       @root_path = root_path
       @load_paths = load_paths
@@ -63,7 +66,7 @@ def initialize(
       @cache_enabled = cache_enabled
       @cache_directory = cache_directory
       @config_path = config_path
-
+      @packages_outside_of_app_dir_enabled = packages_outside_of_app_dir_enabled
       @file_processor = T.let(nil, T.nilable(FileProcessor))
       @context_provider = T.let(nil, T.nilable(ConstantDiscovery))
       @package_set = T.let(nil, T.nilable(PackageSet))
@@ -77,18 +80,27 @@ def initialize(
     def process_file(relative_file:)
       processed_file = file_processor.call(relative_file)
 
+      puts relative_file
+
       references = ReferenceExtractor.get_fully_qualified_references_from(
         processed_file.unresolved_references,
         context_provider
       )
+      puts references
       reference_checker = ReferenceChecking::ReferenceChecker.new(@checkers)
 
       processed_file.offenses + references.flat_map { |reference| reference_checker.call(reference) }
     end
 
     sig { returns(PackageSet) }
     def package_set
-      @package_set ||= ::Packwerk::PackageSet.load_all_from(@root_path, package_pathspec: @package_paths)
+      @package_set ||= ::Packwerk::PackageSet.load_all_from(
+        @root_path,
+        package_pathspec: @package_paths,
+        scan_for_packages_outside_of_app_dir: @packages_outside_of_app_dir_enabled,
+      )
+      puts @package_set.packages
+      @package_set
     end
 
     private
@@ -123,7 +135,7 @@ def resolver
         inflector: @inflector,
       )
     end
-
+    
     sig { returns(T::Array[ConstantNameInspector]) }
     def constant_name_inspectors
       [

lib/packwerk/validator.rb

@@ -61,8 +61,13 @@ def package_manifests_settings_for(configuration, setting)
     end
     def package_manifests(configuration, glob_pattern = nil)
       glob_pattern ||= package_glob(configuration)
-      PackageSet.package_paths(configuration.root_path, glob_pattern, configuration.exclude)
-        .map { |f| File.realpath(f) }
+      package_paths = PackagePaths.new(
+        configuration.root_path,
+        glob_pattern,
+        configuration.exclude,
+        configuration.packages_outside_of_app_dir_enabled
+      )
+      package_paths.all_paths.map { |f| File.realpath(f) }
     end
 
     sig { params(configuration: Configuration).returns(T.any(T::Array[String], String)) }

test/fixtures/external_packages/app/components/timeline/app/models/concerns/has_timeline.rb

@@ -0,0 +1,5 @@
+# typed: ignore
+# frozen_string_literal: true
+
+module HasTimeline
+end

test/fixtures/external_packages/app/components/timeline/app/models/private_thing.rb

@@ -0,0 +1,2 @@
+class PrivateThing
+end

test/fixtures/external_packages/app/components/timeline/nested/package.yml

@@ -0,0 +1 @@
+enforce_dependencies: true

test/fixtures/external_packages/app/components/timeline/package.yml

@@ -0,0 +1,2 @@
+enforce_privacy:
+  - "::PrivateThing"

test/fixtures/external_packages/app/packwerk.yml

@@ -0,0 +1 @@
+packages_outside_of_app_dir_enabled: true

test/fixtures/external_packages/sales/components/sales/app/models/order.rb

@@ -0,0 +1,5 @@
+# typed: ignore
+# frozen_string_literal: true
+
+class Order
+end

test/fixtures/external_packages/sales/components/sales/app/models/sales/order.rb

@@ -0,0 +1,7 @@
+# typed: ignore
+# frozen_string_literal: true
+
+module Sales
+  class Order
+  end
+end

test/fixtures/external_packages/sales/components/sales/app/public/sales/record_new_order.rb

@@ -0,0 +1,7 @@
+# typed: ignore
+# frozen_string_literal: true
+
+module Sales
+  module RecordNewOrder
+  end
+end

test/fixtures/external_packages/sales/components/sales/package.yml

@@ -0,0 +1,8 @@
+---
+enforce_privacy: true
+
+metadata:
+  stewards:
+  - "@Shopify/sales"
+  slack_channels:
+  - "#sales"